From d737e0f8ebda95274395aa6dc00a83f8e6fb1ceb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 12 Sep 2022 19:12:05 +0100 Subject: [PATCH] Updating plugin backend, refactoring a bit, correctly allowing a set of headers to be used for a URL. --- packages/bbui/src/Label/Label.svelte | 11 +- packages/builder/src/constants/index.js | 7 + .../plugins/_components/AddPluginModal.svelte | 63 +++++--- packages/builder/src/stores/portal/plugins.js | 8 +- .../server/src/api/controllers/plugin/file.ts | 15 ++ .../src/api/controllers/plugin/github.ts | 75 +++++++++ .../src/api/controllers/plugin/index.ts | 17 +- .../server/src/api/controllers/plugin/npm.ts | 56 +++++++ .../src/api/controllers/plugin/uploaders.ts | 4 + .../server/src/api/controllers/plugin/url.ts | 12 ++ .../src/api/controllers/plugin/utils.js | 153 ------------------ .../src/api/controllers/plugin/utils.ts | 19 +++ 12 files changed, 251 insertions(+), 189 deletions(-) create mode 100644 packages/server/src/api/controllers/plugin/file.ts create mode 100644 packages/server/src/api/controllers/plugin/github.ts create mode 100644 packages/server/src/api/controllers/plugin/npm.ts create mode 100644 packages/server/src/api/controllers/plugin/uploaders.ts create mode 100644 packages/server/src/api/controllers/plugin/url.ts delete mode 100644 packages/server/src/api/controllers/plugin/utils.js create mode 100644 packages/server/src/api/controllers/plugin/utils.ts diff --git a/packages/bbui/src/Label/Label.svelte b/packages/bbui/src/Label/Label.svelte index 3395ab4179..6b3392ce2d 100644 --- a/packages/bbui/src/Label/Label.svelte +++ b/packages/bbui/src/Label/Label.svelte @@ -4,10 +4,15 @@ export let size = "M" export let tooltip = "" + export let muted - @@ -17,4 +22,8 @@ padding: 0; white-space: nowrap; } + + .muted { + opacity: 0.5; + } diff --git a/packages/builder/src/constants/index.js b/packages/builder/src/constants/index.js index 4e2ca37b9c..151a0cdf8d 100644 --- a/packages/builder/src/constants/index.js +++ b/packages/builder/src/constants/index.js @@ -57,3 +57,10 @@ export const DefaultAppTheme = { navBackground: "var(--spectrum-global-color-gray-50)", navTextColor: "var(--spectrum-global-color-gray-800)", } + +export const PluginSource = { + URL: "URL", + NPM: "NPM", + GITHUB: "Github", + FILE: "File Upload", +} diff --git a/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte b/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte index 6c9dd509fa..7df0ca2a8d 100644 --- a/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte +++ b/packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte @@ -5,40 +5,56 @@ Input, Select, Dropzone, + Body, notifications, } from "@budibase/bbui" + import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte" import { plugins } from "stores/portal" + import { PluginSource } from "constants" - const Sources = { - NPM: "NPM", - GITHUB: "Github", - URL: "URL", - FILE: "File Upload", + function opt(name, optional) { + if (optional) { + return { name, optional } + } + return { name } } let authOptions = { - [Sources.NPM]: ["URL"], - [Sources.GITHUB]: ["Github Token", "URL"], - [Sources.URL]: ["Headers", "URL"], - [Sources.FILE]: ["File Upload"], + [PluginSource.URL]: [opt("URL"), opt("Headers", true)], + [PluginSource.NPM]: [opt("URL")], + [PluginSource.GITHUB]: [opt("URL"), opt("Github Token", true)], + [PluginSource.FILE]: [opt("File Upload")], } let file - let source = Sources.URL + let source = PluginSource.URL let dynamicValues = {} let validation $: validation = source === "File Upload" ? file : dynamicValues["URL"] + function infoMessage(optionName) { + switch (optionName) { + case PluginSource.URL: + return "Please specify a URL which directs to a built plugin TAR archive, you can provide headers if authentication is required." + case PluginSource.NPM: + return "Please specify the URL to a public NPM package which contains the built version of the plugin you wish to install." + case PluginSource.GITHUB: + return "Please specify the URL to a Github repository which contains built plugin releases. If this is a private repo you can provide a token to access it." + case PluginSource.FILE: + return "Please provide a built plugin TAR archive, you can build a plugin locally using the Budibase CLI." + } + } + async function save() { try { - if (source === Sources.FILE) { + if (source === PluginSource.FILE) { await plugins.uploadPlugin(file) } else { const url = dynamicValues["URL"] let auth = - source === Sources.GITHUB + source === PluginSource.GITHUB ? dynamicValues["Github Token"] - : source === Sources.URL + : source === PluginSource.URL ? dynamicValues["Headers"] : undefined await plugins.createPlugin(source, url, auth) @@ -63,14 +79,14 @@ +
+ + {#if option.optional} + + {/if} +
+ {#if option.name === "Headers"} + + {:else} + + {/if} {/if} {/each} diff --git a/packages/builder/src/stores/portal/plugins.js b/packages/builder/src/stores/portal/plugins.js index bfdec9ae6b..8997e8f49d 100644 --- a/packages/builder/src/stores/portal/plugins.js +++ b/packages/builder/src/stores/portal/plugins.js @@ -1,5 +1,6 @@ import { writable } from "svelte/store" import { API } from "api" +import { PluginSource } from "constants" export function createPluginsStore() { const { subscribe, set, update } = writable([]) @@ -24,13 +25,10 @@ export function createPluginsStore() { } switch (source) { - case "url": + case PluginSource.URL: pluginData.headers = auth break - case "npm": - pluginData.npmToken = auth - break - case "github": + case PluginSource.GITHUB: pluginData.githubToken = auth break } diff --git a/packages/server/src/api/controllers/plugin/file.ts b/packages/server/src/api/controllers/plugin/file.ts new file mode 100644 index 0000000000..3b21f08387 --- /dev/null +++ b/packages/server/src/api/controllers/plugin/file.ts @@ -0,0 +1,15 @@ +import { + createTempFolder, + getPluginMetadata, + extractTarball, +} from "../../../utilities/fileSystem" + +export async function fileUpload(file: { name: string; path: string }) { + if (!file.name.endsWith(".tar.gz")) { + throw new Error("Plugin must be compressed into a gzipped tarball.") + } + const path = createTempFolder(file.name.split(".tar.gz")[0]) + await extractTarball(file.path, path) + + return await getPluginMetadata(path) +} diff --git a/packages/server/src/api/controllers/plugin/github.ts b/packages/server/src/api/controllers/plugin/github.ts new file mode 100644 index 0000000000..cbd16f2386 --- /dev/null +++ b/packages/server/src/api/controllers/plugin/github.ts @@ -0,0 +1,75 @@ +import { getPluginMetadata } from "../../../utilities/fileSystem" +import fetch from "node-fetch" +import { downloadUnzipTarball } from "./utils" + +export async function request( + url: string, + headers: { [key: string]: string }, + err: string +) { + const response = await fetch(url, { headers }) + if (response.status >= 300) { + const respErr = await response.text() + throw new Error(`Error: ${err} - ${respErr}`) + } + return response.json() +} + +export async function githubUpload(url: string, name = "", token = "") { + let githubUrl = url + + if (!githubUrl.includes("https://github.com/")) { + throw new Error("The plugin origin must be from Github") + } + + if (url.includes(".git")) { + githubUrl = url.replace(".git", "") + } + + const githubApiUrl = githubUrl.replace( + "https://github.com/", + "https://api.github.com/repos/" + ) + const headers: any = token ? { Authorization: `Bearer ${token}` } : {} + const pluginDetails = await request( + githubApiUrl, + headers, + "Repository not found" + ) + const pluginName = pluginDetails.name || name + const pluginLatestReleaseUrl = pluginDetails?.["releases_url"] + ? pluginDetails?.["releases_url"].replace("{/id}", "/latest") + : undefined + if (!pluginLatestReleaseUrl) { + throw new Error("Github release not found") + } + + const pluginReleaseDetails = await request( + pluginLatestReleaseUrl, + headers, + "Github latest release not found" + ) + const pluginReleaseTarballAsset = pluginReleaseDetails?.assets?.find( + (x: any) => x?.["content_type"] === "application/gzip" + ) + const pluginLastReleaseTarballUrl = + pluginReleaseTarballAsset?.["browser_download_url"] + if (!pluginLastReleaseTarballUrl) { + throw new Error("Github latest release url not found") + } + + try { + const path = await downloadUnzipTarball( + pluginLastReleaseTarballUrl, + pluginName, + headers + ) + return await getPluginMetadata(path) + } catch (err: any) { + let errMsg = err?.message || err + if (errMsg === "unexpected response Not Found") { + errMsg = "Github release tarball not found" + } + throw new Error(errMsg) + } +} diff --git a/packages/server/src/api/controllers/plugin/index.ts b/packages/server/src/api/controllers/plugin/index.ts index 3f3914f5ef..18a64d23a5 100644 --- a/packages/server/src/api/controllers/plugin/index.ts +++ b/packages/server/src/api/controllers/plugin/index.ts @@ -1,11 +1,6 @@ import { ObjectStoreBuckets } from "../../../constants" import { loadJSFile } from "../../../utilities/fileSystem" -import { - uploadedNpmPlugin, - uploadedUrlPlugin, - uploadedGithubPlugin, - uploadedFilePlugin, -} from "./utils" +import { npmUpload, urlUpload, githubUpload, fileUpload } from "./uploaders" import { getGlobalDB } from "@budibase/backend-core/tenancy" import { validate } from "@budibase/backend-core/plugins" import { generatePluginID, getPluginParams } from "../../../db/utils" @@ -70,20 +65,20 @@ export async function create(ctx: any) { switch (source) { case PluginSource.NPM: const { metadata: metadataNpm, directory: directoryNpm } = - await uploadedNpmPlugin(url, name) + await npmUpload(url, name) metadata = metadataNpm directory = directoryNpm break case PluginSource.GITHUB: const { metadata: metadataGithub, directory: directoryGithub } = - await uploadedGithubPlugin(ctx, url, name, githubToken) + await githubUpload(url, name, githubToken) metadata = metadataGithub directory = directoryGithub break case PluginSource.URL: - const headersObj = JSON.parse(headers || null) || {} + const headersObj = headers || {} const { metadata: metadataUrl, directory: directoryUrl } = - await uploadedUrlPlugin(url, name, headersObj) + await urlUpload(url, name, headersObj) metadata = metadataUrl directory = directoryUrl break @@ -202,6 +197,6 @@ export async function processPlugin(plugin: FileType, source?: string) { throw new Error("Plugins not supported outside of self-host.") } - const { metadata, directory } = await uploadedFilePlugin(plugin) + const { metadata, directory } = await fileUpload(plugin) return await storePlugin(metadata, directory, source) } diff --git a/packages/server/src/api/controllers/plugin/npm.ts b/packages/server/src/api/controllers/plugin/npm.ts new file mode 100644 index 0000000000..9463fad44a --- /dev/null +++ b/packages/server/src/api/controllers/plugin/npm.ts @@ -0,0 +1,56 @@ +import { + getPluginMetadata, + findFileRec, + extractTarball, + deleteFolderFileSystem, +} from "../../../utilities/fileSystem" +import fetch from "node-fetch" +import { join } from "path" +import { downloadUnzipTarball } from "./utils" + +export async function npmUpload(url: string, name: string, headers = {}) { + let npmTarballUrl = url + let pluginName = name + + if ( + !npmTarballUrl.includes("https://www.npmjs.com") && + !npmTarballUrl.includes("https://registry.npmjs.org") + ) { + throw new Error("The plugin origin must be from NPM") + } + + if (!npmTarballUrl.includes(".tgz")) { + const npmPackageURl = url.replace( + "https://www.npmjs.com/package/", + "https://registry.npmjs.org/" + ) + const response = await fetch(npmPackageURl) + if (response.status !== 200) { + throw new Error("NPM Package not found") + } + + let npmDetails = await response.json() + pluginName = npmDetails.name + const npmVersion = npmDetails["dist-tags"].latest + npmTarballUrl = npmDetails?.versions?.[npmVersion]?.dist?.tarball + + if (!npmTarballUrl) { + throw new Error("NPM tarball url not found") + } + } + + const path = await downloadUnzipTarball(npmTarballUrl, pluginName, headers) + const tarballPluginFile = findFileRec(path, ".tar.gz") + if (!tarballPluginFile) { + throw new Error("Tarball plugin file not found") + } + + try { + await extractTarball(tarballPluginFile, path) + deleteFolderFileSystem(join(path, "package")) + } catch (err: any) { + throw new Error(err) + } + + return await getPluginMetadata(path) +} diff --git a/packages/server/src/api/controllers/plugin/uploaders.ts b/packages/server/src/api/controllers/plugin/uploaders.ts new file mode 100644 index 0000000000..90b3ab2e64 --- /dev/null +++ b/packages/server/src/api/controllers/plugin/uploaders.ts @@ -0,0 +1,4 @@ +export { fileUpload } from "./file" +export { githubUpload } from "./github" +export { npmUpload } from "./npm" +export { urlUpload } from "./url" diff --git a/packages/server/src/api/controllers/plugin/url.ts b/packages/server/src/api/controllers/plugin/url.ts new file mode 100644 index 0000000000..489631e114 --- /dev/null +++ b/packages/server/src/api/controllers/plugin/url.ts @@ -0,0 +1,12 @@ +import { downloadUnzipTarball } from "./utils" +import { getPluginMetadata } from "../../../utilities/fileSystem" + +export async function urlUpload(url: string, name = "", headers = {}) { + if (!url.includes(".tar.gz")) { + throw new Error("Plugin must be compressed into a gzipped tarball.") + } + + const path = await downloadUnzipTarball(url, name, headers) + + return await getPluginMetadata(path) +} diff --git a/packages/server/src/api/controllers/plugin/utils.js b/packages/server/src/api/controllers/plugin/utils.js deleted file mode 100644 index 37d8ad6c3d..0000000000 --- a/packages/server/src/api/controllers/plugin/utils.js +++ /dev/null @@ -1,153 +0,0 @@ -const { - createTempFolder, - getPluginMetadata, - findFileRec, - downloadTarballDirect, - extractTarball, - deleteFolderFileSystem, -} = require("../../../utilities/fileSystem") -const { join } = require("path") -const fetch = require("node-fetch") - -exports.uploadedFilePlugin = async file => { - if (!file.name.endsWith(".tar.gz")) { - throw new Error("Plugin must be compressed into a gzipped tarball.") - } - const path = createTempFolder(file.name.split(".tar.gz")[0]) - await extractTarball(file.path, path) - - return await getPluginMetadata(path) -} - -exports.uploadedNpmPlugin = async (url, name, headers = {}) => { - let npmTarballUrl = url - let pluginName = name - - if ( - !npmTarballUrl.includes("https://www.npmjs.com") && - !npmTarballUrl.includes("https://registry.npmjs.org") - ) { - throw new Error("The plugin origin must be from NPM") - } - - if (!npmTarballUrl.includes(".tgz")) { - const npmPackageURl = url.replace( - "https://www.npmjs.com/package/", - "https://registry.npmjs.org/" - ) - const response = await fetch(npmPackageURl) - if (response.status !== 200) { - throw new Error("NPM Package not found") - } - - let npmDetails = await response.json() - pluginName = npmDetails.name - const npmVersion = npmDetails["dist-tags"].latest - npmTarballUrl = npmDetails?.versions?.[npmVersion]?.dist?.tarball - - if (!npmTarballUrl) { - throw new Error("NPM tarball url not found") - } - } - - const path = await downloadUnzipTarball(npmTarballUrl, pluginName, headers) - const tarballPluginFile = findFileRec(path, ".tar.gz") - if (!tarballPluginFile) { - throw new Error("Tarball plugin file not found") - } - - try { - await extractTarball(tarballPluginFile, path) - deleteFolderFileSystem(join(path, "package")) - } catch (err) { - throw new Error(err) - } - - return await getPluginMetadata(path) -} - -exports.uploadedUrlPlugin = async (url, name = "", headers = {}) => { - if (!url.includes(".tar.gz")) { - throw new Error("Plugin must be compressed into a gzipped tarball.") - } - - const path = await downloadUnzipTarball(url, name, headers) - - return await getPluginMetadata(path) -} - -exports.uploadedGithubPlugin = async (ctx, url, name = "", token = "") => { - let githubUrl = url - - if (!githubUrl.includes("https://github.com/")) { - throw new Error("The plugin origin must be from Github") - } - - if (url.includes(".git")) { - githubUrl = url.replace(".git", "") - } - - const githubApiUrl = githubUrl.replace( - "https://github.com/", - "https://api.github.com/repos/" - ) - const headers = token ? { Authorization: `Bearer ${token}` } : {} - try { - const pluginRaw = await fetch(githubApiUrl, { headers }) - if (pluginRaw.status !== 200) { - throw new Error(`Repository not found`) - } - - let pluginDetails = await pluginRaw.json() - const pluginName = pluginDetails.name || name - - const pluginLatestReleaseUrl = pluginDetails?.["releases_url"] - ? pluginDetails?.["releases_url"].replace("{/id}", "/latest") - : undefined - if (!pluginLatestReleaseUrl) { - throw new Error("Github release not found") - } - - const pluginReleaseRaw = await fetch(pluginLatestReleaseUrl, { headers }) - if (pluginReleaseRaw.status !== 200) { - throw new Error("Github latest release not found") - } - const pluginReleaseDetails = await pluginReleaseRaw.json() - const pluginReleaseTarballAsset = pluginReleaseDetails?.assets?.find( - x => x?.content_type === "application/gzip" - ) - const pluginLastReleaseTarballUrl = - pluginReleaseTarballAsset?.browser_download_url - if (!pluginLastReleaseTarballUrl) { - throw new Error("Github latest release url not found") - } - - const path = await downloadUnzipTarball( - pluginLastReleaseTarballUrl, - pluginName, - headers - ) - - return await getPluginMetadata(path) - } catch (err) { - let errMsg = err?.message || err - - if (errMsg === "unexpected response Not Found") { - errMsg = "Github release tarbal not found" - } - - throw new Error(errMsg) - } -} - -const downloadUnzipTarball = async (url, name, headers = {}) => { - try { - const path = createTempFolder(name) - - await downloadTarballDirect(url, path, headers) - - return path - } catch (e) { - throw new Error(e.message) - } -} diff --git a/packages/server/src/api/controllers/plugin/utils.ts b/packages/server/src/api/controllers/plugin/utils.ts new file mode 100644 index 0000000000..0e92fbb987 --- /dev/null +++ b/packages/server/src/api/controllers/plugin/utils.ts @@ -0,0 +1,19 @@ +import { + createTempFolder, + downloadTarballDirect, +} from "../../../utilities/fileSystem" + +export async function downloadUnzipTarball( + url: string, + name: string, + headers = {} +) { + try { + const path = createTempFolder(name) + await downloadTarballDirect(url, path, headers) + + return path + } catch (e: any) { + throw new Error(e.message) + } +}