MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Updating plugin backend, refactoring a bit, correctly allowing a set of headers to be used for a URL.

This commit is contained in:
mike12345567 2022-09-12 19:12:05 +01:00
parent e7cfdd4fde
commit 30817751ad
12 changed files with 251 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
packages/bbui/src/Label/Label.svelte
View File

@ -4,10 +4,15 @@
export let size = "M" export let size = "M"
export let tooltip = "" export let tooltip = ""
export let muted
</script> </script>
<TooltipWrapper {tooltip} {size}> <TooltipWrapper {tooltip} {size}>
<label for="" class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`}> <label
class:muted
for=""
class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`}
>
<slot /> <slot />
</label> </label>
</TooltipWrapper> </TooltipWrapper>
@ -17,4 +22,8 @@
padding: 0; padding: 0;
white-space: nowrap; white-space: nowrap;
} }
.muted {
opacity: 0.5;
}
</style> </style>

7
packages/builder/src/constants/index.js
View File

@ -57,3 +57,10 @@ export const DefaultAppTheme = {
navBackground: "var(--spectrum-global-color-gray-50)", navBackground: "var(--spectrum-global-color-gray-50)",
navTextColor: "var(--spectrum-global-color-gray-800)", navTextColor: "var(--spectrum-global-color-gray-800)",
} }
export const PluginSource = {
URL: "URL",
NPM: "NPM",
GITHUB: "Github",
FILE: "File Upload",
}

63
packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte
View File

@ -5,40 +5,56 @@
Input, Input,
Select, Select,
Dropzone, Dropzone,
Body,
notifications, notifications,
} from "@budibase/bbui" } from "@budibase/bbui"
import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
import { plugins } from "stores/portal" import { plugins } from "stores/portal"
import { PluginSource } from "constants"
const Sources = { function opt(name, optional) {
NPM: "NPM", if (optional) {
GITHUB: "Github", return { name, optional }
URL: "URL", }
FILE: "File Upload", return { name }
} }
let authOptions = { let authOptions = {
[Sources.NPM]: ["URL"], [PluginSource.URL]: [opt("URL"), opt("Headers", true)],
[Sources.GITHUB]: ["Github Token", "URL"], [PluginSource.NPM]: [opt("URL")],
[Sources.URL]: ["Headers", "URL"], [PluginSource.GITHUB]: [opt("URL"), opt("Github Token", true)],
[Sources.FILE]: ["File Upload"], [PluginSource.FILE]: [opt("File Upload")],
} }
let file let file
let source = Sources.URL let source = PluginSource.URL
let dynamicValues = {} let dynamicValues = {}
let validation let validation
$: validation = source === "File Upload" ? file : dynamicValues["URL"] $: validation = source === "File Upload" ? file : dynamicValues["URL"]
function infoMessage(optionName) {
switch (optionName) {
case PluginSource.URL:
return "Please specify a URL which directs to a built plugin TAR archive, you can provide headers if authentication is required."
case PluginSource.NPM:
return "Please specify the URL to a public NPM package which contains the built version of the plugin you wish to install."
case PluginSource.GITHUB:
return "Please specify the URL to a Github repository which contains built plugin releases. If this is a private repo you can provide a token to access it."
case PluginSource.FILE:
return "Please provide a built plugin TAR archive, you can build a plugin locally using the Budibase CLI."
}
}
async function save() { async function save() {
try { try {
if (source === Sources.FILE) { if (source === PluginSource.FILE) {
await plugins.uploadPlugin(file) await plugins.uploadPlugin(file)
} else { } else {
const url = dynamicValues["URL"] const url = dynamicValues["URL"]
let auth = let auth =
source === Sources.GITHUB source === PluginSource.GITHUB
? dynamicValues["Github Token"] ? dynamicValues["Github Token"]
: source === Sources.URL : source === PluginSource.URL
? dynamicValues["Headers"] ? dynamicValues["Headers"]
: undefined : undefined
await plugins.createPlugin(source, url, auth) await plugins.createPlugin(source, url, auth)
@ -63,14 +79,14 @@
<Select <Select
placeholder={null} placeholder={null}
bind:value={source} bind:value={source}
options={Object.values(Sources)} options={Object.values(PluginSource)}
/> />
</div> </div>
<Body size="S">{infoMessage(source)}</Body>
{#each authOptions[source] as option} {#each authOptions[source] as option}
{#if option === "File Upload"} {#if option.name === PluginSource.FILE}
<div class="form-row"> <div class="form-row">
<Label size="M">{option}</Label> <Label size="M">{option.name}</Label>
<Dropzone <Dropzone
gallery={false} gallery={false}
value={[file]} value={[file]}
@ -85,8 +101,17 @@
</div> </div>
{:else} {:else}
<div class="form-row"> <div class="form-row">
<Label size="M">{option}</Label> <div>
<Input bind:value={dynamicValues[option]} /> <Label size="M">{option.name}</Label>
{#if option.optional}
<Label size="S" muted><i>Optional</i></Label>
{/if}
</div>
{#if option.name === "Headers"}
<KeyValueBuilder bind:object={dynamicValues[option.name]} />
{:else}
<Input bind:value={dynamicValues[option.name]} />
{/if}
</div> </div>
{/if} {/if}
{/each} {/each}

8
packages/builder/src/stores/portal/plugins.js
View File

@ -1,5 +1,6 @@
import { writable } from "svelte/store" import { writable } from "svelte/store"
import { API } from "api" import { API } from "api"
import { PluginSource } from "constants"
export function createPluginsStore() { export function createPluginsStore() {
const { subscribe, set, update } = writable([]) const { subscribe, set, update } = writable([])
@ -24,13 +25,10 @@ export function createPluginsStore() {
} }
switch (source) { switch (source) {
case "url": case PluginSource.URL:
pluginData.headers = auth pluginData.headers = auth
break break
case "npm": case PluginSource.GITHUB:
pluginData.npmToken = auth
break
case "github":
pluginData.githubToken = auth pluginData.githubToken = auth
break break
} }

15
packages/server/src/api/controllers/plugin/file.ts Normal file
View File

@ -0,0 +1,15 @@
import {
createTempFolder,
getPluginMetadata,
extractTarball,
} from "../../../utilities/fileSystem"
export async function fileUpload(file: { name: string; path: string }) {
if (!file.name.endsWith(".tar.gz")) {
throw new Error("Plugin must be compressed into a gzipped tarball.")
}
const path = createTempFolder(file.name.split(".tar.gz")[0])
await extractTarball(file.path, path)
return await getPluginMetadata(path)
}

75
packages/server/src/api/controllers/plugin/github.ts Normal file
View File

@ -0,0 +1,75 @@
import { getPluginMetadata } from "../../../utilities/fileSystem"
import fetch from "node-fetch"
import { downloadUnzipTarball } from "./utils"
export async function request(
url: string,
headers: { [key: string]: string },
err: string
) {
const response = await fetch(url, { headers })
if (response.status >= 300) {
const respErr = await response.text()
throw new Error(`Error: ${err} - ${respErr}`)
}
return response.json()
}
export async function githubUpload(url: string, name = "", token = "") {
let githubUrl = url
if (!githubUrl.includes("https://github.com/")) {
throw new Error("The plugin origin must be from Github")
}
if (url.includes(".git")) {
githubUrl = url.replace(".git", "")
}
const githubApiUrl = githubUrl.replace(
"https://github.com/",
"https://api.github.com/repos/"
)
const headers: any = token ? { Authorization: `Bearer ${token}` } : {}
const pluginDetails = await request(
githubApiUrl,
headers,
"Repository not found"
)
const pluginName = pluginDetails.name || name
const pluginLatestReleaseUrl = pluginDetails?.["releases_url"]
? pluginDetails?.["releases_url"].replace("{/id}", "/latest")
: undefined
if (!pluginLatestReleaseUrl) {
throw new Error("Github release not found")
}
const pluginReleaseDetails = await request(
pluginLatestReleaseUrl,
headers,
"Github latest release not found"
)
const pluginReleaseTarballAsset = pluginReleaseDetails?.assets?.find(
(x: any) => x?.["content_type"] === "application/gzip"
)
const pluginLastReleaseTarballUrl =
pluginReleaseTarballAsset?.["browser_download_url"]
if (!pluginLastReleaseTarballUrl) {
throw new Error("Github latest release url not found")
}
try {
const path = await downloadUnzipTarball(
pluginLastReleaseTarballUrl,
pluginName,
headers
)
return await getPluginMetadata(path)
} catch (err: any) {
let errMsg = err?.message || err
if (errMsg === "unexpected response Not Found") {
errMsg = "Github release tarball not found"
}
throw new Error(errMsg)
}
}

17
packages/server/src/api/controllers/plugin/index.ts
View File

@ -1,11 +1,6 @@
import { ObjectStoreBuckets } from "../../../constants" import { ObjectStoreBuckets } from "../../../constants"
import { loadJSFile } from "../../../utilities/fileSystem" import { loadJSFile } from "../../../utilities/fileSystem"
import { import { npmUpload, urlUpload, githubUpload, fileUpload } from "./uploaders"
uploadedNpmPlugin,
uploadedUrlPlugin,
uploadedGithubPlugin,
uploadedFilePlugin,
} from "./utils"
import { getGlobalDB } from "@budibase/backend-core/tenancy" import { getGlobalDB } from "@budibase/backend-core/tenancy"
import { validate } from "@budibase/backend-core/plugins" import { validate } from "@budibase/backend-core/plugins"
import { generatePluginID, getPluginParams } from "../../../db/utils" import { generatePluginID, getPluginParams } from "../../../db/utils"
@ -70,20 +65,20 @@ export async function create(ctx: any) {
switch (source) { switch (source) {
case PluginSource.NPM: case PluginSource.NPM:
const { metadata: metadataNpm, directory: directoryNpm } = const { metadata: metadataNpm, directory: directoryNpm } =
await uploadedNpmPlugin(url, name) await npmUpload(url, name)
metadata = metadataNpm metadata = metadataNpm
directory = directoryNpm directory = directoryNpm
break break
case PluginSource.GITHUB: case PluginSource.GITHUB:
const { metadata: metadataGithub, directory: directoryGithub } = const { metadata: metadataGithub, directory: directoryGithub } =
await uploadedGithubPlugin(ctx, url, name, githubToken) await githubUpload(url, name, githubToken)
metadata = metadataGithub metadata = metadataGithub
directory = directoryGithub directory = directoryGithub
break break
case PluginSource.URL: case PluginSource.URL:
const headersObj = JSON.parse(headers || null) || {} const headersObj = headers || {}
const { metadata: metadataUrl, directory: directoryUrl } = const { metadata: metadataUrl, directory: directoryUrl } =
await uploadedUrlPlugin(url, name, headersObj) await urlUpload(url, name, headersObj)
metadata = metadataUrl metadata = metadataUrl
directory = directoryUrl directory = directoryUrl
break break
@ -202,6 +197,6 @@ export async function processPlugin(plugin: FileType, source?: string) {
throw new Error("Plugins not supported outside of self-host.") throw new Error("Plugins not supported outside of self-host.")
} }
const { metadata, directory } = await uploadedFilePlugin(plugin) const { metadata, directory } = await fileUpload(plugin)
return await storePlugin(metadata, directory, source) return await storePlugin(metadata, directory, source)
} }

56
packages/server/src/api/controllers/plugin/npm.ts Normal file
View File

@ -0,0 +1,56 @@
import {
getPluginMetadata,
findFileRec,
extractTarball,
deleteFolderFileSystem,
} from "../../../utilities/fileSystem"
import fetch from "node-fetch"
import { join } from "path"
import { downloadUnzipTarball } from "./utils"
export async function npmUpload(url: string, name: string, headers = {}) {
let npmTarballUrl = url
let pluginName = name
if (
!npmTarballUrl.includes("https://www.npmjs.com") &&
!npmTarballUrl.includes("https://registry.npmjs.org")
) {
throw new Error("The plugin origin must be from NPM")
}
if (!npmTarballUrl.includes(".tgz")) {
const npmPackageURl = url.replace(
"https://www.npmjs.com/package/",
"https://registry.npmjs.org/"
)
const response = await fetch(npmPackageURl)
if (response.status !== 200) {
throw new Error("NPM Package not found")
}
let npmDetails = await response.json()
pluginName = npmDetails.name
const npmVersion = npmDetails["dist-tags"].latest
npmTarballUrl = npmDetails?.versions?.[npmVersion]?.dist?.tarball
if (!npmTarballUrl) {
throw new Error("NPM tarball url not found")
}
}
const path = await downloadUnzipTarball(npmTarballUrl, pluginName, headers)
const tarballPluginFile = findFileRec(path, ".tar.gz")
if (!tarballPluginFile) {
throw new Error("Tarball plugin file not found")
}
try {
await extractTarball(tarballPluginFile, path)
deleteFolderFileSystem(join(path, "package"))
} catch (err: any) {
throw new Error(err)
}
return await getPluginMetadata(path)
}

4
packages/server/src/api/controllers/plugin/uploaders.ts Normal file
View File

@ -0,0 +1,4 @@
export { fileUpload } from "./file"
export { githubUpload } from "./github"
export { npmUpload } from "./npm"
export { urlUpload } from "./url"

12
packages/server/src/api/controllers/plugin/url.ts Normal file
View File

@ -0,0 +1,12 @@
import { downloadUnzipTarball } from "./utils"
import { getPluginMetadata } from "../../../utilities/fileSystem"
export async function urlUpload(url: string, name = "", headers = {}) {
if (!url.includes(".tar.gz")) {
throw new Error("Plugin must be compressed into a gzipped tarball.")
}
const path = await downloadUnzipTarball(url, name, headers)
return await getPluginMetadata(path)
}

153
packages/server/src/api/controllers/plugin/utils.js
View File

@ -1,153 +0,0 @@
const {
createTempFolder,
getPluginMetadata,
findFileRec,
downloadTarballDirect,
extractTarball,
deleteFolderFileSystem,
} = require("../../../utilities/fileSystem")
const { join } = require("path")
const fetch = require("node-fetch")
exports.uploadedFilePlugin = async file => {
if (!file.name.endsWith(".tar.gz")) {
throw new Error("Plugin must be compressed into a gzipped tarball.")
}
const path = createTempFolder(file.name.split(".tar.gz")[0])
await extractTarball(file.path, path)
return await getPluginMetadata(path)
}
exports.uploadedNpmPlugin = async (url, name, headers = {}) => {
let npmTarballUrl = url
let pluginName = name
if (
!npmTarballUrl.includes("https://www.npmjs.com") &&
!npmTarballUrl.includes("https://registry.npmjs.org")
) {
throw new Error("The plugin origin must be from NPM")
}
if (!npmTarballUrl.includes(".tgz")) {
const npmPackageURl = url.replace(
"https://www.npmjs.com/package/",
"https://registry.npmjs.org/"
)
const response = await fetch(npmPackageURl)
if (response.status !== 200) {
throw new Error("NPM Package not found")
}
let npmDetails = await response.json()
pluginName = npmDetails.name
const npmVersion = npmDetails["dist-tags"].latest
npmTarballUrl = npmDetails?.versions?.[npmVersion]?.dist?.tarball
if (!npmTarballUrl) {
throw new Error("NPM tarball url not found")
}
}
const path = await downloadUnzipTarball(npmTarballUrl, pluginName, headers)
const tarballPluginFile = findFileRec(path, ".tar.gz")
if (!tarballPluginFile) {
throw new Error("Tarball plugin file not found")
}
try {
await extractTarball(tarballPluginFile, path)
deleteFolderFileSystem(join(path, "package"))
} catch (err) {
throw new Error(err)
}
return await getPluginMetadata(path)
}
exports.uploadedUrlPlugin = async (url, name = "", headers = {}) => {
if (!url.includes(".tar.gz")) {
throw new Error("Plugin must be compressed into a gzipped tarball.")
}
const path = await downloadUnzipTarball(url, name, headers)
return await getPluginMetadata(path)
}
exports.uploadedGithubPlugin = async (ctx, url, name = "", token = "") => {
let githubUrl = url
if (!githubUrl.includes("https://github.com/")) {
throw new Error("The plugin origin must be from Github")
}
if (url.includes(".git")) {
githubUrl = url.replace(".git", "")
}
const githubApiUrl = githubUrl.replace(
"https://github.com/",
"https://api.github.com/repos/"
)
const headers = token ? { Authorization: `Bearer ${token}` } : {}
try {
const pluginRaw = await fetch(githubApiUrl, { headers })
if (pluginRaw.status !== 200) {
throw new Error(`Repository not found`)
}
let pluginDetails = await pluginRaw.json()
const pluginName = pluginDetails.name || name
const pluginLatestReleaseUrl = pluginDetails?.["releases_url"]
? pluginDetails?.["releases_url"].replace("{/id}", "/latest")
: undefined
if (!pluginLatestReleaseUrl) {
throw new Error("Github release not found")
}
const pluginReleaseRaw = await fetch(pluginLatestReleaseUrl, { headers })
if (pluginReleaseRaw.status !== 200) {
throw new Error("Github latest release not found")
}
const pluginReleaseDetails = await pluginReleaseRaw.json()
const pluginReleaseTarballAsset = pluginReleaseDetails?.assets?.find(
x => x?.content_type === "application/gzip"
)
const pluginLastReleaseTarballUrl =
pluginReleaseTarballAsset?.browser_download_url
if (!pluginLastReleaseTarballUrl) {
throw new Error("Github latest release url not found")
}
const path = await downloadUnzipTarball(
pluginLastReleaseTarballUrl,
pluginName,
headers
)
return await getPluginMetadata(path)
} catch (err) {
let errMsg = err?.message || err
if (errMsg === "unexpected response Not Found") {
errMsg = "Github release tarbal not found"
}
throw new Error(errMsg)
}
}
const downloadUnzipTarball = async (url, name, headers = {}) => {
try {
const path = createTempFolder(name)
await downloadTarballDirect(url, path, headers)
return path
} catch (e) {
throw new Error(e.message)
}
}

19
packages/server/src/api/controllers/plugin/utils.ts Normal file
View File

@ -0,0 +1,19 @@
import {
createTempFolder,
downloadTarballDirect,
} from "../../../utilities/fileSystem"
export async function downloadUnzipTarball(
url: string,
name: string,
headers = {}
) {
try {
const path = createTempFolder(name)
await downloadTarballDirect(url, path, headers)
return path
} catch (e: any) {
throw new Error(e.message)
}
}