From c2bca8a025be85257222a289a4ff93fe36178705 Mon Sep 17 00:00:00 2001 From: NEOLPAR Date: Tue, 6 Sep 2022 16:28:35 +0100 Subject: [PATCH] uploading plugin for github, npm and url --- packages/server/src/api/controllers/plugin.ts | 36 ++-- .../src/api/controllers/plugin/utils.js | 153 +++++++++++++-- .../server/src/utilities/fileSystem/index.js | 184 +++++++----------- 3 files changed, 229 insertions(+), 144 deletions(-) diff --git a/packages/server/src/api/controllers/plugin.ts b/packages/server/src/api/controllers/plugin.ts index 0970bafceb..489c0715a2 100644 --- a/packages/server/src/api/controllers/plugin.ts +++ b/packages/server/src/api/controllers/plugin.ts @@ -1,11 +1,11 @@ import { ObjectStoreBuckets } from "../../constants" +import { loadJSFile } from "../../utilities/fileSystem" import { - extractPluginTarball, - createUrlPlugin, - createGithubPlugin, - loadJSFile, -} from "../../utilities/fileSystem" -import { createNpmPlugin } from "./plugin/utils" + uploadedNpmPlugin, + uploadedUrlPlugin, + uploadedGithubPlugin, + uploadedFilePlugin, +} from "./plugin/utils" import { getGlobalDB } from "@budibase/backend-core/tenancy" import { generatePluginID, getPluginParams } from "../../db/utils" import { @@ -60,22 +60,32 @@ export async function create(ctx: any) { // Generating random name as a backup and needed for url let name = "PLUGIN_" + Math.floor(100000 + Math.random() * 900000) + if (!env.SELF_HOSTED) { + throw new Error("Plugins not supported outside of self-host.") + } + switch (source) { case "npm": - const { metadata: metadataNpm, directory: directoryNpm } = - await createNpmPlugin(url, name) - metadata = metadataNpm - directory = directoryNpm + try { + const { metadata: metadataNpm, directory: directoryNpm } = + await uploadedNpmPlugin(url, name) + metadata = metadataNpm + directory = directoryNpm + } catch (err: any) { + const errMsg = err?.message ? err?.message : err + + ctx.throw(400, `Failed to import plugin: ${errMsg}`) + } break case "github": const { metadata: metadataGithub, directory: directoryGithub } = - await createGithubPlugin(ctx, url, name, githubToken) + await uploadedGithubPlugin(ctx, url, name, githubToken) metadata = metadataGithub directory = directoryGithub break case "url": const { metadata: metadataUrl, directory: directoryUrl } = - await createUrlPlugin(url, name, headers) + await uploadedUrlPlugin(url, name, headers) metadata = metadataUrl directory = directoryUrl break @@ -192,6 +202,6 @@ export async function processPlugin(plugin: FileType, source?: string) { throw new Error("Plugins not supported outside of self-host.") } - const { metadata, directory } = await extractPluginTarball(plugin) + const { metadata, directory } = await uploadedFilePlugin(plugin) return await storePlugin(metadata, directory, source) } diff --git a/packages/server/src/api/controllers/plugin/utils.js b/packages/server/src/api/controllers/plugin/utils.js index 3227a91a98..86f8754a7a 100644 --- a/packages/server/src/api/controllers/plugin/utils.js +++ b/packages/server/src/api/controllers/plugin/utils.js @@ -1,32 +1,153 @@ -const fetch = require("node-fetch") -import { downloadUnzipPlugin } from "../../../utilities/fileSystem" +import fetch from "node-fetch" +import { + createTempFolder, + getPluginMetadata, + findFileRec, + downloadTarballDirect, + extractTarball, + deleteFolderFileSystem, +} from "../../../utilities/fileSystem" +import { join } from "path" -export const createNpmPlugin = async (url, name = "") => { - let npmTarball = url +export const uploadedFilePlugin = async file => { + if (!file.name.endsWith(".tar.gz")) { + throw new Error("Plugin must be compressed into a gzipped tarball.") + } + const path = createTempFolder(file.name.split(".tar.gz")[0]) + await extractTarball(file.path, path) + + return await getPluginMetadata(path) +} + +export const uploadedNpmPlugin = async (url, name, headers = {}) => { + let npmTarballUrl = url let pluginName = name if ( - !npmTarball.includes("https://www.npmjs.com") && - !npmTarball.includes("https://registry.npmjs.org") + !npmTarballUrl.includes("https://www.npmjs.com") && + !npmTarballUrl.includes("https://registry.npmjs.org") ) { - throw "The plugin origin must be from NPM" + throw new Error("The plugin origin must be from NPM") } - if (!npmTarball.includes(".tgz")) { + if (!npmTarballUrl.includes(".tgz")) { const npmPackageURl = url.replace( "https://www.npmjs.com/package/", "https://registry.npmjs.org/" ) const response = await fetch(npmPackageURl) - if (response.status === 200) { - let npmDetails = await response.json() - pluginName = npmDetails.name - const npmVersion = npmDetails["dist-tags"].latest - npmTarball = npmDetails.versions[npmVersion].dist.tarball - } else { - throw "Cannot get package details" + if (response.status !== 200) { + throw new Error("NPM Package not found") + } + + let npmDetails = await response.json() + pluginName = npmDetails.name + const npmVersion = npmDetails["dist-tags"].latest + npmTarballUrl = npmDetails?.versions?.[npmVersion]?.dist?.tarball + + if (!npmTarballUrl) { + throw new Error("NPM tarball url not found") } } - return await downloadUnzipPlugin(pluginName, npmTarball) + const path = await downloadUnzipTarball(npmTarballUrl, pluginName, headers) + const tarballPluginFile = findFileRec(path, ".tar.gz") + if (!tarballPluginFile) { + throw new Error("Tarball plugin file not found") + } + + try { + await extractTarball(tarballPluginFile, path) + deleteFolderFileSystem(join(path, "package")) + } catch (err) { + throw new Error(err) + } + + return await getPluginMetadata(path) +} + +export const uploadedUrlPlugin = async (url, name = "", headers = {}) => { + if (!url.includes(".tar.gz")) { + throw new Error("Plugin must be compressed into a gzipped tarball.") + } + + const path = await downloadUnzipTarball(url, name, headers) + + return await getPluginMetadata(path) +} + +export const uploadedGithubPlugin = async (ctx, url, name = "", token = "") => { + let githubUrl = url + + if (!githubUrl.includes("https://github.com/")) { + throw new Error("The plugin origin must be from Github") + } + + if (url.includes(".git")) { + githubUrl = url.replace(".git", "") + } + + const githubApiUrl = githubUrl.replace( + "https://github.com/", + "https://api.github.com/repos/" + ) + const headers = token ? { Authorization: `Bearer ${token}` } : {} + try { + const pluginRaw = await fetch(githubApiUrl, { headers }) + if (pluginRaw.status !== 200) { + throw new Error(`Repository not found`) + } + + let pluginDetails = await pluginRaw.json() + const pluginName = pluginDetails.name || name + + const pluginLatestReleaseUrl = pluginDetails?.["releases_url"] + ? pluginDetails?.["releases_url"].replace("{/id}", "/latest") + : undefined + if (!pluginLatestReleaseUrl) { + throw new Error("Github release not found") + } + + const pluginReleaseRaw = await fetch(pluginLatestReleaseUrl, { headers }) + if (pluginReleaseRaw.status !== 200) { + throw new Error("Github latest release not found") + } + const pluginReleaseDetails = await pluginReleaseRaw.json() + const pluginReleaseTarballAsset = pluginReleaseDetails?.assets?.find( + x => x?.content_type === "application/gzip" + ) + const pluginLastReleaseTarballUrl = + pluginReleaseTarballAsset?.browser_download_url + if (!pluginLastReleaseTarballUrl) { + throw new Error("Github latest release url not found") + } + + const path = await downloadUnzipTarball( + pluginLastReleaseTarballUrl, + pluginName, + headers + ) + + return await getPluginMetadata(path) + } catch (err) { + let errMsg = err?.message || err + + if (errMsg === "unexpected response Not Found") { + errMsg = "Github release tarbal not found" + } + + throw new Error(errMsg) + } +} + +export const downloadUnzipTarball = async (url, name, headers = {}) => { + try { + const path = createTempFolder(name) + + await downloadTarballDirect(url, path, headers) + + return path + } catch (e) { + throw new Error(e.message) + } } diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 7bbf1db425..578ff1e31a 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -1,8 +1,6 @@ const { budibaseTempDir } = require("../budibaseDir") const fs = require("fs") const { join } = require("path") -const { promisify } = require("util") -const streamPipeline = promisify(require("stream").pipeline) const uuid = require("uuid/v4") const { doWithDB, @@ -17,6 +15,7 @@ const { streamUpload, deleteFolder, downloadTarball, + downloadTarballDirect, deleteFiles, } = require("./utilities") const { updateClientLibrary } = require("./clientLibrary") @@ -31,7 +30,6 @@ const MemoryStream = require("memorystream") const { getAppId } = require("@budibase/backend-core/context") const tar = require("tar") const fetch = require("node-fetch") -const { NodeVM } = require("vm2") const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..") const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules") @@ -341,131 +339,52 @@ exports.cleanup = appIds => { } } -const extractPluginTarball = async (file, ext = ".tar.gz") => { - if (!file.name.endsWith(ext)) { - throw new Error("Plugin must be compressed into a gzipped tarball.") +const createTempFolder = item => { + const path = join(budibaseTempDir(), item) + try { + // remove old tmp directories automatically - don't combine + if (fs.existsSync(path)) { + fs.rmSync(path, { recursive: true, force: true }) + } + fs.mkdirSync(path) + } catch (err) { + throw new Error(`Path cannot be created: ${err.message}`) } - const path = join(budibaseTempDir(), file.name.split(ext)[0]) - // remove old tmp directories automatically - don't combine - if (fs.existsSync(path)) { - fs.rmSync(path, { recursive: true, force: true }) - } - fs.mkdirSync(path) + + return path +} +exports.createTempFolder = createTempFolder + +const extractTarball = async (fromFilePath, toPath) => { await tar.extract({ - file: file.path, - C: path, + file: fromFilePath, + C: toPath, }) - - return await getPluginMetadata(path) } -exports.extractPluginTarball = extractPluginTarball - -exports.createUrlPlugin = async (url, name = "", headers = {}) => { - if (!url.includes(".tgz") && !url.includes(".tar.gz")) { - throw new Error("Plugin must be compressed into a gzipped tarball.") - } - - return await downloadUnzipPlugin(name, url, headers) -} - -exports.createGithubPlugin = async (ctx, url, name = "", token = "") => { - let githubRepositoryUrl - let githubUrl - - if (url.includes(".git")) { - githubRepositoryUrl = token - ? url.replace("https://", `https://${token}@`) - : url - githubUrl = url.replace(".git", "") - } else { - githubRepositoryUrl = token - ? `${url}.git`.replace("https://", `https://${token}@`) - : `${url}.git` - githubUrl = url - } - - const githubApiUrl = githubUrl.replace( - "https://github.com/", - "https://api.github.com/repos/" - ) - const headers = token ? { Authorization: `Bearer ${token}` } : {} - try { - const pluginRaw = await fetch(githubApiUrl, { headers }) - if (pluginRaw.status !== 200) { - throw `Repository not found` - } - - let pluginDetails = await pluginRaw.json() - const pluginName = pluginDetails.name || name - - const path = join(budibaseTempDir(), pluginName) - // Remove first if exists - if (fs.existsSync(path)) { - fs.rmSync(path, { recursive: true, force: true }) - } - fs.mkdirSync(path) - - const script = ` -module.exports = async () => { - const child_process = require('child_process') - child_process.execSync(\`git clone ${githubRepositoryUrl} ${join( - budibaseTempDir(), - pluginName - )}\`); -} -` - const scriptRunner = new NodeVM({ - require: { - external: true, - builtin: ["child_process"], - root: "./", - }, - }).run(script) - - await scriptRunner() - - return await getPluginMetadata(path) - } catch (e) { - throw e.message - } -} - -const downloadUnzipPlugin = async (name, url, headers = {}) => { - const path = join(budibaseTempDir(), name) - try { - // Remove first if exists - if (fs.existsSync(path)) { - fs.rmSync(path, { recursive: true, force: true }) - } - fs.mkdirSync(path) - - const response = await fetch(url, { headers }) - if (!response.ok) - throw new Error(`Loading NPM plugin failed ${response.statusText}`) - - await streamPipeline( - response.body, - tar.x({ - strip: 1, - C: path, - }) - ) - return await getPluginMetadata(path) - } catch (e) { - throw `Cannot store plugin locally: ${e.message}` - } -} -exports.downloadUnzipPlugin = downloadUnzipPlugin +exports.extractTarball = extractTarball const getPluginMetadata = async path => { let metadata = {} try { const pkg = fs.readFileSync(join(path, "package.json"), "utf8") const schema = fs.readFileSync(join(path, "schema.json"), "utf8") + metadata.schema = JSON.parse(schema) metadata.package = JSON.parse(pkg) + + if ( + !metadata.package.name || + !metadata.package.version || + !metadata.package.description + ) { + throw new Error( + "package.json is missing one of 'name', 'version' or 'description'." + ) + } } catch (err) { - throw new Error("Unable to process schema.json/package.json in plugin.") + throw new Error( + `Unable to process schema.json/package.json in plugin. ${err.message}` + ) } return { metadata, directory: path } @@ -504,6 +423,40 @@ exports.getDatasourcePlugin = async (name, url, hash) => { } } +/** + * Find for a file recursively from start path applying filter, return first match + */ +const findFileRec = (startPath, filter) => { + if (!fs.existsSync(startPath)) { + return + } + + var files = fs.readdirSync(startPath) + for (let i = 0, len = files.length; i < len; i++) { + const filename = join(startPath, files[i]) + const stat = fs.lstatSync(filename) + + if (stat.isDirectory()) { + return findFileRec(filename, filter) + } else if (filename.endsWith(filter)) { + return filename + } + } +} +exports.findFileRec = findFileRec + +/** + * Remove a folder which is not empty from the file system + */ +const deleteFolderFileSystem = path => { + if (!fs.existsSync(path)) { + return + } + + fs.rmSync(path, { recursive: true, force: true }) +} +exports.deleteFolderFileSystem = deleteFolderFileSystem + /** * Full function definition for below can be found in the utilities. */ @@ -511,5 +464,6 @@ exports.upload = upload exports.retrieve = retrieve exports.retrieveToTmp = retrieveToTmp exports.deleteFiles = deleteFiles +exports.downloadTarballDirect = downloadTarballDirect exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH exports.NODE_MODULES_PATH = NODE_MODULES_PATH