uploading plugin for github, npm and url

packages/server/src/api/controllers/plugin.ts

@@ -1,11 +1,11 @@
 import { ObjectStoreBuckets } from "../../constants"
+import { loadJSFile } from "../../utilities/fileSystem"
 import {
-  extractPluginTarball,
+  uploadedNpmPlugin,
-  createUrlPlugin,
+  uploadedUrlPlugin,
-  createGithubPlugin,
+  uploadedGithubPlugin,
-  loadJSFile,
+  uploadedFilePlugin,
-} from "../../utilities/fileSystem"
+} from "./plugin/utils"
-import { createNpmPlugin } from "./plugin/utils"
 import { getGlobalDB } from "@budibase/backend-core/tenancy"
 import { generatePluginID, getPluginParams } from "../../db/utils"
 import {
@@ -60,22 +60,32 @@ export async function create(ctx: any) {
   // Generating random name as a backup and needed for url
   let name = "PLUGIN_" + Math.floor(100000 + Math.random() * 900000)
 
+  if (!env.SELF_HOSTED) {
+    throw new Error("Plugins not supported outside of self-host.")
+  }
+
   switch (source) {
     case "npm":
+      try {
         const { metadata: metadataNpm, directory: directoryNpm } =
-        await createNpmPlugin(url, name)
+          await uploadedNpmPlugin(url, name)
         metadata = metadataNpm
         directory = directoryNpm
+      } catch (err: any) {
+        const errMsg = err?.message ? err?.message : err
+
+        ctx.throw(400, `Failed to import plugin: ${errMsg}`)
+      }
       break
     case "github":
       const { metadata: metadataGithub, directory: directoryGithub } =
-        await createGithubPlugin(ctx, url, name, githubToken)
+        await uploadedGithubPlugin(ctx, url, name, githubToken)
       metadata = metadataGithub
       directory = directoryGithub
       break
     case "url":
       const { metadata: metadataUrl, directory: directoryUrl } =
-        await createUrlPlugin(url, name, headers)
+        await uploadedUrlPlugin(url, name, headers)
       metadata = metadataUrl
       directory = directoryUrl
       break
@@ -192,6 +202,6 @@ export async function processPlugin(plugin: FileType, source?: string) {
     throw new Error("Plugins not supported outside of self-host.")
   }
 
-  const { metadata, directory } = await extractPluginTarball(plugin)
+  const { metadata, directory } = await uploadedFilePlugin(plugin)
   return await storePlugin(metadata, directory, source)
 }

packages/server/src/api/controllers/plugin/utils.js

@@ -1,32 +1,153 @@
-const fetch = require("node-fetch")
+import fetch from "node-fetch"
-import { downloadUnzipPlugin } from "../../../utilities/fileSystem"
+import {
+  createTempFolder,
+  getPluginMetadata,
+  findFileRec,
+  downloadTarballDirect,
+  extractTarball,
+  deleteFolderFileSystem,
+} from "../../../utilities/fileSystem"
+import { join } from "path"
 
-export const createNpmPlugin = async (url, name = "") => {
+export const uploadedFilePlugin = async file => {
-  let npmTarball = url
+  if (!file.name.endsWith(".tar.gz")) {
+    throw new Error("Plugin must be compressed into a gzipped tarball.")
+  }
+  const path = createTempFolder(file.name.split(".tar.gz")[0])
+  await extractTarball(file.path, path)
+
+  return await getPluginMetadata(path)
+}
+
+export const uploadedNpmPlugin = async (url, name, headers = {}) => {
+  let npmTarballUrl = url
   let pluginName = name
 
   if (
-    !npmTarball.includes("https://www.npmjs.com") &&
+    !npmTarballUrl.includes("https://www.npmjs.com") &&
-    !npmTarball.includes("https://registry.npmjs.org")
+    !npmTarballUrl.includes("https://registry.npmjs.org")
   ) {
-    throw "The plugin origin must be from NPM"
+    throw new Error("The plugin origin must be from NPM")
   }
 
-  if (!npmTarball.includes(".tgz")) {
+  if (!npmTarballUrl.includes(".tgz")) {
     const npmPackageURl = url.replace(
       "https://www.npmjs.com/package/",
       "https://registry.npmjs.org/"
     )
     const response = await fetch(npmPackageURl)
-    if (response.status === 200) {
+    if (response.status !== 200) {
+      throw new Error("NPM Package not found")
+    }
+
     let npmDetails = await response.json()
     pluginName = npmDetails.name
     const npmVersion = npmDetails["dist-tags"].latest
-      npmTarball = npmDetails.versions[npmVersion].dist.tarball
+    npmTarballUrl = npmDetails?.versions?.[npmVersion]?.dist?.tarball
-    } else {
+
-      throw "Cannot get package details"
+    if (!npmTarballUrl) {
+      throw new Error("NPM tarball url not found")
     }
   }
 
-  return await downloadUnzipPlugin(pluginName, npmTarball)
+  const path = await downloadUnzipTarball(npmTarballUrl, pluginName, headers)
+  const tarballPluginFile = findFileRec(path, ".tar.gz")
+  if (!tarballPluginFile) {
+    throw new Error("Tarball plugin file not found")
+  }
+
+  try {
+    await extractTarball(tarballPluginFile, path)
+    deleteFolderFileSystem(join(path, "package"))
+  } catch (err) {
+    throw new Error(err)
+  }
+
+  return await getPluginMetadata(path)
+}
+
+export const uploadedUrlPlugin = async (url, name = "", headers = {}) => {
+  if (!url.includes(".tar.gz")) {
+    throw new Error("Plugin must be compressed into a gzipped tarball.")
+  }
+
+  const path = await downloadUnzipTarball(url, name, headers)
+
+  return await getPluginMetadata(path)
+}
+
+export const uploadedGithubPlugin = async (ctx, url, name = "", token = "") => {
+  let githubUrl = url
+
+  if (!githubUrl.includes("https://github.com/")) {
+    throw new Error("The plugin origin must be from Github")
+  }
+
+  if (url.includes(".git")) {
+    githubUrl = url.replace(".git", "")
+  }
+
+  const githubApiUrl = githubUrl.replace(
+    "https://github.com/",
+    "https://api.github.com/repos/"
+  )
+  const headers = token ? { Authorization: `Bearer ${token}` } : {}
+  try {
+    const pluginRaw = await fetch(githubApiUrl, { headers })
+    if (pluginRaw.status !== 200) {
+      throw new Error(`Repository not found`)
+    }
+
+    let pluginDetails = await pluginRaw.json()
+    const pluginName = pluginDetails.name || name
+
+    const pluginLatestReleaseUrl = pluginDetails?.["releases_url"]
+      ? pluginDetails?.["releases_url"].replace("{/id}", "/latest")
+      : undefined
+    if (!pluginLatestReleaseUrl) {
+      throw new Error("Github release not found")
+    }
+
+    const pluginReleaseRaw = await fetch(pluginLatestReleaseUrl, { headers })
+    if (pluginReleaseRaw.status !== 200) {
+      throw new Error("Github latest release not found")
+    }
+    const pluginReleaseDetails = await pluginReleaseRaw.json()
+    const pluginReleaseTarballAsset = pluginReleaseDetails?.assets?.find(
+      x => x?.content_type === "application/gzip"
+    )
+    const pluginLastReleaseTarballUrl =
+      pluginReleaseTarballAsset?.browser_download_url
+    if (!pluginLastReleaseTarballUrl) {
+      throw new Error("Github latest release url not found")
+    }
+
+    const path = await downloadUnzipTarball(
+      pluginLastReleaseTarballUrl,
+      pluginName,
+      headers
+    )
+
+    return await getPluginMetadata(path)
+  } catch (err) {
+    let errMsg = err?.message || err
+
+    if (errMsg === "unexpected response Not Found") {
+      errMsg = "Github release tarbal not found"
+    }
+
+    throw new Error(errMsg)
+  }
+}
+
+export const downloadUnzipTarball = async (url, name, headers = {}) => {
+  try {
+    const path = createTempFolder(name)
+
+    await downloadTarballDirect(url, path, headers)
+
+    return path
+  } catch (e) {
+    throw new Error(e.message)
+  }
 }

packages/server/src/utilities/fileSystem/index.js

@@ -1,8 +1,6 @@
 const { budibaseTempDir } = require("../budibaseDir")
 const fs = require("fs")
 const { join } = require("path")
-const { promisify } = require("util")
-const streamPipeline = promisify(require("stream").pipeline)
 const uuid = require("uuid/v4")
 const {
   doWithDB,
@@ -17,6 +15,7 @@ const {
   streamUpload,
   deleteFolder,
   downloadTarball,
+  downloadTarballDirect,
   deleteFiles,
 } = require("./utilities")
 const { updateClientLibrary } = require("./clientLibrary")
@@ -31,7 +30,6 @@ const MemoryStream = require("memorystream")
 const { getAppId } = require("@budibase/backend-core/context")
 const tar = require("tar")
 const fetch = require("node-fetch")
-const { NodeVM } = require("vm2")
 
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
 const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
@@ -341,131 +339,52 @@ exports.cleanup = appIds => {
   }
 }
 
-const extractPluginTarball = async (file, ext = ".tar.gz") => {
+const createTempFolder = item => {
-  if (!file.name.endsWith(ext)) {
+  const path = join(budibaseTempDir(), item)
-    throw new Error("Plugin must be compressed into a gzipped tarball.")
+  try {
-  }
-  const path = join(budibaseTempDir(), file.name.split(ext)[0])
     // remove old tmp directories automatically - don't combine
     if (fs.existsSync(path)) {
       fs.rmSync(path, { recursive: true, force: true })
     }
     fs.mkdirSync(path)
+  } catch (err) {
+    throw new Error(`Path cannot be created: ${err.message}`)
+  }
+
+  return path
+}
+exports.createTempFolder = createTempFolder
+
+const extractTarball = async (fromFilePath, toPath) => {
   await tar.extract({
-    file: file.path,
+    file: fromFilePath,
-    C: path,
+    C: toPath,
   })
-
-  return await getPluginMetadata(path)
 }
-exports.extractPluginTarball = extractPluginTarball
+exports.extractTarball = extractTarball
-
-exports.createUrlPlugin = async (url, name = "", headers = {}) => {
-  if (!url.includes(".tgz") && !url.includes(".tar.gz")) {
-    throw new Error("Plugin must be compressed into a gzipped tarball.")
-  }
-
-  return await downloadUnzipPlugin(name, url, headers)
-}
-
-exports.createGithubPlugin = async (ctx, url, name = "", token = "") => {
-  let githubRepositoryUrl
-  let githubUrl
-
-  if (url.includes(".git")) {
-    githubRepositoryUrl = token
-      ? url.replace("https://", `https://${token}@`)
-      : url
-    githubUrl = url.replace(".git", "")
-  } else {
-    githubRepositoryUrl = token
-      ? `${url}.git`.replace("https://", `https://${token}@`)
-      : `${url}.git`
-    githubUrl = url
-  }
-
-  const githubApiUrl = githubUrl.replace(
-    "https://github.com/",
-    "https://api.github.com/repos/"
-  )
-  const headers = token ? { Authorization: `Bearer ${token}` } : {}
-  try {
-    const pluginRaw = await fetch(githubApiUrl, { headers })
-    if (pluginRaw.status !== 200) {
-      throw `Repository not found`
-    }
-
-    let pluginDetails = await pluginRaw.json()
-    const pluginName = pluginDetails.name || name
-
-    const path = join(budibaseTempDir(), pluginName)
-    // Remove first if exists
-    if (fs.existsSync(path)) {
-      fs.rmSync(path, { recursive: true, force: true })
-    }
-    fs.mkdirSync(path)
-
-    const script = `
-module.exports = async () => {
-  const child_process = require('child_process')
-  child_process.execSync(\`git clone ${githubRepositoryUrl} ${join(
-      budibaseTempDir(),
-      pluginName
-    )}\`);
-}
-`
-    const scriptRunner = new NodeVM({
-      require: {
-        external: true,
-        builtin: ["child_process"],
-        root: "./",
-      },
-    }).run(script)
-
-    await scriptRunner()
-
-    return await getPluginMetadata(path)
-  } catch (e) {
-    throw e.message
-  }
-}
-
-const downloadUnzipPlugin = async (name, url, headers = {}) => {
-  const path = join(budibaseTempDir(), name)
-  try {
-    // Remove first if exists
-    if (fs.existsSync(path)) {
-      fs.rmSync(path, { recursive: true, force: true })
-    }
-    fs.mkdirSync(path)
-
-    const response = await fetch(url, { headers })
-    if (!response.ok)
-      throw new Error(`Loading NPM plugin failed ${response.statusText}`)
-
-    await streamPipeline(
-      response.body,
-      tar.x({
-        strip: 1,
-        C: path,
-      })
-    )
-    return await getPluginMetadata(path)
-  } catch (e) {
-    throw `Cannot store plugin locally: ${e.message}`
-  }
-}
-exports.downloadUnzipPlugin = downloadUnzipPlugin
 
 const getPluginMetadata = async path => {
   let metadata = {}
   try {
     const pkg = fs.readFileSync(join(path, "package.json"), "utf8")
     const schema = fs.readFileSync(join(path, "schema.json"), "utf8")
+
     metadata.schema = JSON.parse(schema)
     metadata.package = JSON.parse(pkg)
+
+    if (
+      !metadata.package.name ||
+      !metadata.package.version ||
+      !metadata.package.description
+    ) {
+      throw new Error(
+        "package.json is missing one of 'name', 'version' or 'description'."
+      )
+    }
   } catch (err) {
-    throw new Error("Unable to process schema.json/package.json in plugin.")
+    throw new Error(
+      `Unable to process schema.json/package.json in plugin. ${err.message}`
+    )
   }
 
   return { metadata, directory: path }
@@ -504,6 +423,40 @@ exports.getDatasourcePlugin = async (name, url, hash) => {
   }
 }
 
+/**
+ * Find for a file recursively from start path applying filter, return first match
+ */
+const findFileRec = (startPath, filter) => {
+  if (!fs.existsSync(startPath)) {
+    return
+  }
+
+  var files = fs.readdirSync(startPath)
+  for (let i = 0, len = files.length; i < len; i++) {
+    const filename = join(startPath, files[i])
+    const stat = fs.lstatSync(filename)
+
+    if (stat.isDirectory()) {
+      return findFileRec(filename, filter)
+    } else if (filename.endsWith(filter)) {
+      return filename
+    }
+  }
+}
+exports.findFileRec = findFileRec
+
+/**
+ * Remove a folder which is not empty from the file system
+ */
+const deleteFolderFileSystem = path => {
+  if (!fs.existsSync(path)) {
+    return
+  }
+
+  fs.rmSync(path, { recursive: true, force: true })
+}
+exports.deleteFolderFileSystem = deleteFolderFileSystem
+
 /**
  * Full function definition for below can be found in the utilities.
  */
@@ -511,5 +464,6 @@ exports.upload = upload
 exports.retrieve = retrieve
 exports.retrieveToTmp = retrieveToTmp
 exports.deleteFiles = deleteFiles
+exports.downloadTarballDirect = downloadTarballDirect
 exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH
 exports.NODE_MODULES_PATH = NODE_MODULES_PATH