Updating plugins to not think about versions, only ever one version of each plugin by name, making plugins self host only and adding error checking for datasource implementations.

packages/server/src/api/controllers/component.js

@@ -2,6 +2,7 @@ const { DocumentType, getPluginParams } = require("../../db/utils")
 const { getComponentLibraryManifest } = require("../../utilities/fileSystem")
 const { getAppDB } = require("@budibase/backend-core/context")
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const env = require("../../environment")
 
 exports.fetchAppComponentDefinitions = async function (ctx) {
   try {
@@ -32,23 +33,26 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
       }
     }
 
-    // Add custom components
-    const globalDB = getGlobalDB()
-    const response = await globalDB.allDocs(
-      getPluginParams(null, {
-        include_docs: true,
-      })
-    )
-    response.rows
-      .map(row => row.doc)
-      .filter(plugin => plugin.schema.type === "component")
-      .forEach(plugin => {
-        const fullComponentName = `plugin/${plugin.name}/${plugin.version}`
-        definitions[fullComponentName] = {
-          component: fullComponentName,
-          ...plugin.schema.schema,
-        }
-      })
+    // for now custom components only supported in self-host
+    if (env.SELF_HOSTED) {
+      // Add custom components
+      const globalDB = getGlobalDB()
+      const response = await globalDB.allDocs(
+        getPluginParams(null, {
+          include_docs: true,
+        })
+      )
+      response.rows
+        .map(row => row.doc)
+        .filter(plugin => plugin.schema.type === "component")
+        .forEach(plugin => {
+          const fullComponentName = `plugin/${plugin.name}/${plugin.version}`
+          definitions[fullComponentName] = {
+            component: fullComponentName,
+            ...plugin.schema.schema,
+          }
+        })
+    }
 
     ctx.body = definitions
   } catch (err) {

packages/server/src/api/controllers/plugin.ts

@@ -1,9 +1,10 @@
 import { ObjectStoreBuckets } from "../../constants"
-import { extractPluginTarball } from "../../utilities/fileSystem"
+import { extractPluginTarball, loadJSFile } from "../../utilities/fileSystem"
 import { getGlobalDB } from "@budibase/backend-core/tenancy"
 import { generatePluginID, getPluginParams } from "../../db/utils"
 import { uploadDirectory } from "@budibase/backend-core/objectStore"
 import { PluginType, FileType } from "@budibase/types"
+import env from "../../environment"
 
 export async function getPlugins(type?: PluginType) {
   const db = getGlobalDB()
@@ -49,6 +50,9 @@ export async function fetch(ctx: any) {
 export async function destroy(ctx: any) {}
 
 export async function processPlugin(plugin: FileType) {
+  if (!env.SELF_HOSTED) {
+    throw new Error("Plugins not supported outside of self-host.")
+  }
   const db = getGlobalDB()
   const { metadata, directory } = await extractPluginTarball(plugin)
   const version = metadata.package.version,
@@ -56,7 +60,7 @@ export async function processPlugin(plugin: FileType) {
     description = metadata.package.description
 
   // first open the tarball into tmp directory
-  const bucketPath = `${name}/${version}/`
+  const bucketPath = `${name}/`
   const files = await uploadDirectory(
     ObjectStoreBuckets.PLUGINS,
     directory,
@@ -66,8 +70,20 @@ export async function processPlugin(plugin: FileType) {
   if (!jsFile) {
     throw new Error(`Plugin missing .js file.`)
   }
+  // validate the JS for a datasource
+  if (metadata.schema.type === PluginType.DATASOURCE) {
+    const js = loadJSFile(directory, jsFile.name)
+    // TODO: this isn't safe - but we need full node environment
+    // in future we should do this in a thread for safety
+    try {
+      eval(js)
+    } catch (err: any) {
+      const message = err?.message ? err.message : JSON.stringify(err)
+      throw new Error(`JS invalid: ${message}`)
+    }
+  }
   const jsFileName = jsFile.name
-  const pluginId = generatePluginID(name, version)
+  const pluginId = generatePluginID(name)
 
   // overwrite existing docs entirely if they exist
   let rev
@@ -80,10 +96,10 @@ export async function processPlugin(plugin: FileType) {
   const doc = {
     _id: pluginId,
     _rev: rev,
+    ...metadata,
     name,
     version,
     description,
-    ...metadata,
     jsUrl: `${bucketPath}${jsFileName}`,
   }
   const response = await db.put(doc)

packages/server/src/db/utils.js

@@ -371,8 +371,8 @@ exports.getMemoryViewParams = (otherProps = {}) => {
   return getDocParams(DocumentType.MEM_VIEW, null, otherProps)
 }
 
-exports.generatePluginID = (name, version) => {
-  return `${DocumentType.PLUGIN}${SEPARATOR}${name}${SEPARATOR}${version}`
+exports.generatePluginID = name => {
+  return `${DocumentType.PLUGIN}${SEPARATOR}${name}`
 }
 
 /**

packages/server/src/integrations/index.ts

@@ -66,18 +66,18 @@ if (environment.SELF_HOSTED) {
   DEFINITIONS[SourceName.GOOGLE_SHEETS] = googlesheets.schema
 }
 
-function isIntegrationAvailable(integration: string) {}
-
 module.exports = {
   getDefinitions: async () => {
-    const plugins = await getPlugins(PluginType.DATASOURCE)
-    // extract the actual schema from each custom
     const pluginSchemas: { [key: string]: Integration } = {}
-    for (let plugin of plugins) {
-      const sourceId = plugin.name
-      pluginSchemas[sourceId] = {
-        ...plugin.schema["schema"],
-        custom: true,
+    if (environment.SELF_HOSTED) {
+      const plugins = await getPlugins(PluginType.DATASOURCE)
+      // extract the actual schema from each custom
+      for (let plugin of plugins) {
+        const sourceId = plugin.name
+        pluginSchemas[sourceId] = {
+          ...plugin.schema["schema"],
+          custom: true,
+        }
       }
     }
     return {
@@ -89,16 +89,19 @@ module.exports = {
     if (INTEGRATIONS[integration]) {
       return INTEGRATIONS[integration]
     }
-    const plugins = await getPlugins(PluginType.DATASOURCE)
-    for (let plugin of plugins) {
-      if (plugin.name === integration) {
-        // need to use commonJS require due to its dynamic runtime nature
-        return getDatasourcePlugin(
-          plugin.name,
-          plugin.jsUrl,
-          plugin.schema?.hash
-        )
+    if (environment.SELF_HOSTED) {
+      const plugins = await getPlugins(PluginType.DATASOURCE)
+      for (let plugin of plugins) {
+        if (plugin.name === integration) {
+          // need to use commonJS require due to its dynamic runtime nature
+          return getDatasourcePlugin(
+            plugin.name,
+            plugin.jsUrl,
+            plugin.schema?.hash
+          )
+        }
       }
     }
+    throw new Error("No datasource implementation found.")
   },
 }

packages/server/src/utilities/fileSystem/index.js

@@ -103,6 +103,13 @@ exports.loadHandlebarsFile = path => {
   return fs.readFileSync(path, "utf8")
 }
 
+/**
+ * Same as above just with a different name.
+ */
+exports.loadJSFile = (directory, name) => {
+  return fs.readFileSync(join(directory, name), "utf8")
+}
+
 /**
  * When return a file from the API need to write the file to the system temporarily so we
  * can create a read stream to send.

packages/server/src/watch.ts

@@ -29,8 +29,9 @@ export function watch() {
           const name = split[split.length - 1]
           console.log("Importing plugin:", path)
           await processPlugin({ name, path })
-        } catch (err) {
-          console.log("Failed to import plugin:", err)
+        } catch (err: any) {
+          const message = err?.message ? err?.message : err
+          console.error("Failed to import plugin:", message)
         }
       })
     })