Merge branch 'plugins-dev-experience' of github.com:Budibase/budibase into plugins-dev-experience

charts/budibase/templates/app-service-deployment.yaml

@@ -124,11 +124,15 @@ spec:
           value: {{ .Values.globals.tenantFeatureFlags | quote }}
         {{ if .Values.globals.bbAdminUserEmail }}
         - name: BB_ADMIN_USER_EMAIL
-          value: { { .Values.globals.bbAdminUserEmail | quote } }
+          value: {{ .Values.globals.bbAdminUserEmail | quote }}
         {{ end }}
         {{ if .Values.globals.bbAdminUserPassword }}
         - name: BB_ADMIN_USER_PASSWORD
-          value: { { .Values.globals.bbAdminUserPassword | quote } }
+          value: {{ .Values.globals.bbAdminUserPassword | quote }}
+        {{ end }}
+        {{ if .Values.globals.pluginsDir }}
+        - name: PLUGINS_DIR
+          value: { { .Values.globals.pluginsDir | quote }}
         {{ end }}
 
         image: budibase/apps:{{ .Values.globals.appVersion }}

hosting/.env

@@ -23,3 +23,6 @@ BUDIBASE_ENVIRONMENT=PRODUCTION
 # An admin user can be automatically created initially if these are set
 BB_ADMIN_USER_EMAIL=
 BB_ADMIN_USER_PASSWORD=
+
+# A path that is watched for plugin bundles. Any bundles found are imported automatically/
+PLUGINS_DIR=

hosting/hosting.properties

@@ -23,3 +23,6 @@ BUDIBASE_ENVIRONMENT=PRODUCTION
 # An admin user can be automatically created initially if these are set
 BB_ADMIN_USER_EMAIL=
 BB_ADMIN_USER_PASSWORD=
+
+# A path that is watched for plugin bundles. Any bundles found are imported automatically/
+PLUGINS_DIR=

packages/builder/src/components/backend/DatasourceNavigator/icons/Custom.svelte

@@ -0,0 +1,39 @@
+<script>
+  export let width = "100"
+  export let height = "100"
+  let color =
+    "var(--spectrum-heading-xxs-text-color, var(--spectrum-alias-heading-text-color))"
+</script>
+
+<svg
+  {width}
+  {height}
+  xmlns="http://www.w3.org/2000/svg"
+  x="0px"
+  y="0px"
+  viewBox="0 0 230.795 230.795"
+  style="enable-background:new 0 0 230.795 230.795;"
+  xml:space="preserve"
+>
+  <g>
+    <path
+      d="M60.357,63.289c-2.929-2.929-7.678-2.93-10.606-0.001L2.197,110.836C0.79,112.243,0,114.151,0,116.14
+		c0,1.989,0.79,3.896,2.196,5.303l47.348,47.35c1.465,1.465,3.384,2.197,5.304,2.197c1.919,0,3.839-0.732,5.303-2.196
+		c2.93-2.929,2.93-7.678,0.001-10.606L18.107,116.14l42.25-42.245C63.286,70.966,63.286,66.217,60.357,63.289z"
+      fill={color}
+    />
+    <path
+      d="M228.598,110.639l-47.355-47.352c-2.928-2.928-7.677-2.929-10.606,0.001c-2.929,2.929-2.929,7.678,0.001,10.607
+		l42.051,42.048l-42.249,42.243c-2.93,2.929-2.93,7.678-0.001,10.606c1.465,1.465,3.384,2.197,5.304,2.197
+		c1.919,0,3.839-0.732,5.303-2.196l47.554-47.547c1.407-1.406,2.197-3.314,2.197-5.304
+		C230.795,113.954,230.005,112.046,228.598,110.639z"
+      fill={color}
+    />
+    <path
+      d="M155.889,61.302c-3.314-2.484-8.017-1.806-10.498,1.51l-71.994,96.184c-2.482,3.316-1.807,8.017,1.51,10.498
+		c1.348,1.01,2.925,1.496,4.488,1.496c2.282,0,4.537-1.038,6.01-3.006L157.398,71.8C159.881,68.484,159.205,63.784,155.889,61.302z"
+      fill={color}
+    />
+  </g>
+  <g />
+</svg>

packages/builder/src/components/backend/DatasourceNavigator/icons/index.js

@@ -15,6 +15,7 @@ import GoogleSheets from "./GoogleSheets.svelte"
 import Firebase from "./Firebase.svelte"
 import Redis from "./Redis.svelte"
 import Snowflake from "./Snowflake.svelte"
+import Custom from "./Custom.svelte"
 
 export default {
   BUDIBASE: Budibase,
@@ -34,4 +35,5 @@ export default {
   FIRESTORE: Firebase,
   REDIS: Redis,
   SNOWFLAKE: Snowflake,
+  CUSTOM: Custom,
 }

packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte

@@ -92,6 +92,14 @@
     }
     integrations = newIntegrations
   }
+
+  function getIcon(integrationType, schema) {
+    if (schema.custom) {
+      return ICONS.CUSTOM
+    } else {
+      return ICONS[integrationType]
+    }
+  }
 </script>
 
 <Modal bind:this={internalTableModal}>
@@ -158,7 +166,7 @@
           >
             <div class="item-body" class:with-type={!!schema.type}>
               <svelte:component
-                this={ICONS[integrationType]}
+                this={getIcon(integrationType, schema)}
                 height="20"
                 width="20"
               />

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/NewComponentPanel.svelte

@@ -56,7 +56,7 @@
     // Add custom components category
     if (customComponents?.length) {
       enrichedStructure.push({
-        name: "Custom components",
+        name: "Plugins",
         isCategory: true,
         children: customComponents.map(x => ({
           ...definitions[x],

packages/server/package.json

@@ -95,6 +95,7 @@
     "bcryptjs": "2.4.3",
     "bull": "3.29.3",
     "chmodr": "1.2.0",
+    "chokidar": "^3.5.3",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
     "dotenv": "8.2.0",

packages/server/scripts/dev/manage.js

@@ -58,6 +58,7 @@ async function init() {
       DEPLOYMENT_ENVIRONMENT: "development",
       BB_ADMIN_USER_EMAIL: "",
       BB_ADMIN_USER_PASSWORD: "",
+      PLUGINS_DIR: "",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

packages/server/src/api/controllers/datasource.js

@@ -7,7 +7,7 @@ const {
   getTableParams,
 } = require("../../db/utils")
 const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
-const { integrations } = require("../../integrations")
+const { getIntegration } = require("../../integrations")
 const { getDatasourceAndQuery } = require("./row/utils")
 const { invalidateDynamicVariables } = require("../../threads/utils")
 const { getAppDB } = require("@budibase/backend-core/context")
@@ -114,7 +114,7 @@ exports.update = async function (ctx) {
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
-    const source = integrations[datasource.source]
+    const source = await getIntegration(datasource.source)
     if (source && source.pool) {
       await source.pool.end()
     }
@@ -149,7 +149,7 @@ exports.save = async function (ctx) {
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
-    const source = integrations[datasource.source]
+    const source = await getIntegration(datasource.source)
     if (source && source.pool) {
       await source.pool.end()
     }
@@ -218,7 +218,7 @@ function updateError(error, newError, tables) {
 }
 
 const buildSchemaHelper = async datasource => {
-  const Connector = integrations[datasource.source]
+  const Connector = await getIntegration(datasource.source)
 
   // Connect to the DB and build the schema
   const connector = new Connector(datasource.config)

packages/server/src/api/controllers/plugin.ts

@@ -3,7 +3,7 @@ import { extractPluginTarball } from "../../utilities/fileSystem"
 import { getGlobalDB } from "@budibase/backend-core/tenancy"
 import { generatePluginID, getPluginParams } from "../../db/utils"
 import { uploadDirectory } from "@budibase/backend-core/objectStore"
-import { PluginType } from "@budibase/types"
+import { PluginType, FileType } from "@budibase/types"
 
 export async function getPlugins(type?: PluginType) {
   const db = getGlobalDB()
@@ -21,56 +21,16 @@ export async function getPlugins(type?: PluginType) {
 }
 
 export async function upload(ctx: any) {
-  const plugins =
+  const plugins: FileType[] =
     ctx.request.files.file.length > 1
       ? Array.from(ctx.request.files.file)
       : [ctx.request.files.file]
-  const db = getGlobalDB()
   try {
     let docs = []
     // can do single or multiple plugins
     for (let plugin of plugins) {
-      const { metadata, directory } = await extractPluginTarball(plugin)
+      const doc = await processPlugin(plugin)
-      const version = metadata.package.version,
+      docs.push(doc)
-        name = metadata.package.name,
-        description = metadata.package.description
-
-      // first open the tarball into tmp directory
-      const bucketPath = `${name}/${version}/`
-      const files = await uploadDirectory(
-        ObjectStoreBuckets.PLUGINS,
-        directory,
-        bucketPath
-      )
-      const jsFile = files.find((file: any) => file.name.endsWith(".js"))
-      if (!jsFile) {
-        throw new Error(`Plugin missing .js file.`)
-      }
-      const jsFileName = jsFile.name
-      const pluginId = generatePluginID(name, version)
-
-      // overwrite existing docs entirely if they exist
-      let rev
-      try {
-        const existing = await db.get(pluginId)
-        rev = existing._rev
-      } catch (err) {
-        rev = undefined
-      }
-      const doc = {
-        _id: pluginId,
-        _rev: rev,
-        name,
-        version,
-        description,
-        ...metadata,
-        jsUrl: `${bucketPath}${jsFileName}`,
-      }
-      const response = await db.put(doc)
-      docs.push({
-        ...doc,
-        _rev: response.rev,
-      })
     }
     ctx.body = {
       message: "Plugin(s) uploaded successfully",
@@ -87,3 +47,48 @@ export async function fetch(ctx: any) {
 }
 
 export async function destroy(ctx: any) {}
+
+export async function processPlugin(plugin: FileType) {
+  const db = getGlobalDB()
+  const { metadata, directory } = await extractPluginTarball(plugin)
+  const version = metadata.package.version,
+    name = metadata.package.name,
+    description = metadata.package.description
+
+  // first open the tarball into tmp directory
+  const bucketPath = `${name}/${version}/`
+  const files = await uploadDirectory(
+    ObjectStoreBuckets.PLUGINS,
+    directory,
+    bucketPath
+  )
+  const jsFile = files.find((file: any) => file.name.endsWith(".js"))
+  if (!jsFile) {
+    throw new Error(`Plugin missing .js file.`)
+  }
+  const jsFileName = jsFile.name
+  const pluginId = generatePluginID(name, version)
+
+  // overwrite existing docs entirely if they exist
+  let rev
+  try {
+    const existing = await db.get(pluginId)
+    rev = existing._rev
+  } catch (err) {
+    rev = undefined
+  }
+  const doc = {
+    _id: pluginId,
+    _rev: rev,
+    name,
+    version,
+    description,
+    ...metadata,
+    jsUrl: `${bucketPath}${jsFileName}`,
+  }
+  const response = await db.put(doc)
+  return {
+    ...doc,
+    _rev: response.rev,
+  }
+}

packages/server/src/app.ts

@@ -17,10 +17,15 @@ const bullboard = require("./automations/bullboard")
 const { logAlert } = require("@budibase/backend-core/logging")
 const { pinoSettings } = require("@budibase/backend-core")
 const { Thread } = require("./threads")
+const chokidar = require("chokidar")
+const fs = require("fs")
+const path = require("path")
 import redis from "./utilities/redis"
 import * as migrations from "./migrations"
 import { events, installation, tenancy } from "@budibase/backend-core"
 import { createAdminUser, getChecklist } from "./utilities/workerRequests"
+import { processPlugin } from "./api/controllers/plugin"
+import { getGlobalDB } from "@budibase/backend-core/tenancy"
 
 const app = new Koa()
 
@@ -132,6 +137,29 @@ module.exports = server.listen(env.PORT || 0, async () => {
     }
   }
 
+  // monitor plugin directory if required
+  if (env.SELF_HOSTED && env.PLUGINS_DIR && fs.existsSync(env.PLUGINS_DIR)) {
+    const watchPath = path.join(env.PLUGINS_DIR, "./**/dist/*.tar.gz")
+    chokidar
+      .watch(watchPath, {
+        ignored: "**/node_modules",
+        awaitWriteFinish: true,
+      })
+      .on("all", async (event: string, path: string) => {
+        const tenantId = tenancy.getTenantId()
+        await tenancy.doInTenant(tenantId, async () => {
+          try {
+            const split = path.split("/")
+            const name = split[split.length - 1]
+            console.log("Importing plugin:", path)
+            await processPlugin({ name, path })
+          } catch (err) {
+            console.log("Failed to import plugin:", err)
+          }
+        })
+      })
+  }
+
   // check for version updates
   await installation.checkInstallVersion()
 

packages/server/src/environment.js

@@ -77,6 +77,7 @@ module.exports = {
   SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
   BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
+  PLUGINS_DIR: process.env.PLUGINS_DIR,
   // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,

packages/server/src/integrations/base/query.ts

@@ -1,11 +1,11 @@
 import { QueryJson, Datasource } from "@budibase/types"
-const { integrations } = require("../index")
+const { getIntegration } = require("../index")
 
 export async function makeExternalQuery(
   datasource: Datasource,
   json: QueryJson
 ) {
-  const Integration = integrations[datasource.source]
+  const Integration = await getIntegration(datasource.source)
   // query is the opinionated function
   if (Integration.prototype.query) {
     const integration = new Integration(datasource.config)

packages/server/src/integrations/index.ts

@@ -67,8 +67,22 @@ if (environment.SELF_HOSTED) {
 
 module.exports = {
   getDefinitions: async () => {
-    const custom = await getPlugins(PluginType.DATASOURCE)
+    const plugins = await getPlugins(PluginType.DATASOURCE)
-    return cloneDeep(DEFINITIONS)
+    // extract the actual schema from each custom
+    const pluginSchemas: { [key: string]: Integration } = {}
+    for (let plugin of plugins) {
+      const sourceId = plugin.name
+      pluginSchemas[sourceId] = {
+        ...plugin.schema["schema"],
+        custom: true,
+      }
+    }
+    return {
+      ...cloneDeep(DEFINITIONS),
+      ...pluginSchemas,
+    }
+  },
+  getIntegration: async () => {
+    return INTEGRATIONS
   },
-  integrations: INTEGRATIONS,
 }

packages/server/src/threads/query.ts

@@ -2,7 +2,7 @@ import { default as threadUtils } from "./utils"
 threadUtils.threadSetup()
 import { WorkerCallback, QueryEvent, QueryVariable } from "./definitions"
 const ScriptRunner = require("../utilities/scriptRunner")
-const { integrations } = require("../integrations")
+const { getIntegration } = require("../integrations")
 const { processStringSync } = require("@budibase/string-templates")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
 const {
@@ -62,7 +62,7 @@ class QueryRunner {
     let datasourceClone = cloneDeep(datasource)
     let fieldsClone = cloneDeep(fields)
 
-    const Integration = integrations[datasourceClone.source]
+    const Integration = await getIntegration(datasourceClone.source)
     if (!Integration) {
       throw "Integration type does not exist."
     }

packages/server/yarn.lock

@@ -4411,7 +4411,7 @@ chmodr@1.2.0:
   resolved "https://registry.yarnpkg.com/chmodr/-/chmodr-1.2.0.tgz#720e96caa09b7f1cdbb01529b7d0ab6bc5e118b9"
   integrity sha512-Y5uI7Iq/Az6HgJEL6pdw7THVd7jbVOTPwsmcPOBjQL8e3N+pz872kzK5QxYGEy21iRys+iHWV0UZQXDFJo1hyA==
 
-chokidar@^3.5.2:
+chokidar@^3.5.2, chokidar@^3.5.3:
   version "3.5.3"
   resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd"
   integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==

packages/types/src/documents/plugin/index.ts

@@ -2,3 +2,8 @@ export enum PluginType {
   DATASOURCE = "datasource",
   COMPONENT = "component",
 }
+
+export interface FileType {
+  path: string
+  name: string
+}