Add hot reloading for component plugins via watched plugin directory

charts/budibase/templates/app-service-deployment.yaml

@@ -124,11 +124,15 @@ spec:
           value: {{ .Values.globals.tenantFeatureFlags | quote }}
         {{ if .Values.globals.bbAdminUserEmail }}
         - name: BB_ADMIN_USER_EMAIL
-          value: { { .Values.globals.bbAdminUserEmail | quote } }
+          value: {{ .Values.globals.bbAdminUserEmail | quote }}
         {{ end }}
         {{ if .Values.globals.bbAdminUserPassword }}
         - name: BB_ADMIN_USER_PASSWORD
-          value: { { .Values.globals.bbAdminUserPassword | quote } }
+          value: {{ .Values.globals.bbAdminUserPassword | quote }}
+        {{ end }}
+        {{ if .Values.globals.pluginsDir }}
+        - name: PLUGINS_DIR
+          value: { { .Values.globals.pluginsDir | quote }}
         {{ end }}
 
         image: budibase/apps:{{ .Values.globals.appVersion }}

hosting/.env

@@ -23,3 +23,6 @@ BUDIBASE_ENVIRONMENT=PRODUCTION
 # An admin user can be automatically created initially if these are set
 BB_ADMIN_USER_EMAIL=
 BB_ADMIN_USER_PASSWORD=
+
+# A path that is watched for plugin bundles. Any bundles found are imported automatically/
+PLUGINS_DIR=

hosting/hosting.properties

@@ -23,3 +23,6 @@ BUDIBASE_ENVIRONMENT=PRODUCTION
 # An admin user can be automatically created initially if these are set
 BB_ADMIN_USER_EMAIL=
 BB_ADMIN_USER_PASSWORD=
+
+# A path that is watched for plugin bundles. Any bundles found are imported automatically/
+PLUGINS_DIR=

packages/server/package.json

@@ -95,6 +95,7 @@
     "bcryptjs": "2.4.3",
     "bull": "3.29.3",
     "chmodr": "1.2.0",
+    "chokidar": "^3.5.3",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
     "dotenv": "8.2.0",

packages/server/scripts/dev/manage.js

@@ -58,6 +58,7 @@ async function init() {
       DEPLOYMENT_ENVIRONMENT: "development",
       BB_ADMIN_USER_EMAIL: "",
       BB_ADMIN_USER_PASSWORD: "",
+      PLUGINS_DIR: "",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

packages/server/src/api/controllers/plugin.ts

@@ -3,7 +3,7 @@ import { extractPluginTarball } from "../../utilities/fileSystem"
 import { getGlobalDB } from "@budibase/backend-core/tenancy"
 import { generatePluginID, getPluginParams } from "../../db/utils"
 import { uploadDirectory } from "@budibase/backend-core/objectStore"
-import { PluginType } from "@budibase/types"
+import { PluginType, FileType } from "@budibase/types"
 
 export async function getPlugins(type?: PluginType) {
   const db = getGlobalDB()
@@ -21,56 +21,16 @@ export async function getPlugins(type?: PluginType) {
 }
 
 export async function upload(ctx: any) {
-  const plugins =
+  const plugins: FileType[] =
     ctx.request.files.file.length > 1
       ? Array.from(ctx.request.files.file)
       : [ctx.request.files.file]
-  const db = getGlobalDB()
   try {
     let docs = []
     // can do single or multiple plugins
     for (let plugin of plugins) {
-      const { metadata, directory } = await extractPluginTarball(plugin)
-      const version = metadata.package.version,
-        name = metadata.package.name,
-        description = metadata.package.description
-
-      // first open the tarball into tmp directory
-      const bucketPath = `${name}/${version}/`
-      const files = await uploadDirectory(
-        ObjectStoreBuckets.PLUGINS,
-        directory,
-        bucketPath
-      )
-      const jsFile = files.find((file: any) => file.name.endsWith(".js"))
-      if (!jsFile) {
-        throw new Error(`Plugin missing .js file.`)
-      }
-      const jsFileName = jsFile.name
-      const pluginId = generatePluginID(name, version)
-
-      // overwrite existing docs entirely if they exist
-      let rev
-      try {
-        const existing = await db.get(pluginId)
-        rev = existing._rev
-      } catch (err) {
-        rev = undefined
-      }
-      const doc = {
-        _id: pluginId,
-        _rev: rev,
-        name,
-        version,
-        description,
-        ...metadata,
-        jsUrl: `${bucketPath}${jsFileName}`,
-      }
-      const response = await db.put(doc)
-      docs.push({
-        ...doc,
-        _rev: response.rev,
-      })
+      const doc = await processPlugin(plugin)
+      docs.push(doc)
     }
     ctx.body = {
       message: "Plugin(s) uploaded successfully",
@@ -87,3 +47,48 @@ export async function fetch(ctx: any) {
 }
 
 export async function destroy(ctx: any) {}
+
+export async function processPlugin(plugin: FileType) {
+  const db = getGlobalDB()
+  const { metadata, directory } = await extractPluginTarball(plugin)
+  const version = metadata.package.version,
+    name = metadata.package.name,
+    description = metadata.package.description
+
+  // first open the tarball into tmp directory
+  const bucketPath = `${name}/${version}/`
+  const files = await uploadDirectory(
+    ObjectStoreBuckets.PLUGINS,
+    directory,
+    bucketPath
+  )
+  const jsFile = files.find((file: any) => file.name.endsWith(".js"))
+  if (!jsFile) {
+    throw new Error(`Plugin missing .js file.`)
+  }
+  const jsFileName = jsFile.name
+  const pluginId = generatePluginID(name, version)
+
+  // overwrite existing docs entirely if they exist
+  let rev
+  try {
+    const existing = await db.get(pluginId)
+    rev = existing._rev
+  } catch (err) {
+    rev = undefined
+  }
+  const doc = {
+    _id: pluginId,
+    _rev: rev,
+    name,
+    version,
+    description,
+    ...metadata,
+    jsUrl: `${bucketPath}${jsFileName}`,
+  }
+  const response = await db.put(doc)
+  return {
+    ...doc,
+    _rev: response.rev,
+  }
+}

packages/server/src/app.ts

@@ -17,10 +17,15 @@ const bullboard = require("./automations/bullboard")
 const { logAlert } = require("@budibase/backend-core/logging")
 const { pinoSettings } = require("@budibase/backend-core")
 const { Thread } = require("./threads")
+const chokidar = require("chokidar")
+const fs = require("fs")
+const path = require("path")
 import redis from "./utilities/redis"
 import * as migrations from "./migrations"
 import { events, installation, tenancy } from "@budibase/backend-core"
 import { createAdminUser, getChecklist } from "./utilities/workerRequests"
+import { processPlugin } from "./api/controllers/plugin"
+import { getGlobalDB } from "@budibase/backend-core/tenancy"
 
 const app = new Koa()
 
@@ -132,6 +137,29 @@ module.exports = server.listen(env.PORT || 0, async () => {
     }
   }
 
+  // monitor plugin directory if required
+  if (env.SELF_HOSTED && env.PLUGINS_DIR && fs.existsSync(env.PLUGINS_DIR)) {
+    const watchPath = path.join(env.PLUGINS_DIR, "./**/dist/*.tar.gz")
+    chokidar
+      .watch(watchPath, {
+        ignored: "**/node_modules",
+        awaitWriteFinish: true,
+      })
+      .on("all", async (event: string, path: string) => {
+        const tenantId = tenancy.getTenantId()
+        await tenancy.doInTenant(tenantId, async () => {
+          try {
+            const split = path.split("/")
+            const name = split[split.length - 1]
+            console.log("Importing plugin:", path)
+            await processPlugin({ name, path })
+          } catch (err) {
+            console.log("Failed to import plugin:", err)
+          }
+        })
+      })
+  }
+
   // check for version updates
   await installation.checkInstallVersion()
 

packages/server/src/environment.js

@@ -77,6 +77,7 @@ module.exports = {
   SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
   BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
+  PLUGINS_DIR: process.env.PLUGINS_DIR,
   // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,

packages/types/src/documents/plugin/index.ts

@@ -2,3 +2,8 @@ export enum PluginType {
   DATASOURCE = "datasource",
   COMPONENT = "component",
 }
+
+export interface FileType {
+  path: string
+  name: string
+}