Add hot reloading for component plugins via watched plugin directory

charts/budibase/templates/app-service-deployment.yaml

@@ -124,11 +124,15 @@ spec:
           value: {{ .Values.globals.tenantFeatureFlags | quote }}
         {{ if .Values.globals.bbAdminUserEmail }}
         - name: BB_ADMIN_USER_EMAIL
-          value: { { .Values.globals.bbAdminUserEmail | quote } }
+          value: {{ .Values.globals.bbAdminUserEmail | quote }}
         {{ end }}
         {{ if .Values.globals.bbAdminUserPassword }}
         - name: BB_ADMIN_USER_PASSWORD
-          value: { { .Values.globals.bbAdminUserPassword | quote } }
+          value: {{ .Values.globals.bbAdminUserPassword | quote }}
+        {{ end }}
+        {{ if .Values.globals.pluginsDir }}
+        - name: PLUGINS_DIR
+          value: { { .Values.globals.pluginsDir | quote }}
         {{ end }}
 
         image: budibase/apps:{{ .Values.globals.appVersion }}

hosting/.env

@@ -23,3 +23,6 @@ BUDIBASE_ENVIRONMENT=PRODUCTION
 # An admin user can be automatically created initially if these are set
 BB_ADMIN_USER_EMAIL=
 BB_ADMIN_USER_PASSWORD=
+
+# A path that is watched for plugin bundles. Any bundles found are imported automatically/
+PLUGINS_DIR=

hosting/hosting.properties

@@ -23,3 +23,6 @@ BUDIBASE_ENVIRONMENT=PRODUCTION
 # An admin user can be automatically created initially if these are set
 BB_ADMIN_USER_EMAIL=
 BB_ADMIN_USER_PASSWORD=
+
+# A path that is watched for plugin bundles. Any bundles found are imported automatically/
+PLUGINS_DIR=

packages/server/package.json

@@ -95,6 +95,7 @@
     "bcryptjs": "2.4.3",
     "bull": "3.29.3",
     "chmodr": "1.2.0",
+    "chokidar": "^3.5.3",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
     "dotenv": "8.2.0",

packages/server/scripts/dev/manage.js

@@ -58,6 +58,7 @@ async function init() {
       DEPLOYMENT_ENVIRONMENT: "development",
       BB_ADMIN_USER_EMAIL: "",
       BB_ADMIN_USER_PASSWORD: "",
+      PLUGINS_DIR: "",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

packages/server/src/api/controllers/plugin.ts

@@ -3,7 +3,7 @@ import { extractPluginTarball } from "../../utilities/fileSystem"
 import { getGlobalDB } from "@budibase/backend-core/tenancy"
 import { generatePluginID, getPluginParams } from "../../db/utils"
 import { uploadDirectory } from "@budibase/backend-core/objectStore"
-import { PluginType } from "@budibase/types"
+import { PluginType, FileType } from "@budibase/types"
 
 export async function getPlugins(type?: PluginType) {
   const db = getGlobalDB()
@@ -21,15 +21,35 @@ export async function getPlugins(type?: PluginType) {
 }
 
 export async function upload(ctx: any) {
-  const plugins =
+  const plugins: FileType[] =
     ctx.request.files.file.length > 1
       ? Array.from(ctx.request.files.file)
       : [ctx.request.files.file]
-  const db = getGlobalDB()
   try {
     let docs = []
     // can do single or multiple plugins
     for (let plugin of plugins) {
+      const doc = await processPlugin(plugin)
+      docs.push(doc)
+    }
+    ctx.body = {
+      message: "Plugin(s) uploaded successfully",
+      plugins: docs,
+    }
+  } catch (err: any) {
+    const errMsg = err?.message ? err?.message : err
+    ctx.throw(400, `Failed to import plugin: ${errMsg}`)
+  }
+}
+
+export async function fetch(ctx: any) {
+  ctx.body = await getPlugins()
+}
+
+export async function destroy(ctx: any) {}
+
+export async function processPlugin(plugin: FileType) {
+  const db = getGlobalDB()
   const { metadata, directory } = await extractPluginTarball(plugin)
   const version = metadata.package.version,
     name = metadata.package.name,
@@ -67,23 +87,8 @@ export async function upload(ctx: any) {
     jsUrl: `${bucketPath}${jsFileName}`,
   }
   const response = await db.put(doc)
-      docs.push({
+  return {
     ...doc,
     _rev: response.rev,
-      })
-    }
-    ctx.body = {
-      message: "Plugin(s) uploaded successfully",
-      plugins: docs,
-    }
-  } catch (err: any) {
-    const errMsg = err?.message ? err?.message : err
-    ctx.throw(400, `Failed to import plugin: ${errMsg}`)
   }
 }
-
-export async function fetch(ctx: any) {
-  ctx.body = await getPlugins()
-}
-
-export async function destroy(ctx: any) {}

packages/server/src/app.ts

@@ -17,10 +17,15 @@ const bullboard = require("./automations/bullboard")
 const { logAlert } = require("@budibase/backend-core/logging")
 const { pinoSettings } = require("@budibase/backend-core")
 const { Thread } = require("./threads")
+const chokidar = require("chokidar")
+const fs = require("fs")
+const path = require("path")
 import redis from "./utilities/redis"
 import * as migrations from "./migrations"
 import { events, installation, tenancy } from "@budibase/backend-core"
 import { createAdminUser, getChecklist } from "./utilities/workerRequests"
+import { processPlugin } from "./api/controllers/plugin"
+import { getGlobalDB } from "@budibase/backend-core/tenancy"
 
 const app = new Koa()
 
@@ -132,6 +137,29 @@ module.exports = server.listen(env.PORT || 0, async () => {
     }
   }
 
+  // monitor plugin directory if required
+  if (env.SELF_HOSTED && env.PLUGINS_DIR && fs.existsSync(env.PLUGINS_DIR)) {
+    const watchPath = path.join(env.PLUGINS_DIR, "./**/dist/*.tar.gz")
+    chokidar
+      .watch(watchPath, {
+        ignored: "**/node_modules",
+        awaitWriteFinish: true,
+      })
+      .on("all", async (event: string, path: string) => {
+        const tenantId = tenancy.getTenantId()
+        await tenancy.doInTenant(tenantId, async () => {
+          try {
+            const split = path.split("/")
+            const name = split[split.length - 1]
+            console.log("Importing plugin:", path)
+            await processPlugin({ name, path })
+          } catch (err) {
+            console.log("Failed to import plugin:", err)
+          }
+        })
+      })
+  }
+
   // check for version updates
   await installation.checkInstallVersion()
 

packages/server/src/environment.js

@@ -77,6 +77,7 @@ module.exports = {
   SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
   BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
+  PLUGINS_DIR: process.env.PLUGINS_DIR,
   // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,

packages/types/src/documents/plugin/index.ts

@@ -2,3 +2,8 @@ export enum PluginType {
   DATASOURCE = "datasource",
   COMPONENT = "component",
 }
+
+export interface FileType {
+  path: string
+  name: string
+}