@@ -158,7 +166,7 @@
>
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/NewComponentPanel.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/NewComponentPanel.svelte
index b0cd544977..c18df34556 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/NewComponentPanel.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/NewComponentPanel.svelte
@@ -56,7 +56,7 @@
// Add custom components category
if (customComponents?.length) {
enrichedStructure.push({
- name: "Custom components",
+ name: "Plugins",
isCategory: true,
children: customComponents.map(x => ({
...definitions[x],
diff --git a/packages/server/package.json b/packages/server/package.json
index 6165cf495e..5c46b1e454 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -95,6 +95,7 @@
"bcryptjs": "2.4.3",
"bull": "3.29.3",
"chmodr": "1.2.0",
+ "chokidar": "^3.5.3",
"csvtojson": "2.0.10",
"curlconverter": "3.21.0",
"dotenv": "8.2.0",
diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js
index fc9fde0a02..8a5f8e9477 100644
--- a/packages/server/scripts/dev/manage.js
+++ b/packages/server/scripts/dev/manage.js
@@ -58,6 +58,7 @@ async function init() {
DEPLOYMENT_ENVIRONMENT: "development",
BB_ADMIN_USER_EMAIL: "",
BB_ADMIN_USER_PASSWORD: "",
+ PLUGINS_DIR: "",
}
let envFile = ""
Object.keys(envFileJson).forEach(key => {
diff --git a/packages/server/src/api/controllers/datasource.js b/packages/server/src/api/controllers/datasource.js
index 8b9b765a5f..36d345912e 100644
--- a/packages/server/src/api/controllers/datasource.js
+++ b/packages/server/src/api/controllers/datasource.js
@@ -7,7 +7,7 @@ const {
getTableParams,
} = require("../../db/utils")
const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
-const { integrations } = require("../../integrations")
+const { getIntegration } = require("../../integrations")
const { getDatasourceAndQuery } = require("./row/utils")
const { invalidateDynamicVariables } = require("../../threads/utils")
const { getAppDB } = require("@budibase/backend-core/context")
@@ -114,7 +114,7 @@ exports.update = async function (ctx) {
// Drain connection pools when configuration is changed
if (datasource.source) {
- const source = integrations[datasource.source]
+ const source = await getIntegration(datasource.source)
if (source && source.pool) {
await source.pool.end()
}
@@ -149,7 +149,7 @@ exports.save = async function (ctx) {
// Drain connection pools when configuration is changed
if (datasource.source) {
- const source = integrations[datasource.source]
+ const source = await getIntegration(datasource.source)
if (source && source.pool) {
await source.pool.end()
}
@@ -218,7 +218,7 @@ function updateError(error, newError, tables) {
}
const buildSchemaHelper = async datasource => {
- const Connector = integrations[datasource.source]
+ const Connector = await getIntegration(datasource.source)
// Connect to the DB and build the schema
const connector = new Connector(datasource.config)
diff --git a/packages/server/src/api/controllers/plugin.ts b/packages/server/src/api/controllers/plugin.ts
index 48104f24b9..93e569e0c8 100644
--- a/packages/server/src/api/controllers/plugin.ts
+++ b/packages/server/src/api/controllers/plugin.ts
@@ -3,7 +3,7 @@ import { extractPluginTarball } from "../../utilities/fileSystem"
import { getGlobalDB } from "@budibase/backend-core/tenancy"
import { generatePluginID, getPluginParams } from "../../db/utils"
import { uploadDirectory } from "@budibase/backend-core/objectStore"
-import { PluginType } from "@budibase/types"
+import { PluginType, FileType } from "@budibase/types"
export async function getPlugins(type?: PluginType) {
const db = getGlobalDB()
@@ -21,56 +21,16 @@ export async function getPlugins(type?: PluginType) {
}
export async function upload(ctx: any) {
- const plugins =
+ const plugins: FileType[] =
ctx.request.files.file.length > 1
? Array.from(ctx.request.files.file)
: [ctx.request.files.file]
- const db = getGlobalDB()
try {
let docs = []
// can do single or multiple plugins
for (let plugin of plugins) {
- const { metadata, directory } = await extractPluginTarball(plugin)
- const version = metadata.package.version,
- name = metadata.package.name,
- description = metadata.package.description
-
- // first open the tarball into tmp directory
- const bucketPath = `${name}/${version}/`
- const files = await uploadDirectory(
- ObjectStoreBuckets.PLUGINS,
- directory,
- bucketPath
- )
- const jsFile = files.find((file: any) => file.name.endsWith(".js"))
- if (!jsFile) {
- throw new Error(`Plugin missing .js file.`)
- }
- const jsFileName = jsFile.name
- const pluginId = generatePluginID(name, version)
-
- // overwrite existing docs entirely if they exist
- let rev
- try {
- const existing = await db.get(pluginId)
- rev = existing._rev
- } catch (err) {
- rev = undefined
- }
- const doc = {
- _id: pluginId,
- _rev: rev,
- name,
- version,
- description,
- ...metadata,
- jsUrl: `${bucketPath}${jsFileName}`,
- }
- const response = await db.put(doc)
- docs.push({
- ...doc,
- _rev: response.rev,
- })
+ const doc = await processPlugin(plugin)
+ docs.push(doc)
}
ctx.body = {
message: "Plugin(s) uploaded successfully",
@@ -87,3 +47,48 @@ export async function fetch(ctx: any) {
}
export async function destroy(ctx: any) {}
+
+export async function processPlugin(plugin: FileType) {
+ const db = getGlobalDB()
+ const { metadata, directory } = await extractPluginTarball(plugin)
+ const version = metadata.package.version,
+ name = metadata.package.name,
+ description = metadata.package.description
+
+ // first open the tarball into tmp directory
+ const bucketPath = `${name}/${version}/`
+ const files = await uploadDirectory(
+ ObjectStoreBuckets.PLUGINS,
+ directory,
+ bucketPath
+ )
+ const jsFile = files.find((file: any) => file.name.endsWith(".js"))
+ if (!jsFile) {
+ throw new Error(`Plugin missing .js file.`)
+ }
+ const jsFileName = jsFile.name
+ const pluginId = generatePluginID(name, version)
+
+ // overwrite existing docs entirely if they exist
+ let rev
+ try {
+ const existing = await db.get(pluginId)
+ rev = existing._rev
+ } catch (err) {
+ rev = undefined
+ }
+ const doc = {
+ _id: pluginId,
+ _rev: rev,
+ name,
+ version,
+ description,
+ ...metadata,
+ jsUrl: `${bucketPath}${jsFileName}`,
+ }
+ const response = await db.put(doc)
+ return {
+ ...doc,
+ _rev: response.rev,
+ }
+}
diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts
index 62301d57ca..2f8adc879d 100644
--- a/packages/server/src/app.ts
+++ b/packages/server/src/app.ts
@@ -17,10 +17,15 @@ const bullboard = require("./automations/bullboard")
const { logAlert } = require("@budibase/backend-core/logging")
const { pinoSettings } = require("@budibase/backend-core")
const { Thread } = require("./threads")
+const chokidar = require("chokidar")
+const fs = require("fs")
+const path = require("path")
import redis from "./utilities/redis"
import * as migrations from "./migrations"
import { events, installation, tenancy } from "@budibase/backend-core"
import { createAdminUser, getChecklist } from "./utilities/workerRequests"
+import { processPlugin } from "./api/controllers/plugin"
+import { getGlobalDB } from "@budibase/backend-core/tenancy"
const app = new Koa()
@@ -132,6 +137,29 @@ module.exports = server.listen(env.PORT || 0, async () => {
}
}
+ // monitor plugin directory if required
+ if (env.SELF_HOSTED && env.PLUGINS_DIR && fs.existsSync(env.PLUGINS_DIR)) {
+ const watchPath = path.join(env.PLUGINS_DIR, "./**/dist/*.tar.gz")
+ chokidar
+ .watch(watchPath, {
+ ignored: "**/node_modules",
+ awaitWriteFinish: true,
+ })
+ .on("all", async (event: string, path: string) => {
+ const tenantId = tenancy.getTenantId()
+ await tenancy.doInTenant(tenantId, async () => {
+ try {
+ const split = path.split("/")
+ const name = split[split.length - 1]
+ console.log("Importing plugin:", path)
+ await processPlugin({ name, path })
+ } catch (err) {
+ console.log("Failed to import plugin:", err)
+ }
+ })
+ })
+ }
+
// check for version updates
await installation.checkInstallVersion()
diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js
index c2e2815e00..5a7aa61b9d 100644
--- a/packages/server/src/environment.js
+++ b/packages/server/src/environment.js
@@ -77,6 +77,7 @@ module.exports = {
SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
+ PLUGINS_DIR: process.env.PLUGINS_DIR,
// flags
ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
DISABLE_THREADING: process.env.DISABLE_THREADING,
diff --git a/packages/server/src/integrations/base/query.ts b/packages/server/src/integrations/base/query.ts
index 1f3ed3dd74..7435b28141 100644
--- a/packages/server/src/integrations/base/query.ts
+++ b/packages/server/src/integrations/base/query.ts
@@ -1,11 +1,11 @@
import { QueryJson, Datasource } from "@budibase/types"
-const { integrations } = require("../index")
+const { getIntegration } = require("../index")
export async function makeExternalQuery(
datasource: Datasource,
json: QueryJson
) {
- const Integration = integrations[datasource.source]
+ const Integration = await getIntegration(datasource.source)
// query is the opinionated function
if (Integration.prototype.query) {
const integration = new Integration(datasource.config)
diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts
index bde7235ac8..bf267e46cc 100644
--- a/packages/server/src/integrations/index.ts
+++ b/packages/server/src/integrations/index.ts
@@ -67,8 +67,22 @@ if (environment.SELF_HOSTED) {
module.exports = {
getDefinitions: async () => {
- const custom = await getPlugins(PluginType.DATASOURCE)
- return cloneDeep(DEFINITIONS)
+ const plugins = await getPlugins(PluginType.DATASOURCE)
+ // extract the actual schema from each custom
+ const pluginSchemas: { [key: string]: Integration } = {}
+ for (let plugin of plugins) {
+ const sourceId = plugin.name
+ pluginSchemas[sourceId] = {
+ ...plugin.schema["schema"],
+ custom: true,
+ }
+ }
+ return {
+ ...cloneDeep(DEFINITIONS),
+ ...pluginSchemas,
+ }
+ },
+ getIntegration: async () => {
+ return INTEGRATIONS
},
- integrations: INTEGRATIONS,
}
diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts
index 6b93a00200..86cbf89c87 100644
--- a/packages/server/src/threads/query.ts
+++ b/packages/server/src/threads/query.ts
@@ -2,7 +2,7 @@ import { default as threadUtils } from "./utils"
threadUtils.threadSetup()
import { WorkerCallback, QueryEvent, QueryVariable } from "./definitions"
const ScriptRunner = require("../utilities/scriptRunner")
-const { integrations } = require("../integrations")
+const { getIntegration } = require("../integrations")
const { processStringSync } = require("@budibase/string-templates")
const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
const {
@@ -62,7 +62,7 @@ class QueryRunner {
let datasourceClone = cloneDeep(datasource)
let fieldsClone = cloneDeep(fields)
- const Integration = integrations[datasourceClone.source]
+ const Integration = await getIntegration(datasourceClone.source)
if (!Integration) {
throw "Integration type does not exist."
}
diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock
index 7fd0fde8b3..b0aebab9a7 100644
--- a/packages/server/yarn.lock
+++ b/packages/server/yarn.lock
@@ -4411,7 +4411,7 @@ chmodr@1.2.0:
resolved "https://registry.yarnpkg.com/chmodr/-/chmodr-1.2.0.tgz#720e96caa09b7f1cdbb01529b7d0ab6bc5e118b9"
integrity sha512-Y5uI7Iq/Az6HgJEL6pdw7THVd7jbVOTPwsmcPOBjQL8e3N+pz872kzK5QxYGEy21iRys+iHWV0UZQXDFJo1hyA==
-chokidar@^3.5.2:
+chokidar@^3.5.2, chokidar@^3.5.3:
version "3.5.3"
resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd"
integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==
diff --git a/packages/types/src/documents/plugin/index.ts b/packages/types/src/documents/plugin/index.ts
index 03a239a6c9..8133e91523 100644
--- a/packages/types/src/documents/plugin/index.ts
+++ b/packages/types/src/documents/plugin/index.ts
@@ -2,3 +2,8 @@ export enum PluginType {
DATASOURCE = "datasource",
COMPONENT = "component",
}
+
+export interface FileType {
+ path: string
+ name: string
+}