From ac37d00f69ef4d70ac303a535792d36beacf3ff3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 11 Apr 2023 16:37:26 +0100 Subject: [PATCH] Updating plugins to have a SDK, which can be used for automation action retrieval as well as datasources. --- packages/cli/.gitignore | 3 + .../src/api/controllers/plugin/index.ts | 49 ++------------ packages/server/src/automations/actions.ts | 66 ++++++++++++------- packages/server/src/integrations/index.ts | 6 +- packages/server/src/sdk/index.ts | 2 + packages/server/src/sdk/plugins/index.ts | 5 ++ packages/server/src/sdk/plugins/plugins.ts | 41 ++++++++++++ .../server/src/tests/utilities/structures.ts | 8 ++- .../server/src/utilities/fileSystem/plugin.ts | 17 +++-- 9 files changed, 120 insertions(+), 77 deletions(-) create mode 100644 packages/server/src/sdk/plugins/index.ts create mode 100644 packages/server/src/sdk/plugins/plugins.ts diff --git a/packages/cli/.gitignore b/packages/cli/.gitignore index 8d8de9a0da..b1d9be617c 100644 --- a/packages/cli/.gitignore +++ b/packages/cli/.gitignore @@ -7,3 +7,6 @@ envoy.yaml *.tar.gz prebuilds/ dist/ +budibase-automation/ +budibase-component/ +budibase-datasource/ diff --git a/packages/server/src/api/controllers/plugin/index.ts b/packages/server/src/api/controllers/plugin/index.ts index faecbc1fd8..e8ae847fc6 100644 --- a/packages/server/src/api/controllers/plugin/index.ts +++ b/packages/server/src/api/controllers/plugin/index.ts @@ -1,31 +1,11 @@ -import { npmUpload, urlUpload, githubUpload, fileUpload } from "./uploaders" -import { - plugins as pluginCore, - db as dbCore, - tenancy, - objectStore, -} from "@budibase/backend-core" -import { PluginType, FileType, PluginSource, Plugin } from "@budibase/types" +import { npmUpload, urlUpload, githubUpload } from "./uploaders" +import { plugins as pluginCore } from "@budibase/backend-core" +import { PluginType, FileType, PluginSource } from "@budibase/types" import env from "../../../environment" import { ClientAppSocket } from "../../../websocket" +import sdk from "../../../sdk" import { sdk as pro } from "@budibase/pro" -export async function getPlugins(type?: PluginType) { - const db = tenancy.getGlobalDB() - const response = await db.allDocs( - dbCore.getPluginParams(null, { - include_docs: true, - }) - ) - let plugins = response.rows.map((row: any) => row.doc) as Plugin[] - plugins = objectStore.enrichPluginURLs(plugins) - if (type) { - return plugins.filter((plugin: Plugin) => plugin.schema?.type === type) - } else { - return plugins - } -} - export async function upload(ctx: any) { const plugins: FileType[] = ctx.request.files.file.length > 1 @@ -35,7 +15,7 @@ export async function upload(ctx: any) { let docs = [] // can do single or multiple plugins for (let plugin of plugins) { - const doc = await processUploadedPlugin(plugin, PluginSource.FILE) + const doc = await sdk.plugins.processUploaded(plugin, PluginSource.FILE) docs.push(doc) } ctx.body = { @@ -105,7 +85,7 @@ export async function create(ctx: any) { } export async function fetch(ctx: any) { - ctx.body = await getPlugins() + ctx.body = await sdk.plugins.fetch() } export async function destroy(ctx: any) { @@ -119,20 +99,3 @@ export async function destroy(ctx: any) { ctx.throw(400, err.message) } } - -export async function processUploadedPlugin( - plugin: FileType, - source?: PluginSource -) { - const { metadata, directory } = await fileUpload(plugin) - pluginCore.validate(metadata?.schema) - - // Only allow components in cloud - if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) { - throw new Error("Only component plugins are supported outside of self-host") - } - - const doc = await pro.plugins.storePlugin(metadata, directory, source) - ClientAppSocket.emit("plugin-update", { name: doc.name, hash: doc.hash }) - return doc -} diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index a82be686c0..2a6b760725 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -15,8 +15,14 @@ import * as delay from "./steps/delay" import * as queryRow from "./steps/queryRows" import * as loop from "./steps/loop" import env from "../environment" -import { AutomationStepSchema, AutomationStepInput, PluginType, AutomationStep } from "@budibase/types" -import { getPlugins } from "../api/controllers/plugin" +import { + AutomationStepSchema, + AutomationStepInput, + PluginType, + AutomationStep, +} from "@budibase/types" +import sdk from "../sdk" +import { getAutomationPlugin } from "../utilities/fileSystem" const ACTION_IMPLS: Record< string, @@ -39,25 +45,26 @@ const ACTION_IMPLS: Record< zapier: zapier.run, integromat: integromat.run, } -export const BUILTIN_ACTION_DEFINITIONS: Record = { - SEND_EMAIL_SMTP: sendSmtpEmail.definition, - CREATE_ROW: createRow.definition, - UPDATE_ROW: updateRow.definition, - DELETE_ROW: deleteRow.definition, - OUTGOING_WEBHOOK: outgoingWebhook.definition, - EXECUTE_SCRIPT: executeScript.definition, - EXECUTE_QUERY: executeQuery.definition, - SERVER_LOG: serverLog.definition, - DELAY: delay.definition, - FILTER: filter.definition, - QUERY_ROWS: queryRow.definition, - LOOP: loop.definition, - // these used to be lowercase step IDs, maintain for backwards compat - discord: discord.definition, - slack: slack.definition, - zapier: zapier.definition, - integromat: integromat.definition, -} +export const BUILTIN_ACTION_DEFINITIONS: Record = + { + SEND_EMAIL_SMTP: sendSmtpEmail.definition, + CREATE_ROW: createRow.definition, + UPDATE_ROW: updateRow.definition, + DELETE_ROW: deleteRow.definition, + OUTGOING_WEBHOOK: outgoingWebhook.definition, + EXECUTE_SCRIPT: executeScript.definition, + EXECUTE_QUERY: executeQuery.definition, + SERVER_LOG: serverLog.definition, + DELAY: delay.definition, + FILTER: filter.definition, + QUERY_ROWS: queryRow.definition, + LOOP: loop.definition, + // these used to be lowercase step IDs, maintain for backwards compat + discord: discord.definition, + slack: slack.definition, + zapier: zapier.definition, + integromat: integromat.definition, + } // don't add the bash script/definitions unless in self host // the fact this isn't included in any definitions means it cannot be @@ -73,7 +80,7 @@ if (env.SELF_HOSTED) { export async function getActionDefinitions() { const actionDefinitions = BUILTIN_ACTION_DEFINITIONS if (env.SELF_HOSTED) { - const plugins = await getPlugins(PluginType.AUTOMATION) + const plugins = await sdk.plugins.fetch(PluginType.AUTOMATION) for (let plugin of plugins) { const schema = plugin.schema.schema as AutomationStep actionDefinitions[schema.stepId] = { @@ -86,8 +93,17 @@ export async function getActionDefinitions() { } /* istanbul ignore next */ -export async function getAction(actionName: string) { - if (ACTION_IMPLS[actionName] != null) { - return ACTION_IMPLS[actionName] +export async function getAction(stepId: string) { + if (ACTION_IMPLS[stepId] != null) { + return ACTION_IMPLS[stepId] + } + // must be a plugin + if (env.SELF_HOSTED) { + const plugins = await sdk.plugins.fetch(PluginType.AUTOMATION) + const found = plugins.find(plugin => plugin.schema.schema.stepId === stepId) + if (!found) { + throw new Error(`Unable to find action implementation for "${stepId}"`) + } + return (await getAutomationPlugin(found)).action } } diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index edbce6db0a..f3285e441f 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -14,11 +14,11 @@ import firebase from "./firebase" import redis from "./redis" import snowflake from "./snowflake" import oracle from "./oracle" -import { getPlugins } from "../api/controllers/plugin" import { SourceName, Integration, PluginType } from "@budibase/types" import { getDatasourcePlugin } from "../utilities/fileSystem" import env from "../environment" import { cloneDeep } from "lodash" +import sdk from "../sdk" const DEFINITIONS: { [key: string]: Integration } = { [SourceName.POSTGRES]: postgres.schema, @@ -79,7 +79,7 @@ export async function getDefinition(source: SourceName): Promise { export async function getDefinitions() { const pluginSchemas: { [key: string]: Integration } = {} if (env.SELF_HOSTED) { - const plugins = await getPlugins(PluginType.DATASOURCE) + const plugins = await sdk.plugins.fetch(PluginType.DATASOURCE) // extract the actual schema from each custom for (let plugin of plugins) { const sourceId = plugin.name @@ -103,7 +103,7 @@ export async function getIntegration(integration: string) { return INTEGRATIONS[integration] } if (env.SELF_HOSTED) { - const plugins = await getPlugins(PluginType.DATASOURCE) + const plugins = await sdk.plugins.fetch(PluginType.DATASOURCE) for (let plugin of plugins) { if (plugin.name === integration) { // need to use commonJS require due to its dynamic runtime nature diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts index 294c99a12c..1bf7d89604 100644 --- a/packages/server/src/sdk/index.ts +++ b/packages/server/src/sdk/index.ts @@ -6,6 +6,7 @@ import { default as datasources } from "./app/datasources" import { default as queries } from "./app/queries" import { default as rows } from "./app/rows" import { default as users } from "./users" +import { default as plugins } from "./plugins" const sdk = { backups, @@ -16,6 +17,7 @@ const sdk = { users, datasources, queries, + plugins, } // default export for TS diff --git a/packages/server/src/sdk/plugins/index.ts b/packages/server/src/sdk/plugins/index.ts new file mode 100644 index 0000000000..863cfa0db6 --- /dev/null +++ b/packages/server/src/sdk/plugins/index.ts @@ -0,0 +1,5 @@ +import * as plugins from "./plugins" + +export default { + ...plugins, +} diff --git a/packages/server/src/sdk/plugins/plugins.ts b/packages/server/src/sdk/plugins/plugins.ts new file mode 100644 index 0000000000..58670146ec --- /dev/null +++ b/packages/server/src/sdk/plugins/plugins.ts @@ -0,0 +1,41 @@ +import { FileType, Plugin, PluginSource, PluginType } from "@budibase/types" +import { + db as dbCore, + objectStore, + plugins as pluginCore, + tenancy, +} from "@budibase/backend-core" +import { fileUpload } from "../../api/controllers/plugin/file" +import env from "../../environment" +import { ClientAppSocket } from "../../websocket" +import { sdk as pro } from "@budibase/pro" + +export async function fetch(type?: PluginType) { + const db = tenancy.getGlobalDB() + const response = await db.allDocs( + dbCore.getPluginParams(null, { + include_docs: true, + }) + ) + let plugins = response.rows.map((row: any) => row.doc) as Plugin[] + plugins = objectStore.enrichPluginURLs(plugins) + if (type) { + return plugins.filter((plugin: Plugin) => plugin.schema?.type === type) + } else { + return plugins + } +} + +export async function processUploaded(plugin: FileType, source?: PluginSource) { + const { metadata, directory } = await fileUpload(plugin) + pluginCore.validate(metadata?.schema) + + // Only allow components in cloud + if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) { + throw new Error("Only component plugins are supported outside of self-host") + } + + const doc = await pro.plugins.storePlugin(metadata, directory, source) + ClientAppSocket.emit("plugin-update", { name: doc.name, hash: doc.hash }) + return doc +} diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts index ed2210270e..e4e03fb85d 100644 --- a/packages/server/src/tests/utilities/structures.ts +++ b/packages/server/src/tests/utilities/structures.ts @@ -2,10 +2,14 @@ import { permissions, roles, utils } from "@budibase/backend-core" import { createHomeScreen } from "../../constants/screens" import { EMPTY_LAYOUT } from "../../constants/layouts" import { cloneDeep } from "lodash/fp" -import { BUILTIN_ACTION_DEFINITIONS, TRIGGER_DEFINITIONS } from "../../automations" +import { + BUILTIN_ACTION_DEFINITIONS, + TRIGGER_DEFINITIONS, +} from "../../automations" import { Automation, - AutomationActionStepId, AutomationStep, + AutomationActionStepId, + AutomationStep, AutomationStepType, AutomationTrigger, AutomationTriggerStepId, diff --git a/packages/server/src/utilities/fileSystem/plugin.ts b/packages/server/src/utilities/fileSystem/plugin.ts index 0bc2b7de44..3e1e9bef4d 100644 --- a/packages/server/src/utilities/fileSystem/plugin.ts +++ b/packages/server/src/utilities/fileSystem/plugin.ts @@ -5,6 +5,7 @@ import { join } from "path" import { objectStore } from "@budibase/backend-core" const DATASOURCE_PATH = join(budibaseTempDir(), "datasource") +const AUTOMATION_PATH = join(budibaseTempDir(), "automation") export const getPluginMetadata = async (path: string) => { let metadata: any = {} @@ -33,12 +34,12 @@ export const getPluginMetadata = async (path: string) => { return { metadata, directory: path } } -export const getDatasourcePlugin = async (plugin: Plugin) => { +async function getPluginImpl(path: string, plugin: Plugin) { const hash = plugin.schema?.hash - if (!fs.existsSync(DATASOURCE_PATH)) { - fs.mkdirSync(DATASOURCE_PATH) + if (!fs.existsSync(path)) { + fs.mkdirSync(path) } - const filename = join(DATASOURCE_PATH, plugin.name) + const filename = join(path, plugin.name) const metadataName = `${filename}.bbmetadata` if (fs.existsSync(filename)) { const currentHash = fs.readFileSync(metadataName, "utf8") @@ -62,3 +63,11 @@ export const getDatasourcePlugin = async (plugin: Plugin) => { return require(filename) } + +export const getDatasourcePlugin = async (plugin: Plugin) => { + return getPluginImpl(DATASOURCE_PATH, plugin) +} + +export const getAutomationPlugin = async (plugin: Plugin) => { + return getPluginImpl(AUTOMATION_PATH, plugin) +}