MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Updating plugins to have a SDK, which can be used for automation action retrieval as well as datasources.

This commit is contained in:
mike12345567 2023-04-11 16:37:26 +01:00
parent e54b48465b
commit ac37d00f69
9 changed files with 120 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
packages/cli/.gitignore vendored
View File

@ -7,3 +7,6 @@ envoy.yaml
*.tar.gz
prebuilds/
dist/
budibase-automation/
budibase-component/
budibase-datasource/

49
packages/server/src/api/controllers/plugin/index.ts
View File

@ -1,31 +1,11 @@
import { npmUpload, urlUpload, githubUpload, fileUpload } from "./uploaders"
import {
plugins as pluginCore,
db as dbCore,
tenancy,
objectStore,
} from "@budibase/backend-core"
import { PluginType, FileType, PluginSource, Plugin } from "@budibase/types"
import { npmUpload, urlUpload, githubUpload } from "./uploaders"
import { plugins as pluginCore } from "@budibase/backend-core"
import { PluginType, FileType, PluginSource } from "@budibase/types"
import env from "../../../environment"
import { ClientAppSocket } from "../../../websocket"
import sdk from "../../../sdk"
import { sdk as pro } from "@budibase/pro"
export async function getPlugins(type?: PluginType) {
const db = tenancy.getGlobalDB()
const response = await db.allDocs(
dbCore.getPluginParams(null, {
include_docs: true,
})
)
let plugins = response.rows.map((row: any) => row.doc) as Plugin[]
plugins = objectStore.enrichPluginURLs(plugins)
if (type) {
return plugins.filter((plugin: Plugin) => plugin.schema?.type === type)
} else {
return plugins
}
}
export async function upload(ctx: any) {
const plugins: FileType[] =
ctx.request.files.file.length > 1
@ -35,7 +15,7 @@ export async function upload(ctx: any) {
let docs = []
// can do single or multiple plugins
for (let plugin of plugins) {
const doc = await processUploadedPlugin(plugin, PluginSource.FILE)
const doc = await sdk.plugins.processUploaded(plugin, PluginSource.FILE)
docs.push(doc)
}
ctx.body = {
@ -105,7 +85,7 @@ export async function create(ctx: any) {
}
export async function fetch(ctx: any) {
ctx.body = await getPlugins()
ctx.body = await sdk.plugins.fetch()
}
export async function destroy(ctx: any) {
@ -119,20 +99,3 @@ export async function destroy(ctx: any) {
ctx.throw(400, err.message)
}
}
export async function processUploadedPlugin(
plugin: FileType,
source?: PluginSource
) {
const { metadata, directory } = await fileUpload(plugin)
pluginCore.validate(metadata?.schema)
// Only allow components in cloud
if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) {
throw new Error("Only component plugins are supported outside of self-host")
}
const doc = await pro.plugins.storePlugin(metadata, directory, source)
ClientAppSocket.emit("plugin-update", { name: doc.name, hash: doc.hash })
return doc
}

66
packages/server/src/automations/actions.ts
View File

@ -15,8 +15,14 @@ import * as delay from "./steps/delay"
import * as queryRow from "./steps/queryRows"
import * as loop from "./steps/loop"
import env from "../environment"
import { AutomationStepSchema, AutomationStepInput, PluginType, AutomationStep } from "@budibase/types"
import { getPlugins } from "../api/controllers/plugin"
import {
AutomationStepSchema,
AutomationStepInput,
PluginType,
AutomationStep,
} from "@budibase/types"
import sdk from "../sdk"
import { getAutomationPlugin } from "../utilities/fileSystem"
const ACTION_IMPLS: Record<
string,
@ -39,25 +45,26 @@ const ACTION_IMPLS: Record<
zapier: zapier.run,
integromat: integromat.run,
}
export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> = {
SEND_EMAIL_SMTP: sendSmtpEmail.definition,
CREATE_ROW: createRow.definition,
UPDATE_ROW: updateRow.definition,
DELETE_ROW: deleteRow.definition,
OUTGOING_WEBHOOK: outgoingWebhook.definition,
EXECUTE_SCRIPT: executeScript.definition,
EXECUTE_QUERY: executeQuery.definition,
SERVER_LOG: serverLog.definition,
DELAY: delay.definition,
FILTER: filter.definition,
QUERY_ROWS: queryRow.definition,
LOOP: loop.definition,
// these used to be lowercase step IDs, maintain for backwards compat
discord: discord.definition,
slack: slack.definition,
zapier: zapier.definition,
integromat: integromat.definition,
}
export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
{
SEND_EMAIL_SMTP: sendSmtpEmail.definition,
CREATE_ROW: createRow.definition,
UPDATE_ROW: updateRow.definition,
DELETE_ROW: deleteRow.definition,
OUTGOING_WEBHOOK: outgoingWebhook.definition,
EXECUTE_SCRIPT: executeScript.definition,
EXECUTE_QUERY: executeQuery.definition,
SERVER_LOG: serverLog.definition,
DELAY: delay.definition,
FILTER: filter.definition,
QUERY_ROWS: queryRow.definition,
LOOP: loop.definition,
// these used to be lowercase step IDs, maintain for backwards compat
discord: discord.definition,
slack: slack.definition,
zapier: zapier.definition,
integromat: integromat.definition,
}
// don't add the bash script/definitions unless in self host
// the fact this isn't included in any definitions means it cannot be
@ -73,7 +80,7 @@ if (env.SELF_HOSTED) {
export async function getActionDefinitions() {
const actionDefinitions = BUILTIN_ACTION_DEFINITIONS
if (env.SELF_HOSTED) {
const plugins = await getPlugins(PluginType.AUTOMATION)
const plugins = await sdk.plugins.fetch(PluginType.AUTOMATION)
for (let plugin of plugins) {
const schema = plugin.schema.schema as AutomationStep
actionDefinitions[schema.stepId] = {
@ -86,8 +93,17 @@ export async function getActionDefinitions() {
}
/* istanbul ignore next */
export async function getAction(actionName: string) {
if (ACTION_IMPLS[actionName] != null) {
return ACTION_IMPLS[actionName]
export async function getAction(stepId: string) {
if (ACTION_IMPLS[stepId] != null) {
return ACTION_IMPLS[stepId]
}
// must be a plugin
if (env.SELF_HOSTED) {
const plugins = await sdk.plugins.fetch(PluginType.AUTOMATION)
const found = plugins.find(plugin => plugin.schema.schema.stepId === stepId)
if (!found) {
throw new Error(`Unable to find action implementation for "${stepId}"`)
}
return (await getAutomationPlugin(found)).action
}
}

6
packages/server/src/integrations/index.ts
View File

@ -14,11 +14,11 @@ import firebase from "./firebase"
import redis from "./redis"
import snowflake from "./snowflake"
import oracle from "./oracle"
import { getPlugins } from "../api/controllers/plugin"
import { SourceName, Integration, PluginType } from "@budibase/types"
import { getDatasourcePlugin } from "../utilities/fileSystem"
import env from "../environment"
import { cloneDeep } from "lodash"
import sdk from "../sdk"
const DEFINITIONS: { [key: string]: Integration } = {
[SourceName.POSTGRES]: postgres.schema,
@ -79,7 +79,7 @@ export async function getDefinition(source: SourceName): Promise<Integration> {
export async function getDefinitions() {
const pluginSchemas: { [key: string]: Integration } = {}
if (env.SELF_HOSTED) {
const plugins = await getPlugins(PluginType.DATASOURCE)
const plugins = await sdk.plugins.fetch(PluginType.DATASOURCE)
// extract the actual schema from each custom
for (let plugin of plugins) {
const sourceId = plugin.name
@ -103,7 +103,7 @@ export async function getIntegration(integration: string) {
return INTEGRATIONS[integration]
}
if (env.SELF_HOSTED) {
const plugins = await getPlugins(PluginType.DATASOURCE)
const plugins = await sdk.plugins.fetch(PluginType.DATASOURCE)
for (let plugin of plugins) {
if (plugin.name === integration) {
// need to use commonJS require due to its dynamic runtime nature

2
packages/server/src/sdk/index.ts
View File

@ -6,6 +6,7 @@ import { default as datasources } from "./app/datasources"
import { default as queries } from "./app/queries"
import { default as rows } from "./app/rows"
import { default as users } from "./users"
import { default as plugins } from "./plugins"
const sdk = {
backups,
@ -16,6 +17,7 @@ const sdk = {
users,
datasources,
queries,
plugins,
}
// default export for TS

5
packages/server/src/sdk/plugins/index.ts Normal file
View File

@ -0,0 +1,5 @@
import * as plugins from "./plugins"
export default {
...plugins,
}

41
packages/server/src/sdk/plugins/plugins.ts Normal file
View File

@ -0,0 +1,41 @@
import { FileType, Plugin, PluginSource, PluginType } from "@budibase/types"
import {
db as dbCore,
objectStore,
plugins as pluginCore,
tenancy,
} from "@budibase/backend-core"
import { fileUpload } from "../../api/controllers/plugin/file"
import env from "../../environment"
import { ClientAppSocket } from "../../websocket"
import { sdk as pro } from "@budibase/pro"
export async function fetch(type?: PluginType) {
const db = tenancy.getGlobalDB()
const response = await db.allDocs(
dbCore.getPluginParams(null, {
include_docs: true,
})
)
let plugins = response.rows.map((row: any) => row.doc) as Plugin[]
plugins = objectStore.enrichPluginURLs(plugins)
if (type) {
return plugins.filter((plugin: Plugin) => plugin.schema?.type === type)
} else {
return plugins
}
}
export async function processUploaded(plugin: FileType, source?: PluginSource) {
const { metadata, directory } = await fileUpload(plugin)
pluginCore.validate(metadata?.schema)
// Only allow components in cloud
if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) {
throw new Error("Only component plugins are supported outside of self-host")
}
const doc = await pro.plugins.storePlugin(metadata, directory, source)
ClientAppSocket.emit("plugin-update", { name: doc.name, hash: doc.hash })
return doc
}

8
packages/server/src/tests/utilities/structures.ts
View File

@ -2,10 +2,14 @@ import { permissions, roles, utils } from "@budibase/backend-core"
import { createHomeScreen } from "../../constants/screens"
import { EMPTY_LAYOUT } from "../../constants/layouts"
import { cloneDeep } from "lodash/fp"
import { BUILTIN_ACTION_DEFINITIONS, TRIGGER_DEFINITIONS } from "../../automations"
import {
BUILTIN_ACTION_DEFINITIONS,
TRIGGER_DEFINITIONS,
} from "../../automations"
import {
Automation,
AutomationActionStepId, AutomationStep,
AutomationActionStepId,
AutomationStep,
AutomationStepType,
AutomationTrigger,
AutomationTriggerStepId,

17
packages/server/src/utilities/fileSystem/plugin.ts
View File

@ -5,6 +5,7 @@ import { join } from "path"
import { objectStore } from "@budibase/backend-core"
const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")
const AUTOMATION_PATH = join(budibaseTempDir(), "automation")
export const getPluginMetadata = async (path: string) => {
let metadata: any = {}
@ -33,12 +34,12 @@ export const getPluginMetadata = async (path: string) => {
return { metadata, directory: path }
}
export const getDatasourcePlugin = async (plugin: Plugin) => {
async function getPluginImpl(path: string, plugin: Plugin) {
const hash = plugin.schema?.hash
if (!fs.existsSync(DATASOURCE_PATH)) {
fs.mkdirSync(DATASOURCE_PATH)
if (!fs.existsSync(path)) {
fs.mkdirSync(path)
}
const filename = join(DATASOURCE_PATH, plugin.name)
const filename = join(path, plugin.name)
const metadataName = `${filename}.bbmetadata`
if (fs.existsSync(filename)) {
const currentHash = fs.readFileSync(metadataName, "utf8")
@ -62,3 +63,11 @@ export const getDatasourcePlugin = async (plugin: Plugin) => {
return require(filename)
}
export const getDatasourcePlugin = async (plugin: Plugin) => {
return getPluginImpl(DATASOURCE_PATH, plugin)
}
export const getAutomationPlugin = async (plugin: Plugin) => {
return getPluginImpl(AUTOMATION_PATH, plugin)
}