Bulk of the automation plugin system, supporting custom plugins.

packages/backend-core/src/logging/pino/middleware.ts

@@ -1,5 +1,5 @@
 import env from "../../environment"
-import logger from "./logger"
+import { logger } from "./logger"
 import { IncomingMessage } from "http"
 const pino = require("koa-pino-logger")
 import { Options } from "pino-http"

packages/backend-core/src/plugin/utils.ts

@@ -91,7 +91,7 @@ function validateDatasource(schema: any) {
 }
 
 function validateAutomation(schema: any) {
-  const basePropsValidator = joi.object({
+  const basePropsValidator = joi.object().pattern(joi.string(), {
     type: joi
       .string()
       .allow(...Object.values(AutomationIOType))

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte

@@ -26,7 +26,7 @@
 
   const external = actions.reduce((acc, elm) => {
     const [k, v] = elm
-    if (!v.internal) {
+    if (!v.internal && !v.custom) {
       acc[k] = v
     }
     return acc
@@ -34,7 +34,7 @@
 
   const internal = actions.reduce((acc, elm) => {
     const [k, v] = elm
-    if (v.internal) {
+    if (v.internal || v.custom) {
       acc[k] = v
     }
     delete acc.LOOP

packages/server/src/api/controllers/automation.ts

@@ -16,9 +16,15 @@ import { setTestFlag, clearTestFlag } from "../../utilities/redis"
 import { context, cache, events } from "@budibase/backend-core"
 import { automations } from "@budibase/pro"
 import { Automation, BBContext } from "@budibase/types"
+import { getActionDefinitions as actionDefs } from "../../automations/actions"
 
-const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
-const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
+async function getActionDefinitions() {
+  return removeDeprecated(await actionDefs())
+}
+
+function getTriggerDefinitions() {
+  return removeDeprecated(triggers.TRIGGER_DEFINITIONS)
+}
 
 /*************************
  *                       *
@@ -228,17 +234,17 @@ export async function clearLogError(ctx: BBContext) {
 }
 
 export async function getActionList(ctx: BBContext) {
-  ctx.body = ACTION_DEFS
+  ctx.body = await getActionDefinitions()
 }
 
 export async function getTriggerList(ctx: BBContext) {
-  ctx.body = TRIGGER_DEFS
+  ctx.body = getTriggerDefinitions()
 }
 
 export async function getDefinitionList(ctx: BBContext) {
   ctx.body = {
-    trigger: TRIGGER_DEFS,
-    action: ACTION_DEFS,
+    trigger: getTriggerDefinitions(),
+    action: await getActionDefinitions(),
   }
 }
 

packages/server/src/api/routes/tests/automation.spec.js

@@ -7,7 +7,7 @@ const {
 const setup = require("./utilities")
 const { basicAutomation, newAutomation, automationTrigger, automationStep } = setup.structures
 const MAX_RETRIES = 4
-const { TRIGGER_DEFINITIONS, ACTION_DEFINITIONS } = require("../../../automations")
+const { TRIGGER_DEFINITIONS, BUILTIN_ACTION_DEFINITIONS } = require("../../../automations")
 const { events } = require("@budibase/backend-core")
 
 
@@ -52,7 +52,7 @@ describe("/automations", () => {
         .expect('Content-Type', /json/)
         .expect(200)
 
-      let definitionsLength = Object.keys(ACTION_DEFINITIONS).length
+      let definitionsLength = Object.keys(BUILTIN_ACTION_DEFINITIONS).length
       definitionsLength-- // OUTGOING_WEBHOOK is deprecated
 
       expect(Object.keys(res.body.action).length).toBeGreaterThanOrEqual(definitionsLength)

packages/server/src/automations/actions.ts

@@ -15,7 +15,8 @@ import * as delay from "./steps/delay"
 import * as queryRow from "./steps/queryRows"
 import * as loop from "./steps/loop"
 import env from "../environment"
-import { AutomationStepSchema, AutomationStepInput } from "@budibase/types"
+import { AutomationStepSchema, AutomationStepInput, PluginType, AutomationStep } from "@budibase/types"
+import { getPlugins } from "../api/controllers/plugin"
 
 const ACTION_IMPLS: Record<
   string,
@@ -38,7 +39,7 @@ const ACTION_IMPLS: Record<
   zapier: zapier.run,
   integromat: integromat.run,
 }
-export const ACTION_DEFINITIONS: Record<string, AutomationStepSchema> = {
+export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> = {
   SEND_EMAIL_SMTP: sendSmtpEmail.definition,
   CREATE_ROW: createRow.definition,
   UPDATE_ROW: updateRow.definition,
@@ -66,7 +67,22 @@ if (env.SELF_HOSTED) {
   // @ts-ignore
   ACTION_IMPLS["EXECUTE_BASH"] = bash.run
   // @ts-ignore
-  ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
+  BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
+}
+
+export async function getActionDefinitions() {
+  const actionDefinitions = BUILTIN_ACTION_DEFINITIONS
+  if (env.SELF_HOSTED) {
+    const plugins = await getPlugins(PluginType.AUTOMATION)
+    for (let plugin of plugins) {
+      const schema = plugin.schema.schema as AutomationStep
+      actionDefinitions[schema.stepId] = {
+        ...schema,
+        custom: true,
+      }
+    }
+  }
+  return actionDefinitions
 }
 
 /* istanbul ignore next */

packages/server/src/automations/index.ts

@@ -6,7 +6,7 @@ import BullQueue from "bull"
 export { automationQueue } from "./bullboard"
 export { shutdown } from "./bullboard"
 export { TRIGGER_DEFINITIONS } from "./triggers"
-export { ACTION_DEFINITIONS } from "./actions"
+export { BUILTIN_ACTION_DEFINITIONS, getActionDefinitions } from "./actions"
 
 /**
  * This module is built purely to kick off the worker farm and manage the inputs/outputs

packages/server/src/automations/steps/bash.ts

@@ -42,8 +42,8 @@ export const definition: AutomationStepSchema = {
           description: "Whether the command was successful",
         },
       },
+      required: ["stdout"],
     },
-    required: ["stdout"],
   },
 }
 

packages/server/src/automations/steps/executeQuery.ts

@@ -53,8 +53,8 @@ export const definition: AutomationStepSchema = {
           description: "Whether the action was successful",
         },
       },
+      required: ["response", "success"],
     },
-    required: ["response", "success"],
   },
 }
 

packages/server/src/automations/tests/utilities/index.ts

@@ -1,6 +1,6 @@
 import TestConfig from "../../../tests/utilities/TestConfiguration"
 import { context } from "@budibase/backend-core"
-import { ACTION_DEFINITIONS, getAction } from "../../actions"
+import { BUILTIN_ACTION_DEFINITIONS, getAction } from "../../actions"
 import emitter from "../../../events/index"
 import env from "../../../environment"
 
@@ -57,4 +57,4 @@ export async function runStep(stepId: string, inputs: any, stepContext?: any) {
 }
 
 export const apiKey = "test"
-export const actions = ACTION_DEFINITIONS
+export const actions = BUILTIN_ACTION_DEFINITIONS

packages/server/src/tests/utilities/structures.ts

@@ -1,18 +1,18 @@
-import { permissions, roles } from "@budibase/backend-core"
+import { permissions, roles, utils } from "@budibase/backend-core"
 import { createHomeScreen } from "../../constants/screens"
 import { EMPTY_LAYOUT } from "../../constants/layouts"
 import { cloneDeep } from "lodash/fp"
-import { ACTION_DEFINITIONS, TRIGGER_DEFINITIONS } from "../../automations"
+import { BUILTIN_ACTION_DEFINITIONS, TRIGGER_DEFINITIONS } from "../../automations"
 import {
   Automation,
-  AutomationActionStepId,
+  AutomationActionStepId, AutomationStep,
+  AutomationStepType,
+  AutomationTrigger,
   AutomationTriggerStepId,
   Datasource,
   SourceName,
 } from "@budibase/types"
 
-const { v4: uuidv4 } = require("uuid")
-
 export function basicTable() {
   return {
     name: "TestTable",
@@ -71,19 +71,19 @@ export function view(tableId: string) {
 }
 
 export function automationStep(
-  actionDefinition = ACTION_DEFINITIONS.CREATE_ROW
-) {
+  actionDefinition = BUILTIN_ACTION_DEFINITIONS.CREATE_ROW
+): AutomationStep {
   return {
-    id: uuidv4(),
+    id: utils.newid(),
     ...actionDefinition,
   }
 }
 
 export function automationTrigger(
   triggerDefinition = TRIGGER_DEFINITIONS.ROW_SAVED
-) {
+): AutomationTrigger {
   return {
-    id: uuidv4(),
+    id: utils.newid(),
     ...triggerDefinition,
   }
 }
@@ -106,7 +106,7 @@ export function newAutomation({ steps, trigger }: any = {}) {
   return automation
 }
 
-export function basicAutomation(appId?: string) {
+export function basicAutomation(appId?: string): Automation {
   return {
     name: "My Automation",
     screenId: "kasdkfldsafkl",
@@ -119,18 +119,22 @@ export function basicAutomation(appId?: string) {
         tagline: "test",
         icon: "test",
         description: "test",
-        type: "trigger",
+        type: AutomationStepType.TRIGGER,
         id: "test",
         inputs: {},
         schema: {
-          inputs: {},
-          outputs: {},
+          inputs: {
+            properties: {},
+          },
+          outputs: {
+            properties: {},
+          },
         },
       },
       steps: [],
     },
     type: "automation",
-    appId,
+    appId: appId!,
   }
 }
 
@@ -154,7 +158,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
           inputs: {
             tableId,
           },
-          schema: ACTION_DEFINITIONS.QUERY_ROWS.schema,
+          schema: BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS.schema,
         },
         {
           id: "c",
@@ -163,7 +167,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
           internal: true,
           inputs: loopOpts,
           blockToLoop: "d",
-          schema: ACTION_DEFINITIONS.LOOP.schema,
+          schema: BUILTIN_ACTION_DEFINITIONS.LOOP.schema,
         },
         {
           id: "d",
@@ -173,7 +177,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
           inputs: {
             text: "log statement",
           },
-          schema: ACTION_DEFINITIONS.SERVER_LOG.schema,
+          schema: BUILTIN_ACTION_DEFINITIONS.SERVER_LOG.schema,
         },
       ],
       trigger: {

packages/server/src/threads/automation.ts

@@ -27,8 +27,8 @@ import { processObject } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import * as sdkUtils from "../sdk/utils"
 import env from "../environment"
-const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
-const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
+const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId
+const LOOP_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.LOOP.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 

packages/types/src/documents/app/automation.ts

@@ -73,9 +73,13 @@ export interface Automation extends Document {
     steps: AutomationStep[]
     trigger: AutomationTrigger
   }
+  screenId?: string
+  uiTree?: any
   appId: string
   live?: boolean
   name: string
+  internal?: boolean
+  type?: string
 }
 
 interface BaseIOStructure {
@@ -115,6 +119,7 @@ export interface AutomationStepSchema {
     inputs: InputOutputBlock
     outputs: InputOutputBlock
   }
+  custom?: boolean
 }
 
 export interface AutomationStep extends AutomationStepSchema {