Refactoring a lot of content around webhooks to Typescript, as well as fixing webhooks and automation app IDs on import of new app.

packages/backend-core/src/context/index.ts

@@ -6,6 +6,7 @@ import { baseGlobalDBName } from "../db/tenancy"
 import { IdentityContext } from "@budibase/types"
 import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants"
 import { ContextKey } from "./constants"
+import PouchDB from "pouchdb"
 import {
   updateUsing,
   closeWithUsing,
@@ -22,16 +23,15 @@ export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID
 let TEST_APP_ID: string | null = null
 
 export const closeTenancy = async () => {
-  let db
   try {
     if (env.USE_COUCH) {
-      db = getGlobalDB()
+      const db = getGlobalDB()
+      await closeDB(db)
     }
   } catch (err) {
     // no DB found - skip closing
     return
   }
-  await closeDB(db)
   // clear from context now that database is closed/task is finished
   cls.setOnContext(ContextKey.TENANT_ID, null)
   cls.setOnContext(ContextKey.GLOBAL_DB, null)

packages/backend-core/src/index.ts

@@ -4,6 +4,7 @@ import * as events from "./events"
 import * as migrations from "./migrations"
 import * as users from "./users"
 import * as roles from "./security/roles"
+import * as permissions from "./security/permissions"
 import * as accounts from "./cloud/accounts"
 import * as installation from "./installation"
 import env from "./environment"
@@ -65,6 +66,7 @@ const core = {
   middleware,
   encryption,
   queue,
+  permissions,
 }
 
 export = core

packages/server/src/api/controllers/webhook.ts

@@ -1,66 +1,51 @@
-const { generateWebhookID, getWebhookParams } = require("../../db/utils")
+import { getWebhookParams } from "../../db/utils"
+import triggers from "../../automations/triggers"
+import { db as dbCore, context } from "@budibase/backend-core"
+import {
+  Webhook,
+  WebhookActionType,
+  BBContext,
+  Automation,
+} from "@budibase/types"
+import sdk from "../../sdk"
 const toJsonSchema = require("to-json-schema")
 const validate = require("jsonschema").validate
-const { WebhookType } = require("../../constants")
-const triggers = require("../../automations/triggers")
-const { getProdAppID } = require("@budibase/backend-core/db")
-const { getAppDB, updateAppId } = require("@budibase/backend-core/context")
 
 const AUTOMATION_DESCRIPTION = "Generated from Webhook Schema"
 
-function Webhook(name, type, target) {
-  this.live = true
-  this.name = name
-  this.action = {
-    type,
-    target,
-  }
-}
-
-exports.Webhook = Webhook
-
-exports.fetch = async ctx => {
-  const db = getAppDB()
+export async function fetch(ctx: BBContext) {
+  const db = context.getAppDB()
   const response = await db.allDocs(
     getWebhookParams(null, {
       include_docs: true,
     })
   )
-  ctx.body = response.rows.map(row => row.doc)
+  ctx.body = response.rows.map((row: any) => row.doc)
 }
 
-exports.save = async ctx => {
-  const db = getAppDB()
-  const webhook = ctx.request.body
-  webhook.appId = ctx.appId
-
-  // check that the webhook exists
-  if (webhook._id) {
-    await db.get(webhook._id)
-  } else {
-    webhook._id = generateWebhookID()
-  }
-  const response = await db.put(webhook)
-  webhook._rev = response.rev
+export async function save(ctx: BBContext) {
+  const webhook = await sdk.automations.webhook.save(ctx.request.body)
   ctx.body = {
     message: "Webhook created successfully",
     webhook,
   }
 }
 
-exports.destroy = async ctx => {
-  const db = getAppDB()
-  ctx.body = await db.remove(ctx.params.id, ctx.params.rev)
+export async function destroy(ctx: BBContext) {
+  ctx.body = await sdk.automations.webhook.destroy(
+    ctx.params.id,
+    ctx.params.rev
+  )
 }
 
-exports.buildSchema = async ctx => {
-  await updateAppId(ctx.params.instance)
-  const db = getAppDB()
-  const webhook = await db.get(ctx.params.id)
+export async function buildSchema(ctx: BBContext) {
+  await context.updateAppId(ctx.params.instance)
+  const db = context.getAppDB()
+  const webhook = (await db.get(ctx.params.id)) as Webhook
   webhook.bodySchema = toJsonSchema(ctx.request.body)
   // update the automation outputs
-  if (webhook.action.type === WebhookType.AUTOMATION) {
-    let automation = await db.get(webhook.action.target)
+  if (webhook.action.type === WebhookActionType.AUTOMATION) {
+    let automation = (await db.get(webhook.action.target)) as Automation
     const autoOutputs = automation.definition.trigger.schema.outputs
     let properties = webhook.bodySchema.properties
     // reset webhook outputs
@@ -78,18 +63,18 @@ exports.buildSchema = async ctx => {
   ctx.body = await db.put(webhook)
 }
 
-exports.trigger = async ctx => {
-  const prodAppId = getProdAppID(ctx.params.instance)
-  await updateAppId(prodAppId)
+export async function trigger(ctx: BBContext) {
+  const prodAppId = dbCore.getProdAppID(ctx.params.instance)
+  await context.updateAppId(prodAppId)
   try {
-    const db = getAppDB()
-    const webhook = await db.get(ctx.params.id)
+    const db = context.getAppDB()
+    const webhook = (await db.get(ctx.params.id)) as Webhook
     // validate against the schema
     if (webhook.bodySchema) {
       validate(ctx.request.body, webhook.bodySchema)
     }
     const target = await db.get(webhook.action.target)
-    if (webhook.action.type === WebhookType.AUTOMATION) {
+    if (webhook.action.type === WebhookActionType.AUTOMATION) {
       // trigger with both the pure request and then expand it
       // incase the user has produced a schema to bind to
       await triggers.externalTrigger(target, {
@@ -102,7 +87,7 @@ exports.trigger = async ctx => {
     ctx.body = {
       message: "Webhook trigger fired successfully",
     }
-  } catch (err) {
+  } catch (err: any) {
     if (err.status === 404) {
       ctx.status = 200
       ctx.body = {

packages/server/src/api/routes/utils/validators.js

@@ -1,10 +1,10 @@
 const { joiValidator } = require("@budibase/backend-core/auth")
 const { DataSourceOperation } = require("../../../constants")
-const { WebhookType } = require("../../../constants")
 const {
   BUILTIN_PERMISSION_IDS,
   PermissionLevels,
 } = require("@budibase/backend-core/permissions")
+const { WebhookActionType } = require("@budibase/types")
 const Joi = require("joi")
 
 const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("")
@@ -126,7 +126,7 @@ exports.webhookValidator = () => {
     name: Joi.string().required(),
     bodySchema: Joi.object().optional(),
     action: Joi.object({
-      type: Joi.string().required().valid(WebhookType.AUTOMATION),
+      type: Joi.string().required().valid(WebhookActionType.AUTOMATION),
       target: Joi.string().required(),
     }).required(),
   }).unknown(true))

packages/server/src/api/routes/webhook.ts

@@ -1,9 +1,10 @@
-const Router = require("@koa/router")
-const controller = require("../controllers/webhook")
-const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/backend-core/permissions")
-const { webhookValidator } = require("./utils/validators")
+import Router from "@koa/router"
+import * as controller from "../controllers/webhook"
+import authorized from "../../middleware/authorized"
+import { permissions } from "@budibase/backend-core"
+import { webhookValidator } from "./utils/validators"
 
+const BUILDER = permissions.BUILDER
 const router = new Router()
 
 router
@@ -23,4 +24,4 @@ router
   // this shouldn't have authorisation, right now its always public
   .post("/api/webhooks/trigger/:instance/:id", controller.trigger)
 
-module.exports = router
+export default router

packages/server/src/automations/utils.ts

@@ -1,10 +1,9 @@
 import { Thread, ThreadType } from "../threads"
 import { definitions } from "./triggerInfo"
-import * as webhooks from "../api/controllers/webhook"
 import { automationQueue } from "./bullboard"
 import newid from "../db/newid"
 import { updateEntityMetadata } from "../utilities"
-import { MetadataTypes, WebhookType } from "../constants"
+import { MetadataTypes } from "../constants"
 import { getProdAppID, doWithDB } from "@budibase/backend-core/db"
 import { getAutomationMetadataParams } from "../db/utils"
 import { cloneDeep } from "lodash/fp"
@@ -15,7 +14,8 @@ import {
 } from "@budibase/backend-core/context"
 import { context } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
-import { Automation } from "@budibase/types"
+import { Automation, WebhookActionType } from "@budibase/types"
+import sdk from "../sdk"
 
 const REBOOT_CRON = "@reboot"
 const WH_STEP_ID = definitions.WEBHOOK.stepId
@@ -197,16 +197,12 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) {
       let db = getAppDB()
       // need to get the webhook to get the rev
       const webhook = await db.get(oldTrigger.webhookId)
-      const ctx = {
-        appId,
-        params: { id: webhook._id, rev: webhook._rev },
-      }
       // might be updating - reset the inputs to remove the URLs
       if (newTrigger) {
         delete newTrigger.webhookId
         newTrigger.inputs = {}
       }
-      await webhooks.destroy(ctx)
+      await sdk.automations.webhook.destroy(webhook._id, webhook._rev)
     } catch (err) {
       // don't worry about not being able to delete, if it doesn't exist all good
     }
@@ -216,18 +212,14 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) {
     (!isWebhookTrigger(oldAuto) || triggerChanged) &&
     isWebhookTrigger(newAuto)
   ) {
-    const ctx: any = {
-      appId,
-      request: {
-        body: new webhooks.Webhook(
+    const webhook = await sdk.automations.webhook.save(
+      sdk.automations.webhook.newDoc(
         "Automation webhook",
-          WebhookType.AUTOMATION,
+        WebhookActionType.AUTOMATION,
         newAuto._id
-        ),
-      },
-    }
-    await webhooks.save(ctx)
-    const id = ctx.body.webhook._id
+      )
+    )
+    const id = webhook._id
     newTrigger.webhookId = id
     // the app ID has to be development for this endpoint
     // it can only be used when building the app

packages/server/src/constants/index.js

@@ -196,10 +196,6 @@ exports.BuildSchemaErrors = {
   INVALID_COLUMN: "invalid_column",
 }
 
-exports.WebhookType = {
-  AUTOMATION: "automation",
-}
-
 exports.AutomationErrors = {
   INCORRECT_TYPE: "INCORRECT_TYPE",
   MAX_ITERATIONS: "MAX_ITERATIONS_REACHED",

packages/server/src/definitions/automations.ts

@@ -1,9 +1,4 @@
-import {
-  Automation,
-  AutomationResults,
-  AutomationStep,
-  Document,
-} from "@budibase/types"
+import { AutomationResults, AutomationStep, Document } from "@budibase/types"
 
 export enum LoopStepType {
   ARRAY = "Array",

packages/server/src/sdk/app/automations/index.ts

@@ -0,0 +1,5 @@
+import * as webhook from "./webhook"
+
+export default {
+  webhook,
+}

packages/server/src/sdk/app/automations/webhook.ts

@@ -0,0 +1,43 @@
+import { Webhook, WebhookActionType } from "@budibase/types"
+import { db as dbCore, context } from "@budibase/backend-core"
+import { generateWebhookID } from "../../../db/utils"
+
+function isWebhookID(id: string) {
+  return id.startsWith(dbCore.DocumentType.WEBHOOK)
+}
+
+export function newDoc(
+  name: string,
+  type: WebhookActionType,
+  target: string
+): Webhook {
+  return {
+    live: true,
+    name,
+    action: {
+      type,
+      target,
+    },
+  }
+}
+
+export async function save(webhook: Webhook) {
+  const db = context.getAppDB()
+  // check that the webhook exists
+  if (webhook._id && isWebhookID(webhook._id)) {
+    await db.get(webhook._id)
+  } else {
+    webhook._id = generateWebhookID()
+  }
+  const response = await db.put(webhook)
+  webhook._rev = response.rev
+  return webhook
+}
+
+export async function destroy(id: string, rev: string) {
+  const db = context.getAppDB()
+  if (!id || !isWebhookID(id)) {
+    throw new Error("Provided webhook ID is not valid.")
+  }
+  return await db.remove(id, rev)
+}

packages/server/src/sdk/app/backups/imports.ts

@@ -1,20 +1,25 @@
 import { db as dbCore } from "@budibase/backend-core"
-import { TABLE_ROW_PREFIX } from "../../../db/utils"
+import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
 import {
-  uploadDirectory,
   upload,
+  uploadDirectory,
 } from "../../../utilities/fileSystem/utilities"
 import { downloadTemplate } from "../../../utilities/fileSystem"
-import { ObjectStoreBuckets, FieldTypes } from "../../../constants"
+import { FieldTypes, ObjectStoreBuckets } from "../../../constants"
 import { join } from "path"
 import fs from "fs"
 import sdk from "../../"
-import { CouchFindOptions, RowAttachment } from "@budibase/types"
+import {
+  Automation,
+  AutomationTriggerStepId,
+  CouchFindOptions,
+  RowAttachment,
+} from "@budibase/types"
+import PouchDB from "pouchdb"
 const uuid = require("uuid/v4")
 const tar = require("tar")
-import PouchDB from "pouchdb"
 
 type TemplateType = {
   file?: {
@@ -81,6 +86,34 @@ async function updateAttachmentColumns(
   }
 }
 
+async function updateAutomations(prodAppId: string, db: PouchDB.Database) {
+  const automations = (
+    await db.allDocs(
+      getAutomationParams(null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc) as Automation[]
+  const devAppId = dbCore.getDevAppID(prodAppId)
+  let toSave: Automation[] = []
+  for (let automation of automations) {
+    const oldDevAppId = automation.appId,
+      oldProdAppId = dbCore.getProdAppID(automation.appId)
+    if (
+      automation.definition.trigger.stepId === AutomationTriggerStepId.WEBHOOK
+    ) {
+      const old = automation.definition.trigger.inputs
+      automation.definition.trigger.inputs = {
+        schemaUrl: old.schemaUrl.replace(oldDevAppId, devAppId),
+        triggerUrl: old.triggerUrl.replace(oldProdAppId, prodAppId),
+      }
+    }
+    automation.appId = devAppId
+    toSave.push(automation)
+  }
+  await db.bulkDocs(toSave)
+}
+
 /**
  * This function manages temporary template files which are stored by Koa.
  * @param {Object} template The template object retrieved from the Koa context object.
@@ -165,5 +198,6 @@ export async function importApp(
     throw "Error loading database dump from template."
   }
   await updateAttachmentColumns(prodAppId, db)
+  await updateAutomations(prodAppId, db)
   return ok
 }

packages/server/src/sdk/index.ts

@@ -1,9 +1,11 @@
 import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
+import { default as automations } from "./app/automations"
 
 const sdk = {
   backups,
   tables,
+  automations,
 }
 
 // default export for TS

packages/types/src/documents/app/automation.ts

@@ -1,5 +1,34 @@
 import { Document } from "../document"
 
+export enum AutomationTriggerStepId {
+  ROW_SAVED = "ROW_SAVED",
+  ROW_UPDATED = "ROW_UPDATED",
+  ROW_DELETED = "ROW_DELETED",
+  WEBHOOK = "WEBHOOK",
+  APP = "APP",
+  CRON = "CRON",
+}
+
+export enum AutomationActionStepId {
+  SEND_EMAIL_SMTP = "SEND_EMAIL_SMTP",
+  CREATE_ROW = "CREATE_ROW",
+  UPDATE_ROW = "UPDATE_ROW",
+  DELETE_ROW = "DELETE_ROW",
+  OUTGOING_WEBHOOK = "OUTGOING_WEBHOOK",
+  EXECUTE_SCRIPT = "EXECUTE_SCRIPT",
+  EXECUTE_QUERY = "EXECUTE_QUERY",
+  SERVER_LOG = "SERVER_LOG",
+  DELAY = "DELAY",
+  FILTER = "FILTER",
+  QUERY_ROWS = "QUERY_ROWS",
+  LOOP = "LOOP",
+  // these used to be lowercase step IDs, maintain for backwards compat
+  discord = "discord",
+  slack = "slack",
+  zapier = "zapier",
+  integromat = "integromat",
+}
+
 export interface Automation extends Document {
   definition: {
     steps: AutomationStep[]
@@ -11,7 +40,7 @@ export interface Automation extends Document {
 
 export interface AutomationStep {
   id: string
-  stepId: string
+  stepId: AutomationTriggerStepId | AutomationActionStepId
   inputs: {
     [key: string]: any
   }
@@ -19,15 +48,13 @@ export interface AutomationStep {
     inputs: {
       [key: string]: any
     }
+    outputs: {
+      [key: string]: any
+    }
   }
 }
 
-export interface AutomationTrigger {
-  id: string
-  stepId: string
-  inputs: {
-    [key: string]: any
-  }
+export interface AutomationTrigger extends AutomationStep {
   cronJobId?: string
 }
 
@@ -43,7 +70,7 @@ export interface AutomationResults {
   status?: AutomationStatus
   trigger?: any
   steps: {
-    stepId: string
+    stepId: AutomationTriggerStepId | AutomationActionStepId
     inputs: {
       [key: string]: any
     }

packages/types/src/documents/app/index.ts

@@ -11,3 +11,4 @@ export * from "../document"
 export * from "./row"
 export * from "./user"
 export * from "./backup"
+export * from "./webhook"

packages/types/src/documents/app/webhook.ts

@@ -0,0 +1,15 @@
+import { Document } from "../document"
+
+export enum WebhookActionType {
+  AUTOMATION = "automation",
+}
+
+export interface Webhook extends Document {
+  live: boolean
+  name: string
+  action: {
+    type: WebhookActionType
+    target: string
+  }
+  bodySchema?: any
+}