Merge pull request #8416 from Budibase/fix/mike-oct-fixes

Various fixes for release

hosting/nginx.prod.conf.hbs

@@ -171,11 +171,13 @@ http {
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $http_host;
 
       proxy_connect_timeout 300;
       proxy_http_version 1.1;
       proxy_set_header Connection "";
       chunked_transfer_encoding off;
+
       proxy_pass      http://$minio:9000;
     }
 

packages/backend-core/src/context/index.ts

@@ -6,6 +6,7 @@ import { baseGlobalDBName } from "../db/tenancy"
 import { IdentityContext } from "@budibase/types"
 import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants"
 import { ContextKey } from "./constants"
+import PouchDB from "pouchdb"
 import {
   updateUsing,
   closeWithUsing,
@@ -22,16 +23,15 @@ export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID
 let TEST_APP_ID: string | null = null
 
 export const closeTenancy = async () => {
-  let db
   try {
     if (env.USE_COUCH) {
-      db = getGlobalDB()
+      const db = getGlobalDB()
+      await closeDB(db)
     }
   } catch (err) {
     // no DB found - skip closing
     return
   }
-  await closeDB(db)
   // clear from context now that database is closed/task is finished
   cls.setOnContext(ContextKey.TENANT_ID, null)
   cls.setOnContext(ContextKey.GLOBAL_DB, null)

packages/backend-core/src/index.ts

@@ -4,6 +4,7 @@ import * as events from "./events"
 import * as migrations from "./migrations"
 import * as users from "./users"
 import * as roles from "./security/roles"
+import * as permissions from "./security/permissions"
 import * as accounts from "./cloud/accounts"
 import * as installation from "./installation"
 import env from "./environment"
@@ -65,6 +66,7 @@ const core = {
   middleware,
   encryption,
   queue,
+  permissions,
 }
 
 export = core

packages/cli/src/backups/index.js

@@ -4,7 +4,7 @@ const fs = require("fs")
 const { join } = require("path")
 const { getAllDbs } = require("../core/db")
 const tar = require("tar")
-const { progressBar } = require("../utils")
+const { progressBar, httpCall } = require("../utils")
 const {
   TEMP_DIR,
   COUCH_DIR,
@@ -86,6 +86,15 @@ async function importBackup(opts) {
   bar.stop()
   console.log("MinIO Import")
   await importObjects()
+  // finish by letting the system know that a restore has occurred
+  try {
+    await httpCall(
+      `http://localhost:${config.MAIN_PORT}/api/system/restored`,
+      "POST"
+    )
+  } catch (err) {
+    // ignore error - it will be an older system
+  }
   console.log("Import complete")
   fs.rmSync(TEMP_DIR, { recursive: true })
 }

packages/cli/src/backups/objectStore.js

@@ -16,16 +16,21 @@ exports.exportObjects = async () => {
   const path = join(TEMP_DIR, MINIO_DIR)
   fs.mkdirSync(path)
   let fullList = []
+  let errorCount = 0
   for (let bucket of bucketList) {
     const client = ObjectStore(bucket)
     try {
       await client.headBucket().promise()
     } catch (err) {
+      errorCount++
       continue
     }
     const list = await client.listObjectsV2().promise()
     fullList = fullList.concat(list.Contents.map(el => ({ ...el, bucket })))
   }
+  if (errorCount === bucketList.length) {
+    throw new Error("Unable to access MinIO/S3 - check environment config.")
+  }
   const bar = progressBar(fullList.length)
   let count = 0
   for (let object of fullList) {

packages/cli/src/backups/utils.js

@@ -2,17 +2,19 @@ const dotenv = require("dotenv")
 const fs = require("fs")
 const { string } = require("../questions")
 const { getPouch } = require("../core/db")
+const { env: environment } = require("@budibase/backend-core")
 
-exports.DEFAULT_COUCH = "http://budibase:budibase@localhost:10000/db/"
-exports.DEFAULT_MINIO = "http://localhost:10000/"
 exports.TEMP_DIR = ".temp"
 exports.COUCH_DIR = "couchdb"
 exports.MINIO_DIR = "minio"
 
 const REQUIRED = [
   { value: "MAIN_PORT", default: "10000" },
-  { value: "COUCH_DB_URL", default: exports.DEFAULT_COUCH },
-  { value: "MINIO_URL", default: exports.DEFAULT_MINIO },
+  {
+    value: "COUCH_DB_URL",
+    default: "http://budibase:budibase@localhost:10000/db/",
+  },
+  { value: "MINIO_URL", default: "http://localhost:10000" },
   { value: "MINIO_ACCESS_KEY" },
   { value: "MINIO_SECRET_KEY" },
 ]
@@ -27,7 +29,7 @@ exports.checkURLs = config => {
     ] = `http://${username}:${password}@localhost:${mainPort}/db/`
   }
   if (!config["MINIO_URL"]) {
-    config["MINIO_URL"] = exports.DEFAULT_MINIO
+    config["MINIO_URL"] = `http://localhost:${mainPort}/`
   }
   return config
 }
@@ -65,6 +67,10 @@ exports.getConfig = async (envFile = true) => {
   } else {
     config = await exports.askQuestions()
   }
+  // fill out environment
+  for (let key of Object.keys(config)) {
+    environment._set(key, config[key])
+  }
   return config
 }
 

packages/cli/src/index.js

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 require("./prebuilds")
 require("./environment")
+const json = require("../package.json")
 const { getCommands } = require("./options")
 const { Command } = require("commander")
 const { getHelpDescription } = require("./utils")
@@ -10,7 +11,7 @@ async function init() {
   const program = new Command()
     .addHelpCommand("help", getHelpDescription("Help with Budibase commands."))
     .helpOption(false)
-  program.helpOption()
+    .version(json.version)
   // add commands
   for (let command of getCommands()) {
     command.configure(program)

packages/cli/src/utils.js

@@ -23,6 +23,14 @@ exports.downloadFile = async (url, filePath) => {
   })
 }
 
+exports.httpCall = async (url, method) => {
+  const response = await axios({
+    url,
+    method,
+  })
+  return response.data
+}
+
 exports.getHelpDescription = string => {
   return chalk.cyan(string)
 }

packages/server/__mocks__/node-fetch.ts

@@ -30,11 +30,21 @@ module FetchMock {
     }
 
     if (url.includes("/api/global")) {
-      return json({
+      const user = {
         email: "test@test.com",
         _id: "us_test@test.com",
         status: "active",
-      })
+        roles: {},
+        builder: {
+          global: false,
+        },
+        admin: {
+          global: false,
+        },
+      }
+      return url.endsWith("/users") && opts.method === "GET"
+        ? json([user])
+        : json(user)
     }
     // mocked data based on url
     else if (url.includes("api/apps")) {

packages/server/src/api/controllers/webhook.ts

@@ -1,66 +1,51 @@
-const { generateWebhookID, getWebhookParams } = require("../../db/utils")
+import { getWebhookParams } from "../../db/utils"
+import triggers from "../../automations/triggers"
+import { db as dbCore, context } from "@budibase/backend-core"
+import {
+  Webhook,
+  WebhookActionType,
+  BBContext,
+  Automation,
+} from "@budibase/types"
+import sdk from "../../sdk"
 const toJsonSchema = require("to-json-schema")
 const validate = require("jsonschema").validate
-const { WebhookType } = require("../../constants")
-const triggers = require("../../automations/triggers")
-const { getProdAppID } = require("@budibase/backend-core/db")
-const { getAppDB, updateAppId } = require("@budibase/backend-core/context")
 
 const AUTOMATION_DESCRIPTION = "Generated from Webhook Schema"
 
-function Webhook(name, type, target) {
-  this.live = true
-  this.name = name
-  this.action = {
-    type,
-    target,
-  }
-}
-
-exports.Webhook = Webhook
-
-exports.fetch = async ctx => {
-  const db = getAppDB()
+export async function fetch(ctx: BBContext) {
+  const db = context.getAppDB()
   const response = await db.allDocs(
     getWebhookParams(null, {
       include_docs: true,
     })
   )
-  ctx.body = response.rows.map(row => row.doc)
+  ctx.body = response.rows.map((row: any) => row.doc)
 }
 
-exports.save = async ctx => {
-  const db = getAppDB()
-  const webhook = ctx.request.body
-  webhook.appId = ctx.appId
-
-  // check that the webhook exists
-  if (webhook._id) {
-    await db.get(webhook._id)
-  } else {
-    webhook._id = generateWebhookID()
-  }
-  const response = await db.put(webhook)
-  webhook._rev = response.rev
+export async function save(ctx: BBContext) {
+  const webhook = await sdk.automations.webhook.save(ctx.request.body)
   ctx.body = {
     message: "Webhook created successfully",
     webhook,
   }
 }
 
-exports.destroy = async ctx => {
-  const db = getAppDB()
-  ctx.body = await db.remove(ctx.params.id, ctx.params.rev)
+export async function destroy(ctx: BBContext) {
+  ctx.body = await sdk.automations.webhook.destroy(
+    ctx.params.id,
+    ctx.params.rev
+  )
 }
 
-exports.buildSchema = async ctx => {
-  await updateAppId(ctx.params.instance)
-  const db = getAppDB()
-  const webhook = await db.get(ctx.params.id)
+export async function buildSchema(ctx: BBContext) {
+  await context.updateAppId(ctx.params.instance)
+  const db = context.getAppDB()
+  const webhook = (await db.get(ctx.params.id)) as Webhook
   webhook.bodySchema = toJsonSchema(ctx.request.body)
   // update the automation outputs
-  if (webhook.action.type === WebhookType.AUTOMATION) {
-    let automation = await db.get(webhook.action.target)
+  if (webhook.action.type === WebhookActionType.AUTOMATION) {
+    let automation = (await db.get(webhook.action.target)) as Automation
     const autoOutputs = automation.definition.trigger.schema.outputs
     let properties = webhook.bodySchema.properties
     // reset webhook outputs
@@ -78,18 +63,18 @@ exports.buildSchema = async ctx => {
   ctx.body = await db.put(webhook)
 }
 
-exports.trigger = async ctx => {
-  const prodAppId = getProdAppID(ctx.params.instance)
-  await updateAppId(prodAppId)
+export async function trigger(ctx: BBContext) {
+  const prodAppId = dbCore.getProdAppID(ctx.params.instance)
+  await context.updateAppId(prodAppId)
   try {
-    const db = getAppDB()
-    const webhook = await db.get(ctx.params.id)
+    const db = context.getAppDB()
+    const webhook = (await db.get(ctx.params.id)) as Webhook
     // validate against the schema
     if (webhook.bodySchema) {
       validate(ctx.request.body, webhook.bodySchema)
     }
     const target = await db.get(webhook.action.target)
-    if (webhook.action.type === WebhookType.AUTOMATION) {
+    if (webhook.action.type === WebhookActionType.AUTOMATION) {
       // trigger with both the pure request and then expand it
       // incase the user has produced a schema to bind to
       await triggers.externalTrigger(target, {
@@ -102,7 +87,7 @@ exports.trigger = async ctx => {
     ctx.body = {
       message: "Webhook trigger fired successfully",
     }
-  } catch (err) {
+  } catch (err: any) {
     if (err.status === 404) {
       ctx.status = 200
       ctx.body = {

packages/server/src/api/routes/utils/validators.js

@@ -1,10 +1,10 @@
 const { joiValidator } = require("@budibase/backend-core/auth")
 const { DataSourceOperation } = require("../../../constants")
-const { WebhookType } = require("../../../constants")
 const {
   BUILTIN_PERMISSION_IDS,
   PermissionLevels,
 } = require("@budibase/backend-core/permissions")
+const { WebhookActionType } = require("@budibase/types")
 const Joi = require("joi")
 
 const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("")
@@ -126,7 +126,7 @@ exports.webhookValidator = () => {
     name: Joi.string().required(),
     bodySchema: Joi.object().optional(),
     action: Joi.object({
-      type: Joi.string().required().valid(WebhookType.AUTOMATION),
+      type: Joi.string().required().valid(WebhookActionType.AUTOMATION),
       target: Joi.string().required(),
     }).required(),
   }).unknown(true))

packages/server/src/api/routes/webhook.ts

@@ -1,9 +1,10 @@
-const Router = require("@koa/router")
-const controller = require("../controllers/webhook")
-const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/backend-core/permissions")
-const { webhookValidator } = require("./utils/validators")
+import Router from "@koa/router"
+import * as controller from "../controllers/webhook"
+import authorized from "../../middleware/authorized"
+import { permissions } from "@budibase/backend-core"
+import { webhookValidator } from "./utils/validators"
 
+const BUILDER = permissions.BUILDER
 const router = new Router()
 
 router
@@ -23,4 +24,4 @@ router
   // this shouldn't have authorisation, right now its always public
   .post("/api/webhooks/trigger/:instance/:id", controller.trigger)
 
-module.exports = router
+export default router

packages/server/src/app.ts

@@ -15,30 +15,16 @@ db.init()
 const Koa = require("koa")
 const destroyable = require("server-destroy")
 const koaBody = require("koa-body")
-const pino = require("koa-pino-logger")
 const http = require("http")
 const api = require("./api")
-const eventEmitter = require("./events")
 const automations = require("./automations/index")
 const Sentry = require("@sentry/node")
-const fileSystem = require("./utilities/fileSystem")
-const bullboard = require("./automations/bullboard")
 const { logAlert } = require("@budibase/backend-core/logging")
-const { pinoSettings } = require("@budibase/backend-core")
 const { Thread } = require("./threads")
-const fs = require("fs")
 import redis from "./utilities/redis"
-import * as migrations from "./migrations"
-import { events, installation, tenancy } from "@budibase/backend-core"
-import {
-  createAdminUser,
-  generateApiKey,
-  getChecklist,
-} from "./utilities/workerRequests"
-import { watch } from "./watch"
+import { events } from "@budibase/backend-core"
 import { initialise as initialiseWebsockets } from "./websocket"
-import sdk from "./sdk"
-import * as pro from "@budibase/pro"
+import { startup } from "./startup"
 
 const app = new Koa()
 
@@ -54,19 +40,6 @@ app.use(
   })
 )
 
-app.use(pino(pinoSettings()))
-
-if (!env.isTest()) {
-  const plugin = bullboard.init()
-  app.use(plugin)
-}
-
-app.context.eventEmitter = eventEmitter
-app.context.auth = {}
-
-// api routes
-app.use(api.router.routes())
-
 if (env.isProd()) {
   env._set("NODE_ENV", "production")
   Sentry.init()
@@ -104,86 +77,8 @@ server.on("close", async () => {
   }
 })
 
-const initPro = async () => {
-  await pro.init({
-    backups: {
-      processing: {
-        exportAppFn: sdk.backups.exportApp,
-        importAppFn: sdk.backups.importApp,
-        statsFn: sdk.backups.calculateBackupStats,
-      },
-    },
-  })
-}
-
 module.exports = server.listen(env.PORT || 0, async () => {
-  console.log(`Budibase running on ${JSON.stringify(server.address())}`)
-  env._set("PORT", server.address().port)
-  eventEmitter.emitPort(env.PORT)
-  fileSystem.init()
-  await redis.init()
-
-  // run migrations on startup if not done via http
-  // not recommended in a clustered environment
-  if (!env.HTTP_MIGRATIONS && !env.isTest()) {
-    try {
-      await migrations.migrate()
-    } catch (e) {
-      logAlert("Error performing migrations. Exiting.", e)
-      shutdown()
-    }
-  }
-
-  // check and create admin user if required
-  if (
-    env.SELF_HOSTED &&
-    !env.MULTI_TENANCY &&
-    env.BB_ADMIN_USER_EMAIL &&
-    env.BB_ADMIN_USER_PASSWORD
-  ) {
-    const checklist = await getChecklist()
-    if (!checklist?.adminUser?.checked) {
-      try {
-        const tenantId = tenancy.getTenantId()
-        const user = await createAdminUser(
-          env.BB_ADMIN_USER_EMAIL,
-          env.BB_ADMIN_USER_PASSWORD,
-          tenantId
-        )
-        // Need to set up an API key for automated integration tests
-        if (env.isTest()) {
-          await generateApiKey(user._id)
-        }
-
-        console.log(
-          "Admin account automatically created for",
-          env.BB_ADMIN_USER_EMAIL
-        )
-      } catch (e) {
-        logAlert("Error creating initial admin user. Exiting.", e)
-        shutdown()
-      }
-    }
-  }
-
-  // monitor plugin directory if required
-  if (
-    env.SELF_HOSTED &&
-    !env.MULTI_TENANCY &&
-    env.PLUGINS_DIR &&
-    fs.existsSync(env.PLUGINS_DIR)
-  ) {
-    watch()
-  }
-
-  // check for version updates
-  await installation.checkInstallVersion()
-
-  // done last - these will never complete
-  let promises = []
-  promises.push(automations.init())
-  promises.push(initPro())
-  await Promise.all(promises)
+  await startup(app, server)
 })
 
 const shutdown = () => {

packages/server/src/automations/bullboard.js

@@ -3,6 +3,7 @@ const { BullAdapter } = require("@bull-board/api/bullAdapter")
 const { KoaAdapter } = require("@bull-board/koa")
 const { queue } = require("@budibase/backend-core")
 const automation = require("../threads/automation")
+const { backups } = require("@budibase/pro")
 
 let automationQueue = queue.createQueue(
   queue.JobQueue.AUTOMATION,
@@ -11,9 +12,13 @@ let automationQueue = queue.createQueue(
 
 const PATH_PREFIX = "/bulladmin"
 
-exports.init = () => {
+exports.init = async () => {
   // Set up queues for bull board admin
+  const backupQueue = await backups.getBackupQueue()
   const queues = [automationQueue]
+  if (backupQueue) {
+    queues.push(backupQueue)
+  }
   const adapters = []
   const serverAdapter = new KoaAdapter()
   for (let queue of queues) {

packages/server/src/automations/utils.ts

@@ -1,10 +1,9 @@
 import { Thread, ThreadType } from "../threads"
 import { definitions } from "./triggerInfo"
-import * as webhooks from "../api/controllers/webhook"
 import { automationQueue } from "./bullboard"
 import newid from "../db/newid"
 import { updateEntityMetadata } from "../utilities"
-import { MetadataTypes, WebhookType } from "../constants"
+import { MetadataTypes } from "../constants"
 import { getProdAppID, doWithDB } from "@budibase/backend-core/db"
 import { getAutomationMetadataParams } from "../db/utils"
 import { cloneDeep } from "lodash/fp"
@@ -15,7 +14,8 @@ import {
 } from "@budibase/backend-core/context"
 import { context } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
-import { Automation } from "@budibase/types"
+import { Automation, WebhookActionType } from "@budibase/types"
+import sdk from "../sdk"
 
 const REBOOT_CRON = "@reboot"
 const WH_STEP_ID = definitions.WEBHOOK.stepId
@@ -197,16 +197,12 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) {
       let db = getAppDB()
       // need to get the webhook to get the rev
       const webhook = await db.get(oldTrigger.webhookId)
-      const ctx = {
-        appId,
-        params: { id: webhook._id, rev: webhook._rev },
-      }
       // might be updating - reset the inputs to remove the URLs
       if (newTrigger) {
         delete newTrigger.webhookId
         newTrigger.inputs = {}
       }
-      await webhooks.destroy(ctx)
+      await sdk.automations.webhook.destroy(webhook._id, webhook._rev)
     } catch (err) {
       // don't worry about not being able to delete, if it doesn't exist all good
     }
@@ -216,18 +212,14 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) {
     (!isWebhookTrigger(oldAuto) || triggerChanged) &&
     isWebhookTrigger(newAuto)
   ) {
-    const ctx: any = {
-      appId,
-      request: {
-        body: new webhooks.Webhook(
-          "Automation webhook",
-          WebhookType.AUTOMATION,
-          newAuto._id
-        ),
-      },
-    }
-    await webhooks.save(ctx)
-    const id = ctx.body.webhook._id
+    const webhook = await sdk.automations.webhook.save(
+      sdk.automations.webhook.newDoc(
+        "Automation webhook",
+        WebhookActionType.AUTOMATION,
+        newAuto._id
+      )
+    )
+    const id = webhook._id
     newTrigger.webhookId = id
     // the app ID has to be development for this endpoint
     // it can only be used when building the app

packages/server/src/constants/index.js

@@ -196,10 +196,6 @@ exports.BuildSchemaErrors = {
   INVALID_COLUMN: "invalid_column",
 }
 
-exports.WebhookType = {
-  AUTOMATION: "automation",
-}
-
 exports.AutomationErrors = {
   INCORRECT_TYPE: "INCORRECT_TYPE",
   MAX_ITERATIONS: "MAX_ITERATIONS_REACHED",

packages/server/src/definitions/automations.ts

@@ -1,9 +1,4 @@
-import {
-  Automation,
-  AutomationResults,
-  AutomationStep,
-  Document,
-} from "@budibase/types"
+import { AutomationResults, AutomationStep, Document } from "@budibase/types"
 
 export enum LoopStepType {
   ARRAY = "Array",

packages/server/src/migrations/functions/tests/syncQuotas.spec.js

@@ -1,13 +1,11 @@
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
-
 const syncApps = jest.fn()
 const syncRows = jest.fn()
 const syncPlugins = jest.fn()
-
 jest.mock("../usageQuotas/syncApps", () => ({ run: syncApps }) )
 jest.mock("../usageQuotas/syncRows", () => ({ run: syncRows }) )
 jest.mock("../usageQuotas/syncPlugins", () => ({ run: syncPlugins }) )
 
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const migration = require("../syncQuotas")
 
 describe("run", () => {

packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js

@@ -1,11 +1,13 @@
+jest.mock("@budibase/backend-core/db", () => ({
+  ...jest.requireActual("@budibase/backend-core/db"),
+  createNewUserEmailView: jest.fn(),
+}))
+const coreDb = require("@budibase/backend-core/db")
 const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const { TENANT_ID } = require("../../../tests/utilities/structures")
 const { getGlobalDB, doInTenant } = require("@budibase/backend-core/tenancy")
 
 // mock email view creation
-const coreDb = require("@budibase/backend-core/db")
-const createNewUserEmailView = jest.fn()
-coreDb.createNewUserEmailView = createNewUserEmailView
 
 const migration = require("../userEmailViewCasing")
 
@@ -22,7 +24,7 @@ describe("run", () => {
       await doInTenant(TENANT_ID, async () => {
         const globalDb = getGlobalDB()
         await migration.run(globalDb)
-        expect(createNewUserEmailView).toHaveBeenCalledTimes(1)
+        expect(coreDb.createNewUserEmailView).toHaveBeenCalledTimes(1)
       })
     })
 })

packages/server/src/sdk/app/automations/index.ts

@@ -0,0 +1,5 @@
+import * as webhook from "./webhook"
+
+export default {
+  webhook,
+}

packages/server/src/sdk/app/automations/webhook.ts

@@ -0,0 +1,43 @@
+import { Webhook, WebhookActionType } from "@budibase/types"
+import { db as dbCore, context } from "@budibase/backend-core"
+import { generateWebhookID } from "../../../db/utils"
+
+function isWebhookID(id: string) {
+  return id.startsWith(dbCore.DocumentType.WEBHOOK)
+}
+
+export function newDoc(
+  name: string,
+  type: WebhookActionType,
+  target: string
+): Webhook {
+  return {
+    live: true,
+    name,
+    action: {
+      type,
+      target,
+    },
+  }
+}
+
+export async function save(webhook: Webhook) {
+  const db = context.getAppDB()
+  // check that the webhook exists
+  if (webhook._id && isWebhookID(webhook._id)) {
+    await db.get(webhook._id)
+  } else {
+    webhook._id = generateWebhookID()
+  }
+  const response = await db.put(webhook)
+  webhook._rev = response.rev
+  return webhook
+}
+
+export async function destroy(id: string, rev: string) {
+  const db = context.getAppDB()
+  if (!id || !isWebhookID(id)) {
+    throw new Error("Provided webhook ID is not valid.")
+  }
+  return await db.remove(id, rev)
+}

packages/server/src/sdk/app/backups/imports.ts

@@ -1,20 +1,25 @@
 import { db as dbCore } from "@budibase/backend-core"
-import { TABLE_ROW_PREFIX } from "../../../db/utils"
+import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
 import {
-  uploadDirectory,
   upload,
+  uploadDirectory,
 } from "../../../utilities/fileSystem/utilities"
 import { downloadTemplate } from "../../../utilities/fileSystem"
-import { ObjectStoreBuckets, FieldTypes } from "../../../constants"
+import { FieldTypes, ObjectStoreBuckets } from "../../../constants"
 import { join } from "path"
 import fs from "fs"
 import sdk from "../../"
-import { CouchFindOptions, RowAttachment } from "@budibase/types"
+import {
+  Automation,
+  AutomationTriggerStepId,
+  CouchFindOptions,
+  RowAttachment,
+} from "@budibase/types"
+import PouchDB from "pouchdb"
 const uuid = require("uuid/v4")
 const tar = require("tar")
-import PouchDB from "pouchdb"
 
 type TemplateType = {
   file?: {
@@ -81,6 +86,34 @@ async function updateAttachmentColumns(
   }
 }
 
+async function updateAutomations(prodAppId: string, db: PouchDB.Database) {
+  const automations = (
+    await db.allDocs(
+      getAutomationParams(null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc) as Automation[]
+  const devAppId = dbCore.getDevAppID(prodAppId)
+  let toSave: Automation[] = []
+  for (let automation of automations) {
+    const oldDevAppId = automation.appId,
+      oldProdAppId = dbCore.getProdAppID(automation.appId)
+    if (
+      automation.definition.trigger.stepId === AutomationTriggerStepId.WEBHOOK
+    ) {
+      const old = automation.definition.trigger.inputs
+      automation.definition.trigger.inputs = {
+        schemaUrl: old.schemaUrl.replace(oldDevAppId, devAppId),
+        triggerUrl: old.triggerUrl.replace(oldProdAppId, prodAppId),
+      }
+    }
+    automation.appId = devAppId
+    toSave.push(automation)
+  }
+  await db.bulkDocs(toSave)
+}
+
 /**
  * This function manages temporary template files which are stored by Koa.
  * @param {Object} template The template object retrieved from the Koa context object.
@@ -165,5 +198,6 @@ export async function importApp(
     throw "Error loading database dump from template."
   }
   await updateAttachmentColumns(prodAppId, db)
+  await updateAutomations(prodAppId, db)
   return ok
 }

packages/server/src/sdk/index.ts

@@ -1,9 +1,11 @@
 import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
+import { default as automations } from "./app/automations"
 
 const sdk = {
   backups,
   tables,
+  automations,
 }
 
 // default export for TS

packages/server/src/startup.ts

@@ -0,0 +1,138 @@
+import * as env from "./environment"
+import redis from "./utilities/redis"
+import {
+  createAdminUser,
+  generateApiKey,
+  getChecklist,
+} from "./utilities/workerRequests"
+import {
+  installation,
+  pinoSettings,
+  tenancy,
+  logging,
+} from "@budibase/backend-core"
+import fs from "fs"
+import { watch } from "./watch"
+import automations from "./automations"
+import fileSystem from "./utilities/fileSystem"
+import eventEmitter from "./events"
+import * as migrations from "./migrations"
+import bullboard from "./automations/bullboard"
+import * as pro from "../../../../budibase-pro/packages/pro"
+import api from "./api"
+import sdk from "./sdk"
+const pino = require("koa-pino-logger")
+
+let STARTUP_RAN = false
+
+async function initRoutes(app: any) {
+  app.use(pino(pinoSettings()))
+
+  if (!env.isTest()) {
+    const plugin = await bullboard.init()
+    app.use(plugin)
+  }
+
+  app.context.eventEmitter = eventEmitter
+  app.context.auth = {}
+
+  // api routes
+  app.use(api.router.routes())
+}
+
+async function initPro() {
+  await pro.init({
+    backups: {
+      processing: {
+        exportAppFn: sdk.backups.exportApp,
+        importAppFn: sdk.backups.importApp,
+        statsFn: sdk.backups.calculateBackupStats,
+      },
+    },
+  })
+}
+
+function shutdown(server?: any) {
+  server.close()
+  server.destroy()
+}
+
+export async function startup(app?: any, server?: any) {
+  if (STARTUP_RAN) {
+    return
+  }
+  STARTUP_RAN = true
+  if (server) {
+    console.log(`Budibase running on ${JSON.stringify(server.address())}`)
+    env._set("PORT", server.address().port)
+  }
+  eventEmitter.emitPort(env.PORT)
+  fileSystem.init()
+  await redis.init()
+
+  // run migrations on startup if not done via http
+  // not recommended in a clustered environment
+  if (!env.HTTP_MIGRATIONS && !env.isTest()) {
+    try {
+      await migrations.migrate()
+    } catch (e) {
+      logging.logAlert("Error performing migrations. Exiting.", e)
+      shutdown()
+    }
+  }
+
+  // check and create admin user if required
+  if (
+    env.SELF_HOSTED &&
+    !env.MULTI_TENANCY &&
+    env.BB_ADMIN_USER_EMAIL &&
+    env.BB_ADMIN_USER_PASSWORD
+  ) {
+    const checklist = await getChecklist()
+    if (!checklist?.adminUser?.checked) {
+      try {
+        const tenantId = tenancy.getTenantId()
+        const user = await createAdminUser(
+          env.BB_ADMIN_USER_EMAIL,
+          env.BB_ADMIN_USER_PASSWORD,
+          tenantId
+        )
+        // Need to set up an API key for automated integration tests
+        if (env.isTest()) {
+          await generateApiKey(user._id)
+        }
+
+        console.log(
+          "Admin account automatically created for",
+          env.BB_ADMIN_USER_EMAIL
+        )
+      } catch (e) {
+        logging.logAlert("Error creating initial admin user. Exiting.", e)
+        shutdown()
+      }
+    }
+  }
+
+  // monitor plugin directory if required
+  if (
+    env.SELF_HOSTED &&
+    !env.MULTI_TENANCY &&
+    env.PLUGINS_DIR &&
+    fs.existsSync(env.PLUGINS_DIR)
+  ) {
+    watch()
+  }
+
+  // check for version updates
+  await installation.checkInstallVersion()
+
+  // get the references to the queue promises, don't await as
+  // they will never end, unless the processing stops
+  let queuePromises = []
+  queuePromises.push(automations.init())
+  queuePromises.push(initPro())
+  if (app) {
+    // bring routes online as final step once everything ready
+    await initRoutes(app)
+  }
+}

packages/server/src/tests/utilities/TestConfiguration.js

@@ -26,6 +26,7 @@ const context = require("@budibase/backend-core/context")
 const { generateDevInfoID, SEPARATOR } = require("@budibase/backend-core/db")
 const { encrypt } = require("@budibase/backend-core/encryption")
 const { DocumentType, generateUserMetadataID } = require("../../db/utils")
+const { startup } = require("../../startup")
 
 const GLOBAL_USER_ID = "us_uuid1"
 const EMAIL = "babs@babs.com"
@@ -41,6 +42,9 @@ class TestConfiguration {
       this.server = require("../../app")
       // we need the request for logging in, involves cookies, hard to fake
       this.request = supertest(this.server)
+      this.started = true
+    } else {
+      this.started = false
     }
     this.appId = null
     this.allApps = []
@@ -95,6 +99,9 @@ class TestConfiguration {
 
   // use a new id as the name to avoid name collisions
   async init(appName = newid()) {
+    if (!this.started) {
+      await startup()
+    }
     this.user = await this.globalUser()
     this.globalUserId = this.user._id
     this.userMetadataId = generateUserMetadataID(this.globalUserId)

packages/types/src/documents/app/automation.ts

@@ -1,5 +1,34 @@
 import { Document } from "../document"
 
+export enum AutomationTriggerStepId {
+  ROW_SAVED = "ROW_SAVED",
+  ROW_UPDATED = "ROW_UPDATED",
+  ROW_DELETED = "ROW_DELETED",
+  WEBHOOK = "WEBHOOK",
+  APP = "APP",
+  CRON = "CRON",
+}
+
+export enum AutomationActionStepId {
+  SEND_EMAIL_SMTP = "SEND_EMAIL_SMTP",
+  CREATE_ROW = "CREATE_ROW",
+  UPDATE_ROW = "UPDATE_ROW",
+  DELETE_ROW = "DELETE_ROW",
+  OUTGOING_WEBHOOK = "OUTGOING_WEBHOOK",
+  EXECUTE_SCRIPT = "EXECUTE_SCRIPT",
+  EXECUTE_QUERY = "EXECUTE_QUERY",
+  SERVER_LOG = "SERVER_LOG",
+  DELAY = "DELAY",
+  FILTER = "FILTER",
+  QUERY_ROWS = "QUERY_ROWS",
+  LOOP = "LOOP",
+  // these used to be lowercase step IDs, maintain for backwards compat
+  discord = "discord",
+  slack = "slack",
+  zapier = "zapier",
+  integromat = "integromat",
+}
+
 export interface Automation extends Document {
   definition: {
     steps: AutomationStep[]
@@ -11,7 +40,7 @@ export interface Automation extends Document {
 
 export interface AutomationStep {
   id: string
-  stepId: string
+  stepId: AutomationTriggerStepId | AutomationActionStepId
   inputs: {
     [key: string]: any
   }
@@ -19,15 +48,13 @@ export interface AutomationStep {
     inputs: {
       [key: string]: any
     }
+    outputs: {
+      [key: string]: any
+    }
   }
 }
 
-export interface AutomationTrigger {
-  id: string
-  stepId: string
-  inputs: {
-    [key: string]: any
-  }
+export interface AutomationTrigger extends AutomationStep {
   cronJobId?: string
 }
 
@@ -43,7 +70,7 @@ export interface AutomationResults {
   status?: AutomationStatus
   trigger?: any
   steps: {
-    stepId: string
+    stepId: AutomationTriggerStepId | AutomationActionStepId
     inputs: {
       [key: string]: any
     }

packages/types/src/documents/app/index.ts

@@ -11,3 +11,4 @@ export * from "../document"
 export * from "./row"
 export * from "./user"
 export * from "./backup"
+export * from "./webhook"

packages/types/src/documents/app/webhook.ts

@@ -0,0 +1,15 @@
+import { Document } from "../document"
+
+export enum WebhookActionType {
+  AUTOMATION = "automation",
+}
+
+export interface Webhook extends Document {
+  live: boolean
+  name: string
+  action: {
+    type: WebhookActionType
+    target: string
+  }
+  bodySchema?: any
+}

packages/worker/src/api/controllers/system/restore.ts

@@ -0,0 +1,13 @@
+import env from "../../../environment"
+import { BBContext } from "@budibase/types"
+import { cache } from "@budibase/backend-core"
+
+export async function systemRestored(ctx: BBContext) {
+  if (!env.SELF_HOSTED) {
+    ctx.throw(405, "This operation is not allowed in cloud.")
+  }
+  await cache.bustCache(cache.CacheKeys.CHECKLIST)
+  ctx.body = {
+    message: "System prepared after restore.",
+  }
+}

packages/worker/src/api/index.ts

@@ -55,6 +55,10 @@ const PUBLIC_ENDPOINTS = [
     route: "/api/global/users/tenant/:id",
     method: "GET",
   },
+  {
+    route: "/api/system/restored",
+    method: "POST",
+  },
 ]
 
 const NO_TENANCY_ENDPOINTS = [

packages/worker/src/api/routes/index.ts

@@ -13,6 +13,7 @@ import selfRoutes from "./global/self"
 import licenseRoutes from "./global/license"
 import migrationRoutes from "./system/migrations"
 import accountRoutes from "./system/accounts"
+import restoreRoutes from "./system/restore"
 
 let userGroupRoutes = api.groups
 export const routes = [
@@ -31,4 +32,5 @@ export const routes = [
   userGroupRoutes,
   migrationRoutes,
   accountRoutes,
+  restoreRoutes,
 ]

packages/worker/src/api/routes/system/restore.ts

@@ -0,0 +1,8 @@
+import * as controller from "../../controllers/system/restore"
+import Router from "@koa/router"
+
+const router = new Router()
+
+router.post("/api/system/restored", controller.systemRestored)
+
+export = router