From e52db231421443b10a84a4c8ba25330f183b09c5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 25 Oct 2022 15:28:05 +0100 Subject: [PATCH 1/6] Adding version handling to CLI, #8335 - useful to see what version of CLI currently installed. --- packages/cli/src/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/index.js b/packages/cli/src/index.js index 691fc71928..9d1004fc20 100644 --- a/packages/cli/src/index.js +++ b/packages/cli/src/index.js @@ -1,6 +1,7 @@ #!/usr/bin/env node require("./prebuilds") require("./environment") +const json = require("../package.json") const { getCommands } = require("./options") const { Command } = require("commander") const { getHelpDescription } = require("./utils") @@ -10,7 +11,7 @@ async function init() { const program = new Command() .addHelpCommand("help", getHelpDescription("Help with Budibase commands.")) .helpOption(false) - program.helpOption() + .version(json.version) // add commands for (let command of getCommands()) { command.configure(program) From a24694a4eaac9d6cc0c3361fd1f5a9113facf440 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 25 Oct 2022 18:19:18 +0100 Subject: [PATCH 2/6] Refactoring a lot of content around webhooks to Typescript, as well as fixing webhooks and automation app IDs on import of new app. --- packages/backend-core/src/context/index.ts | 6 +- packages/backend-core/src/index.ts | 2 + .../controllers/{webhook.js => webhook.ts} | 81 ++++++++----------- .../server/src/api/routes/utils/validators.js | 4 +- .../src/api/routes/{webhook.js => webhook.ts} | 13 +-- packages/server/src/automations/utils.ts | 32 +++----- packages/server/src/constants/index.js | 4 - .../server/src/definitions/automations.ts | 7 +- .../server/src/sdk/app/automations/index.ts | 5 ++ .../server/src/sdk/app/automations/webhook.ts | 43 ++++++++++ .../server/src/sdk/app/backups/imports.ts | 44 ++++++++-- packages/server/src/sdk/index.ts | 2 + .../types/src/documents/app/automation.ts | 43 ++++++++-- packages/types/src/documents/app/index.ts | 1 + packages/types/src/documents/app/webhook.ts | 15 ++++ 15 files changed, 200 insertions(+), 102 deletions(-) rename packages/server/src/api/controllers/{webhook.js => webhook.ts} (51%) rename packages/server/src/api/routes/{webhook.js => webhook.ts} (63%) create mode 100644 packages/server/src/sdk/app/automations/index.ts create mode 100644 packages/server/src/sdk/app/automations/webhook.ts create mode 100644 packages/types/src/documents/app/webhook.ts diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index 7efe0e23f7..c3955c71d9 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -6,6 +6,7 @@ import { baseGlobalDBName } from "../db/tenancy" import { IdentityContext } from "@budibase/types" import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" import { ContextKey } from "./constants" +import PouchDB from "pouchdb" import { updateUsing, closeWithUsing, @@ -22,16 +23,15 @@ export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID let TEST_APP_ID: string | null = null export const closeTenancy = async () => { - let db try { if (env.USE_COUCH) { - db = getGlobalDB() + const db = getGlobalDB() + await closeDB(db) } } catch (err) { // no DB found - skip closing return } - await closeDB(db) // clear from context now that database is closed/task is finished cls.setOnContext(ContextKey.TENANT_ID, null) cls.setOnContext(ContextKey.GLOBAL_DB, null) diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 659a56c051..17393b8ac3 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -4,6 +4,7 @@ import * as events from "./events" import * as migrations from "./migrations" import * as users from "./users" import * as roles from "./security/roles" +import * as permissions from "./security/permissions" import * as accounts from "./cloud/accounts" import * as installation from "./installation" import env from "./environment" @@ -65,6 +66,7 @@ const core = { middleware, encryption, queue, + permissions, } export = core diff --git a/packages/server/src/api/controllers/webhook.js b/packages/server/src/api/controllers/webhook.ts similarity index 51% rename from packages/server/src/api/controllers/webhook.js rename to packages/server/src/api/controllers/webhook.ts index 1698775ab4..26bf16bd4c 100644 --- a/packages/server/src/api/controllers/webhook.js +++ b/packages/server/src/api/controllers/webhook.ts @@ -1,66 +1,51 @@ -const { generateWebhookID, getWebhookParams } = require("../../db/utils") +import { getWebhookParams } from "../../db/utils" +import triggers from "../../automations/triggers" +import { db as dbCore, context } from "@budibase/backend-core" +import { + Webhook, + WebhookActionType, + BBContext, + Automation, +} from "@budibase/types" +import sdk from "../../sdk" const toJsonSchema = require("to-json-schema") const validate = require("jsonschema").validate -const { WebhookType } = require("../../constants") -const triggers = require("../../automations/triggers") -const { getProdAppID } = require("@budibase/backend-core/db") -const { getAppDB, updateAppId } = require("@budibase/backend-core/context") const AUTOMATION_DESCRIPTION = "Generated from Webhook Schema" -function Webhook(name, type, target) { - this.live = true - this.name = name - this.action = { - type, - target, - } -} - -exports.Webhook = Webhook - -exports.fetch = async ctx => { - const db = getAppDB() +export async function fetch(ctx: BBContext) { + const db = context.getAppDB() const response = await db.allDocs( getWebhookParams(null, { include_docs: true, }) ) - ctx.body = response.rows.map(row => row.doc) + ctx.body = response.rows.map((row: any) => row.doc) } -exports.save = async ctx => { - const db = getAppDB() - const webhook = ctx.request.body - webhook.appId = ctx.appId - - // check that the webhook exists - if (webhook._id) { - await db.get(webhook._id) - } else { - webhook._id = generateWebhookID() - } - const response = await db.put(webhook) - webhook._rev = response.rev +export async function save(ctx: BBContext) { + const webhook = await sdk.automations.webhook.save(ctx.request.body) ctx.body = { message: "Webhook created successfully", webhook, } } -exports.destroy = async ctx => { - const db = getAppDB() - ctx.body = await db.remove(ctx.params.id, ctx.params.rev) +export async function destroy(ctx: BBContext) { + ctx.body = await sdk.automations.webhook.destroy( + ctx.params.id, + ctx.params.rev + ) } -exports.buildSchema = async ctx => { - await updateAppId(ctx.params.instance) - const db = getAppDB() - const webhook = await db.get(ctx.params.id) +export async function buildSchema(ctx: BBContext) { + await context.updateAppId(ctx.params.instance) + const db = context.getAppDB() + const webhook = (await db.get(ctx.params.id)) as Webhook webhook.bodySchema = toJsonSchema(ctx.request.body) // update the automation outputs - if (webhook.action.type === WebhookType.AUTOMATION) { - let automation = await db.get(webhook.action.target) + if (webhook.action.type === WebhookActionType.AUTOMATION) { + let automation = (await db.get(webhook.action.target)) as Automation const autoOutputs = automation.definition.trigger.schema.outputs let properties = webhook.bodySchema.properties // reset webhook outputs @@ -78,18 +63,18 @@ exports.buildSchema = async ctx => { ctx.body = await db.put(webhook) } -exports.trigger = async ctx => { - const prodAppId = getProdAppID(ctx.params.instance) - await updateAppId(prodAppId) +export async function trigger(ctx: BBContext) { + const prodAppId = dbCore.getProdAppID(ctx.params.instance) + await context.updateAppId(prodAppId) try { - const db = getAppDB() - const webhook = await db.get(ctx.params.id) + const db = context.getAppDB() + const webhook = (await db.get(ctx.params.id)) as Webhook // validate against the schema if (webhook.bodySchema) { validate(ctx.request.body, webhook.bodySchema) } const target = await db.get(webhook.action.target) - if (webhook.action.type === WebhookType.AUTOMATION) { + if (webhook.action.type === WebhookActionType.AUTOMATION) { // trigger with both the pure request and then expand it // incase the user has produced a schema to bind to await triggers.externalTrigger(target, { @@ -102,7 +87,7 @@ exports.trigger = async ctx => { ctx.body = { message: "Webhook trigger fired successfully", } - } catch (err) { + } catch (err: any) { if (err.status === 404) { ctx.status = 200 ctx.body = { diff --git a/packages/server/src/api/routes/utils/validators.js b/packages/server/src/api/routes/utils/validators.js index ab9f2afaf0..f1d8871805 100644 --- a/packages/server/src/api/routes/utils/validators.js +++ b/packages/server/src/api/routes/utils/validators.js @@ -1,10 +1,10 @@ const { joiValidator } = require("@budibase/backend-core/auth") const { DataSourceOperation } = require("../../../constants") -const { WebhookType } = require("../../../constants") const { BUILTIN_PERMISSION_IDS, PermissionLevels, } = require("@budibase/backend-core/permissions") +const { WebhookActionType } = require("@budibase/types") const Joi = require("joi") const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("") @@ -126,7 +126,7 @@ exports.webhookValidator = () => { name: Joi.string().required(), bodySchema: Joi.object().optional(), action: Joi.object({ - type: Joi.string().required().valid(WebhookType.AUTOMATION), + type: Joi.string().required().valid(WebhookActionType.AUTOMATION), target: Joi.string().required(), }).required(), }).unknown(true)) diff --git a/packages/server/src/api/routes/webhook.js b/packages/server/src/api/routes/webhook.ts similarity index 63% rename from packages/server/src/api/routes/webhook.js rename to packages/server/src/api/routes/webhook.ts index 9d60438a63..103ab98142 100644 --- a/packages/server/src/api/routes/webhook.js +++ b/packages/server/src/api/routes/webhook.ts @@ -1,9 +1,10 @@ -const Router = require("@koa/router") -const controller = require("../controllers/webhook") -const authorized = require("../../middleware/authorized") -const { BUILDER } = require("@budibase/backend-core/permissions") -const { webhookValidator } = require("./utils/validators") +import Router from "@koa/router" +import * as controller from "../controllers/webhook" +import authorized from "../../middleware/authorized" +import { permissions } from "@budibase/backend-core" +import { webhookValidator } from "./utils/validators" +const BUILDER = permissions.BUILDER const router = new Router() router @@ -23,4 +24,4 @@ router // this shouldn't have authorisation, right now its always public .post("/api/webhooks/trigger/:instance/:id", controller.trigger) -module.exports = router +export default router diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index 0eebcb21cf..af4bb8d3af 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -1,10 +1,9 @@ import { Thread, ThreadType } from "../threads" import { definitions } from "./triggerInfo" -import * as webhooks from "../api/controllers/webhook" import { automationQueue } from "./bullboard" import newid from "../db/newid" import { updateEntityMetadata } from "../utilities" -import { MetadataTypes, WebhookType } from "../constants" +import { MetadataTypes } from "../constants" import { getProdAppID, doWithDB } from "@budibase/backend-core/db" import { getAutomationMetadataParams } from "../db/utils" import { cloneDeep } from "lodash/fp" @@ -15,7 +14,8 @@ import { } from "@budibase/backend-core/context" import { context } from "@budibase/backend-core" import { quotas } from "@budibase/pro" -import { Automation } from "@budibase/types" +import { Automation, WebhookActionType } from "@budibase/types" +import sdk from "../sdk" const REBOOT_CRON = "@reboot" const WH_STEP_ID = definitions.WEBHOOK.stepId @@ -197,16 +197,12 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) { let db = getAppDB() // need to get the webhook to get the rev const webhook = await db.get(oldTrigger.webhookId) - const ctx = { - appId, - params: { id: webhook._id, rev: webhook._rev }, - } // might be updating - reset the inputs to remove the URLs if (newTrigger) { delete newTrigger.webhookId newTrigger.inputs = {} } - await webhooks.destroy(ctx) + await sdk.automations.webhook.destroy(webhook._id, webhook._rev) } catch (err) { // don't worry about not being able to delete, if it doesn't exist all good } @@ -216,18 +212,14 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) { (!isWebhookTrigger(oldAuto) || triggerChanged) && isWebhookTrigger(newAuto) ) { - const ctx: any = { - appId, - request: { - body: new webhooks.Webhook( - "Automation webhook", - WebhookType.AUTOMATION, - newAuto._id - ), - }, - } - await webhooks.save(ctx) - const id = ctx.body.webhook._id + const webhook = await sdk.automations.webhook.save( + sdk.automations.webhook.newDoc( + "Automation webhook", + WebhookActionType.AUTOMATION, + newAuto._id + ) + ) + const id = webhook._id newTrigger.webhookId = id // the app ID has to be development for this endpoint // it can only be used when building the app diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index a3bccae754..6d8fe57baa 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -196,10 +196,6 @@ exports.BuildSchemaErrors = { INVALID_COLUMN: "invalid_column", } -exports.WebhookType = { - AUTOMATION: "automation", -} - exports.AutomationErrors = { INCORRECT_TYPE: "INCORRECT_TYPE", MAX_ITERATIONS: "MAX_ITERATIONS_REACHED", diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index 877a1b4579..d4168c020f 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -1,9 +1,4 @@ -import { - Automation, - AutomationResults, - AutomationStep, - Document, -} from "@budibase/types" +import { AutomationResults, AutomationStep, Document } from "@budibase/types" export enum LoopStepType { ARRAY = "Array", diff --git a/packages/server/src/sdk/app/automations/index.ts b/packages/server/src/sdk/app/automations/index.ts new file mode 100644 index 0000000000..1c9ce13455 --- /dev/null +++ b/packages/server/src/sdk/app/automations/index.ts @@ -0,0 +1,5 @@ +import * as webhook from "./webhook" + +export default { + webhook, +} diff --git a/packages/server/src/sdk/app/automations/webhook.ts b/packages/server/src/sdk/app/automations/webhook.ts new file mode 100644 index 0000000000..a6d0691f1f --- /dev/null +++ b/packages/server/src/sdk/app/automations/webhook.ts @@ -0,0 +1,43 @@ +import { Webhook, WebhookActionType } from "@budibase/types" +import { db as dbCore, context } from "@budibase/backend-core" +import { generateWebhookID } from "../../../db/utils" + +function isWebhookID(id: string) { + return id.startsWith(dbCore.DocumentType.WEBHOOK) +} + +export function newDoc( + name: string, + type: WebhookActionType, + target: string +): Webhook { + return { + live: true, + name, + action: { + type, + target, + }, + } +} + +export async function save(webhook: Webhook) { + const db = context.getAppDB() + // check that the webhook exists + if (webhook._id && isWebhookID(webhook._id)) { + await db.get(webhook._id) + } else { + webhook._id = generateWebhookID() + } + const response = await db.put(webhook) + webhook._rev = response.rev + return webhook +} + +export async function destroy(id: string, rev: string) { + const db = context.getAppDB() + if (!id || !isWebhookID(id)) { + throw new Error("Provided webhook ID is not valid.") + } + return await db.remove(id, rev) +} diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index f6f2939b7d..5f4b8c7e41 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -1,20 +1,25 @@ import { db as dbCore } from "@budibase/backend-core" -import { TABLE_ROW_PREFIX } from "../../../db/utils" +import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants" import { - uploadDirectory, upload, + uploadDirectory, } from "../../../utilities/fileSystem/utilities" import { downloadTemplate } from "../../../utilities/fileSystem" -import { ObjectStoreBuckets, FieldTypes } from "../../../constants" +import { FieldTypes, ObjectStoreBuckets } from "../../../constants" import { join } from "path" import fs from "fs" import sdk from "../../" -import { CouchFindOptions, RowAttachment } from "@budibase/types" +import { + Automation, + AutomationTriggerStepId, + CouchFindOptions, + RowAttachment, +} from "@budibase/types" +import PouchDB from "pouchdb" const uuid = require("uuid/v4") const tar = require("tar") -import PouchDB from "pouchdb" type TemplateType = { file?: { @@ -81,6 +86,34 @@ async function updateAttachmentColumns( } } +async function updateAutomations(prodAppId: string, db: PouchDB.Database) { + const automations = ( + await db.allDocs( + getAutomationParams(null, { + include_docs: true, + }) + ) + ).rows.map(row => row.doc) as Automation[] + const devAppId = dbCore.getDevAppID(prodAppId) + let toSave: Automation[] = [] + for (let automation of automations) { + const oldDevAppId = automation.appId, + oldProdAppId = dbCore.getProdAppID(automation.appId) + if ( + automation.definition.trigger.stepId === AutomationTriggerStepId.WEBHOOK + ) { + const old = automation.definition.trigger.inputs + automation.definition.trigger.inputs = { + schemaUrl: old.schemaUrl.replace(oldDevAppId, devAppId), + triggerUrl: old.triggerUrl.replace(oldProdAppId, prodAppId), + } + } + automation.appId = devAppId + toSave.push(automation) + } + await db.bulkDocs(toSave) +} + /** * This function manages temporary template files which are stored by Koa. * @param {Object} template The template object retrieved from the Koa context object. @@ -165,5 +198,6 @@ export async function importApp( throw "Error loading database dump from template." } await updateAttachmentColumns(prodAppId, db) + await updateAutomations(prodAppId, db) return ok } diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts index 8bdc4f8e77..1f7a365e90 100644 --- a/packages/server/src/sdk/index.ts +++ b/packages/server/src/sdk/index.ts @@ -1,9 +1,11 @@ import { default as backups } from "./app/backups" import { default as tables } from "./app/tables" +import { default as automations } from "./app/automations" const sdk = { backups, tables, + automations, } // default export for TS diff --git a/packages/types/src/documents/app/automation.ts b/packages/types/src/documents/app/automation.ts index a038e73d11..b53da956d4 100644 --- a/packages/types/src/documents/app/automation.ts +++ b/packages/types/src/documents/app/automation.ts @@ -1,5 +1,34 @@ import { Document } from "../document" +export enum AutomationTriggerStepId { + ROW_SAVED = "ROW_SAVED", + ROW_UPDATED = "ROW_UPDATED", + ROW_DELETED = "ROW_DELETED", + WEBHOOK = "WEBHOOK", + APP = "APP", + CRON = "CRON", +} + +export enum AutomationActionStepId { + SEND_EMAIL_SMTP = "SEND_EMAIL_SMTP", + CREATE_ROW = "CREATE_ROW", + UPDATE_ROW = "UPDATE_ROW", + DELETE_ROW = "DELETE_ROW", + OUTGOING_WEBHOOK = "OUTGOING_WEBHOOK", + EXECUTE_SCRIPT = "EXECUTE_SCRIPT", + EXECUTE_QUERY = "EXECUTE_QUERY", + SERVER_LOG = "SERVER_LOG", + DELAY = "DELAY", + FILTER = "FILTER", + QUERY_ROWS = "QUERY_ROWS", + LOOP = "LOOP", + // these used to be lowercase step IDs, maintain for backwards compat + discord = "discord", + slack = "slack", + zapier = "zapier", + integromat = "integromat", +} + export interface Automation extends Document { definition: { steps: AutomationStep[] @@ -11,7 +40,7 @@ export interface Automation extends Document { export interface AutomationStep { id: string - stepId: string + stepId: AutomationTriggerStepId | AutomationActionStepId inputs: { [key: string]: any } @@ -19,15 +48,13 @@ export interface AutomationStep { inputs: { [key: string]: any } + outputs: { + [key: string]: any + } } } -export interface AutomationTrigger { - id: string - stepId: string - inputs: { - [key: string]: any - } +export interface AutomationTrigger extends AutomationStep { cronJobId?: string } @@ -43,7 +70,7 @@ export interface AutomationResults { status?: AutomationStatus trigger?: any steps: { - stepId: string + stepId: AutomationTriggerStepId | AutomationActionStepId inputs: { [key: string]: any } diff --git a/packages/types/src/documents/app/index.ts b/packages/types/src/documents/app/index.ts index dad594b804..25c150f9da 100644 --- a/packages/types/src/documents/app/index.ts +++ b/packages/types/src/documents/app/index.ts @@ -11,3 +11,4 @@ export * from "../document" export * from "./row" export * from "./user" export * from "./backup" +export * from "./webhook" diff --git a/packages/types/src/documents/app/webhook.ts b/packages/types/src/documents/app/webhook.ts new file mode 100644 index 0000000000..1ced8627af --- /dev/null +++ b/packages/types/src/documents/app/webhook.ts @@ -0,0 +1,15 @@ +import { Document } from "../document" + +export enum WebhookActionType { + AUTOMATION = "automation", +} + +export interface Webhook extends Document { + live: boolean + name: string + action: { + type: WebhookActionType + target: string + } + bodySchema?: any +} From 51251b1fe2ce2e1566015c2a38c0e475f31d2079 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 26 Oct 2022 18:50:40 +0100 Subject: [PATCH 3/6] Adding backup queue to bullboard if enabled. --- packages/server/src/app.ts | 42 +++++++++++--------- packages/server/src/automations/bullboard.js | 7 +++- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index d8ed2c872b..a5dfcc6e80 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -54,19 +54,6 @@ app.use( }) ) -app.use(pino(pinoSettings())) - -if (!env.isTest()) { - const plugin = bullboard.init() - app.use(plugin) -} - -app.context.eventEmitter = eventEmitter -app.context.auth = {} - -// api routes -app.use(api.router.routes()) - if (env.isProd()) { env._set("NODE_ENV", "production") Sentry.init() @@ -104,7 +91,22 @@ server.on("close", async () => { } }) -const initPro = async () => { +async function initRoutes() { + app.use(pino(pinoSettings())) + + if (!env.isTest()) { + const plugin = await bullboard.init() + app.use(plugin) + } + + app.context.eventEmitter = eventEmitter + app.context.auth = {} + + // api routes + app.use(api.router.routes()) +} + +async function initPro() { await pro.init({ backups: { processing: { @@ -179,11 +181,13 @@ module.exports = server.listen(env.PORT || 0, async () => { // check for version updates await installation.checkInstallVersion() - // done last - these will never complete - let promises = [] - promises.push(automations.init()) - promises.push(initPro()) - await Promise.all(promises) + // get the references to the queue promises, don't await as + // they will never end, unless the processing stops + let queuePromises = [] + queuePromises.push(automations.init()) + queuePromises.push(initPro()) + // bring routes online as final step once everything ready + await initRoutes() }) const shutdown = () => { diff --git a/packages/server/src/automations/bullboard.js b/packages/server/src/automations/bullboard.js index c4f33e07a9..dd4a6aa383 100644 --- a/packages/server/src/automations/bullboard.js +++ b/packages/server/src/automations/bullboard.js @@ -3,6 +3,7 @@ const { BullAdapter } = require("@bull-board/api/bullAdapter") const { KoaAdapter } = require("@bull-board/koa") const { queue } = require("@budibase/backend-core") const automation = require("../threads/automation") +const { backups } = require("@budibase/pro") let automationQueue = queue.createQueue( queue.JobQueue.AUTOMATION, @@ -11,9 +12,13 @@ let automationQueue = queue.createQueue( const PATH_PREFIX = "/bulladmin" -exports.init = () => { +exports.init = async () => { // Set up queues for bull board admin + const backupQueue = await backups.getBackupQueue() const queues = [automationQueue] + if (backupQueue) { + queues.push(backupQueue) + } const adapters = [] const serverAdapter = new KoaAdapter() for (let queue of queues) { From 6d72e51d166a924e5f60dcfedc32074b00bbbcf8 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 26 Oct 2022 18:51:58 +0100 Subject: [PATCH 4/6] Some fixes for CLI to get exporting/importing with different ports working, as well as a fix for access to minio through proxy. --- hosting/nginx.prod.conf.hbs | 2 ++ packages/cli/src/backups/objectStore.js | 5 +++++ packages/cli/src/backups/utils.js | 16 +++++++++++----- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/hosting/nginx.prod.conf.hbs b/hosting/nginx.prod.conf.hbs index f3202ad4a4..114a4575d0 100644 --- a/hosting/nginx.prod.conf.hbs +++ b/hosting/nginx.prod.conf.hbs @@ -171,11 +171,13 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; proxy_connect_timeout 300; proxy_http_version 1.1; proxy_set_header Connection ""; chunked_transfer_encoding off; + proxy_pass http://$minio:9000; } diff --git a/packages/cli/src/backups/objectStore.js b/packages/cli/src/backups/objectStore.js index b0bf99891d..8d616d276a 100644 --- a/packages/cli/src/backups/objectStore.js +++ b/packages/cli/src/backups/objectStore.js @@ -16,16 +16,21 @@ exports.exportObjects = async () => { const path = join(TEMP_DIR, MINIO_DIR) fs.mkdirSync(path) let fullList = [] + let errorCount = 0 for (let bucket of bucketList) { const client = ObjectStore(bucket) try { await client.headBucket().promise() } catch (err) { + errorCount++ continue } const list = await client.listObjectsV2().promise() fullList = fullList.concat(list.Contents.map(el => ({ ...el, bucket }))) } + if (errorCount === bucketList.length) { + throw new Error("Unable to access MinIO/S3 - check environment config.") + } const bar = progressBar(fullList.length) let count = 0 for (let object of fullList) { diff --git a/packages/cli/src/backups/utils.js b/packages/cli/src/backups/utils.js index 5d941aa17c..645b90379b 100644 --- a/packages/cli/src/backups/utils.js +++ b/packages/cli/src/backups/utils.js @@ -2,17 +2,19 @@ const dotenv = require("dotenv") const fs = require("fs") const { string } = require("../questions") const { getPouch } = require("../core/db") +const { env: environment } = require("@budibase/backend-core") -exports.DEFAULT_COUCH = "http://budibase:budibase@localhost:10000/db/" -exports.DEFAULT_MINIO = "http://localhost:10000/" exports.TEMP_DIR = ".temp" exports.COUCH_DIR = "couchdb" exports.MINIO_DIR = "minio" const REQUIRED = [ { value: "MAIN_PORT", default: "10000" }, - { value: "COUCH_DB_URL", default: exports.DEFAULT_COUCH }, - { value: "MINIO_URL", default: exports.DEFAULT_MINIO }, + { + value: "COUCH_DB_URL", + default: "http://budibase:budibase@localhost:10000/db/", + }, + { value: "MINIO_URL", default: "http://localhost:10000" }, { value: "MINIO_ACCESS_KEY" }, { value: "MINIO_SECRET_KEY" }, ] @@ -27,7 +29,7 @@ exports.checkURLs = config => { ] = `http://${username}:${password}@localhost:${mainPort}/db/` } if (!config["MINIO_URL"]) { - config["MINIO_URL"] = exports.DEFAULT_MINIO + config["MINIO_URL"] = `http://localhost:${mainPort}/` } return config } @@ -65,6 +67,10 @@ exports.getConfig = async (envFile = true) => { } else { config = await exports.askQuestions() } + // fill out environment + for (let key of Object.keys(config)) { + environment._set(key, config[key]) + } return config } From a5d980efb1078d825dbc178ed8ef9c816e404f88 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 27 Oct 2022 09:48:37 +0100 Subject: [PATCH 5/6] Adding a fix for checklist being incorrect after restoring from the CLI. --- packages/cli/src/backups/index.js | 11 ++++++++++- packages/cli/src/utils.js | 8 ++++++++ .../worker/src/api/controllers/system/restore.ts | 13 +++++++++++++ packages/worker/src/api/index.ts | 4 ++++ packages/worker/src/api/routes/index.ts | 2 ++ packages/worker/src/api/routes/system/restore.ts | 8 ++++++++ 6 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 packages/worker/src/api/controllers/system/restore.ts create mode 100644 packages/worker/src/api/routes/system/restore.ts diff --git a/packages/cli/src/backups/index.js b/packages/cli/src/backups/index.js index 697dafac6f..47b54fa9a0 100644 --- a/packages/cli/src/backups/index.js +++ b/packages/cli/src/backups/index.js @@ -4,7 +4,7 @@ const fs = require("fs") const { join } = require("path") const { getAllDbs } = require("../core/db") const tar = require("tar") -const { progressBar } = require("../utils") +const { progressBar, httpCall } = require("../utils") const { TEMP_DIR, COUCH_DIR, @@ -86,6 +86,15 @@ async function importBackup(opts) { bar.stop() console.log("MinIO Import") await importObjects() + // finish by letting the system know that a restore has occurred + try { + await httpCall( + `http://localhost:${config.MAIN_PORT}/api/system/restored`, + "POST" + ) + } catch (err) { + // ignore error - it will be an older system + } console.log("Import complete") fs.rmSync(TEMP_DIR, { recursive: true }) } diff --git a/packages/cli/src/utils.js b/packages/cli/src/utils.js index ba793420e7..d041fb498a 100644 --- a/packages/cli/src/utils.js +++ b/packages/cli/src/utils.js @@ -23,6 +23,14 @@ exports.downloadFile = async (url, filePath) => { }) } +exports.httpCall = async (url, method) => { + const response = await axios({ + url, + method, + }) + return response.data +} + exports.getHelpDescription = string => { return chalk.cyan(string) } diff --git a/packages/worker/src/api/controllers/system/restore.ts b/packages/worker/src/api/controllers/system/restore.ts new file mode 100644 index 0000000000..96a7c61cb4 --- /dev/null +++ b/packages/worker/src/api/controllers/system/restore.ts @@ -0,0 +1,13 @@ +import env from "../../../environment" +import { BBContext } from "@budibase/types" +import { cache } from "@budibase/backend-core" + +export async function systemRestored(ctx: BBContext) { + if (!env.SELF_HOSTED) { + ctx.throw(405, "This operation is not allowed in cloud.") + } + await cache.bustCache(cache.CacheKeys.CHECKLIST) + ctx.body = { + message: "System prepared after restore.", + } +} diff --git a/packages/worker/src/api/index.ts b/packages/worker/src/api/index.ts index 21fbf1d993..22ff159dff 100644 --- a/packages/worker/src/api/index.ts +++ b/packages/worker/src/api/index.ts @@ -55,6 +55,10 @@ const PUBLIC_ENDPOINTS = [ route: "/api/global/users/tenant/:id", method: "GET", }, + { + route: "/api/system/restored", + method: "POST", + }, ] const NO_TENANCY_ENDPOINTS = [ diff --git a/packages/worker/src/api/routes/index.ts b/packages/worker/src/api/routes/index.ts index 67edf5d51b..0c107aae26 100644 --- a/packages/worker/src/api/routes/index.ts +++ b/packages/worker/src/api/routes/index.ts @@ -13,6 +13,7 @@ import selfRoutes from "./global/self" import licenseRoutes from "./global/license" import migrationRoutes from "./system/migrations" import accountRoutes from "./system/accounts" +import restoreRoutes from "./system/restore" let userGroupRoutes = api.groups export const routes = [ @@ -31,4 +32,5 @@ export const routes = [ userGroupRoutes, migrationRoutes, accountRoutes, + restoreRoutes, ] diff --git a/packages/worker/src/api/routes/system/restore.ts b/packages/worker/src/api/routes/system/restore.ts new file mode 100644 index 0000000000..ee4bee091d --- /dev/null +++ b/packages/worker/src/api/routes/system/restore.ts @@ -0,0 +1,8 @@ +import * as controller from "../../controllers/system/restore" +import Router from "@koa/router" + +const router = new Router() + +router.post("/api/system/restored", controller.systemRestored) + +export = router From 3d98acc28a6ceb0126676687444a0d0b69e402bc Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 27 Oct 2022 15:15:08 +0100 Subject: [PATCH 6/6] Minor test rework after improving startup, a lot of mocks weren't being used correctly. --- packages/server/__mocks__/node-fetch.ts | 14 +- packages/server/src/app.ts | 115 +-------------- .../functions/tests/syncQuotas.spec.js | 4 +- .../tests/userEmailViewCasing.spec.js | 10 +- packages/server/src/startup.ts | 138 ++++++++++++++++++ .../src/tests/utilities/TestConfiguration.js | 7 + 6 files changed, 167 insertions(+), 121 deletions(-) create mode 100644 packages/server/src/startup.ts diff --git a/packages/server/__mocks__/node-fetch.ts b/packages/server/__mocks__/node-fetch.ts index dfffa7eb58..0e32c39edd 100644 --- a/packages/server/__mocks__/node-fetch.ts +++ b/packages/server/__mocks__/node-fetch.ts @@ -30,11 +30,21 @@ module FetchMock { } if (url.includes("/api/global")) { - return json({ + const user = { email: "test@test.com", _id: "us_test@test.com", status: "active", - }) + roles: {}, + builder: { + global: false, + }, + admin: { + global: false, + }, + } + return url.endsWith("/users") && opts.method === "GET" + ? json([user]) + : json(user) } // mocked data based on url else if (url.includes("api/apps")) { diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index a5dfcc6e80..9253186498 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -15,30 +15,16 @@ db.init() const Koa = require("koa") const destroyable = require("server-destroy") const koaBody = require("koa-body") -const pino = require("koa-pino-logger") const http = require("http") const api = require("./api") -const eventEmitter = require("./events") const automations = require("./automations/index") const Sentry = require("@sentry/node") -const fileSystem = require("./utilities/fileSystem") -const bullboard = require("./automations/bullboard") const { logAlert } = require("@budibase/backend-core/logging") -const { pinoSettings } = require("@budibase/backend-core") const { Thread } = require("./threads") -const fs = require("fs") import redis from "./utilities/redis" -import * as migrations from "./migrations" -import { events, installation, tenancy } from "@budibase/backend-core" -import { - createAdminUser, - generateApiKey, - getChecklist, -} from "./utilities/workerRequests" -import { watch } from "./watch" +import { events } from "@budibase/backend-core" import { initialise as initialiseWebsockets } from "./websocket" -import sdk from "./sdk" -import * as pro from "@budibase/pro" +import { startup } from "./startup" const app = new Koa() @@ -91,103 +77,8 @@ server.on("close", async () => { } }) -async function initRoutes() { - app.use(pino(pinoSettings())) - - if (!env.isTest()) { - const plugin = await bullboard.init() - app.use(plugin) - } - - app.context.eventEmitter = eventEmitter - app.context.auth = {} - - // api routes - app.use(api.router.routes()) -} - -async function initPro() { - await pro.init({ - backups: { - processing: { - exportAppFn: sdk.backups.exportApp, - importAppFn: sdk.backups.importApp, - statsFn: sdk.backups.calculateBackupStats, - }, - }, - }) -} - module.exports = server.listen(env.PORT || 0, async () => { - console.log(`Budibase running on ${JSON.stringify(server.address())}`) - env._set("PORT", server.address().port) - eventEmitter.emitPort(env.PORT) - fileSystem.init() - await redis.init() - - // run migrations on startup if not done via http - // not recommended in a clustered environment - if (!env.HTTP_MIGRATIONS && !env.isTest()) { - try { - await migrations.migrate() - } catch (e) { - logAlert("Error performing migrations. Exiting.", e) - shutdown() - } - } - - // check and create admin user if required - if ( - env.SELF_HOSTED && - !env.MULTI_TENANCY && - env.BB_ADMIN_USER_EMAIL && - env.BB_ADMIN_USER_PASSWORD - ) { - const checklist = await getChecklist() - if (!checklist?.adminUser?.checked) { - try { - const tenantId = tenancy.getTenantId() - const user = await createAdminUser( - env.BB_ADMIN_USER_EMAIL, - env.BB_ADMIN_USER_PASSWORD, - tenantId - ) - // Need to set up an API key for automated integration tests - if (env.isTest()) { - await generateApiKey(user._id) - } - - console.log( - "Admin account automatically created for", - env.BB_ADMIN_USER_EMAIL - ) - } catch (e) { - logAlert("Error creating initial admin user. Exiting.", e) - shutdown() - } - } - } - - // monitor plugin directory if required - if ( - env.SELF_HOSTED && - !env.MULTI_TENANCY && - env.PLUGINS_DIR && - fs.existsSync(env.PLUGINS_DIR) - ) { - watch() - } - - // check for version updates - await installation.checkInstallVersion() - - // get the references to the queue promises, don't await as - // they will never end, unless the processing stops - let queuePromises = [] - queuePromises.push(automations.init()) - queuePromises.push(initPro()) - // bring routes online as final step once everything ready - await initRoutes() + await startup(app, server) }) const shutdown = () => { diff --git a/packages/server/src/migrations/functions/tests/syncQuotas.spec.js b/packages/server/src/migrations/functions/tests/syncQuotas.spec.js index 76a40e4b28..cdffeea8bd 100644 --- a/packages/server/src/migrations/functions/tests/syncQuotas.spec.js +++ b/packages/server/src/migrations/functions/tests/syncQuotas.spec.js @@ -1,13 +1,11 @@ -const TestConfig = require("../../../tests/utilities/TestConfiguration") - const syncApps = jest.fn() const syncRows = jest.fn() const syncPlugins = jest.fn() - jest.mock("../usageQuotas/syncApps", () => ({ run: syncApps }) ) jest.mock("../usageQuotas/syncRows", () => ({ run: syncRows }) ) jest.mock("../usageQuotas/syncPlugins", () => ({ run: syncPlugins }) ) +const TestConfig = require("../../../tests/utilities/TestConfiguration") const migration = require("../syncQuotas") describe("run", () => { diff --git a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js b/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js index a58f8d9114..3c8894f8e5 100644 --- a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js +++ b/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js @@ -1,11 +1,13 @@ +jest.mock("@budibase/backend-core/db", () => ({ + ...jest.requireActual("@budibase/backend-core/db"), + createNewUserEmailView: jest.fn(), +})) +const coreDb = require("@budibase/backend-core/db") const TestConfig = require("../../../tests/utilities/TestConfiguration") const { TENANT_ID } = require("../../../tests/utilities/structures") const { getGlobalDB, doInTenant } = require("@budibase/backend-core/tenancy") // mock email view creation -const coreDb = require("@budibase/backend-core/db") -const createNewUserEmailView = jest.fn() -coreDb.createNewUserEmailView = createNewUserEmailView const migration = require("../userEmailViewCasing") @@ -22,7 +24,7 @@ describe("run", () => { await doInTenant(TENANT_ID, async () => { const globalDb = getGlobalDB() await migration.run(globalDb) - expect(createNewUserEmailView).toHaveBeenCalledTimes(1) + expect(coreDb.createNewUserEmailView).toHaveBeenCalledTimes(1) }) }) }) diff --git a/packages/server/src/startup.ts b/packages/server/src/startup.ts new file mode 100644 index 0000000000..54d8baf098 --- /dev/null +++ b/packages/server/src/startup.ts @@ -0,0 +1,138 @@ +import * as env from "./environment" +import redis from "./utilities/redis" +import { + createAdminUser, + generateApiKey, + getChecklist, +} from "./utilities/workerRequests" +import { + installation, + pinoSettings, + tenancy, + logging, +} from "@budibase/backend-core" +import fs from "fs" +import { watch } from "./watch" +import automations from "./automations" +import fileSystem from "./utilities/fileSystem" +import eventEmitter from "./events" +import * as migrations from "./migrations" +import bullboard from "./automations/bullboard" +import * as pro from "../../../../budibase-pro/packages/pro" +import api from "./api" +import sdk from "./sdk" +const pino = require("koa-pino-logger") + +let STARTUP_RAN = false + +async function initRoutes(app: any) { + app.use(pino(pinoSettings())) + + if (!env.isTest()) { + const plugin = await bullboard.init() + app.use(plugin) + } + + app.context.eventEmitter = eventEmitter + app.context.auth = {} + + // api routes + app.use(api.router.routes()) +} + +async function initPro() { + await pro.init({ + backups: { + processing: { + exportAppFn: sdk.backups.exportApp, + importAppFn: sdk.backups.importApp, + statsFn: sdk.backups.calculateBackupStats, + }, + }, + }) +} + +function shutdown(server?: any) { + server.close() + server.destroy() +} + +export async function startup(app?: any, server?: any) { + if (STARTUP_RAN) { + return + } + STARTUP_RAN = true + if (server) { + console.log(`Budibase running on ${JSON.stringify(server.address())}`) + env._set("PORT", server.address().port) + } + eventEmitter.emitPort(env.PORT) + fileSystem.init() + await redis.init() + + // run migrations on startup if not done via http + // not recommended in a clustered environment + if (!env.HTTP_MIGRATIONS && !env.isTest()) { + try { + await migrations.migrate() + } catch (e) { + logging.logAlert("Error performing migrations. Exiting.", e) + shutdown() + } + } + + // check and create admin user if required + if ( + env.SELF_HOSTED && + !env.MULTI_TENANCY && + env.BB_ADMIN_USER_EMAIL && + env.BB_ADMIN_USER_PASSWORD + ) { + const checklist = await getChecklist() + if (!checklist?.adminUser?.checked) { + try { + const tenantId = tenancy.getTenantId() + const user = await createAdminUser( + env.BB_ADMIN_USER_EMAIL, + env.BB_ADMIN_USER_PASSWORD, + tenantId + ) + // Need to set up an API key for automated integration tests + if (env.isTest()) { + await generateApiKey(user._id) + } + + console.log( + "Admin account automatically created for", + env.BB_ADMIN_USER_EMAIL + ) + } catch (e) { + logging.logAlert("Error creating initial admin user. Exiting.", e) + shutdown() + } + } + } + + // monitor plugin directory if required + if ( + env.SELF_HOSTED && + !env.MULTI_TENANCY && + env.PLUGINS_DIR && + fs.existsSync(env.PLUGINS_DIR) + ) { + watch() + } + + // check for version updates + await installation.checkInstallVersion() + + // get the references to the queue promises, don't await as + // they will never end, unless the processing stops + let queuePromises = [] + queuePromises.push(automations.init()) + queuePromises.push(initPro()) + if (app) { + // bring routes online as final step once everything ready + await initRoutes(app) + } +} diff --git a/packages/server/src/tests/utilities/TestConfiguration.js b/packages/server/src/tests/utilities/TestConfiguration.js index 097b2eabaf..1b529054f5 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.js +++ b/packages/server/src/tests/utilities/TestConfiguration.js @@ -26,6 +26,7 @@ const context = require("@budibase/backend-core/context") const { generateDevInfoID, SEPARATOR } = require("@budibase/backend-core/db") const { encrypt } = require("@budibase/backend-core/encryption") const { DocumentType, generateUserMetadataID } = require("../../db/utils") +const { startup } = require("../../startup") const GLOBAL_USER_ID = "us_uuid1" const EMAIL = "babs@babs.com" @@ -41,6 +42,9 @@ class TestConfiguration { this.server = require("../../app") // we need the request for logging in, involves cookies, hard to fake this.request = supertest(this.server) + this.started = true + } else { + this.started = false } this.appId = null this.allApps = [] @@ -95,6 +99,9 @@ class TestConfiguration { // use a new id as the name to avoid name collisions async init(appName = newid()) { + if (!this.started) { + await startup() + } this.user = await this.globalUser() this.globalUserId = this.user._id this.userMetadataId = generateUserMetadataID(this.globalUserId)