From 2d183e0c6c7552070de51ce102a38011598dc17e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 6 Sep 2022 18:07:18 +0100 Subject: [PATCH 01/11] Adding compaction before each replication to try to reduce size of database, as well as removing the possibility of app metadata being in conflict as per issue #7494. --- packages/backend-core/src/db/Replication.ts | 10 +++++++++ .../server/src/api/controllers/application.ts | 6 +----- .../src/api/controllers/deploy/index.ts | 21 +++++++++++++++---- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index b46f6072be..e0bd3c7a43 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,4 +1,5 @@ import { dangerousGetDB, closeDB } from "." +import { DocumentType } from "./constants" class Replication { source: any @@ -53,6 +54,14 @@ class Replication { return this.replication } + appReplicateOpts() { + return { + filter: (doc: any) => { + return doc._id !== DocumentType.APP_METADATA + }, + } + } + /** * Rollback the target DB back to the state of the source DB */ @@ -60,6 +69,7 @@ class Replication { await this.target.destroy() // Recreate the DB again this.target = dangerousGetDB(this.target.name) + // take the opportunity to remove deleted tombstones await this.replicate() } diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 926fe0ec52..ce3649f082 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -553,11 +553,7 @@ export const sync = async (ctx: any, next: any) => { }) let error try { - await replication.replicate({ - filter: function (doc: any) { - return doc._id !== DocumentType.APP_METADATA - }, - }) + await replication.replicate(replication.appReplicateOpts()) } catch (err) { error = err } finally { diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index d63e167dfb..86718294de 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -15,6 +15,7 @@ import { getAppId, getAppDB, getProdAppDB, + getDevAppDB, } from "@budibase/backend-core/context" import { quotas } from "@budibase/pro" import { events } from "@budibase/backend-core" @@ -110,17 +111,29 @@ async function deployApp(deployment: any) { target: productionAppId, } replication = new Replication(config) - + const devDb = getDevAppDB() + console.log("Compacting development DB") + await devDb.compact() console.log("Replication object created") - await replication.replicate() + await replication.replicate(replication.appReplicateOpts()) console.log("replication complete.. replacing app meta doc") + // app metadata is excluded as it is likely to be in conflict + // replicate the app metadata document manually const db = getProdAppDB() - const appDoc = await db.get(DocumentType.APP_METADATA) + const appDoc = await devDb.get(DocumentType.APP_METADATA) + try { + const prodAppDoc = await db.get(DocumentType.APP_METADATA) + appDoc._rev = prodAppDoc._rev + } catch (err) { + // ignore the error - doesn't exist + } + // switch to production app ID deployment.appUrl = appDoc.url - appDoc.appId = productionAppId appDoc.instance._id = productionAppId + // remove automation errors if they exist + delete appDoc.automationErrors await db.put(appDoc) await appCache.invalidateAppMetadata(productionAppId) console.log("New app doc written successfully.") From 3b64a8f94bf0f228919c8704b8662f3de72469d2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 7 Sep 2022 11:55:03 +0100 Subject: [PATCH 02/11] Adding ellipsis test case. --- packages/string-templates/test/helpers.spec.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/string-templates/test/helpers.spec.js b/packages/string-templates/test/helpers.spec.js index 17e6876bba..1969410993 100644 --- a/packages/string-templates/test/helpers.spec.js +++ b/packages/string-templates/test/helpers.spec.js @@ -272,6 +272,14 @@ describe("test the string helpers", () => { ) expect(output).toBe("Hi!") }) + + it("should allow use of the ellipsis helper", async () => { + const output = await processString( + "{{ ellipsis \"adfasdfasdfasf\" 7 }}", + {}, + ) + expect(output).toBe("adfasdf…") + }) }) describe("test the comparison helpers", () => { From 86c8618e8f220ee65f80dc6ea05d1082355e92f3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 7 Sep 2022 17:05:17 +0100 Subject: [PATCH 03/11] Fix for #7431 - reboot didn't work at all previously which is why apps couldn't be published with it enabled, this is now a self host only feature, I've removed the ability to enable a reboot cron in the Cloud and it will not run the lookup/execution. --- packages/backend-core/src/db/utils.ts | 11 +++- .../automation/SetupPanel/CronBuilder.svelte | 16 ++++-- .../src/api/controllers/deploy/index.ts | 2 +- packages/server/src/automations/index.js | 9 ++-- packages/server/src/automations/triggers.js | 54 +++++++++++++++---- packages/server/src/automations/utils.ts | 30 +++++++---- packages/server/src/threads/automation.ts | 6 +++ .../types/src/documents/app/automation.ts | 4 ++ 8 files changed, 103 insertions(+), 29 deletions(-) diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 321ebd7f58..4926a60150 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -254,7 +254,16 @@ export async function getAllApps({ dev, all, idsOnly, efficient }: any = {}) { return false }) if (idsOnly) { - return appDbNames + const devAppIds = appDbNames.filter(appId => isDevAppID(appId)) + const prodAppIds = appDbNames.filter(appId => !isDevAppID(appId)) + switch (dev) { + case true: + return devAppIds + case false: + return prodAppIds + default: + return appDbNames + } } const appPromises = appDbNames.map((app: any) => // skip setup otherwise databases could be re-created diff --git a/packages/builder/src/components/automation/SetupPanel/CronBuilder.svelte b/packages/builder/src/components/automation/SetupPanel/CronBuilder.svelte index 93b8394b49..f2da863424 100644 --- a/packages/builder/src/components/automation/SetupPanel/CronBuilder.svelte +++ b/packages/builder/src/components/automation/SetupPanel/CronBuilder.svelte @@ -1,6 +1,7 @@
diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index 86718294de..2ac6c8a15c 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -125,7 +125,7 @@ async function deployApp(deployment: any) { const prodAppDoc = await db.get(DocumentType.APP_METADATA) appDoc._rev = prodAppDoc._rev } catch (err) { - // ignore the error - doesn't exist + delete appDoc._rev } // switch to production app ID diff --git a/packages/server/src/automations/index.js b/packages/server/src/automations/index.js index 9c23e35d1c..2baa868890 100644 --- a/packages/server/src/automations/index.js +++ b/packages/server/src/automations/index.js @@ -1,16 +1,19 @@ const { processEvent } = require("./utils") const { queue, shutdown } = require("./bullboard") -const { TRIGGER_DEFINITIONS } = require("./triggers") +const { TRIGGER_DEFINITIONS, rebootTrigger } = require("./triggers") const { ACTION_DEFINITIONS } = require("./actions") /** * This module is built purely to kick off the worker farm and manage the inputs/outputs */ -exports.init = function () { +exports.init = async function () { // this promise will not complete - return queue.process(async job => { + const promise = queue.process(async job => { await processEvent(job) }) + // on init we need to trigger any reboot automations + await rebootTrigger() + return promise } exports.getQueues = () => { diff --git a/packages/server/src/automations/triggers.js b/packages/server/src/automations/triggers.js index 216f24be02..395390113a 100644 --- a/packages/server/src/automations/triggers.js +++ b/packages/server/src/automations/triggers.js @@ -9,6 +9,7 @@ const { checkTestFlag } = require("../utilities/redis") const utils = require("./utils") const env = require("../environment") const { doInAppContext, getAppDB } = require("@budibase/backend-core/context") +const { getAllApps } = require("@budibase/backend-core/db") const TRIGGER_DEFINITIONS = definitions const JOB_OPTS = { @@ -16,24 +17,27 @@ const JOB_OPTS = { removeOnFail: true, } +async function getAllAutomations() { + const db = getAppDB() + let automations = await db.allDocs( + getAutomationParams(null, { include_docs: true }) + ) + return automations.rows.map(row => row.doc) +} + async function queueRelevantRowAutomations(event, eventType) { if (event.appId == null) { throw `No appId specified for ${eventType} - check event emitters.` } doInAppContext(event.appId, async () => { - const db = getAppDB() - let automations = await db.allDocs( - getAutomationParams(null, { include_docs: true }) - ) + let automations = await getAllAutomations() // filter down to the correct event type - automations = automations.rows - .map(automation => automation.doc) - .filter(automation => { - const trigger = automation.definition.trigger - return trigger && trigger.event === eventType - }) + automations = automations.filter(automation => { + const trigger = automation.definition.trigger + return trigger && trigger.event === eventType + }) for (let automation of automations) { let automationDef = automation.definition @@ -110,4 +114,34 @@ exports.externalTrigger = async function ( } } +exports.rebootTrigger = async () => { + // reboot cron option is only available on the main thread at + // startup and only usable in self host + if (env.isInThread() || !env.SELF_HOSTED) { + return + } + // iterate through all production apps, find the reboot crons + // and trigger events for them + const appIds = await getAllApps({ dev: false, idsOnly: true }) + for (let prodAppId of appIds) { + await doInAppContext(prodAppId, async () => { + let automations = await getAllAutomations() + let rebootEvents = [] + for (let automation of automations) { + if (utils.isRebootTrigger(automation)) { + const job = { + automation, + event: { + appId: prodAppId, + timestamp: Date.now(), + }, + } + rebootEvents.push(queue.add(job, JOB_OPTS)) + } + } + await Promise.all(rebootEvents) + }) + } +} + exports.TRIGGER_DEFINITIONS = TRIGGER_DEFINITIONS diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index e0979ac0d9..3093f147dc 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -17,6 +17,7 @@ import { tenancy } from "@budibase/backend-core" import { quotas } from "@budibase/pro" import { Automation } from "@budibase/types" +const REBOOT_CRON = "@reboot" const WH_STEP_ID = definitions.WEBHOOK.stepId const CRON_STEP_ID = definitions.CRON.stepId const Runner = new Thread(ThreadType.AUTOMATION) @@ -109,22 +110,33 @@ export async function clearMetadata() { await db.bulkDocs(automationMetadata) } +export function isCronTrigger(auto: Automation) { + return ( + auto && + auto.definition.trigger && + auto.definition.trigger.stepId === CRON_STEP_ID + ) +} + +export function isRebootTrigger(auto: Automation) { + const trigger = auto ? auto.definition.trigger : null + return isCronTrigger(auto) && trigger?.inputs.cron === REBOOT_CRON +} + /** * This function handles checking of any cron jobs that need to be enabled/updated. * @param {string} appId The ID of the app in which we are checking for webhooks * @param {object|undefined} automation The automation object to be updated. */ -export async function enableCronTrigger(appId: any, automation: any) { +export async function enableCronTrigger(appId: any, automation: Automation) { const trigger = automation ? automation.definition.trigger : null - function isCronTrigger(auto: any) { - return ( - auto && - auto.definition.trigger && - auto.definition.trigger.stepId === CRON_STEP_ID - ) - } + // need to create cron job - if (isCronTrigger(automation) && trigger?.inputs.cron) { + if ( + isCronTrigger(automation) && + !isRebootTrigger(automation) && + trigger?.inputs.cron + ) { // make a job id rather than letting Bull decide, makes it easier to handle on way out const jobId = `${appId}_cron_${newid()}` const job: any = await queue.add( diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 3136155869..f64552a92f 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -458,6 +458,9 @@ class Orchestrator { export function execute(input: AutomationEvent, callback: WorkerCallback) { const appId = input.data.event.appId + if (!appId) { + throw new Error("Unable to execute, event doesn't contain app ID.") + } doInAppContext(appId, async () => { const automationOrchestrator = new Orchestrator( input.data.automation, @@ -475,6 +478,9 @@ export function execute(input: AutomationEvent, callback: WorkerCallback) { export const removeStalled = async (input: AutomationEvent) => { const appId = input.data.event.appId + if (!appId) { + throw new Error("Unable to execute, event doesn't contain app ID.") + } await doInAppContext(appId, async () => { const automationOrchestrator = new Orchestrator( input.data.automation, diff --git a/packages/types/src/documents/app/automation.ts b/packages/types/src/documents/app/automation.ts index 50562461e4..a038e73d11 100644 --- a/packages/types/src/documents/app/automation.ts +++ b/packages/types/src/documents/app/automation.ts @@ -25,6 +25,10 @@ export interface AutomationStep { export interface AutomationTrigger { id: string stepId: string + inputs: { + [key: string]: any + } + cronJobId?: string } export enum AutomationStatus { From 2fb96b29c9cba715298c986b79d91efa93f580f4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 7 Sep 2022 17:31:15 +0100 Subject: [PATCH 04/11] Encoding query string URI parameters for REST requests - #7683. --- packages/builder/src/helpers/data/utils.js | 2 +- packages/server/src/integrations/rest.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/helpers/data/utils.js b/packages/builder/src/helpers/data/utils.js index 647c2be33e..cd6a8cf481 100644 --- a/packages/builder/src/helpers/data/utils.js +++ b/packages/builder/src/helpers/data/utils.js @@ -46,7 +46,7 @@ export function buildQueryString(obj) { if (str !== "") { str += "&" } - str += `${key}=${value || ""}` + str += `${key}=${encodeURIComponent(value || "")}` } } return str diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index 284d2a921a..59eb1e9941 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -215,7 +215,7 @@ module RestModule { } } - const main = `${path}?${queryString}` + const main = `${path}?${encodeURIComponent(queryString)}` let complete = main if (this.config.url && !main.startsWith("http")) { complete = !this.config.url ? main : `${this.config.url}/${main}` From 61fdabc382a7155b5ced03fc025a4e81478a6f08 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 7 Sep 2022 17:45:14 +0100 Subject: [PATCH 05/11] Adding onTop attribute to tabs which allows setting the z-index for the container - #7679. --- packages/bbui/src/Tabs/Tabs.svelte | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/bbui/src/Tabs/Tabs.svelte b/packages/bbui/src/Tabs/Tabs.svelte index 74edc9cd02..7184aedbaf 100644 --- a/packages/bbui/src/Tabs/Tabs.svelte +++ b/packages/bbui/src/Tabs/Tabs.svelte @@ -10,6 +10,7 @@ export let noHorizPadding = false export let quiet = false export let emphasized = false + export let onTop = false export let size = "M" let thisSelected = undefined @@ -75,6 +76,7 @@ bind:this={container} class:spectrum-Tabs--quiet={quiet} class:noHorizPadding + class:onTop class:spectrum-Tabs--vertical={vertical} class:spectrum-Tabs--horizontal={!vertical} class="spectrum-Tabs spectrum-Tabs--size{size}" @@ -122,4 +124,7 @@ .noPadding { margin: 0; } + .onTop { + z-index: 100; + } From 8e56a7a4d9d356ec27bd95a0cf1d87d163425028 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 7 Sep 2022 17:57:02 +0100 Subject: [PATCH 06/11] Fixing an issue with external tables containing time only fields. --- packages/server/src/utilities/rowProcessor/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utilities/rowProcessor/utils.js b/packages/server/src/utilities/rowProcessor/utils.js index c80dae497c..d659cb6822 100644 --- a/packages/server/src/utilities/rowProcessor/utils.js +++ b/packages/server/src/utilities/rowProcessor/utils.js @@ -76,7 +76,7 @@ exports.processDates = (table, rows) => { if (schema.type !== FieldTypes.DATETIME) { continue } - if (!schema.ignoreTimezones) { + if (!schema.timeOnly && !schema.ignoreTimezones) { datesWithTZ.push(column) } } From 7c9d0594f8e37c1cbd102ccf86ae8c0abd9c3faf Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 7 Sep 2022 18:30:17 +0100 Subject: [PATCH 07/11] Fixing #6980 - fixing choice of relational foreign key field name when working with fields named differently to the primary key. --- packages/server/src/api/controllers/row/ExternalRequest.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index a1aecdb0f2..dd5778b9d9 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -534,7 +534,7 @@ module External { }) // this is the response from knex if no rows found const rows = !response[0].read ? response : [] - const storeTo = isMany ? field.throughFrom || linkPrimaryKey : manyKey + const storeTo = isMany ? field.throughFrom || linkPrimaryKey : fieldName related[storeTo] = { rows, isMany, tableId } } return related From 932f119fce48fd193198520aa5c9dbc7e220b6b9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 7 Sep 2022 19:15:05 +0100 Subject: [PATCH 08/11] Updating params in the REST interface so that they can be used in and out of the URL - meaning that updating in one place affects the other. Reduces a bit of the confusing UX around this - discussed in #7683. --- .../bindings/DrawerBindableInput.svelte | 10 ++- .../integration/KeyValueBuilder.svelte | 9 ++- .../rest/[query]/index.svelte | 66 +++++++++++++------ 3 files changed, 62 insertions(+), 23 deletions(-) diff --git a/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte b/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte index b8d418c62b..22d322985d 100644 --- a/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte +++ b/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte @@ -23,6 +23,7 @@ const dispatch = createEventDispatcher() let bindingDrawer let valid = true + let currentVal = value $: readableValue = runtimeToReadableBinding(bindings, value) $: tempValue = readableValue @@ -30,11 +31,17 @@ const saveBinding = () => { onChange(tempValue) + onBlur() bindingDrawer.hide() } const onChange = value => { - dispatch("change", readableToRuntimeBinding(bindings, value)) + currentVal = readableToRuntimeBinding(bindings, value) + dispatch("change", currentVal) + } + + const onBlur = () => { + dispatch("blur", currentVal) } @@ -45,6 +52,7 @@ readonly={isJS} value={isJS ? "(JavaScript function)" : readableValue} on:change={event => onChange(event.detail)} + on:blur={onBlur} {placeholder} {updateOnChange} /> diff --git a/packages/builder/src/components/integration/KeyValueBuilder.svelte b/packages/builder/src/components/integration/KeyValueBuilder.svelte index 4ffb380aa4..28db6b61c6 100644 --- a/packages/builder/src/components/integration/KeyValueBuilder.svelte +++ b/packages/builder/src/components/integration/KeyValueBuilder.svelte @@ -107,7 +107,7 @@ placeholder={keyPlaceholder} readonly={readOnly} bind:value={field.name} - on:change={changed} + on:blur={changed} /> {#if options} +