From 05d4b193006ce32b985b9b6f0b26b5a35fe265af Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 21 Sep 2020 16:45:55 +0100 Subject: [PATCH 01/12] Fix formatter --- .../SetupPanel/ParamInputs/RecordSelector.svelte | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/builder/src/components/workflow/SetupPanel/ParamInputs/RecordSelector.svelte b/packages/builder/src/components/workflow/SetupPanel/ParamInputs/RecordSelector.svelte index a9f9286ef2..1e52ea33a1 100644 --- a/packages/builder/src/components/workflow/SetupPanel/ParamInputs/RecordSelector.svelte +++ b/packages/builder/src/components/workflow/SetupPanel/ParamInputs/RecordSelector.svelte @@ -12,6 +12,10 @@ // Ensure any nullish modelId values get set to empty string so // that the select works $: if (value?.modelId == null) value = { modelId: "" } + + function schemaHasOptions(schema) { + return !!schema.constraints?.inclusion?.length + }
@@ -27,18 +31,15 @@
{#each schemaFields as [field, schema]}
- {#if schema.constraints?.inclusion?.length} + {#if schemaHasOptions(schema)}
{field}
- {#each schema.constraints.inclusion as option} {/each} - {:else if schema.type === "string" || schema.type === "number"} + {:else if schema.type === 'string' || schema.type === 'number'} From 0e4748003e088b314d3cdf92d7ca12cb3e96e5a3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 21 Sep 2020 23:19:45 +0100 Subject: [PATCH 02/12] Updates as per review comments. --- .../server/src/middleware/joi-validator.js | 25 ++++++++++--------- packages/server/src/workflows/thread.js | 4 +-- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/packages/server/src/middleware/joi-validator.js b/packages/server/src/middleware/joi-validator.js index b204d70c84..7ded06fe81 100644 --- a/packages/server/src/middleware/joi-validator.js +++ b/packages/server/src/middleware/joi-validator.js @@ -1,18 +1,19 @@ function validate(schema, property) { // Return a Koa middleware function return (ctx, next) => { - if (schema) { - let params = - ctx[property] != null - ? ctx[property] - : ctx.request[property] != null - ? ctx.request[property] - : null - const { error } = schema.validate(params) - if (error) { - ctx.throw(400, `Invalid ${property} - ${error.message}`) - return - } + if (!schema) { + return next() + } + let params = null + if (ctx[property] != null) { + params = ctx[property] + } else if (ctx.request[property] != null) { + params = ctx.request[property] + } + const { error } = schema.validate(params) + if (error) { + ctx.throw(400, `Invalid ${property} - ${error.message}`) + return } return next() } diff --git a/packages/server/src/workflows/thread.js b/packages/server/src/workflows/thread.js index 7fa0edc7db..1b83a05a39 100644 --- a/packages/server/src/workflows/thread.js +++ b/packages/server/src/workflows/thread.js @@ -10,8 +10,8 @@ function cleanMustache(string) { "]": "", } let regex = new RegExp(/{{[^}}]*}}/g) - let match - while ((match = regex.exec(string)) !== null) { + let matches = [...string.matchAll(regex)] + for (let match of matches) { let baseIdx = string.indexOf(match) for (let key of Object.keys(charToReplace)) { let idxChar = match[0].indexOf(key) From e10a8d1f05b10b6382e88c0072bd595d51be07c7 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 08:58:16 +0100 Subject: [PATCH 03/12] Simplifying review comment changes. --- packages/server/src/workflows/thread.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/workflows/thread.js b/packages/server/src/workflows/thread.js index 1b83a05a39..ab494e327e 100644 --- a/packages/server/src/workflows/thread.js +++ b/packages/server/src/workflows/thread.js @@ -10,8 +10,7 @@ function cleanMustache(string) { "]": "", } let regex = new RegExp(/{{[^}}]*}}/g) - let matches = [...string.matchAll(regex)] - for (let match of matches) { + for (let match of string.matchAll(regex)) { let baseIdx = string.indexOf(match) for (let key of Object.keys(charToReplace)) { let idxChar = match[0].indexOf(key) From 84372c01003e19682bd02ee3dbbdd09799cecc4e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 09:13:19 +0100 Subject: [PATCH 04/12] Adding to test case to retry looking for entries out of the triggered workflow in the case of slow machines. --- .../src/api/routes/tests/workflow.spec.js | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/routes/tests/workflow.spec.js b/packages/server/src/api/routes/tests/workflow.spec.js index 8e0f79a9a6..88498edf33 100644 --- a/packages/server/src/api/routes/tests/workflow.spec.js +++ b/packages/server/src/api/routes/tests/workflow.spec.js @@ -13,6 +13,7 @@ const { const { delay } = require("./testUtils") +const MAX_RETRIES = 4 const TEST_WORKFLOW = { _id: "Test Workflow", name: "My Workflow", @@ -168,11 +169,18 @@ describe("/workflows", () => { expect(res.body.message).toEqual(`Workflow ${workflow._id} has been triggered.`) expect(res.body.workflow.name).toEqual(TEST_WORKFLOW.name) // wait for workflow to complete in background - await delay(500) - let elements = await getAllFromModel(request, app._id, instance._id, model._id) - expect(elements.length).toEqual(1) - expect(elements[0].name).toEqual("Test") - expect(elements[0].description).toEqual("TEST") + for (let tries = 0; tries < MAX_RETRIES; tries++) { + await delay(500) + let elements = await getAllFromModel(request, app._id, instance._id, model._id) + // don't test it unless there are values to test + if (elements.length === 1) { + expect(elements.length).toEqual(1) + expect(elements[0].name).toEqual("Test") + expect(elements[0].description).toEqual("TEST") + return + } + } + throw "Failed to find the records" }) }) From 42d9dd26d9229d682341e0a9ef2c381399d0d1b0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 09:50:16 +0100 Subject: [PATCH 05/12] Confirming suspicions about why a testcase is failing in GH actions only. --- packages/server/src/api/routes/tests/workflow.spec.js | 2 +- packages/server/src/workflows/index.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/workflow.spec.js b/packages/server/src/api/routes/tests/workflow.spec.js index 88498edf33..98627d41f3 100644 --- a/packages/server/src/api/routes/tests/workflow.spec.js +++ b/packages/server/src/api/routes/tests/workflow.spec.js @@ -170,7 +170,6 @@ describe("/workflows", () => { expect(res.body.workflow.name).toEqual(TEST_WORKFLOW.name) // wait for workflow to complete in background for (let tries = 0; tries < MAX_RETRIES; tries++) { - await delay(500) let elements = await getAllFromModel(request, app._id, instance._id, model._id) // don't test it unless there are values to test if (elements.length === 1) { @@ -179,6 +178,7 @@ describe("/workflows", () => { expect(elements[0].description).toEqual("TEST") return } + await delay(500) } throw "Failed to find the records" }) diff --git a/packages/server/src/workflows/index.js b/packages/server/src/workflows/index.js index 81708e8b12..7acc8a1a21 100644 --- a/packages/server/src/workflows/index.js +++ b/packages/server/src/workflows/index.js @@ -23,7 +23,8 @@ function runWorker(job) { module.exports.init = function() { triggers.workflowQueue.process(async job => { if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") { - await runWorker(job) + //await runWorker(job) + await singleThread(job) } else { await singleThread(job) } From cb08efdd5ffa68a1b32b39936c40397cfe4fb506 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 09:54:02 +0100 Subject: [PATCH 06/12] Further testing. --- packages/server/src/workflows/index.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/server/src/workflows/index.js b/packages/server/src/workflows/index.js index 7acc8a1a21..acc2c2871f 100644 --- a/packages/server/src/workflows/index.js +++ b/packages/server/src/workflows/index.js @@ -5,17 +5,17 @@ const singleThread = require("./thread") let workers = workerFarm(require.resolve("./thread")) -function runWorker(job) { - return new Promise((resolve, reject) => { - workers(job, err => { - if (err) { - reject(err) - } else { - resolve() - } - }) - }) -} +// function runWorker(job) { +// return new Promise((resolve, reject) => { +// workers(job, err => { +// if (err) { +// reject(err) +// } else { +// resolve() +// } +// }) +// }) +// } /** * This module is built purely to kick off the worker farm and manage the inputs/outputs From 944d9edbad024f22d8214bb85eea8b13d2bd8d48 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 09:57:05 +0100 Subject: [PATCH 07/12] Adding all fixes for prettier so that test will run. --- packages/server/src/workflows/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/workflows/index.js b/packages/server/src/workflows/index.js index acc2c2871f..596402ef04 100644 --- a/packages/server/src/workflows/index.js +++ b/packages/server/src/workflows/index.js @@ -1,9 +1,9 @@ const triggers = require("./triggers") const environment = require("../environment") -const workerFarm = require("worker-farm") +//const workerFarm = require("worker-farm") const singleThread = require("./thread") -let workers = workerFarm(require.resolve("./thread")) +//let workers = workerFarm(require.resolve("./thread")) // function runWorker(job) { // return new Promise((resolve, reject) => { From 5965a04979b1fc7d67e19513fa7a6ab6daf532d0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 10:12:17 +0100 Subject: [PATCH 08/12] Updating GH actions to use Node 12 LTS to be inline with Node version used in Docker. --- .github/workflows/budibase_ci.yml | 2 +- packages/server/src/workflows/index.js | 30 +++++++++++++------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index f4c17b64ea..e03768a420 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: - node-version: [10.x] + node-version: [12.x] steps: - uses: actions/checkout@v2 diff --git a/packages/server/src/workflows/index.js b/packages/server/src/workflows/index.js index 596402ef04..c4e2ad5cd7 100644 --- a/packages/server/src/workflows/index.js +++ b/packages/server/src/workflows/index.js @@ -1,21 +1,21 @@ const triggers = require("./triggers") const environment = require("../environment") -//const workerFarm = require("worker-farm") +const workerFarm = require("worker-farm") const singleThread = require("./thread") -//let workers = workerFarm(require.resolve("./thread")) +let workers = workerFarm(require.resolve("./thread")) -// function runWorker(job) { -// return new Promise((resolve, reject) => { -// workers(job, err => { -// if (err) { -// reject(err) -// } else { -// resolve() -// } -// }) -// }) -// } +function runWorker(job) { + return new Promise((resolve, reject) => { + workers(job, err => { + if (err) { + reject(err) + } else { + resolve() + } + }) + }) +} /** * This module is built purely to kick off the worker farm and manage the inputs/outputs @@ -23,9 +23,9 @@ const singleThread = require("./thread") module.exports.init = function() { triggers.workflowQueue.process(async job => { if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") { - //await runWorker(job) - await singleThread(job) + await runWorker(job) } else { + console.log("Testing standard thread") await singleThread(job) } }) From fca788f424f0366407fd9e7e4a52d5df07be590a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 10:33:25 +0100 Subject: [PATCH 09/12] One final change, match is likely better than matchAll due to the global nature of the regex and its lack of capture groups currently (in workflow mustache cleansing). --- packages/server/src/workflows/thread.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/workflows/thread.js b/packages/server/src/workflows/thread.js index ab494e327e..afcbd836c8 100644 --- a/packages/server/src/workflows/thread.js +++ b/packages/server/src/workflows/thread.js @@ -10,10 +10,10 @@ function cleanMustache(string) { "]": "", } let regex = new RegExp(/{{[^}}]*}}/g) - for (let match of string.matchAll(regex)) { + for (let match of string.match(regex)) { let baseIdx = string.indexOf(match) for (let key of Object.keys(charToReplace)) { - let idxChar = match[0].indexOf(key) + let idxChar = match.indexOf(key) if (idxChar !== -1) { string = string.slice(baseIdx, baseIdx + idxChar) + From 90027952ea5e579806ea6641261ae81785ee796c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 10:41:56 +0100 Subject: [PATCH 10/12] Quick fix based on test case failure. --- packages/server/src/workflows/thread.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/server/src/workflows/thread.js b/packages/server/src/workflows/thread.js index afcbd836c8..14d0beffc0 100644 --- a/packages/server/src/workflows/thread.js +++ b/packages/server/src/workflows/thread.js @@ -10,7 +10,11 @@ function cleanMustache(string) { "]": "", } let regex = new RegExp(/{{[^}}]*}}/g) - for (let match of string.match(regex)) { + let matches = string.match(regex) + if (matches == null) { + return string + } + for (let match of matches) { let baseIdx = string.indexOf(match) for (let key of Object.keys(charToReplace)) { let idxChar = match.indexOf(key) From 089043cda225896bd5f19cc333b11234d791f930 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 10:49:34 +0100 Subject: [PATCH 11/12] Updating release process to also make use of Node 12.x --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2cfe328312..290f614e20 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: os: [macos-latest, ubuntu-latest, windows-latest] - node-version: [10.x] + node-version: [12.x] steps: - uses: actions/checkout@v2 From cef65a3d9a6bbd7dce5bbc636db22be124cf6fc1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 12:56:48 +0100 Subject: [PATCH 12/12] Removing log statement. --- packages/server/src/automations/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/automations/index.js b/packages/server/src/automations/index.js index b6463e73ec..9685e63f51 100644 --- a/packages/server/src/automations/index.js +++ b/packages/server/src/automations/index.js @@ -25,7 +25,6 @@ module.exports.init = function() { if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") { await runWorker(job) } else { - console.log("Testing standard thread") await singleThread(job) } })