From ce61b979f4d239cd19caa069991c3bc96d1f8081 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 21 Sep 2020 16:45:55 +0100 Subject: [PATCH 01/12] Fix formatter --- .../SetupPanel/ParamInputs/RecordSelector.svelte | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/builder/src/components/workflow/SetupPanel/ParamInputs/RecordSelector.svelte b/packages/builder/src/components/workflow/SetupPanel/ParamInputs/RecordSelector.svelte index a9f9286ef2..1e52ea33a1 100644 --- a/packages/builder/src/components/workflow/SetupPanel/ParamInputs/RecordSelector.svelte +++ b/packages/builder/src/components/workflow/SetupPanel/ParamInputs/RecordSelector.svelte @@ -12,6 +12,10 @@ // Ensure any nullish modelId values get set to empty string so // that the select works $: if (value?.modelId == null) value = { modelId: "" } + + function schemaHasOptions(schema) { + return !!schema.constraints?.inclusion?.length + }
@@ -27,18 +31,15 @@
{#each schemaFields as [field, schema]}
- {#if schema.constraints?.inclusion?.length} + {#if schemaHasOptions(schema)}
{field}
- {#each schema.constraints.inclusion as option} {/each} - {:else if schema.type === "string" || schema.type === "number"} + {:else if schema.type === 'string' || schema.type === 'number'} From 39dbf78359d7df7ec28b8f44c17f9c489c2c4185 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 21 Sep 2020 23:19:45 +0100 Subject: [PATCH 02/12] Updates as per review comments. --- .../server/src/middleware/joi-validator.js | 25 ++++++++++--------- packages/server/src/workflows/thread.js | 4 +-- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/packages/server/src/middleware/joi-validator.js b/packages/server/src/middleware/joi-validator.js index b204d70c84..7ded06fe81 100644 --- a/packages/server/src/middleware/joi-validator.js +++ b/packages/server/src/middleware/joi-validator.js @@ -1,18 +1,19 @@ function validate(schema, property) { // Return a Koa middleware function return (ctx, next) => { - if (schema) { - let params = - ctx[property] != null - ? ctx[property] - : ctx.request[property] != null - ? ctx.request[property] - : null - const { error } = schema.validate(params) - if (error) { - ctx.throw(400, `Invalid ${property} - ${error.message}`) - return - } + if (!schema) { + return next() + } + let params = null + if (ctx[property] != null) { + params = ctx[property] + } else if (ctx.request[property] != null) { + params = ctx.request[property] + } + const { error } = schema.validate(params) + if (error) { + ctx.throw(400, `Invalid ${property} - ${error.message}`) + return } return next() } diff --git a/packages/server/src/workflows/thread.js b/packages/server/src/workflows/thread.js index 7fa0edc7db..1b83a05a39 100644 --- a/packages/server/src/workflows/thread.js +++ b/packages/server/src/workflows/thread.js @@ -10,8 +10,8 @@ function cleanMustache(string) { "]": "", } let regex = new RegExp(/{{[^}}]*}}/g) - let match - while ((match = regex.exec(string)) !== null) { + let matches = [...string.matchAll(regex)] + for (let match of matches) { let baseIdx = string.indexOf(match) for (let key of Object.keys(charToReplace)) { let idxChar = match[0].indexOf(key) From 1a1d09862a54df2e31ea3388c1dcbfab5c17ddde Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 08:58:16 +0100 Subject: [PATCH 03/12] Simplifying review comment changes. --- packages/server/src/workflows/thread.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/workflows/thread.js b/packages/server/src/workflows/thread.js index 1b83a05a39..ab494e327e 100644 --- a/packages/server/src/workflows/thread.js +++ b/packages/server/src/workflows/thread.js @@ -10,8 +10,7 @@ function cleanMustache(string) { "]": "", } let regex = new RegExp(/{{[^}}]*}}/g) - let matches = [...string.matchAll(regex)] - for (let match of matches) { + for (let match of string.matchAll(regex)) { let baseIdx = string.indexOf(match) for (let key of Object.keys(charToReplace)) { let idxChar = match[0].indexOf(key) From cc35099ebd3f31f081438cf612adeb0ac62a6ed5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 09:13:19 +0100 Subject: [PATCH 04/12] Adding to test case to retry looking for entries out of the triggered workflow in the case of slow machines. --- .../src/api/routes/tests/workflow.spec.js | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/routes/tests/workflow.spec.js b/packages/server/src/api/routes/tests/workflow.spec.js index 8e0f79a9a6..88498edf33 100644 --- a/packages/server/src/api/routes/tests/workflow.spec.js +++ b/packages/server/src/api/routes/tests/workflow.spec.js @@ -13,6 +13,7 @@ const { const { delay } = require("./testUtils") +const MAX_RETRIES = 4 const TEST_WORKFLOW = { _id: "Test Workflow", name: "My Workflow", @@ -168,11 +169,18 @@ describe("/workflows", () => { expect(res.body.message).toEqual(`Workflow ${workflow._id} has been triggered.`) expect(res.body.workflow.name).toEqual(TEST_WORKFLOW.name) // wait for workflow to complete in background - await delay(500) - let elements = await getAllFromModel(request, app._id, instance._id, model._id) - expect(elements.length).toEqual(1) - expect(elements[0].name).toEqual("Test") - expect(elements[0].description).toEqual("TEST") + for (let tries = 0; tries < MAX_RETRIES; tries++) { + await delay(500) + let elements = await getAllFromModel(request, app._id, instance._id, model._id) + // don't test it unless there are values to test + if (elements.length === 1) { + expect(elements.length).toEqual(1) + expect(elements[0].name).toEqual("Test") + expect(elements[0].description).toEqual("TEST") + return + } + } + throw "Failed to find the records" }) }) From ab0ee7555bdb05a73c388ba1b791c16674c5aca9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 09:50:16 +0100 Subject: [PATCH 05/12] Confirming suspicions about why a testcase is failing in GH actions only. --- packages/server/src/api/routes/tests/workflow.spec.js | 2 +- packages/server/src/workflows/index.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/workflow.spec.js b/packages/server/src/api/routes/tests/workflow.spec.js index 88498edf33..98627d41f3 100644 --- a/packages/server/src/api/routes/tests/workflow.spec.js +++ b/packages/server/src/api/routes/tests/workflow.spec.js @@ -170,7 +170,6 @@ describe("/workflows", () => { expect(res.body.workflow.name).toEqual(TEST_WORKFLOW.name) // wait for workflow to complete in background for (let tries = 0; tries < MAX_RETRIES; tries++) { - await delay(500) let elements = await getAllFromModel(request, app._id, instance._id, model._id) // don't test it unless there are values to test if (elements.length === 1) { @@ -179,6 +178,7 @@ describe("/workflows", () => { expect(elements[0].description).toEqual("TEST") return } + await delay(500) } throw "Failed to find the records" }) diff --git a/packages/server/src/workflows/index.js b/packages/server/src/workflows/index.js index 81708e8b12..7acc8a1a21 100644 --- a/packages/server/src/workflows/index.js +++ b/packages/server/src/workflows/index.js @@ -23,7 +23,8 @@ function runWorker(job) { module.exports.init = function() { triggers.workflowQueue.process(async job => { if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") { - await runWorker(job) + //await runWorker(job) + await singleThread(job) } else { await singleThread(job) } From c18ac80fe26eb7121066873477a575ffa77d4712 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 09:54:02 +0100 Subject: [PATCH 06/12] Further testing. --- packages/server/src/workflows/index.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/server/src/workflows/index.js b/packages/server/src/workflows/index.js index 7acc8a1a21..acc2c2871f 100644 --- a/packages/server/src/workflows/index.js +++ b/packages/server/src/workflows/index.js @@ -5,17 +5,17 @@ const singleThread = require("./thread") let workers = workerFarm(require.resolve("./thread")) -function runWorker(job) { - return new Promise((resolve, reject) => { - workers(job, err => { - if (err) { - reject(err) - } else { - resolve() - } - }) - }) -} +// function runWorker(job) { +// return new Promise((resolve, reject) => { +// workers(job, err => { +// if (err) { +// reject(err) +// } else { +// resolve() +// } +// }) +// }) +// } /** * This module is built purely to kick off the worker farm and manage the inputs/outputs From 3be61180a8b3d662074e4a7ff74b5813ebeaa751 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 09:57:05 +0100 Subject: [PATCH 07/12] Adding all fixes for prettier so that test will run. --- packages/server/src/workflows/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/workflows/index.js b/packages/server/src/workflows/index.js index acc2c2871f..596402ef04 100644 --- a/packages/server/src/workflows/index.js +++ b/packages/server/src/workflows/index.js @@ -1,9 +1,9 @@ const triggers = require("./triggers") const environment = require("../environment") -const workerFarm = require("worker-farm") +//const workerFarm = require("worker-farm") const singleThread = require("./thread") -let workers = workerFarm(require.resolve("./thread")) +//let workers = workerFarm(require.resolve("./thread")) // function runWorker(job) { // return new Promise((resolve, reject) => { From b5f42384e3c4cb60afb51ce971e2b6b012bb9603 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 10:12:17 +0100 Subject: [PATCH 08/12] Updating GH actions to use Node 12 LTS to be inline with Node version used in Docker. --- .github/workflows/budibase_ci.yml | 2 +- packages/server/src/workflows/index.js | 30 +++++++++++++------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index f4c17b64ea..e03768a420 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: - node-version: [10.x] + node-version: [12.x] steps: - uses: actions/checkout@v2 diff --git a/packages/server/src/workflows/index.js b/packages/server/src/workflows/index.js index 596402ef04..c4e2ad5cd7 100644 --- a/packages/server/src/workflows/index.js +++ b/packages/server/src/workflows/index.js @@ -1,21 +1,21 @@ const triggers = require("./triggers") const environment = require("../environment") -//const workerFarm = require("worker-farm") +const workerFarm = require("worker-farm") const singleThread = require("./thread") -//let workers = workerFarm(require.resolve("./thread")) +let workers = workerFarm(require.resolve("./thread")) -// function runWorker(job) { -// return new Promise((resolve, reject) => { -// workers(job, err => { -// if (err) { -// reject(err) -// } else { -// resolve() -// } -// }) -// }) -// } +function runWorker(job) { + return new Promise((resolve, reject) => { + workers(job, err => { + if (err) { + reject(err) + } else { + resolve() + } + }) + }) +} /** * This module is built purely to kick off the worker farm and manage the inputs/outputs @@ -23,9 +23,9 @@ const singleThread = require("./thread") module.exports.init = function() { triggers.workflowQueue.process(async job => { if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") { - //await runWorker(job) - await singleThread(job) + await runWorker(job) } else { + console.log("Testing standard thread") await singleThread(job) } }) From c0442b25e1e3f50fe503b77db560b6fee7139e8d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 10:33:25 +0100 Subject: [PATCH 09/12] One final change, match is likely better than matchAll due to the global nature of the regex and its lack of capture groups currently (in workflow mustache cleansing). --- packages/server/src/workflows/thread.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/workflows/thread.js b/packages/server/src/workflows/thread.js index ab494e327e..afcbd836c8 100644 --- a/packages/server/src/workflows/thread.js +++ b/packages/server/src/workflows/thread.js @@ -10,10 +10,10 @@ function cleanMustache(string) { "]": "", } let regex = new RegExp(/{{[^}}]*}}/g) - for (let match of string.matchAll(regex)) { + for (let match of string.match(regex)) { let baseIdx = string.indexOf(match) for (let key of Object.keys(charToReplace)) { - let idxChar = match[0].indexOf(key) + let idxChar = match.indexOf(key) if (idxChar !== -1) { string = string.slice(baseIdx, baseIdx + idxChar) + From 0501acc82356f28e662c5ea0f884357f1fafdf48 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 10:41:56 +0100 Subject: [PATCH 10/12] Quick fix based on test case failure. --- packages/server/src/workflows/thread.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/server/src/workflows/thread.js b/packages/server/src/workflows/thread.js index afcbd836c8..14d0beffc0 100644 --- a/packages/server/src/workflows/thread.js +++ b/packages/server/src/workflows/thread.js @@ -10,7 +10,11 @@ function cleanMustache(string) { "]": "", } let regex = new RegExp(/{{[^}}]*}}/g) - for (let match of string.match(regex)) { + let matches = string.match(regex) + if (matches == null) { + return string + } + for (let match of matches) { let baseIdx = string.indexOf(match) for (let key of Object.keys(charToReplace)) { let idxChar = match.indexOf(key) From 9840585b907d218bc88e4de0e14bc77a72c27607 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 10:49:34 +0100 Subject: [PATCH 11/12] Updating release process to also make use of Node 12.x --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2cfe328312..290f614e20 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: os: [macos-latest, ubuntu-latest, windows-latest] - node-version: [10.x] + node-version: [12.x] steps: - uses: actions/checkout@v2 From 9c5b4b42119c120d0bfc52fe9b9fa156b4e3b1f5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 22 Sep 2020 12:56:48 +0100 Subject: [PATCH 12/12] Removing log statement. --- packages/server/src/automations/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/automations/index.js b/packages/server/src/automations/index.js index b6463e73ec..9685e63f51 100644 --- a/packages/server/src/automations/index.js +++ b/packages/server/src/automations/index.js @@ -25,7 +25,6 @@ module.exports.init = function() { if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") { await runWorker(job) } else { - console.log("Testing standard thread") await singleThread(job) } })