From 2ae262a0805e7916cb2206a7464b4fc086719cf6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 11 Jul 2022 12:54:45 +0100 Subject: [PATCH 1/5] Quick fix for string based searches, attempting to convert doc type if they differ. --- packages/frontend-core/src/utils/lucene.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/frontend-core/src/utils/lucene.js b/packages/frontend-core/src/utils/lucene.js index 1001ec26a8..d02d6e60a3 100644 --- a/packages/frontend-core/src/utils/lucene.js +++ b/packages/frontend-core/src/utils/lucene.js @@ -170,7 +170,10 @@ export const runLuceneQuery = (docs, query) => { const filters = Object.entries(query[type] || {}) for (let i = 0; i < filters.length; i++) { const [key, testValue] = filters[i] - const docValue = Helpers.deepGet(doc, key) + let docValue = Helpers.deepGet(doc, key) + if (typeof docValue !== "string" && typeof testValue === "string") { + docValue = docValue.toString() + } if (failFn(docValue, testValue)) { return false } From 22d2e41f7b2162ac3c49ec37dbdd9ccc471c8854 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 13 Jul 2022 10:53:50 +0100 Subject: [PATCH 2/5] Allowing query backend to attempt to convert types in returned JSON to build a better base schema. --- .../builder/src/stores/backend/queries.js | 4 +-- packages/frontend-core/src/utils/lucene.js | 5 +-- .../server/src/api/controllers/query/index.ts | 31 +++++++++++++++++-- .../server/src/api/routes/tests/query.spec.js | 23 +++++++++++--- 4 files changed, 51 insertions(+), 12 deletions(-) diff --git a/packages/builder/src/stores/backend/queries.js b/packages/builder/src/stores/backend/queries.js index e5b79fb165..bb456ce405 100644 --- a/packages/builder/src/stores/backend/queries.js +++ b/packages/builder/src/stores/backend/queries.js @@ -90,8 +90,8 @@ export function createQueriesStore() { // Assume all the fields are strings and create a basic schema from the // unique fields returned by the server const schema = {} - for (let field of result.schemaFields) { - schema[field] = "string" + for (let [field, type] of Object.entries(result.schemaFields)) { + schema[field] = type || "string" } return { ...result, schema, rows: result.rows || [] } }, diff --git a/packages/frontend-core/src/utils/lucene.js b/packages/frontend-core/src/utils/lucene.js index d02d6e60a3..1001ec26a8 100644 --- a/packages/frontend-core/src/utils/lucene.js +++ b/packages/frontend-core/src/utils/lucene.js @@ -170,10 +170,7 @@ export const runLuceneQuery = (docs, query) => { const filters = Object.entries(query[type] || {}) for (let i = 0; i < filters.length; i++) { const [key, testValue] = filters[i] - let docValue = Helpers.deepGet(doc, key) - if (typeof docValue !== "string" && typeof testValue === "string") { - docValue = docValue.toString() - } + const docValue = Helpers.deepGet(doc, key) if (failFn(docValue, testValue)) { return false } diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 8dcf570343..412bda62d3 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -1,5 +1,5 @@ import { generateQueryID, getQueryParams, isProdAppID } from "../../../db/utils" -import { BaseQueryVerbs } from "../../../constants" +import { BaseQueryVerbs, FieldTypes } from "../../../constants" import { Thread, ThreadType } from "../../../threads" import { save as saveDatasource } from "../datasource" import { RestImporter } from "./import" @@ -154,10 +154,37 @@ export async function preview(ctx: any) { }, }) const { rows, keys, info, extra } = await quotas.addQuery(runFn) + const schemaFields: any = {} + if (rows?.length > 0) { + for (let key of [...new Set(keys)] as string[]) { + const field = rows[0][key] + let type = typeof field, + fieldType = FieldTypes.STRING + if (field) + switch (type) { + case "boolean": + schemaFields[key] = FieldTypes.BOOLEAN + break + case "object": + if (field instanceof Date) { + fieldType = FieldTypes.DATETIME + } else if (Array.isArray(field)) { + fieldType = FieldTypes.ARRAY + } else { + fieldType = FieldTypes.JSON + } + break + case "number": + fieldType = FieldTypes.NUMBER + break + } + schemaFields[key] = fieldType + } + } await events.query.previewed(datasource, query) ctx.body = { rows, - schemaFields: [...new Set(keys)], + schemaFields, info, extra, } diff --git a/packages/server/src/api/routes/tests/query.spec.js b/packages/server/src/api/routes/tests/query.spec.js index 00cae3a9c9..45f69de5e9 100644 --- a/packages/server/src/api/routes/tests/query.spec.js +++ b/packages/server/src/api/routes/tests/query.spec.js @@ -215,7 +215,10 @@ describe("/queries", () => { .expect("Content-Type", /json/) .expect(200) // these responses come from the mock - expect(res.body.schemaFields).toEqual(["a", "b"]) + expect(res.body.schemaFields).toEqual({ + "a": "string", + "b": "number", + }) expect(res.body.rows.length).toEqual(1) expect(events.query.previewed).toBeCalledTimes(1) expect(events.query.previewed).toBeCalledWith(datasource, query) @@ -289,7 +292,11 @@ describe("/queries", () => { queryString: "test={{ variable2 }}", }) // these responses come from the mock - expect(res.body.schemaFields).toEqual(["url", "opts", "value"]) + expect(res.body.schemaFields).toEqual({ + "opts": "json", + "url": "string", + "value": "string", + }) expect(res.body.rows[0].url).toEqual("http://www.google.com?test=1") }) @@ -299,7 +306,11 @@ describe("/queries", () => { path: "www.google.com", queryString: "test={{ variable3 }}", }) - expect(res.body.schemaFields).toEqual(["url", "opts", "value"]) + expect(res.body.schemaFields).toEqual({ + "opts": "json", + "url": "string", + "value": "string" + }) expect(res.body.rows[0].url).toContain("doctype html") }) @@ -318,7 +329,11 @@ describe("/queries", () => { path: "www.failonce.com", queryString: "test={{ variable3 }}", }) - expect(res.body.schemaFields).toEqual(["fails", "url", "opts"]) + expect(res.body.schemaFields).toEqual({ + "fails": "number", + "opts": "json", + "url": "string" + }) expect(res.body.rows[0].fails).toEqual(1) }) From eadd37e22dbf83b3926ccfa9043900848587d2a7 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 13 Jul 2022 11:57:14 +0100 Subject: [PATCH 3/5] Fix for #6681 - email save validation not working. --- packages/server/yarn.lock | 30 +++++++++---------- .../worker/src/api/routes/global/configs.js | 6 ++-- packages/worker/yarn.lock | 30 +++++++++---------- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index a9194a988d..cd309ad3a4 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1080,12 +1080,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.220-alpha.3": - version "1.0.220-alpha.3" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.220-alpha.3.tgz#62ca7f5c592e1efa9ed25abcdbafd60207961333" - integrity sha512-spX86t09B0h8vAiKDpm65b/wW22NRQtjRhJLFaD4bkV+rFoKi3WCVmxp/ah4k6AIQ4Q3aZQPEIsxAw4TNfQjMA== +"@budibase/backend-core@1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.1.2.tgz#a95703605db2a757913c669427a1d3d7894b0cc6" + integrity sha512-8dQTom7YCFySm40DHKA22YNKW9g1v6CkTg20vUmM8rMga5RrxDqQYJ3Ikgz2QRjQ9aT5kw/MIhHXM/Xgi8LrDw== dependencies: - "@budibase/types" "^1.0.220-alpha.3" + "@budibase/types" "^1.1.2" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" bcrypt "5.0.1" @@ -1162,13 +1162,13 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@1.0.220-alpha.3": - version "1.0.220-alpha.3" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.220-alpha.3.tgz#8384237f8ab72c1494d3b6b728b58efa2a29c999" - integrity sha512-mYnuw9HWmJM4IrS8uhJuOyX7oCfEclQQ8ak7UlAZoTmM3zxWTzJL1dIbMNKo8YvAv/w45FvW6QgiYjajXxG3XA== +"@budibase/pro@1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.1.2.tgz#6c061579eb9d34dcb6509e83596bb7019fc345fd" + integrity sha512-gdZasKfcC/eCqkiXBlErogHJN+5Sdxf3tA9xRMxehHXb/4eY+gOBEugAt/1/mI5GoiP509XWbO71JcGImPyRzg== dependencies: - "@budibase/backend-core" "1.0.220-alpha.3" - "@budibase/types" "1.0.220-alpha.3" + "@budibase/backend-core" "1.1.2" + "@budibase/types" "1.1.2" node-fetch "^2.6.1" "@budibase/standard-components@^0.9.139": @@ -1189,10 +1189,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@1.0.220-alpha.3", "@budibase/types@^1.0.220-alpha.3": - version "1.0.220-alpha.3" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.220-alpha.3.tgz#c63931cd25308e13143c1673850323cb3b60c159" - integrity sha512-iLcXUVaVfJE69LQGJFOBCjLS4pXceE0hAIQFio9kc2sP2w0SwvvhMwOVfyNERKycy1EQ+qw6BhlY00ceMZ7szQ== +"@budibase/types@1.1.2", "@budibase/types@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.1.2.tgz#696cf142856a4a60e0c3f48cc95eae965c43ae44" + integrity sha512-u377MZHErqaCnRdqX7NIi4CU090j7SkbIPJ2XYghcMzW7jcRz87N+Ehi8do3JnrXN6LZS98PiKo2hI33GTD0DQ== "@bull-board/api@3.7.0": version "3.7.0" diff --git a/packages/worker/src/api/routes/global/configs.js b/packages/worker/src/api/routes/global/configs.js index f6cac4d3b2..5505d03753 100644 --- a/packages/worker/src/api/routes/global/configs.js +++ b/packages/worker/src/api/routes/global/configs.js @@ -75,7 +75,7 @@ function buildConfigSaveValidation() { { is: Configs.OIDC, then: oidcValidation() } ], }), - }).required(), + }).required().unknown(true), ) } @@ -84,14 +84,14 @@ function buildUploadValidation() { return joiValidator.params(Joi.object({ type: Joi.string().valid(...Object.values(Configs)).required(), name: Joi.string().required(), - }).required()) + }).required().unknown(true)) } function buildConfigGetValidation() { // prettier-ignore return joiValidator.params(Joi.object({ type: Joi.string().valid(...Object.values(Configs)).required() - }).unknown(true).required()) + }).required().unknown(true)) } router diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index a51d1038f3..fe80aef008 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -293,12 +293,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.220-alpha.3": - version "1.0.220-alpha.3" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.220-alpha.3.tgz#62ca7f5c592e1efa9ed25abcdbafd60207961333" - integrity sha512-spX86t09B0h8vAiKDpm65b/wW22NRQtjRhJLFaD4bkV+rFoKi3WCVmxp/ah4k6AIQ4Q3aZQPEIsxAw4TNfQjMA== +"@budibase/backend-core@1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.1.2.tgz#a95703605db2a757913c669427a1d3d7894b0cc6" + integrity sha512-8dQTom7YCFySm40DHKA22YNKW9g1v6CkTg20vUmM8rMga5RrxDqQYJ3Ikgz2QRjQ9aT5kw/MIhHXM/Xgi8LrDw== dependencies: - "@budibase/types" "^1.0.220-alpha.3" + "@budibase/types" "^1.1.2" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" bcrypt "5.0.1" @@ -326,19 +326,19 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@1.0.220-alpha.3": - version "1.0.220-alpha.3" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.220-alpha.3.tgz#8384237f8ab72c1494d3b6b728b58efa2a29c999" - integrity sha512-mYnuw9HWmJM4IrS8uhJuOyX7oCfEclQQ8ak7UlAZoTmM3zxWTzJL1dIbMNKo8YvAv/w45FvW6QgiYjajXxG3XA== +"@budibase/pro@1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.1.2.tgz#6c061579eb9d34dcb6509e83596bb7019fc345fd" + integrity sha512-gdZasKfcC/eCqkiXBlErogHJN+5Sdxf3tA9xRMxehHXb/4eY+gOBEugAt/1/mI5GoiP509XWbO71JcGImPyRzg== dependencies: - "@budibase/backend-core" "1.0.220-alpha.3" - "@budibase/types" "1.0.220-alpha.3" + "@budibase/backend-core" "1.1.2" + "@budibase/types" "1.1.2" node-fetch "^2.6.1" -"@budibase/types@1.0.220-alpha.3", "@budibase/types@^1.0.220-alpha.3": - version "1.0.220-alpha.3" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.220-alpha.3.tgz#c63931cd25308e13143c1673850323cb3b60c159" - integrity sha512-iLcXUVaVfJE69LQGJFOBCjLS4pXceE0hAIQFio9kc2sP2w0SwvvhMwOVfyNERKycy1EQ+qw6BhlY00ceMZ7szQ== +"@budibase/types@1.1.2", "@budibase/types@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.1.2.tgz#696cf142856a4a60e0c3f48cc95eae965c43ae44" + integrity sha512-u377MZHErqaCnRdqX7NIi4CU090j7SkbIPJ2XYghcMzW7jcRz87N+Ehi8do3JnrXN6LZS98PiKo2hI33GTD0DQ== "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" From cff42331bcd26ae0087ec0fb31948f9172d2d5b8 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 13 Jul 2022 12:37:55 +0100 Subject: [PATCH 4/5] Fix for ARM processors. --- packages/server/src/integrations/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index c83e295ec5..6edc65f08c 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -53,7 +53,7 @@ const INTEGRATIONS = { } // optionally add oracle integration if the oracle binary can be installed -if (!(process.arch === "arm64" && process.platform === "darwin")) { +if (process.arch && !process.arch.startsWith("arm")) { const oracle = require("./oracle") DEFINITIONS[SourceNames.ORACLE] = oracle.schema INTEGRATIONS[SourceNames.ORACLE] = oracle.integration From 18fde073e48c2700c8d2667fed4c22d4bb97143a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 13 Jul 2022 13:32:06 +0100 Subject: [PATCH 5/5] Increasing real email test timeout. --- packages/worker/src/api/routes/tests/realEmail.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/worker/src/api/routes/tests/realEmail.spec.js b/packages/worker/src/api/routes/tests/realEmail.spec.js index 6f2af0010b..32508604ea 100644 --- a/packages/worker/src/api/routes/tests/realEmail.spec.js +++ b/packages/worker/src/api/routes/tests/realEmail.spec.js @@ -4,7 +4,7 @@ const nodemailer = require("nodemailer") const fetch = require("node-fetch") // for the real email tests give them a long time to try complete/fail -jest.setTimeout(30000) +jest.setTimeout(60000) describe("/api/global/email", () => {