From 496e4d5e87f49b76962cae3ecc1f9328aed45693 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 21 Mar 2022 12:32:04 +0000 Subject: [PATCH 1/8] Handle error when running an automation with a deleted column --- packages/server/src/automations/steps/queryRows.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/automations/steps/queryRows.js b/packages/server/src/automations/steps/queryRows.js index 6584cda8d3..d7fca116ed 100644 --- a/packages/server/src/automations/steps/queryRows.js +++ b/packages/server/src/automations/steps/queryRows.js @@ -85,7 +85,7 @@ exports.run = async function ({ inputs, appId }) { const { tableId, filters, sortColumn, sortOrder, limit } = inputs const table = await getTable(appId, tableId) let sortType = FieldTypes.STRING - if (table && table.schema && sortColumn) { + if (table && table.schema && table.schema[sortColumn] && sortColumn) { const fieldType = table.schema[sortColumn].type sortType = fieldType === FieldTypes.NUMBER ? FieldTypes.NUMBER : FieldTypes.STRING From 11b3d52857ef86982d74f5a24b67c572452f3798 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 21 Mar 2022 15:24:39 +0000 Subject: [PATCH 2/8] increasing client max body size for uploads --- hosting/nginx.prod.conf.hbs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hosting/nginx.prod.conf.hbs b/hosting/nginx.prod.conf.hbs index f3009baf40..0aaecc609d 100644 --- a/hosting/nginx.prod.conf.hbs +++ b/hosting/nginx.prod.conf.hbs @@ -22,9 +22,8 @@ http { resolver {{ resolver }} valid=10s ipv6=off; # buffering - client_body_buffer_size 1K; client_header_buffer_size 1k; - client_max_body_size 10M; + client_max_body_size 20M; ignore_invalid_headers off; proxy_buffering off; From 10699edd3bb8f08810a2075f97fdcb93b50b0729 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Mon, 21 Mar 2022 16:08:25 +0000 Subject: [PATCH 3/8] Remove automation usage increment --- packages/server/src/threads/automation.js | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/server/src/threads/automation.js b/packages/server/src/threads/automation.js index c0843a286c..f1df5c3dab 100644 --- a/packages/server/src/threads/automation.js +++ b/packages/server/src/threads/automation.js @@ -1,13 +1,11 @@ require("./utils").threadSetup() -const env = require("../environment") const actions = require("../automations/actions") const automationUtils = require("../automations/automationUtils") const AutomationEmitter = require("../events/AutomationEmitter") const { processObject } = require("@budibase/string-templates") const { DEFAULT_TENANT_ID } = require("@budibase/backend-core/constants") -const { DocumentTypes, isDevAppID } = require("../db/utils") +const { DocumentTypes } = require("../db/utils") const { doInTenant } = require("@budibase/backend-core/tenancy") -const usage = require("../utilities/usageQuota") const { definitions: triggerDefs } = require("../automations/triggerInfo") const { doInAppContext, getAppDB } = require("@budibase/backend-core/context") @@ -120,11 +118,6 @@ class Orchestrator { return err } } - - // Increment quota for automation runs - if (!env.SELF_HOSTED && !isDevAppID(this._appId)) { - await usage.update(usage.Properties.AUTOMATION, 1) - } return this.executionOutput } } From 38f53a4148845f1aded8a72311363032c177d2c6 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Mon, 21 Mar 2022 17:15:53 +0000 Subject: [PATCH 4/8] v1.0.93 --- lerna.json | 2 +- packages/backend-core/package.json | 2 +- packages/bbui/package.json | 4 ++-- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 2 +- packages/client/package.json | 8 ++++---- packages/frontend-core/package.json | 4 ++-- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 10 files changed, 24 insertions(+), 24 deletions(-) diff --git a/lerna.json b/lerna.json index cd88fd373f..49830d0647 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.92", + "version": "1.0.93", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index ec55941fdf..6a14ca5036 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.92", + "version": "1.0.93", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 278b8640cd..04cb290008 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.92", + "version": "1.0.93", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^1.0.92", + "@budibase/string-templates": "^1.0.93", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index 51d323395c..fab2c23465 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.92", + "version": "1.0.93", "license": "GPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.92", - "@budibase/client": "^1.0.92", - "@budibase/frontend-core": "^1.0.92", - "@budibase/string-templates": "^1.0.92", + "@budibase/bbui": "^1.0.93", + "@budibase/client": "^1.0.93", + "@budibase/frontend-core": "^1.0.93", + "@budibase/string-templates": "^1.0.93", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 8a047ceb16..ae2f9412bd 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.92", + "version": "1.0.93", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 11b879de79..fc21569e60 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.92", + "version": "1.0.93", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.92", - "@budibase/frontend-core": "^1.0.92", - "@budibase/string-templates": "^1.0.92", + "@budibase/bbui": "^1.0.93", + "@budibase/frontend-core": "^1.0.93", + "@budibase/string-templates": "^1.0.93", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 14e85a16f5..a487d2a56b 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "1.0.92", + "version": "1.0.93", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^1.0.92", + "@budibase/bbui": "^1.0.93", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/server/package.json b/packages/server/package.json index 53fe4338ce..9e1c079d2b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.92", + "version": "1.0.93", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -71,9 +71,9 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.92", - "@budibase/client": "^1.0.92", - "@budibase/string-templates": "^1.0.92", + "@budibase/backend-core": "^1.0.93", + "@budibase/client": "^1.0.93", + "@budibase/string-templates": "^1.0.93", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index c9838a5196..692472315e 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.92", + "version": "1.0.93", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 83ae213258..1d7ace07a3 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.92", + "version": "1.0.93", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -34,8 +34,8 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.92", - "@budibase/string-templates": "^1.0.92", + "@budibase/backend-core": "^1.0.93", + "@budibase/string-templates": "^1.0.93", "@koa/router": "^8.0.0", "@sentry/node": "^6.0.0", "@techpass/passport-openidconnect": "^0.3.0", From 3eb7ab4e5b6f3f896eb02836897b369d309c8bdd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 21 Mar 2022 17:16:18 +0000 Subject: [PATCH 5/8] Fixing issue #4978 - fixing an issue with using the keyword 'in' as part of an SQL query. --- packages/server/src/threads/query.js | 42 ++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/packages/server/src/threads/query.js b/packages/server/src/threads/query.js index 3ce138b72f..a2f2ed84a8 100644 --- a/packages/server/src/threads/query.js +++ b/packages/server/src/threads/query.js @@ -33,10 +33,12 @@ class QueryRunner { return fields } const bindings = findHBSBlocks(sql) - let variables = [] + let variables = [], + arrays = [] for (let binding of bindings) { - let variable = integration.getBindingIdentifier() - variables.push(binding) + // look for array/list operations in the SQL statement, which will need handled later + const listRegex = new RegExp(`(in|IN|In|iN) ${binding}`) + const listRegexMatch = sql.match(listRegex) // check if the variable was used as part of a string concat e.g. 'Hello {{binding}}' const charConstRegex = new RegExp(`'[^']*${binding}[^']*'`) const charConstMatch = sql.match(charConstRegex) @@ -46,15 +48,45 @@ class QueryRunner { part2 = `'${part2.substring(0, part2.length - 1)}'` sql = sql.replace( charConstMatch[0], - integration.getStringConcat([part1, variable, part2]) + integration.getStringConcat([ + part1, + integration.getBindingIdentifier(), + part2, + ]) + ) + } + // generate SQL parameterised array + else if (listRegexMatch) { + arrays.push(binding) + // determine the length of the array + const value = this.enrichQueryFields([binding], parameters)[0].split( + "," + ) + // build a string like ($1, $2, $3) + sql = sql.replace( + binding, + `(${Array.apply(null, Array(value.length)) + .map(() => integration.getBindingIdentifier()) + .join(",")})` ) } else { - sql = sql.replace(binding, variable) + sql = sql.replace(binding, integration.getBindingIdentifier()) } + variables.push(binding) } // replicate the knex structure fields.sql = sql fields.bindings = this.enrichQueryFields(variables, parameters) + // check for arrays in the data + let updated = [] + for (let i = 0; i < variables.length; i++) { + if (arrays.includes(variables[i])) { + updated = updated.concat(fields.bindings[i].split(",")) + } else { + updated.push(fields.bindings[i]) + } + } + fields.bindings = updated return fields } From d52f6dea053d14f63b20a82068fe410639cc1df5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 21 Mar 2022 17:44:43 +0000 Subject: [PATCH 6/8] Adding the ability to expand the max number of rows retrieved by SQL queries (say for table exports). --- packages/server/src/environment.js | 1 + packages/server/src/integrations/base/sql.ts | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 312ecf313d..212a54b87c 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -70,6 +70,7 @@ module.exports = { ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS, DISABLE_THREADING: process.env.DISABLE_THREADING, QUERY_THREAD_TIMEOUT: process.env.QUERY_THREAD_TIMEOUT, + SQL_MAX_ROWS: process.env.SQL_MAX_ROWS, _set(key, value) { process.env[key] = value module.exports[key] = value diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index ffa405f016..3c3132c887 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -9,8 +9,12 @@ import { } from "../../definitions/datasource" import { isIsoDateString, SqlClients } from "../utils" import SqlTableQueryBuilder from "./sqlTable" +import environment from "../../environment" -const BASE_LIMIT = 5000 +const envLimit = environment.SQL_MAX_ROWS + ? parseInt(environment.SQL_MAX_ROWS) + : null +const BASE_LIMIT = envLimit || 5000 type KnexQuery = Knex.QueryBuilder | Knex // these are invalid dates sent by the client, need to convert them to a real max date From 114c9cf96a1505975565d2507f1d2972f852a1c2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 21 Mar 2022 17:57:26 +0000 Subject: [PATCH 7/8] Fixing review comment. --- packages/server/src/threads/query.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/threads/query.js b/packages/server/src/threads/query.js index a2f2ed84a8..270c2cc713 100644 --- a/packages/server/src/threads/query.js +++ b/packages/server/src/threads/query.js @@ -37,7 +37,7 @@ class QueryRunner { arrays = [] for (let binding of bindings) { // look for array/list operations in the SQL statement, which will need handled later - const listRegex = new RegExp(`(in|IN|In|iN) ${binding}`) + const listRegex = new RegExp(`(in|IN|In|iN)( )+${binding}`) const listRegexMatch = sql.match(listRegex) // check if the variable was used as part of a string concat e.g. 'Hello {{binding}}' const charConstRegex = new RegExp(`'[^']*${binding}[^']*'`) From 01c2b9e4968883f6fb29e8c8d5b1f82f0ecbf4bb Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Tue, 22 Mar 2022 13:08:40 +0000 Subject: [PATCH 8/8] v1.0.94 --- lerna.json | 2 +- packages/backend-core/package.json | 2 +- packages/bbui/package.json | 4 ++-- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 2 +- packages/client/package.json | 8 ++++---- packages/frontend-core/package.json | 4 ++-- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 10 files changed, 24 insertions(+), 24 deletions(-) diff --git a/lerna.json b/lerna.json index 49830d0647..0ab734ac77 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.93", + "version": "1.0.94", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 6a14ca5036..f63207850b 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.93", + "version": "1.0.94", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 04cb290008..de4269d1aa 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.93", + "version": "1.0.94", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^1.0.93", + "@budibase/string-templates": "^1.0.94", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index fab2c23465..56151f018f 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.93", + "version": "1.0.94", "license": "GPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.93", - "@budibase/client": "^1.0.93", - "@budibase/frontend-core": "^1.0.93", - "@budibase/string-templates": "^1.0.93", + "@budibase/bbui": "^1.0.94", + "@budibase/client": "^1.0.94", + "@budibase/frontend-core": "^1.0.94", + "@budibase/string-templates": "^1.0.94", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index ae2f9412bd..6297b5001c 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.93", + "version": "1.0.94", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index fc21569e60..4b2ccf84e4 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.93", + "version": "1.0.94", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.93", - "@budibase/frontend-core": "^1.0.93", - "@budibase/string-templates": "^1.0.93", + "@budibase/bbui": "^1.0.94", + "@budibase/frontend-core": "^1.0.94", + "@budibase/string-templates": "^1.0.94", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index a487d2a56b..851eba6e28 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "1.0.93", + "version": "1.0.94", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^1.0.93", + "@budibase/bbui": "^1.0.94", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/server/package.json b/packages/server/package.json index 9e1c079d2b..f12064c6e5 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.93", + "version": "1.0.94", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -71,9 +71,9 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.93", - "@budibase/client": "^1.0.93", - "@budibase/string-templates": "^1.0.93", + "@budibase/backend-core": "^1.0.94", + "@budibase/client": "^1.0.94", + "@budibase/string-templates": "^1.0.94", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 692472315e..bc28ac69b5 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.93", + "version": "1.0.94", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 1d7ace07a3..e1909e2e04 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.93", + "version": "1.0.94", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -34,8 +34,8 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.93", - "@budibase/string-templates": "^1.0.93", + "@budibase/backend-core": "^1.0.94", + "@budibase/string-templates": "^1.0.94", "@koa/router": "^8.0.0", "@sentry/node": "^6.0.0", "@techpass/passport-openidconnect": "^0.3.0",