diff --git a/hosting/nginx.prod.conf.hbs b/hosting/nginx.prod.conf.hbs index 0f3a41929b..0ab7ed2c7e 100644 --- a/hosting/nginx.prod.conf.hbs +++ b/hosting/nginx.prod.conf.hbs @@ -22,9 +22,8 @@ http { resolver {{ resolver }} valid=10s ipv6=off; # buffering - client_body_buffer_size 1K; client_header_buffer_size 1k; - client_max_body_size 10M; + client_max_body_size 20M; ignore_invalid_headers off; proxy_buffering off; diff --git a/lerna.json b/lerna.json index cd88fd373f..0ab734ac77 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.92", + "version": "1.0.94", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index ec55941fdf..f63207850b 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.92", + "version": "1.0.94", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 278b8640cd..de4269d1aa 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.92", + "version": "1.0.94", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^1.0.92", + "@budibase/string-templates": "^1.0.94", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index 51d323395c..56151f018f 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.92", + "version": "1.0.94", "license": "GPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.92", - "@budibase/client": "^1.0.92", - "@budibase/frontend-core": "^1.0.92", - "@budibase/string-templates": "^1.0.92", + "@budibase/bbui": "^1.0.94", + "@budibase/client": "^1.0.94", + "@budibase/frontend-core": "^1.0.94", + "@budibase/string-templates": "^1.0.94", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 8a047ceb16..6297b5001c 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.92", + "version": "1.0.94", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 11b879de79..4b2ccf84e4 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.92", + "version": "1.0.94", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.92", - "@budibase/frontend-core": "^1.0.92", - "@budibase/string-templates": "^1.0.92", + "@budibase/bbui": "^1.0.94", + "@budibase/frontend-core": "^1.0.94", + "@budibase/string-templates": "^1.0.94", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 14e85a16f5..851eba6e28 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "1.0.92", + "version": "1.0.94", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^1.0.92", + "@budibase/bbui": "^1.0.94", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/server/package.json b/packages/server/package.json index 53fe4338ce..f12064c6e5 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.92", + "version": "1.0.94", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -71,9 +71,9 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.92", - "@budibase/client": "^1.0.92", - "@budibase/string-templates": "^1.0.92", + "@budibase/backend-core": "^1.0.94", + "@budibase/client": "^1.0.94", + "@budibase/string-templates": "^1.0.94", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/server/src/automations/steps/queryRows.js b/packages/server/src/automations/steps/queryRows.js index 5bc19d0022..62f7659589 100644 --- a/packages/server/src/automations/steps/queryRows.js +++ b/packages/server/src/automations/steps/queryRows.js @@ -85,7 +85,7 @@ exports.run = async function ({ inputs, appId }) { const { tableId, filters, sortColumn, sortOrder, limit } = inputs const table = await getTable(appId, tableId) let sortType = FieldTypes.STRING - if (table && table.schema && sortColumn) { + if (table && table.schema && table.schema[sortColumn] && sortColumn) { const fieldType = table.schema[sortColumn].type sortType = fieldType === FieldTypes.NUMBER ? FieldTypes.NUMBER : FieldTypes.STRING diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 312ecf313d..212a54b87c 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -70,6 +70,7 @@ module.exports = { ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS, DISABLE_THREADING: process.env.DISABLE_THREADING, QUERY_THREAD_TIMEOUT: process.env.QUERY_THREAD_TIMEOUT, + SQL_MAX_ROWS: process.env.SQL_MAX_ROWS, _set(key, value) { process.env[key] = value module.exports[key] = value diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index ffa405f016..3c3132c887 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -9,8 +9,12 @@ import { } from "../../definitions/datasource" import { isIsoDateString, SqlClients } from "../utils" import SqlTableQueryBuilder from "./sqlTable" +import environment from "../../environment" -const BASE_LIMIT = 5000 +const envLimit = environment.SQL_MAX_ROWS + ? parseInt(environment.SQL_MAX_ROWS) + : null +const BASE_LIMIT = envLimit || 5000 type KnexQuery = Knex.QueryBuilder | Knex // these are invalid dates sent by the client, need to convert them to a real max date diff --git a/packages/server/src/threads/automation.js b/packages/server/src/threads/automation.js index c0843a286c..f1df5c3dab 100644 --- a/packages/server/src/threads/automation.js +++ b/packages/server/src/threads/automation.js @@ -1,13 +1,11 @@ require("./utils").threadSetup() -const env = require("../environment") const actions = require("../automations/actions") const automationUtils = require("../automations/automationUtils") const AutomationEmitter = require("../events/AutomationEmitter") const { processObject } = require("@budibase/string-templates") const { DEFAULT_TENANT_ID } = require("@budibase/backend-core/constants") -const { DocumentTypes, isDevAppID } = require("../db/utils") +const { DocumentTypes } = require("../db/utils") const { doInTenant } = require("@budibase/backend-core/tenancy") -const usage = require("../utilities/usageQuota") const { definitions: triggerDefs } = require("../automations/triggerInfo") const { doInAppContext, getAppDB } = require("@budibase/backend-core/context") @@ -120,11 +118,6 @@ class Orchestrator { return err } } - - // Increment quota for automation runs - if (!env.SELF_HOSTED && !isDevAppID(this._appId)) { - await usage.update(usage.Properties.AUTOMATION, 1) - } return this.executionOutput } } diff --git a/packages/server/src/threads/query.js b/packages/server/src/threads/query.js index 3ce138b72f..270c2cc713 100644 --- a/packages/server/src/threads/query.js +++ b/packages/server/src/threads/query.js @@ -33,10 +33,12 @@ class QueryRunner { return fields } const bindings = findHBSBlocks(sql) - let variables = [] + let variables = [], + arrays = [] for (let binding of bindings) { - let variable = integration.getBindingIdentifier() - variables.push(binding) + // look for array/list operations in the SQL statement, which will need handled later + const listRegex = new RegExp(`(in|IN|In|iN)( )+${binding}`) + const listRegexMatch = sql.match(listRegex) // check if the variable was used as part of a string concat e.g. 'Hello {{binding}}' const charConstRegex = new RegExp(`'[^']*${binding}[^']*'`) const charConstMatch = sql.match(charConstRegex) @@ -46,15 +48,45 @@ class QueryRunner { part2 = `'${part2.substring(0, part2.length - 1)}'` sql = sql.replace( charConstMatch[0], - integration.getStringConcat([part1, variable, part2]) + integration.getStringConcat([ + part1, + integration.getBindingIdentifier(), + part2, + ]) + ) + } + // generate SQL parameterised array + else if (listRegexMatch) { + arrays.push(binding) + // determine the length of the array + const value = this.enrichQueryFields([binding], parameters)[0].split( + "," + ) + // build a string like ($1, $2, $3) + sql = sql.replace( + binding, + `(${Array.apply(null, Array(value.length)) + .map(() => integration.getBindingIdentifier()) + .join(",")})` ) } else { - sql = sql.replace(binding, variable) + sql = sql.replace(binding, integration.getBindingIdentifier()) } + variables.push(binding) } // replicate the knex structure fields.sql = sql fields.bindings = this.enrichQueryFields(variables, parameters) + // check for arrays in the data + let updated = [] + for (let i = 0; i < variables.length; i++) { + if (arrays.includes(variables[i])) { + updated = updated.concat(fields.bindings[i].split(",")) + } else { + updated.push(fields.bindings[i]) + } + } + fields.bindings = updated return fields } diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index c9838a5196..bc28ac69b5 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.92", + "version": "1.0.94", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 83ae213258..e1909e2e04 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.92", + "version": "1.0.94", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -34,8 +34,8 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.92", - "@budibase/string-templates": "^1.0.92", + "@budibase/backend-core": "^1.0.94", + "@budibase/string-templates": "^1.0.94", "@koa/router": "^8.0.0", "@sentry/node": "^6.0.0", "@techpass/passport-openidconnect": "^0.3.0",