From 8e8c913dfb97d91d860d01e7c653da8c398873cb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 6 Sep 2023 18:17:44 +0100 Subject: [PATCH 1/5] Updating the yarn dev process to not include a removal of the dist directory as part of the streamed watchers - this seems to create a problem for shared core and moving this up a level to a parallel run before the watchers massively improves stability. --- package.json | 2 +- packages/server/package.json | 2 +- packages/shared-core/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index b0c428fe70..b263fb6585 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "kill-builder": "kill-port 3000", "kill-server": "kill-port 4001 4002", "kill-all": "yarn run kill-builder && yarn run kill-server", - "dev": "yarn run kill-all && lerna run --stream dev:builder", + "dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev:builder", "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", diff --git a/packages/server/package.json b/packages/server/package.json index ecf31175dc..39f52342a4 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -26,7 +26,7 @@ "dev:stack:up": "node scripts/dev/manage.js up", "dev:stack:down": "node scripts/dev/manage.js down", "dev:stack:nuke": "node scripts/dev/manage.js nuke", - "dev:builder": "yarn run dev:stack:up && rimraf dist/ && nodemon", + "dev:builder": "yarn run dev:stack:up && nodemon", "dev:built": "yarn run dev:stack:up && yarn run run:docker", "specs": "ts-node specs/generate.ts && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts", "initialise": "node scripts/initialise.js", diff --git a/packages/shared-core/package.json b/packages/shared-core/package.json index 8edea4870a..160a42f086 100644 --- a/packages/shared-core/package.json +++ b/packages/shared-core/package.json @@ -10,7 +10,7 @@ "prebuild": "rimraf dist/", "build": "node ../../scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly --paths null", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", - "dev:builder": "yarn prebuild && tsc -p tsconfig.json --watch --preserveWatchOutput", + "dev:builder": "tsc -p tsconfig.json --watch --preserveWatchOutput", "check:types": "tsc -p tsconfig.json --noEmit --paths null" }, "dependencies": { diff --git a/packages/types/package.json b/packages/types/package.json index bb0163ae13..919cc736fc 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -10,7 +10,7 @@ "prebuild": "rimraf dist/", "build": "node ../../scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", - "dev:builder": "yarn prebuild && tsc -p tsconfig.json --watch --preserveWatchOutput", + "dev:builder": "tsc -p tsconfig.json --watch --preserveWatchOutput", "check:types": "tsc -p tsconfig.json --noEmit --paths null" }, "jest": {}, diff --git a/packages/worker/package.json b/packages/worker/package.json index 3b66df264e..15ab1c1ff9 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -22,7 +22,7 @@ "predocker": "yarn build && cp ../../yarn.lock ./dist/", "build:docker": "yarn predocker && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION", "dev:stack:init": "node ./scripts/dev/manage.js init", - "dev:builder": "npm run dev:stack:init && rimraf dist/ && nodemon", + "dev:builder": "npm run dev:stack:init && nodemon", "dev:built": "yarn run dev:stack:init && yarn run run:docker", "test": "bash scripts/test.sh", "test:watch": "jest --watch", From aeac7aaa0c0f8845bbfd187e207795c25af7237d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 12 Sep 2023 13:54:39 +0100 Subject: [PATCH 2/5] Making sure that not all falsy values are considered invalid filter inputs. --- .../server/scripts/integrations/postgres/init.sql | 3 ++- packages/server/src/api/controllers/row/external.ts | 4 ++-- packages/server/src/api/controllers/row/utils.ts | 6 ++++++ yarn.lock | 13 +++++++++++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/packages/server/scripts/integrations/postgres/init.sql b/packages/server/scripts/integrations/postgres/init.sql index 6cb3f51269..f89ad2812d 100644 --- a/packages/server/scripts/integrations/postgres/init.sql +++ b/packages/server/scripts/integrations/postgres/init.sql @@ -1,7 +1,7 @@ SELECT 'CREATE DATABASE main' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec CREATE SCHEMA "test-1"; -CREATE TYPE person_job AS ENUM ('qa', 'programmer', 'designer'); +CREATE TYPE person_job AS ENUM ('qa', 'programmer', 'designer', 'support'); CREATE TABLE Persons ( PersonID SERIAL PRIMARY KEY, LastName varchar(255), @@ -51,6 +51,7 @@ CREATE TABLE CompositeTable ( ); INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('Mike', 'Hughes', '123 Fake Street', 'Belfast', 'qa'); INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('John', 'Smith', '64 Updown Road', 'Dublin', 'programmer'); +INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age) VALUES ('Foo', 'Bar', 'Foo Street', 'Bartown', 'support', 0); INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (1, 2, 'assembling', TRUE); INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (2, 1, 'processing', FALSE); INSERT INTO Products (ProductName) VALUES ('Computers'); diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 491cbfa8f9..44921a5114 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -32,12 +32,12 @@ export async function handleRequest( } // @ts-ignore for (let [key, value] of Object.entries(opts.filters[filterField])) { - if (!value || value === "") { + if (utils.invalidFilter(value)) { // @ts-ignore delete opts.filters[filterField][key] } } - } + } if ( diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index e85ec4553c..12c84bfb7b 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -139,3 +139,9 @@ export async function validate({ } return { valid: Object.keys(errors).length === 0, errors } } + +// don't do a pure falsy check, as 0 is included +// https://github.com/Budibase/budibase/issues/10118 +export function invalidFilter(value: any) { + return value == null || value === "" +} diff --git a/yarn.lock b/yarn.lock index ab86a87560..8c93661665 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6269,6 +6269,14 @@ "@types/tedious" "*" tarn "^3.0.1" +"@types/node-fetch@2.6.1": + version "2.6.1" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975" + integrity sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA== + dependencies: + "@types/node" "*" + form-data "^3.0.0" + "@types/node-fetch@2.6.4": version "2.6.4" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.4.tgz#1bc3a26de814f6bf466b25aeb1473fa1afe6a660" @@ -6290,6 +6298,11 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f" integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA== +"@types/node@14.18.20": + version "14.18.20" + resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.20.tgz#268f028b36eaf51181c3300252f605488c4f0650" + integrity sha512-Q8KKwm9YqEmUBRsqJ2GWJDtXltBDxTdC4m5vTdXBolu2PeQh8LX+f6BTwU+OuXPu37fLxoN6gidqBmnky36FXA== + "@types/node@16.9.1": version "16.9.1" resolved "https://registry.yarnpkg.com/@types/node/-/node-16.9.1.tgz#0611b37db4246c937feef529ddcc018cf8e35708" From aed5694ac2715dae82e5f43f314d16b5647806a7 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 12 Sep 2023 14:06:20 +0100 Subject: [PATCH 3/5] Adding dedicated function for removing empty filters. --- .../src/api/controllers/row/external.ts | 15 +------- .../server/src/api/controllers/row/utils.ts | 38 +++++++++++++++++-- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 44921a5114..a04584e6bd 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -18,6 +18,7 @@ import { import sdk from "../../../sdk" import * as utils from "./utils" import { dataFilters } from "@budibase/shared-core" +import { removeEmptyFilters } from "./utils" export async function handleRequest( operation: Operation, @@ -26,20 +27,8 @@ export async function handleRequest( ) { // make sure the filters are cleaned up, no empty strings for equals, fuzzy or string if (opts && opts.filters) { - for (let filterField of NoEmptyFilterStrings) { - if (!opts.filters[filterField]) { - continue - } - // @ts-ignore - for (let [key, value] of Object.entries(opts.filters[filterField])) { - if (utils.invalidFilter(value)) { - // @ts-ignore - delete opts.filters[filterField][key] - } - } - + opts.filters = utils.removeEmptyFilters(opts.filters) } - if ( !dataFilters.hasFilters(opts?.filters) && opts?.filters?.onEmptyFilter === EmptyFilterOption.RETURN_NONE diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index 12c84bfb7b..cc27f4c2a3 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -1,8 +1,15 @@ import { InternalTables } from "../../../db/utils" import * as userController from "../user" import { context } from "@budibase/backend-core" -import { Ctx, FieldType, Row, Table, UserCtx } from "@budibase/types" -import { FieldTypes } from "../../../constants" +import { + Ctx, + FieldType, + Row, + SearchFilters, + Table, + UserCtx, +} from "@budibase/types" +import { FieldTypes, NoEmptyFilterStrings } from "../../../constants" import sdk from "../../../sdk" import validateJs from "validate.js" @@ -142,6 +149,29 @@ export async function validate({ // don't do a pure falsy check, as 0 is included // https://github.com/Budibase/budibase/issues/10118 -export function invalidFilter(value: any) { - return value == null || value === "" +export function removeEmptyFilters(filters: SearchFilters) { + for (let filterField of NoEmptyFilterStrings) { + if (!filters[filterField]) { + continue + } + + for (let filterType of Object.keys(filters)) { + if (filterType !== filterField) { + continue + } + // don't know which one we're checking, type could be anything + const value = filters[filterType] as unknown + if (typeof value === "object") { + for (let [key, value] of Object.entries( + filters[filterType] as object + )) { + if (value == null || value === "") { + // @ts-ignore + delete filters[filterField][key] + } + } + } + } + } + return filters } From 80e217afda1325ccf8b23f8650eead02a26fb0d0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 12 Sep 2023 14:23:08 +0100 Subject: [PATCH 4/5] Quick test to avoid scenario in future. --- .../api/controllers/row/tests/utils.spec.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 packages/server/src/api/controllers/row/tests/utils.spec.ts diff --git a/packages/server/src/api/controllers/row/tests/utils.spec.ts b/packages/server/src/api/controllers/row/tests/utils.spec.ts new file mode 100644 index 0000000000..e0ad637e9d --- /dev/null +++ b/packages/server/src/api/controllers/row/tests/utils.spec.ts @@ -0,0 +1,21 @@ +import * as utils from "../utils" + +describe("removeEmptyFilters", () => { + it("0 should not be removed", () => { + const filters = utils.removeEmptyFilters({ + equal: { + column: 0, + }, + }) + expect((filters.equal as any).column).toBe(0) + }) + + it("empty string should be removed", () => { + const filters = utils.removeEmptyFilters({ + equal: { + column: "", + }, + }) + expect(Object.values(filters.equal as any).length).toBe(0) + }) +}) From 6d74bb4063bc7580bad2c822a325f291d638aa7d Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 13 Sep 2023 18:01:20 +0000 Subject: [PATCH 5/5] Bump version to 2.10.8 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 77f38e67ed..30aa2afa3c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.10.7", + "version": "2.10.8", "npmClient": "yarn", "packages": [ "packages/*"