From e8dbb8f6817d6a551fd32bf2a6e727cee9bd0ad7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 Apr 2024 09:14:04 +0200 Subject: [PATCH 01/96] Remove todo --- packages/shared-core/src/filters.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index d9fe533c88..c2a8cfd416 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -21,7 +21,7 @@ const HBS_REGEX = /{{([^{].*?)}}/g export const getValidOperatorsForType = ( fieldType: { type: FieldType; subtype?: FieldSubtype }, field: string, - datasource: Datasource & { tableId: any } // TODO: is this table id ever populated? + datasource: Datasource & { tableId: any } ) => { const Op = OperatorOptions const stringOps = [ From cf2778cbaac83d34998ad6fcdd39e75d2c191568 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 Apr 2024 09:30:57 +0200 Subject: [PATCH 02/96] Filter out fields without filtering options --- .../design/settings/controls/FilterEditor/FilterDrawer.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte index 7f1ee8010d..22b8ede31d 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte @@ -46,7 +46,9 @@ $: parseFilters(filters) $: dispatch("change", enrichFilters(rawFilters, matchAny, onEmptyFilter)) - $: enrichedSchemaFields = getFields(schemaFields || [], { allowLinks: true }) + $: enrichedSchemaFields = getFields(schemaFields || [], { + allowLinks: true, + }).filter(f => !!getValidOperatorsForType({ field: f.name, ...f }).length) $: fieldOptions = enrichedSchemaFields.map(field => field.name) || [] $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"] From 63dd0b5d6598ba8422205dd0fcaeec4215f39eca Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 Apr 2024 10:19:07 +0200 Subject: [PATCH 03/96] Display no available filter --- .../controls/FilterEditor/FilterDrawer.svelte | 232 +++++++++--------- 1 file changed, 119 insertions(+), 113 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte index 22b8ede31d..b4e45c9487 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte @@ -207,130 +207,136 @@
- {#if !rawFilters?.length} - Add your first filter expression. - {:else} -
- opt.label} getOptionValue={opt => opt.value} - on:change={e => (onEmptyFilter = e.detail)} + on:change={e => (matchAny = e.detail === "or")} placeholder={null} /> - {/if} -
-
-
- -
-
- {#each rawFilters as filter} + {#if datasource?.type === "table"} onOperatorChange(filter)} + label="When filter empty" + value={onEmptyFilter} + options={onEmptyOptions} + getOptionLabel={opt => opt.label} + getOptionValue={opt => opt.value} + on:change={e => (onEmptyFilter = e.detail)} placeholder={null} /> - - {:else if filter.type === "array" || (filter.type === "options" && filter.operator === "oneOf")} - - {:else if filter.type === "options"} - - {:else if filter.type === "boolean"} - - {:else if filter.type === "datetime"} - - {:else if filter.type === FieldType.BB_REFERENCE} - - {:else} - - {/if} - duplicateFilter(filter.id)} - /> - removeFilter(filter.id)} - /> - {/each} + {/if}
+
+
+ +
+
+ {#each rawFilters as filter} + onOperatorChange(filter)} + placeholder={null} + /> + + {:else if filter.type === "array" || (filter.type === "options" && filter.operator === "oneOf")} + + {:else if filter.type === "options"} + + {:else if filter.type === "boolean"} + + {:else if filter.type === "datetime"} + + {:else if filter.type === FieldType.BB_REFERENCE} + + {:else} + + {/if} + duplicateFilter(filter.id)} + /> + removeFilter(filter.id)} + /> + {/each} +
+
+ {/if} +
+
+ {:else} + + None of the table column can be used for filtering. + {/if} -
- -
From a4a095b6a1cf5668ea906020c1c3fdd77b50df09 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 Apr 2024 11:37:59 +0200 Subject: [PATCH 04/96] Handle formulas for getValidOperatorsForType --- packages/shared-core/src/filters.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index c2a8cfd416..95d5269c4b 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -2,6 +2,7 @@ import { Datasource, FieldSubtype, FieldType, + FormulaType, SearchFilter, SearchQuery, SearchQueryFields, @@ -19,7 +20,11 @@ const HBS_REGEX = /{{([^{].*?)}}/g * Returns the valid operator options for a certain data type */ export const getValidOperatorsForType = ( - fieldType: { type: FieldType; subtype?: FieldSubtype }, + fieldType: { + type: FieldType + subtype?: FieldSubtype + formulaType?: FormulaType + }, field: string, datasource: Datasource & { tableId: any } ) => { @@ -46,7 +51,7 @@ export const getValidOperatorsForType = ( value: string label: string }[] = [] - const { type, subtype } = fieldType + const { type, subtype, formulaType } = fieldType if (type === FieldType.STRING) { ops = stringOps } else if (type === FieldType.NUMBER || type === FieldType.BIGINT) { @@ -61,7 +66,7 @@ export const getValidOperatorsForType = ( ops = stringOps } else if (type === FieldType.DATETIME) { ops = numOps - } else if (type === FieldType.FORMULA) { + } else if (type === FieldType.FORMULA && formulaType === FormulaType.STATIC) { ops = stringOps.concat([Op.MoreThan, Op.LessThan]) } else if (type === FieldType.BB_REFERENCE && subtype == FieldSubtype.USER) { ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In] From 709457477e801a35963a1b80191cbdb099a0cc38 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 Apr 2024 12:29:45 +0200 Subject: [PATCH 05/96] Move filterbuilder to frontend-core --- .../controls/FilterEditor/FilterDrawer.svelte | 325 +++--------------- packages/frontend-core/package.json | 1 + .../src/components/FilterBuilder.svelte | 281 +++++++++++++++ .../frontend-core/src/components/index.js | 1 + 4 files changed, 325 insertions(+), 283 deletions(-) create mode 100644 packages/frontend-core/src/components/FilterBuilder.svelte diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte index b4e45c9487..c235ea4591 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte @@ -1,25 +1,12 @@ - -
- - {#if fieldOptions?.length} - {#if !rawFilters?.length} - Add your first filter expression. - {:else} -
- opt.label} - getOptionValue={opt => opt.value} - on:change={e => (onEmptyFilter = e.detail)} - placeholder={null} - /> - {/if} -
-
-
- -
-
- {#each rawFilters as filter} - onOperatorChange(filter)} - placeholder={null} - /> - - {:else if filter.type === "array" || (filter.type === "options" && filter.operator === "oneOf")} - - {:else if filter.type === "options"} - - {:else if filter.type === "boolean"} - - {:else if filter.type === "datetime"} - - {:else if filter.type === FieldType.BB_REFERENCE} - - {:else} - - {/if} - duplicateFilter(filter.id)} - /> - removeFilter(filter.id)} - /> - {/each} -
-
- {/if} -
- -
- {:else} - - None of the table column can be used for filtering. - + + +
+ opt.label} + getOptionValue={opt => opt.value} + on:change={e => (onEmptyFilter = e.detail)} + placeholder={null} + /> {/if} - -
+
+
+ (filter.value = event.detail)} + /> +
+ diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 4ca88de8f2..3f97573d4a 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -11,6 +11,7 @@ "@budibase/types": "0.0.0", "dayjs": "^1.10.8", "lodash": "4.17.21", + "shortid": "2.2.15", "socket.io-client": "^4.6.1" } } diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte new file mode 100644 index 0000000000..67bfca524d --- /dev/null +++ b/packages/frontend-core/src/components/FilterBuilder.svelte @@ -0,0 +1,281 @@ + + +
+ + + {#if !filters?.length} + Add your first filter expression. + {:else} + + {/if} + + {#if filters?.length} +
+ +
+
+ {#each filters as filter} + onOperatorChange(filter)} + placeholder={null} + /> + {#if allowBindings} + + {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === SearchQueryOperators.ONE_OF)} + + {:else if filter.type === FieldType.OPTIONS} + + {:else if filter.type === FieldType.BOOLEAN} + + {:else if filter.type === FieldType.DATETIME} + + {:else} + + {/if} +
+ duplicateFilter(filter.id)} + /> +
+
+ removeFilter(filter.id)} + /> +
+ {/each} +
+ {/if} +
+ +
+ +
+ + diff --git a/packages/frontend-core/src/components/index.js b/packages/frontend-core/src/components/index.js index f71420b12b..0d4ff8ea35 100644 --- a/packages/frontend-core/src/components/index.js +++ b/packages/frontend-core/src/components/index.js @@ -6,3 +6,4 @@ export { default as UserAvatars } from "./UserAvatars.svelte" export { default as Updating } from "./Updating.svelte" export { Grid } from "./grid" export { default as ClientAppSkeleton } from "./ClientAppSkeleton.svelte" +export { default as FilterBuilder } from "./FilterBuilder.svelte" From 2149b7dc5768d281d88dd07402a554d8d274e825 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 11 Apr 2024 11:31:26 +0100 Subject: [PATCH 06/96] fix issue where automation meta fields were being overridden --- packages/server/src/threads/automation.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 4e33fadce6..469d0845c9 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -374,11 +374,13 @@ class Orchestrator { for (let [innerObject, innerValue] of Object.entries( originalStepInput[key][innerKey] )) { - originalStepInput[key][innerKey][innerObject] = - automationUtils.substituteLoopStep( - innerValue as string, - `steps.${loopStepNumber}` - ) + if (typeof innerValue === "string") { + originalStepInput[key][innerKey][innerObject] = + automationUtils.substituteLoopStep( + innerValue, + `steps.${loopStepNumber}` + ) + } } } } @@ -458,7 +460,6 @@ class Orchestrator { inputs, step.schema.inputs ) - try { // appId is always passed const outputs = await stepFn({ From 3f85514a0a999ad8996921dfc995c12c17aa4d91 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 Apr 2024 12:44:33 +0200 Subject: [PATCH 07/96] Fix --- .../settings/controls/FilterEditor/FilterDrawer.svelte | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte index c235ea4591..af2b13d7e2 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte @@ -31,7 +31,7 @@ let matchAny = false let onEmptyFilter = "all" - $: parseFilters(filters) + $: parseFilters(rawFilters) $: dispatch("change", enrichFilters(rawFilters, matchAny, onEmptyFilter)) // Remove field key prefixes and determine which behaviours to use @@ -79,7 +79,12 @@ - +
onFieldChange(filter)} - placeholder="Column" - /> - onValueTypeChange(filter)} + bind:value={filter.field} + options={fieldOptions} + on:change={() => onFieldChange(filter)} + placeholder="Column" + /> + - {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === SearchQueryOperators.ONE_OF)} - - {:else if filter.type === FieldType.OPTIONS} - - {:else if filter.type === FieldType.BOOLEAN} - - {:else if filter.type === FieldType.DATETIME} - - {:else} - - {/if} -
- duplicateFilter(filter.id)} - /> -
-
- removeFilter(filter.id)} - /> -
- {/each} + {#if allowBindings} + + {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === SearchQueryOperators.ONE_OF)} + + {:else if filter.type === FieldType.OPTIONS} + + {:else if filter.type === FieldType.BOOLEAN} + + {:else if filter.type === FieldType.DATETIME} + + {:else} + + {/if} +
+ duplicateFilter(filter.id)} + /> +
+
+ removeFilter(filter.id)} + /> +
+ {/each} +
+ {/if} +
+
+ {:else} + None of the table column can be used for filtering. {/if} -
- -
From 6945ed5674e6bcc333ada1e6a9d4b5a032313b2d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 Apr 2024 12:51:40 +0200 Subject: [PATCH 09/96] Format --- .../src/components/FilterBuilder.svelte | 156 +++++++++--------- 1 file changed, 79 insertions(+), 77 deletions(-) diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte index 5aef3d0fd0..9fee854649 100644 --- a/packages/frontend-core/src/components/FilterBuilder.svelte +++ b/packages/frontend-core/src/components/FilterBuilder.svelte @@ -157,86 +157,88 @@ {/if} {#if filters?.length} -
- -
-
- {#each filters as filter} - onOperatorChange(filter)} - placeholder={null} - /> - {#if allowBindings} +
+
+ +
+
+ {#each filters as filter} onOperatorChange(filter)} placeholder={null} /> - {/if} - {#if allowBindings && filter.field && filter.valueType === "Binding"} - - {:else if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)} - - {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === SearchQueryOperators.ONE_OF)} - - {:else if filter.type === FieldType.OPTIONS} - - {:else if filter.type === FieldType.BOOLEAN} - - {:else if filter.type === FieldType.DATETIME} - - {:else} - - {/if} -
- duplicateFilter(filter.id)} - /> -
-
- removeFilter(filter.id)} - /> -
- {/each} + {#if allowBindings} + + {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === SearchQueryOperators.ONE_OF)} + + {:else if filter.type === FieldType.OPTIONS} + + {:else if filter.type === FieldType.BOOLEAN} + + {:else if filter.type === FieldType.DATETIME} + + {:else} + + {/if} +
+ duplicateFilter(filter.id)} + /> +
+
+ removeFilter(filter.id)} + /> +
+ {/each} +
{/if}
From 2fcdf2602ec5bdc0421e77634f995c574343aa0f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 Apr 2024 16:00:56 +0200 Subject: [PATCH 10/96] Handle extra filters in base --- .../controls/FilterEditor/FilterDrawer.svelte | 70 +++++-------------- .../src/components/FilterBuilder.svelte | 66 ++++++++++++++++- 2 files changed, 82 insertions(+), 54 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte index af2b13d7e2..49c2f17c59 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte @@ -1,5 +1,5 @@ -
- opt.label} - getOptionValue={opt => opt.value} - on:change={e => (onEmptyFilter = e.detail)} - placeholder={null} - /> - {/if} -
+
filter.operator === "allOr") != null + $: onEmptyFilter = + filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all" + + $: console.warn(filters) + + const behaviourOptions = [ + { value: "and", label: "Match all filters" }, + { value: "or", label: "Match any filter" }, + ] + const onEmptyOptions = [ + { value: "all", label: "Return all table rows" }, + { value: "none", label: "Return no rows" }, + ] + const context = getContext("context") $: fieldOptions = (schemaFields ?? []) @@ -144,6 +160,22 @@ filter.value = filter.type === FieldType.ARRAY ? [] : null } } + + function handleAllOr(option) { + filters = filters.filter(f => f.operator !== "allOr") + if (option === "or") { + filters.push({ operator: "allOr" }) + } + } + + function handleOnEmptyFilter(value) { + const existingFilter = filters?.find(filter => filter.onEmptyFilter) + if (existingFilter) { + existingFilter.onEmptyFilter = value + } else { + filters.push({ onEmptyFilter: value }) + } + }
@@ -154,6 +186,30 @@ Add your first filter expression. {:else} + {#if behaviourFilters} +
+ opt.label} + getOptionValue={opt => opt.value} + on:change={e => handleOnEmptyFilter(e.detail)} + placeholder={null} + /> + {/if} +
+ {/if} {/if} {#if filters?.length} @@ -162,7 +218,7 @@
- {#each filters as filter} + {#each filters.filter(filter => filter.operator !== "allOr" && !filter.onEmptyFilter) as filter} {/if} diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte b/packages/frontend-core/src/components/FilterUsers.svelte similarity index 100% rename from packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte rename to packages/frontend-core/src/components/FilterUsers.svelte From 1f77b09eed7a644136da163e39d5ec2ab7acaedb Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 11 Apr 2024 16:22:16 +0200 Subject: [PATCH 12/96] Handle drawers --- .../SetupPanel/AutomationBlockSetup.svelte | 4 +- .../buttons/TableFilterButton.svelte | 4 +- ...lterDrawer.svelte => FilterBuilder.svelte} | 53 +++++++------------ .../controls/FilterEditor/FilterEditor.svelte | 27 ++++++---- .../app/dynamic-filter/FilterModal.svelte | 7 ++- 5 files changed, 47 insertions(+), 48 deletions(-) rename packages/builder/src/components/design/settings/controls/FilterEditor/{FilterDrawer.svelte => FilterBuilder.svelte} (67%) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 0632993cf0..717905eb90 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -37,7 +37,7 @@ hbAutocomplete, EditorModes, } from "components/common/CodeEditor" - import FilterDrawer from "components/design/settings/controls/FilterEditor/FilterDrawer.svelte" + import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte" import { LuceneUtils, Utils } from "@budibase/frontend-core" import { getSchemaForDatasourcePlus, @@ -442,7 +442,7 @@ - import { createEventDispatcher } from "svelte" import { ActionButton, Modal, ModalContent } from "@budibase/bbui" - import FilterDrawer from "components/design/settings/controls/FilterEditor/FilterDrawer.svelte" + import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte" export let schema export let filters @@ -40,7 +40,7 @@ onConfirm={() => dispatch("change", tempValue)} >
- - import { DrawerContent } from "@budibase/bbui" import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte" import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte" @@ -61,35 +60,23 @@ } - - -
-
- (filter.value = event.detail)} - /> -
- - - - + +
+
+ (filter.value = event.detail)} + /> +
+ diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte index 0f1f08d823..e481bb4381 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte @@ -1,8 +1,14 @@ From 6e4a66b2e1101566533f6397af9877a5882f20b3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 11 Apr 2024 18:19:47 +0100 Subject: [PATCH 15/96] Initial implementation of generating SQS junction table definitions. --- hosting/.env | 3 +- hosting/docker-compose.dev.yaml | 3 +- packages/backend-core/src/environment.ts | 2 +- packages/server/src/environment.ts | 2 + .../server/src/sdk/app/tables/internal/sqs.ts | 60 ++++++++++++++++--- packages/types/src/documents/app/sqlite.ts | 20 ++++--- 6 files changed, 72 insertions(+), 18 deletions(-) diff --git a/hosting/.env b/hosting/.env index 8a0756c0e3..173d409d04 100644 --- a/hosting/.env +++ b/hosting/.env @@ -17,6 +17,7 @@ APP_PORT=4002 WORKER_PORT=4003 MINIO_PORT=4004 COUCH_DB_PORT=4005 +COUCH_DB_SQS_PORT=4006 REDIS_PORT=6379 WATCHTOWER_PORT=6161 BUDIBASE_ENVIRONMENT=PRODUCTION @@ -28,4 +29,4 @@ BB_ADMIN_USER_PASSWORD= # A path that is watched for plugin bundles. Any bundles found are imported automatically/ PLUGINS_DIR= -ROLLING_LOG_MAX_SIZE= \ No newline at end of file +ROLLING_LOG_MAX_SIZE= diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index 9dba5d427c..77f6bd053b 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -42,12 +42,13 @@ services: couchdb-service: container_name: budi-couchdb3-dev restart: on-failure - image: budibase/couchdb + image: budibase/couchdb:v3.2.1-sqs environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} ports: - "${COUCH_DB_PORT}:5984" + - "${COUCH_DB_SQS_PORT}:4984" volumes: - couchdb_data:/data diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 2da2a77d67..8dbc904643 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -107,7 +107,7 @@ const environment = { ENCRYPTION_KEY: process.env.ENCRYPTION_KEY, API_ENCRYPTION_KEY: getAPIEncryptionKey(), COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", - COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984", + COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006", COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index f8adcbe0ee..d9d299d5fa 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -28,6 +28,7 @@ const DEFAULTS = { PLUGINS_DIR: "/plugins", FORKED_PROCESS_NAME: "main", JS_RUNNER_MEMORY_LIMIT: 64, + COUCH_DB_SQL_URL: "http://localhost:4006", } const QUERY_THREAD_TIMEOUT = @@ -39,6 +40,7 @@ const environment = { // important - prefer app port to generic port PORT: process.env.APP_PORT || process.env.PORT, COUCH_DB_URL: process.env.COUCH_DB_URL, + COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || DEFAULTS.COUCH_DB_SQL_URL, MINIO_URL: process.env.MINIO_URL, WORKER_URL: process.env.WORKER_URL, AWS_REGION: process.env.AWS_REGION, diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index 79d9be2348..5dd16f516c 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -1,8 +1,19 @@ import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core" -import { FieldType, SQLiteDefinition, SQLiteType, Table } from "@budibase/types" +import { + FieldType, + RelationshipFieldMetadata, + SQLiteDefinition, + SQLiteTable, + SQLiteTables, + SQLiteType, + Table, +} from "@budibase/types" import { cloneDeep } from "lodash" import tablesSdk from "../" -import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils" +import { + CONSTANT_INTERNAL_ROW_COLS, + generateJunctionTableID, +} from "../../../../db/utils" const BASIC_SQLITE_DOC: SQLiteDefinition = { _id: SQLITE_DESIGN_DOC_ID, @@ -36,9 +47,38 @@ const FieldTypeMap: Record = { [FieldType.BB_REFERENCE]: SQLiteType.TEXT, } -function mapTable(table: Table): { [key: string]: SQLiteType } { +function buildRelationshipDefinitions( + table: Table, + relationshipColumn: RelationshipFieldMetadata +): { + tableId: string + definition: SQLiteTable +} { + const tableId = table._id!, + relatedTableId = relationshipColumn.tableId + return { + tableId: generateJunctionTableID(tableId, relatedTableId), + definition: { + doc1: SQLiteType.BLOB, + doc2: SQLiteType.BLOB, + tableId: SQLiteType.TEXT, + }, + } +} + +// this can generate relationship tables as part of the mapping +function mapTable(table: Table): SQLiteTables { + const tables: SQLiteTables = {} const fields: Record = {} for (let [key, column] of Object.entries(table.schema)) { + // relationships should be handled differently + if (column.type === FieldType.LINK) { + const { tableId, definition } = buildRelationshipDefinitions( + table, + column + ) + tables[tableId] = { fields: definition } + } if (!FieldTypeMap[column.type]) { throw new Error(`Unable to map type "${column.type}" to SQLite type`) } @@ -49,10 +89,12 @@ function mapTable(table: Table): { [key: string]: SQLiteType } { CONSTANT_INTERNAL_ROW_COLS.forEach(col => { constantMap[col] = SQLiteType.TEXT }) - return { + const thisTable: SQLiteTable = { ...constantMap, ...fields, } + tables[table._id!] = { fields: thisTable } + return tables } // nothing exists, need to iterate though existing tables @@ -60,8 +102,9 @@ async function buildBaseDefinition(): Promise { const tables = await tablesSdk.getAllInternalTables() const definition = cloneDeep(BASIC_SQLITE_DOC) for (let table of tables) { - definition.sql.tables[table._id!] = { - fields: mapTable(table), + definition.sql.tables = { + ...definition.sql.tables, + ...mapTable(table), } } return definition @@ -75,8 +118,9 @@ export async function addTableToSqlite(table: Table) { } catch (err) { definition = await buildBaseDefinition() } - definition.sql.tables[table._id!] = { - fields: mapTable(table), + definition.sql.tables = { + ...definition.sql.tables, + ...mapTable(table), } await db.put(definition) } diff --git a/packages/types/src/documents/app/sqlite.ts b/packages/types/src/documents/app/sqlite.ts index 76c47bbd74..e23a68b336 100644 --- a/packages/types/src/documents/app/sqlite.ts +++ b/packages/types/src/documents/app/sqlite.ts @@ -6,17 +6,23 @@ export enum SQLiteType { NUMERIC = "NUMERIC", } +export type SQLiteTable = Record< + string, + SQLiteType | { field: string; type: SQLiteType } +> + +export type SQLiteTables = Record< + string, + { + fields: SQLiteTable + } +> + export interface SQLiteDefinition { _id: string language: string sql: { - tables: { - [tableName: string]: { - fields: { - [key: string]: SQLiteType | { field: string; type: SQLiteType } - } - } - } + tables: SQLiteTables options: { table_name: string } From d6b252013b80d22bb87677a8494c55663c799de6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 11 Apr 2024 18:25:18 +0100 Subject: [PATCH 16/96] Quick fix to link document structure in sqlite. --- packages/server/src/sdk/app/tables/internal/sqs.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index 5dd16f516c..99240c28d4 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -59,8 +59,12 @@ function buildRelationshipDefinitions( return { tableId: generateJunctionTableID(tableId, relatedTableId), definition: { - doc1: SQLiteType.BLOB, - doc2: SQLiteType.BLOB, + ["doc1.rowId"]: SQLiteType.TEXT, + ["doc1.tableId"]: SQLiteType.TEXT, + ["doc1.fieldName"]: SQLiteType.TEXT, + ["doc2.rowId"]: SQLiteType.TEXT, + ["doc2.tableId"]: SQLiteType.TEXT, + ["doc2.fieldName"]: SQLiteType.TEXT, tableId: SQLiteType.TEXT, }, } From 173debd403c8ba1de9a2eae873ea7371a31a7107 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 Apr 2024 09:07:10 +0200 Subject: [PATCH 17/96] Fix bindings --- .../FilterEditor/FilterBuilder.svelte | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte index 0ae3928875..f2f61fdb99 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte @@ -68,15 +68,22 @@ {allowBindings} >
-
- (filter.value = event.detail)} - /> -
+ + { + const indexToUpdate = rawFilters.findIndex(f => f.id === filter.id) + rawFilters[indexToUpdate] = { + ...rawFilters[indexToUpdate], + value: event.detail, + } + }} + /> From db9809207e8c4ee0757dcc9ae9f47a562c119cbb Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 Apr 2024 09:13:16 +0200 Subject: [PATCH 18/96] Fix client --- .../app/dynamic-filter/FilterModal.svelte | 216 +----------------- .../src/components/FilterBuilder.svelte | 2 - 2 files changed, 2 insertions(+), 216 deletions(-) diff --git a/packages/client/src/components/app/dynamic-filter/FilterModal.svelte b/packages/client/src/components/app/dynamic-filter/FilterModal.svelte index 496f95f2d7..93fe4acbd5 100644 --- a/packages/client/src/components/app/dynamic-filter/FilterModal.svelte +++ b/packages/client/src/components/app/dynamic-filter/FilterModal.svelte @@ -1,221 +1,9 @@ -
- - - - {#if !filters?.length} - Add your first filter expression. - {:else} - Results are filtered to only those which match all of the following - constraints. - {/if} - - {#if filters?.length} -
- {#each filters as filter} - onOperatorChange(filter, e.detail)} - placeholder={null} - /> - {#if ["string", "longform", "number", "bigint", "formula"].includes(filter.type)} - - {:else if ["options", "array"].includes(filter.type)} - - {:else if filter.type === "boolean"} - - {:else if filter.type === "datetime"} - - {:else} - - {/if} -
- duplicateFilter(filter.id)} - /> - removeFilter(filter.id)} - /> -
- {/each} -
- {/if} -
- -
- -
- - + diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte index 14bbc47e86..9125be401b 100644 --- a/packages/frontend-core/src/components/FilterBuilder.svelte +++ b/packages/frontend-core/src/components/FilterBuilder.svelte @@ -29,8 +29,6 @@ $: onEmptyFilter = filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all" - $: console.warn(filters) - const behaviourOptions = [ { value: "and", label: "Match all filters" }, { value: "or", label: "Match any filter" }, From 4b382200481d3e040c1149d417ae4e01e02101dc Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 Apr 2024 09:50:11 +0200 Subject: [PATCH 19/96] Fix drawer --- .../frontend-core/src/components/FilterBuilder.svelte | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte index 9125be401b..dfdf48a6ec 100644 --- a/packages/frontend-core/src/components/FilterBuilder.svelte +++ b/packages/frontend-core/src/components/FilterBuilder.svelte @@ -29,6 +29,10 @@ $: onEmptyFilter = filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all" + $: fieldFilters = filters.filter( + filter => filter.operator !== "allOr" && !filter.onEmptyFilter + ) + const behaviourOptions = [ { value: "and", label: "Match all filters" }, { value: "or", label: "Match any filter" }, @@ -177,7 +181,7 @@ {#if fieldOptions?.length} - {#if !filters?.length} + {#if !fieldFilters?.length} Add your first filter expression. {:else} @@ -207,13 +211,13 @@ {/if} {/if} - {#if filters?.length} + {#if fieldFilters?.length}
- {#each filters.filter(filter => filter.operator !== "allOr" && !filter.onEmptyFilter) as filter} + {#each fieldFilters as filter} {#if fieldFilters?.length}
-
- -
+ {#if filtersLabel} +
+ +
+ {/if}
{#each fieldFilters as filter} Date: Fri, 12 Apr 2024 10:27:34 +0200 Subject: [PATCH 22/96] Fix api --- packages/frontend-core/src/components/FilterUsers.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/frontend-core/src/components/FilterUsers.svelte b/packages/frontend-core/src/components/FilterUsers.svelte index 88383ba170..1712d7ebdf 100644 --- a/packages/frontend-core/src/components/FilterUsers.svelte +++ b/packages/frontend-core/src/components/FilterUsers.svelte @@ -1,9 +1,9 @@ diff --git a/packages/client/src/components/app/forms/AttachmentField.svelte b/packages/client/src/components/app/forms/AttachmentField.svelte index 644630810d..3489fd809c 100644 --- a/packages/client/src/components/app/forms/AttachmentField.svelte +++ b/packages/client/src/components/app/forms/AttachmentField.svelte @@ -58,17 +58,6 @@ } } - const deleteAttachments = async fileList => { - try { - return await API.deleteAttachments({ - keys: fileList, - tableId: formContext?.dataSource?.tableId, - }) - } catch (error) { - return [] - } - } - const handleChange = e => { const value = fieldApiMapper.set(e.detail) const changed = fieldApi.setValue(value) @@ -98,7 +87,6 @@ error={fieldState.error} on:change={handleChange} {processFiles} - {deleteAttachments} {handleFileTooLarge} {handleTooManyFiles} {maximum} diff --git a/packages/frontend-core/src/api/attachments.js b/packages/frontend-core/src/api/attachments.js index f79b461574..e3b1b74e5b 100644 --- a/packages/frontend-core/src/api/attachments.js +++ b/packages/frontend-core/src/api/attachments.js @@ -61,32 +61,5 @@ export const buildAttachmentEndpoints = API => { }) return { publicUrl } }, - - /** - * Deletes attachments from the bucket. - * @param keys the attachments to delete - * @param tableId the associated table ID - */ - deleteAttachments: async ({ keys, tableId }) => { - return await API.post({ - url: `/api/attachments/${tableId}/delete`, - body: { - keys, - }, - }) - }, - - /** - * Deletes attachments from the builder bucket. - * @param keys the attachments to delete - */ - deleteBuilderAttachments: async keys => { - return await API.post({ - url: `/api/attachments/delete`, - body: { - keys, - }, - }) - }, } } diff --git a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte index 3a1f165b6e..e7dc51e5d5 100644 --- a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte @@ -61,14 +61,6 @@ } } - const deleteAttachments = async fileList => { - try { - return await API.deleteBuilderAttachments(fileList) - } catch (error) { - return [] - } - } - onMount(() => { api = { focus: () => open(), @@ -101,7 +93,6 @@ on:change={e => onChange(e.detail)} maximum={maximum || schema.constraints?.length?.maximum} {processFiles} - {deleteAttachments} {handleFileTooLarge} />
From ebb79c16fe7d626d9d201fe5d1090da6e4ab078f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 12 Apr 2024 16:15:36 +0100 Subject: [PATCH 25/96] Aliasing support for SQS. --- .../api/controllers/row/ExternalRequest.ts | 3 +- packages/server/src/db/utils.ts | 1 + .../src/integrations/tests/sqlAlias.spec.ts | 4 +- packages/server/src/sdk/app/rows/index.ts | 2 + .../server/src/sdk/app/rows/search/sqs.ts | 99 +++++++++++-------- .../row/alias.ts => sdk/app/rows/sqlAlias.ts} | 28 ++++-- 6 files changed, 86 insertions(+), 51 deletions(-) rename packages/server/src/{api/controllers/row/alias.ts => sdk/app/rows/sqlAlias.ts} (87%) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 7fc0333de1..4adbb72c7a 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -36,7 +36,6 @@ import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils" import { processObjectSync } from "@budibase/string-templates" import { cloneDeep } from "lodash/fp" import { db as dbCore } from "@budibase/backend-core" -import AliasTables from "./alias" import sdk from "../../../sdk" import env from "../../../environment" @@ -618,7 +617,7 @@ export class ExternalRequest { if (env.SQL_ALIASING_DISABLE) { response = await getDatasourceAndQuery(json) } else { - const aliasing = new AliasTables(Object.keys(this.tables)) + const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables)) response = await aliasing.queryWithAliasing(json) } diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index b1c02b1764..ce8d0accbb 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -40,6 +40,7 @@ export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.Inte export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}` export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}` export const AUTOMATION_LOG_PREFIX = `${DocumentType.AUTOMATION_LOG}${SEPARATOR}` +export const SQS_DATASOURCE_INTERNAL = "internal" export const ViewName = dbCore.ViewName export const InternalTables = dbCore.InternalTable export const UNICODE_MAX = dbCore.UNICODE_MAX diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index bfca24ff7d..58c3a05245 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -8,8 +8,10 @@ import { import { join } from "path" import Sql from "../base/sql" import { SqlClient } from "../utils" -import AliasTables from "../../api/controllers/row/alias" import { generator } from "@budibase/backend-core/tests" +import sdk from "../../sdk" + +const AliasTables = sdk.rows.AliasTables function multiline(sql: string) { return sql.replace(/\n/g, "").replace(/ +/g, " ") diff --git a/packages/server/src/sdk/app/rows/index.ts b/packages/server/src/sdk/app/rows/index.ts index ea501e93d9..c117941419 100644 --- a/packages/server/src/sdk/app/rows/index.ts +++ b/packages/server/src/sdk/app/rows/index.ts @@ -3,6 +3,7 @@ import * as rows from "./rows" import * as search from "./search" import * as utils from "./utils" import * as external from "./external" +import AliasTables from "./sqlAlias" export default { ...attachments, @@ -10,4 +11,5 @@ export default { ...search, utils, external, + AliasTables, } diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 5b0b6e3bc7..20edb988d3 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -20,7 +20,12 @@ import { } from "../../../../api/controllers/row/utils" import sdk from "../../../index" import { context } from "@budibase/backend-core" -import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils" +import { + CONSTANT_INTERNAL_ROW_COLS, + SQS_DATASOURCE_INTERNAL, +} from "../../../../db/utils" +import AliasTables from "../sqlAlias" +import { outputProcessing } from "../../../../utilities/rowProcessor" function buildInternalFieldList( table: Table, @@ -31,19 +36,19 @@ function buildInternalFieldList( fieldList = fieldList.concat( CONSTANT_INTERNAL_ROW_COLS.map(col => `${table._id}.${col}`) ) - if (opts.relationships) { - for (let col of Object.values(table.schema)) { - if (col.type === FieldType.LINK) { - const linkCol = col as RelationshipFieldMetadata - const relatedTable = tables.find( - table => table._id === linkCol.tableId - )! - fieldList = fieldList.concat( - buildInternalFieldList(relatedTable, tables, { relationships: false }) - ) - } else { - fieldList.push(`${table._id}.${col.name}`) - } + for (let col of Object.values(table.schema)) { + const isRelationship = col.type === FieldType.LINK + if (!opts.relationships && isRelationship) { + continue + } + if (isRelationship) { + const linkCol = col as RelationshipFieldMetadata + const relatedTable = tables.find(table => table._id === linkCol.tableId)! + fieldList = fieldList.concat( + buildInternalFieldList(relatedTable, tables, { relationships: false }) + ) + } else { + fieldList.push(`${table._id}.${col.name}`) } } return fieldList @@ -94,14 +99,14 @@ function buildTableMap(tables: Table[]) { } export async function search( - options: RowSearchParams + options: RowSearchParams, + table: Table ): Promise> { - const { tableId, paginate, query, ...params } = options + const { paginate, query, ...params } = options const builder = new SqlQueryBuilder(SqlClient.SQL_LITE) const allTables = await sdk.tables.getAllInternalTables() const allTablesMap = buildTableMap(allTables) - const table = allTables.find(table => table._id === tableId) if (!table) { throw new Error("Unable to find table") } @@ -111,7 +116,7 @@ export async function search( const request: QueryJson = { endpoint: { // not important, we query ourselves - datasourceId: "internal", + datasourceId: SQS_DATASOURCE_INTERNAL, entityId: table._id!, operation: Operation.READ, }, @@ -154,34 +159,44 @@ export async function search( } } try { - const query = builder._query(request, { - disableReturning: true, + const alias = new AliasTables(allTables.map(table => table.name)) + const rows = await alias.queryWithAliasing(request, async json => { + const query = builder._query(json, { + disableReturning: true, + }) + + if (Array.isArray(query)) { + throw new Error("SQS cannot currently handle multiple queries") + } + + let sql = query.sql, + bindings = query.bindings + + // quick hack for docIds + sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`") + sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`") + + const db = context.getAppDB() + return await db.sql(sql, bindings) }) - if (Array.isArray(query)) { - throw new Error("SQS cannot currently handle multiple queries") - } - - let sql = query.sql, - bindings = query.bindings - - // quick hack for docIds - sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`") - sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`") - - const db = context.getAppDB() - const rows = await db.sql(sql, bindings) + // process from the format of tableId.column to expected format + const processed = await sqlOutputProcessing( + rows, + table!, + allTablesMap, + relationships, + { + sqs: true, + } + ) return { - rows: await sqlOutputProcessing( - rows, - table!, - allTablesMap, - relationships, - { - sqs: true, - } - ), + // final row processing for response + rows: await outputProcessing(table, processed, { + preserveLinks: true, + squash: true, + }), } } catch (err: any) { const msg = typeof err === "string" ? err : err.message diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts similarity index 87% rename from packages/server/src/api/controllers/row/alias.ts rename to packages/server/src/sdk/app/rows/sqlAlias.ts index 0ec9d1a09c..0fc338ecbe 100644 --- a/packages/server/src/api/controllers/row/alias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -6,11 +6,12 @@ import { Row, SearchFilters, } from "@budibase/types" -import { getSQLClient } from "../../../sdk/app/rows/utils" +import { getSQLClient } from "./utils" import { cloneDeep } from "lodash" -import sdk from "../../../sdk" +import datasources from "../datasources" import { makeExternalQuery } from "../../../integrations/base/query" import { SqlClient } from "../../../integrations/utils" +import { SQS_DATASOURCE_INTERNAL } from "../../../db/utils" const WRITE_OPERATIONS: Operation[] = [ Operation.CREATE, @@ -156,12 +157,19 @@ export default class AliasTables { } async queryWithAliasing( - json: QueryJson + json: QueryJson, + queryFn?: (json: QueryJson) => Promise ): Promise { const datasourceId = json.endpoint.datasourceId - const datasource = await sdk.datasources.get(datasourceId) + const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL + let aliasingEnabled: boolean, datasource: Datasource | undefined + if (isSqs) { + aliasingEnabled = true + } else { + datasource = await datasources.get(datasourceId) + aliasingEnabled = this.isAliasingEnabled(json, datasource) + } - const aliasingEnabled = this.isAliasingEnabled(json, datasource) if (aliasingEnabled) { json = cloneDeep(json) // run through the query json to update anywhere a table may be used @@ -207,7 +215,15 @@ export default class AliasTables { } json.tableAliases = invertedTableAliases } - const response = await makeExternalQuery(datasource, json) + + let response: DatasourcePlusQueryResponse + if (datasource && !isSqs) { + response = await makeExternalQuery(datasource, json) + } else if (queryFn) { + response = await queryFn(json) + } else { + throw new Error("No supplied method to perform aliased query") + } if (Array.isArray(response) && aliasingEnabled) { return this.reverse(response) } else { From c40e9656345f5e95ead009976ab00755973b4845 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 12 Apr 2024 16:16:31 +0100 Subject: [PATCH 26/96] Getting relationships working properly as well as renaming internal -> sqs in function opts. --- packages/server/src/api/controllers/row/utils/basic.ts | 9 +++++---- .../server/src/api/controllers/row/utils/sqlUtils.ts | 4 ++-- packages/server/src/api/controllers/row/utils/utils.ts | 6 ++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index 1fc84de9c7..6255e13c1c 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -62,12 +62,12 @@ export function basicProcessing({ row, table, isLinked, - internal, + sqs, }: { row: Row table: Table isLinked: boolean - internal?: boolean + sqs?: boolean }): Row { const thisRow: Row = {} // filter the row down to what is actually the row (not joined) @@ -84,12 +84,13 @@ export function basicProcessing({ thisRow[fieldName] = value } } - if (!internal) { + if (!sqs) { thisRow._id = generateIdForRow(row, table, isLinked) thisRow.tableId = table._id thisRow._rev = "rev" } else { - for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) { + const columns = Object.keys(table.schema) + for (let internalColumn of [...CONSTANT_INTERNAL_ROW_COLS, ...columns]) { thisRow[internalColumn] = extractFieldValue({ row, tableName: table._id!, diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index 6f9837e0ab..372b8394ff 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -51,11 +51,11 @@ export async function updateRelationshipColumns( continue } - let linked = await basicProcessing({ + let linked = basicProcessing({ row, table: linkedTable, isLinked: true, - internal: opts?.sqs, + sqs: opts?.sqs, }) if (!linked._id) { continue diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index f387a468cf..bf9ede6fe3 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -132,6 +132,7 @@ export async function sqlOutputProcessing( let rowId = row._id if (opts?.sqs) { rowId = getInternalRowId(row, table) + row._id = rowId } else if (!rowId) { rowId = generateIdForRow(row, table) row._id = rowId @@ -153,7 +154,7 @@ export async function sqlOutputProcessing( row, table, isLinked: false, - internal: opts?.sqs, + sqs: opts?.sqs, }), table ) @@ -167,7 +168,8 @@ export async function sqlOutputProcessing( tables, row, finalRows, - relationships + relationships, + opts ) } From bfb7750213400e833bfb4ffeae0be9462b66e0bc Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 12 Apr 2024 16:17:06 +0100 Subject: [PATCH 27/96] Getting search input mapping up a level in the search SDK - avoids having to call it for every search type. --- packages/server/src/sdk/app/rows/search.ts | 11 +++++++--- .../src/sdk/app/rows/search/external.ts | 13 +++++++----- .../src/sdk/app/rows/search/internal.ts | 20 ++++++++++--------- .../app/rows/search/tests/external.spec.ts | 6 +++--- .../app/rows/search/tests/internal.spec.ts | 4 ++-- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index f681bfeb90..5d8f7ef80b 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -13,6 +13,8 @@ import * as sqs from "./search/sqs" import env from "../../../environment" import { ExportRowsParams, ExportRowsResult } from "./search/types" import { dataFilters } from "@budibase/shared-core" +import sdk from "../../index" +import { searchInputMapping } from "./search/utils" export { isValidFilter } from "../../../integrations/utils" @@ -72,12 +74,15 @@ export async function search( } } + const table = await sdk.tables.getTable(options.tableId) + options = searchInputMapping(table, options) + if (isExternalTable) { - return external.search(options) + return external.search(options, table) } else if (env.SQS_SEARCH_ENABLE) { - return sqs.search(options) + return sqs.search(options, table) } else { - return internal.search(options) + return internal.search(options, table) } } diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index e0a3bad94e..077f971903 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -8,6 +8,7 @@ import { SearchFilters, RowSearchParams, SearchResponse, + Table, } from "@budibase/types" import * as exporters from "../../../../api/controllers/view/exporters" import { handleRequest } from "../../../../api/controllers/row/external" @@ -18,13 +19,13 @@ import { import { utils } from "@budibase/shared-core" import { ExportRowsParams, ExportRowsResult } from "./types" import { HTTPError, db } from "@budibase/backend-core" -import { searchInputMapping } from "./utils" import pick from "lodash/pick" import { outputProcessing } from "../../../../utilities/rowProcessor" import sdk from "../../../" export async function search( - options: RowSearchParams + options: RowSearchParams, + table: Table ): Promise> { const { tableId } = options const { paginate, query, ...params } = options @@ -68,8 +69,6 @@ export async function search( } try { - const table = await sdk.tables.getTable(tableId) - options = searchInputMapping(table, options) let rows = await handleRequest(Operation.READ, tableId, { filters: query, sort, @@ -150,11 +149,15 @@ export async function exportRows( } const datasource = await sdk.datasources.get(datasourceId!) + const table = await sdk.tables.getTable(tableId) if (!datasource || !datasource.entities) { throw new HTTPError("Datasource has not been configured for plus API.", 400) } - let result = await search({ tableId, query: requestQuery, sort, sortOrder }) + let result = await search( + { tableId, query: requestQuery, sort, sortOrder }, + table + ) let rows: Row[] = [] let headers diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 610807a10e..ffd13ed731 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -33,7 +33,8 @@ import pick from "lodash/pick" import { breakRowIdField } from "../../../../integrations/utils" export async function search( - options: RowSearchParams + options: RowSearchParams, + table: Table ): Promise> { const { tableId } = options @@ -51,8 +52,6 @@ export async function search( query: {}, } - let table = await sdk.tables.getTable(tableId) - options = searchInputMapping(table, options) if (params.sort && !params.sortType) { const schema = table.schema const sortField = schema[params.sort] @@ -122,12 +121,15 @@ export async function exportRows( result = await outputProcessing(table, response) } else if (query) { - let searchResponse = await search({ - tableId, - query, - sort, - sortOrder, - }) + let searchResponse = await search( + { + tableId, + query, + sort, + sortOrder, + }, + table + ) result = searchResponse.rows } diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts index f2bdec4692..53bc049a9b 100644 --- a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts @@ -112,7 +112,7 @@ describe("external search", () => { tableId, query: {}, } - const result = await search(searchParams) + const result = await search(searchParams, config.table!) expect(result.rows).toHaveLength(10) expect(result.rows).toEqual( @@ -130,7 +130,7 @@ describe("external search", () => { query: {}, fields: ["name", "age"], } - const result = await search(searchParams) + const result = await search(searchParams, config.table!) expect(result.rows).toHaveLength(10) expect(result.rows).toEqual( @@ -157,7 +157,7 @@ describe("external search", () => { }, }, } - const result = await search(searchParams) + const result = await search(searchParams, config.table!) expect(result.rows).toHaveLength(3) expect(result.rows.map(row => row.id)).toEqual([1, 4, 8]) diff --git a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts index 5be0f4a258..1c5f396737 100644 --- a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts @@ -81,7 +81,7 @@ describe("internal", () => { tableId, query: {}, } - const result = await search(searchParams) + const result = await search(searchParams, config.table!) expect(result.rows).toHaveLength(10) expect(result.rows).toEqual( @@ -99,7 +99,7 @@ describe("internal", () => { query: {}, fields: ["name", "age"], } - const result = await search(searchParams) + const result = await search(searchParams, config.table!) expect(result.rows).toHaveLength(10) expect(result.rows).toEqual( From 7d7de33cabbcfcefb7ea668a92813b96c1e28b3b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 12 Apr 2024 16:29:48 +0100 Subject: [PATCH 28/96] Removing CouchDB SQS image for now. --- hosting/docker-compose.dev.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index 77f6bd053b..9dba5d427c 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -42,13 +42,12 @@ services: couchdb-service: container_name: budi-couchdb3-dev restart: on-failure - image: budibase/couchdb:v3.2.1-sqs + image: budibase/couchdb environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} ports: - "${COUCH_DB_PORT}:5984" - - "${COUCH_DB_SQS_PORT}:4984" volumes: - couchdb_data:/data From aeda5931c07c84ed826e5e37bd54b3382d955653 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 12 Apr 2024 16:34:33 +0100 Subject: [PATCH 29/96] Fixing lint. --- packages/server/src/sdk/app/rows/search/internal.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index ffd13ed731..906ca016d1 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -1,6 +1,6 @@ import { context, db, HTTPError } from "@budibase/backend-core" import env from "../../../../environment" -import { fullSearch, paginatedSearch, searchInputMapping } from "./utils" +import { fullSearch, paginatedSearch } from "./utils" import { getRowParams, InternalTables } from "../../../../db/utils" import { Database, From 68c5e657ddd0fb5b45948e418531d72cd16b178e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 15 Apr 2024 13:46:31 +0100 Subject: [PATCH 30/96] Updating @types/archiver to be more specific. --- packages/server/package.json | 2 +- yarn.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index ad03033e67..76402785d7 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -125,7 +125,7 @@ "@babel/preset-env": "7.16.11", "@swc/core": "1.3.71", "@swc/jest": "0.2.27", - "@types/archiver": "^6.0.2", + "@types/archiver": "6.0.2", "@types/global-agent": "2.1.1", "@types/google-spreadsheet": "3.1.5", "@types/jest": "29.5.5", diff --git a/yarn.lock b/yarn.lock index a36b54d3be..ce39c89075 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5174,7 +5174,7 @@ dependencies: "@types/node" "*" -"@types/archiver@^6.0.2": +"@types/archiver@6.0.2": version "6.0.2" resolved "https://registry.yarnpkg.com/@types/archiver/-/archiver-6.0.2.tgz#0daf8c83359cbde69de1e4b33dcade6a48a929e2" integrity sha512-KmROQqbQzKGuaAbmK+ZcytkJ51+YqDa7NmbXjmtC5YBLSyQYo21YaUnQ3HbaPFKL1ooo6RQ6OPYPIDyxfpDDXw== From b13b7df678768a460c6c4b4dc464bbb4a25dcda1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 15 Apr 2024 18:23:39 +0100 Subject: [PATCH 31/96] Correctly handling aliasing for sorting/json field types with SQS. --- packages/server/src/integrations/base/sql.ts | 31 ++++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 259abec106..b622ca39be 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -22,6 +22,8 @@ import { SortDirection, SqlQueryBinding, Table, + TableSourceType, + INTERNAL_TABLE_SOURCE_ID, } from "@budibase/types" import environment from "../../environment" @@ -135,6 +137,21 @@ function generateSelectStatement( }) } +function getTableName(table?: Table): string { + if (!table) { + throw new Error("No table name available.") + } + // SQS uses the table ID rather than the table name + if ( + table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID + ) { + return table._id! + } else { + return table.name + } +} + class InternalBuilder { private readonly client: string @@ -149,7 +166,7 @@ class InternalBuilder { tableName: string, opts: { aliases?: Record; relationship?: boolean } ): Knex.QueryBuilder { - function getTableName(name: string) { + function getTableAlias(name: string) { const alias = opts.aliases?.[name] return alias || name } @@ -161,11 +178,11 @@ class InternalBuilder { const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") if (!opts.relationship && !isRelationshipField) { - fn(`${getTableName(tableName)}.${updatedKey}`, value) + fn(`${getTableAlias(tableName)}.${updatedKey}`, value) } if (opts.relationship && isRelationshipField) { const [filterTableName, property] = updatedKey.split(".") - fn(`${getTableName(filterTableName)}.${property}`, value) + fn(`${getTableAlias(filterTableName)}.${property}`, value) } } } @@ -346,9 +363,10 @@ class InternalBuilder { addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder { let { sort, paginate } = json const table = json.meta?.table + const tableName = getTableName(table) const aliases = json.tableAliases const aliased = - table?.name && aliases?.[table.name] ? aliases[table.name] : table?.name + table?.name && aliases?.[tableName] ? aliases[tableName] : table?.name if (sort && Object.keys(sort || {}).length > 0) { for (let [key, value] of Object.entries(sort)) { const direction = @@ -729,12 +747,13 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { results: Record[], aliases?: Record ): Record[] { + const tableName = getTableName(table) for (const [name, field] of Object.entries(table.schema)) { if (!this._isJsonColumn(field)) { continue } - const tableName = aliases?.[table.name] || table.name - const fullName = `${tableName}.${name}` + const aliasedTableName = aliases?.[tableName] || tableName + const fullName = `${aliasedTableName}.${name}` for (let row of results) { if (typeof row[fullName] === "string") { row[fullName] = JSON.parse(row[fullName]) From 69ae603fa4d3aae6e61427149057edbbd885e64a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 15 Apr 2024 18:24:11 +0100 Subject: [PATCH 32/96] Updating errors from sql table actions and making sure SQS tables cannot be reach the sql table actions/will error if they do. --- .../server/src/integrations/base/sqlTable.ts | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts index 4ff336421f..3c55d75b8b 100644 --- a/packages/server/src/integrations/base/sqlTable.ts +++ b/packages/server/src/integrations/base/sqlTable.ts @@ -1,19 +1,20 @@ import { Knex, knex } from "knex" import { - RelationshipType, FieldSubtype, + FieldType, NumberFieldMetadata, Operation, QueryJson, + RelationshipType, RenameColumn, - Table, - FieldType, SqlQuery, + Table, + TableSourceType, } from "@budibase/types" import { breakExternalTableId, getNativeSql, SqlClient } from "../utils" +import { utils } from "@budibase/shared-core" import SchemaBuilder = Knex.SchemaBuilder import CreateTableBuilder = Knex.CreateTableBuilder -import { utils } from "@budibase/shared-core" function isIgnoredType(type: FieldType) { const ignored = [FieldType.LINK, FieldType.FORMULA] @@ -105,13 +106,13 @@ function generateSchema( column.relationshipType !== RelationshipType.MANY_TO_MANY ) { if (!column.foreignKey || !column.tableId) { - throw "Invalid relationship schema" + throw new Error("Invalid relationship schema") } const { tableName } = breakExternalTableId(column.tableId) // @ts-ignore const relatedTable = tables[tableName] if (!relatedTable) { - throw "Referenced table doesn't exist" + throw new Error("Referenced table doesn't exist") } const relatedPrimary = relatedTable.primary[0] const externalType = relatedTable.schema[relatedPrimary].externalType @@ -209,15 +210,19 @@ class SqlTableQueryBuilder { let query: Knex.SchemaBuilder if (!json.table || !json.meta || !json.meta.tables) { - throw "Cannot execute without table being specified" + throw new Error("Cannot execute without table being specified") } + if (json.table.sourceType === TableSourceType.INTERNAL) { + throw new Error("Cannot perform table actions for SQS.") + } + switch (this._operation(json)) { case Operation.CREATE_TABLE: query = buildCreateTable(client, json.table, json.meta.tables) break case Operation.UPDATE_TABLE: if (!json.meta || !json.meta.table) { - throw "Must specify old table for update" + throw new Error("Must specify old table for update") } // renameColumn does not work for MySQL, so return a raw query if (this.sqlClient === SqlClient.MY_SQL && json.meta.renamed) { @@ -264,7 +269,7 @@ class SqlTableQueryBuilder { query = buildDeleteTable(client, json.table) break default: - throw "Table operation is of unknown type" + throw new Error("Table operation is of unknown type") } return getNativeSql(query) } From 128596cd4174320ae9381593ffcc6830a4105ba9 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 Apr 2024 10:41:11 +0100 Subject: [PATCH 33/96] Ensure normal spectrum colors are used for nested links in top navs --- packages/client/src/components/app/NavItem.svelte | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/client/src/components/app/NavItem.svelte b/packages/client/src/components/app/NavItem.svelte index 360ea4f8bb..fcdda57ace 100644 --- a/packages/client/src/components/app/NavItem.svelte +++ b/packages/client/src/components/app/NavItem.svelte @@ -157,6 +157,11 @@ width: 100%; } + /* Use normal theme colors for links when using a top nav */ + .dropdown:not(.left) .sublinks a { + color: var(--spectrum-alias-text-color); + } + /* Left dropdowns */ .dropdown.left .sublinks-wrapper { display: none; From e2ca21053e52c88994e2ec75845548673e0707e1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 16 Apr 2024 11:38:00 +0100 Subject: [PATCH 34/96] Fixing build issue caught in CI. --- packages/server/src/integrations/base/sql.ts | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index b622ca39be..59684422e7 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -137,18 +137,15 @@ function generateSelectStatement( }) } -function getTableName(table?: Table): string { - if (!table) { - throw new Error("No table name available.") - } +function getTableName(table?: Table): string | undefined { // SQS uses the table ID rather than the table name if ( - table.sourceType === TableSourceType.INTERNAL || - table.sourceId === INTERNAL_TABLE_SOURCE_ID + table?.sourceType === TableSourceType.INTERNAL || + table?.sourceId === INTERNAL_TABLE_SOURCE_ID ) { - return table._id! + return table?._id } else { - return table.name + return table?.name } } @@ -366,7 +363,7 @@ class InternalBuilder { const tableName = getTableName(table) const aliases = json.tableAliases const aliased = - table?.name && aliases?.[tableName] ? aliases[tableName] : table?.name + tableName && aliases?.[tableName] ? aliases[tableName] : table?.name if (sort && Object.keys(sort || {}).length > 0) { for (let [key, value] of Object.entries(sort)) { const direction = @@ -752,7 +749,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { if (!this._isJsonColumn(field)) { continue } - const aliasedTableName = aliases?.[tableName] || tableName + const aliasedTableName = (tableName && aliases?.[tableName]) || tableName const fullName = `${aliasedTableName}.${name}` for (let row of results) { if (typeof row[fullName] === "string") { From 253fa0def8ce95e11cb38edb613c5270e09d7704 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 16 Apr 2024 16:20:44 +0100 Subject: [PATCH 35/96] In progress: bigint tests. --- .../src/api/routes/tests/search.spec.ts | 97 ++++++++++++++++++- .../src/sdk/app/tables/external/index.ts | 6 ++ .../server/src/sdk/app/tables/internal/sqs.ts | 2 +- 3 files changed, 99 insertions(+), 6 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5b71ec9044..39ba0b589d 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -18,14 +18,15 @@ import _ from "lodash" jest.unmock("mssql") describe.each([ - ["internal", undefined], + // ["internal", undefined], ["internal-sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/api/:sourceId/search (%s)", (name, dsProvider) => { const isSqs = name === "internal-sqs" + const isInternal = name === "internal" const config = setup.getConfig() let envCleanup: (() => void) | undefined @@ -550,4 +551,90 @@ describe.each([ ])) }) }) + + describe("bigints", () => { + const SMALL = "1" + const MEDIUM = "10000000" + + // Our bigints are int64s in most datasources. + const BIG = "9223372036854775807" + + beforeAll(async () => { + await createTable({ + num: { name: "num", type: FieldType.BIGINT }, + }) + await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }]) + }) + + describe("equal", () => { + it("successfully finds a row", () => + expectQuery({ equal: { num: SMALL } }).toContainExactly([ + { num: SMALL }, + ])) + + it("successfully finds a big value", () => + expectQuery({ equal: { num: BIG } }).toContainExactly([{ num: BIG }])) + + it("fails to find nonexistent row", () => + expectQuery({ equal: { num: "2" } }).toFindNothing()) + }) + + describe("notEqual", () => { + it("successfully finds a row", () => + expectQuery({ notEqual: { num: SMALL } }).toContainExactly([ + { num: MEDIUM }, + { num: BIG }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ notEqual: { num: 10 } }).toContainExactly([ + { num: SMALL }, + { num: MEDIUM }, + { num: BIG }, + ])) + }) + + describe("oneOf", () => { + it("successfully finds a row", () => + expectQuery({ oneOf: { num: [SMALL] } }).toContainExactly([ + { num: SMALL }, + ])) + + it("successfully finds all rows", () => + expectQuery({ oneOf: { num: [SMALL, MEDIUM, BIG] } }).toContainExactly([ + { num: SMALL }, + { num: MEDIUM }, + { num: BIG }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ oneOf: { num: [2] } }).toFindNothing()) + }) + + // Range searches against bigints don't seem to work at all in Lucene, and I + // couldn't figure out why. Given that we're replacing Lucene with SQS, + // we've decided not to spend time on it. + !isInternal && + describe("range", () => { + it.only("successfully finds a row", () => + expectQuery({ + range: { num: { low: SMALL, high: "5" } }, + }).toContainExactly([{ num: SMALL }])) + + it("successfully finds multiple rows", () => + expectQuery({ + range: { num: { low: SMALL, high: MEDIUM } }, + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }])) + + it("successfully finds a row with a high bound", () => + expectQuery({ + range: { num: { low: MEDIUM, high: BIG } }, + }).toContainExactly([{ num: MEDIUM }, { num: BIG }])) + + it("successfully finds no rows", () => + expectQuery({ + range: { num: { low: "5", high: "5" } }, + }).toFindNothing()) + }) + }) }) diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 65cd4a07c1..bc8430c72c 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -52,6 +52,12 @@ export async function save( !oldTable && (tableToSave.primary == null || tableToSave.primary.length === 0) ) { + if (tableToSave.schema.id) { + throw new Error( + "External tables with no `primary` column set will define an `id` column, but we found an `id` column in the supplied schema. Either set a `primary` column or remove the `id` column." + ) + } + tableToSave.primary = ["id"] tableToSave.schema.id = { type: FieldType.NUMBER, diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index 79d9be2348..14313973c1 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -31,7 +31,7 @@ const FieldTypeMap: Record = { [FieldType.ATTACHMENT_SINGLE]: SQLiteType.BLOB, [FieldType.ARRAY]: SQLiteType.BLOB, [FieldType.LINK]: SQLiteType.BLOB, - [FieldType.BIGINT]: SQLiteType.REAL, + [FieldType.BIGINT]: SQLiteType.TEXT, // TODO: consider the difference between multi-user and single user types (subtyping) [FieldType.BB_REFERENCE]: SQLiteType.TEXT, } From 7d2861718a3511963a02bbb9737eafb504bd4f7f Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 16 Apr 2024 16:41:39 +0100 Subject: [PATCH 36/96] Making meta required in query JSON. --- .../api/controllers/row/ExternalRequest.ts | 16 ++++++++++ .../routes/tests/queries/generic-sql.spec.ts | 3 ++ .../server/src/integrations/tests/sql.spec.ts | 30 ++++++++++++++----- .../src/integrations/tests/sqlAlias.spec.ts | 15 ++++++++++ packages/types/src/sdk/search.ts | 2 +- 5 files changed, 57 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 4adbb72c7a..da620a92e6 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -119,6 +119,9 @@ async function removeManyToManyRelationships( endpoint: getEndpoint(tableId, Operation.DELETE), body: { [colName]: null }, filters, + meta: { + table, + } }) } else { return [] @@ -133,6 +136,9 @@ async function removeOneToManyRelationships(rowId: string, table: Table) { return getDatasourceAndQuery({ endpoint: getEndpoint(tableId, Operation.UPDATE), filters, + meta: { + table, + } }) } else { return [] @@ -248,6 +254,9 @@ export class ExternalRequest { const response = await getDatasourceAndQuery({ endpoint: getEndpoint(table._id!, Operation.READ), filters: buildFilters(rowId, {}, table), + meta: { + table, + } }) if (Array.isArray(response) && response.length > 0) { return response[0] @@ -395,6 +404,9 @@ export class ExternalRequest { [fieldName]: row[lookupField], }, }, + meta: { + table, + } }) // this is the response from knex if no rows found const rows: Row[] = @@ -425,6 +437,7 @@ export class ExternalRequest { // if we're creating (in a through table) need to wipe the existing ones first const promises = [] const related = await this.lookupRelations(mainTableId, row) + const table = this.getTable(mainTableId) for (let relationship of relationships) { const { key, tableId, isUpdate, id, ...rest } = relationship const body: { [key: string]: any } = processObjectSync(rest, row, {}) @@ -470,6 +483,9 @@ export class ExternalRequest { // if we're doing many relationships then we're writing, only one response body, filters: buildFilters(id, {}, linkTable), + meta: { + table, + } }) ) } else { diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 7790f909e7..1f83adc663 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -755,6 +755,9 @@ describe.each( name: "two", }, }, + meta: { + table: config.table, + }, }) expect(res).toHaveLength(1) expect(res[0]).toEqual({ diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index dc2a06446b..d056159d7d 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -9,6 +9,14 @@ import { } from "@budibase/types" const TABLE_NAME = "test" +const TABLE: Table = { + type: "table", + sourceType: TableSourceType.EXTERNAL, + sourceId: "SOURCE_ID", + schema: {}, + name: TABLE_NAME, + primary: ["id"], +} function endpoint(table: any, operation: any) { return { @@ -25,6 +33,10 @@ function generateReadJson({ sort, paginate, }: any = {}): QueryJson { + const tableObj = { ...TABLE } + if (table) { + tableObj.name = table + } return { endpoint: endpoint(table || TABLE_NAME, "READ"), resource: { @@ -34,14 +46,7 @@ function generateReadJson({ sort: sort || {}, paginate: paginate || {}, meta: { - table: { - type: "table", - sourceType: TableSourceType.EXTERNAL, - sourceId: "SOURCE_ID", - schema: {}, - name: table || TABLE_NAME, - primary: ["id"], - } as any, + table: tableObj, }, } } @@ -49,6 +54,9 @@ function generateReadJson({ function generateCreateJson(table = TABLE_NAME, body = {}): QueryJson { return { endpoint: endpoint(table, "CREATE"), + meta: { + table: TABLE, + }, body, } } @@ -70,6 +78,9 @@ function generateUpdateJson({ function generateDeleteJson(table = TABLE_NAME, filters = {}): QueryJson { return { endpoint: endpoint(table, "DELETE"), + meta: { + table: TABLE, + }, filters, } } @@ -102,6 +113,9 @@ function generateRelationshipJson(config: { schema?: string } = {}): QueryJson { }, ], extra: { idFilter: {} }, + meta: { + table: TABLE, + }, } } diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 58c3a05245..f4edab8dad 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -4,6 +4,8 @@ import { QueryJson, SourceName, SqlQuery, + Table, + TableSourceType, } from "@budibase/types" import { join } from "path" import Sql from "../base/sql" @@ -11,6 +13,16 @@ import { SqlClient } from "../utils" import { generator } from "@budibase/backend-core/tests" import sdk from "../../sdk" +// this doesn't exist strictly +const TABLE: Table = { + type: "table", + sourceType: TableSourceType.EXTERNAL, + sourceId: "SOURCE_ID", + schema: {}, + name: "tableName", + primary: ["id"], +} + const AliasTables = sdk.rows.AliasTables function multiline(sql: string) { @@ -222,6 +234,9 @@ describe("Captures of real examples", () => { resource: { fields, }, + meta: { + table: TABLE, + }, } } diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 9325f09eed..6cac76e01d 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -90,7 +90,7 @@ export interface QueryJson { paginate?: PaginationJson body?: Row | Row[] table?: Table - meta?: { + meta: { table?: Table tables?: Record renamed?: RenameColumn From ccb56c8788391d046243d059ce48898275b073ca Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 16 Apr 2024 17:05:09 +0100 Subject: [PATCH 37/96] Linting. --- .../server/src/api/controllers/row/ExternalRequest.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index da620a92e6..835e5bb20d 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -121,7 +121,7 @@ async function removeManyToManyRelationships( filters, meta: { table, - } + }, }) } else { return [] @@ -138,7 +138,7 @@ async function removeOneToManyRelationships(rowId: string, table: Table) { filters, meta: { table, - } + }, }) } else { return [] @@ -256,7 +256,7 @@ export class ExternalRequest { filters: buildFilters(rowId, {}, table), meta: { table, - } + }, }) if (Array.isArray(response) && response.length > 0) { return response[0] @@ -406,7 +406,7 @@ export class ExternalRequest { }, meta: { table, - } + }, }) // this is the response from knex if no rows found const rows: Row[] = @@ -485,7 +485,7 @@ export class ExternalRequest { filters: buildFilters(id, {}, linkTable), meta: { table, - } + }, }) ) } else { From 8b0fc5ed5d1b640a15adcf7fd1932ca71909cc1c Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 16 Apr 2024 17:22:17 +0100 Subject: [PATCH 38/96] Making table required. --- .../server/src/api/controllers/row/ExternalRequest.ts | 2 +- .../server/src/api/controllers/table/ExternalRequest.ts | 1 + .../src/api/routes/tests/queries/generic-sql.spec.ts | 2 +- packages/server/src/integrations/tests/sql.spec.ts | 8 ++++++++ packages/types/src/sdk/search.ts | 2 +- 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 835e5bb20d..3dd3f9b8e7 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -437,7 +437,7 @@ export class ExternalRequest { // if we're creating (in a through table) need to wipe the existing ones first const promises = [] const related = await this.lookupRelations(mainTableId, row) - const table = this.getTable(mainTableId) + const table = this.getTable(mainTableId)! for (let relationship of relationships) { const { key, tableId, isUpdate, id, ...rest } = relationship const body: { [key: string]: any } = processObjectSync(rest, row, {}) diff --git a/packages/server/src/api/controllers/table/ExternalRequest.ts b/packages/server/src/api/controllers/table/ExternalRequest.ts index 65cead3a1d..1e57ea3294 100644 --- a/packages/server/src/api/controllers/table/ExternalRequest.ts +++ b/packages/server/src/api/controllers/table/ExternalRequest.ts @@ -22,6 +22,7 @@ export async function makeTableRequest( operation, }, meta: { + table, tables, }, table, diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 1f83adc663..718f18203c 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -756,7 +756,7 @@ describe.each( }, }, meta: { - table: config.table, + table: config.table!, }, }) expect(res).toHaveLength(1) diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index d056159d7d..4ee544cc5e 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -66,7 +66,15 @@ function generateUpdateJson({ body = {}, filters = {}, meta = {}, +}: { + table: string + body?: any + filters?: any + meta?: any }): QueryJson { + if (!meta.table) { + meta.table = table + } return { endpoint: endpoint(table, "UPDATE"), filters, diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 6cac76e01d..0b93fb9215 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -91,7 +91,7 @@ export interface QueryJson { body?: Row | Row[] table?: Table meta: { - table?: Table + table: Table tables?: Record renamed?: RenameColumn } From 2efbd6726c5878fc2e99793427ca6079e4ae8265 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 16 Apr 2024 17:28:13 +0100 Subject: [PATCH 39/96] Removing meta from test API, it cannot be supplied, it is an internal property. --- .../server/src/api/routes/tests/queries/generic-sql.spec.ts | 3 --- packages/server/src/tests/utilities/api/datasource.ts | 5 ++++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 718f18203c..7790f909e7 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -755,9 +755,6 @@ describe.each( name: "two", }, }, - meta: { - table: config.table!, - }, }) expect(res).toHaveLength(1) expect(res[0]).toEqual({ diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts index 0296f58f7d..0362a25940 100644 --- a/packages/server/src/tests/utilities/api/datasource.ts +++ b/packages/server/src/tests/utilities/api/datasource.ts @@ -60,7 +60,10 @@ export class DatasourceAPI extends TestAPI { }) } - query = async (query: QueryJson, expectations?: Expectations) => { + query = async ( + query: Omit, + expectations?: Expectations + ) => { return await this._post(`/api/datasources/query`, { body: query, expectations, From 03b18234635445da730dc3f01bfbf7bb95c3de3b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 16 Apr 2024 17:28:21 +0100 Subject: [PATCH 40/96] Range tests passing. --- .../src/api/routes/tests/search.spec.ts | 50 ++++++++++++++-- packages/server/src/integrations/base/sql.ts | 57 +++++++++++++++---- .../server/src/sdk/app/rows/search/sqs.ts | 2 +- packages/types/src/sdk/search.ts | 11 ++-- 4 files changed, 97 insertions(+), 23 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 39ba0b589d..698ea0c10b 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -18,12 +18,12 @@ import _ from "lodash" jest.unmock("mssql") describe.each([ - // ["internal", undefined], + ["internal", undefined], ["internal-sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/api/:sourceId/search (%s)", (name, dsProvider) => { const isSqs = name === "internal-sqs" const isInternal = name === "internal" @@ -337,6 +337,20 @@ describe.each([ expectQuery({ range: { age: { low: 5, high: 9 } }, }).toFindNothing()) + + // We never implemented half-open ranges in Lucene. + !isInternal && + it("can search using just a low value", () => + expectQuery({ + range: { age: { low: 5 } }, + }).toContainExactly([{ age: 10 }])) + + // We never implemented half-open ranges in Lucene. + !isInternal && + it("can search using just a high value", () => + expectQuery({ + range: { age: { high: 5 } }, + }).toContainExactly([{ age: 1 }])) }) describe("sort", () => { @@ -441,6 +455,20 @@ describe.each([ expectQuery({ range: { dob: { low: JAN_5TH, high: JAN_9TH } }, }).toFindNothing()) + + // We never implemented half-open ranges in Lucene. + !isInternal && + it("can search using just a low value", () => + expectQuery({ + range: { dob: { low: JAN_5TH } }, + }).toContainExactly([{ dob: JAN_10TH }])) + + // We never implemented half-open ranges in Lucene. + !isInternal && + it("can search using just a high value", () => + expectQuery({ + range: { dob: { high: JAN_5TH } }, + }).toContainExactly([{ dob: JAN_1ST }])) }) describe("sort", () => { @@ -616,7 +644,7 @@ describe.each([ // we've decided not to spend time on it. !isInternal && describe("range", () => { - it.only("successfully finds a row", () => + it("successfully finds a row", () => expectQuery({ range: { num: { low: SMALL, high: "5" } }, }).toContainExactly([{ num: SMALL }])) @@ -635,6 +663,16 @@ describe.each([ expectQuery({ range: { num: { low: "5", high: "5" } }, }).toFindNothing()) + + it("can search using just a low value", () => + expectQuery({ + range: { num: { low: MEDIUM } }, + }).toContainExactly([{ num: MEDIUM }, { num: BIG }])) + + it("can search using just a high value", () => + expectQuery({ + range: { num: { high: MEDIUM } }, + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }])) }) }) }) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 259abec106..a3454d7a56 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -146,7 +146,7 @@ class InternalBuilder { addFilters( query: Knex.QueryBuilder, filters: SearchFilters | undefined, - tableName: string, + table: Table, opts: { aliases?: Record; relationship?: boolean } ): Knex.QueryBuilder { function getTableName(name: string) { @@ -161,7 +161,7 @@ class InternalBuilder { const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") if (!opts.relationship && !isRelationshipField) { - fn(`${getTableName(tableName)}.${updatedKey}`, value) + fn(`${getTableName(table.name)}.${updatedKey}`, value) } if (opts.relationship && isRelationshipField) { const [filterTableName, property] = updatedKey.split(".") @@ -276,6 +276,9 @@ class InternalBuilder { } if (filters.range) { iterate(filters.range, (key, value) => { + const fieldName = key.split(".")[1] + const field = table.schema[fieldName] + const isEmptyObject = (val: any) => { return ( val && @@ -293,16 +296,46 @@ class InternalBuilder { highValid = isValidFilter(value.high) if (lowValid && highValid) { // Use a between operator if we have 2 valid range values - const fnc = allOr ? "orWhereBetween" : "whereBetween" - query = query[fnc](key, [value.low, value.high]) + if ( + field.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw( + `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, + [value.low, value.high] + ) + } else { + const fnc = allOr ? "orWhereBetween" : "whereBetween" + query = query[fnc](key, [value.low, value.high]) + } } else if (lowValid) { // Use just a single greater than operator if we only have a low - const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, ">", value.low) + if ( + field.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw( + `CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, + [value.low] + ) + } else { + const fnc = allOr ? "orWhere" : "where" + query = query[fnc](key, ">=", value.low) + } } else if (highValid) { // Use just a single less than operator if we only have a high - const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, "<", value.high) + if ( + field.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw( + `CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, + [value.high] + ) + } else { + const fnc = allOr ? "orWhere" : "where" + query = query[fnc](key, "<=", value.high) + } } }) } @@ -532,7 +565,7 @@ class InternalBuilder { if (foundOffset) { query = query.offset(foundOffset) } - query = this.addFilters(query, filters, tableName, { + query = this.addFilters(query, filters, json.meta?.table!, { aliases: tableAliases, }) // add sorting to pre-query @@ -553,7 +586,7 @@ class InternalBuilder { endpoint.schema, tableAliases ) - return this.addFilters(query, filters, tableName, { + return this.addFilters(query, filters, json.meta?.table!, { relationship: true, aliases: tableAliases, }) @@ -563,7 +596,7 @@ class InternalBuilder { const { endpoint, body, filters, tableAliases } = json let query = this.knexWithAlias(knex, endpoint, tableAliases) const parsedBody = parseBody(body) - query = this.addFilters(query, filters, endpoint.entityId, { + query = this.addFilters(query, filters, json.meta?.table!, { aliases: tableAliases, }) // mysql can't use returning @@ -577,7 +610,7 @@ class InternalBuilder { delete(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder { const { endpoint, filters, tableAliases } = json let query = this.knexWithAlias(knex, endpoint, tableAliases) - query = this.addFilters(query, filters, endpoint.entityId, { + query = this.addFilters(query, filters, json.meta?.table!, { aliases: tableAliases, }) // mysql can't use returning diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 7abd7d9e72..4517739d26 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -185,6 +185,6 @@ export async function search( } } catch (err: any) { const msg = typeof err === "string" ? err : err.message - throw new Error(`Unable to search by SQL - ${msg}`) + throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) } } diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 9325f09eed..288618647f 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -13,10 +13,13 @@ export interface SearchFilters { [key: string]: string } range?: { - [key: string]: { - high: number | string - low: number | string - } + [key: string]: + | { + high: number | string + low: number | string + } + | { high: number | string } + | { low: number | string } } equal?: { [key: string]: any From a3a13d21d9847f5bd180a84c7aea57d9c2364349 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 16 Apr 2024 17:35:15 +0100 Subject: [PATCH 41/96] bumping pro sm --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index ef186d0024..78bd9cd090 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit ef186d00241f96037f9fd34d7a3826041977ab3a +Subproject commit 78bd9cd0909cba873253ebaf4affe0af4cf27d1c From b01b260e396c5597c4cbe201bd3d92ec71124558 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 16 Apr 2024 17:36:51 +0100 Subject: [PATCH 42/96] Fix some of the broken tests. --- packages/server/src/integrations/base/sql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 65709d8a82..e157590c5e 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -175,7 +175,7 @@ class InternalBuilder { const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") if (!opts.relationship && !isRelationshipField) { - fn(`${getTableName(table)}.${updatedKey}`, value) + fn(`${getTableAlias(table.name)}.${updatedKey}`, value) } if (opts.relationship && isRelationshipField) { const [filterTableName, property] = updatedKey.split(".") From 38dc7ae39142e718aebdb767c1120ee8f49abc8e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 16 Apr 2024 17:39:05 +0100 Subject: [PATCH 43/96] Remove extraneous ? and ! operators. --- packages/server/src/integrations/base/sql.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index e157590c5e..4b85f1cea2 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -392,7 +392,7 @@ class InternalBuilder { addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder { let { sort, paginate } = json - const table = json.meta?.table + const table = json.meta.table const tableName = getTableName(table) const aliases = json.tableAliases const aliased = @@ -580,7 +580,7 @@ class InternalBuilder { if (foundOffset) { query = query.offset(foundOffset) } - query = this.addFilters(query, filters, json.meta?.table!, { + query = this.addFilters(query, filters, json.meta.table, { aliases: tableAliases, }) // add sorting to pre-query @@ -601,7 +601,7 @@ class InternalBuilder { endpoint.schema, tableAliases ) - return this.addFilters(query, filters, json.meta?.table!, { + return this.addFilters(query, filters, json.meta.table, { relationship: true, aliases: tableAliases, }) @@ -611,7 +611,7 @@ class InternalBuilder { const { endpoint, body, filters, tableAliases } = json let query = this.knexWithAlias(knex, endpoint, tableAliases) const parsedBody = parseBody(body) - query = this.addFilters(query, filters, json.meta?.table!, { + query = this.addFilters(query, filters, json.meta.table, { aliases: tableAliases, }) // mysql can't use returning @@ -625,7 +625,7 @@ class InternalBuilder { delete(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder { const { endpoint, filters, tableAliases } = json let query = this.knexWithAlias(knex, endpoint, tableAliases) - query = this.addFilters(query, filters, json.meta?.table!, { + query = this.addFilters(query, filters, json.meta.table, { aliases: tableAliases, }) // mysql can't use returning @@ -717,7 +717,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { // when creating if an ID has been inserted need to make sure // the id filter is enriched with it before trying to retrieve the row checkLookupKeys(id: any, json: QueryJson) { - if (!id || !json.meta?.table || !json.meta.table.primary) { + if (!id || !json.meta.table || !json.meta.table.primary) { return json } const primaryKey = json.meta.table.primary?.[0] From aff495e0bc1c038079d6ddb082db9eaec1ab8e0e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 16 Apr 2024 17:40:35 +0100 Subject: [PATCH 44/96] bump pro to master ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 78bd9cd090..c68183402b 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 78bd9cd0909cba873253ebaf4affe0af4cf27d1c +Subproject commit c68183402b8fb17248572006531d5293ffc8a9ac From affa546159c48ddf3e7f7ff309fff5aadef9813d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 16 Apr 2024 17:45:06 +0100 Subject: [PATCH 45/96] Fix tests. --- packages/server/src/integrations/base/sql.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 4b85f1cea2..7ee22b5933 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -163,6 +163,9 @@ class InternalBuilder { table: Table, opts: { aliases?: Record; relationship?: boolean } ): Knex.QueryBuilder { + const tableName = + this.client === SqlClient.SQL_LITE ? table._id! : table.name + function getTableAlias(name: string) { const alias = opts.aliases?.[name] return alias || name @@ -175,7 +178,7 @@ class InternalBuilder { const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") if (!opts.relationship && !isRelationshipField) { - fn(`${getTableAlias(table.name)}.${updatedKey}`, value) + fn(`${getTableAlias(tableName)}.${updatedKey}`, value) } if (opts.relationship && isRelationshipField) { const [filterTableName, property] = updatedKey.split(".") From 141718d53200f8afd7bc7881b3e527d2d1ed9373 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 16 Apr 2024 18:10:18 +0100 Subject: [PATCH 46/96] bump account portal module --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index bd0e01d639..328c84234d 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit bd0e01d639ec3b2547e7c859a1c43b622dce8344 +Subproject commit 328c84234d11d97d840f0eb2c72665b04ba9e4f8 From b529e98b057eb7704fd0f01ca83690ca1be18f9f Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 16 Apr 2024 17:20:33 +0000 Subject: [PATCH 47/96] Bump version to 2.23.6 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 9839b8b166..a2be7be7b4 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.23.5", + "version": "2.23.6", "npmClient": "yarn", "packages": [ "packages/*", From ca158c33e81860e8cbaec328c934e0458dc0c7d3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 10:33:20 +0200 Subject: [PATCH 48/96] DRY --- packages/backend-core/src/db/lucene.ts | 14 ++------------ .../controls/FilterEditor/FilterBuilder.svelte | 9 ++------- packages/shared-core/src/filters.ts | 3 ++- 3 files changed, 6 insertions(+), 20 deletions(-) diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index 4a2cfd34e2..d9dddd0097 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -8,19 +8,9 @@ import { SearchParams, WithRequired, } from "@budibase/types" +import { dataFilters } from "@budibase/shared-core" -const QUERY_START_REGEX = /\d[0-9]*:/g - -export function removeKeyNumbering(key: any): string { - if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) { - const parts = key.split(":") - // remove the number - parts.shift() - return parts.join(":") - } else { - return key - } -} +export const removeKeyNumbering = dataFilters.removeKeyNumbering /** * Class to build lucene query URLs. diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte index 6eb17df059..0ab67cbada 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte @@ -2,6 +2,7 @@ import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte" import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte" + import { dataFilters } from "@budibase/shared-core" import { FilterBuilder } from "@budibase/frontend-core" import { createEventDispatcher, onMount } from "svelte" @@ -15,8 +16,6 @@ const dispatch = createEventDispatcher() - const KeyedFieldRegex = /\d[0-9]*:/g - let rawFilters $: parseFilters(rawFilters) @@ -28,11 +27,7 @@ const { field } = filter let newFilter = { ...filter } delete newFilter.allOr - if (typeof field === "string" && field.match(KeyedFieldRegex) != null) { - const parts = field.split(":") - parts.shift() - newFilter.field = parts.join(":") - } + newFilter.field = dataFilters.removeKeyNumbering(field) return newFilter }) } diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 95d5269c4b..4416b28b08 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -120,9 +120,10 @@ const cleanupQuery = (query: SearchQuery) => { /** * Removes a numeric prefix on field names designed to give fields uniqueness */ -const removeKeyNumbering = (key: string) => { +export const removeKeyNumbering = (key: string): string => { if (typeof key === "string" && key.match(/\d[0-9]*:/g) != null) { const parts = key.split(":") + // remove the number parts.shift() return parts.join(":") } else { From 8161dfc0a909c1ef99590ef6d359d52af23bcf1b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 14:16:13 +0200 Subject: [PATCH 49/96] Create SWITCHABLE_TYPES in shared-core --- packages/shared-core/src/constants/fields.ts | 33 ++++++++++++++++++++ packages/shared-core/src/constants/index.ts | 1 + 2 files changed, 34 insertions(+) create mode 100644 packages/shared-core/src/constants/fields.ts diff --git a/packages/shared-core/src/constants/fields.ts b/packages/shared-core/src/constants/fields.ts new file mode 100644 index 0000000000..89504de473 --- /dev/null +++ b/packages/shared-core/src/constants/fields.ts @@ -0,0 +1,33 @@ +import { FieldType } from "@budibase/types" + +type SwitchableTypes = Partial<{ + [K in FieldType]: [K, ...FieldType[]] +}> + +export const SWITCHABLE_TYPES: Partial = { + [FieldType.STRING]: [ + FieldType.STRING, + FieldType.OPTIONS, + FieldType.LONGFORM, + FieldType.BARCODEQR, + ], + [FieldType.OPTIONS]: [ + FieldType.OPTIONS, + FieldType.STRING, + FieldType.LONGFORM, + FieldType.BARCODEQR, + ], + [FieldType.LONGFORM]: [ + FieldType.LONGFORM, + FieldType.STRING, + FieldType.OPTIONS, + FieldType.BARCODEQR, + ], + [FieldType.BARCODEQR]: [ + FieldType.BARCODEQR, + FieldType.STRING, + FieldType.OPTIONS, + FieldType.LONGFORM, + ], + [FieldType.NUMBER]: [FieldType.NUMBER, FieldType.BOOLEAN], +} diff --git a/packages/shared-core/src/constants/index.ts b/packages/shared-core/src/constants/index.ts index 922f0d4387..afb7e659e1 100644 --- a/packages/shared-core/src/constants/index.ts +++ b/packages/shared-core/src/constants/index.ts @@ -1,4 +1,5 @@ export * from "./api" +export * from "./fields" export const OperatorOptions = { Equals: { From 3e32ce4d242b375190fbed56f60802827e6593fd Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 14:21:37 +0200 Subject: [PATCH 50/96] Use shared-core --- .../DataTable/modals/CreateEditColumn.svelte | 22 +++++-------------- .../builder/src/constants/backend/index.js | 20 ----------------- packages/builder/src/stores/builder/tables.js | 4 ++-- packages/shared-core/src/constants/fields.ts | 2 +- 4 files changed, 9 insertions(+), 39 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 92501bec3b..16a26b60c1 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -13,6 +13,7 @@ Layout, AbsTooltip, } from "@budibase/bbui" + import { SWITCHABLE_TYPES } from "@budibase/shared-core" import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" import { tables, datasources } from "stores/builder" @@ -20,11 +21,6 @@ import { FIELDS, RelationshipType, - ALLOWABLE_STRING_OPTIONS, - ALLOWABLE_NUMBER_OPTIONS, - ALLOWABLE_STRING_TYPES, - ALLOWABLE_NUMBER_TYPES, - SWITCHABLE_TYPES, PrettyRelationshipDefinitions, DB_TYPE_EXTERNAL, } from "constants/backend" @@ -175,7 +171,7 @@ $: typeEnabled = !originalName || (originalName && - SWITCHABLE_TYPES.indexOf(editableColumn.type) !== -1 && + SWITCHABLE_TYPES[editableColumn.type] && !editableColumn?.autocolumn) const fieldDefinitions = Object.values(FIELDS).reduce( @@ -367,16 +363,10 @@ } function getAllowedTypes() { - if ( - originalName && - ALLOWABLE_STRING_TYPES.indexOf(editableColumn.type) !== -1 - ) { - return ALLOWABLE_STRING_OPTIONS - } else if ( - originalName && - ALLOWABLE_NUMBER_TYPES.indexOf(editableColumn.type) !== -1 - ) { - return ALLOWABLE_NUMBER_OPTIONS + if (originalName) { + return ( + SWITCHABLE_TYPES[editableColumn.type] || [editableColumn.type] + ).map(f => FIELDS[f.toUpperCase()]) } const isUsers = diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index ffa6ef36c6..84975d93e2 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -202,26 +202,6 @@ export const PrettyRelationshipDefinitions = { ONE: "One row", } -export const ALLOWABLE_STRING_OPTIONS = [ - FIELDS.STRING, - FIELDS.OPTIONS, - FIELDS.LONGFORM, - FIELDS.BARCODEQR, -] -export const ALLOWABLE_STRING_TYPES = ALLOWABLE_STRING_OPTIONS.map( - opt => opt.type -) - -export const ALLOWABLE_NUMBER_OPTIONS = [FIELDS.NUMBER, FIELDS.BOOLEAN] -export const ALLOWABLE_NUMBER_TYPES = ALLOWABLE_NUMBER_OPTIONS.map( - opt => opt.type -) - -export const SWITCHABLE_TYPES = [ - ...ALLOWABLE_STRING_TYPES, - ...ALLOWABLE_NUMBER_TYPES, -] - export const BUDIBASE_INTERNAL_DB_ID = INTERNAL_TABLE_SOURCE_ID export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" export const BUDIBASE_DATASOURCE_TYPE = "budibase" diff --git a/packages/builder/src/stores/builder/tables.js b/packages/builder/src/stores/builder/tables.js index 0163281480..8a246a8ac2 100644 --- a/packages/builder/src/stores/builder/tables.js +++ b/packages/builder/src/stores/builder/tables.js @@ -1,8 +1,8 @@ import { FieldType } from "@budibase/types" +import { SWITCHABLE_TYPES } from "@budibase/shared-core" import { get, writable, derived } from "svelte/store" import { cloneDeep } from "lodash/fp" import { API } from "api" -import { SWITCHABLE_TYPES } from "constants/backend" export function createTablesStore() { const store = writable({ @@ -64,7 +64,7 @@ export function createTablesStore() { if ( oldField != null && oldField?.type !== field.type && - SWITCHABLE_TYPES.indexOf(oldField?.type) === -1 + SWITCHABLE_TYPES[oldField?.type] ) { updatedTable.schema[key] = oldField } diff --git a/packages/shared-core/src/constants/fields.ts b/packages/shared-core/src/constants/fields.ts index 89504de473..5acf07d863 100644 --- a/packages/shared-core/src/constants/fields.ts +++ b/packages/shared-core/src/constants/fields.ts @@ -4,7 +4,7 @@ type SwitchableTypes = Partial<{ [K in FieldType]: [K, ...FieldType[]] }> -export const SWITCHABLE_TYPES: Partial = { +export const SWITCHABLE_TYPES: SwitchableTypes = { [FieldType.STRING]: [ FieldType.STRING, FieldType.OPTIONS, From 4aba988ca97682c337f6a3e8576636cf23922602 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 14:34:06 +0200 Subject: [PATCH 51/96] Keep schema by default --- packages/server/src/integrations/utils.ts | 41 +++++++++++++---------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index d5f6d191e1..f0f96c82a5 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -284,8 +284,8 @@ export function isIsoDateString(str: string) { * @param column The column to check, to see if it is a valid relationship. * @param tableIds The IDs of the tables which currently exist. */ -export function shouldCopyRelationship( - column: { type: string; tableId?: string }, +function shouldCopyRelationship( + column: { type: FieldType.LINK; tableId?: string }, tableIds: string[] ) { return ( @@ -303,28 +303,18 @@ export function shouldCopyRelationship( * @param column The column to check for options or boolean type. * @param fetchedColumn The fetched column to check for the type in the external database. */ -export function shouldCopySpecialColumn( +function shouldCopySpecialColumn( column: { type: string }, fetchedColumn: { type: string } | undefined ) { const isFormula = column.type === FieldType.FORMULA - const specialTypes = [ - FieldType.OPTIONS, - FieldType.LONGFORM, - FieldType.ARRAY, - FieldType.FORMULA, - FieldType.BB_REFERENCE, - ] // column has been deleted, remove - formulas will never exist, always copy if (!isFormula && column && !fetchedColumn) { return false } const fetchedIsNumber = !fetchedColumn || fetchedColumn.type === FieldType.NUMBER - return ( - specialTypes.indexOf(column.type as FieldType) !== -1 || - (fetchedIsNumber && column.type === FieldType.BOOLEAN) - ) + return fetchedIsNumber && column.type === FieldType.BOOLEAN } /** @@ -357,12 +347,29 @@ function copyExistingPropsOver( continue } const column = existingTableSchema[key] + if ( - shouldCopyRelationship(column, tableIds) || - shouldCopySpecialColumn(column, table.schema[key]) + column.type === FieldType.LINK && + !shouldCopyRelationship(column, tableIds) ) { - table.schema[key] = existingTableSchema[key] + continue } + + const specialTypes = [ + FieldType.OPTIONS, + FieldType.LONGFORM, + FieldType.ARRAY, + FieldType.FORMULA, + FieldType.BB_REFERENCE, + ] + if ( + specialTypes.includes(column.type) && + !shouldCopySpecialColumn(column, table.schema[key]) + ) { + continue + } + + table.schema[key] = existingTableSchema[key] } } return table From b34227039900ad430f732dee8dff7f0754019c37 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 14:43:52 +0200 Subject: [PATCH 52/96] Lint --- .../backend/DataTable/modals/CreateEditColumn.svelte | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 16a26b60c1..8716a00660 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -13,7 +13,7 @@ Layout, AbsTooltip, } from "@budibase/bbui" - import { SWITCHABLE_TYPES } from "@budibase/shared-core" + import { SWITCHABLE_TYPES, ValidColumnNameRegex } from "@budibase/shared-core" import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" import { tables, datasources } from "stores/builder" @@ -29,7 +29,6 @@ import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" import { getBindings } from "components/backend/DataTable/formula" import JSONSchemaModal from "./JSONSchemaModal.svelte" - import { ValidColumnNameRegex } from "@budibase/shared-core" import { FieldType, FieldSubtype, SourceName } from "@budibase/types" import RelationshipSelector from "components/common/RelationshipSelector.svelte" import { RowUtils } from "@budibase/frontend-core" From 92c307105c2527377ddb8e878bcaa32312411b6b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 16:20:58 +0200 Subject: [PATCH 53/96] Skip test --- packages/server/src/integration-test/mysql.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index 7e54b53b15..fc9bb1aafa 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -235,7 +235,8 @@ describe("mysql integrations", () => { describe("POST /api/tables/", () => { const emitDatasourceUpdateMock = jest.fn() - it("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => { + // TODO: This is not actually required, will fix after cleaning the `_add` logic + xit("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => { const addColumnToTable: TableRequest = { type: "table", sourceType: TableSourceType.EXTERNAL, From a6132c28042681bac22198ff6fc4900804ec6d68 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 16:27:05 +0200 Subject: [PATCH 54/96] Lint --- packages/server/src/integration-test/mysql.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index fc9bb1aafa..59340eb202 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -236,7 +236,7 @@ describe("mysql integrations", () => { const emitDatasourceUpdateMock = jest.fn() // TODO: This is not actually required, will fix after cleaning the `_add` logic - xit("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => { + it.skip("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => { const addColumnToTable: TableRequest = { type: "table", sourceType: TableSourceType.EXTERNAL, From 764235469cecfec95b0543dd34b318b0e5fd38e5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 16:32:45 +0200 Subject: [PATCH 55/96] Lint --- packages/server/src/integration-test/mysql.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index 59340eb202..964bc8fce7 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -236,6 +236,7 @@ describe("mysql integrations", () => { const emitDatasourceUpdateMock = jest.fn() // TODO: This is not actually required, will fix after cleaning the `_add` logic + // eslint-disable-next-line jest/no-disabled-tests it.skip("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => { const addColumnToTable: TableRequest = { type: "table", From e1c092d65d498769f8e368f6e84fe4bde4f32417 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 16:36:52 +0200 Subject: [PATCH 56/96] Refetch when type changed --- packages/server/src/integrations/utils.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index f0f96c82a5..165fffc652 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -7,7 +7,7 @@ import { } from "@budibase/types" import { DocumentType, SEPARATOR } from "../db/utils" import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../constants" -import { helpers } from "@budibase/shared-core" +import { SWITCHABLE_TYPES, helpers } from "@budibase/shared-core" import env from "../environment" import { Knex } from "knex" @@ -348,6 +348,16 @@ function copyExistingPropsOver( } const column = existingTableSchema[key] + // If the db column type changed to a non-compatible one, we want to re-fetch it + if ( + table.schema[key].type !== existingTableSchema[key].type && + !SWITCHABLE_TYPES[existingTableSchema[key].type]?.includes( + table.schema[key].type + ) + ) { + continue + } + if ( column.type === FieldType.LINK && !shouldCopyRelationship(column, tableIds) From 8afb0e3c91d4d029b2b53f7e69c481d6a96a3e66 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 16:50:19 +0200 Subject: [PATCH 57/96] Fix tests --- packages/server/src/integrations/utils.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 165fffc652..b8bd1db897 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -304,8 +304,8 @@ function shouldCopyRelationship( * @param fetchedColumn The fetched column to check for the type in the external database. */ function shouldCopySpecialColumn( - column: { type: string }, - fetchedColumn: { type: string } | undefined + column: { type: FieldType }, + fetchedColumn: { type: FieldType } | undefined ) { const isFormula = column.type === FieldType.FORMULA // column has been deleted, remove - formulas will never exist, always copy @@ -348,12 +348,13 @@ function copyExistingPropsOver( } const column = existingTableSchema[key] + const existingColumnType = column?.type + const updatedColumnType = table.schema[key]?.type + // If the db column type changed to a non-compatible one, we want to re-fetch it if ( - table.schema[key].type !== existingTableSchema[key].type && - !SWITCHABLE_TYPES[existingTableSchema[key].type]?.includes( - table.schema[key].type - ) + updatedColumnType !== existingColumnType && + !SWITCHABLE_TYPES[existingColumnType]?.includes(updatedColumnType) ) { continue } From 54f2c825fc7d2cf69ce20b79f76087782e266440 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 12:03:46 +0200 Subject: [PATCH 58/96] Remove _add --- packages/builder/src/stores/builder/tables.js | 6 ------ .../server/src/api/controllers/table/external.ts | 3 +-- packages/server/src/api/controllers/table/index.ts | 1 - packages/server/src/sdk/app/tables/external/index.ts | 12 ++---------- packages/types/src/documents/app/table/table.ts | 3 +-- packages/types/src/sdk/search.ts | 4 ---- 6 files changed, 4 insertions(+), 25 deletions(-) diff --git a/packages/builder/src/stores/builder/tables.js b/packages/builder/src/stores/builder/tables.js index 8a246a8ac2..c2aab5634b 100644 --- a/packages/builder/src/stores/builder/tables.js +++ b/packages/builder/src/stores/builder/tables.js @@ -148,12 +148,6 @@ export function createTablesStore() { if (indexes) { draft.indexes = indexes } - // Add object to indicate if column is being added - if (draft.schema[field.name] === undefined) { - draft._add = { - name: field.name, - } - } draft.schema = { ...draft.schema, [field.name]: cloneDeep(field), diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 7c036bec9d..e526af4ecb 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -31,7 +31,6 @@ export async function save( renaming?: RenameColumn ) { const inputs = ctx.request.body - const adding = inputs?._add // can't do this right now delete inputs.rows const tableId = ctx.request.body._id @@ -44,7 +43,7 @@ export async function save( const { datasource, table } = await sdk.tables.external.save( datasourceId!, inputs, - { tableId, renaming, adding } + { tableId, renaming } ) builderSocket?.emitDatasourceUpdate(ctx, datasource) return table diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index f799113333..cae3a3fe11 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -79,7 +79,6 @@ export async function save(ctx: UserCtx) { const api = pickApi({ table }) // do not pass _rename or _add if saving to CouchDB if (api === internal) { - delete ctx.request.body._add delete ctx.request.body._rename } let savedTable = await api.save(ctx, renaming) diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 65cd4a07c1..f3f2f070be 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -3,7 +3,6 @@ import { Operation, RelationshipType, RenameColumn, - AddColumn, Table, TableRequest, ViewV2, @@ -33,7 +32,7 @@ import * as viewSdk from "../../views" export async function save( datasourceId: string, update: Table, - opts?: { tableId?: string; renaming?: RenameColumn; adding?: AddColumn } + opts?: { tableId?: string; renaming?: RenameColumn } ) { let tableToSave: TableRequest = { ...update, @@ -179,14 +178,7 @@ export async function save( // remove the rename prop delete tableToSave._rename - // if adding a new column, we need to rebuild the schema for that table to get the 'externalType' of the column - if (opts?.adding) { - datasource.entities[tableToSave.name] = ( - await datasourceSdk.buildFilteredSchema(datasource, [tableToSave.name]) - ).tables[tableToSave.name] - } else { - datasource.entities[tableToSave.name] = tableToSave - } + datasource.entities[tableToSave.name] = tableToSave // store it into couch now for budibase reference await db.put(populateExternalTableSchemas(datasource)) diff --git a/packages/types/src/documents/app/table/table.ts b/packages/types/src/documents/app/table/table.ts index b284e9a840..f0e6079aef 100644 --- a/packages/types/src/documents/app/table/table.ts +++ b/packages/types/src/documents/app/table/table.ts @@ -1,6 +1,6 @@ import { Document } from "../../document" import { View, ViewV2 } from "../view" -import { AddColumn, RenameColumn } from "../../../sdk" +import { RenameColumn } from "../../../sdk" import { TableSchema } from "./schema" export const INTERNAL_TABLE_SOURCE_ID = "bb_internal" @@ -30,6 +30,5 @@ export interface Table extends Document { export interface TableRequest extends Table { _rename?: RenameColumn - _add?: AddColumn created?: boolean } diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 0b93fb9215..b9aedede09 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -61,10 +61,6 @@ export interface RenameColumn { updated: string } -export interface AddColumn { - name: string -} - export interface RelationshipsJson { through?: string from?: string From 88fc133f45464556f8c971b7a56e44193a457d12 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 12:04:33 +0200 Subject: [PATCH 59/96] Move responsability --- packages/server/src/api/controllers/table/index.ts | 4 ---- packages/server/src/api/controllers/table/internal.ts | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index cae3a3fe11..63ce00c5ef 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -77,10 +77,6 @@ export async function save(ctx: UserCtx) { const renaming = ctx.request.body._rename const api = pickApi({ table }) - // do not pass _rename or _add if saving to CouchDB - if (api === internal) { - delete ctx.request.body._rename - } let savedTable = await api.save(ctx, renaming) if (!table._id) { savedTable = sdk.tables.enrichViewSchemas(savedTable) diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index eb5e4b6c41..a06cc4dee3 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -16,7 +16,7 @@ export async function save( ctx: UserCtx, renaming?: RenameColumn ) { - const { rows, ...rest } = ctx.request.body + const { _rename, rows, ...rest } = ctx.request.body let tableToSave: Table = { _id: generateTableID(), ...rest, From 33c40a897a2297be1d513633bb419b82963a8f17 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 12:38:46 +0200 Subject: [PATCH 60/96] Fix test --- packages/server/src/api/routes/tests/table.spec.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 7639b840dc..77e05b8e07 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -219,9 +219,6 @@ describe.each([ it("should add a new column for an internal DB table", async () => { const saveTableRequest: SaveTableRequest = { - _add: { - name: "NEW_COLUMN", - }, ...basicTable(), } @@ -235,7 +232,6 @@ describe.each([ updatedAt: expect.stringMatching(ISO_REGEX_PATTERN), views: {}, } - delete expectedResponse._add expect(response).toEqual(expectedResponse) }) }) From de47f44959bf9705197fcc1983e0be7c059b46b7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 13:04:16 +0200 Subject: [PATCH 61/96] Remove _add references --- packages/server/src/integration-test/mysql.spec.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index 964bc8fce7..04a7b548a6 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -255,9 +255,6 @@ describe("mysql integrations", () => { name: "new_column", }, }, - _add: { - name: "new_column", - }, } jest @@ -273,25 +270,18 @@ describe("mysql integrations", () => { type: FieldType.NUMBER, name: "id", autocolumn: true, - constraints: { - presence: false, - }, externalType: "int unsigned", }, new_column: { type: FieldType.NUMBER, name: "new_column", autocolumn: false, - constraints: { - presence: false, - }, externalType: "float(8,2)", }, }, created: true, _id: `${datasource._id}__${addColumnToTable.name}`, } - delete expectedTable._add expect(emitDatasourceUpdateMock).toHaveBeenCalledTimes(1) const emittedDatasource: Datasource = From da399970abcc94917ffa5e6d02e1965090b8b7db Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 17:02:58 +0200 Subject: [PATCH 62/96] Lint --- .eslintrc.json | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index 525072dc6c..624c2b8f26 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -42,7 +42,15 @@ }, "rules": { "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": "error", + "@typescript-eslint/no-unused-vars": [ + "error", + { + "varsIgnorePattern": "^_", + "argsIgnorePattern": "^_", + "destructuredArrayIgnorePattern": "^_", + "ignoreRestSiblings": true + } + ], "local-rules/no-budibase-imports": "error" } }, @@ -59,7 +67,15 @@ }, "rules": { "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": "error", + "@typescript-eslint/no-unused-vars": [ + "error", + { + "varsIgnorePattern": "^_", + "argsIgnorePattern": "^_", + "destructuredArrayIgnorePattern": "^_", + "ignoreRestSiblings": true + } + ], "local-rules/no-test-com": "error", "local-rules/email-domain-example-com": "error", "no-console": "warn", @@ -89,7 +105,8 @@ { "varsIgnorePattern": "^_", "argsIgnorePattern": "^_", - "destructuredArrayIgnorePattern": "^_" + "destructuredArrayIgnorePattern": "^_", + "ignoreRestSiblings": true } ], "import/no-relative-packages": "error", From 2da0daafe51239d56ace0fa169d0550685e366ef Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 10:15:07 +0200 Subject: [PATCH 63/96] Clean --- packages/server/src/sdk/app/datasources/datasources.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index 336a94636b..84e1601152 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -348,8 +348,7 @@ const preSaveAction: Partial> = { * Make sure all datasource entities have a display name selected */ export function setDefaultDisplayColumns(datasource: Datasource) { - // - for (let entity of Object.values(datasource.entities || {})) { + for (const entity of Object.values(datasource.entities || {})) { if (entity.primaryDisplay) { continue } From 21898afb29df6d5cec7e652cdfd0dfb76186507e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 10:15:42 +0200 Subject: [PATCH 64/96] Persist externalType changes --- packages/server/src/integrations/utils.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index b8bd1db897..39f2bd3cb0 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -380,7 +380,12 @@ function copyExistingPropsOver( continue } - table.schema[key] = existingTableSchema[key] + table.schema[key] = { + ...existingTableSchema[key], + externalType: + existingTableSchema[key].externalType || + table.schema[key].externalType, + } } } return table From 9be86e8b05d3e0b2cd91a6354f0d911206c4c731 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 17:02:58 +0200 Subject: [PATCH 65/96] Lint --- .eslintrc.json | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index 525072dc6c..624c2b8f26 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -42,7 +42,15 @@ }, "rules": { "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": "error", + "@typescript-eslint/no-unused-vars": [ + "error", + { + "varsIgnorePattern": "^_", + "argsIgnorePattern": "^_", + "destructuredArrayIgnorePattern": "^_", + "ignoreRestSiblings": true + } + ], "local-rules/no-budibase-imports": "error" } }, @@ -59,7 +67,15 @@ }, "rules": { "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": "error", + "@typescript-eslint/no-unused-vars": [ + "error", + { + "varsIgnorePattern": "^_", + "argsIgnorePattern": "^_", + "destructuredArrayIgnorePattern": "^_", + "ignoreRestSiblings": true + } + ], "local-rules/no-test-com": "error", "local-rules/email-domain-example-com": "error", "no-console": "warn", @@ -89,7 +105,8 @@ { "varsIgnorePattern": "^_", "argsIgnorePattern": "^_", - "destructuredArrayIgnorePattern": "^_" + "destructuredArrayIgnorePattern": "^_", + "ignoreRestSiblings": true } ], "import/no-relative-packages": "error", From ec93ea59c11c19285a885aee877834b973388649 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 10:22:49 +0200 Subject: [PATCH 66/96] Remove // eslint-disable-next-line @typescript-eslint/no-unused-vars --- packages/backend-core/src/queue/inMemoryQueue.ts | 3 +-- packages/pro | 2 +- .../server/src/automations/steps/triggerAutomationRun.ts | 1 - packages/server/src/integrations/googlesheets.ts | 3 +-- packages/server/src/jsRunner/index.ts | 1 - packages/server/src/sdk/app/views/tests/views.spec.ts | 5 ----- packages/server/src/websockets/websocket.ts | 6 ++---- packages/worker/src/api/routes/global/tests/scim.spec.ts | 3 --- 8 files changed, 5 insertions(+), 19 deletions(-) diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index 87e43b324d..06fe036da6 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -166,8 +166,7 @@ class InMemoryQueue implements Partial { return [] } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async removeJobs(pattern: string) { + async removeJobs(_pattern: string) { // no-op } diff --git a/packages/pro b/packages/pro index c68183402b..e6c7cde321 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit c68183402b8fb17248572006531d5293ffc8a9ac +Subproject commit e6c7cde321b6943e3f0f464957b1c1c5b22fbb50 diff --git a/packages/server/src/automations/steps/triggerAutomationRun.ts b/packages/server/src/automations/steps/triggerAutomationRun.ts index f45a60600f..83e1722877 100644 --- a/packages/server/src/automations/steps/triggerAutomationRun.ts +++ b/packages/server/src/automations/steps/triggerAutomationRun.ts @@ -62,7 +62,6 @@ export const definition: AutomationStepSchema = { } export async function run({ inputs }: AutomationStepInput) { - // eslint-disable-next-line @typescript-eslint/no-unused-vars const { automationId, ...fieldParams } = inputs.automation if (await features.isTriggerAutomationRunEnabled()) { diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 1573c98f16..7215c337d7 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -168,8 +168,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { return "" } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - getStringConcat(parts: string[]) { + getStringConcat(_parts: string[]) { return "" } diff --git a/packages/server/src/jsRunner/index.ts b/packages/server/src/jsRunner/index.ts index 7065febcb4..b2cae1cfbc 100644 --- a/packages/server/src/jsRunner/index.ts +++ b/packages/server/src/jsRunner/index.ts @@ -35,7 +35,6 @@ export function init() { // Because we can't pass functions into an Isolate, we remove them from // the passed context and rely on the withHelpers() method to add them // back in. - // eslint-disable-next-line @typescript-eslint/no-unused-vars const { helpers, snippets, ...rest } = ctx return vm.withContext(rest, () => vm.execute(js)) } catch (error: any) { diff --git a/packages/server/src/sdk/app/views/tests/views.spec.ts b/packages/server/src/sdk/app/views/tests/views.spec.ts index a610d34ec2..508285651a 100644 --- a/packages/server/src/sdk/app/views/tests/views.spec.ts +++ b/packages/server/src/sdk/app/views/tests/views.spec.ts @@ -351,7 +351,6 @@ describe("table sdk", () => { const view: ViewV2 = { ...basicView, } - // eslint-disable-next-line @typescript-eslint/no-unused-vars const { name, description, ...newTableSchema } = basicTable.schema const result = syncSchema(_.cloneDeep(view), newTableSchema, undefined) @@ -365,7 +364,6 @@ describe("table sdk", () => { const view: ViewV2 = { ...basicView, } - // eslint-disable-next-line @typescript-eslint/no-unused-vars const { description, ...newTableSchema } = { ...basicTable.schema, updatedDescription: { @@ -450,7 +448,6 @@ describe("table sdk", () => { hiddenField: { visible: false }, }, } - // eslint-disable-next-line @typescript-eslint/no-unused-vars const { name, description, ...newTableSchema } = basicTable.schema const result = syncSchema(_.cloneDeep(view), newTableSchema, undefined) @@ -474,7 +471,6 @@ describe("table sdk", () => { hiddenField: { visible: false }, }, } - // eslint-disable-next-line @typescript-eslint/no-unused-vars const { name, description, ...newTableSchema } = { ...basicTable.schema, newField1: { @@ -506,7 +502,6 @@ describe("table sdk", () => { hiddenField: { visible: false }, }, } - // eslint-disable-next-line @typescript-eslint/no-unused-vars const { description, ...newTableSchema } = { ...basicTable.schema, updatedDescription: { diff --git a/packages/server/src/websockets/websocket.ts b/packages/server/src/websockets/websocket.ts index 871122678d..5dcf8c8f6f 100644 --- a/packages/server/src/websockets/websocket.ts +++ b/packages/server/src/websockets/websocket.ts @@ -262,13 +262,11 @@ export class BaseSocket { } } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async onConnect(socket: Socket) { + async onConnect(_socket: Socket) { // Override } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async onDisconnect(socket: Socket) { + async onDisconnect(_socket: Socket) { // Override } diff --git a/packages/worker/src/api/routes/global/tests/scim.spec.ts b/packages/worker/src/api/routes/global/tests/scim.spec.ts index 85c70b7b63..258702a3b3 100644 --- a/packages/worker/src/api/routes/global/tests/scim.spec.ts +++ b/packages/worker/src/api/routes/global/tests/scim.spec.ts @@ -704,7 +704,6 @@ describe("scim", () => { expect(response).toEqual({ Resources: expect.arrayContaining( groups.map(g => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars const { members, ...groupData } = g return groupData }) @@ -724,7 +723,6 @@ describe("scim", () => { expect(response).toEqual({ Resources: expect.arrayContaining( groups.map(g => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars const { members, displayName, ...groupData } = g return groupData }) @@ -874,7 +872,6 @@ describe("scim", () => { qs: "excludedAttributes=members", }) - // eslint-disable-next-line @typescript-eslint/no-unused-vars const { members, ...expectedResponse } = group expect(response).toEqual(expectedResponse) From 7e1afd8b737e85270d21a9b12b02690eb377cb43 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 12:19:58 +0200 Subject: [PATCH 67/96] Remove // eslint-disable-next-line no-unused-vars in svelte --- packages/account-portal | 2 +- .../src/components/common/users/PasswordRepeatInput.svelte | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/account-portal b/packages/account-portal index 328c84234d..e525e1c873 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 328c84234d11d97d840f0eb2c72665b04ba9e4f8 +Subproject commit e525e1c87322689436dba62794dd984f66e433f8 diff --git a/packages/builder/src/components/common/users/PasswordRepeatInput.svelte b/packages/builder/src/components/common/users/PasswordRepeatInput.svelte index 496bee14ec..4a453ef049 100644 --- a/packages/builder/src/components/common/users/PasswordRepeatInput.svelte +++ b/packages/builder/src/components/common/users/PasswordRepeatInput.svelte @@ -9,7 +9,6 @@ "", requiredValidator ) - // eslint-disable-next-line no-unused-vars const [repeatPassword, _, repeatTouched] = createValidationStore( "", requiredValidator From a88924a41873ce43bd65ecdb470e24c7466af6e9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 12:24:41 +0200 Subject: [PATCH 68/96] Remove // eslint-disable-next-line no-unused-vars --- packages/account-portal | 2 +- packages/backend-core/src/queue/inMemoryQueue.ts | 1 - .../stores/datasourceCreation.test.js | 2 -- .../CreateExternalDatasourceModal/stores/onGoogleAuth.test.js | 4 ++-- packages/cli/src/hosting/utils.ts | 1 - packages/frontend-core/src/fetch/DataFetch.js | 3 +-- packages/server/src/features.ts | 3 --- packages/string-templates/src/processors/postprocessor.ts | 1 - packages/string-templates/src/processors/preprocessor.ts | 1 - packages/worker/src/features.ts | 1 - 10 files changed, 4 insertions(+), 15 deletions(-) diff --git a/packages/account-portal b/packages/account-portal index e525e1c873..e2357a67de 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit e525e1c87322689436dba62794dd984f66e433f8 +Subproject commit e2357a67de047c430908901889bbcb83767970ef diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index 06fe036da6..333accc985 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -115,7 +115,6 @@ class InMemoryQueue implements Partial { * a JSON message as this is required by Bull. * @param repeat serves no purpose for the import queue. */ - // eslint-disable-next-line no-unused-vars async add(data: any, opts?: JobOptions) { const jobId = opts?.jobId?.toString() if (jobId && this._queuedJobIds.has(jobId)) { diff --git a/packages/builder/src/pages/builder/app/[application]/data/_components/CreateExternalDatasourceModal/stores/datasourceCreation.test.js b/packages/builder/src/pages/builder/app/[application]/data/_components/CreateExternalDatasourceModal/stores/datasourceCreation.test.js index 4c3a958134..75a3afc2ff 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/_components/CreateExternalDatasourceModal/stores/datasourceCreation.test.js +++ b/packages/builder/src/pages/builder/app/[application]/data/_components/CreateExternalDatasourceModal/stores/datasourceCreation.test.js @@ -4,8 +4,6 @@ import { createDatasourceCreationStore, } from "./datasourceCreation" import { get } from "svelte/store" -// eslint-disable-next-line no-unused-vars -import { shouldIntegrationFetchTableNames } from "stores/selectors" vi.mock("stores/selectors", () => ({ shouldIntegrationFetchTableNames: vi.fn(), diff --git a/packages/builder/src/pages/builder/app/[application]/data/_components/CreateExternalDatasourceModal/stores/onGoogleAuth.test.js b/packages/builder/src/pages/builder/app/[application]/data/_components/CreateExternalDatasourceModal/stores/onGoogleAuth.test.js index f3e921c698..edf133ea0d 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/_components/CreateExternalDatasourceModal/stores/onGoogleAuth.test.js +++ b/packages/builder/src/pages/builder/app/[application]/data/_components/CreateExternalDatasourceModal/stores/onGoogleAuth.test.js @@ -1,9 +1,9 @@ import { it, expect, describe, beforeEach, vi } from "vitest" import { createOnGoogleAuthStore } from "./onGoogleAuth" import { writable, get } from "svelte/store" -// eslint-disable-next-line no-unused-vars +// eslint-disable-next-line import { params } from "@roxi/routify" -// eslint-disable-next-line no-unused-vars +// eslint-disable-next-line import { integrations } from "stores/builder" import { IntegrationTypes } from "constants/backend" diff --git a/packages/cli/src/hosting/utils.ts b/packages/cli/src/hosting/utils.ts index a4b28539e9..cbf6d9b0c3 100644 --- a/packages/cli/src/hosting/utils.ts +++ b/packages/cli/src/hosting/utils.ts @@ -105,7 +105,6 @@ export function getAppService(path: string) { } export function updateDockerComposeService( - // eslint-disable-next-line no-unused-vars updateFn: (service: DockerCompose) => void ) { const opts = ["docker-compose.yaml", "docker-compose.yml"] diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js index 92115efef0..3a45543c60 100644 --- a/packages/frontend-core/src/fetch/DataFetch.js +++ b/packages/frontend-core/src/fetch/DataFetch.js @@ -348,8 +348,7 @@ export default class DataFetch { * Determine the feature flag for this datasource definition * @param definition */ - // eslint-disable-next-line no-unused-vars - determineFeatureFlags(definition) { + determineFeatureFlags(_definition) { return { supportsSearch: false, supportsSort: false, diff --git a/packages/server/src/features.ts b/packages/server/src/features.ts index 3ab9410f53..f040cf82a2 100644 --- a/packages/server/src/features.ts +++ b/packages/server/src/features.ts @@ -1,11 +1,8 @@ import { features } from "@budibase/backend-core" import env from "./environment" -// eslint-disable-next-line no-unused-vars enum AppFeature { - // eslint-disable-next-line no-unused-vars API = "api", - // eslint-disable-next-line no-unused-vars AUTOMATIONS = "automations", } diff --git a/packages/string-templates/src/processors/postprocessor.ts b/packages/string-templates/src/processors/postprocessor.ts index 6f7260718b..b8d99682b1 100644 --- a/packages/string-templates/src/processors/postprocessor.ts +++ b/packages/string-templates/src/processors/postprocessor.ts @@ -4,7 +4,6 @@ export const PostProcessorNames = { CONVERT_LITERALS: "convert-literals", } -/* eslint-disable no-unused-vars */ class Postprocessor { name: string private fn: any diff --git a/packages/string-templates/src/processors/preprocessor.ts b/packages/string-templates/src/processors/preprocessor.ts index 141b2be3a9..010c259e12 100644 --- a/packages/string-templates/src/processors/preprocessor.ts +++ b/packages/string-templates/src/processors/preprocessor.ts @@ -9,7 +9,6 @@ export const PreprocessorNames = { FINALISE: "finalise", } -/* eslint-disable no-unused-vars */ class Preprocessor { name: string private fn: any diff --git a/packages/worker/src/features.ts b/packages/worker/src/features.ts index 33fce3aebe..075b3b81ca 100644 --- a/packages/worker/src/features.ts +++ b/packages/worker/src/features.ts @@ -1,7 +1,6 @@ import { features } from "@budibase/backend-core" import env from "./environment" -// eslint-disable-next-line no-unused-vars enum WorkerFeature {} const featureList: WorkerFeature[] = features.processFeatureEnvVar( From 25dba0be356bb77ced51c465e28e0933afd6dff8 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Apr 2024 13:54:23 +0100 Subject: [PATCH 69/96] This removes the SearchQuery which was a duplication of the SearchFilters interface - the duplicated types were not correctly getting overlay on each other, causing build errors. --- packages/backend-core/src/users/users.ts | 12 ++-- .../src/components/FilterBuilder.svelte | 4 +- .../src/api/routes/tests/viewV2.spec.ts | 12 ++-- packages/shared-core/src/filters.ts | 68 ++++++++++--------- .../shared-core/src/tests/filters.test.ts | 30 ++++---- packages/types/src/api/web/searchFilter.ts | 61 +---------------- packages/types/src/api/web/user.ts | 4 +- packages/types/src/sdk/search.ts | 42 ++++++++---- packages/worker/src/tests/api/users.ts | 4 +- 9 files changed, 99 insertions(+), 138 deletions(-) diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 48920a3771..7d62a6ef39 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -17,8 +17,8 @@ import { ContextUser, CouchFindOptions, DatabaseQueryOpts, - SearchQuery, - SearchQueryOperators, + SearchFilters, + SearchFilterOperator, SearchUsersRequest, User, } from "@budibase/types" @@ -44,11 +44,11 @@ function removeUserPassword(users: User | User[]) { return users } -export function isSupportedUserSearch(query: SearchQuery) { +export function isSupportedUserSearch(query: SearchFilters) { const allowed = [ - { op: SearchQueryOperators.STRING, key: "email" }, - { op: SearchQueryOperators.EQUAL, key: "_id" }, - { op: SearchQueryOperators.ONE_OF, key: "_id" }, + { op: SearchFilterOperator.STRING, key: "email" }, + { op: SearchFilterOperator.EQUAL, key: "_id" }, + { op: SearchFilterOperator.ONE_OF, key: "_id" }, ] for (let [key, operation] of Object.entries(query)) { if (typeof operation !== "object") { diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte index 45b81f2a78..1b252d5b06 100644 --- a/packages/frontend-core/src/components/FilterBuilder.svelte +++ b/packages/frontend-core/src/components/FilterBuilder.svelte @@ -11,7 +11,7 @@ Label, Multiselect, } from "@budibase/bbui" - import { FieldType, SearchQueryOperators } from "@budibase/types" + import { FieldType, SearchFilterOperator } from "@budibase/types" import { generate } from "shortid" import { LuceneUtils, Constants } from "@budibase/frontend-core" import { getContext } from "svelte" @@ -247,7 +247,7 @@ {:else if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)} - {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === SearchQueryOperators.ONE_OF)} + {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === SearchFilterOperator.ONE_OF)} { +const cleanupQuery = (query: SearchFilters) => { if (!query) { return query } for (let filterField of NoEmptyFilterStrings) { - if (!query[filterField]) { + const operator = filterField as SearchFilterOperator + if (!query[operator]) { continue } - for (let [key, value] of Object.entries(query[filterField]!)) { + for (let [key, value] of Object.entries(query[operator]!)) { if (value == null || value === "") { - delete query[filterField]![key] + delete query[operator]![key] } } } @@ -136,7 +137,7 @@ export const removeKeyNumbering = (key: string): string => { * @param filter the builder filter structure */ export const buildLuceneQuery = (filter: SearchFilter[]) => { - let query: SearchQuery = { + let query: SearchFilters = { string: {}, fuzzy: {}, range: {}, @@ -157,6 +158,7 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => { filter.forEach(expression => { let { operator, field, type, value, externalType, onEmptyFilter } = expression + const queryOperator = operator as SearchFilterOperator const isHbs = typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0 // Parse all values into correct types @@ -171,8 +173,8 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => { if ( type === "datetime" && !isHbs && - operator !== "empty" && - operator !== "notEmpty" + queryOperator !== "empty" && + queryOperator !== "notEmpty" ) { // Ensure date value is a valid date and parse into correct format if (!value) { @@ -185,7 +187,7 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => { } } if (type === "number" && typeof value === "string" && !isHbs) { - if (operator === "oneOf") { + if (queryOperator === "oneOf") { value = value.split(",").map(item => parseFloat(item)) } else { value = parseFloat(value) @@ -225,24 +227,24 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => { ) { query.range[field].high = value } - } else if (query[operator] && operator !== "onEmptyFilter") { + } else if (query[queryOperator] && operator !== "onEmptyFilter") { if (type === "boolean") { // Transform boolean filters to cope with null. // "equals false" needs to be "not equals true" // "not equals false" needs to be "equals true" - if (operator === "equal" && value === false) { + if (queryOperator === "equal" && value === false) { query.notEqual = query.notEqual || {} query.notEqual[field] = true - } else if (operator === "notEqual" && value === false) { + } else if (queryOperator === "notEqual" && value === false) { query.equal = query.equal || {} query.equal[field] = true } else { - query[operator] = query[operator] || {} - query[operator]![field] = value + query[queryOperator] = query[queryOperator] || {} + query[queryOperator]![field] = value } } else { - query[operator] = query[operator] || {} - query[operator]![field] = value + query[queryOperator] = query[queryOperator] || {} + query[queryOperator]![field] = value } } }) @@ -255,7 +257,7 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => { * @param docs the data * @param query the JSON lucene query */ -export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { +export const runLuceneQuery = (docs: any[], query?: SearchFilters) => { if (!docs || !Array.isArray(docs)) { return [] } @@ -269,7 +271,7 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { // Iterates over a set of filters and evaluates a fail function against a doc const match = ( - type: keyof SearchQueryFields, + type: SearchFilterOperator, failFn: (docValue: any, testValue: any) => boolean ) => (doc: any) => { @@ -286,7 +288,7 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { // Process a string match (fails if the value does not start with the string) const stringMatch = match( - SearchQueryOperators.STRING, + SearchFilterOperator.STRING, (docValue: string, testValue: string) => { return ( !docValue || @@ -297,7 +299,7 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { // Process a fuzzy match (treat the same as starts with when running locally) const fuzzyMatch = match( - SearchQueryOperators.FUZZY, + SearchFilterOperator.FUZZY, (docValue: string, testValue: string) => { return ( !docValue || @@ -308,7 +310,7 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { // Process a range match const rangeMatch = match( - SearchQueryOperators.RANGE, + SearchFilterOperator.RANGE, ( docValue: string | number | null, testValue: { low: number; high: number } @@ -331,7 +333,7 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { // Process an equal match (fails if the value is different) const equalMatch = match( - SearchQueryOperators.EQUAL, + SearchFilterOperator.EQUAL, (docValue: any, testValue: string | null) => { return testValue != null && testValue !== "" && docValue !== testValue } @@ -339,7 +341,7 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { // Process a not-equal match (fails if the value is the same) const notEqualMatch = match( - SearchQueryOperators.NOT_EQUAL, + SearchFilterOperator.NOT_EQUAL, (docValue: any, testValue: string | null) => { return testValue != null && testValue !== "" && docValue === testValue } @@ -347,7 +349,7 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { // Process an empty match (fails if the value is not empty) const emptyMatch = match( - SearchQueryOperators.EMPTY, + SearchFilterOperator.EMPTY, (docValue: string | null) => { return docValue != null && docValue !== "" } @@ -355,7 +357,7 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { // Process a not-empty match (fails is the value is empty) const notEmptyMatch = match( - SearchQueryOperators.NOT_EMPTY, + SearchFilterOperator.NOT_EMPTY, (docValue: string | null) => { return docValue == null || docValue === "" } @@ -363,7 +365,7 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { // Process an includes match (fails if the value is not included) const oneOf = match( - SearchQueryOperators.ONE_OF, + SearchFilterOperator.ONE_OF, (docValue: any, testValue: any) => { if (typeof testValue === "string") { testValue = testValue.split(",") @@ -376,28 +378,28 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { ) const containsAny = match( - SearchQueryOperators.CONTAINS_ANY, + SearchFilterOperator.CONTAINS_ANY, (docValue: any, testValue: any) => { return !docValue?.includes(...testValue) } ) const contains = match( - SearchQueryOperators.CONTAINS, + SearchFilterOperator.CONTAINS, (docValue: string | any[], testValue: any[]) => { return !testValue?.every((item: any) => docValue?.includes(item)) } ) const notContains = match( - SearchQueryOperators.NOT_CONTAINS, + SearchFilterOperator.NOT_CONTAINS, (docValue: string | any[], testValue: any[]) => { return testValue?.every((item: any) => docValue?.includes(item)) } ) const docMatch = (doc: any) => { - const filterFunctions: Record boolean> = + const filterFunctions: Record boolean> = { string: stringMatch, fuzzy: fuzzyMatch, @@ -412,7 +414,7 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { notContains: notContains, } - const activeFilterKeys: SearchQueryOperators[] = Object.entries(query || {}) + const activeFilterKeys: SearchFilterOperator[] = Object.entries(query || {}) .filter( ([key, value]: [string, any]) => !["allOr", "onEmptyFilter"].includes(key) && @@ -480,7 +482,7 @@ export const luceneLimit = (docs: any[], limit: string) => { return docs.slice(0, numLimit) } -export const hasFilters = (query?: SearchQuery) => { +export const hasFilters = (query?: SearchFilters) => { if (!query) { return false } diff --git a/packages/shared-core/src/tests/filters.test.ts b/packages/shared-core/src/tests/filters.test.ts index e74e37d681..f188c5f951 100644 --- a/packages/shared-core/src/tests/filters.test.ts +++ b/packages/shared-core/src/tests/filters.test.ts @@ -1,6 +1,6 @@ import { - SearchQuery, - SearchQueryOperators, + SearchFilters, + SearchFilterOperator, FieldType, SearchFilter, } from "@budibase/types" @@ -46,8 +46,8 @@ describe("runLuceneQuery", () => { }, ] - function buildQuery(filters: { [filterKey: string]: any }): SearchQuery { - const query: SearchQuery = { + function buildQuery(filters: { [filterKey: string]: any }): SearchFilters { + const query: SearchFilters = { string: {}, fuzzy: {}, range: {}, @@ -63,7 +63,7 @@ describe("runLuceneQuery", () => { } for (const filterKey in filters) { - query[filterKey as SearchQueryOperators] = filters[filterKey] + query[filterKey as SearchFilterOperator] = filters[filterKey] } return query @@ -265,13 +265,13 @@ describe("buildLuceneQuery", () => { it("should parseFloat if the type is a number, but the value is a numeric string", () => { const filter: SearchFilter[] = [ { - operator: SearchQueryOperators.EQUAL, + operator: SearchFilterOperator.EQUAL, field: "customer_id", type: FieldType.NUMBER, value: "1212", }, { - operator: SearchQueryOperators.ONE_OF, + operator: SearchFilterOperator.ONE_OF, field: "customer_id", type: FieldType.NUMBER, value: "1000,1212,3400", @@ -299,13 +299,13 @@ describe("buildLuceneQuery", () => { it("should not parseFloat if the type is a number, but the value is a handlebars binding string", () => { const filter: SearchFilter[] = [ { - operator: SearchQueryOperators.EQUAL, + operator: SearchFilterOperator.EQUAL, field: "customer_id", type: FieldType.NUMBER, value: "{{ customer_id }}", }, { - operator: SearchQueryOperators.ONE_OF, + operator: SearchFilterOperator.ONE_OF, field: "customer_id", type: FieldType.NUMBER, value: "{{ list_of_customer_ids }}", @@ -333,19 +333,19 @@ describe("buildLuceneQuery", () => { it("should cast string to boolean if the type is boolean", () => { const filter: SearchFilter[] = [ { - operator: SearchQueryOperators.EQUAL, + operator: SearchFilterOperator.EQUAL, field: "a", type: FieldType.BOOLEAN, value: "not_true", }, { - operator: SearchQueryOperators.NOT_EQUAL, + operator: SearchFilterOperator.NOT_EQUAL, field: "b", type: FieldType.BOOLEAN, value: "not_true", }, { - operator: SearchQueryOperators.EQUAL, + operator: SearchFilterOperator.EQUAL, field: "c", type: FieldType.BOOLEAN, value: "true", @@ -374,19 +374,19 @@ describe("buildLuceneQuery", () => { it("should split the string for contains operators", () => { const filter: SearchFilter[] = [ { - operator: SearchQueryOperators.CONTAINS, + operator: SearchFilterOperator.CONTAINS, field: "description", type: FieldType.ARRAY, value: "Large box,Heavy box,Small box", }, { - operator: SearchQueryOperators.NOT_CONTAINS, + operator: SearchFilterOperator.NOT_CONTAINS, field: "description", type: FieldType.ARRAY, value: "Large box,Heavy box,Small box", }, { - operator: SearchQueryOperators.CONTAINS_ANY, + operator: SearchFilterOperator.CONTAINS_ANY, field: "description", type: FieldType.ARRAY, value: "Large box,Heavy box,Small box", diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts index ac3c446e36..5223204a7f 100644 --- a/packages/types/src/api/web/searchFilter.ts +++ b/packages/types/src/api/web/searchFilter.ts @@ -1,68 +1,11 @@ import { FieldType } from "../../documents" -import { EmptyFilterOption } from "../../sdk" +import { EmptyFilterOption, SearchFilters } from "../../sdk" export type SearchFilter = { - operator: keyof SearchQuery + operator: keyof SearchFilters | "rangeLow" | "rangeHigh" onEmptyFilter?: EmptyFilterOption field: string type?: FieldType value: any externalType?: string } - -export enum SearchQueryOperators { - STRING = "string", - FUZZY = "fuzzy", - RANGE = "range", - EQUAL = "equal", - NOT_EQUAL = "notEqual", - EMPTY = "empty", - NOT_EMPTY = "notEmpty", - ONE_OF = "oneOf", - CONTAINS = "contains", - NOT_CONTAINS = "notContains", - CONTAINS_ANY = "containsAny", -} - -export type SearchQuery = { - allOr?: boolean - onEmptyFilter?: EmptyFilterOption - [SearchQueryOperators.STRING]?: { - [key: string]: string - } - [SearchQueryOperators.FUZZY]?: { - [key: string]: string - } - [SearchQueryOperators.RANGE]?: { - [key: string]: { - high: number | string - low: number | string - } - } - [SearchQueryOperators.EQUAL]?: { - [key: string]: any - } - [SearchQueryOperators.NOT_EQUAL]?: { - [key: string]: any - } - [SearchQueryOperators.EMPTY]?: { - [key: string]: any - } - [SearchQueryOperators.NOT_EMPTY]?: { - [key: string]: any - } - [SearchQueryOperators.ONE_OF]?: { - [key: string]: any[] - } - [SearchQueryOperators.CONTAINS]?: { - [key: string]: any[] - } - [SearchQueryOperators.NOT_CONTAINS]?: { - [key: string]: any[] - } - [SearchQueryOperators.CONTAINS_ANY]?: { - [key: string]: any[] - } -} - -export type SearchQueryFields = Omit diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts index 0ef7493016..10630c272c 100644 --- a/packages/types/src/api/web/user.ts +++ b/packages/types/src/api/web/user.ts @@ -1,5 +1,5 @@ import { User } from "../../documents" -import { SearchQuery } from "./searchFilter" +import { SearchFilters } from "../../sdk" export interface SaveUserResponse { _id: string @@ -55,7 +55,7 @@ export interface InviteUsersResponse { export interface SearchUsersRequest { bookmark?: string - query?: SearchQuery + query?: SearchFilters appId?: string limit?: number paginate?: boolean diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 0b93fb9215..51d866c9de 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -3,47 +3,63 @@ import { Row, Table } from "../documents" import { SortType } from "../api" import { Knex } from "knex" +export enum SearchFilterOperator { + STRING = "string", + FUZZY = "fuzzy", + RANGE = "range", + EQUAL = "equal", + NOT_EQUAL = "notEqual", + EMPTY = "empty", + NOT_EMPTY = "notEmpty", + ONE_OF = "oneOf", + CONTAINS = "contains", + NOT_CONTAINS = "notContains", + CONTAINS_ANY = "containsAny", +} + export interface SearchFilters { allOr?: boolean onEmptyFilter?: EmptyFilterOption - string?: { + [SearchFilterOperator.STRING]?: { [key: string]: string } - fuzzy?: { + [SearchFilterOperator.FUZZY]?: { [key: string]: string } - range?: { + [SearchFilterOperator.RANGE]?: { [key: string]: { high: number | string low: number | string } } - equal?: { + [SearchFilterOperator.EQUAL]?: { [key: string]: any } - notEqual?: { + [SearchFilterOperator.NOT_EQUAL]?: { [key: string]: any } - empty?: { + [SearchFilterOperator.EMPTY]?: { [key: string]: any } - notEmpty?: { + [SearchFilterOperator.NOT_EMPTY]?: { [key: string]: any } - oneOf?: { + [SearchFilterOperator.ONE_OF]?: { [key: string]: any[] } - contains?: { - [key: string]: any[] | any - } - notContains?: { + [SearchFilterOperator.CONTAINS]?: { [key: string]: any[] } - containsAny?: { + [SearchFilterOperator.NOT_CONTAINS]?: { + [key: string]: any[] + } + [SearchFilterOperator.CONTAINS_ANY]?: { [key: string]: any[] } } +export type SearchQueryFields = Omit + export interface SortJson { [key: string]: { direction: SortDirection diff --git a/packages/worker/src/tests/api/users.ts b/packages/worker/src/tests/api/users.ts index d08a4ef8c7..541004391d 100644 --- a/packages/worker/src/tests/api/users.ts +++ b/packages/worker/src/tests/api/users.ts @@ -4,7 +4,7 @@ import { InviteUsersRequest, User, CreateAdminUserRequest, - SearchQuery, + SearchFilters, InviteUsersResponse, } from "@budibase/types" import structures from "../structures" @@ -150,7 +150,7 @@ export class UserAPI extends TestAPI { } searchUsers = ( - { query }: { query?: SearchQuery }, + { query }: { query?: SearchFilters }, opts?: { status?: number; noHeaders?: boolean } ) => { const req = this.request From c51df0eceb32f5448533a5212984e2e98af9f102 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Apr 2024 14:05:47 +0100 Subject: [PATCH 70/96] Fixing test case. --- .../sdk/app/rows/search/tests/{lucene.ts => lucene.spec.ts} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename packages/server/src/sdk/app/rows/search/tests/{lucene.ts => lucene.spec.ts} (98%) diff --git a/packages/server/src/sdk/app/rows/search/tests/lucene.ts b/packages/server/src/sdk/app/rows/search/tests/lucene.spec.ts similarity index 98% rename from packages/server/src/sdk/app/rows/search/tests/lucene.ts rename to packages/server/src/sdk/app/rows/search/tests/lucene.spec.ts index 708f362198..d9c1c79177 100644 --- a/packages/server/src/sdk/app/rows/search/tests/lucene.ts +++ b/packages/server/src/sdk/app/rows/search/tests/lucene.spec.ts @@ -160,7 +160,7 @@ describe("internal search", () => { const response = await search.paginatedSearch( { contains: { - column: "a", + column: ["a"], colArr: [1, 2, 3], }, }, @@ -168,7 +168,7 @@ describe("internal search", () => { ) checkLucene( response, - `(*:* AND column:a AND colArr:(1 AND 2 AND 3))`, + `(*:* AND column:(a) AND colArr:(1 AND 2 AND 3))`, PARAMS ) }) From 828d78f2afa63fc1a5932c8ce733bd9d835bad76 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Apr 2024 14:47:56 +0100 Subject: [PATCH 71/96] Fixing build after recent type updates. --- .../src/integrations/tests/sqlAlias.spec.ts | 3 ++- packages/shared-core/src/filters.ts | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index f4edab8dad..fda2a091fa 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -117,7 +117,8 @@ describe("Captures of real examples", () => { let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) const filters = queryJson.filters const notEqualsValue = Object.values(filters?.notEqual!)[0] - const rangeValue = Object.values(filters?.range!)[0] + const rangeValue: { high?: string | number; low?: string | number } = + Object.values(filters?.range!)[0] const equalValue = Object.values(filters?.equal!)[0] expect(query).toEqual({ diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 6010f064bf..0554e0c1e4 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -218,14 +218,16 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => { high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z", } } - if ((operator as any) === "rangeLow" && value != null && value !== "") { - query.range[field].low = value - } else if ( - (operator as any) === "rangeHigh" && - value != null && - value !== "" - ) { - query.range[field].high = value + if (operator === "rangeLow" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + low: value, + } + } else if (operator === "rangeHigh" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + high: value, + } } } else if (query[queryOperator] && operator !== "onEmptyFilter") { if (type === "boolean") { From 160c3913a2cd35d6ff200c2634f0f18d6c4c43e5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 15:48:44 +0200 Subject: [PATCH 72/96] Update submodule refs --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index e6c7cde321..c78bab4ca1 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit e6c7cde321b6943e3f0f464957b1c1c5b22fbb50 +Subproject commit c78bab4ca13176263510cbdbc7d115d9d9d22ad9 From fcb04c6be128dffb6450a62a891fee78a1f55b2d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 15:50:29 +0200 Subject: [PATCH 73/96] Update submodule refs --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index e2357a67de..266807f80b 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit e2357a67de047c430908901889bbcb83767970ef +Subproject commit 266807f80b63cddfdcdcff2c83ff0732a68bf3bb From c763d3758e083aeaf93e4b21b4946b3959a7b818 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 15:59:21 +0200 Subject: [PATCH 74/96] Update submodule refs --- packages/account-portal | 2 +- packages/pro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/account-portal b/packages/account-portal index 266807f80b..eb7d5da233 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 266807f80b63cddfdcdcff2c83ff0732a68bf3bb +Subproject commit eb7d5da233885c5cffd9c255d3e954d0cd39185e diff --git a/packages/pro b/packages/pro index c78bab4ca1..06b1064f7e 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit c78bab4ca13176263510cbdbc7d115d9d9d22ad9 +Subproject commit 06b1064f7e2f7cac5d4bef2ee999796a2a1f0f2c From b3ff97df7b32567adb1fc2f0491a9325f1241b0c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 16:13:28 +0200 Subject: [PATCH 75/96] Fix typo --- .../backend/DataTable/modals/CreateEditColumn.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 8716a00660..3034b40ae6 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -56,8 +56,8 @@ let primaryDisplay let indexes = [...($tables.selected.indexes || [])] let isCreating = undefined - let relationshipPart1 = PrettyRelationshipDefinitions.Many - let relationshipPart2 = PrettyRelationshipDefinitions.One + let relationshipPart1 = PrettyRelationshipDefinitions.MANY + let relationshipPart2 = PrettyRelationshipDefinitions.ONE let relationshipTableIdPrimary = null let relationshipTableIdSecondary = null let table = $tables.selected From f7a1b4cb12185c3ac8553ac68ef2714971be635e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 16:13:46 +0200 Subject: [PATCH 76/96] Fix switching types --- .../backend/DataTable/modals/CreateEditColumn.svelte | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 3034b40ae6..e56f28dca6 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -170,7 +170,7 @@ $: typeEnabled = !originalName || (originalName && - SWITCHABLE_TYPES[editableColumn.type] && + SWITCHABLE_TYPES[field.type] && !editableColumn?.autocolumn) const fieldDefinitions = Object.values(FIELDS).reduce( @@ -363,9 +363,9 @@ function getAllowedTypes() { if (originalName) { - return ( - SWITCHABLE_TYPES[editableColumn.type] || [editableColumn.type] - ).map(f => FIELDS[f.toUpperCase()]) + return (SWITCHABLE_TYPES[field.type] || [editableColumn.type]).map( + f => FIELDS[f.toUpperCase()] + ) } const isUsers = From ae137ca67711050eddbd86a17c0eb2439a8311ca Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 16:34:08 +0200 Subject: [PATCH 77/96] Fix update --- packages/builder/src/stores/builder/tables.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/stores/builder/tables.js b/packages/builder/src/stores/builder/tables.js index c2aab5634b..88b26929ad 100644 --- a/packages/builder/src/stores/builder/tables.js +++ b/packages/builder/src/stores/builder/tables.js @@ -64,7 +64,7 @@ export function createTablesStore() { if ( oldField != null && oldField?.type !== field.type && - SWITCHABLE_TYPES[oldField?.type] + !SWITCHABLE_TYPES[oldField?.type]?.includes(field.type) ) { updatedTable.schema[key] = oldField } From 7c3c82013c0b27e4ecd3d08e5b81e72ea2fa2df1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 16:43:13 +0200 Subject: [PATCH 78/96] Sort types --- .../backend/DataTable/modals/CreateEditColumn.svelte | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index e56f28dca6..ff9830c92a 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -363,9 +363,14 @@ function getAllowedTypes() { if (originalName) { - return (SWITCHABLE_TYPES[field.type] || [editableColumn.type]).map( - f => FIELDS[f.toUpperCase()] - ) + const possibleTypes = ( + SWITCHABLE_TYPES[field.type] || [editableColumn.type] + ).map(t => t.toLowerCase()) + return Object.entries(FIELDS) + .filter(([fieldType]) => + possibleTypes.includes(fieldType.toLowerCase()) + ) + .map(([_, fieldDefinition]) => fieldDefinition) } const isUsers = From 1bc7072a720bd44001353033f75e419a873bda5a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 17:18:57 +0200 Subject: [PATCH 79/96] Fix type swap on refetch schema --- packages/server/src/integrations/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 39f2bd3cb0..cc832d8062 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -354,7 +354,7 @@ function copyExistingPropsOver( // If the db column type changed to a non-compatible one, we want to re-fetch it if ( updatedColumnType !== existingColumnType && - !SWITCHABLE_TYPES[existingColumnType]?.includes(updatedColumnType) + !SWITCHABLE_TYPES[updatedColumnType]?.includes(existingColumnType) ) { continue } From a33c2599b51b793d13ba25a614a94d18b5d07898 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Apr 2024 16:25:27 +0100 Subject: [PATCH 80/96] Adding error to catch scenario that caused tests to fail - fixing cases of table metadata not aligning with entityId --- .../api/controllers/row/ExternalRequest.ts | 28 +++++++++++++------ .../server/src/integrations/base/query.ts | 12 ++++++++ packages/types/src/sdk/datasources.ts | 8 ++++++ 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 3dd3f9b8e7..132028b86f 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -374,38 +374,44 @@ export class ExternalRequest { ) { continue } - let tableId: string | undefined, + let relatedTableId: string | undefined, lookupField: string | undefined, fieldName: string | undefined if (isManyToMany(field)) { - tableId = field.through + relatedTableId = field.through lookupField = primaryKey fieldName = field.throughTo || primaryKey } else if (isManyToOne(field)) { - tableId = field.tableId + relatedTableId = field.tableId lookupField = field.foreignKey fieldName = field.fieldName } - if (!tableId || !lookupField || !fieldName) { + if (!relatedTableId || !lookupField || !fieldName) { throw new Error( "Unable to lookup relationships - undefined column properties." ) } - const { tableName: relatedTableName } = breakExternalTableId(tableId) + const { tableName: relatedTableName } = + breakExternalTableId(relatedTableId) // @ts-ignore const linkPrimaryKey = this.tables[relatedTableName].primary[0] if (!lookupField || !row[lookupField]) { continue } + const endpoint = getEndpoint(relatedTableId, Operation.READ) + const relatedTable = this.tables[endpoint.entityId] + if (!relatedTable) { + throw new Error("unable to find related table") + } const response = await getDatasourceAndQuery({ - endpoint: getEndpoint(tableId, Operation.READ), + endpoint: endpoint, filters: { equal: { [fieldName]: row[lookupField], }, }, meta: { - table, + table: relatedTable, }, }) // this is the response from knex if no rows found @@ -414,7 +420,11 @@ export class ExternalRequest { const storeTo = isManyToMany(field) ? field.throughFrom || linkPrimaryKey : fieldName - related[storeTo] = { rows, isMany: isManyToMany(field), tableId } + related[storeTo] = { + rows, + isMany: isManyToMany(field), + tableId: relatedTableId, + } } return related } @@ -484,7 +494,7 @@ export class ExternalRequest { body, filters: buildFilters(id, {}, linkTable), meta: { - table, + table: linkTable, }, }) ) diff --git a/packages/server/src/integrations/base/query.ts b/packages/server/src/integrations/base/query.ts index 03e6028e32..371592bece 100644 --- a/packages/server/src/integrations/base/query.ts +++ b/packages/server/src/integrations/base/query.ts @@ -2,6 +2,7 @@ import { QueryJson, Datasource, DatasourcePlusQueryResponse, + RowOperations, } from "@budibase/types" import { getIntegration } from "../index" import sdk from "../../sdk" @@ -10,6 +11,17 @@ export async function makeExternalQuery( datasource: Datasource, json: QueryJson ): Promise { + const entityId = json.endpoint.entityId, + tableName = json.meta.table.name, + tableId = json.meta.table._id + // case found during testing - make sure this doesn't happen again + if ( + RowOperations.includes(json.endpoint.operation) && + entityId !== tableId && + entityId !== tableName + ) { + throw new Error("Entity ID and table metadata do not align") + } datasource = await sdk.datasources.enrich(datasource) const Integration = await getIntegration(datasource.source) // query is the opinionated function diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index e1a012d81e..77e4877dfa 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -14,6 +14,14 @@ export enum Operation { DELETE_TABLE = "DELETE_TABLE", } +export const RowOperations = [ + Operation.CREATE, + Operation.READ, + Operation.UPDATE, + Operation.DELETE, + Operation.BULK_CREATE, +] + export enum SortDirection { ASCENDING = "ASCENDING", DESCENDING = "DESCENDING", From 30077418eb267a66a2ff5d8c7594dec88ff73219 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Apr 2024 16:31:45 +0100 Subject: [PATCH 81/96] Fixing linting issue. --- packages/server/src/api/controllers/row/ExternalRequest.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 132028b86f..be6ac885df 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -447,7 +447,6 @@ export class ExternalRequest { // if we're creating (in a through table) need to wipe the existing ones first const promises = [] const related = await this.lookupRelations(mainTableId, row) - const table = this.getTable(mainTableId)! for (let relationship of relationships) { const { key, tableId, isUpdate, id, ...rest } = relationship const body: { [key: string]: any } = processObjectSync(rest, row, {}) From 60ed4d8443938ee6a72c8205413105db0ea74b14 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Apr 2024 17:12:26 +0100 Subject: [PATCH 82/96] Updating how between/less than/greater than are handled for sqlite. --- packages/server/src/integrations/base/sql.ts | 59 +++----------- .../server/src/integrations/tests/sql.spec.ts | 24 ++++-- .../server/src/integrations/utils/index.ts | 2 + .../src/integrations/utils/sqlStatements.ts | 80 +++++++++++++++++++ .../src/integrations/{ => utils}/utils.ts | 6 +- 5 files changed, 112 insertions(+), 59 deletions(-) create mode 100644 packages/server/src/integrations/utils/index.ts create mode 100644 packages/server/src/integrations/utils/sqlStatements.ts rename packages/server/src/integrations/{ => utils}/utils.ts (98%) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 7ee22b5933..4c975cbef7 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -6,6 +6,7 @@ import { SqlClient, isValidFilter, getNativeSql, + SqlStatements, } from "../utils" import SqlTableQueryBuilder from "./sqlTable" import { @@ -163,6 +164,13 @@ class InternalBuilder { table: Table, opts: { aliases?: Record; relationship?: boolean } ): Knex.QueryBuilder { + if (!filters) { + return query + } + filters = parseFilters(filters) + // if all or specified in filters, then everything is an or + const allOr = filters.allOr + const sqlStatements = new SqlStatements(this.client, table, { allOr }) const tableName = this.client === SqlClient.SQL_LITE ? table._id! : table.name @@ -261,12 +269,6 @@ class InternalBuilder { } } - if (!filters) { - return query - } - filters = parseFilters(filters) - // if all or specified in filters, then everything is an or - const allOr = filters.allOr if (filters.oneOf) { iterate(filters.oneOf, (key, array) => { const fnc = allOr ? "orWhereIn" : "whereIn" @@ -293,9 +295,6 @@ class InternalBuilder { } if (filters.range) { iterate(filters.range, (key, value) => { - const fieldName = key.split(".")[1] - const field = table.schema[fieldName] - const isEmptyObject = (val: any) => { return ( val && @@ -312,47 +311,11 @@ class InternalBuilder { const lowValid = isValidFilter(value.low), highValid = isValidFilter(value.high) if (lowValid && highValid) { - // Use a between operator if we have 2 valid range values - if ( - field.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - query = query.whereRaw( - `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, - [value.low, value.high] - ) - } else { - const fnc = allOr ? "orWhereBetween" : "whereBetween" - query = query[fnc](key, [value.low, value.high]) - } + query = sqlStatements.between(query, key, value.low, value.high) } else if (lowValid) { - // Use just a single greater than operator if we only have a low - if ( - field.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - query = query.whereRaw( - `CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, - [value.low] - ) - } else { - const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, ">=", value.low) - } + query = sqlStatements.lower(query, key, value.low) } else if (highValid) { - // Use just a single less than operator if we only have a high - if ( - field.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - query = query.whereRaw( - `CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, - [value.high] - ) - } else { - const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, "<=", value.high) - } + query = sqlStatements.higher(query, key, value.high) } }) } diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index 4ee544cc5e..5de9cc4fbc 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -1,11 +1,11 @@ import { SqlClient } from "../utils" import Sql from "../base/sql" import { + FieldType, Operation, QueryJson, - TableSourceType, Table, - FieldType, + TableSourceType, } from "@budibase/types" const TABLE_NAME = "test" @@ -13,7 +13,12 @@ const TABLE: Table = { type: "table", sourceType: TableSourceType.EXTERNAL, sourceId: "SOURCE_ID", - schema: {}, + schema: { + id: { + name: "id", + type: FieldType.NUMBER, + }, + }, name: TABLE_NAME, primary: ["id"], } @@ -73,7 +78,7 @@ function generateUpdateJson({ meta?: any }): QueryJson { if (!meta.table) { - meta.table = table + meta.table = TABLE } return { endpoint: endpoint(table, "UPDATE"), @@ -158,6 +163,9 @@ function generateManyRelationshipJson(config: { schema?: string } = {}) { }, ], extra: { idFilter: {} }, + meta: { + table: TABLE, + }, } } @@ -341,7 +349,7 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ bindings: [date, limit], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" > $1 limit $2) as "${TABLE_NAME}"`, + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" >= $1 limit $2) as "${TABLE_NAME}"`, }) }) @@ -360,7 +368,7 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ bindings: [date, limit], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" < $1 limit $2) as "${TABLE_NAME}"`, + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" <= $1 limit $2) as "${TABLE_NAME}"`, }) }) @@ -594,7 +602,7 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ bindings: ["2000-01-01 00:00:00", 500], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" > $1 limit $2) as "${TABLE_NAME}"`, + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" >= $1 limit $2) as "${TABLE_NAME}"`, }) }) @@ -613,7 +621,7 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ bindings: ["2010-01-01 00:00:00", 500], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" < $1 limit $2) as "${TABLE_NAME}"`, + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" <= $1 limit $2) as "${TABLE_NAME}"`, }) }) diff --git a/packages/server/src/integrations/utils/index.ts b/packages/server/src/integrations/utils/index.ts new file mode 100644 index 0000000000..a9c2019ba2 --- /dev/null +++ b/packages/server/src/integrations/utils/index.ts @@ -0,0 +1,2 @@ +export * from "./utils" +export { SqlStatements } from "./sqlStatements" diff --git a/packages/server/src/integrations/utils/sqlStatements.ts b/packages/server/src/integrations/utils/sqlStatements.ts new file mode 100644 index 0000000000..b05196f56f --- /dev/null +++ b/packages/server/src/integrations/utils/sqlStatements.ts @@ -0,0 +1,80 @@ +import { FieldType, Table, FieldSchema } from "@budibase/types" +import { SqlClient } from "./utils" +import { Knex } from "knex" + +export class SqlStatements { + client: string + table: Table + allOr: boolean | undefined + constructor( + client: string, + table: Table, + { allOr }: { allOr?: boolean } = {} + ) { + this.client = client + this.table = table + this.allOr = allOr + } + + getField(key: string): FieldSchema | undefined { + const fieldName = key.split(".")[1] + return this.table.schema[fieldName] + } + + between( + query: Knex.QueryBuilder, + key: string, + low: number | string, + high: number | string + ) { + // Use a between operator if we have 2 valid range values + const field = this.getField(key) + if ( + field?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw( + `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, + [low, high] + ) + } else { + const fnc = this.allOr ? "orWhereBetween" : "whereBetween" + query = query[fnc](key, [low, high]) + } + return query + } + + lower(query: Knex.QueryBuilder, key: string, low: number | string) { + // Use just a single greater than operator if we only have a low + const field = this.getField(key) + if ( + field?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [ + low, + ]) + } else { + const fnc = this.allOr ? "orWhere" : "where" + query = query[fnc](key, ">=", low) + } + return query + } + + higher(query: Knex.QueryBuilder, key: string, high: number | string) { + const field = this.getField(key) + // Use just a single less than operator if we only have a high + if ( + field?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [ + high, + ]) + } else { + const fnc = this.allOr ? "orWhere" : "where" + query = query[fnc](key, "<=", high) + } + return query + } +} diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils/utils.ts similarity index 98% rename from packages/server/src/integrations/utils.ts rename to packages/server/src/integrations/utils/utils.ts index d5f6d191e1..8f2bed04a2 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -5,10 +5,10 @@ import { FieldType, TableSourceType, } from "@budibase/types" -import { DocumentType, SEPARATOR } from "../db/utils" -import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../constants" +import { DocumentType, SEPARATOR } from "../../db/utils" +import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../../constants" import { helpers } from "@budibase/shared-core" -import env from "../environment" +import env from "../../environment" import { Knex } from "knex" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` From 58d4b2b56e79d2f09b6e96ce4efd6f3cf53d481d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Apr 2024 17:13:08 +0100 Subject: [PATCH 83/96] renaming some sqlstatement generation to lte/gte. --- packages/server/src/integrations/base/sql.ts | 4 ++-- packages/server/src/integrations/utils/sqlStatements.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 4c975cbef7..a8d746add9 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -313,9 +313,9 @@ class InternalBuilder { if (lowValid && highValid) { query = sqlStatements.between(query, key, value.low, value.high) } else if (lowValid) { - query = sqlStatements.lower(query, key, value.low) + query = sqlStatements.lte(query, key, value.low) } else if (highValid) { - query = sqlStatements.higher(query, key, value.high) + query = sqlStatements.gte(query, key, value.high) } }) } diff --git a/packages/server/src/integrations/utils/sqlStatements.ts b/packages/server/src/integrations/utils/sqlStatements.ts index b05196f56f..7a5482830b 100644 --- a/packages/server/src/integrations/utils/sqlStatements.ts +++ b/packages/server/src/integrations/utils/sqlStatements.ts @@ -44,7 +44,7 @@ export class SqlStatements { return query } - lower(query: Knex.QueryBuilder, key: string, low: number | string) { + lte(query: Knex.QueryBuilder, key: string, low: number | string) { // Use just a single greater than operator if we only have a low const field = this.getField(key) if ( @@ -61,7 +61,7 @@ export class SqlStatements { return query } - higher(query: Knex.QueryBuilder, key: string, high: number | string) { + gte(query: Knex.QueryBuilder, key: string, high: number | string) { const field = this.getField(key) // Use just a single less than operator if we only have a high if ( From e90e2b214e413a832902cfc55b45e5f269f66c9c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Apr 2024 17:36:19 +0100 Subject: [PATCH 84/96] Making sure meta.table is always available. --- .../routes/tests/queries/generic-sql.spec.ts | 18 ++++++++++++++++-- packages/server/src/sdk/app/rows/utils.ts | 6 ++++++ .../src/tests/utilities/api/datasource.ts | 2 +- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 7790f909e7..e8a38dcfaa 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -4,6 +4,7 @@ import { Query, QueryPreview, SourceName, + TableSourceType, } from "@budibase/types" import * as setup from "../utilities" import { @@ -740,12 +741,25 @@ describe.each( }) describe("query through datasource", () => { - it("should be able to query a pg datasource", async () => { + it("should be able to query the datasource", async () => { + const entityId = "test_table" + await config.api.datasource.update({ + ...datasource, + entities: { + [entityId]: { + name: entityId, + schema: {}, + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + }, + }, + }) const res = await config.api.datasource.query({ endpoint: { datasourceId: datasource._id!, operation: Operation.READ, - entityId: "test_table", + entityId, }, resource: { fields: ["id", "name"], diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index d307b17947..a9df4f89cd 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -52,6 +52,12 @@ export async function getDatasourceAndQuery( ): Promise { const datasourceId = json.endpoint.datasourceId const datasource = await sdk.datasources.get(datasourceId) + const table = datasource.entities?.[json.endpoint.entityId] + if (!json.meta && table) { + json.meta = { + table, + } + } return makeExternalQuery(datasource, json) } diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts index 0362a25940..6ac624f0db 100644 --- a/packages/server/src/tests/utilities/api/datasource.ts +++ b/packages/server/src/tests/utilities/api/datasource.ts @@ -61,7 +61,7 @@ export class DatasourceAPI extends TestAPI { } query = async ( - query: Omit, + query: Omit & Partial>, expectations?: Expectations ) => { return await this._post(`/api/datasources/query`, { From 17c6b4ab68770a5118229b435f5e80eb564d21cf Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 19:51:22 +0200 Subject: [PATCH 85/96] Remove skipped test --- .../server/src/integration-test/mysql.spec.ts | 58 ------------------- 1 file changed, 58 deletions(-) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index 04a7b548a6..69093c8668 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -233,64 +233,6 @@ describe("mysql integrations", () => { }) describe("POST /api/tables/", () => { - const emitDatasourceUpdateMock = jest.fn() - - // TODO: This is not actually required, will fix after cleaning the `_add` logic - // eslint-disable-next-line jest/no-disabled-tests - it.skip("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => { - const addColumnToTable: TableRequest = { - type: "table", - sourceType: TableSourceType.EXTERNAL, - name: uniqueTableName(), - sourceId: datasource._id!, - primary: ["id"], - schema: { - id: { - type: FieldType.AUTO, - name: "id", - autocolumn: true, - }, - new_column: { - type: FieldType.NUMBER, - name: "new_column", - }, - }, - } - - jest - .spyOn(builderSocket!, "emitDatasourceUpdate") - .mockImplementation(emitDatasourceUpdateMock) - - await makeRequest("post", "/api/tables/", addColumnToTable) - - const expectedTable: TableRequest = { - ...addColumnToTable, - schema: { - id: { - type: FieldType.NUMBER, - name: "id", - autocolumn: true, - externalType: "int unsigned", - }, - new_column: { - type: FieldType.NUMBER, - name: "new_column", - autocolumn: false, - externalType: "float(8,2)", - }, - }, - created: true, - _id: `${datasource._id}__${addColumnToTable.name}`, - } - - expect(emitDatasourceUpdateMock).toHaveBeenCalledTimes(1) - const emittedDatasource: Datasource = - emitDatasourceUpdateMock.mock.calls[0][1] - expect(emittedDatasource.entities![expectedTable.name]).toEqual( - expectedTable - ) - }) - it("will rename a column", async () => { await makeRequest("post", "/api/tables/", primaryMySqlTable) From 5e094dd3bca9225b0263af8c831836642924cd23 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 20:06:54 +0200 Subject: [PATCH 86/96] Lint --- packages/server/src/integration-test/mysql.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index 69093c8668..b4eb1035d6 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -16,7 +16,6 @@ import { getDatasource, rawQuery, } from "../integrations/tests/utils" -import { builderSocket } from "../websockets" import { generator } from "@budibase/backend-core/tests" // @ts-ignore fetch.mockSearch() From c36ca27292375c5a85f991aeae3306224d802216 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 Apr 2024 13:48:50 +0200 Subject: [PATCH 87/96] Use field types --- .../DataTable/modals/CreateEditColumn.svelte | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index ff9830c92a..e8e4180c3d 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -34,15 +34,15 @@ import { RowUtils } from "@budibase/frontend-core" import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte" - const AUTO_TYPE = FIELDS.AUTO.type - const FORMULA_TYPE = FIELDS.FORMULA.type - const LINK_TYPE = FIELDS.LINK.type - const STRING_TYPE = FIELDS.STRING.type - const NUMBER_TYPE = FIELDS.NUMBER.type - const JSON_TYPE = FIELDS.JSON.type - const DATE_TYPE = FIELDS.DATETIME.type - const USER_TYPE = FIELDS.USER.subtype - const USERS_TYPE = FIELDS.USERS.subtype + const AUTO_TYPE = FieldType.AUTO + const FORMULA_TYPE = FieldType.FORMULA + const LINK_TYPE = FieldType.LINK + const STRING_TYPE = FieldType.STRING + const NUMBER_TYPE = FieldType.NUMBER + const JSON_TYPE = FieldType.JSON + const DATE_TYPE = FieldType.DATETIME + const USER_TYPE = FieldSubtype.USER + const USERS_TYPE = FieldSubtype.USERS const dispatch = createEventDispatcher() const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"] From e327ebd613ce25736c375ed60b18eccc33a2e521 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 18 Apr 2024 10:06:34 +0200 Subject: [PATCH 88/96] Do not display relationship selector if autocolumn links --- .../components/backend/DataTable/modals/CreateEditColumn.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index e8e4180c3d..d271462f3e 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -626,7 +626,7 @@ />
- {:else if editableColumn.type === FieldType.LINK} + {:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn} Date: Thu, 18 Apr 2024 08:26:29 +0000 Subject: [PATCH 89/96] Bump version to 2.23.7 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index a2be7be7b4..19f6486f22 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.23.6", + "version": "2.23.7", "npmClient": "yarn", "packages": [ "packages/*", From e90d40c0be5aa16dd893b9528e6bedb7ecec6cc9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 18 Apr 2024 10:49:22 +0200 Subject: [PATCH 90/96] Add all scripts as nx cache inputs --- nx.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nx.json b/nx.json index 618395ec90..8ba8798946 100644 --- a/nx.json +++ b/nx.json @@ -9,10 +9,7 @@ }, "targetDefaults": { "build": { - "inputs": [ - "{workspaceRoot}/scripts/build.js", - "{workspaceRoot}/lerna.json" - ] + "inputs": ["{workspaceRoot}/scripts/*", "{workspaceRoot}/lerna.json"] } } } From 8e2fb778233a9b8a3d7373655f80db2cd0968c81 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 18 Apr 2024 10:53:23 +0200 Subject: [PATCH 91/96] Prevent dev:docker running for oss --- package.json | 2 +- scripts/devDocker.sh | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100755 scripts/devDocker.sh diff --git a/package.json b/package.json index e520b7c2cf..e60a086e17 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "dev:camunda": "./scripts/deploy-camunda.sh", "dev:all": "yarn run kill-all && lerna run --stream dev", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", - "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", + "dev:docker": "./scripts/devDocker.sh", "test": "REUSE_CONTAINERS=1 lerna run --concurrency 1 --stream test --stream", "lint:eslint": "eslint packages --max-warnings=0", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"", diff --git a/scripts/devDocker.sh b/scripts/devDocker.sh new file mode 100755 index 0000000000..3d454bafb5 --- /dev/null +++ b/scripts/devDocker.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Check if the pro submodule is loaded +if [ ! -d "./packages/pro/src" ]; then + echo "Submodule is not loaded. This is only allowed with loaded submodules." + exit 1 +fi + +yarn build --scope @budibase/server --scope @budibase/worker +docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0 + + From ccbbb2e1ee278a556543c32a4812c02dce06a5f4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 18 Apr 2024 10:53:58 +0200 Subject: [PATCH 92/96] Add message --- scripts/devDocker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/devDocker.sh b/scripts/devDocker.sh index 3d454bafb5..5e01e5813a 100755 --- a/scripts/devDocker.sh +++ b/scripts/devDocker.sh @@ -2,7 +2,7 @@ # Check if the pro submodule is loaded if [ ! -d "./packages/pro/src" ]; then - echo "Submodule is not loaded. This is only allowed with loaded submodules." + echo "[ERROR] Submodule is not loaded. This is only allowed with loaded submodules." exit 1 fi From 6bdc726d55d9a4def42c67f00ab0e2f37be8e410 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Thu, 18 Apr 2024 13:52:00 +0100 Subject: [PATCH 93/96] Allow a displayName to be passed when creating admin user (#13516) * Allow a displayName to be passed when creating admin user * Set the first and last names * Don't format handlebars files on save * Use familyName and givenName --- .vscode/settings.json | 3 +++ packages/backend-core/src/users/db.ts | 4 ++++ packages/types/src/api/web/user.ts | 2 ++ packages/types/src/documents/global/user.ts | 7 +++++++ packages/worker/src/api/controllers/global/users.ts | 5 ++++- packages/worker/src/api/routes/global/users.ts | 2 ++ 6 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index e22d5a8866..0723219a8b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -24,5 +24,8 @@ }, "[svelte]": { "editor.defaultFormatter": "svelte.svelte-vscode" + }, + "[handlebars]": { + "editor.formatOnSave": false } } diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 6165a68e57..f77c6385ba 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -50,6 +50,8 @@ type CreateAdminUserOpts = { hashPassword?: boolean requirePassword?: boolean skipPasswordValidation?: boolean + firstName?: string + lastName?: string } type FeatureFns = { isSSOEnforced: FeatureFn; isAppBuildersEnabled: FeatureFn } @@ -517,6 +519,8 @@ export class UserDB { global: true, }, tenantId, + firstName: opts?.firstName, + lastName: opts?.lastName, } if (opts?.ssoId) { user.ssoId = opts.ssoId diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts index 10630c272c..75f00760bf 100644 --- a/packages/types/src/api/web/user.ts +++ b/packages/types/src/api/web/user.ts @@ -66,6 +66,8 @@ export interface CreateAdminUserRequest { password?: string tenantId: string ssoId?: string + familyName?: string + givenName?: string } export interface AddSSoUserRequest { diff --git a/packages/types/src/documents/global/user.ts b/packages/types/src/documents/global/user.ts index a4e6b613c6..9c7dc80e49 100644 --- a/packages/types/src/documents/global/user.ts +++ b/packages/types/src/documents/global/user.ts @@ -22,6 +22,13 @@ export interface UserSSO { providerType: SSOProviderType oauth2?: OAuth2 thirdPartyProfile?: SSOProfileJson + profile?: { + displayName?: string + name?: { + givenName?: string + familyName?: string + } + } } export type SSOUser = User & UserSSO diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 4c1af90d38..b610ecce1a 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -116,7 +116,8 @@ const parseBooleanParam = (param: any) => { export const adminUser = async ( ctx: Ctx ) => { - const { email, password, tenantId, ssoId } = ctx.request.body + const { email, password, tenantId, ssoId, givenName, familyName } = + ctx.request.body if (await platform.tenants.exists(tenantId)) { ctx.throw(403, "Organisation already exists.") @@ -151,6 +152,8 @@ export const adminUser = async ( ssoId, hashPassword, requirePassword, + firstName: givenName, + lastName: familyName, }) // events diff --git a/packages/worker/src/api/routes/global/users.ts b/packages/worker/src/api/routes/global/users.ts index e7c77678fc..b40c491830 100644 --- a/packages/worker/src/api/routes/global/users.ts +++ b/packages/worker/src/api/routes/global/users.ts @@ -16,6 +16,8 @@ function buildAdminInitValidation() { password: OPTIONAL_STRING, tenantId: Joi.string().required(), ssoId: Joi.string(), + familyName: OPTIONAL_STRING, + givenName: OPTIONAL_STRING, }) .required() .unknown(false) From 21f554a6b89049b402cf05e3cf12de847dde2065 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 18 Apr 2024 12:53:32 +0000 Subject: [PATCH 94/96] Bump version to 2.23.8 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 19f6486f22..99a3e46ab2 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.23.7", + "version": "2.23.8", "npmClient": "yarn", "packages": [ "packages/*", From dafa20ce47f6619985f95c75bd1fce2e53bc770a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 18 Apr 2024 11:20:11 +0200 Subject: [PATCH 95/96] Add platform docs to mssql docker-compose --- packages/server/scripts/integrations/mssql/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/scripts/integrations/mssql/docker-compose.yaml b/packages/server/scripts/integrations/mssql/docker-compose.yaml index 89222eddaa..0f10eee456 100644 --- a/packages/server/scripts/integrations/mssql/docker-compose.yaml +++ b/packages/server/scripts/integrations/mssql/docker-compose.yaml @@ -4,6 +4,7 @@ services: # user: sa # database: master mssql: + # platform: linux/amd64 image: bb/mssql build: context: . From 642f30416a19eae3c36882b469459720a749ac7b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 18 Apr 2024 16:40:43 +0200 Subject: [PATCH 96/96] Fix schema and aliasing usage --- packages/server/src/integrations/base/sql.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index a8d746add9..28b7eb910b 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -472,14 +472,13 @@ class InternalBuilder { ): Knex.QueryBuilder { const tableName = endpoint.entityId const tableAlias = aliases?.[tableName] - let table: string | Record = tableName - if (tableAlias) { - table = { [tableAlias]: tableName } - } - let query = knex(table) - if (endpoint.schema) { - query = query.withSchema(endpoint.schema) - } + + const query = knex( + this.tableNameWithSchema(tableName, { + alias: tableAlias, + schema: endpoint.schema, + }) + ) return query }