From fd21503f3e665c826e4174b957945e3e229df1aa Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 24 Sep 2024 12:43:41 +0100 Subject: [PATCH 01/85] Enable default values on options columns with validation and fix default values being available on primary display columns --- .../DataTable/modals/CreateEditColumn.svelte | 57 ++++++++++++------- packages/shared-core/src/table.ts | 2 +- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index a956d09ee6..b04ccdb6a9 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -136,9 +136,8 @@ } $: initialiseField(field, savingColumn) $: checkConstraints(editableColumn) - $: required = hasDefault - ? false - : !!editableColumn?.constraints?.presence || primaryDisplay + $: required = + primaryDisplay || (!hasDefault && !!editableColumn?.constraints?.presence) $: uneditable = $tables.selected?._id === TableNames.USERS && UNEDITABLE_USER_FIELDS.includes(editableColumn.name) @@ -168,7 +167,9 @@ $: canBeDisplay = canBeDisplayColumn(editableColumn.type) && !editableColumn.autocolumn $: canHaveDefault = - isEnabled("DEFAULT_VALUES") && canHaveDefaultColumn(editableColumn.type) + !required && + isEnabled("DEFAULT_VALUES") && + canHaveDefaultColumn(editableColumn.type) $: canBeRequired = editableColumn?.type !== LINK_TYPE && !uneditable && @@ -187,11 +188,11 @@ (originalName && SWITCHABLE_TYPES[field.type] && !editableColumn?.autocolumn) - $: allowedTypes = getAllowedTypes(datasource).map(t => ({ fieldId: makeFieldId(t.type, t.subtype), ...t, })) + $: bindings = getBindings({ table }) const fieldDefinitions = Object.values(FIELDS).reduce( // Storing the fields by complex field id @@ -281,6 +282,20 @@ delete saveColumn.fieldName } + // Ensure the field is not required if we have a default value + if (saveColumn.default) { + saveColumn.constraints.presence = false + } + + // Delete default value for options fields if the option is no longer available + if ( + saveColumn.type === FieldType.OPTIONS && + saveColumn.default && + !saveColumn.constraints.inclusion?.includes(saveColumn.default) + ) { + delete saveColumn.default + } + try { await tables.saveField({ originalName, @@ -727,7 +742,7 @@ formula: e.detail, } }} - bindings={getBindings({ table })} + {bindings} allowJS context={rowGoldenSample} /> @@ -766,27 +781,27 @@ {/if} {#if canHaveDefault} -
+ {#if editableColumn.type === FieldType.OPTIONS} + {:else} Date: Fri, 27 Sep 2024 08:33:09 +0100 Subject: [PATCH 05/85] Account for both shapes of the required constraint and ensure required trumps default values --- .../backend/DataTable/modals/CreateEditColumn.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index c4c3d661a3..05e0eba042 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -137,7 +137,9 @@ $: initialiseField(field, savingColumn) $: checkConstraints(editableColumn) $: required = - primaryDisplay || (!hasDefault && !!editableColumn?.constraints?.presence) + primaryDisplay || + editableColumn?.constraints?.presence === true || + editableColumn?.constraints?.presence?.allowEmpty === false $: uneditable = $tables.selected?._id === TableNames.USERS && UNEDITABLE_USER_FIELDS.includes(editableColumn.name) From 5d319768359953035226f53d9ac5ad5f027dc8f6 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 30 Sep 2024 13:07:32 +0100 Subject: [PATCH 06/85] updated automation thread to use ids and test --- .../tests/scenarios/branching.spec.ts | 57 +++++++++++------ .../tests/utilities/AutomationTestBuilder.ts | 61 ++++++++++++------- .../server/src/definitions/automations.ts | 1 + packages/server/src/threads/automation.ts | 18 ++++-- 4 files changed, 91 insertions(+), 46 deletions(-) diff --git a/packages/server/src/automations/tests/scenarios/branching.spec.ts b/packages/server/src/automations/tests/scenarios/branching.spec.ts index ae89fc18b5..76e04afdd3 100644 --- a/packages/server/src/automations/tests/scenarios/branching.spec.ts +++ b/packages/server/src/automations/tests/scenarios/branching.spec.ts @@ -17,44 +17,65 @@ describe("Branching automations", () => { afterAll(setup.afterAll) it("should run a multiple nested branching automation", async () => { + const firstLogId = "11111111-1111-1111-1111-111111111111" + const branch1LogId = "22222222-2222-2222-2222-222222222222" + const branch2LogId = "33333333-3333-3333-3333-333333333333" + const branch2Id = "44444444-4444-4444-4444-444444444444" + const builder = createAutomationBuilder({ name: "Test Trigger with Loop and Create Row", }) const results = await builder .appAction({ fields: {} }) - .serverLog({ text: "Starting automation" }) + .serverLog( + { text: "Starting automation" }, + { stepName: "FirstLog", id: firstLogId } + ) .branch({ topLevelBranch1: { steps: stepBuilder => - stepBuilder.serverLog({ text: "Branch 1" }).branch({ - branch1: { - steps: stepBuilder => - stepBuilder.serverLog({ text: "Branch 1.1" }), - condition: { - equal: { "{{steps.1.success}}": true }, + stepBuilder + .serverLog( + { text: "Branch 1" }, + { id: "66666666-6666-6666-6666-666666666666" } + ) + .branch({ + branch1: { + steps: stepBuilder => + stepBuilder.serverLog( + { text: "Branch 1.1" }, + { id: branch1LogId } + ), + condition: { + equal: { [`{{ steps.${firstLogId}.success }}`]: true }, + }, }, - }, - branch2: { - steps: stepBuilder => - stepBuilder.serverLog({ text: "Branch 1.2" }), - condition: { - equal: { "{{steps.1.success}}": false }, + branch2: { + steps: stepBuilder => + stepBuilder.serverLog( + { text: "Branch 1.2" }, + { id: branch2LogId } + ), + condition: { + equal: { [`{{ steps.${firstLogId}.success }}`]: false }, + }, }, - }, - }), + }), condition: { - equal: { "{{steps.1.success}}": true }, + equal: { [`{{ steps.${firstLogId}.success }}`]: true }, }, }, topLevelBranch2: { - steps: stepBuilder => stepBuilder.serverLog({ text: "Branch 2" }), + steps: stepBuilder => + stepBuilder.serverLog({ text: "Branch 2" }, { id: branch2Id }), condition: { - equal: { "{{steps.1.success}}": false }, + equal: { [`{{ steps.${firstLogId}.success }}`]: false }, }, }, }) .run() + expect(results.steps[3].outputs.status).toContain("branch1 branch taken") expect(results.steps[4].outputs.message).toContain("Branch 1.1") }) diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index 2269f075b2..6af18cd27e 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -64,18 +64,18 @@ class BaseStepBuilder { stepId: TStep, stepSchema: Omit, inputs: AutomationStepInputs, - stepName?: string + opts?: { stepName?: string; id?: string } ): this { - const id = uuidv4() + const id = opts?.id || uuidv4() this.steps.push({ ...stepSchema, inputs: inputs as any, id, stepId, - name: stepName || stepSchema.name, + name: opts?.stepName || stepSchema.name, }) - if (stepName) { - this.stepNames[id] = stepName + if (opts?.stepName) { + this.stepNames[id] = opts.stepName } return this } @@ -95,7 +95,6 @@ class BaseStepBuilder { }) branchStepInputs.children![key] = stepBuilder.build() }) - const branchStep: AutomationStep = { ...definition, id: uuidv4(), @@ -106,80 +105,98 @@ class BaseStepBuilder { } // STEPS - createRow(inputs: CreateRowStepInputs, opts?: { stepName?: string }): this { + createRow( + inputs: CreateRowStepInputs, + opts?: { stepName?: string; id?: string } + ): this { return this.step( AutomationActionStepId.CREATE_ROW, BUILTIN_ACTION_DEFINITIONS.CREATE_ROW, inputs, - opts?.stepName + opts ) } - updateRow(inputs: UpdateRowStepInputs, opts?: { stepName?: string }): this { + updateRow( + inputs: UpdateRowStepInputs, + opts?: { stepName?: string; id?: string } + ): this { return this.step( AutomationActionStepId.UPDATE_ROW, BUILTIN_ACTION_DEFINITIONS.UPDATE_ROW, inputs, - opts?.stepName + opts ) } - deleteRow(inputs: DeleteRowStepInputs, opts?: { stepName?: string }): this { + deleteRow( + inputs: DeleteRowStepInputs, + opts?: { stepName?: string; id?: string } + ): this { return this.step( AutomationActionStepId.DELETE_ROW, BUILTIN_ACTION_DEFINITIONS.DELETE_ROW, inputs, - opts?.stepName + opts ) } sendSmtpEmail( inputs: SmtpEmailStepInputs, - opts?: { stepName?: string } + opts?: { stepName?: string; id?: string } ): this { return this.step( AutomationActionStepId.SEND_EMAIL_SMTP, BUILTIN_ACTION_DEFINITIONS.SEND_EMAIL_SMTP, inputs, - opts?.stepName + opts ) } executeQuery( inputs: ExecuteQueryStepInputs, - opts?: { stepName?: string } + opts?: { stepName?: string; id?: string } ): this { return this.step( AutomationActionStepId.EXECUTE_QUERY, BUILTIN_ACTION_DEFINITIONS.EXECUTE_QUERY, inputs, - opts?.stepName + opts ) } - queryRows(inputs: QueryRowsStepInputs, opts?: { stepName?: string }): this { + queryRows( + inputs: QueryRowsStepInputs, + opts?: { stepName?: string; id?: string } + ): this { return this.step( AutomationActionStepId.QUERY_ROWS, BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS, inputs, - opts?.stepName + opts ) } - loop(inputs: LoopStepInputs, opts?: { stepName?: string }): this { + loop( + inputs: LoopStepInputs, + opts?: { stepName?: string; id?: string } + ): this { return this.step( AutomationActionStepId.LOOP, BUILTIN_ACTION_DEFINITIONS.LOOP, inputs, - opts?.stepName + opts ) } - serverLog(input: ServerLogStepInputs, opts?: { stepName?: string }): this { + serverLog( + input: ServerLogStepInputs, + opts?: { stepName?: string; id?: string } + ): this { return this.step( AutomationActionStepId.SERVER_LOG, BUILTIN_ACTION_DEFINITIONS.SERVER_LOG, input, - opts?.stepName + opts ) } diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index 6488e604e9..44758b727b 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -15,6 +15,7 @@ export interface TriggerOutput { export interface AutomationContext extends AutomationResults { steps: any[] + stepsById?: Record stepsByName?: Record env?: Record trigger: any diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index e2a5a1c192..7788744ea2 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -91,6 +91,7 @@ class Orchestrator { // step zero is never used as the template string is zero indexed for customer facing this.context = { steps: [{}], + stepsById: {}, stepsByName: {}, trigger: triggerOutput, } @@ -457,6 +458,7 @@ class Orchestrator { inputs: steps[stepToLoopIndex].inputs, }) + this.context.stepsById![steps[stepToLoopIndex].id] = tempOutput const stepName = steps[stepToLoopIndex].name || steps[stepToLoopIndex].id this.context.stepsByName![stepName] = tempOutput this.context.steps[this.context.steps.length] = tempOutput @@ -517,7 +519,10 @@ class Orchestrator { Object.entries(filter).forEach(([_, value]) => { Object.entries(value).forEach(([field, _]) => { const updatedField = field.replace("{{", "{{ literal ") - const fromContext = processStringSync(updatedField, this.context) + const fromContext = processStringSync( + updatedField, + this.processContext(this.context) + ) toFilter[field] = fromContext }) }) @@ -563,9 +568,9 @@ class Orchestrator { } const stepFn = await this.getStepFunctionality(step.stepId) - let inputs = await this.addContextAndProcess( + let inputs = await processObject( originalStepInput, - this.context + this.processContext(this.context) ) inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs) @@ -594,16 +599,16 @@ class Orchestrator { return null } - private async addContextAndProcess(inputs: any, context: any) { + private processContext(context: AutomationContext) { const processContext = { ...context, steps: { ...context.steps, + ...context.stepsById, ...context.stepsByName, }, } - - return processObject(inputs, processContext) + return processContext } private handleStepOutput( @@ -623,6 +628,7 @@ class Orchestrator { } else { this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs) this.context.steps[this.context.steps.length] = outputs + this.context.stepsById![step.id] = outputs const stepName = step.name || step.id this.context.stepsByName![stepName] = outputs } From e281250569a9bcd67d9d2e998959682018097b3e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 30 Sep 2024 14:00:12 +0100 Subject: [PATCH 07/85] ai cron helper E2E --- packages/builder/package.json | 1 + .../SetupPanel/AutomationBlockSetup.svelte | 2 +- .../automation/SetupPanel/CronBuilder.svelte | 179 ++++++++--- .../SetupPanel/test/CronBuilder.spec.js | 0 packages/frontend-core/src/api/ai.js | 11 + packages/frontend-core/src/api/index.js | 2 + packages/server/src/api/routes/index.ts | 2 + packages/shared-core/src/helpers/cron.ts | 14 + yarn.lock | 295 ++++++++++-------- 9 files changed, 339 insertions(+), 167 deletions(-) create mode 100644 packages/builder/src/components/automation/SetupPanel/test/CronBuilder.spec.js create mode 100644 packages/frontend-core/src/api/ai.js diff --git a/packages/builder/package.json b/packages/builder/package.json index f9e6becbab..aec0b509f0 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -67,6 +67,7 @@ "@spectrum-css/vars": "^3.0.1", "@zerodevx/svelte-json-view": "^1.0.7", "codemirror": "^5.65.16", + "cron-parser": "^4.9.0", "dayjs": "^1.10.8", "downloadjs": "1.4.7", "fast-json-patch": "^3.1.1", diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index aceb980786..7f8a68bf37 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -1048,7 +1048,7 @@ {:else if value.customType === "cron"} onChange({ [key]: e.detail })} - value={inputData[key]} + cronExpression={inputData[key]} /> {:else if value.customType === "automationFields"} - import { Button, Select, Input, Label } from "@budibase/bbui" + import { Button, Select, Icon, InlineAlert, Input, Label, Layout, Popover } from "@budibase/bbui" import { onMount, createEventDispatcher } from "svelte" import { flags } from "stores/builder" + import { licensing } from "stores/portal" + import { API } from "api" + import { helpers, REBOOT_CRON } from "@budibase/shared-core" const dispatch = createEventDispatcher() - export let value + export let cronExpression + let error + let nextExecutions + // AI prompt + let aiCronPrompt = "" + let loadingAICronExpression = false + + $: aiEnabled = $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled $: { - const exists = CRON_EXPRESSIONS.some(cron => cron.value === value) - const customIndex = CRON_EXPRESSIONS.findIndex( - cron => cron.label === "Custom" - ) - - if (!exists && customIndex === -1) { - CRON_EXPRESSIONS[0] = { label: "Custom", value: value } - } else if (exists && customIndex !== -1) { - CRON_EXPRESSIONS.splice(customIndex, 1) + if (cronExpression) { + try { + nextExecutions = helpers.cron.getNextExecutionDates(cronExpression).join("\n") + } catch (err) { + nextExecutions = null + } } } const onChange = e => { - if (value !== REBOOT_CRON) { + if (e.detail !== REBOOT_CRON) { error = helpers.cron.validate(e.detail).err } - if (e.detail === value || error) { + if (e.detail === cronExpression || error) { return } - value = e.detail + cronExpression = e.detail dispatch("change", e.detail) } + const updatePreset = e => { + aiCronPrompt = "" + onChange(e) + } + + const updateCronExpression = e => { + aiCronPrompt = "" + cronExpression = null + onChange(e) + } + let touched = false - let presets = false const CRON_EXPRESSIONS = [ { @@ -64,45 +81,129 @@ }) } }) + + async function generateAICronExpression() { + loadingAICronExpression = true + // make the API call to generate the cron expression + const response = await API.generateCronExpression({ prompt: aiCronPrompt }) + // return it and set it in the field + cronExpression = response.message + dispatch("change", response.message) + loadingAICronExpression = false + } -
+ + + {#if aiCronPrompt} +
+ +
+ {/if} +
+ {/if} (touched = true)} updateOnChange={false} /> - {#if touched && !value} + {#if touched && !cronExpression} {/if} -
- - {#if presets} - =0.1.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" - integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== - sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -20031,33 +20045,13 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: - version "5.7.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" - integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== - -semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: +"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" -semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: - version "6.3.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" - integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== - -semver@^7.5.4, semver@^7.6.0, semver@^7.6.3: - version "7.6.3" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" - integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== - -semver@~2.3.1: - version "2.3.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52" - integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA== - seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -21620,7 +21614,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: +tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -21630,14 +21624,6 @@ touch@^3.1.0: universalify "^0.2.0" url-parse "^1.5.3" -tough-cookie@~2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" - integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== - dependencies: - psl "^1.1.28" - punycode "^2.1.1" - tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -22166,6 +22152,14 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== +unset-value@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" + integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== + dependencies: + has-value "^2.0.2" + isobject "^4.0.0" + untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -22940,33 +22934,10 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x: - version "0.1.14" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" - integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== - dependencies: - sax ">=0.1.1" - -xml2js@0.4.19: - version "0.4.19" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" - integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== - dependencies: - sax ">=0.6.0" - xmlbuilder "~9.0.1" - -xml2js@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" - integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== - dependencies: - sax ">=0.6.0" - xmlbuilder "~11.0.0" - -xml2js@^0.4.19, xml2js@^0.4.5: - version "0.4.23" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" - integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== +xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5: + version "0.6.2" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" + integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" @@ -22976,11 +22947,6 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== -xmlbuilder@~9.0.1: - version "9.0.7" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" - integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== - xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" From 6e660151bdef6cd5c9130e09a695bc47b80d4b0c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 30 Sep 2024 18:06:47 +0100 Subject: [PATCH 18/85] backport of V3 backend changes for search filters on view, giving this the correct type to support conditionals. --- .../server/src/api/controllers/row/views.ts | 48 ++-- .../src/api/controllers/view/viewsV2.ts | 4 +- packages/shared-core/src/filters.ts | 208 +++++++++++++++++- packages/shared-core/src/utils.ts | 110 ++++++++- packages/types/src/api/web/app/view.ts | 5 +- packages/types/src/api/web/searchFilter.ts | 16 +- packages/types/src/documents/app/view.ts | 8 +- packages/types/src/sdk/search.ts | 5 + 8 files changed, 372 insertions(+), 32 deletions(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 68958da8e7..398121f49b 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -5,6 +5,9 @@ import { SearchViewRowRequest, SearchFilterKey, LogicalOperator, + RequiredKeys, + RowSearchParams, + LegacyFilter, } from "@budibase/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../../sdk" @@ -17,7 +20,7 @@ export async function searchView( ) { const { viewId } = ctx.params - const view = await sdk.views.get(viewId) + const view: ViewV2 = await sdk.views.get(viewId) if (!view) { ctx.throw(404, `View ${viewId} not found`) } @@ -25,23 +28,35 @@ export async function searchView( ctx.throw(400, `This method only supports viewsV2`) } + const viewFields = Object.entries(view.schema || {}) + .filter(([_, value]) => value.visible) + .map(([key]) => key) const { body } = ctx.request + const sqsEnabled = await features.flags.isEnabled("SQS") + const supportsLogicalOperators = isExternalTableID(view.tableId) || sqsEnabled + // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let query = dataFilters.buildQuery(view.query || []) + let query = dataFilters.buildQueryLegacy(view.query) + + delete query?.onEmptyFilter + if (body.query) { // Delete extraneous search params that cannot be overridden delete body.query.onEmptyFilter - if ( - !isExternalTableID(view.tableId) && - !(await features.flags.isEnabled("SQS")) - ) { + if (!supportsLogicalOperators) { + // In the unlikely event that a Grouped Filter is in a non-SQS environment + // It needs to be ignored entirely + let queryFilters: LegacyFilter[] = Array.isArray(view.query) + ? view.query + : [] + // Extract existing fields const existingFields = - view.query + queryFilters ?.filter(filter => filter.field) .map(filter => db.removeKeyNumbering(filter.field)) || [] @@ -49,15 +64,16 @@ export async function searchView( Object.keys(body.query).forEach(key => { const operator = key as Exclude Object.keys(body.query[operator] || {}).forEach(field => { - if (!existingFields.includes(db.removeKeyNumbering(field))) { + if (query && !existingFields.includes(db.removeKeyNumbering(field))) { query[operator]![field] = body.query[operator]![field] } }) }) } else { + const conditions = query ? [query] : [] query = { $and: { - conditions: [query, body.query], + conditions: [...conditions, body.query], }, } } @@ -65,25 +81,29 @@ export async function searchView( await context.ensureSnippetContext(true) - const enrichedQuery = await enrichSearchContext(query, { + const enrichedQuery = await enrichSearchContext(query || {}, { user: sdk.users.getUserContextBindings(ctx.user), }) - const result = await sdk.rows.search({ - viewId: view.id, + const searchOptions: RequiredKeys & + RequiredKeys< + Pick + > = { tableId: view.tableId, + viewId: view.id, query: enrichedQuery, + fields: viewFields, ...getSortOptions(body, view), limit: body.limit, bookmark: body.bookmark, paginate: body.paginate, countRows: body.countRows, - }) + } + const result = await sdk.rows.search(searchOptions) result.rows.forEach(r => (r._viewId = view.id)) ctx.body = result } - function getSortOptions(request: SearchViewRowRequest, view: ViewV2) { if (request.sort) { return { diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 7f6f638541..3df7172de2 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -99,7 +99,7 @@ export async function create(ctx: Ctx) { const schema = await parseSchema(view) - const parsedView: Omit, "id" | "version"> = { + const parsedView: Omit, "id" | "version" | "queryUI"> = { name: view.name, tableId: view.tableId, query: view.query, @@ -132,7 +132,7 @@ export async function update(ctx: Ctx) { const { tableId } = view const schema = await parseSchema(view) - const parsedView: RequiredKeys = { + const parsedView: RequiredKeys> = { id: view.id, name: view.name, version: view.version, diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index ef0500b01a..18ce4b6ed7 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -3,7 +3,7 @@ import { BBReferenceFieldSubType, FieldType, FormulaType, - SearchFilter, + LegacyFilter, SearchFilters, SearchQueryFields, ArrayOperator, @@ -19,9 +19,12 @@ import { RangeOperator, LogicalOperator, isLogicalSearchOperator, + SearchFilterGroup, + FilterGroupLogicalOperator, } from "@budibase/types" import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" +import { processSearchFilters } from "./utils" import { deepGet, schema } from "./helpers" import { isPlainObject, isEmpty } from "lodash" import { decodeNonAscii } from "./helpers/schema" @@ -124,7 +127,7 @@ export function recurseLogicalOperators( fn: (f: SearchFilters) => SearchFilters ) { for (const logical of LOGICAL_OPERATORS) { - if (filters?.[logical]) { + if (filters[logical]) { filters[logical]!.conditions = filters[logical]!.conditions.map( condition => fn(condition) ) @@ -304,10 +307,143 @@ export class ColumnSplitter { } /** - * Builds a JSON query from the filter structure generated in the builder + * Builds a JSON query from the filter a SearchFilter definition * @param filter the builder filter structure */ -export const buildQuery = (filter: SearchFilter[]) => { + +const buildCondition = (expression: LegacyFilter) => { + // Filter body + let query: SearchFilters = { + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + notEqual: {}, + empty: {}, + notEmpty: {}, + contains: {}, + notContains: {}, + oneOf: {}, + containsAny: {}, + } + let { operator, field, type, value, externalType, onEmptyFilter } = expression + + if (!operator || !field) { + return + } + + const queryOperator = operator as SearchFilterOperator + const isHbs = + typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0 + // Parse all values into correct types + if (operator === "allOr") { + query.allOr = true + return + } + if (onEmptyFilter) { + query.onEmptyFilter = onEmptyFilter + return + } + + // Default the value for noValue fields to ensure they are correctly added + // to the final query + if (queryOperator === "empty" || queryOperator === "notEmpty") { + value = null + } + + if ( + type === "datetime" && + !isHbs && + queryOperator !== "empty" && + queryOperator !== "notEmpty" + ) { + // Ensure date value is a valid date and parse into correct format + if (!value) { + return + } + try { + value = new Date(value).toISOString() + } catch (error) { + return + } + } + if (type === "number" && typeof value === "string" && !isHbs) { + if (queryOperator === "oneOf") { + value = value.split(",").map(item => parseFloat(item)) + } else { + value = parseFloat(value) + } + } + if (type === "boolean") { + value = `${value}`?.toLowerCase() === "true" + } + if ( + ["contains", "notContains", "containsAny"].includes( + operator.toLocaleString() + ) && + type === "array" && + typeof value === "string" + ) { + value = value.split(",") + } + if (operator.toLocaleString().startsWith("range") && query.range) { + const minint = + SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap] + ?.min || Number.MIN_SAFE_INTEGER + const maxint = + SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap] + ?.max || Number.MAX_SAFE_INTEGER + if (!query.range[field]) { + query.range[field] = { + low: type === "number" ? minint : "0000-00-00T00:00:00.000Z", + high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z", + } + } + if (operator === "rangeLow" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + low: value, + } + } else if (operator === "rangeHigh" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + high: value, + } + } + } else if (isLogicalSearchOperator(queryOperator)) { + // TODO + } else if (query[queryOperator] && operator !== "onEmptyFilter") { + if (type === "boolean") { + // Transform boolean filters to cope with null. + // "equals false" needs to be "not equals true" + // "not equals false" needs to be "equals true" + if (queryOperator === "equal" && value === false) { + query.notEqual = query.notEqual || {} + query.notEqual[field] = true + } else if (queryOperator === "notEqual" && value === false) { + query.equal = query.equal || {} + query.equal[field] = true + } else { + query[queryOperator] ??= {} + query[queryOperator]![field] = value + } + } else { + query[queryOperator] ??= {} + query[queryOperator]![field] = value + } + } + + return query +} + +export const buildQueryLegacy = ( + filter?: LegacyFilter[] | SearchFilters +): SearchFilters | undefined => { + // this is of type SearchFilters or is undefined + if (!Array.isArray(filter)) { + return filter + } + let query: SearchFilters = { string: {}, fuzzy: {}, @@ -368,13 +504,15 @@ export const buildQuery = (filter: SearchFilter[]) => { value = `${value}`?.toLowerCase() === "true" } if ( - ["contains", "notContains", "containsAny"].includes(operator) && + ["contains", "notContains", "containsAny"].includes( + operator.toLocaleString() + ) && type === "array" && typeof value === "string" ) { value = value.split(",") } - if (operator.startsWith("range") && query.range) { + if (operator.toLocaleString().startsWith("range") && query.range) { const minint = SqlNumberTypeRangeMap[ externalType as keyof typeof SqlNumberTypeRangeMap @@ -401,7 +539,7 @@ export const buildQuery = (filter: SearchFilter[]) => { } } } else if (isLogicalSearchOperator(queryOperator)) { - // TODO + // ignore } else if (query[queryOperator] && operator !== "onEmptyFilter") { if (type === "boolean") { // Transform boolean filters to cope with null. @@ -423,14 +561,68 @@ export const buildQuery = (filter: SearchFilter[]) => { } } }) - return query } +/** + * Converts a **SearchFilterGroup** filter definition into a grouped + * search query of type **SearchFilters** + * + * Legacy support remains for the old **SearchFilter[]** format. + * These will be migrated to an appropriate **SearchFilters** object, if encountered + * + * @param filter + * + * @returns {SearchFilters} + */ + +export const buildQuery = ( + filter?: SearchFilterGroup | LegacyFilter[] +): SearchFilters | undefined => { + const parsedFilter: SearchFilterGroup | undefined = + processSearchFilters(filter) + + if (!parsedFilter) { + return + } + + const operatorMap: { [key in FilterGroupLogicalOperator]: LogicalOperator } = + { + [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND, + [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR, + } + + const globalOnEmpty = parsedFilter.onEmptyFilter + ? parsedFilter.onEmptyFilter + : null + + const globalOperator: LogicalOperator = + operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator] + + const coreRequest: SearchFilters = { + ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), + [globalOperator]: { + conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { + return { + [operatorMap[group.logicalOperator]]: { + conditions: group.filters + ?.map(x => buildCondition(x)) + .filter(filter => filter), + }, + } + }), + }, + } + return coreRequest +} + // The frontend can send single values for array fields sometimes, so to handle // this we convert them to arrays at the controller level so that nothing below // this has to worry about the non-array values. export function fixupFilterArrays(filters: SearchFilters) { + if (!filters) { + return filters + } for (const searchField of Object.values(ArrayOperator)) { const field = filters[searchField] if (field == null || !isPlainObject(field)) { diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 81fab659c6..b441791751 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -1,5 +1,20 @@ -import { ArrayOperator, BasicOperator, SearchFilters } from "@budibase/types" +import { + LegacyFilter, + SearchFilterGroup, + FilterGroupLogicalOperator, + SearchFilters, + BasicOperator, + ArrayOperator, +} from "@budibase/types" import * as Constants from "./constants" +import { removeKeyNumbering } from "./filters" + +// an array of keys from filter type to properties that are in the type +// this can then be converted using .fromEntries to an object +type WhitelistedFilters = [ + keyof LegacyFilter, + LegacyFilter[keyof LegacyFilter] +][] export function unreachable( value: never, @@ -104,3 +119,96 @@ export function isSupportedUserSearch(query: SearchFilters) { } return true } + +/** + * Processes the filter config. Filters are migrated from + * SearchFilter[] to SearchFilterGroup + * + * If filters is not an array, the migration is skipped + * + * @param {LegacyFilter[] | SearchFilterGroup} filters + */ +export const processSearchFilters = ( + filters: LegacyFilter[] | SearchFilterGroup | undefined +): SearchFilterGroup | undefined => { + if (!filters) { + return + } + + // Base search config. + const defaultCfg: SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator.ALL, + groups: [], + } + + const filterWhitelistKeys = [ + "field", + "operator", + "value", + "type", + "externalType", + "valueType", + "noValue", + "formulaType", + ] + + if (Array.isArray(filters)) { + let baseGroup: SearchFilterGroup = { + filters: [], + logicalOperator: FilterGroupLogicalOperator.ALL, + } + + return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => { + // Sort the properties for easier debugging + const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[]) + .sort((a, b) => { + return a.localeCompare(b) + }) + .filter(key => key in filter) + + if (filterPropertyKeys.length == 1) { + const key = filterPropertyKeys[0], + value = filter[key] + // Global + if (key === "onEmptyFilter") { + // unset otherwise + acc.onEmptyFilter = value + } else if (key === "operator" && value === "allOr") { + // Group 1 logical operator + baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY + } + + return acc + } + + const whiteListedFilterSettings: WhitelistedFilters = + filterPropertyKeys.reduce((acc: WhitelistedFilters, key) => { + const value = filter[key] + if (filterWhitelistKeys.includes(key)) { + if (key === "field") { + acc.push([key, removeKeyNumbering(value)]) + } else { + acc.push([key, value]) + } + } + return acc + }, []) + + const migratedFilter: LegacyFilter = Object.fromEntries( + whiteListedFilterSettings + ) as LegacyFilter + + baseGroup.filters!.push(migratedFilter) + + if (!acc.groups || !acc.groups.length) { + // init the base group + acc.groups = [baseGroup] + } + + return acc + }, defaultCfg) + } else if (!filters?.groups) { + return + } + return filters +} diff --git a/packages/types/src/api/web/app/view.ts b/packages/types/src/api/web/app/view.ts index a6be5e2986..a99f2938ab 100644 --- a/packages/types/src/api/web/app/view.ts +++ b/packages/types/src/api/web/app/view.ts @@ -9,6 +9,7 @@ export interface ViewResponseEnriched { data: ViewV2Enriched } -export interface CreateViewRequest extends Omit {} +export interface CreateViewRequest + extends Omit {} -export interface UpdateViewRequest extends ViewV2 {} +export interface UpdateViewRequest extends Omit {} diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts index 5223204a7f..23c599027e 100644 --- a/packages/types/src/api/web/searchFilter.ts +++ b/packages/types/src/api/web/searchFilter.ts @@ -1,7 +1,11 @@ import { FieldType } from "../../documents" -import { EmptyFilterOption, SearchFilters } from "../../sdk" +import { + EmptyFilterOption, + FilterGroupLogicalOperator, + SearchFilters, +} from "../../sdk" -export type SearchFilter = { +export type LegacyFilter = { operator: keyof SearchFilters | "rangeLow" | "rangeHigh" onEmptyFilter?: EmptyFilterOption field: string @@ -9,3 +13,11 @@ export type SearchFilter = { value: any externalType?: string } + +// this is a type purely used by the UI +export type SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator + onEmptyFilter?: EmptyFilterOption + groups?: SearchFilterGroup[] + filters?: LegacyFilter[] +} diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index a957564039..87667a71e0 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -1,7 +1,7 @@ -import { SearchFilter, SortOrder, SortType } from "../../api" +import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api" import { UIFieldMetadata } from "./table" import { Document } from "../document" -import { DBView } from "../../sdk" +import { DBView, SearchFilters } from "../../sdk" export type ViewTemplateOpts = { field: string @@ -65,7 +65,9 @@ export interface ViewV2 { name: string primaryDisplay?: string tableId: string - query?: SearchFilter[] + query?: LegacyFilter[] | SearchFilters + // duplicate to store UI information about filters + queryUI?: SearchFilterGroup sort?: { field: string order?: SortOrder diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 647a9e7d00..d41bb0fb99 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -191,6 +191,11 @@ export enum EmptyFilterOption { RETURN_NONE = "none", } +export enum FilterGroupLogicalOperator { + ALL = "all", + ANY = "any", +} + export enum SqlClient { MS_SQL = "mssql", POSTGRES = "pg", From f31c7c3487d2b2cd91454f1e89a9be12d2ddae7e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 1 Oct 2024 10:55:25 +0200 Subject: [PATCH 19/85] Add test --- .../src/api/routes/tests/viewV2.spec.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index aab846e704..09273abdce 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -1738,6 +1738,40 @@ describe.each([ }) }) + it("views filters are respected even if the column is hidden", async () => { + await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + const two = await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + query: [ + { + operator: BasicOperator.EQUAL, + field: "two", + value: "bar2", + }, + ], + schema: { + id: { visible: true }, + one: { visible: false }, + two: { visible: false }, + }, + }) + + const response = await config.api.viewV2.search(view.id) + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual([ + expect.objectContaining({ _id: two._id }), + ]) + }) + it("views without data can be returned", async () => { const response = await config.api.viewV2.search(view.id) expect(response.rows).toHaveLength(0) From 975e348de5b9bed62030ed73b7db00cef700df0d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 10:25:15 +0100 Subject: [PATCH 20/85] Check options.fields are in the table. --- packages/server/src/sdk/app/rows/search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 809bd73d1f..eb04e9fe62 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -89,7 +89,7 @@ export async function search( if (options.query) { const visibleFields = ( options.fields || Object.keys(table.schema) - ).filter(field => table.schema[field].visible !== false) + ).filter(field => table.schema[field]?.visible !== false) const queryableFields = await getQueryableFields(table, visibleFields) options.query = removeInvalidFilters(options.query, queryableFields) From 4b65ce4f8b51e55c9dc33de08337c8b8e0cbd7dd Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 1 Oct 2024 09:31:57 +0000 Subject: [PATCH 21/85] Bump version to 2.32.10 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 5a279b3e44..092e9a133e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.9", + "version": "2.32.10", "npmClient": "yarn", "packages": [ "packages/*", From 119767a30e970220a64f7aa8a43c25cc349c56b0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 1 Oct 2024 12:20:18 +0200 Subject: [PATCH 22/85] Cleanup --- packages/server/src/api/controllers/row/views.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index b8d01424f2..622688deb6 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -14,7 +14,7 @@ export async function searchView( ) { const { viewId } = ctx.params - const view: ViewV2 = await sdk.views.get(viewId) + const view = await sdk.views.get(viewId) if (!view) { ctx.throw(404, `View ${viewId} not found`) } From 522941abf004cae1e04f650a0128df73aca2814a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 11:31:41 +0100 Subject: [PATCH 23/85] PR comments. --- packages/shared-core/src/filters.ts | 9 +-------- packages/shared-core/src/utils.ts | 19 +++++++++---------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 18ce4b6ed7..b10375acb0 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -163,9 +163,6 @@ export function recurseSearchFilters( * https://github.com/Budibase/budibase/issues/10118 */ export const cleanupQuery = (query: SearchFilters) => { - if (!query) { - return query - } for (let filterField of NoEmptyFilterStrings) { if (!query[filterField]) { continue @@ -599,7 +596,7 @@ export const buildQuery = ( const globalOperator: LogicalOperator = operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator] - const coreRequest: SearchFilters = { + return { ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), [globalOperator]: { conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { @@ -613,16 +610,12 @@ export const buildQuery = ( }), }, } - return coreRequest } // The frontend can send single values for array fields sometimes, so to handle // this we convert them to arrays at the controller level so that nothing below // this has to worry about the non-array values. export function fixupFilterArrays(filters: SearchFilters) { - if (!filters) { - return filters - } for (const searchField of Object.values(ArrayOperator)) { const field = filters[searchField] if (field == null || !isPlainObject(field)) { diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index b441791751..14b3c84425 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -11,10 +11,7 @@ import { removeKeyNumbering } from "./filters" // an array of keys from filter type to properties that are in the type // this can then be converted using .fromEntries to an object -type WhitelistedFilters = [ - keyof LegacyFilter, - LegacyFilter[keyof LegacyFilter] -][] +type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][] export function unreachable( value: never, @@ -141,7 +138,7 @@ export const processSearchFilters = ( groups: [], } - const filterWhitelistKeys = [ + const filterAllowedKeys = [ "field", "operator", "value", @@ -181,10 +178,10 @@ export const processSearchFilters = ( return acc } - const whiteListedFilterSettings: WhitelistedFilters = - filterPropertyKeys.reduce((acc: WhitelistedFilters, key) => { + const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce( + (acc: AllowedFilters, key) => { const value = filter[key] - if (filterWhitelistKeys.includes(key)) { + if (filterAllowedKeys.includes(key)) { if (key === "field") { acc.push([key, removeKeyNumbering(value)]) } else { @@ -192,10 +189,12 @@ export const processSearchFilters = ( } } return acc - }, []) + }, + [] + ) const migratedFilter: LegacyFilter = Object.fromEntries( - whiteListedFilterSettings + allowedFilterSettings ) as LegacyFilter baseGroup.filters!.push(migratedFilter) From 19407d5e37bc4a2e937c4e153a8bdf74fd46a83f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 11:38:02 +0100 Subject: [PATCH 24/85] Check filters have been provided. --- packages/server/src/sdk/app/rows/search/utils.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 1dba420a28..90303a6ca7 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) { } return dataFilters.recurseLogicalOperators(filters, checkFilters) } - options.query = checkFilters(options.query) + if (options.query) { + options.query = checkFilters(options.query) + } return options } From d7873c5c6e5cd6908f4fc75bc44a53b9af6436aa Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 11:42:16 +0100 Subject: [PATCH 25/85] Test fix. --- .../server/src/api/routes/tests/search.spec.ts | 14 +++++++------- packages/server/src/sdk/app/rows/search.ts | 5 +++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 1ec5ca792a..092a851e14 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -45,14 +45,14 @@ import { generateRowIdField } from "../../../integrations/utils" import { cloneDeep } from "lodash/fp" describe.each([ - ["in-memory", undefined], - ["lucene", undefined], + // ["in-memory", undefined], + // ["lucene", undefined], ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 87129fdbc8..7e73a51889 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -81,12 +81,13 @@ export async function search( options.query = {} } + // need to make sure filters in correct shape before checking for view + options = searchInputMapping(table, options) + if (options.viewId) { // Delete extraneous search params that cannot be overridden delete options.query.onEmptyFilter - options = searchInputMapping(table, options) - const view = source as ViewV2 // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as From 4d33106b450781573083fd6359f6c1a3297e374e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 11:42:44 +0100 Subject: [PATCH 26/85] Undo commenting out other DBs. --- .../server/src/api/routes/tests/search.spec.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 092a851e14..1ec5ca792a 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -45,14 +45,14 @@ import { generateRowIdField } from "../../../integrations/utils" import { cloneDeep } from "lodash/fp" describe.each([ - // ["in-memory", undefined], - // ["lucene", undefined], + ["in-memory", undefined], + ["lucene", undefined], ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" From 987a24fabc85d76bb3332e364d877a4349083692 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 11:48:14 +0100 Subject: [PATCH 27/85] wip --- packages/backend-core/src/sql/sql.ts | 87 ++++++++++++------- .../src/api/routes/tests/viewV2.spec.ts | 39 +++++++-- 2 files changed, 88 insertions(+), 38 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b7bf5bc102..54ca5a0135 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -71,18 +71,6 @@ function prioritisedArraySort(toSort: string[], priorities: string[]) { }) } -function getTableName(table?: Table): string | undefined { - // SQS uses the table ID rather than the table name - if ( - table?.sourceType === TableSourceType.INTERNAL || - table?.sourceId === INTERNAL_TABLE_SOURCE_ID - ) { - return table?._id - } else { - return table?.name - } -} - function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] { if (Array.isArray(query)) { return query.map((q: SqlQuery) => convertBooleans(q) as SqlQuery) @@ -180,15 +168,13 @@ class InternalBuilder { } private generateSelectStatement(): (string | Knex.Raw)[] | "*" { - const { meta, endpoint, resource, tableAliases } = this.query + const { meta, endpoint, resource } = this.query if (!resource || !resource.fields || resource.fields.length === 0) { return "*" } - const alias = tableAliases?.[endpoint.entityId] - ? tableAliases?.[endpoint.entityId] - : endpoint.entityId + const alias = this.getTableName(endpoint.entityId) const schema = meta.table.schema if (!this.isFullSelectStatementRequired()) { return [this.knex.raw(`${this.quote(alias)}.*`)] @@ -813,17 +799,39 @@ class InternalBuilder { return query } + getTableName(t?: Table | string): string { + let table: Table + if (typeof t === "string") { + if (!this.query.meta.tables?.[t]) { + throw new Error(`Table ${t} not found`) + } + table = this.query.meta.tables[t] + } else if (t) { + table = t + } else { + table = this.table + } + + let name = table.name + if ( + (table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID) && + table._id + ) { + // SQS uses the table ID rather than the table name + name = table._id + } + const aliases = this.query.tableAliases || {} + return aliases[name] ? aliases[name] : name + } + addDistinctCount(query: Knex.QueryBuilder): Knex.QueryBuilder { - const primary = this.table.primary - const aliases = this.query.tableAliases - const aliased = - this.table.name && aliases?.[this.table.name] - ? aliases[this.table.name] - : this.table.name - if (!primary) { + if (!this.table.primary) { throw new Error("SQL counting requires primary key to be supplied") } - return query.countDistinct(`${aliased}.${primary[0]} as total`) + return query.countDistinct( + `${this.getTableName(this.table)}.${this.table.primary[0]} as total` + ) } addAggregations( @@ -831,8 +839,9 @@ class InternalBuilder { aggregations: Aggregation[] ): Knex.QueryBuilder { const fields = this.query.resource?.fields || [] + const tableName = this.getTableName() if (fields.length > 0) { - query = query.groupBy(fields.map(field => `${this.table.name}.${field}`)) + query = query.groupBy(fields.map(field => `${tableName}.${field}`)) } for (const aggregation of aggregations) { const op = aggregation.calculationType @@ -861,10 +870,7 @@ class InternalBuilder { addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder { let { sort, resource } = this.query const primaryKey = this.table.primary - const tableName = getTableName(this.table) - const aliases = this.query.tableAliases - const aliased = - tableName && aliases?.[tableName] ? aliases[tableName] : this.table?.name + const aliased = this.getTableName() if (!Array.isArray(primaryKey)) { throw new Error("Sorting requires primary key to be specified for table") } @@ -1508,18 +1514,35 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { return results.length ? results : [{ [operation.toLowerCase()]: true }] } + private getTableName( + table: Table, + aliases?: Record + ): string | undefined { + let name = table.name + if ( + table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID + ) { + if (!table._id) { + return + } + // SQS uses the table ID rather than the table name + name = table._id + } + return aliases?.[name] ? aliases[name] : name + } + convertJsonStringColumns>( table: Table, results: T[], aliases?: Record ): T[] { - const tableName = getTableName(table) + const tableName = this.getTableName(table, aliases) for (const [name, field] of Object.entries(table.schema)) { if (!this._isJsonColumn(field)) { continue } - const aliasedTableName = (tableName && aliases?.[tableName]) || tableName - const fullName = `${aliasedTableName}.${name}` + const fullName = `${tableName}.${name}` for (let row of results) { if (typeof row[fullName as keyof T] === "string") { row[fullName as keyof T] = JSON.parse(row[fullName]) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 09273abdce..0f4e6c961c 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -39,13 +39,13 @@ import { } from "@budibase/backend-core" describe.each([ - ["lucene", undefined], - ["sqs", undefined], + // ["lucene", undefined], + // ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -2458,6 +2458,33 @@ describe.each([ expect("_id" in row).toBe(false) } }) + + it.only("should be able to group by a basic field", async () => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + quantity: { + visible: true, + field: "quantity", + }, + "Total Price": { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + for (const row of response.rows) { + expect(row["quantity"]).toBeGreaterThan(0) + expect(row["Total Price"]).toBeGreaterThan(0) + } + }) }) }) From 84f7a477a1a39953ce11bc3aeedce5ec0e872a34 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 12:00:03 +0100 Subject: [PATCH 28/85] Fix the binding drawer for default values. --- .../DataTable/modals/CreateEditColumn.svelte | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 0130c39715..a1bd54715b 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -21,6 +21,7 @@ PROTECTED_EXTERNAL_COLUMNS, canHaveDefaultColumn, } from "@budibase/shared-core" + import { makePropSafe } from "@budibase/string-templates" import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" import { tables, datasources } from "stores/builder" @@ -46,6 +47,7 @@ import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte" import OptionsEditor from "./OptionsEditor.svelte" import { isEnabled } from "helpers/featureFlags" + import { getUserBindings } from "dataBinding" const AUTO_TYPE = FieldType.AUTO const FORMULA_TYPE = FieldType.FORMULA @@ -191,6 +193,19 @@ fieldId: makeFieldId(t.type, t.subtype), ...t, })) + $: defaultValueBindings = [ + { + type: "context", + runtimeBinding: `${makePropSafe("now")}`, + readableBinding: `Date`, + category: "Date", + icon: "Date", + display: { + name: "Server date", + }, + }, + ...getUserBindings(), + ] const fieldDefinitions = Object.values(FIELDS).reduce( // Storing the fields by complex field id @@ -781,9 +796,8 @@ setRequired(false) } }} - bindings={getBindings({ table })} + bindings={defaultValueBindings} allowJS - context={rowGoldenSample} />
{/if} From 97b70e1f5a0282f7dd997c78e91e57eb8fe24103 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 1 Oct 2024 13:21:04 +0200 Subject: [PATCH 29/85] Change tableid for source id --- .../src/api/routes/tests/search.spec.ts | 161 +++++++++--------- 1 file changed, 80 insertions(+), 81 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 1ec5ca792a..ac189464a3 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -64,7 +64,7 @@ describe.each([ let envCleanup: (() => void) | undefined let datasource: Datasource | undefined let client: Knex | undefined - let table: Table + let sourceId: string let rows: Row[] async function basicRelationshipTables(type: RelationshipType) { @@ -74,7 +74,7 @@ describe.each([ }, generator.guid().substring(0, 10) ) - table = await createTable( + sourceId = await createTable( { name: { name: "name", type: FieldType.STRING }, //@ts-ignore - API accepts this structure, will build out rest of definition @@ -83,7 +83,7 @@ describe.each([ relationshipType: type, name: "productCat", fieldName: "product", - tableId: relatedTable._id!, + tableId: relatedTable, constraints: { type: "array", }, @@ -92,8 +92,7 @@ describe.each([ generator.guid().substring(0, 10) ) return { - relatedTable: await config.api.table.get(relatedTable._id!), - table, + relatedTable: await config.api.table.get(relatedTable), } } @@ -137,17 +136,18 @@ describe.each([ }) async function createTable(schema: TableSchema, name?: string) { - return await config.api.table.save( + const table = await config.api.table.save( tableForDatasource(datasource, { schema, name }) ) + return table._id! } async function createRows(arr: Record[]) { // Shuffling to avoid false positives given a fixed order - await config.api.row.bulkImport(table._id!, { + await config.api.row.bulkImport(sourceId, { rows: _.shuffle(arr), }) - rows = await config.api.row.fetch(table._id!) + rows = await config.api.row.fetch(sourceId) } class SearchAssertion { @@ -332,7 +332,7 @@ describe.each([ } function expectSearch(query: Omit) { - return new SearchAssertion({ ...query, tableId: table._id! }) + return new SearchAssertion({ ...query, tableId: sourceId }) } function expectQuery(query: SearchFilters) { @@ -341,7 +341,7 @@ describe.each([ describe("boolean", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ isTrue: { name: "isTrue", type: FieldType.BOOLEAN }, }) await createRows([{ isTrue: true }, { isTrue: false }]) @@ -482,7 +482,7 @@ describe.each([ }) ) - table = await createTable({ + sourceId = await createTable({ name: { name: "name", type: FieldType.STRING }, appointment: { name: "appointment", type: FieldType.DATETIME }, single_user: { @@ -764,7 +764,7 @@ describe.each([ describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ name: { name: "name", type: FieldType.STRING }, }) await createRows([{ name: "foo" }, { name: "bar" }]) @@ -1055,7 +1055,7 @@ describe.each([ datasourceId: datasource!._id!, }) - table = resp.datasource.entities![tableName] + sourceId = resp.datasource.entities![tableName]._id! await createRows([{ name: "foo" }, { name: "bar" }]) }) @@ -1079,7 +1079,7 @@ describe.each([ describe("numbers", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ age: { name: "age", type: FieldType.NUMBER }, }) await createRows([{ age: 1 }, { age: 10 }]) @@ -1252,7 +1252,7 @@ describe.each([ const JAN_10TH = "2020-01-10T00:00:00.000Z" beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ dob: { name: "dob", type: FieldType.DATETIME }, }) @@ -1399,7 +1399,7 @@ describe.each([ const NULL_TIME__ID = `null_time__id` beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ timeid: { name: "timeid", type: FieldType.STRING }, time: { name: "time", type: FieldType.DATETIME, timeOnly: true }, }) @@ -1560,7 +1560,7 @@ describe.each([ describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ numbers: { name: "numbers", type: FieldType.ARRAY, @@ -1657,7 +1657,7 @@ describe.each([ let BIG = "9223372036854775807" beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ num: { name: "num", type: FieldType.BIGINT }, }) await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }]) @@ -1762,7 +1762,7 @@ describe.each([ isInternal && describe("auto", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ auto: { name: "auto", type: FieldType.AUTO, @@ -1912,21 +1912,18 @@ describe.each([ // be stable or pagination will break. We don't want the user to need // to specify an order for pagination to work. it("is stable without a sort specified", async () => { - let { rows: fullRowList } = await config.api.row.search( - table._id!, - { - tableId: table._id!, - query: {}, - } - ) + let { rows: fullRowList } = await config.api.row.search(sourceId, { + tableId: sourceId, + query: {}, + }) // repeat the search many times to check the first row is always the same let bookmark: string | number | undefined, hasNextPage: boolean | undefined = true, rowCount: number = 0 do { - const response = await config.api.row.search(table._id!, { - tableId: table._id!, + const response = await config.api.row.search(sourceId, { + tableId: sourceId, limit: 1, paginate: true, query: {}, @@ -1949,8 +1946,8 @@ describe.each([ // eslint-disable-next-line no-constant-condition while (true) { - const response = await config.api.row.search(table._id!, { - tableId: table._id!, + const response = await config.api.row.search(sourceId, { + tableId: sourceId, limit: 3, query: {}, bookmark, @@ -1973,7 +1970,7 @@ describe.each([ describe("field name 1:name", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ "1:name": { name: "1:name", type: FieldType.STRING }, }) await createRows([{ "1:name": "bar" }, { "1:name": "foo" }]) @@ -2007,14 +2004,14 @@ describe.each([ }, "array" ) - table = await createTable( + sourceId = await createTable( { relationship: { type: FieldType.LINK, relationshipType: RelationshipType.MANY_TO_ONE, name: "relationship", fieldName: "relate", - tableId: arrayTable._id!, + tableId: arrayTable, constraints: { type: "array", }, @@ -2030,17 +2027,17 @@ describe.each([ "main" ) const arrayRows = await Promise.all([ - config.api.row.save(arrayTable._id!, { + config.api.row.save(arrayTable, { name: "foo", array: ["option 1"], }), - config.api.row.save(arrayTable._id!, { + config.api.row.save(arrayTable, { name: "bar", array: ["option 2"], }), ]) await Promise.all([ - config.api.row.save(table._id!, { + config.api.row.save(sourceId, { relationship: [arrayRows[0]._id, arrayRows[1]._id], }), ]) @@ -2059,7 +2056,7 @@ describe.each([ user1 = await config.createUser({ _id: `us_${utils.newid()}` }) user2 = await config.createUser({ _id: `us_${utils.newid()}` }) - table = await createTable({ + sourceId = await createTable({ user: { name: "user", type: FieldType.BB_REFERENCE_SINGLE, @@ -2139,7 +2136,7 @@ describe.each([ user1 = await config.createUser({ _id: `us_${utils.newid()}` }) user2 = await config.createUser({ _id: `us_${utils.newid()}` }) - table = await createTable({ + sourceId = await createTable({ users: { name: "users", type: FieldType.BB_REFERENCE, @@ -2260,15 +2257,15 @@ describe.each([ ]) await Promise.all([ - config.api.row.save(table._id!, { + config.api.row.save(sourceId, { name: "foo", productCat: [productCatRows[0]._id], }), - config.api.row.save(table._id!, { + config.api.row.save(sourceId, { name: "bar", productCat: [productCatRows[1]._id], }), - config.api.row.save(table._id!, { + config.api.row.save(sourceId, { name: "baz", productCat: [], }), @@ -2304,7 +2301,7 @@ describe.each([ const { relatedTable } = await basicRelationshipTables( RelationshipType.MANY_TO_ONE ) - const mainRow = await config.api.row.save(table._id!, { + const mainRow = await config.api.row.save(sourceId, { name: "foo", }) for (let i = 0; i < 11; i++) { @@ -2329,7 +2326,7 @@ describe.each([ }) ;(isSqs || isLucene) && describe("relations to same table", () => { - let relatedTable: Table, relatedRows: Row[] + let relatedTable: string, relatedRows: Row[] beforeAll(async () => { relatedTable = await createTable( @@ -2338,36 +2335,36 @@ describe.each([ }, "productCategory" ) - table = await createTable({ + sourceId = await createTable({ name: { name: "name", type: FieldType.STRING }, related1: { type: FieldType.LINK, name: "related1", fieldName: "main1", - tableId: relatedTable._id!, + tableId: relatedTable, relationshipType: RelationshipType.MANY_TO_MANY, }, related2: { type: FieldType.LINK, name: "related2", fieldName: "main2", - tableId: relatedTable._id!, + tableId: relatedTable, relationshipType: RelationshipType.MANY_TO_MANY, }, }) relatedRows = await Promise.all([ - config.api.row.save(relatedTable._id!, { name: "foo" }), - config.api.row.save(relatedTable._id!, { name: "bar" }), - config.api.row.save(relatedTable._id!, { name: "baz" }), - config.api.row.save(relatedTable._id!, { name: "boo" }), + config.api.row.save(relatedTable, { name: "foo" }), + config.api.row.save(relatedTable, { name: "bar" }), + config.api.row.save(relatedTable, { name: "baz" }), + config.api.row.save(relatedTable, { name: "boo" }), ]) await Promise.all([ - config.api.row.save(table._id!, { + config.api.row.save(sourceId, { name: "test", related1: [relatedRows[0]._id!], related2: [relatedRows[1]._id!], }), - config.api.row.save(table._id!, { + config.api.row.save(sourceId, { name: "test2", related1: [relatedRows[2]._id!], related2: [relatedRows[3]._id!], @@ -2430,7 +2427,7 @@ describe.each([ isInternal && describe("no column error backwards compat", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ name: { name: "name", type: FieldType.STRING, @@ -2453,7 +2450,7 @@ describe.each([ !isLucene && describe("row counting", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ name: { name: "name", type: FieldType.STRING, @@ -2488,7 +2485,7 @@ describe.each([ describe("Invalid column definitions", () => { beforeAll(async () => { // need to create an invalid table - means ignoring typescript - table = await createTable({ + sourceId = await createTable({ // @ts-ignore invalid: { type: FieldType.STRING, @@ -2518,7 +2515,7 @@ describe.each([ "special (%s) case", column => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ [column]: { name: column, type: FieldType.STRING, @@ -2543,8 +2540,8 @@ describe.each([ describe("sample data", () => { beforeAll(async () => { await config.api.application.addSampleData(config.appId!) - table = DEFAULT_EMPLOYEE_TABLE_SCHEMA - rows = await config.api.row.fetch(table._id!) + sourceId = DEFAULT_EMPLOYEE_TABLE_SCHEMA._id! + rows = await config.api.row.fetch(sourceId) }) it("should be able to search sample data", async () => { @@ -2567,7 +2564,7 @@ describe.each([ const earlyDate = "2024-07-03T10:00:00.000Z", laterDate = "2024-07-03T11:00:00.000Z" beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ date: { name: "date", type: FieldType.DATETIME, @@ -2610,7 +2607,7 @@ describe.each([ "ชื่อผู้ใช้", // Thai for "username" ])("non-ascii column name: %s", name => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ [name]: { name, type: FieldType.STRING, @@ -2637,7 +2634,7 @@ describe.each([ isInternal && describe("space at end of column name", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ "name ": { name: "name ", type: FieldType.STRING, @@ -2672,7 +2669,7 @@ describe.each([ ;(isSqs || isInMemory) && describe("space at start of column name", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ " name": { name: " name", type: FieldType.STRING, @@ -2705,7 +2702,7 @@ describe.each([ isSqs && describe("duplicate columns", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ name: { name: "name", type: FieldType.STRING, @@ -2713,7 +2710,7 @@ describe.each([ }) await context.doInAppContext(config.getAppId(), async () => { const db = context.getAppDB() - const tableDoc = await db.get(table._id!) + const tableDoc = await db.get
(sourceId) tableDoc.schema.Name = { name: "Name", type: FieldType.STRING, @@ -2747,7 +2744,7 @@ describe.each([ type: FieldType.STRING, }, }) - table = await createTable({ + sourceId = await createTable({ name: { name: "name", type: FieldType.STRING, @@ -2756,15 +2753,15 @@ describe.each([ name: "rel", type: FieldType.LINK, relationshipType: RelationshipType.MANY_TO_MANY, - tableId: toRelateTable._id!, + tableId: toRelateTable, fieldName: "rel", }, }) const [row1, row2] = await Promise.all([ - config.api.row.save(toRelateTable._id!, { name: "tag 1" }), - config.api.row.save(toRelateTable._id!, { name: "tag 2" }), + config.api.row.save(toRelateTable, { name: "tag 1" }), + config.api.row.save(toRelateTable, { name: "tag 2" }), ]) - row = await config.api.row.save(table._id!, { + row = await config.api.row.save(sourceId, { name: "product 1", rel: [row1._id, row2._id], }) @@ -2783,7 +2780,7 @@ describe.each([ !isInternal && describe("search by composite key", () => { beforeAll(async () => { - table = await config.api.table.save( + const table = await config.api.table.save( tableForDatasource(datasource, { schema: { idColumn1: { @@ -2798,6 +2795,7 @@ describe.each([ primary: ["idColumn1", "idColumn2"], }) ) + sourceId = table._id! await createRows([{ idColumn1: 1, idColumn2: 2 }]) }) @@ -2819,13 +2817,13 @@ describe.each([ isSql && describe("primaryDisplay", () => { beforeAll(async () => { - let toRelateTable = await createTable({ + let toRelateTableId = await createTable({ name: { name: "name", type: FieldType.STRING, }, }) - table = await config.api.table.save( + const table = await config.api.table.save( tableForDatasource(datasource, { schema: { name: { @@ -2836,13 +2834,14 @@ describe.each([ name: "link", type: FieldType.LINK, relationshipType: RelationshipType.MANY_TO_ONE, - tableId: toRelateTable._id!, + tableId: toRelateTableId, fieldName: "link", }, }, }) ) - toRelateTable = await config.api.table.get(toRelateTable._id!) + sourceId = table._id! + const toRelateTable = await config.api.table.get(toRelateTableId) await config.api.table.save({ ...toRelateTable, primaryDisplay: "link", @@ -2851,7 +2850,7 @@ describe.each([ config.api.row.save(toRelateTable._id!, { name: "test" }), ]) await Promise.all([ - config.api.row.save(table._id!, { + config.api.row.save(sourceId, { name: "test", link: relatedRows.map(row => row._id), }), @@ -2870,7 +2869,7 @@ describe.each([ !isLucene && describe("$and", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ age: { name: "age", type: FieldType.NUMBER }, name: { name: "name", type: FieldType.STRING }, }) @@ -2999,7 +2998,7 @@ describe.each([ !isLucene && describe("$or", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ age: { name: "age", type: FieldType.NUMBER }, name: { name: "name", type: FieldType.STRING }, }) @@ -3159,20 +3158,20 @@ describe.each([ row[name] = i } const relatedTable = await createTable(relatedSchema) - table = await createTable({ + sourceId = await createTable({ name: { name: "name", type: FieldType.STRING }, related1: { type: FieldType.LINK, name: "related1", fieldName: "main1", - tableId: relatedTable._id!, + tableId: relatedTable, relationshipType: RelationshipType.MANY_TO_MANY, }, }) relatedRows = await Promise.all([ - config.api.row.save(relatedTable._id!, row), + config.api.row.save(relatedTable, row), ]) - await config.api.row.save(table._id!, { + await config.api.row.save(sourceId, { name: "foo", related1: [relatedRows[0]._id], }) From b88e63d490923a6c11a000d58bdfe733dc3c5ced Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 1 Oct 2024 13:24:38 +0200 Subject: [PATCH 30/85] Helpers not changing state --- packages/server/src/api/routes/tests/search.spec.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index ac189464a3..09ae252115 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -74,7 +74,7 @@ describe.each([ }, generator.guid().substring(0, 10) ) - sourceId = await createTable( + const tableId = await createTable( { name: { name: "name", type: FieldType.STRING }, //@ts-ignore - API accepts this structure, will build out rest of definition @@ -93,6 +93,7 @@ describe.each([ ) return { relatedTable: await config.api.table.get(relatedTable), + tableId, } } @@ -2246,9 +2247,10 @@ describe.each([ let productCategoryTable: Table, productCatRows: Row[] beforeAll(async () => { - const { relatedTable } = await basicRelationshipTables( + const { relatedTable, tableId } = await basicRelationshipTables( RelationshipType.ONE_TO_MANY ) + sourceId = tableId productCategoryTable = relatedTable productCatRows = await Promise.all([ @@ -2298,9 +2300,10 @@ describe.each([ isSql && describe("big relations", () => { beforeAll(async () => { - const { relatedTable } = await basicRelationshipTables( + const { relatedTable, tableId } = await basicRelationshipTables( RelationshipType.MANY_TO_ONE ) + sourceId = tableId const mainRow = await config.api.row.save(sourceId, { name: "foo", }) From f00593ff26ca42e2e1521048aab079b29a3742d3 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 1 Oct 2024 12:25:41 +0100 Subject: [PATCH 31/85] pr comments --- .../tests/scenarios/branching.spec.ts | 10 +++++----- .../tests/utilities/AutomationTestBuilder.ts | 20 +++++++++---------- .../server/src/definitions/automations.ts | 4 ++-- packages/server/src/threads/automation.ts | 6 +++--- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/server/src/automations/tests/scenarios/branching.spec.ts b/packages/server/src/automations/tests/scenarios/branching.spec.ts index 76e04afdd3..032b729e44 100644 --- a/packages/server/src/automations/tests/scenarios/branching.spec.ts +++ b/packages/server/src/automations/tests/scenarios/branching.spec.ts @@ -30,7 +30,7 @@ describe("Branching automations", () => { .appAction({ fields: {} }) .serverLog( { text: "Starting automation" }, - { stepName: "FirstLog", id: firstLogId } + { stepName: "FirstLog", stepId: firstLogId } ) .branch({ topLevelBranch1: { @@ -38,14 +38,14 @@ describe("Branching automations", () => { stepBuilder .serverLog( { text: "Branch 1" }, - { id: "66666666-6666-6666-6666-666666666666" } + { stepId: "66666666-6666-6666-6666-666666666666" } ) .branch({ branch1: { steps: stepBuilder => stepBuilder.serverLog( { text: "Branch 1.1" }, - { id: branch1LogId } + { stepId: branch1LogId } ), condition: { equal: { [`{{ steps.${firstLogId}.success }}`]: true }, @@ -55,7 +55,7 @@ describe("Branching automations", () => { steps: stepBuilder => stepBuilder.serverLog( { text: "Branch 1.2" }, - { id: branch2LogId } + { stepId: branch2LogId } ), condition: { equal: { [`{{ steps.${firstLogId}.success }}`]: false }, @@ -68,7 +68,7 @@ describe("Branching automations", () => { }, topLevelBranch2: { steps: stepBuilder => - stepBuilder.serverLog({ text: "Branch 2" }, { id: branch2Id }), + stepBuilder.serverLog({ text: "Branch 2" }, { stepId: branch2Id }), condition: { equal: { [`{{ steps.${firstLogId}.success }}`]: false }, }, diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index 6af18cd27e..6aaf22cd6a 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -64,9 +64,9 @@ class BaseStepBuilder { stepId: TStep, stepSchema: Omit, inputs: AutomationStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { - const id = opts?.id || uuidv4() + const id = opts?.stepId || uuidv4() this.steps.push({ ...stepSchema, inputs: inputs as any, @@ -107,7 +107,7 @@ class BaseStepBuilder { // STEPS createRow( inputs: CreateRowStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.CREATE_ROW, @@ -119,7 +119,7 @@ class BaseStepBuilder { updateRow( inputs: UpdateRowStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.UPDATE_ROW, @@ -131,7 +131,7 @@ class BaseStepBuilder { deleteRow( inputs: DeleteRowStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.DELETE_ROW, @@ -143,7 +143,7 @@ class BaseStepBuilder { sendSmtpEmail( inputs: SmtpEmailStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.SEND_EMAIL_SMTP, @@ -155,7 +155,7 @@ class BaseStepBuilder { executeQuery( inputs: ExecuteQueryStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.EXECUTE_QUERY, @@ -167,7 +167,7 @@ class BaseStepBuilder { queryRows( inputs: QueryRowsStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.QUERY_ROWS, @@ -178,7 +178,7 @@ class BaseStepBuilder { } loop( inputs: LoopStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.LOOP, @@ -190,7 +190,7 @@ class BaseStepBuilder { serverLog( input: ServerLogStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.SERVER_LOG, diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index 44758b727b..9433075da7 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -15,8 +15,8 @@ export interface TriggerOutput { export interface AutomationContext extends AutomationResults { steps: any[] - stepsById?: Record - stepsByName?: Record + stepsById: Record + stepsByName: Record env?: Record trigger: any } diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 7788744ea2..3b47634663 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -74,7 +74,7 @@ class Orchestrator { private job: Job private loopStepOutputs: LoopStep[] private stopped: boolean - private executionOutput: AutomationContext + private executionOutput: Omit constructor(job: AutomationJob) { let automation = job.data.automation @@ -458,9 +458,9 @@ class Orchestrator { inputs: steps[stepToLoopIndex].inputs, }) - this.context.stepsById![steps[stepToLoopIndex].id] = tempOutput + this.context.stepsById[steps[stepToLoopIndex].id] = tempOutput const stepName = steps[stepToLoopIndex].name || steps[stepToLoopIndex].id - this.context.stepsByName![stepName] = tempOutput + this.context.stepsByName[stepName] = tempOutput this.context.steps[this.context.steps.length] = tempOutput this.context.steps = this.context.steps.filter( item => !item.hasOwnProperty.call(item, "currentItem") From ae4f7ae4b45ab4e30778eddff0938017924a22d0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 15:04:01 +0100 Subject: [PATCH 32/85] Implement group by and add a test for it. --- packages/backend-core/src/sql/sql.ts | 19 +++++++++------ .../src/api/routes/tests/viewV2.spec.ts | 23 +++++++++++-------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 54ca5a0135..14e32623e3 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -799,6 +799,14 @@ class InternalBuilder { return query } + isSqs(t?: Table): boolean { + const table = t || this.table + return ( + table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID + ) + } + getTableName(t?: Table | string): string { let table: Table if (typeof t === "string") { @@ -813,11 +821,7 @@ class InternalBuilder { } let name = table.name - if ( - (table.sourceType === TableSourceType.INTERNAL || - table.sourceId === INTERNAL_TABLE_SOURCE_ID) && - table._id - ) { + if (this.isSqs(table) && table._id) { // SQS uses the table ID rather than the table name name = table._id } @@ -830,7 +834,7 @@ class InternalBuilder { throw new Error("SQL counting requires primary key to be supplied") } return query.countDistinct( - `${this.getTableName(this.table)}.${this.table.primary[0]} as total` + `${this.getTableName()}.${this.table.primary[0]} as total` ) } @@ -842,10 +846,11 @@ class InternalBuilder { const tableName = this.getTableName() if (fields.length > 0) { query = query.groupBy(fields.map(field => `${tableName}.${field}`)) + query = query.select(fields.map(field => `${tableName}.${field}`)) } for (const aggregation of aggregations) { const op = aggregation.calculationType - const field = `${this.table.name}.${aggregation.field} as ${aggregation.name}` + const field = `${tableName}.${aggregation.field} as ${aggregation.name}` switch (op) { case CalculationType.COUNT: query = query.count(field) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 0f4e6c961c..b03c445b78 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -39,13 +39,13 @@ import { } from "@budibase/backend-core" describe.each([ - // ["lucene", undefined], - // ["sqs", undefined], + ["lucene", undefined], + ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -2459,7 +2459,7 @@ describe.each([ } }) - it.only("should be able to group by a basic field", async () => { + it("should be able to group by a basic field", async () => { const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), @@ -2480,9 +2480,14 @@ describe.each([ query: {}, }) + const priceByQuantity: Record = {} + for (const row of rows) { + priceByQuantity[row.quantity] ??= 0 + priceByQuantity[row.quantity] += row.price + } + for (const row of response.rows) { - expect(row["quantity"]).toBeGreaterThan(0) - expect(row["Total Price"]).toBeGreaterThan(0) + expect(row["Total Price"]).toEqual(priceByQuantity[row.quantity]) } }) }) From addd54a8e8baab79811e6f71aab728f32239577b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 15:39:33 +0100 Subject: [PATCH 33/85] Fix generic-sql.spec.ts --- packages/backend-core/src/sql/sql.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 14e32623e3..0f72eea96e 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -811,7 +811,10 @@ class InternalBuilder { let table: Table if (typeof t === "string") { if (!this.query.meta.tables?.[t]) { - throw new Error(`Table ${t} not found`) + // This can legitimately happen in custom queries, where the user is + // querying against a table that may not have been imported into + // Budibase. + return t } table = this.query.meta.tables[t] } else if (t) { @@ -1547,12 +1550,12 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { if (!this._isJsonColumn(field)) { continue } - const fullName = `${tableName}.${name}` + const fullName = `${tableName}.${name}` as keyof T for (let row of results) { - if (typeof row[fullName as keyof T] === "string") { - row[fullName as keyof T] = JSON.parse(row[fullName]) + if (typeof row[fullName] === "string") { + row[fullName] = JSON.parse(row[fullName]) } - if (typeof row[name as keyof T] === "string") { + if (typeof row[name] === "string") { row[name as keyof T] = JSON.parse(row[name]) } } From 7cee1509aa3ee47677e3116121beab3b57a6deef Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 16:17:11 +0100 Subject: [PATCH 34/85] Fix sqlAlias.spec.ts --- packages/backend-core/src/sql/sql.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 0f72eea96e..105116e828 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -810,13 +810,18 @@ class InternalBuilder { getTableName(t?: Table | string): string { let table: Table if (typeof t === "string") { - if (!this.query.meta.tables?.[t]) { + if (this.query.table?.name === t) { + table = this.query.table + } else if (this.query.meta.table?.name === t) { + table = this.query.meta.table + } else if (!this.query.meta.tables?.[t]) { // This can legitimately happen in custom queries, where the user is // querying against a table that may not have been imported into // Budibase. return t + } else { + table = this.query.meta.tables[t] } - table = this.query.meta.tables[t] } else if (t) { table = t } else { From 4165c6cab42affee7f40f5493e29e06a90c67ec4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 16:06:40 +0100 Subject: [PATCH 35/85] Test all aggregation types. --- .../src/api/routes/tests/viewV2.spec.ts | 55 +++++++++++++++++++ packages/server/src/integrations/postgres.ts | 3 +- .../src/utilities/rowProcessor/index.ts | 19 +++++++ 3 files changed, 76 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index b03c445b78..1d6c1d50cd 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -2490,6 +2490,61 @@ describe.each([ expect(row["Total Price"]).toEqual(priceByQuantity[row.quantity]) } }) + + it.each([ + CalculationType.COUNT, + CalculationType.SUM, + CalculationType.AVG, + CalculationType.MIN, + CalculationType.MAX, + ])("should be able to calculate $type", async type => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + aggregate: { + visible: true, + calculationType: type, + field: "price", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + function calculate( + type: CalculationType, + numbers: number[] + ): number { + switch (type) { + case CalculationType.COUNT: + return numbers.length + case CalculationType.SUM: + return numbers.reduce((a, b) => a + b, 0) + case CalculationType.AVG: + return numbers.reduce((a, b) => a + b, 0) / numbers.length + case CalculationType.MIN: + return Math.min(...numbers) + case CalculationType.MAX: + return Math.max(...numbers) + } + } + + const prices = rows.map(row => row.price) + const expected = calculate(type, prices) + const actual = response.rows[0].aggregate + + if (type === CalculationType.AVG) { + // The average calculation can introduce floating point rounding + // errors, so we need to compare to within a small margin of + // error. + expect(actual).toBeCloseTo(expected) + } else { + expect(actual).toEqual(expected) + } + }) }) }) diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 3652864991..ce8b21eede 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -272,7 +272,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus { try { const bindings = query.bindings || [] this.log(query.sql, bindings) - return await client.query(query.sql, bindings) + const result = await client.query(query.sql, bindings) + return result } catch (err: any) { await this.closeConnection() let readableMessage = getReadableErrorMessage( diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index bddd590f25..7332f8b244 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -426,6 +426,25 @@ export async function coreOutputProcessing( } } } + + if (sdk.views.isView(source)) { + const calculationFields = Object.keys( + helpers.views.calculationFields(source) + ) + + // We ensure all calculation fields are returned as numbers. During the + // testing of this feature it was discovered that the COUNT operation + // returns a string for MySQL, MariaDB, and Postgres. But given that all + // calculation fields should be numbers, we blanket make sure of that + // here. + for (const key of calculationFields) { + for (const row of rows) { + if (typeof row[key] === "string") { + row[key] = parseFloat(row[key]) + } + } + } + } } if (!isUserMetadataTable(table._id!)) { From 1dea53f5976aeee63c3fe5f5b8087ac64a42fb90 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 1 Oct 2024 16:25:48 +0100 Subject: [PATCH 36/85] Refresh data when adding columns --- .../DataTable/modals/grid/GridCreateColumnModal.svelte | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte b/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte index 2040f66706..d031e752cd 100644 --- a/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte @@ -2,7 +2,12 @@ import { getContext } from "svelte" import CreateEditColumn from "components/backend/DataTable/modals/CreateEditColumn.svelte" - const { datasource } = getContext("grid") + const { datasource, rows } = getContext("grid") + + const onUpdate = async () => { + await datasource.actions.refreshDefinition() + await rows.actions.refreshData() + } - + From 13248c409f358cb3c0a791bce59b8e05c01247da Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 16:44:16 +0100 Subject: [PATCH 37/85] Respond to PR comment. --- packages/server/src/integrations/postgres.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index ce8b21eede..3652864991 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -272,8 +272,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus { try { const bindings = query.bindings || [] this.log(query.sql, bindings) - const result = await client.query(query.sql, bindings) - return result + return await client.query(query.sql, bindings) } catch (err: any) { await this.closeConnection() let readableMessage = getReadableErrorMessage( From c4a6a92bdbbfee4768c5a77b10aef3c5c8986a43 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 1 Oct 2024 17:03:06 +0100 Subject: [PATCH 38/85] PR comments --- packages/frontend-core/src/api/ai.js | 2 +- packages/pro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/frontend-core/src/api/ai.js b/packages/frontend-core/src/api/ai.js index 702ce87cdb..7fa756a19e 100644 --- a/packages/frontend-core/src/api/ai.js +++ b/packages/frontend-core/src/api/ai.js @@ -4,7 +4,7 @@ export const buildAIEndpoints = API => ({ */ generateCronExpression: async ({ prompt }) => { return await API.post({ - url: "/api/ai/generate/cron", + url: "/api/ai/cron", body: { prompt }, }) }, diff --git a/packages/pro b/packages/pro index dcc9e50b80..f35190a594 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit dcc9e50b8064a2097d408771462ad80f48de7ff6 +Subproject commit f35190a594afb04525d0bc4405bea8a04bd62b42 From a28a64f9d8b1053509f58db6acc6ee0c89d92109 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 1 Oct 2024 17:05:38 +0100 Subject: [PATCH 39/85] update pro ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index f35190a594..aca9828117 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit f35190a594afb04525d0bc4405bea8a04bd62b42 +Subproject commit aca9828117bb97f54f40ee359f1a3f6e259174e7 From 77856eb35a8072c8c71d63178c6cc8a13ef3de0b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 17:23:21 +0100 Subject: [PATCH 40/85] Add a test to make sure fields on the underlying table that are required are not required on the view. --- .../src/api/routes/tests/viewV2.spec.ts | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 1d6c1d50cd..f76b6eb470 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -2546,6 +2546,51 @@ describe.each([ } }) }) + + !isLucene && + it("should not need required fields to be present", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + age: { + name: "age", + type: FieldType.NUMBER, + }, + }, + }) + ) + + await Promise.all([ + config.api.row.save(table._id!, { name: "Steve", age: 30 }), + config.api.row.save(table._id!, { name: "Jane", age: 31 }), + ]) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "age", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows[0].sum).toEqual(61) + }) }) describe("permissions", () => { From 08f1c4dadc668db7fe645f01336832341546828e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 09:35:15 +0100 Subject: [PATCH 41/85] Update packages/backend-core/src/sql/sql.ts Co-authored-by: Adria Navarro --- packages/backend-core/src/sql/sql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 105116e828..e55524a67e 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1542,7 +1542,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { // SQS uses the table ID rather than the table name name = table._id } - return aliases?.[name] ? aliases[name] : name + return aliases?.[name] || name } convertJsonStringColumns>( From cc6b2f6717cce5ea7dfb2830aa1f1e4de9410e68 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 09:35:47 +0100 Subject: [PATCH 42/85] add failing test --- .../src/api/routes/tests/viewV2.spec.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index f76b6eb470..954047d536 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -25,6 +25,7 @@ import { ViewFieldMetadata, FeatureFlag, BBReferenceFieldSubType, + ViewCalculationFieldMetadata, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -540,6 +541,31 @@ describe.each([ status: 201, }) }) + + it.only("can create a view with calculation fields", async () => { + let view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "Price", + }, + }, + }) + + let sum = view.schema!.sum as ViewCalculationFieldMetadata + expect(sum).toBeDefined() + expect(sum.calculationType).toEqual(CalculationType.SUM) + expect(sum.field).toEqual("Price") + + view = await config.api.viewV2.get(view.id) + sum = view.schema!.sum as ViewCalculationFieldMetadata + expect(sum).toBeDefined() + expect(sum.calculationType).toEqual(CalculationType.SUM) + expect(sum.field).toEqual("Price") + }) }) describe("update", () => { From ddd229062c20e37f2860c8a07583688f67c0ed1c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 09:39:54 +0100 Subject: [PATCH 43/85] Rename total field when doing row counts. --- packages/backend-core/src/sql/sql.ts | 2 +- packages/server/src/sdk/app/rows/utils.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index e55524a67e..701797329f 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -842,7 +842,7 @@ class InternalBuilder { throw new Error("SQL counting requires primary key to be supplied") } return query.countDistinct( - `${this.getTableName()}.${this.table.primary[0]} as total` + `${this.getTableName()}.${this.table.primary[0]} as __bb_total` ) } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index d5c0560d9b..3d6bf39d3f 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -57,8 +57,8 @@ export function getSQLClient(datasource: Datasource): SqlClient { export function processRowCountResponse( response: DatasourcePlusQueryResponse ): number { - if (response && response.length === 1 && "total" in response[0]) { - const total = response[0].total + if (response && response.length === 1 && "__bb_total" in response[0]) { + const total = response[0].__bb_total return typeof total === "number" ? total : parseInt(total) } else { throw new Error("Unable to count rows in query - no count response") From 7b9af81fd510f2aa87c757173abd028c068cebba Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 09:44:20 +0100 Subject: [PATCH 44/85] Clean up params and isSqs --- packages/backend-core/src/sql/sql.ts | 36 +++++++++++++++------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 701797329f..627be039ca 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -87,6 +87,13 @@ function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] { return query } +function isSqs(table: Table): boolean { + return ( + table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID + ) +} + class InternalBuilder { private readonly client: SqlClient private readonly query: QueryJson @@ -799,37 +806,34 @@ class InternalBuilder { return query } - isSqs(t?: Table): boolean { - const table = t || this.table - return ( - table.sourceType === TableSourceType.INTERNAL || - table.sourceId === INTERNAL_TABLE_SOURCE_ID - ) + isSqs(): boolean { + return isSqs(this.table) } - getTableName(t?: Table | string): string { + getTableName(tableOrName?: Table | string): string { let table: Table - if (typeof t === "string") { - if (this.query.table?.name === t) { + if (typeof tableOrName === "string") { + const name = tableOrName + if (this.query.table?.name === name) { table = this.query.table - } else if (this.query.meta.table?.name === t) { + } else if (this.query.meta.table?.name === name) { table = this.query.meta.table - } else if (!this.query.meta.tables?.[t]) { + } else if (!this.query.meta.tables?.[name]) { // This can legitimately happen in custom queries, where the user is // querying against a table that may not have been imported into // Budibase. - return t + return name } else { - table = this.query.meta.tables[t] + table = this.query.meta.tables[name] } - } else if (t) { - table = t + } else if (tableOrName) { + table = tableOrName } else { table = this.table } let name = table.name - if (this.isSqs(table) && table._id) { + if (isSqs(table) && table._id) { // SQS uses the table ID rather than the table name name = table._id } From 61a0db0984ee4bd59094c214ebd55c43e763ce45 Mon Sep 17 00:00:00 2001 From: Dean Date: Wed, 2 Oct 2024 09:51:13 +0100 Subject: [PATCH 45/85] Add explicit typing for view search filter config --- packages/server/src/sdk/app/rows/search.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 8de5818805..dae24c6bc0 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -3,6 +3,8 @@ import { LogicalOperator, Row, RowSearchParams, + SearchFilter, + SearchFilterGroup, SearchFilterKey, SearchFilters, SearchResponse, @@ -91,11 +93,12 @@ export async function search( if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) { // Lucene does not accept conditional filters, so we need to keep the old logic - const query: SearchFilters = viewQuery + const query: SearchFilters = viewQuery || {} + const viewFilters = view.query as SearchFilter[] // Extract existing fields const existingFields = - view.query + viewFilters ?.filter(filter => filter.field) .map(filter => db.removeKeyNumbering(filter.field)) || [] @@ -112,7 +115,7 @@ export async function search( } else { options.query = { $and: { - conditions: [viewQuery, options.query], + conditions: [viewQuery as SearchFilterGroup, options.query], }, } } From 4dd6afd4352ba774bde40764cf6d9a315d983d7b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 09:57:18 +0100 Subject: [PATCH 46/85] Symbolise the special __bb_total count field name. --- packages/backend-core/src/sql/sql.ts | 4 +++- packages/server/src/sdk/app/rows/utils.ts | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 627be039ca..3585dacbed 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -43,6 +43,8 @@ import { cloneDeep } from "lodash" type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any +export const COUNT_FIELD_NAME = "__bb_total" + function getBaseLimit() { const envLimit = environment.SQL_MAX_ROWS ? parseInt(environment.SQL_MAX_ROWS) @@ -846,7 +848,7 @@ class InternalBuilder { throw new Error("SQL counting requires primary key to be supplied") } return query.countDistinct( - `${this.getTableName()}.${this.table.primary[0]} as __bb_total` + `${this.getTableName()}.${this.table.primary[0]} as ${COUNT_FIELD_NAME}` ) } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 3d6bf39d3f..e1b5615046 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -22,6 +22,7 @@ import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils" import { isSQL } from "../../../integrations/utils" import { docIds } from "@budibase/backend-core" import { getTableFromSource } from "../../../api/controllers/row/utils" +import { COUNT_FIELD_NAME } from "@budibase/backend-core/src/sql/sql" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, @@ -57,8 +58,8 @@ export function getSQLClient(datasource: Datasource): SqlClient { export function processRowCountResponse( response: DatasourcePlusQueryResponse ): number { - if (response && response.length === 1 && "__bb_total" in response[0]) { - const total = response[0].__bb_total + if (response && response.length === 1 && COUNT_FIELD_NAME in response[0]) { + const total = response[0][COUNT_FIELD_NAME] return typeof total === "number" ? total : parseInt(total) } else { throw new Error("Unable to count rows in query - no count response") From ee897e4d7ef6afbeb56bb4068c9e1e4ba44959f6 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 10:05:56 +0100 Subject: [PATCH 47/85] Fix imports. --- packages/backend-core/src/sql/index.ts | 2 +- packages/server/src/sdk/app/rows/utils.ts | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/sql/index.ts b/packages/backend-core/src/sql/index.ts index 16b718d2e6..816b3d60a5 100644 --- a/packages/backend-core/src/sql/index.ts +++ b/packages/backend-core/src/sql/index.ts @@ -1,5 +1,5 @@ export * as utils from "./utils" -export { default as Sql } from "./sql" +export { default as Sql, COUNT_FIELD_NAME } from "./sql" export { default as SqlTable } from "./sqlTable" export * as designDoc from "./designDoc" diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index e1b5615046..6ef4dcbc8e 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -20,9 +20,8 @@ import { Format } from "../../../api/controllers/view/exporters" import sdk from "../.." import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils" import { isSQL } from "../../../integrations/utils" -import { docIds } from "@budibase/backend-core" +import { docIds, sql } from "@budibase/backend-core" import { getTableFromSource } from "../../../api/controllers/row/utils" -import { COUNT_FIELD_NAME } from "@budibase/backend-core/src/sql/sql" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, @@ -58,8 +57,12 @@ export function getSQLClient(datasource: Datasource): SqlClient { export function processRowCountResponse( response: DatasourcePlusQueryResponse ): number { - if (response && response.length === 1 && COUNT_FIELD_NAME in response[0]) { - const total = response[0][COUNT_FIELD_NAME] + if ( + response && + response.length === 1 && + sql.COUNT_FIELD_NAME in response[0] + ) { + const total = response[0][sql.COUNT_FIELD_NAME] return typeof total === "number" ? total : parseInt(total) } else { throw new Error("Unable to count rows in query - no count response") From f6a783a9f13a7c0cfc601ba23664e010f9b29a70 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 2 Oct 2024 10:25:22 +0100 Subject: [PATCH 48/85] Update submodules --- .../components/backend/DataTable/modals/CreateEditColumn.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index c7a9fecd65..7172689d69 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -767,7 +767,7 @@ formula: e.detail, } }} - {bindings} + bindings={getBindings({ table })} allowJS context={rowGoldenSample} /> From 0679ec89931879158f2ef3fd43bc9de89540321b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 10:36:45 +0100 Subject: [PATCH 49/85] Make sure calculation views are created and returned correctly. --- .../src/api/routes/tests/viewV2.spec.ts | 4 ++- packages/server/src/sdk/app/views/index.ts | 31 +++++++++---------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 954047d536..669d35ba5b 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -542,7 +542,7 @@ describe.each([ }) }) - it.only("can create a view with calculation fields", async () => { + it("can create a view with calculation fields", async () => { let view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), @@ -555,6 +555,8 @@ describe.each([ }, }) + expect(Object.keys(view.schema!)).toHaveLength(1) + let sum = view.schema!.sum as ViewCalculationFieldMetadata expect(sum).toBeDefined() expect(sum.calculationType).toEqual(CalculationType.SUM) diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index 24e4da3172..d218a3c7e8 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -258,19 +258,12 @@ export async function enrichSchema( view: ViewV2, tableSchema: TableSchema ): Promise { - const tableCache: Record = {} - async function populateRelTableSchema( tableId: string, viewFields: Record ) { - if (!tableCache[tableId]) { - tableCache[tableId] = await sdk.tables.getTable(tableId) - } - const relTable = tableCache[tableId] - + const relTable = await sdk.tables.getTable(tableId) const result: Record = {} - for (const relTableFieldName of Object.keys(relTable.schema)) { const relTableField = relTable.schema[relTableFieldName] if ([FieldType.LINK, FieldType.FORMULA].includes(relTableField.type)) { @@ -299,15 +292,22 @@ export async function enrichSchema( const viewSchema = view.schema || {} const anyViewOrder = Object.values(viewSchema).some(ui => ui.order != null) - for (const key of Object.keys(tableSchema).filter( - k => tableSchema[k].visible !== false - )) { + + const visibleSchemaFields = Object.keys(viewSchema).filter( + key => viewSchema[key].visible !== false + ) + const visibleTableFields = Object.keys(tableSchema).filter( + key => tableSchema[key].visible !== false + ) + const visibleFields = new Set([...visibleSchemaFields, ...visibleTableFields]) + + for (const key of visibleFields) { // if nothing specified in view, then it is not visible const ui = viewSchema[key] || { visible: false } schema[key] = { - ...tableSchema[key], + ...(tableSchema[key] || {}), ...ui, - order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key].order, + order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key]?.order, columns: undefined, } @@ -319,10 +319,7 @@ export async function enrichSchema( } } - return { - ...view, - schema: schema, - } + return { ...view, schema } } export function syncSchema( From a49e779d111f051f42a718debdea2e1a58c8d43f Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 2 Oct 2024 10:50:22 +0100 Subject: [PATCH 50/85] Add default value for multi-selects and improve default value validation --- packages/account-portal | 2 +- .../DataTable/modals/CreateEditColumn.svelte | 39 +++++++---- packages/pro | 2 +- .../src/utilities/rowProcessor/index.ts | 6 +- packages/shared-core/src/table.ts | 2 +- yarn.lock | 64 ++++++++++++++++++- 6 files changed, 93 insertions(+), 22 deletions(-) diff --git a/packages/account-portal b/packages/account-portal index 905773d708..3e24f6293f 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 905773d70854a43c6ef2461c7a49671bff56fedc +Subproject commit 3e24f6293ff5ee5f9b42822e001504e3bbf19cc0 diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 7172689d69..19cc4ec1c0 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -207,6 +207,11 @@ }, ...getUserBindings(), ] + $: sanitiseDefaultValue( + editableColumn.type, + editableColumn.constraints?.inclusion || [], + editableColumn.default + ) const fieldDefinitions = Object.values(FIELDS).reduce( // Storing the fields by complex field id @@ -301,15 +306,6 @@ delete saveColumn.default } - // Delete default value for options fields if the option is no longer available - if ( - saveColumn.type === FieldType.OPTIONS && - saveColumn.default && - !saveColumn.constraints.inclusion?.includes(saveColumn.default) - ) { - delete saveColumn.default - } - // Ensure primary display columns are always required and don't have default values if (primaryDisplay) { saveColumn.constraints.presence = { allowEmpty: false } @@ -318,7 +314,7 @@ // Ensure the field is not required if we have a default value if (saveColumn.default) { - saveColumn.constraints.presence = { allowEmpty: true } + saveColumn.constraints.presence = false } try { @@ -567,6 +563,20 @@ return newError } + const sanitiseDefaultValue = (type, options, defaultValue) => { + if (!defaultValue?.length) { + return + } + // Delete default value for options fields if the option is no longer available + if (type === FieldType.OPTIONS && !options.includes(defaultValue)) { + delete editableColumn.default + } + // Filter array default values to only valid options + if (type === FieldType.ARRAY) { + editableColumn.default = defaultValue.filter(x => options.includes(x)) + } + } + onMount(() => { mounted = true }) @@ -774,9 +784,9 @@ {:else if editableColumn.type === JSON_TYPE} - + {/if} {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}