From cb3937b5c10c439ffec2c2c4910434a5927d2a6c Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 18 Apr 2024 15:58:40 +0100 Subject: [PATCH 01/40] Auto fill card blocks with bindings and reset when changing datasource --- .../builder/src/stores/builder/components.js | 57 ++++++++++++++++--- packages/client/manifest.json | 9 +-- 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js index fe5f4e8a05..24789b6b05 100644 --- a/packages/builder/src/stores/builder/components.js +++ b/packages/builder/src/stores/builder/components.js @@ -1,4 +1,4 @@ -import { get, derived } from "svelte/store" +import { get, derived, readable } from "svelte/store" import { cloneDeep } from "lodash/fp" import { API } from "api" import { Helpers } from "@budibase/bbui" @@ -20,7 +20,7 @@ import { previewStore, tables, componentTreeNodesStore, -} from "stores/builder/index" +} from "stores/builder" import { buildFormSchema, getSchemaForDatasource } from "dataBinding" import { BUDIBASE_INTERNAL_DB_ID, @@ -30,6 +30,7 @@ import { } from "constants/backend" import BudiStore from "../BudiStore" import { Utils } from "@budibase/frontend-core" +import { FieldType } from "@budibase/types" export const INITIAL_COMPONENTS_STATE = { components: {}, @@ -295,6 +296,44 @@ export class ComponentStore extends BudiStore { } } }) + + // Add default bindings to card blocks + if (component._component.endsWith("/cardsblock")) { + const { _id, dataSource } = component + if (dataSource) { + const { schema, table } = getSchemaForDatasource(screen, dataSource) + const readableTypes = [ + FieldType.STRING, + FieldType.OPTIONS, + FieldType.DATETIME, + FieldType.NUMBER, + ] + + // Extract good field candidates to prefil our cards with + const fields = Object.entries(schema || {}) + .filter(([name, fieldSchema]) => { + return ( + readableTypes.includes(fieldSchema.type) && + !fieldSchema.autoColumn && + name !== table?.primaryDisplay + ) + }) + .map(([name]) => name) + + // Use the primary display as the best field, if it exists + if (schema?.[table?.primaryDisplay]) { + fields.unshift(table.primaryDisplay) + } + + // Fill our cards with as many bindings as we can + const cardKeys = ["cardTitle", "cardSubtitle", "cardDescription"] + cardKeys.forEach(key => { + if (!fields[0] || component[key]) return + component[key] = `{{ ${safe(`${_id}-repeater`)}.${safe(fields[0])} }}` + fields.shift() + }) + } + } } /** @@ -323,21 +362,21 @@ export class ComponentStore extends BudiStore { ...presetProps, } - // Enrich empty settings + // Standard post processing this.enrichEmptySettings(instance, { parent, screen: get(selectedScreen), useDefaultValues: true, }) - - // Migrate nested component settings this.migrateSettings(instance) - // Add any extra properties the component needs + // Custom post processing for creation only let extras = {} if (definition.hasChildren) { extras._children = [] } + + // Add step name to form steps if (componentName.endsWith("/formstep")) { const parentForm = findClosestMatchingComponent( get(selectedScreen).props, @@ -350,6 +389,7 @@ export class ComponentStore extends BudiStore { extras.step = formSteps.length + 1 extras._instanceName = `Step ${formSteps.length + 1}` } + return { ...cloneDeep(instance), ...extras, @@ -460,7 +500,6 @@ export class ComponentStore extends BudiStore { if (!componentId || !screenId) { const state = get(this.store) componentId = componentId || state.selectedComponentId - const screenState = get(screenStore) screenId = screenId || screenState.selectedScreenId } @@ -468,7 +507,6 @@ export class ComponentStore extends BudiStore { return } const patchScreen = screen => { - // findComponent looks in the tree not comp.settings[0] let component = findComponent(screen.props, componentId) if (!component) { return false @@ -477,7 +515,8 @@ export class ComponentStore extends BudiStore { // Mutates the fetched component with updates const patchResult = patchFn(component, screen) - // Mutates the component with any required settings updates + // Post processing + this.enrichEmptySettings(component, { screen, useDefaultValues: false }) const migrated = this.migrateSettings(component) // Returning an explicit false signifies that we should skip this diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 40abc7a9a0..91fd141704 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -5990,27 +5990,28 @@ "key": "cardTitle", "label": "Title", "nested": true, - "defaultValue": "Title" + "resetOn": "dataSource" }, { "type": "text", "key": "cardSubtitle", "label": "Subtitle", "nested": true, - "defaultValue": "Subtitle" + "resetOn": "dataSource" }, { "type": "text", "key": "cardDescription", "label": "Description", "nested": true, - "defaultValue": "Description" + "resetOn": "dataSource" }, { "type": "text", "key": "cardImageURL", "label": "Image URL", - "nested": true + "nested": true, + "resetOn": "dataSource" }, { "type": "boolean", From e98a9f7f80792086cda9f4e1f0b470375673fddb Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 19 Apr 2024 10:55:58 +0100 Subject: [PATCH 02/40] Update when empty setting enrichment runs to ensure card blocks properly reset when changing datasource --- .../builder/src/stores/builder/components.js | 83 ++++++++++++------- 1 file changed, 54 insertions(+), 29 deletions(-) diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js index 24789b6b05..33d21f57cc 100644 --- a/packages/builder/src/stores/builder/components.js +++ b/packages/builder/src/stores/builder/components.js @@ -299,39 +299,64 @@ export class ComponentStore extends BudiStore { // Add default bindings to card blocks if (component._component.endsWith("/cardsblock")) { - const { _id, dataSource } = component - if (dataSource) { - const { schema, table } = getSchemaForDatasource(screen, dataSource) - const readableTypes = [ - FieldType.STRING, - FieldType.OPTIONS, - FieldType.DATETIME, - FieldType.NUMBER, - ] + // Only proceed if the card is empty, i.e. we just changed datasource or + // just created the card + const cardKeys = ["cardTitle", "cardSubtitle", "cardDescription"] + if (cardKeys.every(key => !component[key]) && !component.cardImageURL) { + const { _id, dataSource } = component + if (dataSource) { + const { schema, table } = getSchemaForDatasource(screen, dataSource) + const findFieldTypes = fieldTypes => { + if (!Array.isArray(fieldTypes)) { + fieldTypes = [fieldTypes] + } + return Object.entries(schema || {}) + .filter(([name, fieldSchema]) => { + return ( + fieldTypes.includes(fieldSchema.type) && + !fieldSchema.autoColumn && + name !== table?.primaryDisplay + ) + }) + .map(([name]) => name) + } - // Extract good field candidates to prefil our cards with - const fields = Object.entries(schema || {}) - .filter(([name, fieldSchema]) => { - return ( - readableTypes.includes(fieldSchema.type) && - !fieldSchema.autoColumn && - name !== table?.primaryDisplay - ) + // Extract good field candidates to prefil our cards with + const fields = findFieldTypes([ + FieldType.STRING, + FieldType.OPTIONS, + FieldType.DATETIME, + FieldType.NUMBER, + ]) + + // Use the primary display as the best field, if it exists + if (schema?.[table?.primaryDisplay]) { + fields.unshift(table.primaryDisplay) + } + + // Fill our cards with as many bindings as we can + const prefix = safe(`${_id}-repeater`) + cardKeys.forEach(key => { + if (!fields[0]) return + component[key] = `{{ ${prefix}.${safe(fields[0])} }}` + fields.shift() }) - .map(([name]) => name) - // Use the primary display as the best field, if it exists - if (schema?.[table?.primaryDisplay]) { - fields.unshift(table.primaryDisplay) + // Attempt to fill the image setting + let imgFields = findFieldTypes([FieldType.ATTACHMENT_SINGLE]) + if (imgFields[0]) { + component.cardImageURL = `{{ ${prefix}.${safe( + imgFields[0] + )}.[url] }}` + } else { + imgFields = findFieldTypes([FieldType.ATTACHMENTS]) + if (imgFields[0]) { + component.cardImageURL = `{{ ${prefix}.${safe( + imgFields[0] + )}.[0].[url] }}` + } + } } - - // Fill our cards with as many bindings as we can - const cardKeys = ["cardTitle", "cardSubtitle", "cardDescription"] - cardKeys.forEach(key => { - if (!fields[0] || component[key]) return - component[key] = `{{ ${safe(`${_id}-repeater`)}.${safe(fields[0])} }}` - fields.shift() - }) } } } From 75bf928242386844a000a906e916ba08f8a4bbcd Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 19 Apr 2024 11:08:59 +0100 Subject: [PATCH 03/40] Tidy up card binding logic --- .../builder/src/stores/builder/components.js | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js index 33d21f57cc..c1c16193c2 100644 --- a/packages/builder/src/stores/builder/components.js +++ b/packages/builder/src/stores/builder/components.js @@ -321,39 +321,42 @@ export class ComponentStore extends BudiStore { .map(([name]) => name) } - // Extract good field candidates to prefil our cards with + // Inserts a card binding for a certain setting + const addBinding = (key, ...parts) => { + parts.unshift(`${_id}-repeater`) + component[key] = `{{ ${parts.map(safe).join(".")} }}` + } + + // Extract good field candidates to prefil our cards with. + // Use the primary display as the best field, if it exists. const fields = findFieldTypes([ FieldType.STRING, FieldType.OPTIONS, FieldType.DATETIME, FieldType.NUMBER, ]) - - // Use the primary display as the best field, if it exists if (schema?.[table?.primaryDisplay]) { fields.unshift(table.primaryDisplay) } // Fill our cards with as many bindings as we can - const prefix = safe(`${_id}-repeater`) cardKeys.forEach(key => { - if (!fields[0]) return - component[key] = `{{ ${prefix}.${safe(fields[0])} }}` - fields.shift() + if (fields[0]) { + addBinding(key, fields[0]) + fields.shift() + } }) - // Attempt to fill the image setting - let imgFields = findFieldTypes([FieldType.ATTACHMENT_SINGLE]) - if (imgFields[0]) { - component.cardImageURL = `{{ ${prefix}.${safe( - imgFields[0] - )}.[url] }}` + // Attempt to fill the image setting. + // Check single attachment fields first. + let imgField = findFieldTypes(FieldType.ATTACHMENT_SINGLE)[0] + if (imgField) { + addBinding("cardImageURL", imgField, "url") } else { - imgFields = findFieldTypes([FieldType.ATTACHMENTS]) - if (imgFields[0]) { - component.cardImageURL = `{{ ${prefix}.${safe( - imgFields[0] - )}.[0].[url] }}` + // Then try multi-attachment fields if no single ones exist + imgField = findFieldTypes(FieldType.ATTACHMENTS)[0] + if (imgField) { + addBinding("cardImageURL", imgField, 0, "url") } } } From bc29d3515fcbf6acc78263b59838c58195ebead4 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 19 Apr 2024 11:35:16 +0100 Subject: [PATCH 04/40] Improve card binding autofill field selection --- .../builder/src/stores/builder/components.js | 63 +++++++++++-------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js index c1c16193c2..7eb212d81e 100644 --- a/packages/builder/src/stores/builder/components.js +++ b/packages/builder/src/stores/builder/components.js @@ -30,7 +30,7 @@ import { } from "constants/backend" import BudiStore from "../BudiStore" import { Utils } from "@budibase/frontend-core" -import { FieldType } from "@budibase/types" +import { FieldSubtype, FieldType } from "@budibase/types" export const INITIAL_COMPONENTS_STATE = { components: {}, @@ -306,6 +306,8 @@ export class ComponentStore extends BudiStore { const { _id, dataSource } = component if (dataSource) { const { schema, table } = getSchemaForDatasource(screen, dataSource) + + // Finds fields by types from the schema of the configured datasource const findFieldTypes = fieldTypes => { if (!Array.isArray(fieldTypes)) { fieldTypes = [fieldTypes] @@ -315,48 +317,55 @@ export class ComponentStore extends BudiStore { return ( fieldTypes.includes(fieldSchema.type) && !fieldSchema.autoColumn && - name !== table?.primaryDisplay + name !== table?.primaryDisplay && + !name.startsWith("_") ) }) .map(([name]) => name) } // Inserts a card binding for a certain setting - const addBinding = (key, ...parts) => { - parts.unshift(`${_id}-repeater`) - component[key] = `{{ ${parts.map(safe).join(".")} }}` - } - - // Extract good field candidates to prefil our cards with. - // Use the primary display as the best field, if it exists. - const fields = findFieldTypes([ - FieldType.STRING, - FieldType.OPTIONS, - FieldType.DATETIME, - FieldType.NUMBER, - ]) - if (schema?.[table?.primaryDisplay]) { - fields.unshift(table.primaryDisplay) - } - - // Fill our cards with as many bindings as we can - cardKeys.forEach(key => { - if (fields[0]) { - addBinding(key, fields[0]) - fields.shift() + const addBinding = (key, fallback, ...parts) => { + if (parts.some(x => x == null)) { + component[key] = fallback + } else { + parts.unshift(`${_id}-repeater`) + component[key] = `{{ ${parts.map(safe).join(".")} }}` } - }) + } + + // Extract good field candidates to prefill our cards with. + // Use the primary display as the best field, if it exists. + const shortFields = [ + ...findFieldTypes(FieldType.STRING), + ...findFieldTypes(FieldType.OPTIONS), + ...findFieldTypes(FieldType.ARRAY), + ...findFieldTypes(FieldType.DATETIME), + ...findFieldTypes(FieldType.NUMBER), + ] + const longFields = findFieldTypes(FieldType.LONGFORM) + if (schema?.[table?.primaryDisplay]) { + shortFields.unshift(table.primaryDisplay) + } + + // Fill title and subtitle with short fields + addBinding("cardTitle", "Title", shortFields[0]) + addBinding("cardSubtitle", "Subtitle", shortFields[1]) + + // Fill description with a long field if possible + const longField = longFields[0] ?? shortFields[2] + addBinding("cardDescription", "Description", longField) // Attempt to fill the image setting. // Check single attachment fields first. let imgField = findFieldTypes(FieldType.ATTACHMENT_SINGLE)[0] if (imgField) { - addBinding("cardImageURL", imgField, "url") + addBinding("cardImageURL", null, imgField, "url") } else { // Then try multi-attachment fields if no single ones exist imgField = findFieldTypes(FieldType.ATTACHMENTS)[0] if (imgField) { - addBinding("cardImageURL", imgField, 0, "url") + addBinding("cardImageURL", null, imgField, 0, "url") } } } From e670dc1e1b5c6601e5128e70308ca78ae4e716d5 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 19 Apr 2024 11:43:25 +0100 Subject: [PATCH 05/40] Lint --- packages/builder/src/stores/builder/components.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js index 7eb212d81e..7f5d7a2022 100644 --- a/packages/builder/src/stores/builder/components.js +++ b/packages/builder/src/stores/builder/components.js @@ -1,4 +1,4 @@ -import { get, derived, readable } from "svelte/store" +import { get, derived } from "svelte/store" import { cloneDeep } from "lodash/fp" import { API } from "api" import { Helpers } from "@budibase/bbui" @@ -30,7 +30,7 @@ import { } from "constants/backend" import BudiStore from "../BudiStore" import { Utils } from "@budibase/frontend-core" -import { FieldSubtype, FieldType } from "@budibase/types" +import { FieldType } from "@budibase/types" export const INITIAL_COMPONENTS_STATE = { components: {}, From ebbd0a87d4b28c9bbae80b31753f99593fb22029 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 19 Apr 2024 11:48:26 +0100 Subject: [PATCH 06/40] Remove dates from card block autofill because the timestamps don't look nice --- packages/builder/src/stores/builder/components.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js index 7f5d7a2022..7ee3db08cf 100644 --- a/packages/builder/src/stores/builder/components.js +++ b/packages/builder/src/stores/builder/components.js @@ -340,7 +340,6 @@ export class ComponentStore extends BudiStore { ...findFieldTypes(FieldType.STRING), ...findFieldTypes(FieldType.OPTIONS), ...findFieldTypes(FieldType.ARRAY), - ...findFieldTypes(FieldType.DATETIME), ...findFieldTypes(FieldType.NUMBER), ] const longFields = findFieldTypes(FieldType.LONGFORM) From 16e58a38eaec8f483f6e08b0044bcd5dd13befa2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 17 May 2024 10:51:40 +0200 Subject: [PATCH 07/40] Support pg time types --- packages/server/src/integrations/utils/utils.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts index 892d8ae034..466f539ef3 100644 --- a/packages/server/src/integrations/utils/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -71,7 +71,11 @@ const SQL_DATE_TYPE_MAP: Record = { } const SQL_DATE_ONLY_TYPES = ["date"] -const SQL_TIME_ONLY_TYPES = ["time"] +const SQL_TIME_ONLY_TYPES = [ + "time", + "time without time zone", + "time with time zone", +] const SQL_STRING_TYPE_MAP: Record = { varchar: FieldType.STRING, From a81626005c7b65c1321b462d861ca64dac977561 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 17 May 2024 15:55:27 +0200 Subject: [PATCH 08/40] Save timeonly on external db --- .../api/controllers/row/ExternalRequest.ts | 120 ++++++++++-------- 1 file changed, 64 insertions(+), 56 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index bd92413851..823330d601 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -1,3 +1,4 @@ +import dayjs from "dayjs" import { AutoFieldSubType, AutoReason, @@ -285,65 +286,72 @@ export class ExternalRequest { // parse floats/numbers if (field.type === FieldType.NUMBER && !isNaN(parseFloat(row[key]))) { newRow[key] = parseFloat(row[key]) - } - // if its not a link then just copy it over - if (field.type !== FieldType.LINK) { - newRow[key] = row[key] - continue - } - const { tableName: linkTableName } = breakExternalTableId(field?.tableId) - // table has to exist for many to many - if (!linkTableName || !this.tables[linkTableName]) { - continue - } - const linkTable = this.tables[linkTableName] - // @ts-ignore - const linkTablePrimary = linkTable.primary[0] - // one to many - if (isOneSide(field)) { - let id = row[key][0] - if (id) { - if (typeof row[key] === "string") { - id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1] - } - newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(id)[0] - } else { - // Removing from both new and row, as we don't know if it has already been processed - row[field.foreignKey || linkTablePrimary] = null - newRow[field.foreignKey || linkTablePrimary] = null + } else if (field.type === FieldType.LINK) { + const { tableName: linkTableName } = breakExternalTableId( + field?.tableId + ) + // table has to exist for many to many + if (!linkTableName || !this.tables[linkTableName]) { + continue } - } - // many to many - else if (isManyToMany(field)) { - // we're not inserting a doc, will be a bunch of update calls - const otherKey: string = field.throughFrom || linkTablePrimary - const thisKey: string = field.throughTo || tablePrimary - for (const relationship of row[key]) { - manyRelationships.push({ - tableId: field.through || field.tableId, - isUpdate: false, - key: otherKey, - [otherKey]: breakRowIdField(relationship)[0], - // leave the ID for enrichment later - [thisKey]: `{{ literal ${tablePrimary} }}`, - }) - } - } - // many to one - else { - const thisKey: string = "id" + const linkTable = this.tables[linkTableName] // @ts-ignore - const otherKey: string = field.fieldName - for (const relationship of row[key]) { - manyRelationships.push({ - tableId: field.tableId, - isUpdate: true, - key: otherKey, - [thisKey]: breakRowIdField(relationship)[0], - // leave the ID for enrichment later - [otherKey]: `{{ literal ${tablePrimary} }}`, - }) + const linkTablePrimary = linkTable.primary[0] + // one to many + if (isOneSide(field)) { + let id = row[key][0] + if (id) { + if (typeof row[key] === "string") { + id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1] + } + newRow[field.foreignKey || linkTablePrimary] = + breakRowIdField(id)[0] + } else { + // Removing from both new and row, as we don't know if it has already been processed + row[field.foreignKey || linkTablePrimary] = null + newRow[field.foreignKey || linkTablePrimary] = null + } } + // many to many + else if (isManyToMany(field)) { + // we're not inserting a doc, will be a bunch of update calls + const otherKey: string = field.throughFrom || linkTablePrimary + const thisKey: string = field.throughTo || tablePrimary + for (const relationship of row[key]) { + manyRelationships.push({ + tableId: field.through || field.tableId, + isUpdate: false, + key: otherKey, + [otherKey]: breakRowIdField(relationship)[0], + // leave the ID for enrichment later + [thisKey]: `{{ literal ${tablePrimary} }}`, + }) + } + } + // many to one + else { + const thisKey: string = "id" + // @ts-ignore + const otherKey: string = field.fieldName + for (const relationship of row[key]) { + manyRelationships.push({ + tableId: field.tableId, + isUpdate: true, + key: otherKey, + [thisKey]: breakRowIdField(relationship)[0], + // leave the ID for enrichment later + [otherKey]: `{{ literal ${tablePrimary} }}`, + }) + } + } + } else if ( + field.type === FieldType.DATETIME && + field.timeOnly && + row[key] + ) { + newRow[key] = dayjs(row[key]).format("HH:mm") + } else { + newRow[key] = row[key] } } // we return the relationships that may need to be created in the through table From 76ac300cf004d947272cc95a9745e4a03c3021ba Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 17 May 2024 16:53:28 +0200 Subject: [PATCH 09/40] Save only if valid --- packages/server/src/api/controllers/row/ExternalRequest.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 823330d601..b30c97e289 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -347,7 +347,8 @@ export class ExternalRequest { } else if ( field.type === FieldType.DATETIME && field.timeOnly && - row[key] + row[key] && + dayjs(row[key]).isValid() ) { newRow[key] = dayjs(row[key]).format("HH:mm") } else { From a74f82a5359a01cd0ba02b67d0dcf8660c9ae79a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 May 2024 13:06:45 +0200 Subject: [PATCH 10/40] Use native inputs --- .../Form/Core/DatePicker/NumberInput.svelte | 3 +- .../Form/Core/DatePicker/TimePicker.svelte | 37 +++++-------------- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte b/packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte index dc4886d28d..7e013341ac 100644 --- a/packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte +++ b/packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte @@ -4,13 +4,14 @@ export let max export let hideArrows = false export let width + export let type = "number" $: style = width ? `width:${width}px;` : "" - import { cleanInput } from "./utils" - import dayjs from "dayjs" import NumberInput from "./NumberInput.svelte" import { createEventDispatcher } from "svelte" @@ -8,39 +6,22 @@ const dispatch = createEventDispatcher() - $: displayValue = value || dayjs() + $: displayValue = value?.format("HH:mm") - const handleHourChange = e => { - dispatch("change", displayValue.hour(parseInt(e.target.value))) + const handleChange = e => { + const [hour, minute] = e.target.value.split(":").map(x => parseInt(x)) + dispatch("change", value.hour(hour).minute(minute)) } - - const handleMinuteChange = e => { - dispatch("change", displayValue.minute(parseInt(e.target.value))) - } - - const cleanHour = cleanInput({ max: 23, pad: 2, fallback: "00" }) - const cleanMinute = cleanInput({ max: 59, pad: 2, fallback: "00" })
- : -
From 5dc75582d9532042dc46db5acc76f08333a1bb1d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 May 2024 13:09:28 +0200 Subject: [PATCH 11/40] Fix timezone issues --- packages/bbui/src/helpers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/bbui/src/helpers.js b/packages/bbui/src/helpers.js index 90b447f3c1..66bc6551d8 100644 --- a/packages/bbui/src/helpers.js +++ b/packages/bbui/src/helpers.js @@ -166,7 +166,7 @@ export const stringifyDate = ( const offsetForTimezone = (enableTime && ignoreTimezones) || timeOnly if (offsetForTimezone) { // Ensure we use the correct offset for the date - const referenceDate = timeOnly ? new Date() : value.toDate() + const referenceDate = value.toDate() const offset = referenceDate.getTimezoneOffset() * 60000 return new Date(value.valueOf() - offset).toISOString().slice(0, -1) } From 995d2154ca3edf983b5993ee283be62e852126ae Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 20 May 2024 14:09:07 +0100 Subject: [PATCH 12/40] Included blob: in worker csp security header. Required for atrament to load --- hosting/proxy/nginx.prod.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf index 79007da311..12b8df049f 100644 --- a/hosting/proxy/nginx.prod.conf +++ b/hosting/proxy/nginx.prod.conf @@ -61,7 +61,7 @@ http { set $csp_img "img-src http: https: data: blob:"; set $csp_manifest "manifest-src 'self'"; set $csp_media "media-src 'self' https://js.intercomcdn.com https://cdn.budi.live"; - set $csp_worker "worker-src 'none'"; + set $csp_worker "worker-src blob:"; error_page 502 503 504 /error.html; location = /error.html { From 9f759220118d603b7a16259ca81fba4d6d2ce933 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 May 2024 16:31:11 +0200 Subject: [PATCH 13/40] SQL time only column creation as time --- packages/server/src/integrations/base/sqlTable.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts index a82a9fcea8..9871d57a3a 100644 --- a/packages/server/src/integrations/base/sqlTable.ts +++ b/packages/server/src/integrations/base/sqlTable.ts @@ -79,9 +79,13 @@ function generateSchema( schema.boolean(key) break case FieldType.DATETIME: - schema.datetime(key, { - useTz: !column.ignoreTimezones, - }) + if (!column.timeOnly) { + schema.datetime(key, { + useTz: !column.ignoreTimezones, + }) + } else { + schema.time(key) + } break case FieldType.ARRAY: case FieldType.BB_REFERENCE: From 57e73488b572a2834a01d717743acb046e51b7d0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 May 2024 16:34:00 +0200 Subject: [PATCH 14/40] Add tests --- .../src/api/routes/tests/search.spec.ts | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 87d0aa72c7..9f7209129a 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -962,6 +962,36 @@ describe.each([ }) }) + describe("datetime - time only", () => { + const T_1000 = "10:00" + const T_1045 = "10:45" + const T_1200 = "12:00" + const T_1530 = "15:30" + const T_0000 = "00:00" + + beforeAll(async () => { + await createTable({ + time: { name: "time", type: FieldType.DATETIME, timeOnly: true }, + }) + + await createRows( + _.shuffle([T_1000, T_1045, T_1200, T_1530, T_0000]).map(time => ({ + time, + })) + ) + }) + + describe("equal", () => { + it("successfully finds a row", () => + expectQuery({ equal: { time: T_1000 } }).toContainExactly([ + { time: "10:00:00" }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ equal: { time: "10:01" } }).toFindNothing()) + }) + }) + describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => { beforeAll(async () => { await createTable({ From 318dd5e6288d8fc70e5850c2805239112e51feab Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 May 2024 16:34:22 +0200 Subject: [PATCH 15/40] Fix time and mssql tests --- packages/server/src/integrations/base/sql.ts | 26 +++++++++++++++----- packages/server/src/utilities/schema.ts | 5 ++-- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 85db642e47..dd663dc918 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -122,11 +122,8 @@ function generateSelectStatement( const fieldNames = field.split(/\./g) const tableName = fieldNames[0] const columnName = fieldNames[1] - if ( - columnName && - schema?.[columnName] && - knex.client.config.client === SqlClient.POSTGRES - ) { + const columnSchema = schema?.[columnName] + if (columnSchema && knex.client.config.client === SqlClient.POSTGRES) { const externalType = schema[columnName].externalType if (externalType?.includes("money")) { return knex.raw( @@ -134,6 +131,13 @@ function generateSelectStatement( ) } } + if ( + knex.client.config.client === SqlClient.MS_SQL && + columnSchema.type === FieldType.DATETIME && + columnSchema.timeOnly + ) { + return knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`) + } return `${field} as ${field}` }) } @@ -634,13 +638,23 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { */ _query(json: QueryJson, opts: QueryOptions = {}): SqlQuery | SqlQuery[] { const sqlClient = this.getSqlClient() - const config: { client: string; useNullAsDefault?: boolean } = { + const config: Knex.Config = { client: sqlClient, } if (sqlClient === SqlClient.SQL_LITE) { config.useNullAsDefault = true } + + if (sqlClient === SqlClient.MS_SQL) { + // config.connection ??= {} + // config.connection.typeCast = (field: any, next: any): any => { + // if (field.type === "TIME") return field.string() + // return next() + // } + } + const client = knex(config) + let query: Knex.QueryBuilder const builder = new InternalBuilder(sqlClient) switch (this._operation(json)) { diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts index 4f0feb3c93..a205bf8c11 100644 --- a/packages/server/src/utilities/schema.ts +++ b/packages/server/src/utilities/schema.ts @@ -129,11 +129,12 @@ export function parse(rows: Rows, schema: TableSchema): Rows { return } - const { type: columnType } = schema[columnName] + const columnSchema = schema[columnName] + const { type: columnType } = columnSchema if (columnType === FieldType.NUMBER) { // If provided must be a valid number parsedRow[columnName] = columnData ? Number(columnData) : columnData - } else if (columnType === FieldType.DATETIME) { + } else if (columnType === FieldType.DATETIME && !columnSchema.timeOnly) { // If provided must be a valid date parsedRow[columnName] = columnData ? new Date(columnData).toISOString() From 684e3607157c576a3ee0cdbb3bae03e43b14ddd5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 May 2024 16:49:38 +0200 Subject: [PATCH 16/40] Add other tests --- .../src/api/routes/tests/search.spec.ts | 159 +++++++++++++++--- 1 file changed, 136 insertions(+), 23 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 9f7209129a..5e306739c9 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -962,35 +962,148 @@ describe.each([ }) }) - describe("datetime - time only", () => { - const T_1000 = "10:00" - const T_1045 = "10:45" - const T_1200 = "12:00" - const T_1530 = "15:30" - const T_0000 = "00:00" + !isInternal && + describe("datetime - time only", () => { + const T_1000 = "10:00" + const T_1045 = "10:45" + const T_1200 = "12:00" + const T_1530 = "15:30" + const T_0000 = "00:00" - beforeAll(async () => { - await createTable({ - time: { name: "time", type: FieldType.DATETIME, timeOnly: true }, + const UNEXISTING_TIME = "10:01" + + beforeAll(async () => { + await createTable({ + time: { name: "time", type: FieldType.DATETIME, timeOnly: true }, + }) + + await createRows( + _.shuffle([T_1000, T_1045, T_1200, T_1530, T_0000]).map(time => ({ + time, + })) + ) }) - await createRows( - _.shuffle([T_1000, T_1045, T_1200, T_1530, T_0000]).map(time => ({ - time, - })) - ) - }) + describe("equal", () => { + it("successfully finds a row", () => + expectQuery({ equal: { time: T_1000 } }).toContainExactly([ + { time: "10:00:00" }, + ])) - describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { time: T_1000 } }).toContainExactly([ - { time: "10:00:00" }, - ])) + it("fails to find nonexistent row", () => + expectQuery({ equal: { time: UNEXISTING_TIME } }).toFindNothing()) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { time: "10:01" } }).toFindNothing()) + describe("notEqual", () => { + it("successfully finds a row", () => + expectQuery({ notEqual: { time: T_1000 } }).toContainExactly([ + { time: "10:45:00" }, + { time: "12:00:00" }, + { time: "15:30:00" }, + { time: "00:00:00" }, + ])) + + it("return all when requesting non-existing", () => + expectQuery({ notEqual: { time: UNEXISTING_TIME } }).toContainExactly( + [ + { time: "10:00:00" }, + { time: "10:45:00" }, + { time: "12:00:00" }, + { time: "15:30:00" }, + { time: "00:00:00" }, + ] + )) + }) + + describe("oneOf", () => { + it("successfully finds a row", () => + expectQuery({ oneOf: { time: [T_1000] } }).toContainExactly([ + { time: "10:00:00" }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ oneOf: { time: [UNEXISTING_TIME] } }).toFindNothing()) + }) + + describe("range", () => { + it("successfully finds a row", () => + expectQuery({ + range: { time: { low: T_1045, high: T_1045 } }, + }).toContainExactly([{ time: "10:45:00" }])) + + it("successfully finds multiple rows", () => + expectQuery({ + range: { time: { low: T_1045, high: T_1530 } }, + }).toContainExactly([ + { time: "10:45:00" }, + { time: "12:00:00" }, + { time: "15:30:00" }, + ])) + + it("successfully finds no rows", () => + expectQuery({ + range: { time: { low: UNEXISTING_TIME, high: UNEXISTING_TIME } }, + }).toFindNothing()) + }) + + describe("sort", () => { + it("sorts ascending", () => + expectSearch({ + query: {}, + sort: "time", + sortOrder: SortOrder.ASCENDING, + }).toMatchExactly([ + { time: "00:00:00" }, + { time: "10:00:00" }, + { time: "10:45:00" }, + { time: "12:00:00" }, + { time: "15:30:00" }, + ])) + + it("sorts descending", () => + expectSearch({ + query: {}, + sort: "time", + sortOrder: SortOrder.DESCENDING, + }).toMatchExactly([ + { time: "15:30:00" }, + { time: "12:00:00" }, + { time: "10:45:00" }, + { time: "10:00:00" }, + { time: "00:00:00" }, + ])) + + describe("sortType STRING", () => { + it("sorts ascending", () => + expectSearch({ + query: {}, + sort: "time", + sortType: SortType.STRING, + sortOrder: SortOrder.ASCENDING, + }).toMatchExactly([ + { time: "00:00:00" }, + { time: "10:00:00" }, + { time: "10:45:00" }, + { time: "12:00:00" }, + { time: "15:30:00" }, + ])) + + it("sorts descending", () => + expectSearch({ + query: {}, + sort: "time", + sortType: SortType.STRING, + sortOrder: SortOrder.DESCENDING, + }).toMatchExactly([ + { time: "15:30:00" }, + { time: "12:00:00" }, + { time: "10:45:00" }, + { time: "10:00:00" }, + { time: "00:00:00" }, + ])) + }) + }) }) - }) describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => { beforeAll(async () => { From 7d709d0d221963234c1c3e97c6f3440d1f4c11d0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 May 2024 16:57:03 +0200 Subject: [PATCH 17/40] Fix flaky test --- packages/server/src/api/routes/tests/search.spec.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5e306739c9..fec0c4738e 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -407,8 +407,7 @@ describe.each([ }) it("should parse the encoded js binding. Return rows with appointments 2 weeks in the past", async () => { - const jsBinding = - "const currentTime = new Date()\ncurrentTime.setDate(currentTime.getDate()-14);\nreturn currentTime.toISOString();" + const jsBinding = `const currentTime = new Date(${Date.now()})\ncurrentTime.setDate(currentTime.getDate()-14);\nreturn currentTime.toISOString();` const encodedBinding = encodeJSBinding(jsBinding) await expectQuery({ From 028aaa0bb40d1788bb4539b1c681a82c6bf03234 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 May 2024 17:00:12 +0200 Subject: [PATCH 18/40] Clean --- packages/server/src/integrations/base/sql.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index dd663dc918..1b8d2ea4ae 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -645,14 +645,6 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { config.useNullAsDefault = true } - if (sqlClient === SqlClient.MS_SQL) { - // config.connection ??= {} - // config.connection.typeCast = (field: any, next: any): any => { - // if (field.type === "TIME") return field.string() - // return next() - // } - } - const client = knex(config) let query: Knex.QueryBuilder From 356da44b4b4133370fcd9b89e578d2ee144c9be2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 May 2024 17:00:51 +0200 Subject: [PATCH 19/40] Clean code --- packages/server/src/integrations/base/sql.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 1b8d2ea4ae..59efe9a9ba 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -646,7 +646,6 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { } const client = knex(config) - let query: Knex.QueryBuilder const builder = new InternalBuilder(sqlClient) switch (this._operation(json)) { From 1d00604674d0c85fc8489f2015c34ef0a6557a8b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 May 2024 17:06:34 +0200 Subject: [PATCH 20/40] Lint --- packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte index af38eaef7a..6f1490a573 100644 --- a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte +++ b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte @@ -31,10 +31,4 @@ flex-direction: row; align-items: center; } - .time-picker span { - font-weight: bold; - font-size: 18px; - z-index: 0; - margin-bottom: 1px; - } From 2b1df81649dde09fd9d8974720818133e9a669c7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 May 2024 11:24:04 +0200 Subject: [PATCH 21/40] Fix null references --- packages/server/src/integrations/base/sql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 59efe9a9ba..25f2d13a35 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -133,7 +133,7 @@ function generateSelectStatement( } if ( knex.client.config.client === SqlClient.MS_SQL && - columnSchema.type === FieldType.DATETIME && + columnSchema?.type === FieldType.DATETIME && columnSchema.timeOnly ) { return knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`) From d97f3b03781e796c2eff7288de823bb6bea9baa4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 May 2024 11:54:42 +0200 Subject: [PATCH 22/40] Handle undefineds --- packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte index 6f1490a573..14f3559e9a 100644 --- a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte +++ b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte @@ -9,6 +9,11 @@ $: displayValue = value?.format("HH:mm") const handleChange = e => { + if (!e.target.value) { + dispatch("change", undefined) + return + } + const [hour, minute] = e.target.value.split(":").map(x => parseInt(x)) dispatch("change", value.hour(hour).minute(minute)) } From fc99fad3d2e85a0090724150ea9cb541ddd65bcb Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 May 2024 11:54:55 +0200 Subject: [PATCH 23/40] Fix display in chromium --- packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte | 3 +++ packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte b/packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte index 7e013341ac..6c06ce4e79 100644 --- a/packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte +++ b/packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte @@ -52,4 +52,7 @@ input.hide-arrows { -moz-appearance: textfield; } + input[type="time"]::-webkit-calendar-picker-indicator { + display: none; + } diff --git a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte index 14f3559e9a..e1ea4d625b 100644 --- a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte +++ b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte @@ -24,7 +24,6 @@ hideArrows type={"time"} value={displayValue} - width={60} on:input={handleChange} on:change={handleChange} /> From 6bd984ab9cef22d036ee43fd8f4245a451bc247d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 21 May 2024 11:51:42 +0100 Subject: [PATCH 24/40] Making progress towards deleting sql.spec.ts. --- .../src/api/routes/tests/search.spec.ts | 81 +++ .../server/src/integrations/tests/sql.spec.ts | 537 ------------------ .../server/src/sdk/app/rows/search/sqs.ts | 11 +- .../app/rows/search/tests/external.spec.ts | 166 ------ .../sdk/app/rows/search/tests/lucene.spec.ts | 249 -------- .../sdk/app/rows/search/tests/search.spec.ts | 131 +++++ .../server/src/tests/utilities/api/row.ts | 17 +- 7 files changed, 234 insertions(+), 958 deletions(-) delete mode 100644 packages/server/src/sdk/app/rows/search/tests/external.spec.ts delete mode 100644 packages/server/src/sdk/app/rows/search/tests/lucene.spec.ts create mode 100644 packages/server/src/sdk/app/rows/search/tests/search.spec.ts diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 0dc244cfee..2ddc6ef1ff 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -16,6 +16,7 @@ import { Table, TableSchema, User, + Row, } from "@budibase/types" import _ from "lodash" import tk from "timekeeper" @@ -629,6 +630,19 @@ describe.each([ it("fails to find nonexistent row", () => expectQuery({ equal: { name: "none" } }).toFindNothing()) + + it("works as an or condition", () => + expectQuery({ + allOr: true, + equal: { name: "foo" }, + oneOf: { name: ["bar"] }, + }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + + it("can have multiple values for same column", () => + expectQuery({ + allOr: true, + equal: { "1:name": "foo", "2:name": "bar" }, + }).toContainExactly([{ name: "foo" }, { name: "bar" }])) }) describe("notEqual", () => { @@ -663,6 +677,21 @@ describe.each([ expectQuery({ fuzzy: { name: "none" } }).toFindNothing()) }) + describe("string", () => { + it("successfully finds a row", () => + expectQuery({ string: { name: "fo" } }).toContainExactly([ + { name: "foo" }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ string: { name: "none" } }).toFindNothing()) + + it("is case-insensitive", () => + expectQuery({ string: { name: "FO" } }).toContainExactly([ + { name: "foo" }, + ])) + }) + describe("range", () => { it("successfully finds multiple rows", () => expectQuery({ @@ -1267,5 +1296,57 @@ describe.each([ { auto: 1 }, ])) }) + + // TODO(samwho): fix for SQS + !isSqs && + describe("pagination", () => { + it("should paginate through all rows", async () => { + // @ts-ignore + let bookmark: string | number = undefined + let rows: Row[] = [] + + // eslint-disable-next-line no-constant-condition + while (true) { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + limit: 3, + query: {}, + bookmark, + paginate: true, + }) + + rows.push(...response.rows) + + if (!response.bookmark || !response.hasNextPage) { + break + } + bookmark = response.bookmark + } + + expect(rows).toHaveLength(10) + expect(rows.map(row => row.auto)).toEqual( + expect.arrayContaining([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + ) + }) + }) }) + + describe("field name 1:name", () => { + beforeAll(async () => { + await createTable({ + "1:name": { name: "1:name", type: FieldType.STRING }, + }) + await createRows([{ "1:name": "bar" }, { "1:name": "foo" }]) + }) + + describe("equal", () => { + it("successfully finds a row", () => + expectQuery({ equal: { "1:1:name": "bar" } }).toContainExactly([ + { "1:name": "bar" }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ equal: { "1:1:name": "none" } }).toFindNothing()) + }) + }) }) diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index 5de9cc4fbc..78f6cd5d57 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -66,38 +66,6 @@ function generateCreateJson(table = TABLE_NAME, body = {}): QueryJson { } } -function generateUpdateJson({ - table = TABLE_NAME, - body = {}, - filters = {}, - meta = {}, -}: { - table: string - body?: any - filters?: any - meta?: any -}): QueryJson { - if (!meta.table) { - meta.table = TABLE - } - return { - endpoint: endpoint(table, "UPDATE"), - filters, - body, - meta, - } -} - -function generateDeleteJson(table = TABLE_NAME, filters = {}): QueryJson { - return { - endpoint: endpoint(table, "DELETE"), - meta: { - table: TABLE, - }, - filters, - } -} - function generateRelationshipJson(config: { schema?: string } = {}): QueryJson { return { endpoint: { @@ -178,81 +146,6 @@ describe("SQL query builder", () => { sql = new Sql(client, limit) }) - it("should test a basic read", () => { - const query = sql._query(generateReadJson()) - expect(query).toEqual({ - bindings: [limit], - sql: `select * from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"`, - }) - }) - - it("should test a read with specific columns", () => { - const nameProp = `${TABLE_NAME}.name`, - ageProp = `${TABLE_NAME}.age` - const query = sql._query( - generateReadJson({ - fields: [nameProp, ageProp], - }) - ) - expect(query).toEqual({ - bindings: [limit], - sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"`, - }) - }) - - it("should test a where string starts with read", () => { - const query = sql._query( - generateReadJson({ - filters: { - string: { - name: "John", - }, - }, - }) - ) - expect(query).toEqual({ - bindings: ["John%", limit], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2) as "${TABLE_NAME}"`, - }) - }) - - it("should test a where range read", () => { - const query = sql._query( - generateReadJson({ - filters: { - range: { - age: { - low: 2, - high: 10, - }, - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [2, 10, limit], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3) as "${TABLE_NAME}"`, - }) - }) - - it("should test for multiple IDs with OR", () => { - const query = sql._query( - generateReadJson({ - filters: { - equal: { - age: 10, - name: "John", - }, - allOr: true, - }, - }) - ) - expect(query).toEqual({ - bindings: [10, "John", limit], - sql: `select * from (select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3) as "${TABLE_NAME}"`, - }) - }) - it("should allow filtering on a related field", () => { const query = sql._query( generateReadJson({ @@ -271,260 +164,6 @@ describe("SQL query builder", () => { }) }) - it("should test an create statement", () => { - const query = sql._query( - generateCreateJson(TABLE_NAME, { - name: "Michael", - age: 45, - }) - ) - expect(query).toEqual({ - bindings: [45, "Michael"], - sql: `insert into "${TABLE_NAME}" ("age", "name") values ($1, $2) returning *`, - }) - }) - - it("should test an update statement", () => { - const query = sql._query( - generateUpdateJson({ - table: TABLE_NAME, - body: { - name: "John", - }, - filters: { - equal: { - id: 1001, - }, - }, - }) - ) - expect(query).toEqual({ - bindings: ["John", 1001], - sql: `update "${TABLE_NAME}" set "name" = $1 where "${TABLE_NAME}"."id" = $2 returning *`, - }) - }) - - it("should test a delete statement", () => { - const query = sql._query( - generateDeleteJson(TABLE_NAME, { - equal: { - id: 1001, - }, - }) - ) - expect(query).toEqual({ - bindings: [1001], - sql: `delete from "${TABLE_NAME}" where "${TABLE_NAME}"."id" = $1 returning *`, - }) - }) - - it("should work with MS-SQL", () => { - const query = new Sql(SqlClient.MS_SQL, 10)._query(generateReadJson()) - expect(query).toEqual({ - bindings: [10], - sql: `select * from (select top (@p0) * from [${TABLE_NAME}]) as [${TABLE_NAME}]`, - }) - }) - - it("should work with MySQL", () => { - const query = new Sql(SqlClient.MY_SQL, 10)._query(generateReadJson()) - expect(query).toEqual({ - bindings: [10], - sql: `select * from (select * from \`${TABLE_NAME}\` limit ?) as \`${TABLE_NAME}\``, - }) - }) - - it("should use greater than when only low range specified", () => { - const date = new Date() - const query = sql._query( - generateReadJson({ - filters: { - range: { - property: { - low: date, - }, - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [date, limit], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" >= $1 limit $2) as "${TABLE_NAME}"`, - }) - }) - - it("should use less than when only high range specified", () => { - const date = new Date() - const query = sql._query( - generateReadJson({ - filters: { - range: { - property: { - high: date, - }, - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [date, limit], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" <= $1 limit $2) as "${TABLE_NAME}"`, - }) - }) - - it("should use AND like expression for MS-SQL when filter is contains", () => { - const query = new Sql(SqlClient.MS_SQL, 10)._query( - generateReadJson({ - filters: { - contains: { - age: [20, 25], - name: ["John", "Mary"], - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [10, "%20%", "%25%", `%"john"%`, `%"mary"%`], - sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where (LOWER([${TABLE_NAME}].[age]) LIKE @p1 AND LOWER([${TABLE_NAME}].[age]) LIKE @p2) and (LOWER([${TABLE_NAME}].[name]) LIKE @p3 AND LOWER([${TABLE_NAME}].[name]) LIKE @p4)) as [${TABLE_NAME}]`, - }) - }) - - it("should use JSON_CONTAINS expression for MySQL when filter is contains", () => { - const query = new Sql(SqlClient.MY_SQL, 10)._query( - generateReadJson({ - filters: { - contains: { - age: [20], - name: ["John"], - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [10], - sql: `select * from (select * from \`${TABLE_NAME}\` where JSON_CONTAINS(${TABLE_NAME}.age, '[20]') and JSON_CONTAINS(${TABLE_NAME}.name, '["John"]') limit ?) as \`${TABLE_NAME}\``, - }) - }) - - it("should use jsonb operator expression for PostgreSQL when filter is contains", () => { - const query = new Sql(SqlClient.POSTGRES, 10)._query( - generateReadJson({ - filters: { - contains: { - age: [20], - name: ["John"], - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [10], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age"::jsonb @> '[20]' and "${TABLE_NAME}"."name"::jsonb @> '["John"]' limit $1) as "${TABLE_NAME}"`, - }) - }) - - it("should use NOT like expression for MS-SQL when filter is notContains", () => { - const query = new Sql(SqlClient.MS_SQL, 10)._query( - generateReadJson({ - filters: { - notContains: { - age: [20], - name: ["John"], - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [10, "%20%", `%"john"%`], - sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where NOT (LOWER([${TABLE_NAME}].[age]) LIKE @p1) and NOT (LOWER([${TABLE_NAME}].[name]) LIKE @p2)) as [${TABLE_NAME}]`, - }) - }) - - it("should use NOT JSON_CONTAINS expression for MySQL when filter is notContains", () => { - const query = new Sql(SqlClient.MY_SQL, 10)._query( - generateReadJson({ - filters: { - notContains: { - age: [20], - name: ["John"], - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [10], - sql: `select * from (select * from \`${TABLE_NAME}\` where NOT JSON_CONTAINS(${TABLE_NAME}.age, '[20]') and NOT JSON_CONTAINS(${TABLE_NAME}.name, '["John"]') limit ?) as \`${TABLE_NAME}\``, - }) - }) - - it("should use jsonb operator NOT expression for PostgreSQL when filter is notContains", () => { - const query = new Sql(SqlClient.POSTGRES, 10)._query( - generateReadJson({ - filters: { - notContains: { - age: [20], - name: ["John"], - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [10], - sql: `select * from (select * from "${TABLE_NAME}" where NOT "${TABLE_NAME}"."age"::jsonb @> '[20]' and NOT "${TABLE_NAME}"."name"::jsonb @> '["John"]' limit $1) as "${TABLE_NAME}"`, - }) - }) - - it("should use OR like expression for MS-SQL when filter is containsAny", () => { - const query = new Sql(SqlClient.MS_SQL, 10)._query( - generateReadJson({ - filters: { - containsAny: { - age: [20, 25], - name: ["John", "Mary"], - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [10, "%20%", "%25%", `%"john"%`, `%"mary"%`], - sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where (LOWER([${TABLE_NAME}].[age]) LIKE @p1 OR LOWER([${TABLE_NAME}].[age]) LIKE @p2) and (LOWER([${TABLE_NAME}].[name]) LIKE @p3 OR LOWER([${TABLE_NAME}].[name]) LIKE @p4)) as [${TABLE_NAME}]`, - }) - }) - - it("should use JSON_OVERLAPS expression for MySQL when filter is containsAny", () => { - const query = new Sql(SqlClient.MY_SQL, 10)._query( - generateReadJson({ - filters: { - containsAny: { - age: [20, 25], - name: ["John", "Mary"], - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [10], - sql: `select * from (select * from \`${TABLE_NAME}\` where JSON_OVERLAPS(${TABLE_NAME}.age, '[20,25]') and JSON_OVERLAPS(${TABLE_NAME}.name, '["John","Mary"]') limit ?) as \`${TABLE_NAME}\``, - }) - }) - - it("should use ?| operator expression for PostgreSQL when filter is containsAny", () => { - const query = new Sql(SqlClient.POSTGRES, 10)._query( - generateReadJson({ - filters: { - containsAny: { - age: [20, 25], - name: ["John", "Mary"], - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [10], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age"::jsonb ?| array [20,25] and "${TABLE_NAME}"."name"::jsonb ?| array ['John','Mary'] limit $1) as "${TABLE_NAME}"`, - }) - }) - it("should add the schema to the LEFT JOIN", () => { const query = sql._query(generateRelationshipJson({ schema: "production" })) expect(query).toEqual({ @@ -551,42 +190,6 @@ describe("SQL query builder", () => { }) }) - it("should handle table names with dashes when performing a LIKE in MySQL", () => { - const tableName = "Table-Name-With-Dashes" - const query = new Sql(SqlClient.MY_SQL, limit)._query( - generateReadJson({ - table: tableName, - filters: { - string: { - name: "John", - }, - }, - }) - ) - expect(query).toEqual({ - bindings: ["john%", limit], - sql: `select * from (select * from \`${tableName}\` where LOWER(\`${tableName}\`.\`name\`) LIKE ? limit ?) as \`${tableName}\``, - }) - }) - - it("should handle table names with dashes when performing a LIKE in SQL Server", () => { - const tableName = "Table-Name-With-Dashes" - const query = new Sql(SqlClient.MS_SQL, limit)._query( - generateReadJson({ - table: tableName, - filters: { - string: { - name: "John", - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [limit, "john%"], - sql: `select * from (select top (@p0) * from [${tableName}] where LOWER([${tableName}].[name]) LIKE @p1) as [${tableName}]`, - }) - }) - it("should ignore high range value if it is an empty object", () => { const query = sql._query( generateReadJson({ @@ -625,51 +228,6 @@ describe("SQL query builder", () => { }) }) - it("should lowercase the values for Oracle LIKE statements", () => { - let query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - filters: { - string: { - name: "John", - }, - }, - }) - ) - expect(query).toEqual({ - bindings: ["john%", limit], - sql: `select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1) where rownum <= :2) "test"`, - }) - - query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - filters: { - contains: { - age: [20, 25], - name: ["John", "Mary"], - }, - }, - }) - ) - expect(query).toEqual({ - bindings: ["%20%", "%25%", `%"john"%`, `%"mary"%`, limit], - sql: `select * from (select * from (select * from "test" where (LOWER("test"."age") LIKE :1 AND LOWER("test"."age") LIKE :2) and (LOWER("test"."name") LIKE :3 AND LOWER("test"."name") LIKE :4)) where rownum <= :5) "test"`, - }) - - query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - filters: { - fuzzy: { - name: "Jo", - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [`%jo%`, limit], - sql: `select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1) where rownum <= :2) "test"`, - }) - }) - it("should sort SQL Server tables by the primary key if no sort data is provided", () => { let query = new Sql(SqlClient.MS_SQL, limit)._query( generateReadJson({ @@ -709,99 +267,4 @@ describe("SQL query builder", () => { sql: `insert into "test" ("name") values ($1) returning *`, }) }) - - it("should be able to rename column for MySQL", () => { - const table: Table = { - type: "table", - sourceType: TableSourceType.EXTERNAL, - name: TABLE_NAME, - schema: { - first_name: { - type: FieldType.STRING, - name: "first_name", - externalType: "varchar(45)", - }, - }, - sourceId: "SOURCE_ID", - } - const oldTable: Table = { - ...table, - schema: { - name: { - type: FieldType.STRING, - name: "name", - externalType: "varchar(45)", - }, - }, - } - const query = new Sql(SqlClient.MY_SQL, limit)._query({ - table, - endpoint: { - datasourceId: "MySQL", - operation: Operation.UPDATE_TABLE, - entityId: TABLE_NAME, - }, - meta: { - table: oldTable, - tables: { [oldTable.name]: oldTable }, - renamed: { - old: "name", - updated: "first_name", - }, - }, - }) - expect(query).toEqual({ - bindings: [], - sql: `alter table \`${TABLE_NAME}\` rename column \`name\` to \`first_name\`;`, - }) - }) - - it("should be able to delete a column", () => { - const table: Table = { - type: "table", - sourceType: TableSourceType.EXTERNAL, - name: TABLE_NAME, - schema: { - first_name: { - type: FieldType.STRING, - name: "first_name", - externalType: "varchar(45)", - }, - }, - sourceId: "SOURCE_ID", - } - const oldTable: Table = { - ...table, - schema: { - first_name: { - type: FieldType.STRING, - name: "first_name", - externalType: "varchar(45)", - }, - last_name: { - type: FieldType.STRING, - name: "last_name", - externalType: "varchar(45)", - }, - }, - } - const query = sql._query({ - table, - endpoint: { - datasourceId: "Postgres", - operation: Operation.UPDATE_TABLE, - entityId: TABLE_NAME, - }, - meta: { - table: oldTable, - tables: [oldTable], - }, - }) - expect(query).toEqual([ - { - bindings: [], - sql: `alter table "${TABLE_NAME}" drop column "last_name"`, - }, - ]) - }) }) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index a94ce265c5..c4dc408cac 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -26,6 +26,7 @@ import { } from "../../../../db/utils" import AliasTables from "../sqlAlias" import { outputProcessing } from "../../../../utilities/rowProcessor" +import pick from "lodash/pick" function buildInternalFieldList( table: Table, @@ -186,13 +187,19 @@ export async function search( } ) - return { - // final row processing for response + const output = { rows: await outputProcessing(table, processed, { preserveLinks: true, squash: true, }), } + + if (options.fields) { + const fields = [...options.fields, ...CONSTANT_INTERNAL_ROW_COLS] + output.rows = output.rows.map((r: any) => pick(r, fields)) + } + + return output } catch (err: any) { const msg = typeof err === "string" ? err : err.message if (err.status === 404 && err.message?.includes(SQLITE_DESIGN_DOC_ID)) { diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts deleted file mode 100644 index 53bc049a9b..0000000000 --- a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { GenericContainer } from "testcontainers" - -import { - Datasource, - FieldType, - Row, - SourceName, - Table, - RowSearchParams, - TableSourceType, -} from "@budibase/types" - -import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" -import { search } from "../external" -import { - expectAnyExternalColsAttributes, - generator, -} from "@budibase/backend-core/tests" - -describe("external search", () => { - const config = new TestConfiguration() - - let externalDatasource: Datasource, tableData: Table - const rows: Row[] = [] - - beforeAll(async () => { - const container = await new GenericContainer("mysql:8.3") - .withExposedPorts(3306) - .withEnvironment({ - MYSQL_ROOT_PASSWORD: "admin", - MYSQL_DATABASE: "db", - MYSQL_USER: "user", - MYSQL_PASSWORD: "password", - }) - .start() - - const host = container.getHost() - const port = container.getMappedPort(3306) - - await config.init() - - externalDatasource = await config.createDatasource({ - datasource: { - type: "datasource", - name: "Test", - source: SourceName.MYSQL, - plus: true, - config: { - host, - port, - user: "user", - database: "db", - password: "password", - rejectUnauthorized: true, - }, - }, - }) - - tableData = { - name: generator.word(), - type: "table", - primary: ["id"], - sourceId: externalDatasource._id!, - sourceType: TableSourceType.EXTERNAL, - schema: { - id: { - name: "id", - type: FieldType.AUTO, - autocolumn: true, - }, - name: { - name: "name", - type: FieldType.STRING, - }, - surname: { - name: "surname", - type: FieldType.STRING, - }, - age: { - name: "age", - type: FieldType.NUMBER, - }, - address: { - name: "address", - type: FieldType.STRING, - }, - }, - } - - const table = await config.createExternalTable({ - ...tableData, - sourceId: externalDatasource._id, - }) - for (let i = 0; i < 10; i++) { - rows.push( - await config.createRow({ - tableId: table._id, - name: generator.first(), - surname: generator.last(), - age: generator.age(), - address: generator.address(), - }) - ) - } - }) - - it("default search returns all the data", async () => { - await config.doInContext(config.appId, async () => { - const tableId = config.table!._id! - - const searchParams: RowSearchParams = { - tableId, - query: {}, - } - const result = await search(searchParams, config.table!) - - expect(result.rows).toHaveLength(10) - expect(result.rows).toEqual( - expect.arrayContaining(rows.map(r => expect.objectContaining(r))) - ) - }) - }) - - it("querying by fields will always return data attribute columns", async () => { - await config.doInContext(config.appId, async () => { - const tableId = config.table!._id! - - const searchParams: RowSearchParams = { - tableId, - query: {}, - fields: ["name", "age"], - } - const result = await search(searchParams, config.table!) - - expect(result.rows).toHaveLength(10) - expect(result.rows).toEqual( - expect.arrayContaining( - rows.map(r => ({ - ...expectAnyExternalColsAttributes, - name: r.name, - age: r.age, - })) - ) - ) - }) - }) - - it("will decode _id in oneOf query", async () => { - await config.doInContext(config.appId, async () => { - const tableId = config.table!._id! - - const searchParams: RowSearchParams = { - tableId, - query: { - oneOf: { - _id: ["%5B1%5D", "%5B4%5D", "%5B8%5D"], - }, - }, - } - const result = await search(searchParams, config.table!) - - expect(result.rows).toHaveLength(3) - expect(result.rows.map(row => row.id)).toEqual([1, 4, 8]) - }) - }) -}) diff --git a/packages/server/src/sdk/app/rows/search/tests/lucene.spec.ts b/packages/server/src/sdk/app/rows/search/tests/lucene.spec.ts deleted file mode 100644 index d9c1c79177..0000000000 --- a/packages/server/src/sdk/app/rows/search/tests/lucene.spec.ts +++ /dev/null @@ -1,249 +0,0 @@ -const nodeFetch = require("node-fetch") - -nodeFetch.mockSearch() -import * as search from "../utils" -import { RowSearchParams, SortOrder, SortType } from "@budibase/types" - -// this will be mocked out for _search endpoint -const PARAMS: RowSearchParams = { - query: {}, - tableId: "ta_12345679abcdef", - version: "1", - bookmark: undefined, - sort: undefined, - sortOrder: SortOrder.ASCENDING, - sortType: SortType.STRING, -} - -function checkLucene(resp: any, expected: any, params = PARAMS) { - const query = resp.rows[0].query - const json = JSON.parse(query) - if (PARAMS.sort) { - expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`) - } - if (PARAMS.bookmark) { - expect(json.bookmark).toBe(PARAMS.bookmark) - } - expect(json.include_docs).toBe(true) - expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`) - expect(json.limit).toBe(params.limit || 50) -} - -describe("internal search", () => { - it("default query", async () => { - const response = await search.paginatedSearch({}, PARAMS) - checkLucene(response, `*:*`) - }) - - it("test equal query", async () => { - const response = await search.paginatedSearch( - { - equal: { - column: "1", - }, - }, - PARAMS - ) - checkLucene(response, `*:* AND column:"1"`) - }) - - it("test notEqual query", async () => { - const response = await search.paginatedSearch( - { - notEqual: { - column: "1", - }, - }, - PARAMS - ) - checkLucene(response, `*:* AND !column:"1"`) - }) - - it("test OR query", async () => { - const response = await search.paginatedSearch( - { - allOr: true, - equal: { - column: "2", - }, - notEqual: { - column: "1", - }, - }, - PARAMS - ) - checkLucene(response, `(column:"2" OR !column:"1")`) - }) - - it("test AND query", async () => { - const response = await search.paginatedSearch( - { - equal: { - column: "2", - }, - notEqual: { - column: "1", - }, - }, - PARAMS - ) - checkLucene(response, `(*:* AND column:"2" AND !column:"1")`) - }) - - it("test pagination query", async () => { - const updatedParams = { - ...PARAMS, - limit: 100, - bookmark: "awd", - sort: "column", - } - const response = await search.paginatedSearch( - { - string: { - column: "2", - }, - }, - updatedParams - ) - checkLucene(response, `*:* AND column:2*`, updatedParams) - }) - - it("test range query", async () => { - const response = await search.paginatedSearch( - { - range: { - column: { low: 1, high: 2 }, - }, - }, - PARAMS - ) - checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS) - }) - - it("test empty query", async () => { - const response = await search.paginatedSearch( - { - empty: { - column: "", - }, - }, - PARAMS - ) - checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS) - }) - - it("test notEmpty query", async () => { - const response = await search.paginatedSearch( - { - notEmpty: { - column: "", - }, - }, - PARAMS - ) - checkLucene(response, `*:* AND column:["" TO *]`, PARAMS) - }) - - it("test oneOf query", async () => { - const response = await search.paginatedSearch( - { - oneOf: { - column: ["a", "b"], - }, - }, - PARAMS - ) - checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS) - }) - - it("test contains query", async () => { - const response = await search.paginatedSearch( - { - contains: { - column: ["a"], - colArr: [1, 2, 3], - }, - }, - PARAMS - ) - checkLucene( - response, - `(*:* AND column:(a) AND colArr:(1 AND 2 AND 3))`, - PARAMS - ) - }) - - it("test multiple of same column", async () => { - const response = await search.paginatedSearch( - { - allOr: true, - equal: { - "1:column": "a", - "2:column": "b", - "3:column": "c", - }, - }, - PARAMS - ) - checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS) - }) - - it("check a weird case for lucene building", async () => { - const response = await search.paginatedSearch( - { - equal: { - "1:1:column": "a", - }, - }, - PARAMS - ) - checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS) - }) - - it("test containsAny query", async () => { - const response = await search.paginatedSearch( - { - containsAny: { - column: ["a", "b", "c"], - }, - }, - PARAMS - ) - checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS) - }) - - it("test notContains query", async () => { - const response = await search.paginatedSearch( - { - notContains: { - column: ["a", "b", "c"], - }, - }, - PARAMS - ) - checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS) - }) - - it("test equal without version query", async () => { - PARAMS.version = undefined - const response = await search.paginatedSearch( - { - equal: { - column: "1", - }, - }, - PARAMS - ) - - const query = response.rows[0].query - const json = JSON.parse(query) - if (PARAMS.sort) { - expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`) - } - if (PARAMS.bookmark) { - expect(json.bookmark).toBe(PARAMS.bookmark) - } - expect(json.include_docs).toBe(true) - expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`) - }) -}) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts new file mode 100644 index 0000000000..2f347475f4 --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -0,0 +1,131 @@ +import { Datasource, FieldType, Row, Table } from "@budibase/types" + +import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" +import { search } from "../../../../../sdk/app/rows/search" +import { generator } from "@budibase/backend-core/tests" +import { + DatabaseName, + getDatasource, +} from "../../../../../integrations/tests/utils" +import { tableForDatasource } from "../../../../../tests/utilities/structures" + +// These test cases are only for things that cannot be tested through the API +// (e.g. limiting searches to returning specific fields). If it's possible to +// test through the API, it should be done there instead. +describe.each([ + ["lucene", undefined], + ["sqs", undefined], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], +])("search sdk (%s)", (name, dsProvider) => { + const isSqs = name === "sqs" + const isLucene = name === "lucene" + const isInternal = isLucene || isSqs + const config = new TestConfiguration() + + let envCleanup: (() => void) | undefined + let datasource: Datasource | undefined + let table: Table + let rows: Row[] + + beforeAll(async () => { + if (isSqs) { + envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" }) + } + await config.init() + + if (dsProvider) { + datasource = await config.createDatasource({ + datasource: await dsProvider, + }) + } + + table = await config.api.table.save( + tableForDatasource(datasource, { + primary: ["id"], + schema: { + id: { + name: "id", + type: FieldType.AUTO, + autocolumn: true, + }, + name: { + name: "name", + type: FieldType.STRING, + }, + surname: { + name: "surname", + type: FieldType.STRING, + }, + age: { + name: "age", + type: FieldType.NUMBER, + }, + address: { + name: "address", + type: FieldType.STRING, + }, + }, + }) + ) + + rows = [] + for (let i = 0; i < 10; i++) { + rows.push( + await config.api.row.save(table._id!, { + name: generator.first(), + surname: generator.last(), + age: generator.age(), + address: generator.address(), + }) + ) + } + }) + + afterAll(async () => { + config.end() + if (envCleanup) { + envCleanup() + } + }) + + it("querying by fields will always return data attribute columns", async () => { + await config.doInContext(config.appId, async () => { + const { rows } = await search({ + tableId: table._id!, + query: {}, + fields: ["name", "age"], + }) + + expect(rows).toHaveLength(10) + for (const row of rows) { + const keys = Object.keys(row) + expect(keys).toContain("name") + expect(keys).toContain("age") + expect(keys).not.toContain("surname") + expect(keys).not.toContain("address") + } + }) + }) + + !isInternal && + it("will decode _id in oneOf query", async () => { + await config.doInContext(config.appId, async () => { + const result = await search({ + tableId: table._id!, + query: { + oneOf: { + _id: ["%5B1%5D", "%5B4%5D", "%5B8%5D"], + }, + }, + }) + + expect(result.rows).toHaveLength(3) + expect(result.rows.map(row => row.id)).toEqual( + expect.arrayContaining([1, 4, 8]) + ) + }) + }) +}) diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index fc40262363..17d21f0996 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -10,6 +10,7 @@ import { RowSearchParams, DeleteRows, DeleteRow, + PaginatedSearchRowResponse, } from "@budibase/types" import { Expectations, TestAPI } from "./base" @@ -133,12 +134,20 @@ export class RowAPI extends TestAPI { ) } - search = async ( + search = async ( sourceId: string, - params?: RowSearchParams, + params?: T, expectations?: Expectations - ): Promise => { - return await this._post(`/api/${sourceId}/search`, { + ): Promise< + T extends { paginate: true } + ? PaginatedSearchRowResponse + : SearchRowResponse + > => { + return await this._post< + T extends { paginate: true } + ? PaginatedSearchRowResponse + : SearchRowResponse + >(`/api/${sourceId}/search`, { body: params, expectations, }) From 221c8a3f0a7734bd53446855a23e0226cc5eb666 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 May 2024 13:31:15 +0200 Subject: [PATCH 25/40] Shuffle all test createRows --- .../server/src/api/routes/tests/search.spec.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 2a5890decf..62d55e7e32 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -78,7 +78,7 @@ describe.each([ } async function createRows(rows: Record[]) { - await config.api.row.bulkImport(table._id!, { rows }) + await config.api.row.bulkImport(table._id!, { rows: _.shuffle(rows) }) } class SearchAssertion { @@ -981,11 +981,13 @@ describe.each([ time: { name: "time", type: FieldType.DATETIME, timeOnly: true }, }) - await createRows( - _.shuffle([T_1000, T_1045, T_1200, T_1530, T_0000]).map(time => ({ - time, - })) - ) + await createRows([ + { time: T_1000 }, + { time: T_1045 }, + { time: T_1200 }, + { time: T_1530 }, + { time: T_0000 }, + ]) }) describe("equal", () => { From 7c7f88bd47a3b9b776a3efcef40ad946e54fb8a8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 May 2024 13:41:46 +0200 Subject: [PATCH 26/40] Pop on asserts --- .../server/src/api/routes/tests/search.spec.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 62d55e7e32..74eca789cb 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -84,7 +84,7 @@ describe.each([ class SearchAssertion { constructor(private readonly query: RowSearchParams) {} - private findRow(expectedRow: any, foundRows: any[]) { + private popRow(expectedRow: any, foundRows: any[]) { const row = foundRows.find(foundRow => _.isMatch(foundRow, expectedRow)) if (!row) { const fields = Object.keys(expectedRow) @@ -97,6 +97,9 @@ describe.each([ )} in ${JSON.stringify(searchedObjects)}` ) } + + // Ensuring the same row is not matched twice + foundRows.splice(foundRows.indexOf(row), 1) return row } @@ -113,9 +116,9 @@ describe.each([ // eslint-disable-next-line jest/no-standalone-expect expect(foundRows).toHaveLength(expectedRows.length) // eslint-disable-next-line jest/no-standalone-expect - expect(foundRows).toEqual( + expect([...foundRows]).toEqual( expectedRows.map((expectedRow: any) => - expect.objectContaining(this.findRow(expectedRow, foundRows)) + expect.objectContaining(this.popRow(expectedRow, foundRows)) ) ) } @@ -132,10 +135,10 @@ describe.each([ // eslint-disable-next-line jest/no-standalone-expect expect(foundRows).toHaveLength(expectedRows.length) // eslint-disable-next-line jest/no-standalone-expect - expect(foundRows).toEqual( + expect([...foundRows]).toEqual( expect.arrayContaining( expectedRows.map((expectedRow: any) => - expect.objectContaining(this.findRow(expectedRow, foundRows)) + expect.objectContaining(this.popRow(expectedRow, foundRows)) ) ) ) @@ -151,10 +154,10 @@ describe.each([ }) // eslint-disable-next-line jest/no-standalone-expect - expect(foundRows).toEqual( + expect([...foundRows]).toEqual( expect.arrayContaining( expectedRows.map((expectedRow: any) => - expect.objectContaining(this.findRow(expectedRow, foundRows)) + expect.objectContaining(this.popRow(expectedRow, foundRows)) ) ) ) From 6eefa1afe81d4ebd503dffdd25d00909f3cb7aa6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 May 2024 14:30:03 +0200 Subject: [PATCH 27/40] Add comment --- packages/server/src/api/routes/tests/search.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 74eca789cb..00864e7454 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -78,6 +78,7 @@ describe.each([ } async function createRows(rows: Record[]) { + // Shuffling to avoid false positives given a fixed order await config.api.row.bulkImport(table._id!, { rows: _.shuffle(rows) }) } From 8cf021f254f8144b0ffa14624871ecd3fb3c4505 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 May 2024 14:30:37 +0200 Subject: [PATCH 28/40] Add null row --- packages/server/src/api/routes/tests/search.spec.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 00864e7454..15220a9c1a 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -980,12 +980,16 @@ describe.each([ const UNEXISTING_TIME = "10:01" + const NULL_TIME__ID = `null_time__id` + beforeAll(async () => { await createTable({ + timeid: { name: "timeid", type: FieldType.STRING }, time: { name: "time", type: FieldType.DATETIME, timeOnly: true }, }) await createRows([ + { timeid: NULL_TIME__ID, time: null }, { time: T_1000 }, { time: T_1045 }, { time: T_1200 }, From 682cdfffe8a7cfb8405c345da8b27362c83707b0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 21 May 2024 14:03:07 +0100 Subject: [PATCH 29/40] Respond to PR feedback. --- .../server/src/api/routes/tests/table.spec.ts | 14 +++++- .../server/src/integrations/tests/sql.spec.ts | 45 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 059b5cde72..f23e0de6db 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -52,14 +52,24 @@ describe.each([ jest.clearAllMocks() }) - it("creates a table successfully", async () => { - const name = generator.guid() + it.each([ + "alphanum", + "with spaces", + "with-dashes", + "with_underscores", + 'with "double quotes"', + "with 'single quotes'", + "with `backticks`", + ])("creates a table with name: %s", async name => { const table = await config.api.table.save( tableForDatasource(datasource, { name }) ) expect(table.name).toEqual(name) expect(events.table.created).toHaveBeenCalledTimes(1) expect(events.table.created).toHaveBeenCalledWith(table) + + const res = await config.api.table.get(table._id!) + expect(res.name).toEqual(name) }) it("creates a table via data import", async () => { diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index 78f6cd5d57..cf433725ce 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -228,6 +228,51 @@ describe("SQL query builder", () => { }) }) + it("should lowercase the values for Oracle LIKE statements", () => { + let query = new Sql(SqlClient.ORACLE, limit)._query( + generateReadJson({ + filters: { + string: { + name: "John", + }, + }, + }) + ) + expect(query).toEqual({ + bindings: ["john%", limit], + sql: `select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1) where rownum <= :2) "test"`, + }) + + query = new Sql(SqlClient.ORACLE, limit)._query( + generateReadJson({ + filters: { + contains: { + age: [20, 25], + name: ["John", "Mary"], + }, + }, + }) + ) + expect(query).toEqual({ + bindings: ["%20%", "%25%", `%"john"%`, `%"mary"%`, limit], + sql: `select * from (select * from (select * from "test" where (LOWER("test"."age") LIKE :1 AND LOWER("test"."age") LIKE :2) and (LOWER("test"."name") LIKE :3 AND LOWER("test"."name") LIKE :4)) where rownum <= :5) "test"`, + }) + + query = new Sql(SqlClient.ORACLE, limit)._query( + generateReadJson({ + filters: { + fuzzy: { + name: "Jo", + }, + }, + }) + ) + expect(query).toEqual({ + bindings: [`%jo%`, limit], + sql: `select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1) where rownum <= :2) "test"`, + }) + }) + it("should sort SQL Server tables by the primary key if no sort data is provided", () => { let query = new Sql(SqlClient.MS_SQL, limit)._query( generateReadJson({ From a074cb6befe600bd56bf6c1ce9cd0b0261f818cb Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 21 May 2024 14:17:35 +0100 Subject: [PATCH 30/40] Remove enrichEmptySettings from component patch function as any screen update already invokes empty setting enrichment on every single component --- packages/builder/src/stores/builder/components.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js index 1311c26d5d..c281c73dfe 100644 --- a/packages/builder/src/stores/builder/components.js +++ b/packages/builder/src/stores/builder/components.js @@ -555,7 +555,6 @@ export class ComponentStore extends BudiStore { const patchResult = patchFn(component, screen) // Post processing - this.enrichEmptySettings(component, { screen, useDefaultValues: false }) const migrated = this.migrateSettings(component) // Returning an explicit false signifies that we should skip this From 53605ec8ed68c06c5baab65e98706e2642bec2ef Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 May 2024 16:08:22 +0200 Subject: [PATCH 31/40] Fix sorting in pg --- packages/server/src/integrations/base/sql.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 25f2d13a35..1d0d909829 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -387,7 +387,13 @@ class InternalBuilder { for (let [key, value] of Object.entries(sort)) { const direction = value.direction === SortDirection.ASCENDING ? "asc" : "desc" - query = query.orderBy(`${aliased}.${key}`, direction) + let nulls + if (this.client === SqlClient.POSTGRES) { + // All other clients already sort this as expected by default, and adding this to the rest of the clients is causing issues + nulls = value.direction === SortDirection.ASCENDING ? "first" : "last" + } + + query = query.orderBy(`${aliased}.${key}`, direction, nulls) } } else if (this.client === SqlClient.MS_SQL && paginate?.limit) { // @ts-ignore From a55f975489abebad2c020c497aac32f1f85f862b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 May 2024 16:11:36 +0200 Subject: [PATCH 32/40] Fix tests --- packages/server/src/api/routes/tests/search.spec.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 15220a9c1a..709f8a7597 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1067,6 +1067,7 @@ describe.each([ sort: "time", sortOrder: SortOrder.ASCENDING, }).toMatchExactly([ + { timeid: NULL_TIME__ID }, { time: "00:00:00" }, { time: "10:00:00" }, { time: "10:45:00" }, @@ -1085,6 +1086,7 @@ describe.each([ { time: "10:45:00" }, { time: "10:00:00" }, { time: "00:00:00" }, + { timeid: NULL_TIME__ID }, ])) describe("sortType STRING", () => { @@ -1095,6 +1097,7 @@ describe.each([ sortType: SortType.STRING, sortOrder: SortOrder.ASCENDING, }).toMatchExactly([ + { timeid: NULL_TIME__ID }, { time: "00:00:00" }, { time: "10:00:00" }, { time: "10:45:00" }, @@ -1114,6 +1117,7 @@ describe.each([ { time: "10:45:00" }, { time: "10:00:00" }, { time: "00:00:00" }, + { timeid: NULL_TIME__ID }, ])) }) }) From 4bf0a43c64ed73d9d213d24aa91ed9c6be2adbd5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 May 2024 16:35:11 +0200 Subject: [PATCH 33/40] Add comment --- packages/server/src/integrations/base/sql.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 1d0d909829..c3dca2a39c 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -136,6 +136,7 @@ function generateSelectStatement( columnSchema?.type === FieldType.DATETIME && columnSchema.timeOnly ) { + // Time gets returned as timestamp from mssql, not matching the expected HH:mm format return knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`) } return `${field} as ${field}` From 7bbb920aa7fc17ccd91d8c0c8da555042b4be934 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 21 May 2024 15:49:48 +0100 Subject: [PATCH 34/40] Optimise the `reverse` method in sqlAlias.ts. --- packages/server/src/sdk/app/rows/sqlAlias.ts | 23 ++++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index 0fc338ecbe..79d1ff485d 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -126,16 +126,25 @@ export default class AliasTables { } reverse(rows: T): T { + const mapping = new Map() + const process = (row: Row) => { const final: Row = {} - for (let [key, value] of Object.entries(row)) { - if (!key.includes(".")) { - final[key] = value - } else { - const [alias, column] = key.split(".") - const tableName = this.tableAliases[alias] || alias - final[`${tableName}.${column}`] = value + for (const key of Object.keys(row)) { + let mappedKey = mapping.get(key) + if (!mappedKey) { + const dotLocation = key.indexOf(".") + if (dotLocation === -1) { + mappedKey = key + } else { + const alias = key.slice(0, dotLocation) + const column = key.slice(dotLocation + 1) + const tableName = this.tableAliases[alias] || alias + mappedKey = `${tableName}.${column}` + } + mapping.set(key, mappedKey) } + final[mappedKey] = row[key] } return final } From d93a9e2c4f6d7cb531a6dfd84582d1dc776ef682 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 May 2024 16:52:40 +0200 Subject: [PATCH 35/40] Fix value from null --- packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte index e1ea4d625b..4f070bdcfb 100644 --- a/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte +++ b/packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte @@ -1,4 +1,5 @@ From 229af2e695cc3decd037f1980a05d3a3c5a15f22 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 21 May 2024 16:12:16 +0000 Subject: [PATCH 36/40] Bump version to 2.27.0 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index cba15492eb..d4e12438ee 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.26.4", + "version": "2.27.0", "npmClient": "yarn", "packages": [ "packages/*", From 35c52203ce4b432fc811212f55f4043cdf0c35cc Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 21 May 2024 17:29:32 +0100 Subject: [PATCH 37/40] Add tests for enriching empty card block settings --- .../stores/builder/tests/component.test.js | 23 +++++++ .../stores/builder/tests/fixtures/index.js | 60 ++++++++++++++++++- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/stores/builder/tests/component.test.js b/packages/builder/src/stores/builder/tests/component.test.js index b8baefc5e6..80a0c8077d 100644 --- a/packages/builder/src/stores/builder/tests/component.test.js +++ b/packages/builder/src/stores/builder/tests/component.test.js @@ -23,6 +23,7 @@ import { DB_TYPE_EXTERNAL, DEFAULT_BB_DATASOURCE_ID, } from "constants/backend" +import { makePropSafe as safe } from "@budibase/string-templates" // Could move to fixtures const COMP_PREFIX = "@budibase/standard-components" @@ -360,8 +361,30 @@ describe("Component store", () => { resourceId: internalTableDoc._id, type: "table", }) + + return comp } + it("enrichEmptySettings - initialise cards blocks with correct fields", async ctx => { + const comp = enrichSettingsDS("cardsblock", ctx) + const expectBinding = (setting, ...parts) => { + expect(comp[setting]).toStrictEqual( + `{{ ${safe(`${comp._id}-repeater`)}.${parts.map(safe).join(".")} }}` + ) + } + expectBinding("cardTitle", internalTableDoc.schema.MediaTitle.name) + expectBinding("cardSubtitle", internalTableDoc.schema.MediaVersion.name) + expectBinding( + "cardDescription", + internalTableDoc.schema.MediaDescription.name + ) + expectBinding( + "cardImageURL", + internalTableDoc.schema.MediaImage.name, + "url" + ) + }) + it("enrichEmptySettings - set default datasource for 'table' setting type", async ctx => { enrichSettingsDS("formblock", ctx) }) diff --git a/packages/builder/src/stores/builder/tests/fixtures/index.js b/packages/builder/src/stores/builder/tests/fixtures/index.js index f636790f53..fbad17e374 100644 --- a/packages/builder/src/stores/builder/tests/fixtures/index.js +++ b/packages/builder/src/stores/builder/tests/fixtures/index.js @@ -8,6 +8,7 @@ import { DB_TYPE_EXTERNAL, DEFAULT_BB_DATASOURCE_ID, } from "constants/backend" +import { FieldType } from "@budibase/types" const getDocId = () => { return v4().replace(/-/g, "") @@ -45,6 +46,52 @@ export const COMPONENT_DEFINITIONS = { }, ], }, + cardsblock: { + block: true, + name: "Cards Block", + settings: [ + { + type: "dataSource", + label: "Data", + key: "dataSource", + required: true, + }, + { + section: true, + name: "Cards", + settings: [ + { + type: "text", + key: "cardTitle", + label: "Title", + nested: true, + resetOn: "dataSource", + }, + { + type: "text", + key: "cardSubtitle", + label: "Subtitle", + nested: true, + resetOn: "dataSource", + }, + { + type: "text", + key: "cardDescription", + label: "Description", + nested: true, + resetOn: "dataSource", + }, + { + type: "text", + key: "cardImageURL", + label: "Image URL", + nested: true, + resetOn: "dataSource", + }, + ], + }, + ], + }, container: { name: "Container", }, @@ -262,14 +309,23 @@ export const internalTableDoc = { name: "Media", sourceId: BUDIBASE_INTERNAL_DB_ID, sourceType: DB_TYPE_INTERNAL, + primaryDisplay: "MediaTitle", schema: { MediaTitle: { name: "MediaTitle", - type: "string", + type: FieldType.STRING, }, MediaVersion: { name: "MediaVersion", - type: "string", + type: FieldType.STRING, + }, + MediaDescription: { + name: "MediaDescription", + type: FieldType.LONGFORM, + }, + MediaImage: { + name: "MediaImage", + type: FieldType.ATTACHMENT_SINGLE, }, }, } From 9efffdf9d6929ab7a81d2f474e8506bd4b5110dd Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 21 May 2024 18:47:10 +0000 Subject: [PATCH 38/40] Bump version to 2.27.1 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index d4e12438ee..f0b3f51d47 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.27.0", + "version": "2.27.1", "npmClient": "yarn", "packages": [ "packages/*", From a920161e92e5aaf283733c6e17681d77bb952d64 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 May 2024 22:02:47 +0200 Subject: [PATCH 39/40] Fix tests --- .../src/integrations/tests/sqlAlias.spec.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index fda2a091fa..0de4d0a151 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -61,9 +61,9 @@ describe("Captures of real examples", () => { "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" - from (select * from "persons" as "a" order by "a"."firstname" asc limit $1) as "a" + from (select * from "persons" as "a" order by "a"."firstname" asc nulls first limit $1) as "a" left join "tasks" as "b" on "a"."personid" = "b"."qaid" or "a"."personid" = "b"."executorid" - order by "a"."firstname" asc limit $2`), + order by "a"."firstname" asc nulls first limit $2`), }) }) @@ -75,10 +75,10 @@ describe("Captures of real examples", () => { sql: multiline(`select "a"."productname" as "a.productname", "a"."productid" as "a.productid", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" - from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" + from (select * from "products" as "a" order by "a"."productname" asc nulls first limit $1) as "a" left join "products_tasks" as "c" on "a"."productid" = "c"."productid" left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where "b"."taskname" = $2 - order by "a"."productname" asc limit $3`), + order by "a"."productname" asc nulls first limit $3`), }) }) @@ -90,10 +90,10 @@ describe("Captures of real examples", () => { sql: multiline(`select "a"."productname" as "a.productname", "a"."productid" as "a.productid", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" - from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" + from (select * from "products" as "a" order by "a"."productname" asc nulls first limit $1) as "a" left join "products_tasks" as "c" on "a"."productid" = "c"."productid" left join "tasks" as "b" on "b"."taskid" = "c"."taskid" - order by "a"."productname" asc limit $2`), + order by "a"."productname" asc nulls first limit $2`), }) }) @@ -138,11 +138,11 @@ describe("Captures of real examples", () => { "c"."personid" as "c.personid", "c"."address" as "c.address", "c"."age" as "c.age", "c"."type" as "c.type", "c"."city" as "c.city", "c"."lastname" as "c.lastname" from (select * from "tasks" as "a" where not "a"."completed" = $1 - order by "a"."taskname" asc limit $2) as "a" + order by "a"."taskname" asc nulls first limit $2) as "a" left join "products_tasks" as "d" on "a"."taskid" = "d"."taskid" left join "products" as "b" on "b"."productid" = "d"."productid" left join "persons" as "c" on "a"."executorid" = "c"."personid" or "a"."qaid" = "c"."personid" - where "c"."year" between $3 and $4 and "b"."productname" = $5 order by "a"."taskname" asc limit $6`), + where "c"."year" between $3 and $4 and "b"."productname" = $5 order by "a"."taskname" asc nulls first limit $6`), }) }) }) From 6dc463610a8533008ebbf0cb62f884654001624e Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 22 May 2024 10:22:02 +0000 Subject: [PATCH 40/40] Bump version to 2.27.2 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index f0b3f51d47..a943000cea 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.27.1", + "version": "2.27.2", "npmClient": "yarn", "packages": [ "packages/*",