From 21b28a3de85c8fc4aac0c9f3e042d0b4d979b0de Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 4 Jul 2024 12:05:56 +0100 Subject: [PATCH 01/28] Added new Generate automation button and added it to the table --- .../backend/DataTable/TableDataTable.svelte | 4 + .../grid/GridCreateAutomationButton.svelte | 112 ++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte diff --git a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte index e8e1008e3c..525421f996 100644 --- a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte @@ -5,6 +5,7 @@ import { TableNames } from "constants" import { Grid } from "@budibase/frontend-core" import { API } from "api" + import GridCreateAutomationButton from "./buttons/grid/GridCreateAutomationButton.svelte" import GridAddColumnModal from "components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte" import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte" import GridEditUserModal from "components/backend/DataTable/modals/grid/GridEditUserModal.svelte" @@ -81,6 +82,9 @@ {/if} + {#if !isUsersTable} + + {/if} {#if relationshipsEnabled} {/if} diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte new file mode 100644 index 0000000000..60aae9e195 --- /dev/null +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte @@ -0,0 +1,112 @@ + + +
+ (open = !open)} + selected={open} + > + Generate + +
+ + + + { + open = false + createAutomation(TriggerStepID.ROW_SAVED) + console.log("create") + }} + > + Automation: when row is created + + { + open = false + createAutomation(TriggerStepID.ROW_UPDATED) + console.log("update") + }} + > + Automation: when row is updated + + + + + From a764bfb6a54ac1fda4be86ff295ce8136ffce0e5 Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 4 Jul 2024 14:30:51 +0100 Subject: [PATCH 02/28] Tidying up debugging console statements and comments --- .../grid/GridCreateAutomationButton.svelte | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte index 60aae9e195..3ee1edc4b6 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte @@ -7,22 +7,15 @@ notifications, } from "@budibase/bbui" import { getContext } from "svelte" - import { automationStore } from "stores/builder" + import { automationStore, tables } from "stores/builder" import { TriggerStepID } from "constants/backend/automations" - import { tables } from "stores/builder" import { goto } from "@roxi/routify" const { datasource } = getContext("grid") - // ROW_SAVED - // ROW_UPDATED - $: console.log($datasource) $: triggers = $automationStore.blockDefinitions.TRIGGER $: table = $tables.list.find(table => table._id === $datasource.tableId) - $: console.log("table", table) - // $: rowCreateTrigger = triggers[TriggerStepID.ROW_SAVED] - // $: rowUpdateTrigger = triggers[TriggerStepID.ROW_UPDATED] async function createAutomation(type) { const triggerType = triggers[type] @@ -48,13 +41,7 @@ triggerBlock.inputs = { tableId: $datasource.tableId } - // need to set inputs to { "tableId": "ta_bb_employee" }, - try { - console.log("REQ", { - automationName, - triggerBlock, - }) const response = await automationStore.actions.create( automationName, triggerBlock @@ -90,7 +77,6 @@ on:click={() => { open = false createAutomation(TriggerStepID.ROW_SAVED) - console.log("create") }} > Automation: when row is created @@ -100,7 +86,6 @@ on:click={() => { open = false createAutomation(TriggerStepID.ROW_UPDATED) - console.log("update") }} > Automation: when row is updated From 77abe6da839657709566745a41f232177292c763 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 4 Jul 2024 18:29:08 +0100 Subject: [PATCH 03/28] Handling invalid time values when ISO strings are input as filter options. --- .../backend-core/src/db/couch/DatabaseImpl.ts | 3 ++ packages/backend-core/src/sql/sql.ts | 22 ++++---------- packages/backend-core/src/sql/utils.ts | 29 +++++++++++++++++-- .../src/api/routes/tests/search.spec.ts | 29 +++++++++++++++++++ 4 files changed, 65 insertions(+), 18 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 274c09438d..f3c3beeaab 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -13,6 +13,7 @@ import { isDocument, RowResponse, RowValue, + SqlClient, SQLiteDefinition, SqlQueryBinding, } from "@budibase/types" @@ -25,6 +26,7 @@ import { SQLITE_DESIGN_DOC_ID } from "../../constants" import { DDInstrumentedDatabase } from "../instrumentation" import { checkSlashesInUrl } from "../../helpers" import env from "../../environment" +import { sqlLog } from "../../sql/utils" const DATABASE_NOT_FOUND = "Database does not exist." @@ -322,6 +324,7 @@ export class DatabaseImpl implements Database { ): Promise { const dbName = this.name const url = `/${dbName}/${SQLITE_DESIGN_DOC_ID}` + sqlLog(SqlClient.SQL_LITE, sql, parameters) return await this._sqlQuery(url, "POST", { query: sql, args: parameters, diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 615753efc3..e8881c22d4 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -3,8 +3,10 @@ import * as dbCore from "../db" import { getNativeSql, isExternalTable, - isIsoDateString, + isValidISODateString, isValidFilter, + sqlLog, + isInvalidISODateString, } from "./utils" import { SqlStatements } from "./sqlStatements" import SqlTableQueryBuilder from "./sqlTable" @@ -38,10 +40,6 @@ const envLimit = environment.SQL_MAX_ROWS : null const BASE_LIMIT = envLimit || 5000 -// these are invalid dates sent by the client, need to convert them to a real max date -const MIN_ISO_DATE = "0000-00-00T00:00:00.000Z" -const MAX_ISO_DATE = "9999-00-00T00:00:00.000Z" - function likeKey(client: string, key: string): string { let start: string, end: string switch (client) { @@ -75,10 +73,10 @@ function parse(input: any) { if (typeof input !== "string") { return input } - if (input === MAX_ISO_DATE || input === MIN_ISO_DATE) { + if (isInvalidISODateString(input)) { return null } - if (isIsoDateString(input)) { + if (isValidISODateString(input)) { return new Date(input.trim()) } return input @@ -938,15 +936,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { } log(query: string, values?: SqlQueryBinding) { - if (!environment.SQL_LOGGING_ENABLE) { - return - } - const sqlClient = this.getSqlClient() - let string = `[SQL] [${sqlClient.toUpperCase()}] query="${query}"` - if (values) { - string += ` values="${values.join(", ")}"` - } - console.log(string) + sqlLog(this.getSqlClient(), query, values) } } diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts index 45ab510948..e73c6ac445 100644 --- a/packages/backend-core/src/sql/utils.ts +++ b/packages/backend-core/src/sql/utils.ts @@ -2,10 +2,12 @@ import { DocumentType, SqlQuery, Table, TableSourceType } from "@budibase/types" import { DEFAULT_BB_DATASOURCE_ID } from "../constants" import { Knex } from "knex" import { SEPARATOR } from "../db" +import environment from "../environment" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` const ROW_ID_REGEX = /^\[.*]$/g const ENCODED_SPACE = encodeURIComponent(" ") +const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/ export function isExternalTableID(tableId: string) { return tableId.startsWith(DocumentType.DATASOURCE + SEPARATOR) @@ -120,15 +122,38 @@ export function breakRowIdField(_id: string | { _id: string }): any[] { } } -export function isIsoDateString(str: string) { +export function isInvalidISODateString(str: string) { const trimmedValue = str.trim() - if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/.test(trimmedValue)) { + if (!ISO_DATE_REGEX.test(trimmedValue)) { return false } let d = new Date(trimmedValue) + return isNaN(d.getTime()) +} + +export function isValidISODateString(str: string) { + const trimmedValue = str.trim() + if (!ISO_DATE_REGEX.test(trimmedValue)) { + return false + } + let d = new Date(trimmedValue) + if (isNaN(d.getTime())) { + return false + } return d.toISOString() === trimmedValue } export function isValidFilter(value: any) { return value != null && value !== "" } + +export function sqlLog(client: string, query: string, values?: any[]) { + if (!environment.SQL_LOGGING_ENABLE) { + return + } + let string = `[SQL] [${client.toUpperCase()}] query="${query}"` + if (values) { + string += ` values="${values.join(", ")}"` + } + console.log(string) +} diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 145db9b4a3..bdc8fc1af6 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2167,6 +2167,35 @@ describe.each([ } ) + describe.each([ + { low: "2024-07-03T00:00:00.000Z", high: "9999-00-00T00:00:00.000Z" }, + { low: "2024-07-03T00:00:00.000Z", high: "9998-00-00T00:00:00.000Z" }, + { low: "0000-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" }, + { low: "0001-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" }, + ])("date special cases", ({ low, high }) => { + const earlyDate = "2024-07-03T10:00:00.000Z", + laterDate = "2024-07-03T11:00:00.000Z" + beforeAll(async () => { + table = await createTable({ + date: { + name: "date", + type: FieldType.DATETIME, + }, + }) + await createRows([{ date: earlyDate }, { date: laterDate }]) + }) + + it("should be able to handle a date search", async () => { + await expectSearch({ + query: { + range: { + "1:date": { low, high }, + }, + }, + }).toContainExactly([{ date: earlyDate }, { date: laterDate }]) + }) + }) + describe.each([ "名前", // Japanese for "name" "Benutzer-ID", // German for "user ID", includes a hyphen From f26a8d410aef9e3994962843655f2a172f982273 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 5 Jul 2024 11:19:01 +0200 Subject: [PATCH 04/28] Add export composite key --- .../server/src/api/routes/tests/row.spec.ts | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index b6e3edf5ff..d92438a310 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1452,6 +1452,43 @@ describe.each([ { status: 404 } ) }) + + it("can export rows with composite primary keys", async () => { + const tableRequest = saveTableRequest({ + primary: ["number", "string"], + schema: { + string: { + type: FieldType.STRING, + name: "string", + }, + number: { + type: FieldType.NUMBER, + name: "number", + }, + }, + }) + delete tableRequest.schema.id + + const table = await config.api.table.save(tableRequest) + + const rows = await Promise.all( + generator + .unique( + () => ({ + string: generator.word({ length: 30 }), + number: generator.integer({ min: 0, max: 10000 }), + }), + 10 + ) + .map(d => config.api.row.save(table._id!, d)) + ) + + const res = await config.api.row.exportRows(table._id!, { + rows: _.sampleSize(rows, 3).map(r => r._id!), + }) + const results = JSON.parse(res) + expect(results.length).toEqual(3) + }) }) let o2mTable: Table From 9518680d1277f1b5cbf9733dd63a5c6102bd9311 Mon Sep 17 00:00:00 2001 From: Dean Date: Fri, 5 Jul 2024 14:33:09 +0100 Subject: [PATCH 05/28] Update the topnav history when navigating to the automation sections on create --- .../buttons/grid/GridCreateAutomationButton.svelte | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte index 3ee1edc4b6..94ac41fd24 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte @@ -7,9 +7,9 @@ notifications, } from "@budibase/bbui" import { getContext } from "svelte" - import { automationStore, tables } from "stores/builder" + import { automationStore, tables, builderStore } from "stores/builder" import { TriggerStepID } from "constants/backend/automations" - import { goto } from "@roxi/routify" + import { goto, layout, isActive } from "@roxi/routify" const { datasource } = getContext("grid") @@ -46,10 +46,14 @@ automationName, triggerBlock ) - + builderStore.setPreviousTopNavPath( + "/builder/app/:application/data", + window.location.pathname + ) $goto(`/builder/app/${response.appId}/automation/${response.id}`) notifications.success(`Automation created`) - } catch { + } catch (e) { + console.error("Error creating automation", e) notifications.error("Error creating automation") } } From 4d11f62e007899abb8317e305ea057f27160aa08 Mon Sep 17 00:00:00 2001 From: Dean Date: Fri, 5 Jul 2024 14:36:20 +0100 Subject: [PATCH 06/28] Lint --- .../DataTable/buttons/grid/GridCreateAutomationButton.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte index 94ac41fd24..8e3d90be41 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte @@ -9,7 +9,7 @@ import { getContext } from "svelte" import { automationStore, tables, builderStore } from "stores/builder" import { TriggerStepID } from "constants/backend/automations" - import { goto, layout, isActive } from "@roxi/routify" + import { goto } from "@roxi/routify" const { datasource } = getContext("grid") From e4375c21966aee6c99018889d54862db27399ee2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Jul 2024 15:27:54 +0100 Subject: [PATCH 07/28] Fixing a build issue uncovered by tests. --- packages/backend-core/src/db/constants.ts | 5 ----- packages/backend-core/src/db/couch/index.ts | 1 - .../backend-core/tests/core/utilities/jestUtils.ts | 9 ++++++--- packages/server/src/sdk/app/rows/search/internal.ts | 5 +++-- packages/server/src/sdk/app/tables/migration.ts | 3 ++- packages/server/src/sdk/app/views/index.ts | 12 ++++++++---- 6 files changed, 19 insertions(+), 16 deletions(-) delete mode 100644 packages/backend-core/src/db/constants.ts diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts deleted file mode 100644 index 69c98fe569..0000000000 --- a/packages/backend-core/src/db/constants.ts +++ /dev/null @@ -1,5 +0,0 @@ -export { - CONSTANT_INTERNAL_ROW_COLS, - CONSTANT_EXTERNAL_ROW_COLS, - isInternalColumnName, -} from "@budibase/shared-core" diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts index 932efed3f7..c731d20d6c 100644 --- a/packages/backend-core/src/db/couch/index.ts +++ b/packages/backend-core/src/db/couch/index.ts @@ -2,4 +2,3 @@ export * from "./connections" export * from "./DatabaseImpl" export * from "./utils" export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB" -export * from "../constants" diff --git a/packages/backend-core/tests/core/utilities/jestUtils.ts b/packages/backend-core/tests/core/utilities/jestUtils.ts index 4a3da8db8c..a49c2a795e 100644 --- a/packages/backend-core/tests/core/utilities/jestUtils.ts +++ b/packages/backend-core/tests/core/utilities/jestUtils.ts @@ -1,4 +1,7 @@ -import { db } from "../../../src" +import { + CONSTANT_EXTERNAL_ROW_COLS, + CONSTANT_INTERNAL_ROW_COLS, +} from "@budibase/shared-core" export function expectFunctionWasCalledTimesWith( jestFunction: any, @@ -11,7 +14,7 @@ export function expectFunctionWasCalledTimesWith( } export const expectAnyInternalColsAttributes: { - [K in (typeof db.CONSTANT_INTERNAL_ROW_COLS)[number]]: any + [K in (typeof CONSTANT_INTERNAL_ROW_COLS)[number]]: any } = { tableId: expect.anything(), type: expect.anything(), @@ -22,7 +25,7 @@ export const expectAnyInternalColsAttributes: { } export const expectAnyExternalColsAttributes: { - [K in (typeof db.CONSTANT_EXTERNAL_ROW_COLS)[number]]: any + [K in (typeof CONSTANT_EXTERNAL_ROW_COLS)[number]]: any } = { tableId: expect.anything(), _id: expect.anything(), diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 906ca016d1..097b16b104 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -1,4 +1,5 @@ -import { context, db, HTTPError } from "@budibase/backend-core" +import { context, HTTPError } from "@budibase/backend-core" +import { CONSTANT_INTERNAL_ROW_COLS } from "@budibase/shared-core" import env from "../../../../environment" import { fullSearch, paginatedSearch } from "./utils" import { getRowParams, InternalTables } from "../../../../db/utils" @@ -74,7 +75,7 @@ export async function search( } if (options.fields) { - const fields = [...options.fields, ...db.CONSTANT_INTERNAL_ROW_COLS] + const fields = [...options.fields, ...CONSTANT_INTERNAL_ROW_COLS] response.rows = response.rows.map((r: any) => pick(r, fields)) } diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts index d0decf01f6..8e084381aa 100644 --- a/packages/server/src/sdk/app/tables/migration.ts +++ b/packages/server/src/sdk/app/tables/migration.ts @@ -18,6 +18,7 @@ import sdk from "../../../sdk" import { isExternalTableID } from "../../../integrations/utils" import { EventType, updateLinks } from "../../../db/linkedRows" import { cloneDeep } from "lodash" +import { isInternalColumnName } from "@budibase/shared-core" export interface MigrationResult { tablesUpdated: Table[] @@ -36,7 +37,7 @@ export async function migrate( throw new BadRequestError(`Column name cannot be empty`) } - if (dbCore.isInternalColumnName(newColumnName)) { + if (isInternalColumnName(newColumnName)) { throw new BadRequestError(`Column name cannot be a reserved column name`) } diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index fce57a390d..3bdfec7448 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -6,9 +6,13 @@ import { ViewV2, ViewV2Enriched, } from "@budibase/types" -import { HTTPError, db as dbCore } from "@budibase/backend-core" +import { HTTPError } from "@budibase/backend-core" import { features } from "@budibase/pro" -import { helpers } from "@budibase/shared-core" +import { + helpers, + CONSTANT_EXTERNAL_ROW_COLS, + CONSTANT_INTERNAL_ROW_COLS, +} from "@budibase/shared-core" import { cloneDeep } from "lodash/fp" import * as utils from "../../../db/utils" @@ -144,8 +148,8 @@ export function allowedFields(view: View | ViewV2) { const fieldSchema = view.schema![key] return fieldSchema.visible && !fieldSchema.readonly }), - ...dbCore.CONSTANT_EXTERNAL_ROW_COLS, - ...dbCore.CONSTANT_INTERNAL_ROW_COLS, + ...CONSTANT_EXTERNAL_ROW_COLS, + ...CONSTANT_INTERNAL_ROW_COLS, ] } From c48f5c6d8016f082324bc49b4447557393220430 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Jul 2024 15:45:00 +0100 Subject: [PATCH 08/28] Fixing build issue. --- packages/server/src/sdk/app/rows/search/external.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 93c46d8cc3..1b6638f671 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -16,9 +16,9 @@ import { breakExternalTableId, breakRowIdField, } from "../../../../integrations/utils" -import { utils } from "@budibase/shared-core" +import { utils, CONSTANT_EXTERNAL_ROW_COLS } from "@budibase/shared-core" import { ExportRowsParams, ExportRowsResult } from "./types" -import { db, HTTPError } from "@budibase/backend-core" +import { HTTPError } from "@budibase/backend-core" import pick from "lodash/pick" import { outputProcessing } from "../../../../utilities/rowProcessor" import sdk from "../../../" @@ -99,7 +99,7 @@ export async function search( } if (options.fields) { - const fields = [...options.fields, ...db.CONSTANT_EXTERNAL_ROW_COLS] + const fields = [...options.fields, ...CONSTANT_EXTERNAL_ROW_COLS] rows = rows.map((r: any) => pick(r, fields)) } From 5066da2630787d24ac17f09b33798da5c0c601d0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Jul 2024 16:03:03 +0100 Subject: [PATCH 09/28] Grouping and exporting default tables. --- packages/server/src/constants/index.ts | 6 ++++++ .../server/src/db/defaultData/datasource_bb_default.ts | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index 60875b3daa..4fcc9c5d06 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -173,3 +173,9 @@ export const DEFAULT_INVENTORY_TABLE_ID = constants.DEFAULT_INVENTORY_TABLE_ID export const DEFAULT_EXPENSES_TABLE_ID = constants.DEFAULT_EXPENSES_TABLE_ID export const DEFAULT_EMPLOYEE_TABLE_ID = constants.DEFAULT_EMPLOYEE_TABLE_ID export const DEFAULT_BB_DATASOURCE_ID = constants.DEFAULT_BB_DATASOURCE_ID +export const DEFAULT_TABLE_IDS = [ + DEFAULT_JOBS_TABLE_ID, + DEFAULT_INVENTORY_TABLE_ID, + DEFAULT_EXPENSES_TABLE_ID, + DEFAULT_EMPLOYEE_TABLE_ID, +] diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts index 68d49b2d8b..fc79b90c00 100644 --- a/packages/server/src/db/defaultData/datasource_bb_default.ts +++ b/packages/server/src/db/defaultData/datasource_bb_default.ts @@ -619,6 +619,13 @@ export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = { }, } +export const DEFAULT_TABLES: Table[] = [ + DEFAULT_INVENTORY_TABLE_SCHEMA, + DEFAULT_EMPLOYEE_TABLE_SCHEMA, + DEFAULT_JOBS_TABLE_SCHEMA, + DEFAULT_EXPENSES_TABLE_SCHEMA, +] + export async function buildDefaultDocs() { const inventoryData = await tableImport( DEFAULT_INVENTORY_TABLE_SCHEMA, From 34d073bcb7ef073de97044056f0adca842edc6fe Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Jul 2024 16:03:38 +0100 Subject: [PATCH 10/28] Adding default tables to sync. --- packages/server/src/sdk/app/rows/search/sqs.ts | 18 ++++++++++++++---- .../server/src/sdk/app/tables/internal/sqs.ts | 4 +++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 2c9ee1356c..72a1557cc9 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -41,6 +41,7 @@ import { getTableIDList, } from "./filters" import { dataFilters } from "@budibase/shared-core" +import { DEFAULT_TABLE_IDS } from "../../../../constants" const builder = new sql.Sql(SqlClient.SQL_LITE) const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`) @@ -211,6 +212,18 @@ async function runSqlQuery( return response } +function resyncDefinitionsRequired(status: number, message: string) { + // pre data_ prefix on column names, need to resync + return ( + (status === 400 && message?.match(USER_COLUMN_PREFIX_REGEX)) || + // default tables aren't included in definition + (status === 400 && + DEFAULT_TABLE_IDS.find(tableId => message?.includes(tableId))) || + // no design document found, needs a full sync + (status === 404 && message?.includes(SQLITE_DESIGN_DOC_ID)) + ) +} + export async function search( options: RowSearchParams, table: Table @@ -338,10 +351,7 @@ export async function search( return response } catch (err: any) { const msg = typeof err === "string" ? err : err.message - const syncAndRepeat = - (err.status === 400 && msg?.match(USER_COLUMN_PREFIX_REGEX)) || - (err.status === 404 && msg?.includes(SQLITE_DESIGN_DOC_ID)) - if (syncAndRepeat) { + if (resyncDefinitionsRequired(err.status, msg)) { await sdk.tables.sqs.syncDefinition() return search(options, table) } diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index 9e831f4af7..2d49adf96e 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -15,6 +15,7 @@ import { generateJunctionTableID, } from "../../../../db/utils" import { isEqual } from "lodash" +import { DEFAULT_TABLES } from "../../../../db/defaultData/datasource_bb_default" const FieldTypeMap: Record = { [FieldType.BOOLEAN]: SQLiteType.NUMERIC, @@ -126,8 +127,9 @@ function mapTable(table: Table): SQLiteTables { // nothing exists, need to iterate though existing tables async function buildBaseDefinition(): Promise { const tables = await tablesSdk.getAllInternalTables() + const defaultTables = DEFAULT_TABLES const definition = sql.designDoc.base("tableId") - for (let table of tables) { + for (let table of tables.concat(defaultTables)) { definition.sql.tables = { ...definition.sql.tables, ...mapTable(table), From 39523685d696b8828c1890dfbf2061cce7e3fa8e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Jul 2024 16:20:27 +0100 Subject: [PATCH 11/28] Adding test case for searching sample data. --- .../src/api/routes/tests/search.spec.ts | 20 +++++++++++++++++++ .../src/tests/utilities/api/application.ts | 4 ++++ 2 files changed, 24 insertions(+) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 145db9b4a3..6b1fd2a198 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -30,6 +30,8 @@ import { encodeJSBinding } from "@budibase/string-templates" import { dataFilters } from "@budibase/shared-core" import { Knex } from "knex" import { structures } from "@budibase/backend-core/tests" +import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default" +import { employeeImport } from "../../../db/defaultData/employeeImport" describe.each([ ["in-memory", undefined], @@ -2167,6 +2169,24 @@ describe.each([ } ) + isInternal && + describe("sample data", () => { + beforeAll(async () => { + await config.api.application.addSampleData(config.appId!) + table = DEFAULT_EMPLOYEE_TABLE_SCHEMA + }) + + it("should be able to search sample data", async () => { + await expectSearch({ + query: {}, + }).toContain([ + { + "First Name": "Mandy", + }, + ]) + }) + }) + describe.each([ "名前", // Japanese for "name" "Benutzer-ID", // German for "user ID", includes a hyphen diff --git a/packages/server/src/tests/utilities/api/application.ts b/packages/server/src/tests/utilities/api/application.ts index bb9357c893..516af5c973 100644 --- a/packages/server/src/tests/utilities/api/application.ts +++ b/packages/server/src/tests/utilities/api/application.ts @@ -149,4 +149,8 @@ export class ApplicationAPI extends TestAPI { query: { status }, }) } + + addSampleData = async (appId: string): Promise => { + await this._post(`/api/applications/${appId}/sample`) + } } From eafe66d01e638c2e06a55a8380394b6e2cb6d32f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Jul 2024 16:26:26 +0100 Subject: [PATCH 12/28] Linting. --- packages/server/src/api/routes/tests/search.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 6b1fd2a198..a832c26f42 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -31,7 +31,6 @@ import { dataFilters } from "@budibase/shared-core" import { Knex } from "knex" import { structures } from "@budibase/backend-core/tests" import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default" -import { employeeImport } from "../../../db/defaultData/employeeImport" describe.each([ ["in-memory", undefined], From 37d53cff2036e25ad225ed771527e8ea61cafff6 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 5 Jul 2024 15:43:04 +0000 Subject: [PATCH 13/28] Bump version to 2.29.14 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index c3419a3f87..b3831d0886 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.29.13", + "version": "2.29.14", "npmClient": "yarn", "packages": [ "packages/*", From 85827bbf9309a3894b7ab215f8277bda5aaa0f5e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 8 Jul 2024 13:27:30 +0200 Subject: [PATCH 14/28] Refactor breakRowIdField --- packages/backend-core/src/sql/sql.ts | 3 +-- packages/backend-core/src/sql/utils.ts | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 615753efc3..c17cbae86f 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -217,8 +217,7 @@ class InternalBuilder { if (!opts.relationship && !isRelationshipField) { const alias = getTableAlias(tableName) fn(alias ? `${alias}.${updatedKey}` : updatedKey, value) - } - if (opts.relationship && isRelationshipField) { + } else if (opts.relationship && isRelationshipField) { const [filterTableName, property] = updatedKey.split(".") const alias = getTableAlias(filterTableName) fn(alias ? `${alias}.${property}` : property, value) diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts index 45ab510948..4b0ac82ffe 100644 --- a/packages/backend-core/src/sql/utils.ts +++ b/packages/backend-core/src/sql/utils.ts @@ -116,6 +116,9 @@ export function breakRowIdField(_id: string | { _id: string }): any[] { return Array.isArray(parsed) ? parsed : [parsed] } catch (err) { // wasn't json - likely was handlebars for a many to many + if (Array.isArray(_id)) { + return _id + } return [_id] } } From 5be8882122fec80594a80bc1c63217c86e34a6a2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 9 Jul 2024 10:43:45 +0200 Subject: [PATCH 15/28] Handle composite keys on exports --- packages/backend-core/src/sql/sql.ts | 14 ++++++++++- .../api/controllers/row/ExternalRequest.ts | 25 +++++++++++++++---- .../server/src/api/routes/tests/row.spec.ts | 18 +------------ .../src/sdk/app/rows/search/external.ts | 5 +--- 4 files changed, 35 insertions(+), 27 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index c17cbae86f..7410ca5dbf 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -172,6 +172,8 @@ function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] { return query } +const COMPLEX_ID_OPERATOR = "_complexIdOperator" + class InternalBuilder { private readonly client: string @@ -214,7 +216,14 @@ class InternalBuilder { for (let [key, value] of Object.entries(structure)) { const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") - if (!opts.relationship && !isRelationshipField) { + + if (updatedKey === COMPLEX_ID_OPERATOR) { + const alias = getTableAlias(tableName) + fn( + value.id.map((x: string) => (alias ? `${alias}.${x}` : x)), + value.values + ) + } else if (!opts.relationship && !isRelationshipField) { const alias = getTableAlias(tableName) fn(alias ? `${alias}.${updatedKey}` : updatedKey, value) } else if (opts.relationship && isRelationshipField) { @@ -745,6 +754,9 @@ class InternalBuilder { class SqlQueryBuilder extends SqlTableQueryBuilder { private readonly limit: number + + public static COMPLEX_ID_OPERATOR = COMPLEX_ID_OPERATOR + // pass through client to get flavour of SQL constructor(client: string, limit: number = BASE_LIMIT) { super(client) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index b51de46e99..a50297f7b0 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -40,7 +40,7 @@ import { } from "../../../sdk/app/rows/utils" import { processObjectSync } from "@budibase/string-templates" import { cloneDeep } from "lodash/fp" -import { db as dbCore } from "@budibase/backend-core" +import { db as dbCore, sql } from "@budibase/backend-core" import sdk from "../../../sdk" import env from "../../../environment" import { makeExternalQuery } from "../../../integrations/base/query" @@ -193,11 +193,26 @@ export class ExternalRequest { for (let field of Object.keys(operator || {})) { if (dbCore.removeKeyNumbering(field) === "_id") { if (primary) { - const parts = breakRowIdField(operator[field]) - for (let field of primary) { - operator[`${prefix}:${field}`] = parts.shift() + let idField = operator[field] + try { + // Make sure _id queries decode the Row IDs + idField = JSON.parse(idField) + } catch { + // It is not a JSON value + } + + const parts = breakRowIdField(idField) + if (primary.length > 1) { + operator[sql.Sql.COMPLEX_ID_OPERATOR] = { + id: primary, + values: parts, + } + } else { + for (let field of primary) { + operator[`${prefix}:${field}`] = parts.shift() + } + prefix++ } - prefix++ } // make sure this field doesn't exist on any filter delete operator[field] diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index d92438a310..9628757d16 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1428,22 +1428,6 @@ describe.each([ expect(row._id).toEqual(existing._id) }) - it("should return an error on composite keys", async () => { - const existing = await config.api.row.save(table._id!, {}) - await config.api.row.exportRows( - table._id!, - { - rows: [`['${existing._id!}']`, "['d001', '10111']"], - }, - { - status: 400, - body: { - message: "Export data does not support composite keys.", - }, - } - ) - }) - it("should return an error if no table is found", async () => { const existing = await config.api.row.save(table._id!, {}) await config.api.row.exportRows( @@ -1453,7 +1437,7 @@ describe.each([ ) }) - it("can export rows with composite primary keys", async () => { + it("should handle filtering by composite primary keys", async () => { const tableRequest = saveTableRequest({ primary: ["number", "string"], schema: { diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 93c46d8cc3..84306f572f 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -158,10 +158,7 @@ export async function exportRows( _id: rowIds.map((row: string) => { const ids = breakRowIdField(row) if (ids.length > 1) { - throw new HTTPError( - "Export data does not support composite keys.", - 400 - ) + return ids } return ids[0] }), From 0f1c8eb788eef9d4b35751b7b4385fd7777816e5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 8 Jul 2024 21:22:03 +0200 Subject: [PATCH 16/28] Disable mssql test --- .../server/src/api/routes/tests/row.spec.ts | 71 ++++++++++--------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 9628757d16..9fc7fb05c1 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1437,42 +1437,45 @@ describe.each([ ) }) - it("should handle filtering by composite primary keys", async () => { - const tableRequest = saveTableRequest({ - primary: ["number", "string"], - schema: { - string: { - type: FieldType.STRING, - name: "string", + // MSSQL needs a setting called IDENTITY_INSERT to be set to ON to allow writing + // to identity columns. This is not something Budibase does currently. + providerType !== DatabaseName.SQL_SERVER && + it("should handle filtering by composite primary keys", async () => { + const tableRequest = saveTableRequest({ + primary: ["number", "string"], + schema: { + string: { + type: FieldType.STRING, + name: "string", + }, + number: { + type: FieldType.NUMBER, + name: "number", + }, }, - number: { - type: FieldType.NUMBER, - name: "number", - }, - }, + }) + delete tableRequest.schema.id + + const table = await config.api.table.save(tableRequest) + + const rows = await Promise.all( + generator + .unique( + () => ({ + string: generator.word({ length: 30 }), + number: generator.integer({ min: 0, max: 10000 }), + }), + 10 + ) + .map(d => config.api.row.save(table._id!, d)) + ) + + const res = await config.api.row.exportRows(table._id!, { + rows: _.sampleSize(rows, 3).map(r => r._id!), + }) + const results = JSON.parse(res) + expect(results.length).toEqual(3) }) - delete tableRequest.schema.id - - const table = await config.api.table.save(tableRequest) - - const rows = await Promise.all( - generator - .unique( - () => ({ - string: generator.word({ length: 30 }), - number: generator.integer({ min: 0, max: 10000 }), - }), - 10 - ) - .map(d => config.api.row.save(table._id!, d)) - ) - - const res = await config.api.row.exportRows(table._id!, { - rows: _.sampleSize(rows, 3).map(r => r._id!), - }) - const results = JSON.parse(res) - expect(results.length).toEqual(3) - }) }) let o2mTable: Table From 5387717183dd7c31bc78927c17600d102f91ee60 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 9 Jul 2024 10:21:46 +0200 Subject: [PATCH 17/28] Fix --- packages/backend-core/src/sql/utils.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts index 4b0ac82ffe..45ab510948 100644 --- a/packages/backend-core/src/sql/utils.ts +++ b/packages/backend-core/src/sql/utils.ts @@ -116,9 +116,6 @@ export function breakRowIdField(_id: string | { _id: string }): any[] { return Array.isArray(parsed) ? parsed : [parsed] } catch (err) { // wasn't json - likely was handlebars for a many to many - if (Array.isArray(_id)) { - return _id - } return [_id] } } From 58bd346885a9114ba2baffe6ae78c4b4d76b9cf8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 9 Jul 2024 10:27:56 +0200 Subject: [PATCH 18/28] Fix --- packages/server/src/api/controllers/row/ExternalRequest.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index a50297f7b0..46de4ec350 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -205,7 +205,7 @@ export class ExternalRequest { if (primary.length > 1) { operator[sql.Sql.COMPLEX_ID_OPERATOR] = { id: primary, - values: parts, + values: parts[0], } } else { for (let field of primary) { From e23cfd3e4d9362fe22fa48bd7d0c879cd41d9956 Mon Sep 17 00:00:00 2001 From: Gerard Burns Date: Tue, 9 Jul 2024 09:44:41 +0100 Subject: [PATCH 19/28] wire up outside click handler (#14125) --- packages/client/src/components/app/Modal.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/components/app/Modal.svelte b/packages/client/src/components/app/Modal.svelte index 5c60735959..591c4da971 100644 --- a/packages/client/src/components/app/Modal.svelte +++ b/packages/client/src/components/app/Modal.svelte @@ -56,7 +56,7 @@
From 62f5790a81465592fabd239ca52297b3148fd6f8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 9 Jul 2024 10:49:57 +0200 Subject: [PATCH 20/28] Remove unnecessary cast --- .../server/src/api/controllers/row/ExternalRequest.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 46de4ec350..dfad5dcf58 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -193,15 +193,7 @@ export class ExternalRequest { for (let field of Object.keys(operator || {})) { if (dbCore.removeKeyNumbering(field) === "_id") { if (primary) { - let idField = operator[field] - try { - // Make sure _id queries decode the Row IDs - idField = JSON.parse(idField) - } catch { - // It is not a JSON value - } - - const parts = breakRowIdField(idField) + const parts = breakRowIdField(operator[field]) if (primary.length > 1) { operator[sql.Sql.COMPLEX_ID_OPERATOR] = { id: primary, From d1f04548318d741b62b75dde3494fd873ab3efb9 Mon Sep 17 00:00:00 2001 From: Conor Webb <126772285+ConorWebb96@users.noreply.github.com> Date: Tue, 9 Jul 2024 09:55:19 +0100 Subject: [PATCH 21/28] Switch the notification messages around as it was stating it was enable/disabled at the wrong times. (#14124) --- packages/builder/src/stores/builder/automations.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/stores/builder/automations.js b/packages/builder/src/stores/builder/automations.js index fc55b8cb67..fb46b90235 100644 --- a/packages/builder/src/stores/builder/automations.js +++ b/packages/builder/src/stores/builder/automations.js @@ -146,13 +146,13 @@ const automationActions = store => ({ await store.actions.save(automation) notifications.success( `Automation ${ - automation.disabled ? "enabled" : "disabled" + automation.disabled ? "disabled" : "enabled" } successfully` ) } catch (error) { notifications.error( `Error ${ - automation && automation.disabled ? "enabling" : "disabling" + automation && automation.disabled ? "disabling" : "enabling" } automation` ) } From 9c6347f7fd6345bce83bd924dc7df354f1fec9e9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 9 Jul 2024 12:24:59 +0200 Subject: [PATCH 22/28] Move constants to types --- packages/backend-core/src/sql/sql.ts | 7 ++----- packages/server/src/api/controllers/row/ExternalRequest.ts | 5 +++-- packages/types/src/sdk/search.ts | 4 ++++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b274e40de7..76d73c42fe 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -15,6 +15,7 @@ import { FieldSchema, FieldType, INTERNAL_TABLE_SOURCE_ID, + InternalSearchFilterOperator, JsonFieldMetadata, JsonTypes, Operation, @@ -170,8 +171,6 @@ function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] { return query } -const COMPLEX_ID_OPERATOR = "_complexIdOperator" - class InternalBuilder { private readonly client: string @@ -215,7 +214,7 @@ class InternalBuilder { const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") - if (updatedKey === COMPLEX_ID_OPERATOR) { + if (updatedKey === InternalSearchFilterOperator.COMPLEX_ID_OPERATOR) { const alias = getTableAlias(tableName) fn( value.id.map((x: string) => (alias ? `${alias}.${x}` : x)), @@ -753,8 +752,6 @@ class InternalBuilder { class SqlQueryBuilder extends SqlTableQueryBuilder { private readonly limit: number - public static COMPLEX_ID_OPERATOR = COMPLEX_ID_OPERATOR - // pass through client to get flavour of SQL constructor(client: string, limit: number = BASE_LIMIT) { super(client) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index dfad5dcf58..87c9d27ee0 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -7,6 +7,7 @@ import { FieldType, FilterType, IncludeRelationship, + InternalSearchFilterOperator, isManyToOne, OneToManyRelationshipFieldMetadata, Operation, @@ -40,7 +41,7 @@ import { } from "../../../sdk/app/rows/utils" import { processObjectSync } from "@budibase/string-templates" import { cloneDeep } from "lodash/fp" -import { db as dbCore, sql } from "@budibase/backend-core" +import { db as dbCore } from "@budibase/backend-core" import sdk from "../../../sdk" import env from "../../../environment" import { makeExternalQuery } from "../../../integrations/base/query" @@ -195,7 +196,7 @@ export class ExternalRequest { if (primary) { const parts = breakRowIdField(operator[field]) if (primary.length > 1) { - operator[sql.Sql.COMPLEX_ID_OPERATOR] = { + operator[InternalSearchFilterOperator.COMPLEX_ID_OPERATOR] = { id: primary, values: parts[0], } diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index c40f1c3b84..5ea1f359f1 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -17,6 +17,10 @@ export enum SearchFilterOperator { CONTAINS_ANY = "containsAny", } +export enum InternalSearchFilterOperator { + COMPLEX_ID_OPERATOR = "_complexIdOperator", +} + export interface SearchFilters { allOr?: boolean // TODO: this is just around for now - we need a better way to do or/and From 63bd83457ede90eee5c18ed2957b5bbc4493f2b5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 9 Jul 2024 11:26:54 +0100 Subject: [PATCH 23/28] Always allow creating views - don't limit it causing confusion. --- .../DataTable/buttons/grid/GridCreateViewButton.svelte | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateViewButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateViewButton.svelte index 3441d8de17..d4f4bcd1b1 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateViewButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateViewButton.svelte @@ -3,12 +3,11 @@ import { Modal, ActionButton, TooltipType, TempTooltip } from "@budibase/bbui" import GridCreateViewModal from "../../modals/grid/GridCreateViewModal.svelte" - const { rows, columns, filter } = getContext("grid") + const { filter } = getContext("grid") let modal let firstFilterUsage = false - $: disabled = !$columns.length || !$rows.length $: { if ($filter?.length && !firstFilterUsage) { firstFilterUsage = true @@ -21,7 +20,7 @@ type={TooltipType.Info} condition={firstFilterUsage} > - + Create view From 38f7b88735f20c9e580ea60523397f85e5b1bd4c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 9 Jul 2024 14:01:44 +0200 Subject: [PATCH 24/28] Type filters --- packages/backend-core/src/sql/sql.ts | 48 ++++++++---- .../api/controllers/row/ExternalRequest.ts | 4 +- packages/types/src/sdk/search.ts | 75 +++++++++---------- 3 files changed, 70 insertions(+), 57 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 76d73c42fe..4936e4da68 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -11,6 +11,7 @@ import { import { SqlStatements } from "./sqlStatements" import SqlTableQueryBuilder from "./sqlTable" import { + AnySearchFilter, BBReferenceFieldMetadata, FieldSchema, FieldType, @@ -41,7 +42,7 @@ const envLimit = environment.SQL_MAX_ROWS : null const BASE_LIMIT = envLimit || 5000 -function likeKey(client: string, key: string): string { +function likeKey(client: string | string[], key: string): string { let start: string, end: string switch (client) { case SqlClient.MY_SQL: @@ -207,18 +208,27 @@ class InternalBuilder { return alias || name } function iterate( - structure: { [key: string]: any }, - fn: (key: string, value: any) => void + structure: AnySearchFilter, + fn: (key: string, value: any) => void, + complexKeyFn?: (key: string[], value: any) => void ) { - for (let [key, value] of Object.entries(structure)) { + for (const key in structure) { + const value = structure[key] const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") - if (updatedKey === InternalSearchFilterOperator.COMPLEX_ID_OPERATOR) { + let castedTypeValue + if ( + key === InternalSearchFilterOperator.COMPLEX_ID_OPERATOR && + (castedTypeValue = structure[key]) && + complexKeyFn + ) { const alias = getTableAlias(tableName) - fn( - value.id.map((x: string) => (alias ? `${alias}.${x}` : x)), - value.values + complexKeyFn( + castedTypeValue.id.map((x: string) => + alias ? `${alias}.${x}` : x + ), + castedTypeValue.values ) } else if (!opts.relationship && !isRelationshipField) { const alias = getTableAlias(tableName) @@ -246,7 +256,7 @@ class InternalBuilder { } } - const contains = (mode: object, any: boolean = false) => { + const contains = (mode: AnySearchFilter, any: boolean = false) => { const rawFnc = allOr ? "orWhereRaw" : "whereRaw" const not = mode === filters?.notContains ? "NOT " : "" function stringifyArray(value: Array, quoteStyle = '"'): string { @@ -258,7 +268,7 @@ class InternalBuilder { return `[${value.join(",")}]` } if (this.client === SqlClient.POSTGRES) { - iterate(mode, (key: string, value: Array) => { + iterate(mode, (key, value) => { const wrap = any ? "" : "'" const op = any ? "\\?| array" : "@>" const fieldNames = key.split(/\./g) @@ -273,7 +283,7 @@ class InternalBuilder { }) } else if (this.client === SqlClient.MY_SQL) { const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS" - iterate(mode, (key: string, value: Array) => { + iterate(mode, (key, value) => { query = query[rawFnc]( `${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray( value @@ -282,7 +292,7 @@ class InternalBuilder { }) } else { const andOr = mode === filters?.containsAny ? " OR " : " AND " - iterate(mode, (key: string, value: Array) => { + iterate(mode, (key, value) => { let statement = "" for (let i in value) { if (typeof value[i] === "string") { @@ -306,10 +316,16 @@ class InternalBuilder { } if (filters.oneOf) { - iterate(filters.oneOf, (key, array) => { - const fnc = allOr ? "orWhereIn" : "whereIn" - query = query[fnc](key, Array.isArray(array) ? array : [array]) - }) + const fnc = allOr ? "orWhereIn" : "whereIn" + iterate( + filters.oneOf, + (key: string, array) => { + query = query[fnc](key, Array.isArray(array) ? array : [array]) + }, + (key: string[], array) => { + query = query[fnc](key, Array.isArray(array) ? array : [array]) + } + ) } if (filters.string) { iterate(filters.string, (key, value) => { diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 87c9d27ee0..2ecdf9a4cb 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -190,8 +190,8 @@ export class ExternalRequest { if (filters) { // need to map over the filters and make sure the _id field isn't present let prefix = 1 - for (let operator of Object.values(filters)) { - for (let field of Object.keys(operator || {})) { + for (const operator of Object.values(filters)) { + for (const field of Object.keys(operator || {})) { if (dbCore.removeKeyNumbering(field) === "_id") { if (primary) { const parts = breakRowIdField(operator[field]) diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 5ea1f359f1..856011284f 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -21,51 +21,48 @@ export enum InternalSearchFilterOperator { COMPLEX_ID_OPERATOR = "_complexIdOperator", } +type BasicFilter = Record & { + [InternalSearchFilterOperator.COMPLEX_ID_OPERATOR]?: never +} + +type ArrayFilter = Record & { + [InternalSearchFilterOperator.COMPLEX_ID_OPERATOR]?: { + id: string[] + values: string[] + } +} + +type RangeFilter = Record< + string, + | { + high: number | string + low: number | string + } + | { high: number | string } + | { low: number | string } +> & { + [InternalSearchFilterOperator.COMPLEX_ID_OPERATOR]?: never +} + +export type AnySearchFilter = BasicFilter | ArrayFilter | RangeFilter + export interface SearchFilters { allOr?: boolean // TODO: this is just around for now - we need a better way to do or/and // allows just fuzzy to be or - all the fuzzy/like parameters fuzzyOr?: boolean onEmptyFilter?: EmptyFilterOption - [SearchFilterOperator.STRING]?: { - [key: string]: string - } - [SearchFilterOperator.FUZZY]?: { - [key: string]: string - } - [SearchFilterOperator.RANGE]?: { - [key: string]: - | { - high: number | string - low: number | string - } - | { high: number | string } - | { low: number | string } - } - [SearchFilterOperator.EQUAL]?: { - [key: string]: any - } - [SearchFilterOperator.NOT_EQUAL]?: { - [key: string]: any - } - [SearchFilterOperator.EMPTY]?: { - [key: string]: any - } - [SearchFilterOperator.NOT_EMPTY]?: { - [key: string]: any - } - [SearchFilterOperator.ONE_OF]?: { - [key: string]: any[] - } - [SearchFilterOperator.CONTAINS]?: { - [key: string]: any[] - } - [SearchFilterOperator.NOT_CONTAINS]?: { - [key: string]: any[] - } - [SearchFilterOperator.CONTAINS_ANY]?: { - [key: string]: any[] - } + [SearchFilterOperator.STRING]?: BasicFilter + [SearchFilterOperator.FUZZY]?: BasicFilter + [SearchFilterOperator.RANGE]?: RangeFilter + [SearchFilterOperator.EQUAL]?: BasicFilter + [SearchFilterOperator.NOT_EQUAL]?: BasicFilter + [SearchFilterOperator.EMPTY]?: BasicFilter + [SearchFilterOperator.NOT_EMPTY]?: BasicFilter + [SearchFilterOperator.ONE_OF]?: ArrayFilter + [SearchFilterOperator.CONTAINS]?: ArrayFilter + [SearchFilterOperator.NOT_CONTAINS]?: ArrayFilter + [SearchFilterOperator.CONTAINS_ANY]?: ArrayFilter // specific to SQS/SQLite search on internal tables this can be used // to make sure the documents returned are always filtered down to a // specific document type (such as just rows) From 288d48c60df759d1204fd895b45b55abe9f9b957 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 9 Jul 2024 15:04:41 +0200 Subject: [PATCH 25/28] Fix build --- packages/shared-core/src/filters.ts | 12 ++++-------- packages/types/src/sdk/search.ts | 6 +++--- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 3c6901e195..28f0b28425 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -310,16 +310,12 @@ export const buildQuery = (filter: SearchFilter[]) => { query.equal = query.equal || {} query.equal[field] = true } else { - query[queryOperator] = { - ...query[queryOperator], - [field]: value, - } + query[queryOperator] ??= {} + query[queryOperator]![field] = value } } else { - query[queryOperator] = { - ...query[queryOperator], - [field]: value, - } + query[queryOperator] ??= {} + query[queryOperator]![field] = value } } }) diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 856011284f..713b33ff54 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -21,7 +21,7 @@ export enum InternalSearchFilterOperator { COMPLEX_ID_OPERATOR = "_complexIdOperator", } -type BasicFilter = Record & { +type BasicFilter = Record & { [InternalSearchFilterOperator.COMPLEX_ID_OPERATOR]?: never } @@ -52,8 +52,8 @@ export interface SearchFilters { // allows just fuzzy to be or - all the fuzzy/like parameters fuzzyOr?: boolean onEmptyFilter?: EmptyFilterOption - [SearchFilterOperator.STRING]?: BasicFilter - [SearchFilterOperator.FUZZY]?: BasicFilter + [SearchFilterOperator.STRING]?: BasicFilter + [SearchFilterOperator.FUZZY]?: BasicFilter [SearchFilterOperator.RANGE]?: RangeFilter [SearchFilterOperator.EQUAL]?: BasicFilter [SearchFilterOperator.NOT_EQUAL]?: BasicFilter From a7b6004c1c24b34563ebd98c58541e2225a4de03 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 9 Jul 2024 15:09:54 +0200 Subject: [PATCH 26/28] Fix more typings --- packages/types/src/sdk/search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 713b33ff54..ccb73a7fba 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -25,7 +25,7 @@ type BasicFilter = Record & { [InternalSearchFilterOperator.COMPLEX_ID_OPERATOR]?: never } -type ArrayFilter = Record & { +type ArrayFilter = Record & { [InternalSearchFilterOperator.COMPLEX_ID_OPERATOR]?: { id: string[] values: string[] From eae559d24959f57261abdcb60064a7f2ace2c372 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 9 Jul 2024 15:14:37 +0200 Subject: [PATCH 27/28] Fix scim sync issues --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 11379517b7..7dbe323aec 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 11379517b76264a7f938c2d520bd259f586edada +Subproject commit 7dbe323aec724ae6336b13c06aaefa4a89837edf From 49fc362c340f2cba54536e3da9e2c8181e0b1ec2 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 9 Jul 2024 14:51:23 +0000 Subject: [PATCH 28/28] Bump version to 2.29.15 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index b3831d0886..4d2b5a6cf8 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.29.14", + "version": "2.29.15", "npmClient": "yarn", "packages": [ "packages/*",