From 5066da2630787d24ac17f09b33798da5c0c601d0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Jul 2024 16:03:03 +0100 Subject: [PATCH 1/4] Grouping and exporting default tables. --- packages/server/src/constants/index.ts | 6 ++++++ .../server/src/db/defaultData/datasource_bb_default.ts | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index 60875b3daa..4fcc9c5d06 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -173,3 +173,9 @@ export const DEFAULT_INVENTORY_TABLE_ID = constants.DEFAULT_INVENTORY_TABLE_ID export const DEFAULT_EXPENSES_TABLE_ID = constants.DEFAULT_EXPENSES_TABLE_ID export const DEFAULT_EMPLOYEE_TABLE_ID = constants.DEFAULT_EMPLOYEE_TABLE_ID export const DEFAULT_BB_DATASOURCE_ID = constants.DEFAULT_BB_DATASOURCE_ID +export const DEFAULT_TABLE_IDS = [ + DEFAULT_JOBS_TABLE_ID, + DEFAULT_INVENTORY_TABLE_ID, + DEFAULT_EXPENSES_TABLE_ID, + DEFAULT_EMPLOYEE_TABLE_ID, +] diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts index 68d49b2d8b..fc79b90c00 100644 --- a/packages/server/src/db/defaultData/datasource_bb_default.ts +++ b/packages/server/src/db/defaultData/datasource_bb_default.ts @@ -619,6 +619,13 @@ export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = { }, } +export const DEFAULT_TABLES: Table[] = [ + DEFAULT_INVENTORY_TABLE_SCHEMA, + DEFAULT_EMPLOYEE_TABLE_SCHEMA, + DEFAULT_JOBS_TABLE_SCHEMA, + DEFAULT_EXPENSES_TABLE_SCHEMA, +] + export async function buildDefaultDocs() { const inventoryData = await tableImport( DEFAULT_INVENTORY_TABLE_SCHEMA, From 34d073bcb7ef073de97044056f0adca842edc6fe Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Jul 2024 16:03:38 +0100 Subject: [PATCH 2/4] Adding default tables to sync. --- packages/server/src/sdk/app/rows/search/sqs.ts | 18 ++++++++++++++---- .../server/src/sdk/app/tables/internal/sqs.ts | 4 +++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 2c9ee1356c..72a1557cc9 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -41,6 +41,7 @@ import { getTableIDList, } from "./filters" import { dataFilters } from "@budibase/shared-core" +import { DEFAULT_TABLE_IDS } from "../../../../constants" const builder = new sql.Sql(SqlClient.SQL_LITE) const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`) @@ -211,6 +212,18 @@ async function runSqlQuery( return response } +function resyncDefinitionsRequired(status: number, message: string) { + // pre data_ prefix on column names, need to resync + return ( + (status === 400 && message?.match(USER_COLUMN_PREFIX_REGEX)) || + // default tables aren't included in definition + (status === 400 && + DEFAULT_TABLE_IDS.find(tableId => message?.includes(tableId))) || + // no design document found, needs a full sync + (status === 404 && message?.includes(SQLITE_DESIGN_DOC_ID)) + ) +} + export async function search( options: RowSearchParams, table: Table @@ -338,10 +351,7 @@ export async function search( return response } catch (err: any) { const msg = typeof err === "string" ? err : err.message - const syncAndRepeat = - (err.status === 400 && msg?.match(USER_COLUMN_PREFIX_REGEX)) || - (err.status === 404 && msg?.includes(SQLITE_DESIGN_DOC_ID)) - if (syncAndRepeat) { + if (resyncDefinitionsRequired(err.status, msg)) { await sdk.tables.sqs.syncDefinition() return search(options, table) } diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index 9e831f4af7..2d49adf96e 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -15,6 +15,7 @@ import { generateJunctionTableID, } from "../../../../db/utils" import { isEqual } from "lodash" +import { DEFAULT_TABLES } from "../../../../db/defaultData/datasource_bb_default" const FieldTypeMap: Record = { [FieldType.BOOLEAN]: SQLiteType.NUMERIC, @@ -126,8 +127,9 @@ function mapTable(table: Table): SQLiteTables { // nothing exists, need to iterate though existing tables async function buildBaseDefinition(): Promise { const tables = await tablesSdk.getAllInternalTables() + const defaultTables = DEFAULT_TABLES const definition = sql.designDoc.base("tableId") - for (let table of tables) { + for (let table of tables.concat(defaultTables)) { definition.sql.tables = { ...definition.sql.tables, ...mapTable(table), From 39523685d696b8828c1890dfbf2061cce7e3fa8e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Jul 2024 16:20:27 +0100 Subject: [PATCH 3/4] Adding test case for searching sample data. --- .../src/api/routes/tests/search.spec.ts | 20 +++++++++++++++++++ .../src/tests/utilities/api/application.ts | 4 ++++ 2 files changed, 24 insertions(+) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 145db9b4a3..6b1fd2a198 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -30,6 +30,8 @@ import { encodeJSBinding } from "@budibase/string-templates" import { dataFilters } from "@budibase/shared-core" import { Knex } from "knex" import { structures } from "@budibase/backend-core/tests" +import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default" +import { employeeImport } from "../../../db/defaultData/employeeImport" describe.each([ ["in-memory", undefined], @@ -2167,6 +2169,24 @@ describe.each([ } ) + isInternal && + describe("sample data", () => { + beforeAll(async () => { + await config.api.application.addSampleData(config.appId!) + table = DEFAULT_EMPLOYEE_TABLE_SCHEMA + }) + + it("should be able to search sample data", async () => { + await expectSearch({ + query: {}, + }).toContain([ + { + "First Name": "Mandy", + }, + ]) + }) + }) + describe.each([ "名前", // Japanese for "name" "Benutzer-ID", // German for "user ID", includes a hyphen diff --git a/packages/server/src/tests/utilities/api/application.ts b/packages/server/src/tests/utilities/api/application.ts index bb9357c893..516af5c973 100644 --- a/packages/server/src/tests/utilities/api/application.ts +++ b/packages/server/src/tests/utilities/api/application.ts @@ -149,4 +149,8 @@ export class ApplicationAPI extends TestAPI { query: { status }, }) } + + addSampleData = async (appId: string): Promise => { + await this._post(`/api/applications/${appId}/sample`) + } } From eafe66d01e638c2e06a55a8380394b6e2cb6d32f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Jul 2024 16:26:26 +0100 Subject: [PATCH 4/4] Linting. --- packages/server/src/api/routes/tests/search.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 6b1fd2a198..a832c26f42 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -31,7 +31,6 @@ import { dataFilters } from "@budibase/shared-core" import { Knex } from "knex" import { structures } from "@budibase/backend-core/tests" import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default" -import { employeeImport } from "../../../db/defaultData/employeeImport" describe.each([ ["in-memory", undefined],