From 6db0379504f52552bc6f8d918936a61d323946bc Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 11 Jul 2024 16:35:57 +0100 Subject: [PATCH 1/4] Resync if it is found that a table or column is missing, this was previously done for specific cases but have expanded to cover all, but only retry once (not get into an infinite loop). --- packages/server/src/sdk/app/rows/search/sqs.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 4745aee7fb..0ae43563de 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -240,10 +240,10 @@ async function runSqlQuery( function resyncDefinitionsRequired(status: number, message: string) { // pre data_ prefix on column names, need to resync return ( - (status === 400 && message?.match(USER_COLUMN_PREFIX_REGEX)) || - // default tables aren't included in definition - (status === 400 && - DEFAULT_TABLE_IDS.find(tableId => message?.includes(tableId))) || + // there are tables missing - try a resync + (status === 400 && message.includes("no such table: ")) || + // there are columns missing - try a resync + (status === 400 && message.includes("no such column: ")) || // no design document found, needs a full sync (status === 404 && message?.includes(SQLITE_DESIGN_DOC_ID)) ) @@ -251,7 +251,8 @@ function resyncDefinitionsRequired(status: number, message: string) { export async function search( options: RowSearchParams, - table: Table + table: Table, + opts?: { retry?: boolean } ): Promise> { let { paginate, query, ...params } = options @@ -376,9 +377,9 @@ export async function search( return response } catch (err: any) { const msg = typeof err === "string" ? err : err.message - if (resyncDefinitionsRequired(err.status, msg)) { + if (!opts?.retry && resyncDefinitionsRequired(err.status, msg)) { await sdk.tables.sqs.syncDefinition() - return search(options, table) + return search(options, table, { retry: true }) } // previously the internal table didn't error when a column didn't exist in search if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) { From 1e189450c08feded0b36ee28e2c178608a70950e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 11 Jul 2024 16:38:30 +0100 Subject: [PATCH 2/4] Switching to regex. --- packages/server/src/sdk/app/rows/search/sqs.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 0ae43563de..ead970ca82 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -45,13 +45,10 @@ import { getTableIDList, } from "./filters" import { dataFilters } from "@budibase/shared-core" -import { DEFAULT_TABLE_IDS } from "../../../../constants" const builder = new sql.Sql(SqlClient.SQL_LITE) const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`) -const USER_COLUMN_PREFIX_REGEX = new RegExp( - `no such column: .+${USER_COLUMN_PREFIX}` -) +const MISSING_TABLE_REGX = new RegExp(`no such table: .+`) function buildInternalFieldList( table: Table, @@ -241,9 +238,9 @@ function resyncDefinitionsRequired(status: number, message: string) { // pre data_ prefix on column names, need to resync return ( // there are tables missing - try a resync - (status === 400 && message.includes("no such table: ")) || + (status === 400 && message.match(MISSING_TABLE_REGX)) || // there are columns missing - try a resync - (status === 400 && message.includes("no such column: ")) || + (status === 400 && message.match(MISSING_COLUMN_REGEX)) || // no design document found, needs a full sync (status === 404 && message?.includes(SQLITE_DESIGN_DOC_ID)) ) From 5b1850d28f8146a4d1ea0031ff9b93aa742e177f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 11 Jul 2024 16:41:12 +0100 Subject: [PATCH 3/4] PR comments. --- packages/server/src/sdk/app/rows/search/sqs.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index ead970ca82..4bf3c374e4 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -249,7 +249,7 @@ function resyncDefinitionsRequired(status: number, message: string) { export async function search( options: RowSearchParams, table: Table, - opts?: { retry?: boolean } + opts?: { retries?: number } ): Promise> { let { paginate, query, ...params } = options @@ -374,9 +374,10 @@ export async function search( return response } catch (err: any) { const msg = typeof err === "string" ? err : err.message - if (!opts?.retry && resyncDefinitionsRequired(err.status, msg)) { + const firstTry = !opts?.retries || opts.retries === 0 + if (firstTry && resyncDefinitionsRequired(err.status, msg)) { await sdk.tables.sqs.syncDefinition() - return search(options, table, { retry: true }) + return search(options, table, { retries: 1 }) } // previously the internal table didn't error when a column didn't exist in search if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) { From 9129e47dd941213391fba1b59c8f46431a867e74 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 11 Jul 2024 17:09:41 +0100 Subject: [PATCH 4/4] PR comment 2. --- packages/server/src/sdk/app/rows/search/sqs.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 4bf3c374e4..7042d6fa2c 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -249,7 +249,7 @@ function resyncDefinitionsRequired(status: number, message: string) { export async function search( options: RowSearchParams, table: Table, - opts?: { retries?: number } + opts?: { retrying?: boolean } ): Promise> { let { paginate, query, ...params } = options @@ -374,10 +374,9 @@ export async function search( return response } catch (err: any) { const msg = typeof err === "string" ? err : err.message - const firstTry = !opts?.retries || opts.retries === 0 - if (firstTry && resyncDefinitionsRequired(err.status, msg)) { + if (!opts?.retrying && resyncDefinitionsRequired(err.status, msg)) { await sdk.tables.sqs.syncDefinition() - return search(options, table, { retries: 1 }) + return search(options, table, { retrying: true }) } // previously the internal table didn't error when a column didn't exist in search if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) {