From 80a772f39fb8d7cc23bbd03cee40947cd2e7c3c6 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 5 Apr 2024 13:15:06 +0100 Subject: [PATCH 01/16] Add snippets to app imports --- packages/server/src/api/controllers/application.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index ceef421fab..6acdfcd465 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -320,6 +320,7 @@ async function performAppCreate(ctx: UserCtx) { "theme", "customTheme", "icon", + "snippets", ] keys.forEach(key => { if (existing[key]) { From 6e4a66b2e1101566533f6397af9877a5882f20b3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 11 Apr 2024 18:19:47 +0100 Subject: [PATCH 02/16] Initial implementation of generating SQS junction table definitions. --- hosting/.env | 3 +- hosting/docker-compose.dev.yaml | 3 +- packages/backend-core/src/environment.ts | 2 +- packages/server/src/environment.ts | 2 + .../server/src/sdk/app/tables/internal/sqs.ts | 60 ++++++++++++++++--- packages/types/src/documents/app/sqlite.ts | 20 ++++--- 6 files changed, 72 insertions(+), 18 deletions(-) diff --git a/hosting/.env b/hosting/.env index 8a0756c0e3..173d409d04 100644 --- a/hosting/.env +++ b/hosting/.env @@ -17,6 +17,7 @@ APP_PORT=4002 WORKER_PORT=4003 MINIO_PORT=4004 COUCH_DB_PORT=4005 +COUCH_DB_SQS_PORT=4006 REDIS_PORT=6379 WATCHTOWER_PORT=6161 BUDIBASE_ENVIRONMENT=PRODUCTION @@ -28,4 +29,4 @@ BB_ADMIN_USER_PASSWORD= # A path that is watched for plugin bundles. Any bundles found are imported automatically/ PLUGINS_DIR= -ROLLING_LOG_MAX_SIZE= \ No newline at end of file +ROLLING_LOG_MAX_SIZE= diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index 9dba5d427c..77f6bd053b 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -42,12 +42,13 @@ services: couchdb-service: container_name: budi-couchdb3-dev restart: on-failure - image: budibase/couchdb + image: budibase/couchdb:v3.2.1-sqs environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} ports: - "${COUCH_DB_PORT}:5984" + - "${COUCH_DB_SQS_PORT}:4984" volumes: - couchdb_data:/data diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 2da2a77d67..8dbc904643 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -107,7 +107,7 @@ const environment = { ENCRYPTION_KEY: process.env.ENCRYPTION_KEY, API_ENCRYPTION_KEY: getAPIEncryptionKey(), COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", - COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984", + COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006", COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index f8adcbe0ee..d9d299d5fa 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -28,6 +28,7 @@ const DEFAULTS = { PLUGINS_DIR: "/plugins", FORKED_PROCESS_NAME: "main", JS_RUNNER_MEMORY_LIMIT: 64, + COUCH_DB_SQL_URL: "http://localhost:4006", } const QUERY_THREAD_TIMEOUT = @@ -39,6 +40,7 @@ const environment = { // important - prefer app port to generic port PORT: process.env.APP_PORT || process.env.PORT, COUCH_DB_URL: process.env.COUCH_DB_URL, + COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || DEFAULTS.COUCH_DB_SQL_URL, MINIO_URL: process.env.MINIO_URL, WORKER_URL: process.env.WORKER_URL, AWS_REGION: process.env.AWS_REGION, diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index 79d9be2348..5dd16f516c 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -1,8 +1,19 @@ import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core" -import { FieldType, SQLiteDefinition, SQLiteType, Table } from "@budibase/types" +import { + FieldType, + RelationshipFieldMetadata, + SQLiteDefinition, + SQLiteTable, + SQLiteTables, + SQLiteType, + Table, +} from "@budibase/types" import { cloneDeep } from "lodash" import tablesSdk from "../" -import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils" +import { + CONSTANT_INTERNAL_ROW_COLS, + generateJunctionTableID, +} from "../../../../db/utils" const BASIC_SQLITE_DOC: SQLiteDefinition = { _id: SQLITE_DESIGN_DOC_ID, @@ -36,9 +47,38 @@ const FieldTypeMap: Record = { [FieldType.BB_REFERENCE]: SQLiteType.TEXT, } -function mapTable(table: Table): { [key: string]: SQLiteType } { +function buildRelationshipDefinitions( + table: Table, + relationshipColumn: RelationshipFieldMetadata +): { + tableId: string + definition: SQLiteTable +} { + const tableId = table._id!, + relatedTableId = relationshipColumn.tableId + return { + tableId: generateJunctionTableID(tableId, relatedTableId), + definition: { + doc1: SQLiteType.BLOB, + doc2: SQLiteType.BLOB, + tableId: SQLiteType.TEXT, + }, + } +} + +// this can generate relationship tables as part of the mapping +function mapTable(table: Table): SQLiteTables { + const tables: SQLiteTables = {} const fields: Record = {} for (let [key, column] of Object.entries(table.schema)) { + // relationships should be handled differently + if (column.type === FieldType.LINK) { + const { tableId, definition } = buildRelationshipDefinitions( + table, + column + ) + tables[tableId] = { fields: definition } + } if (!FieldTypeMap[column.type]) { throw new Error(`Unable to map type "${column.type}" to SQLite type`) } @@ -49,10 +89,12 @@ function mapTable(table: Table): { [key: string]: SQLiteType } { CONSTANT_INTERNAL_ROW_COLS.forEach(col => { constantMap[col] = SQLiteType.TEXT }) - return { + const thisTable: SQLiteTable = { ...constantMap, ...fields, } + tables[table._id!] = { fields: thisTable } + return tables } // nothing exists, need to iterate though existing tables @@ -60,8 +102,9 @@ async function buildBaseDefinition(): Promise { const tables = await tablesSdk.getAllInternalTables() const definition = cloneDeep(BASIC_SQLITE_DOC) for (let table of tables) { - definition.sql.tables[table._id!] = { - fields: mapTable(table), + definition.sql.tables = { + ...definition.sql.tables, + ...mapTable(table), } } return definition @@ -75,8 +118,9 @@ export async function addTableToSqlite(table: Table) { } catch (err) { definition = await buildBaseDefinition() } - definition.sql.tables[table._id!] = { - fields: mapTable(table), + definition.sql.tables = { + ...definition.sql.tables, + ...mapTable(table), } await db.put(definition) } diff --git a/packages/types/src/documents/app/sqlite.ts b/packages/types/src/documents/app/sqlite.ts index 76c47bbd74..e23a68b336 100644 --- a/packages/types/src/documents/app/sqlite.ts +++ b/packages/types/src/documents/app/sqlite.ts @@ -6,17 +6,23 @@ export enum SQLiteType { NUMERIC = "NUMERIC", } +export type SQLiteTable = Record< + string, + SQLiteType | { field: string; type: SQLiteType } +> + +export type SQLiteTables = Record< + string, + { + fields: SQLiteTable + } +> + export interface SQLiteDefinition { _id: string language: string sql: { - tables: { - [tableName: string]: { - fields: { - [key: string]: SQLiteType | { field: string; type: SQLiteType } - } - } - } + tables: SQLiteTables options: { table_name: string } From d6b252013b80d22bb87677a8494c55663c799de6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 11 Apr 2024 18:25:18 +0100 Subject: [PATCH 03/16] Quick fix to link document structure in sqlite. --- packages/server/src/sdk/app/tables/internal/sqs.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index 5dd16f516c..99240c28d4 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -59,8 +59,12 @@ function buildRelationshipDefinitions( return { tableId: generateJunctionTableID(tableId, relatedTableId), definition: { - doc1: SQLiteType.BLOB, - doc2: SQLiteType.BLOB, + ["doc1.rowId"]: SQLiteType.TEXT, + ["doc1.tableId"]: SQLiteType.TEXT, + ["doc1.fieldName"]: SQLiteType.TEXT, + ["doc2.rowId"]: SQLiteType.TEXT, + ["doc2.tableId"]: SQLiteType.TEXT, + ["doc2.fieldName"]: SQLiteType.TEXT, tableId: SQLiteType.TEXT, }, } From ebb79c16fe7d626d9d201fe5d1090da6e4ab078f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 12 Apr 2024 16:15:36 +0100 Subject: [PATCH 04/16] Aliasing support for SQS. --- .../api/controllers/row/ExternalRequest.ts | 3 +- packages/server/src/db/utils.ts | 1 + .../src/integrations/tests/sqlAlias.spec.ts | 4 +- packages/server/src/sdk/app/rows/index.ts | 2 + .../server/src/sdk/app/rows/search/sqs.ts | 99 +++++++++++-------- .../row/alias.ts => sdk/app/rows/sqlAlias.ts} | 28 ++++-- 6 files changed, 86 insertions(+), 51 deletions(-) rename packages/server/src/{api/controllers/row/alias.ts => sdk/app/rows/sqlAlias.ts} (87%) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 7fc0333de1..4adbb72c7a 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -36,7 +36,6 @@ import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils" import { processObjectSync } from "@budibase/string-templates" import { cloneDeep } from "lodash/fp" import { db as dbCore } from "@budibase/backend-core" -import AliasTables from "./alias" import sdk from "../../../sdk" import env from "../../../environment" @@ -618,7 +617,7 @@ export class ExternalRequest { if (env.SQL_ALIASING_DISABLE) { response = await getDatasourceAndQuery(json) } else { - const aliasing = new AliasTables(Object.keys(this.tables)) + const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables)) response = await aliasing.queryWithAliasing(json) } diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index b1c02b1764..ce8d0accbb 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -40,6 +40,7 @@ export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.Inte export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}` export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}` export const AUTOMATION_LOG_PREFIX = `${DocumentType.AUTOMATION_LOG}${SEPARATOR}` +export const SQS_DATASOURCE_INTERNAL = "internal" export const ViewName = dbCore.ViewName export const InternalTables = dbCore.InternalTable export const UNICODE_MAX = dbCore.UNICODE_MAX diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index bfca24ff7d..58c3a05245 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -8,8 +8,10 @@ import { import { join } from "path" import Sql from "../base/sql" import { SqlClient } from "../utils" -import AliasTables from "../../api/controllers/row/alias" import { generator } from "@budibase/backend-core/tests" +import sdk from "../../sdk" + +const AliasTables = sdk.rows.AliasTables function multiline(sql: string) { return sql.replace(/\n/g, "").replace(/ +/g, " ") diff --git a/packages/server/src/sdk/app/rows/index.ts b/packages/server/src/sdk/app/rows/index.ts index ea501e93d9..c117941419 100644 --- a/packages/server/src/sdk/app/rows/index.ts +++ b/packages/server/src/sdk/app/rows/index.ts @@ -3,6 +3,7 @@ import * as rows from "./rows" import * as search from "./search" import * as utils from "./utils" import * as external from "./external" +import AliasTables from "./sqlAlias" export default { ...attachments, @@ -10,4 +11,5 @@ export default { ...search, utils, external, + AliasTables, } diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 5b0b6e3bc7..20edb988d3 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -20,7 +20,12 @@ import { } from "../../../../api/controllers/row/utils" import sdk from "../../../index" import { context } from "@budibase/backend-core" -import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils" +import { + CONSTANT_INTERNAL_ROW_COLS, + SQS_DATASOURCE_INTERNAL, +} from "../../../../db/utils" +import AliasTables from "../sqlAlias" +import { outputProcessing } from "../../../../utilities/rowProcessor" function buildInternalFieldList( table: Table, @@ -31,19 +36,19 @@ function buildInternalFieldList( fieldList = fieldList.concat( CONSTANT_INTERNAL_ROW_COLS.map(col => `${table._id}.${col}`) ) - if (opts.relationships) { - for (let col of Object.values(table.schema)) { - if (col.type === FieldType.LINK) { - const linkCol = col as RelationshipFieldMetadata - const relatedTable = tables.find( - table => table._id === linkCol.tableId - )! - fieldList = fieldList.concat( - buildInternalFieldList(relatedTable, tables, { relationships: false }) - ) - } else { - fieldList.push(`${table._id}.${col.name}`) - } + for (let col of Object.values(table.schema)) { + const isRelationship = col.type === FieldType.LINK + if (!opts.relationships && isRelationship) { + continue + } + if (isRelationship) { + const linkCol = col as RelationshipFieldMetadata + const relatedTable = tables.find(table => table._id === linkCol.tableId)! + fieldList = fieldList.concat( + buildInternalFieldList(relatedTable, tables, { relationships: false }) + ) + } else { + fieldList.push(`${table._id}.${col.name}`) } } return fieldList @@ -94,14 +99,14 @@ function buildTableMap(tables: Table[]) { } export async function search( - options: RowSearchParams + options: RowSearchParams, + table: Table ): Promise> { - const { tableId, paginate, query, ...params } = options + const { paginate, query, ...params } = options const builder = new SqlQueryBuilder(SqlClient.SQL_LITE) const allTables = await sdk.tables.getAllInternalTables() const allTablesMap = buildTableMap(allTables) - const table = allTables.find(table => table._id === tableId) if (!table) { throw new Error("Unable to find table") } @@ -111,7 +116,7 @@ export async function search( const request: QueryJson = { endpoint: { // not important, we query ourselves - datasourceId: "internal", + datasourceId: SQS_DATASOURCE_INTERNAL, entityId: table._id!, operation: Operation.READ, }, @@ -154,34 +159,44 @@ export async function search( } } try { - const query = builder._query(request, { - disableReturning: true, + const alias = new AliasTables(allTables.map(table => table.name)) + const rows = await alias.queryWithAliasing(request, async json => { + const query = builder._query(json, { + disableReturning: true, + }) + + if (Array.isArray(query)) { + throw new Error("SQS cannot currently handle multiple queries") + } + + let sql = query.sql, + bindings = query.bindings + + // quick hack for docIds + sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`") + sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`") + + const db = context.getAppDB() + return await db.sql(sql, bindings) }) - if (Array.isArray(query)) { - throw new Error("SQS cannot currently handle multiple queries") - } - - let sql = query.sql, - bindings = query.bindings - - // quick hack for docIds - sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`") - sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`") - - const db = context.getAppDB() - const rows = await db.sql(sql, bindings) + // process from the format of tableId.column to expected format + const processed = await sqlOutputProcessing( + rows, + table!, + allTablesMap, + relationships, + { + sqs: true, + } + ) return { - rows: await sqlOutputProcessing( - rows, - table!, - allTablesMap, - relationships, - { - sqs: true, - } - ), + // final row processing for response + rows: await outputProcessing(table, processed, { + preserveLinks: true, + squash: true, + }), } } catch (err: any) { const msg = typeof err === "string" ? err : err.message diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts similarity index 87% rename from packages/server/src/api/controllers/row/alias.ts rename to packages/server/src/sdk/app/rows/sqlAlias.ts index 0ec9d1a09c..0fc338ecbe 100644 --- a/packages/server/src/api/controllers/row/alias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -6,11 +6,12 @@ import { Row, SearchFilters, } from "@budibase/types" -import { getSQLClient } from "../../../sdk/app/rows/utils" +import { getSQLClient } from "./utils" import { cloneDeep } from "lodash" -import sdk from "../../../sdk" +import datasources from "../datasources" import { makeExternalQuery } from "../../../integrations/base/query" import { SqlClient } from "../../../integrations/utils" +import { SQS_DATASOURCE_INTERNAL } from "../../../db/utils" const WRITE_OPERATIONS: Operation[] = [ Operation.CREATE, @@ -156,12 +157,19 @@ export default class AliasTables { } async queryWithAliasing( - json: QueryJson + json: QueryJson, + queryFn?: (json: QueryJson) => Promise ): Promise { const datasourceId = json.endpoint.datasourceId - const datasource = await sdk.datasources.get(datasourceId) + const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL + let aliasingEnabled: boolean, datasource: Datasource | undefined + if (isSqs) { + aliasingEnabled = true + } else { + datasource = await datasources.get(datasourceId) + aliasingEnabled = this.isAliasingEnabled(json, datasource) + } - const aliasingEnabled = this.isAliasingEnabled(json, datasource) if (aliasingEnabled) { json = cloneDeep(json) // run through the query json to update anywhere a table may be used @@ -207,7 +215,15 @@ export default class AliasTables { } json.tableAliases = invertedTableAliases } - const response = await makeExternalQuery(datasource, json) + + let response: DatasourcePlusQueryResponse + if (datasource && !isSqs) { + response = await makeExternalQuery(datasource, json) + } else if (queryFn) { + response = await queryFn(json) + } else { + throw new Error("No supplied method to perform aliased query") + } if (Array.isArray(response) && aliasingEnabled) { return this.reverse(response) } else { From c40e9656345f5e95ead009976ab00755973b4845 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 12 Apr 2024 16:16:31 +0100 Subject: [PATCH 05/16] Getting relationships working properly as well as renaming internal -> sqs in function opts. --- packages/server/src/api/controllers/row/utils/basic.ts | 9 +++++---- .../server/src/api/controllers/row/utils/sqlUtils.ts | 4 ++-- packages/server/src/api/controllers/row/utils/utils.ts | 6 ++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index 1fc84de9c7..6255e13c1c 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -62,12 +62,12 @@ export function basicProcessing({ row, table, isLinked, - internal, + sqs, }: { row: Row table: Table isLinked: boolean - internal?: boolean + sqs?: boolean }): Row { const thisRow: Row = {} // filter the row down to what is actually the row (not joined) @@ -84,12 +84,13 @@ export function basicProcessing({ thisRow[fieldName] = value } } - if (!internal) { + if (!sqs) { thisRow._id = generateIdForRow(row, table, isLinked) thisRow.tableId = table._id thisRow._rev = "rev" } else { - for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) { + const columns = Object.keys(table.schema) + for (let internalColumn of [...CONSTANT_INTERNAL_ROW_COLS, ...columns]) { thisRow[internalColumn] = extractFieldValue({ row, tableName: table._id!, diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index 6f9837e0ab..372b8394ff 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -51,11 +51,11 @@ export async function updateRelationshipColumns( continue } - let linked = await basicProcessing({ + let linked = basicProcessing({ row, table: linkedTable, isLinked: true, - internal: opts?.sqs, + sqs: opts?.sqs, }) if (!linked._id) { continue diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index f387a468cf..bf9ede6fe3 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -132,6 +132,7 @@ export async function sqlOutputProcessing( let rowId = row._id if (opts?.sqs) { rowId = getInternalRowId(row, table) + row._id = rowId } else if (!rowId) { rowId = generateIdForRow(row, table) row._id = rowId @@ -153,7 +154,7 @@ export async function sqlOutputProcessing( row, table, isLinked: false, - internal: opts?.sqs, + sqs: opts?.sqs, }), table ) @@ -167,7 +168,8 @@ export async function sqlOutputProcessing( tables, row, finalRows, - relationships + relationships, + opts ) } From bfb7750213400e833bfb4ffeae0be9462b66e0bc Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 12 Apr 2024 16:17:06 +0100 Subject: [PATCH 06/16] Getting search input mapping up a level in the search SDK - avoids having to call it for every search type. --- packages/server/src/sdk/app/rows/search.ts | 11 +++++++--- .../src/sdk/app/rows/search/external.ts | 13 +++++++----- .../src/sdk/app/rows/search/internal.ts | 20 ++++++++++--------- .../app/rows/search/tests/external.spec.ts | 6 +++--- .../app/rows/search/tests/internal.spec.ts | 4 ++-- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index f681bfeb90..5d8f7ef80b 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -13,6 +13,8 @@ import * as sqs from "./search/sqs" import env from "../../../environment" import { ExportRowsParams, ExportRowsResult } from "./search/types" import { dataFilters } from "@budibase/shared-core" +import sdk from "../../index" +import { searchInputMapping } from "./search/utils" export { isValidFilter } from "../../../integrations/utils" @@ -72,12 +74,15 @@ export async function search( } } + const table = await sdk.tables.getTable(options.tableId) + options = searchInputMapping(table, options) + if (isExternalTable) { - return external.search(options) + return external.search(options, table) } else if (env.SQS_SEARCH_ENABLE) { - return sqs.search(options) + return sqs.search(options, table) } else { - return internal.search(options) + return internal.search(options, table) } } diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index e0a3bad94e..077f971903 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -8,6 +8,7 @@ import { SearchFilters, RowSearchParams, SearchResponse, + Table, } from "@budibase/types" import * as exporters from "../../../../api/controllers/view/exporters" import { handleRequest } from "../../../../api/controllers/row/external" @@ -18,13 +19,13 @@ import { import { utils } from "@budibase/shared-core" import { ExportRowsParams, ExportRowsResult } from "./types" import { HTTPError, db } from "@budibase/backend-core" -import { searchInputMapping } from "./utils" import pick from "lodash/pick" import { outputProcessing } from "../../../../utilities/rowProcessor" import sdk from "../../../" export async function search( - options: RowSearchParams + options: RowSearchParams, + table: Table ): Promise> { const { tableId } = options const { paginate, query, ...params } = options @@ -68,8 +69,6 @@ export async function search( } try { - const table = await sdk.tables.getTable(tableId) - options = searchInputMapping(table, options) let rows = await handleRequest(Operation.READ, tableId, { filters: query, sort, @@ -150,11 +149,15 @@ export async function exportRows( } const datasource = await sdk.datasources.get(datasourceId!) + const table = await sdk.tables.getTable(tableId) if (!datasource || !datasource.entities) { throw new HTTPError("Datasource has not been configured for plus API.", 400) } - let result = await search({ tableId, query: requestQuery, sort, sortOrder }) + let result = await search( + { tableId, query: requestQuery, sort, sortOrder }, + table + ) let rows: Row[] = [] let headers diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 610807a10e..ffd13ed731 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -33,7 +33,8 @@ import pick from "lodash/pick" import { breakRowIdField } from "../../../../integrations/utils" export async function search( - options: RowSearchParams + options: RowSearchParams, + table: Table ): Promise> { const { tableId } = options @@ -51,8 +52,6 @@ export async function search( query: {}, } - let table = await sdk.tables.getTable(tableId) - options = searchInputMapping(table, options) if (params.sort && !params.sortType) { const schema = table.schema const sortField = schema[params.sort] @@ -122,12 +121,15 @@ export async function exportRows( result = await outputProcessing(table, response) } else if (query) { - let searchResponse = await search({ - tableId, - query, - sort, - sortOrder, - }) + let searchResponse = await search( + { + tableId, + query, + sort, + sortOrder, + }, + table + ) result = searchResponse.rows } diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts index f2bdec4692..53bc049a9b 100644 --- a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts @@ -112,7 +112,7 @@ describe("external search", () => { tableId, query: {}, } - const result = await search(searchParams) + const result = await search(searchParams, config.table!) expect(result.rows).toHaveLength(10) expect(result.rows).toEqual( @@ -130,7 +130,7 @@ describe("external search", () => { query: {}, fields: ["name", "age"], } - const result = await search(searchParams) + const result = await search(searchParams, config.table!) expect(result.rows).toHaveLength(10) expect(result.rows).toEqual( @@ -157,7 +157,7 @@ describe("external search", () => { }, }, } - const result = await search(searchParams) + const result = await search(searchParams, config.table!) expect(result.rows).toHaveLength(3) expect(result.rows.map(row => row.id)).toEqual([1, 4, 8]) diff --git a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts index 5be0f4a258..1c5f396737 100644 --- a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts @@ -81,7 +81,7 @@ describe("internal", () => { tableId, query: {}, } - const result = await search(searchParams) + const result = await search(searchParams, config.table!) expect(result.rows).toHaveLength(10) expect(result.rows).toEqual( @@ -99,7 +99,7 @@ describe("internal", () => { query: {}, fields: ["name", "age"], } - const result = await search(searchParams) + const result = await search(searchParams, config.table!) expect(result.rows).toHaveLength(10) expect(result.rows).toEqual( From 7d7de33cabbcfcefb7ea668a92813b96c1e28b3b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 12 Apr 2024 16:29:48 +0100 Subject: [PATCH 07/16] Removing CouchDB SQS image for now. --- hosting/docker-compose.dev.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index 77f6bd053b..9dba5d427c 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -42,13 +42,12 @@ services: couchdb-service: container_name: budi-couchdb3-dev restart: on-failure - image: budibase/couchdb:v3.2.1-sqs + image: budibase/couchdb environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} ports: - "${COUCH_DB_PORT}:5984" - - "${COUCH_DB_SQS_PORT}:4984" volumes: - couchdb_data:/data From aeda5931c07c84ed826e5e37bd54b3382d955653 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 12 Apr 2024 16:34:33 +0100 Subject: [PATCH 08/16] Fixing lint. --- packages/server/src/sdk/app/rows/search/internal.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index ffd13ed731..906ca016d1 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -1,6 +1,6 @@ import { context, db, HTTPError } from "@budibase/backend-core" import env from "../../../../environment" -import { fullSearch, paginatedSearch, searchInputMapping } from "./utils" +import { fullSearch, paginatedSearch } from "./utils" import { getRowParams, InternalTables } from "../../../../db/utils" import { Database, From 68c5e657ddd0fb5b45948e418531d72cd16b178e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 15 Apr 2024 13:46:31 +0100 Subject: [PATCH 09/16] Updating @types/archiver to be more specific. --- packages/server/package.json | 2 +- yarn.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index ad03033e67..76402785d7 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -125,7 +125,7 @@ "@babel/preset-env": "7.16.11", "@swc/core": "1.3.71", "@swc/jest": "0.2.27", - "@types/archiver": "^6.0.2", + "@types/archiver": "6.0.2", "@types/global-agent": "2.1.1", "@types/google-spreadsheet": "3.1.5", "@types/jest": "29.5.5", diff --git a/yarn.lock b/yarn.lock index a36b54d3be..ce39c89075 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5174,7 +5174,7 @@ dependencies: "@types/node" "*" -"@types/archiver@^6.0.2": +"@types/archiver@6.0.2": version "6.0.2" resolved "https://registry.yarnpkg.com/@types/archiver/-/archiver-6.0.2.tgz#0daf8c83359cbde69de1e4b33dcade6a48a929e2" integrity sha512-KmROQqbQzKGuaAbmK+ZcytkJ51+YqDa7NmbXjmtC5YBLSyQYo21YaUnQ3HbaPFKL1ooo6RQ6OPYPIDyxfpDDXw== From 203e32ecc6c39f836f74711f2a3b89f70a4ef167 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 15 Apr 2024 15:09:17 +0100 Subject: [PATCH 10/16] Commenting the field type enumeration to better explain what all of the types do and how they are represented within Budibase. --- packages/types/src/documents/app/row.ts | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts index 222c346591..ccdf001965 100644 --- a/packages/types/src/documents/app/row.ts +++ b/packages/types/src/documents/app/row.ts @@ -1,22 +1,76 @@ import { Document } from "../document" export enum FieldType { + // a primitive type, stores a string, called Text within Budibase. This is one of the default + // types of Budibase, if an external type is not fully understood, we will treat it as text. STRING = "string", + // similar to string type, called Long Form Text within Budibase. This is mainly a frontend + // orientated type which disables a larger text input area. This can also be used + // in conjunction with the 'useRichText' option to support a markdown editor/viewer. LONGFORM = "longform", + // similar to string type, called Options within Budibase. This works very similarly to + // the string type within the backend, but is validated to a list of options. This will + // be displayed a select input within the builder/client. OPTIONS = "options", + // a primitive type, stores a number, as a floating point, called Number within Budibase. + // this type will always represent numbers as reals/floating point - there is no integer only + // type within Budibase. NUMBER = "number", + // a primitive type, stores a boolean, called Boolean within Budibase. This is often represented + // as a toggle or checkbox within forms/grids. BOOLEAN = "boolean", + // a JSON type, this type is always an array of strings, called Multi-select within Budibase. + // This type can be compared to the options type, as it functions similarly, but allows picking + // multiple options rather than a single option. ARRAY = "array", + // a string type, this is always a string when input/returned from the API, called Date/Time within + // Budibase. We utilise ISO date strings for representing dates, this type has a range of sub-types + // to restrict it to date only, time only and ignore timezone capabilities. DATETIME = "datetime", + // a JSON type, an array of metadata about files held in object storage, called Attachment List within + // Budibase. To utilise this type there is an API for uploading files to Budibase, which returns metadata + // that can be stored against columns of this type. Currently this is not supported on external databases. ATTACHMENTS = "attachment", + // a JSON type, similar to the attachments type, called Attachment within Budibase. This type functions + // much the same as the attachment list, but only holds a single attachment metadata as an object. + // This simpifies the binding experience of using this column type. ATTACHMENT_SINGLE = "attachment_single", + // a complex type, called Relationships within Budibase. This is the most complex type of Budibase, + // nothing should be stored against rows under link columns; this type simply represents the + // relationship between tables as part of the table schema. When rows are input to the Budibase API + // relationships to be made are represented as a list of row IDs to link. When rows are returned + // from the Budibase API it will contain a list of row IDs and display column values of the related rows. LINK = "link", + // a complex type, called Formulas within Budibase. This type has two variants, static and dynamic, with + // static only being supported against internal tables. Dynamic formulas calculate a provided HBS/JS binding + // based on the row context and enrich it when rows are being returned from the API. Static bindings calculate + // this when rows are being stored, so that the formula output can be searched upon within the DB. FORMULA = "formula", + // a complex type, called Auto Column within Budibase. This type has a few variants, with options such as a + // date for created at/updated at, an auto ID column with auto-increments as rows are saved and a user + // relationship type which stores the created by/updated by user details. This sub-types all depend on the + // date, number of link types respectively. AUTO = "auto", + // a JSON type, called JSON within Budibase. This type allows any arbitrary JSON to be input to this column + // type, which will be represented a string in the row. This type depends on a schema being provided to make the + // JSON searchable/bindable, the JSON cannot be fully dynamic. JSON = "json", + // an internal type, this is an old deprecated type which is no longer used - still represented to note it + // could appear in very old tables. INTERNAL = "internal", + // a string type, called Barcode/QR within Budibase. This type is used to denote to forms to that this column + // should be filled in using a camera to read a barcode, there is a form component which will be used when this + // type is found. The column will contain the contents of any barcode scanned. BARCODEQR = "barcodeqr", + // a string type, this allows representing very large integers, but they are held/managed within Budibase as + // strings. When stored in external databases Budibase will attempt to use a real big integer type and depend + // on the database parsing the string to this type as part of saving. BIGINT = "bigint", + // a JSON type, called User within Budibase. This type is used to represent a link to an internal Budibase + // resource, like a user or group, today only users are supported. This type will be represented as an + // array of internal resource IDs (e.g. user IDs) within the row - this ID list will be enriched with + // the full resources when rows are returned from the API. The full resources can be input to the API, or + // an array of resource IDs, the API will squash these down and validate them before saving the row. BB_REFERENCE = "bb_reference", } From 81425b3d287340054d4796e7e6763ce72e7e7d23 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 15 Apr 2024 15:50:25 +0100 Subject: [PATCH 11/16] Addressing PR comment.s --- packages/types/src/documents/app/row.ts | 142 +++++++++++++++--------- 1 file changed, 88 insertions(+), 54 deletions(-) diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts index ccdf001965..4f2f9f99ef 100644 --- a/packages/types/src/documents/app/row.ts +++ b/packages/types/src/documents/app/row.ts @@ -1,76 +1,110 @@ import { Document } from "../document" export enum FieldType { - // a primitive type, stores a string, called Text within Budibase. This is one of the default - // types of Budibase, if an external type is not fully understood, we will treat it as text. + /** + * a primitive type, stores a string, called Text within Budibase. This is one of the default + * types of Budibase, if an external type is not fully understood, we will treat it as text. + */ STRING = "string", - // similar to string type, called Long Form Text within Budibase. This is mainly a frontend - // orientated type which disables a larger text input area. This can also be used - // in conjunction with the 'useRichText' option to support a markdown editor/viewer. + /** + * similar to string type, called Long Form Text within Budibase. This is mainly a frontend + * orientated type which enables a larger text input area. This can also be used + * in conjunction with the 'useRichText' option to support a markdown editor/viewer. + */ LONGFORM = "longform", - // similar to string type, called Options within Budibase. This works very similarly to - // the string type within the backend, but is validated to a list of options. This will - // be displayed a select input within the builder/client. + /** + * similar to string type, called Options within Budibase. This works very similarly to + * the string type within the backend, but is validated to a list of options. This will + * display a