From 6e4a66b2e1101566533f6397af9877a5882f20b3 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 11 Apr 2024 18:19:47 +0100
Subject: [PATCH 01/11] Initial implementation of generating SQS junction table
definitions.
---
hosting/.env | 3 +-
hosting/docker-compose.dev.yaml | 3 +-
packages/backend-core/src/environment.ts | 2 +-
packages/server/src/environment.ts | 2 +
.../server/src/sdk/app/tables/internal/sqs.ts | 60 ++++++++++++++++---
packages/types/src/documents/app/sqlite.ts | 20 ++++---
6 files changed, 72 insertions(+), 18 deletions(-)
diff --git a/hosting/.env b/hosting/.env
index 8a0756c0e3..173d409d04 100644
--- a/hosting/.env
+++ b/hosting/.env
@@ -17,6 +17,7 @@ APP_PORT=4002
WORKER_PORT=4003
MINIO_PORT=4004
COUCH_DB_PORT=4005
+COUCH_DB_SQS_PORT=4006
REDIS_PORT=6379
WATCHTOWER_PORT=6161
BUDIBASE_ENVIRONMENT=PRODUCTION
@@ -28,4 +29,4 @@ BB_ADMIN_USER_PASSWORD=
# A path that is watched for plugin bundles. Any bundles found are imported automatically/
PLUGINS_DIR=
-ROLLING_LOG_MAX_SIZE=
\ No newline at end of file
+ROLLING_LOG_MAX_SIZE=
diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml
index 9dba5d427c..77f6bd053b 100644
--- a/hosting/docker-compose.dev.yaml
+++ b/hosting/docker-compose.dev.yaml
@@ -42,12 +42,13 @@ services:
couchdb-service:
container_name: budi-couchdb3-dev
restart: on-failure
- image: budibase/couchdb
+ image: budibase/couchdb:v3.2.1-sqs
environment:
- COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
- COUCHDB_USER=${COUCH_DB_USER}
ports:
- "${COUCH_DB_PORT}:5984"
+ - "${COUCH_DB_SQS_PORT}:4984"
volumes:
- couchdb_data:/data
diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index 2da2a77d67..8dbc904643 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -107,7 +107,7 @@ const environment = {
ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
API_ENCRYPTION_KEY: getAPIEncryptionKey(),
COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
- COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984",
+ COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006",
COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts
index f8adcbe0ee..d9d299d5fa 100644
--- a/packages/server/src/environment.ts
+++ b/packages/server/src/environment.ts
@@ -28,6 +28,7 @@ const DEFAULTS = {
PLUGINS_DIR: "/plugins",
FORKED_PROCESS_NAME: "main",
JS_RUNNER_MEMORY_LIMIT: 64,
+ COUCH_DB_SQL_URL: "http://localhost:4006",
}
const QUERY_THREAD_TIMEOUT =
@@ -39,6 +40,7 @@ const environment = {
// important - prefer app port to generic port
PORT: process.env.APP_PORT || process.env.PORT,
COUCH_DB_URL: process.env.COUCH_DB_URL,
+ COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || DEFAULTS.COUCH_DB_SQL_URL,
MINIO_URL: process.env.MINIO_URL,
WORKER_URL: process.env.WORKER_URL,
AWS_REGION: process.env.AWS_REGION,
diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index 79d9be2348..5dd16f516c 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -1,8 +1,19 @@
import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
-import { FieldType, SQLiteDefinition, SQLiteType, Table } from "@budibase/types"
+import {
+ FieldType,
+ RelationshipFieldMetadata,
+ SQLiteDefinition,
+ SQLiteTable,
+ SQLiteTables,
+ SQLiteType,
+ Table,
+} from "@budibase/types"
import { cloneDeep } from "lodash"
import tablesSdk from "../"
-import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
+import {
+ CONSTANT_INTERNAL_ROW_COLS,
+ generateJunctionTableID,
+} from "../../../../db/utils"
const BASIC_SQLITE_DOC: SQLiteDefinition = {
_id: SQLITE_DESIGN_DOC_ID,
@@ -36,9 +47,38 @@ const FieldTypeMap: Record<FieldType, SQLiteType> = {
[FieldType.BB_REFERENCE]: SQLiteType.TEXT,
}
-function mapTable(table: Table): { [key: string]: SQLiteType } {
+function buildRelationshipDefinitions(
+ table: Table,
+ relationshipColumn: RelationshipFieldMetadata
+): {
+ tableId: string
+ definition: SQLiteTable
+} {
+ const tableId = table._id!,
+ relatedTableId = relationshipColumn.tableId
+ return {
+ tableId: generateJunctionTableID(tableId, relatedTableId),
+ definition: {
+ doc1: SQLiteType.BLOB,
+ doc2: SQLiteType.BLOB,
+ tableId: SQLiteType.TEXT,
+ },
+ }
+}
+
+// this can generate relationship tables as part of the mapping
+function mapTable(table: Table): SQLiteTables {
+ const tables: SQLiteTables = {}
const fields: Record<string, SQLiteType> = {}
for (let [key, column] of Object.entries(table.schema)) {
+ // relationships should be handled differently
+ if (column.type === FieldType.LINK) {
+ const { tableId, definition } = buildRelationshipDefinitions(
+ table,
+ column
+ )
+ tables[tableId] = { fields: definition }
+ }
if (!FieldTypeMap[column.type]) {
throw new Error(`Unable to map type "${column.type}" to SQLite type`)
}
@@ -49,10 +89,12 @@ function mapTable(table: Table): { [key: string]: SQLiteType } {
CONSTANT_INTERNAL_ROW_COLS.forEach(col => {
constantMap[col] = SQLiteType.TEXT
})
- return {
+ const thisTable: SQLiteTable = {
...constantMap,
...fields,
}
+ tables[table._id!] = { fields: thisTable }
+ return tables
}
// nothing exists, need to iterate though existing tables
@@ -60,8 +102,9 @@ async function buildBaseDefinition(): Promise<SQLiteDefinition> {
const tables = await tablesSdk.getAllInternalTables()
const definition = cloneDeep(BASIC_SQLITE_DOC)
for (let table of tables) {
- definition.sql.tables[table._id!] = {
- fields: mapTable(table),
+ definition.sql.tables = {
+ ...definition.sql.tables,
+ ...mapTable(table),
}
}
return definition
@@ -75,8 +118,9 @@ export async function addTableToSqlite(table: Table) {
} catch (err) {
definition = await buildBaseDefinition()
}
- definition.sql.tables[table._id!] = {
- fields: mapTable(table),
+ definition.sql.tables = {
+ ...definition.sql.tables,
+ ...mapTable(table),
}
await db.put(definition)
}
diff --git a/packages/types/src/documents/app/sqlite.ts b/packages/types/src/documents/app/sqlite.ts
index 76c47bbd74..e23a68b336 100644
--- a/packages/types/src/documents/app/sqlite.ts
+++ b/packages/types/src/documents/app/sqlite.ts
@@ -6,17 +6,23 @@ export enum SQLiteType {
NUMERIC = "NUMERIC",
}
+export type SQLiteTable = Record<
+ string,
+ SQLiteType | { field: string; type: SQLiteType }
+>
+
+export type SQLiteTables = Record<
+ string,
+ {
+ fields: SQLiteTable
+ }
+>
+
export interface SQLiteDefinition {
_id: string
language: string
sql: {
- tables: {
- [tableName: string]: {
- fields: {
- [key: string]: SQLiteType | { field: string; type: SQLiteType }
- }
- }
- }
+ tables: SQLiteTables
options: {
table_name: string
}
From d6b252013b80d22bb87677a8494c55663c799de6 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 11 Apr 2024 18:25:18 +0100
Subject: [PATCH 02/11] Quick fix to link document structure in sqlite.
---
packages/server/src/sdk/app/tables/internal/sqs.ts | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index 5dd16f516c..99240c28d4 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -59,8 +59,12 @@ function buildRelationshipDefinitions(
return {
tableId: generateJunctionTableID(tableId, relatedTableId),
definition: {
- doc1: SQLiteType.BLOB,
- doc2: SQLiteType.BLOB,
+ ["doc1.rowId"]: SQLiteType.TEXT,
+ ["doc1.tableId"]: SQLiteType.TEXT,
+ ["doc1.fieldName"]: SQLiteType.TEXT,
+ ["doc2.rowId"]: SQLiteType.TEXT,
+ ["doc2.tableId"]: SQLiteType.TEXT,
+ ["doc2.fieldName"]: SQLiteType.TEXT,
tableId: SQLiteType.TEXT,
},
}
From ebb79c16fe7d626d9d201fe5d1090da6e4ab078f Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 12 Apr 2024 16:15:36 +0100
Subject: [PATCH 03/11] Aliasing support for SQS.
---
.../api/controllers/row/ExternalRequest.ts | 3 +-
packages/server/src/db/utils.ts | 1 +
.../src/integrations/tests/sqlAlias.spec.ts | 4 +-
packages/server/src/sdk/app/rows/index.ts | 2 +
.../server/src/sdk/app/rows/search/sqs.ts | 99 +++++++++++--------
.../row/alias.ts => sdk/app/rows/sqlAlias.ts} | 28 ++++--
6 files changed, 86 insertions(+), 51 deletions(-)
rename packages/server/src/{api/controllers/row/alias.ts => sdk/app/rows/sqlAlias.ts} (87%)
diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 7fc0333de1..4adbb72c7a 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -36,7 +36,6 @@ import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
import { processObjectSync } from "@budibase/string-templates"
import { cloneDeep } from "lodash/fp"
import { db as dbCore } from "@budibase/backend-core"
-import AliasTables from "./alias"
import sdk from "../../../sdk"
import env from "../../../environment"
@@ -618,7 +617,7 @@ export class ExternalRequest<T extends Operation> {
if (env.SQL_ALIASING_DISABLE) {
response = await getDatasourceAndQuery(json)
} else {
- const aliasing = new AliasTables(Object.keys(this.tables))
+ const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables))
response = await aliasing.queryWithAliasing(json)
}
diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts
index b1c02b1764..ce8d0accbb 100644
--- a/packages/server/src/db/utils.ts
+++ b/packages/server/src/db/utils.ts
@@ -40,6 +40,7 @@ export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.Inte
export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}`
export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}`
export const AUTOMATION_LOG_PREFIX = `${DocumentType.AUTOMATION_LOG}${SEPARATOR}`
+export const SQS_DATASOURCE_INTERNAL = "internal"
export const ViewName = dbCore.ViewName
export const InternalTables = dbCore.InternalTable
export const UNICODE_MAX = dbCore.UNICODE_MAX
diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index bfca24ff7d..58c3a05245 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -8,8 +8,10 @@ import {
import { join } from "path"
import Sql from "../base/sql"
import { SqlClient } from "../utils"
-import AliasTables from "../../api/controllers/row/alias"
import { generator } from "@budibase/backend-core/tests"
+import sdk from "../../sdk"
+
+const AliasTables = sdk.rows.AliasTables
function multiline(sql: string) {
return sql.replace(/\n/g, "").replace(/ +/g, " ")
diff --git a/packages/server/src/sdk/app/rows/index.ts b/packages/server/src/sdk/app/rows/index.ts
index ea501e93d9..c117941419 100644
--- a/packages/server/src/sdk/app/rows/index.ts
+++ b/packages/server/src/sdk/app/rows/index.ts
@@ -3,6 +3,7 @@ import * as rows from "./rows"
import * as search from "./search"
import * as utils from "./utils"
import * as external from "./external"
+import AliasTables from "./sqlAlias"
export default {
...attachments,
@@ -10,4 +11,5 @@ export default {
...search,
utils,
external,
+ AliasTables,
}
diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts
index 5b0b6e3bc7..20edb988d3 100644
--- a/packages/server/src/sdk/app/rows/search/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/sqs.ts
@@ -20,7 +20,12 @@ import {
} from "../../../../api/controllers/row/utils"
import sdk from "../../../index"
import { context } from "@budibase/backend-core"
-import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
+import {
+ CONSTANT_INTERNAL_ROW_COLS,
+ SQS_DATASOURCE_INTERNAL,
+} from "../../../../db/utils"
+import AliasTables from "../sqlAlias"
+import { outputProcessing } from "../../../../utilities/rowProcessor"
function buildInternalFieldList(
table: Table,
@@ -31,19 +36,19 @@ function buildInternalFieldList(
fieldList = fieldList.concat(
CONSTANT_INTERNAL_ROW_COLS.map(col => `${table._id}.${col}`)
)
- if (opts.relationships) {
- for (let col of Object.values(table.schema)) {
- if (col.type === FieldType.LINK) {
- const linkCol = col as RelationshipFieldMetadata
- const relatedTable = tables.find(
- table => table._id === linkCol.tableId
- )!
- fieldList = fieldList.concat(
- buildInternalFieldList(relatedTable, tables, { relationships: false })
- )
- } else {
- fieldList.push(`${table._id}.${col.name}`)
- }
+ for (let col of Object.values(table.schema)) {
+ const isRelationship = col.type === FieldType.LINK
+ if (!opts.relationships && isRelationship) {
+ continue
+ }
+ if (isRelationship) {
+ const linkCol = col as RelationshipFieldMetadata
+ const relatedTable = tables.find(table => table._id === linkCol.tableId)!
+ fieldList = fieldList.concat(
+ buildInternalFieldList(relatedTable, tables, { relationships: false })
+ )
+ } else {
+ fieldList.push(`${table._id}.${col.name}`)
}
}
return fieldList
@@ -94,14 +99,14 @@ function buildTableMap(tables: Table[]) {
}
export async function search(
- options: RowSearchParams
+ options: RowSearchParams,
+ table: Table
): Promise<SearchResponse<Row>> {
- const { tableId, paginate, query, ...params } = options
+ const { paginate, query, ...params } = options
const builder = new SqlQueryBuilder(SqlClient.SQL_LITE)
const allTables = await sdk.tables.getAllInternalTables()
const allTablesMap = buildTableMap(allTables)
- const table = allTables.find(table => table._id === tableId)
if (!table) {
throw new Error("Unable to find table")
}
@@ -111,7 +116,7 @@ export async function search(
const request: QueryJson = {
endpoint: {
// not important, we query ourselves
- datasourceId: "internal",
+ datasourceId: SQS_DATASOURCE_INTERNAL,
entityId: table._id!,
operation: Operation.READ,
},
@@ -154,34 +159,44 @@ export async function search(
}
}
try {
- const query = builder._query(request, {
- disableReturning: true,
+ const alias = new AliasTables(allTables.map(table => table.name))
+ const rows = await alias.queryWithAliasing(request, async json => {
+ const query = builder._query(json, {
+ disableReturning: true,
+ })
+
+ if (Array.isArray(query)) {
+ throw new Error("SQS cannot currently handle multiple queries")
+ }
+
+ let sql = query.sql,
+ bindings = query.bindings
+
+ // quick hack for docIds
+ sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
+ sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
+
+ const db = context.getAppDB()
+ return await db.sql<Row>(sql, bindings)
})
- if (Array.isArray(query)) {
- throw new Error("SQS cannot currently handle multiple queries")
- }
-
- let sql = query.sql,
- bindings = query.bindings
-
- // quick hack for docIds
- sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
- sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
-
- const db = context.getAppDB()
- const rows = await db.sql<Row>(sql, bindings)
+ // process from the format of tableId.column to expected format
+ const processed = await sqlOutputProcessing(
+ rows,
+ table!,
+ allTablesMap,
+ relationships,
+ {
+ sqs: true,
+ }
+ )
return {
- rows: await sqlOutputProcessing(
- rows,
- table!,
- allTablesMap,
- relationships,
- {
- sqs: true,
- }
- ),
+ // final row processing for response
+ rows: await outputProcessing<Row[]>(table, processed, {
+ preserveLinks: true,
+ squash: true,
+ }),
}
} catch (err: any) {
const msg = typeof err === "string" ? err : err.message
diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts
similarity index 87%
rename from packages/server/src/api/controllers/row/alias.ts
rename to packages/server/src/sdk/app/rows/sqlAlias.ts
index 0ec9d1a09c..0fc338ecbe 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/sdk/app/rows/sqlAlias.ts
@@ -6,11 +6,12 @@ import {
Row,
SearchFilters,
} from "@budibase/types"
-import { getSQLClient } from "../../../sdk/app/rows/utils"
+import { getSQLClient } from "./utils"
import { cloneDeep } from "lodash"
-import sdk from "../../../sdk"
+import datasources from "../datasources"
import { makeExternalQuery } from "../../../integrations/base/query"
import { SqlClient } from "../../../integrations/utils"
+import { SQS_DATASOURCE_INTERNAL } from "../../../db/utils"
const WRITE_OPERATIONS: Operation[] = [
Operation.CREATE,
@@ -156,12 +157,19 @@ export default class AliasTables {
}
async queryWithAliasing(
- json: QueryJson
+ json: QueryJson,
+ queryFn?: (json: QueryJson) => Promise<DatasourcePlusQueryResponse>
): Promise<DatasourcePlusQueryResponse> {
const datasourceId = json.endpoint.datasourceId
- const datasource = await sdk.datasources.get(datasourceId)
+ const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL
+ let aliasingEnabled: boolean, datasource: Datasource | undefined
+ if (isSqs) {
+ aliasingEnabled = true
+ } else {
+ datasource = await datasources.get(datasourceId)
+ aliasingEnabled = this.isAliasingEnabled(json, datasource)
+ }
- const aliasingEnabled = this.isAliasingEnabled(json, datasource)
if (aliasingEnabled) {
json = cloneDeep(json)
// run through the query json to update anywhere a table may be used
@@ -207,7 +215,15 @@ export default class AliasTables {
}
json.tableAliases = invertedTableAliases
}
- const response = await makeExternalQuery(datasource, json)
+
+ let response: DatasourcePlusQueryResponse
+ if (datasource && !isSqs) {
+ response = await makeExternalQuery(datasource, json)
+ } else if (queryFn) {
+ response = await queryFn(json)
+ } else {
+ throw new Error("No supplied method to perform aliased query")
+ }
if (Array.isArray(response) && aliasingEnabled) {
return this.reverse(response)
} else {
From c40e9656345f5e95ead009976ab00755973b4845 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 12 Apr 2024 16:16:31 +0100
Subject: [PATCH 04/11] Getting relationships working properly as well as
renaming internal -> sqs in function opts.
---
packages/server/src/api/controllers/row/utils/basic.ts | 9 +++++----
.../server/src/api/controllers/row/utils/sqlUtils.ts | 4 ++--
packages/server/src/api/controllers/row/utils/utils.ts | 6 ++++--
3 files changed, 11 insertions(+), 8 deletions(-)
diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts
index 1fc84de9c7..6255e13c1c 100644
--- a/packages/server/src/api/controllers/row/utils/basic.ts
+++ b/packages/server/src/api/controllers/row/utils/basic.ts
@@ -62,12 +62,12 @@ export function basicProcessing({
row,
table,
isLinked,
- internal,
+ sqs,
}: {
row: Row
table: Table
isLinked: boolean
- internal?: boolean
+ sqs?: boolean
}): Row {
const thisRow: Row = {}
// filter the row down to what is actually the row (not joined)
@@ -84,12 +84,13 @@ export function basicProcessing({
thisRow[fieldName] = value
}
}
- if (!internal) {
+ if (!sqs) {
thisRow._id = generateIdForRow(row, table, isLinked)
thisRow.tableId = table._id
thisRow._rev = "rev"
} else {
- for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) {
+ const columns = Object.keys(table.schema)
+ for (let internalColumn of [...CONSTANT_INTERNAL_ROW_COLS, ...columns]) {
thisRow[internalColumn] = extractFieldValue({
row,
tableName: table._id!,
diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index 6f9837e0ab..372b8394ff 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -51,11 +51,11 @@ export async function updateRelationshipColumns(
continue
}
- let linked = await basicProcessing({
+ let linked = basicProcessing({
row,
table: linkedTable,
isLinked: true,
- internal: opts?.sqs,
+ sqs: opts?.sqs,
})
if (!linked._id) {
continue
diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts
index f387a468cf..bf9ede6fe3 100644
--- a/packages/server/src/api/controllers/row/utils/utils.ts
+++ b/packages/server/src/api/controllers/row/utils/utils.ts
@@ -132,6 +132,7 @@ export async function sqlOutputProcessing(
let rowId = row._id
if (opts?.sqs) {
rowId = getInternalRowId(row, table)
+ row._id = rowId
} else if (!rowId) {
rowId = generateIdForRow(row, table)
row._id = rowId
@@ -153,7 +154,7 @@ export async function sqlOutputProcessing(
row,
table,
isLinked: false,
- internal: opts?.sqs,
+ sqs: opts?.sqs,
}),
table
)
@@ -167,7 +168,8 @@ export async function sqlOutputProcessing(
tables,
row,
finalRows,
- relationships
+ relationships,
+ opts
)
}
From bfb7750213400e833bfb4ffeae0be9462b66e0bc Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 12 Apr 2024 16:17:06 +0100
Subject: [PATCH 05/11] Getting search input mapping up a level in the search
SDK - avoids having to call it for every search type.
---
packages/server/src/sdk/app/rows/search.ts | 11 +++++++---
.../src/sdk/app/rows/search/external.ts | 13 +++++++-----
.../src/sdk/app/rows/search/internal.ts | 20 ++++++++++---------
.../app/rows/search/tests/external.spec.ts | 6 +++---
.../app/rows/search/tests/internal.spec.ts | 4 ++--
5 files changed, 32 insertions(+), 22 deletions(-)
diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index f681bfeb90..5d8f7ef80b 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -13,6 +13,8 @@ import * as sqs from "./search/sqs"
import env from "../../../environment"
import { ExportRowsParams, ExportRowsResult } from "./search/types"
import { dataFilters } from "@budibase/shared-core"
+import sdk from "../../index"
+import { searchInputMapping } from "./search/utils"
export { isValidFilter } from "../../../integrations/utils"
@@ -72,12 +74,15 @@ export async function search(
}
}
+ const table = await sdk.tables.getTable(options.tableId)
+ options = searchInputMapping(table, options)
+
if (isExternalTable) {
- return external.search(options)
+ return external.search(options, table)
} else if (env.SQS_SEARCH_ENABLE) {
- return sqs.search(options)
+ return sqs.search(options, table)
} else {
- return internal.search(options)
+ return internal.search(options, table)
}
}
diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts
index e0a3bad94e..077f971903 100644
--- a/packages/server/src/sdk/app/rows/search/external.ts
+++ b/packages/server/src/sdk/app/rows/search/external.ts
@@ -8,6 +8,7 @@ import {
SearchFilters,
RowSearchParams,
SearchResponse,
+ Table,
} from "@budibase/types"
import * as exporters from "../../../../api/controllers/view/exporters"
import { handleRequest } from "../../../../api/controllers/row/external"
@@ -18,13 +19,13 @@ import {
import { utils } from "@budibase/shared-core"
import { ExportRowsParams, ExportRowsResult } from "./types"
import { HTTPError, db } from "@budibase/backend-core"
-import { searchInputMapping } from "./utils"
import pick from "lodash/pick"
import { outputProcessing } from "../../../../utilities/rowProcessor"
import sdk from "../../../"
export async function search(
- options: RowSearchParams
+ options: RowSearchParams,
+ table: Table
): Promise<SearchResponse<Row>> {
const { tableId } = options
const { paginate, query, ...params } = options
@@ -68,8 +69,6 @@ export async function search(
}
try {
- const table = await sdk.tables.getTable(tableId)
- options = searchInputMapping(table, options)
let rows = await handleRequest(Operation.READ, tableId, {
filters: query,
sort,
@@ -150,11 +149,15 @@ export async function exportRows(
}
const datasource = await sdk.datasources.get(datasourceId!)
+ const table = await sdk.tables.getTable(tableId)
if (!datasource || !datasource.entities) {
throw new HTTPError("Datasource has not been configured for plus API.", 400)
}
- let result = await search({ tableId, query: requestQuery, sort, sortOrder })
+ let result = await search(
+ { tableId, query: requestQuery, sort, sortOrder },
+ table
+ )
let rows: Row[] = []
let headers
diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts
index 610807a10e..ffd13ed731 100644
--- a/packages/server/src/sdk/app/rows/search/internal.ts
+++ b/packages/server/src/sdk/app/rows/search/internal.ts
@@ -33,7 +33,8 @@ import pick from "lodash/pick"
import { breakRowIdField } from "../../../../integrations/utils"
export async function search(
- options: RowSearchParams
+ options: RowSearchParams,
+ table: Table
): Promise<SearchResponse<Row>> {
const { tableId } = options
@@ -51,8 +52,6 @@ export async function search(
query: {},
}
- let table = await sdk.tables.getTable(tableId)
- options = searchInputMapping(table, options)
if (params.sort && !params.sortType) {
const schema = table.schema
const sortField = schema[params.sort]
@@ -122,12 +121,15 @@ export async function exportRows(
result = await outputProcessing<Row[]>(table, response)
} else if (query) {
- let searchResponse = await search({
- tableId,
- query,
- sort,
- sortOrder,
- })
+ let searchResponse = await search(
+ {
+ tableId,
+ query,
+ sort,
+ sortOrder,
+ },
+ table
+ )
result = searchResponse.rows
}
diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts
index f2bdec4692..53bc049a9b 100644
--- a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts
+++ b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts
@@ -112,7 +112,7 @@ describe("external search", () => {
tableId,
query: {},
}
- const result = await search(searchParams)
+ const result = await search(searchParams, config.table!)
expect(result.rows).toHaveLength(10)
expect(result.rows).toEqual(
@@ -130,7 +130,7 @@ describe("external search", () => {
query: {},
fields: ["name", "age"],
}
- const result = await search(searchParams)
+ const result = await search(searchParams, config.table!)
expect(result.rows).toHaveLength(10)
expect(result.rows).toEqual(
@@ -157,7 +157,7 @@ describe("external search", () => {
},
},
}
- const result = await search(searchParams)
+ const result = await search(searchParams, config.table!)
expect(result.rows).toHaveLength(3)
expect(result.rows.map(row => row.id)).toEqual([1, 4, 8])
diff --git a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts
index 5be0f4a258..1c5f396737 100644
--- a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts
+++ b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts
@@ -81,7 +81,7 @@ describe("internal", () => {
tableId,
query: {},
}
- const result = await search(searchParams)
+ const result = await search(searchParams, config.table!)
expect(result.rows).toHaveLength(10)
expect(result.rows).toEqual(
@@ -99,7 +99,7 @@ describe("internal", () => {
query: {},
fields: ["name", "age"],
}
- const result = await search(searchParams)
+ const result = await search(searchParams, config.table!)
expect(result.rows).toHaveLength(10)
expect(result.rows).toEqual(
From 7d7de33cabbcfcefb7ea668a92813b96c1e28b3b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 12 Apr 2024 16:29:48 +0100
Subject: [PATCH 06/11] Removing CouchDB SQS image for now.
---
hosting/docker-compose.dev.yaml | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml
index 77f6bd053b..9dba5d427c 100644
--- a/hosting/docker-compose.dev.yaml
+++ b/hosting/docker-compose.dev.yaml
@@ -42,13 +42,12 @@ services:
couchdb-service:
container_name: budi-couchdb3-dev
restart: on-failure
- image: budibase/couchdb:v3.2.1-sqs
+ image: budibase/couchdb
environment:
- COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
- COUCHDB_USER=${COUCH_DB_USER}
ports:
- "${COUCH_DB_PORT}:5984"
- - "${COUCH_DB_SQS_PORT}:4984"
volumes:
- couchdb_data:/data
From aeda5931c07c84ed826e5e37bd54b3382d955653 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 12 Apr 2024 16:34:33 +0100
Subject: [PATCH 07/11] Fixing lint.
---
packages/server/src/sdk/app/rows/search/internal.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts
index ffd13ed731..906ca016d1 100644
--- a/packages/server/src/sdk/app/rows/search/internal.ts
+++ b/packages/server/src/sdk/app/rows/search/internal.ts
@@ -1,6 +1,6 @@
import { context, db, HTTPError } from "@budibase/backend-core"
import env from "../../../../environment"
-import { fullSearch, paginatedSearch, searchInputMapping } from "./utils"
+import { fullSearch, paginatedSearch } from "./utils"
import { getRowParams, InternalTables } from "../../../../db/utils"
import {
Database,
From 68c5e657ddd0fb5b45948e418531d72cd16b178e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 15 Apr 2024 13:46:31 +0100
Subject: [PATCH 08/11] Updating @types/archiver to be more specific.
---
packages/server/package.json | 2 +-
yarn.lock | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/packages/server/package.json b/packages/server/package.json
index ad03033e67..76402785d7 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -125,7 +125,7 @@
"@babel/preset-env": "7.16.11",
"@swc/core": "1.3.71",
"@swc/jest": "0.2.27",
- "@types/archiver": "^6.0.2",
+ "@types/archiver": "6.0.2",
"@types/global-agent": "2.1.1",
"@types/google-spreadsheet": "3.1.5",
"@types/jest": "29.5.5",
diff --git a/yarn.lock b/yarn.lock
index a36b54d3be..ce39c89075 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -5174,7 +5174,7 @@
dependencies:
"@types/node" "*"
-"@types/archiver@^6.0.2":
+"@types/archiver@6.0.2":
version "6.0.2"
resolved "https://registry.yarnpkg.com/@types/archiver/-/archiver-6.0.2.tgz#0daf8c83359cbde69de1e4b33dcade6a48a929e2"
integrity sha512-KmROQqbQzKGuaAbmK+ZcytkJ51+YqDa7NmbXjmtC5YBLSyQYo21YaUnQ3HbaPFKL1ooo6RQ6OPYPIDyxfpDDXw==
From b13b7df678768a460c6c4b4dc464bbb4a25dcda1 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 15 Apr 2024 18:23:39 +0100
Subject: [PATCH 09/11] Correctly handling aliasing for sorting/json field
types with SQS.
---
packages/server/src/integrations/base/sql.ts | 31 ++++++++++++++++----
1 file changed, 25 insertions(+), 6 deletions(-)
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 259abec106..b622ca39be 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -22,6 +22,8 @@ import {
SortDirection,
SqlQueryBinding,
Table,
+ TableSourceType,
+ INTERNAL_TABLE_SOURCE_ID,
} from "@budibase/types"
import environment from "../../environment"
@@ -135,6 +137,21 @@ function generateSelectStatement(
})
}
+function getTableName(table?: Table): string {
+ if (!table) {
+ throw new Error("No table name available.")
+ }
+ // SQS uses the table ID rather than the table name
+ if (
+ table.sourceType === TableSourceType.INTERNAL ||
+ table.sourceId === INTERNAL_TABLE_SOURCE_ID
+ ) {
+ return table._id!
+ } else {
+ return table.name
+ }
+}
+
class InternalBuilder {
private readonly client: string
@@ -149,7 +166,7 @@ class InternalBuilder {
tableName: string,
opts: { aliases?: Record<string, string>; relationship?: boolean }
): Knex.QueryBuilder {
- function getTableName(name: string) {
+ function getTableAlias(name: string) {
const alias = opts.aliases?.[name]
return alias || name
}
@@ -161,11 +178,11 @@ class InternalBuilder {
const updatedKey = dbCore.removeKeyNumbering(key)
const isRelationshipField = updatedKey.includes(".")
if (!opts.relationship && !isRelationshipField) {
- fn(`${getTableName(tableName)}.${updatedKey}`, value)
+ fn(`${getTableAlias(tableName)}.${updatedKey}`, value)
}
if (opts.relationship && isRelationshipField) {
const [filterTableName, property] = updatedKey.split(".")
- fn(`${getTableName(filterTableName)}.${property}`, value)
+ fn(`${getTableAlias(filterTableName)}.${property}`, value)
}
}
}
@@ -346,9 +363,10 @@ class InternalBuilder {
addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder {
let { sort, paginate } = json
const table = json.meta?.table
+ const tableName = getTableName(table)
const aliases = json.tableAliases
const aliased =
- table?.name && aliases?.[table.name] ? aliases[table.name] : table?.name
+ table?.name && aliases?.[tableName] ? aliases[tableName] : table?.name
if (sort && Object.keys(sort || {}).length > 0) {
for (let [key, value] of Object.entries(sort)) {
const direction =
@@ -729,12 +747,13 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
results: Record<string, any>[],
aliases?: Record<string, string>
): Record<string, any>[] {
+ const tableName = getTableName(table)
for (const [name, field] of Object.entries(table.schema)) {
if (!this._isJsonColumn(field)) {
continue
}
- const tableName = aliases?.[table.name] || table.name
- const fullName = `${tableName}.${name}`
+ const aliasedTableName = aliases?.[tableName] || tableName
+ const fullName = `${aliasedTableName}.${name}`
for (let row of results) {
if (typeof row[fullName] === "string") {
row[fullName] = JSON.parse(row[fullName])
From 69ae603fa4d3aae6e61427149057edbbd885e64a Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 15 Apr 2024 18:24:11 +0100
Subject: [PATCH 10/11] Updating errors from sql table actions and making sure
SQS tables cannot be reach the sql table actions/will error if they do.
---
.../server/src/integrations/base/sqlTable.ts | 23 +++++++++++--------
1 file changed, 14 insertions(+), 9 deletions(-)
diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts
index 4ff336421f..3c55d75b8b 100644
--- a/packages/server/src/integrations/base/sqlTable.ts
+++ b/packages/server/src/integrations/base/sqlTable.ts
@@ -1,19 +1,20 @@
import { Knex, knex } from "knex"
import {
- RelationshipType,
FieldSubtype,
+ FieldType,
NumberFieldMetadata,
Operation,
QueryJson,
+ RelationshipType,
RenameColumn,
- Table,
- FieldType,
SqlQuery,
+ Table,
+ TableSourceType,
} from "@budibase/types"
import { breakExternalTableId, getNativeSql, SqlClient } from "../utils"
+import { utils } from "@budibase/shared-core"
import SchemaBuilder = Knex.SchemaBuilder
import CreateTableBuilder = Knex.CreateTableBuilder
-import { utils } from "@budibase/shared-core"
function isIgnoredType(type: FieldType) {
const ignored = [FieldType.LINK, FieldType.FORMULA]
@@ -105,13 +106,13 @@ function generateSchema(
column.relationshipType !== RelationshipType.MANY_TO_MANY
) {
if (!column.foreignKey || !column.tableId) {
- throw "Invalid relationship schema"
+ throw new Error("Invalid relationship schema")
}
const { tableName } = breakExternalTableId(column.tableId)
// @ts-ignore
const relatedTable = tables[tableName]
if (!relatedTable) {
- throw "Referenced table doesn't exist"
+ throw new Error("Referenced table doesn't exist")
}
const relatedPrimary = relatedTable.primary[0]
const externalType = relatedTable.schema[relatedPrimary].externalType
@@ -209,15 +210,19 @@ class SqlTableQueryBuilder {
let query: Knex.SchemaBuilder
if (!json.table || !json.meta || !json.meta.tables) {
- throw "Cannot execute without table being specified"
+ throw new Error("Cannot execute without table being specified")
}
+ if (json.table.sourceType === TableSourceType.INTERNAL) {
+ throw new Error("Cannot perform table actions for SQS.")
+ }
+
switch (this._operation(json)) {
case Operation.CREATE_TABLE:
query = buildCreateTable(client, json.table, json.meta.tables)
break
case Operation.UPDATE_TABLE:
if (!json.meta || !json.meta.table) {
- throw "Must specify old table for update"
+ throw new Error("Must specify old table for update")
}
// renameColumn does not work for MySQL, so return a raw query
if (this.sqlClient === SqlClient.MY_SQL && json.meta.renamed) {
@@ -264,7 +269,7 @@ class SqlTableQueryBuilder {
query = buildDeleteTable(client, json.table)
break
default:
- throw "Table operation is of unknown type"
+ throw new Error("Table operation is of unknown type")
}
return getNativeSql(query)
}
From e2ca21053e52c88994e2ec75845548673e0707e1 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 16 Apr 2024 11:38:00 +0100
Subject: [PATCH 11/11] Fixing build issue caught in CI.
---
packages/server/src/integrations/base/sql.ts | 17 +++++++----------
1 file changed, 7 insertions(+), 10 deletions(-)
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index b622ca39be..59684422e7 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -137,18 +137,15 @@ function generateSelectStatement(
})
}
-function getTableName(table?: Table): string {
- if (!table) {
- throw new Error("No table name available.")
- }
+function getTableName(table?: Table): string | undefined {
// SQS uses the table ID rather than the table name
if (
- table.sourceType === TableSourceType.INTERNAL ||
- table.sourceId === INTERNAL_TABLE_SOURCE_ID
+ table?.sourceType === TableSourceType.INTERNAL ||
+ table?.sourceId === INTERNAL_TABLE_SOURCE_ID
) {
- return table._id!
+ return table?._id
} else {
- return table.name
+ return table?.name
}
}
@@ -366,7 +363,7 @@ class InternalBuilder {
const tableName = getTableName(table)
const aliases = json.tableAliases
const aliased =
- table?.name && aliases?.[tableName] ? aliases[tableName] : table?.name
+ tableName && aliases?.[tableName] ? aliases[tableName] : table?.name
if (sort && Object.keys(sort || {}).length > 0) {
for (let [key, value] of Object.entries(sort)) {
const direction =
@@ -752,7 +749,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
if (!this._isJsonColumn(field)) {
continue
}
- const aliasedTableName = aliases?.[tableName] || tableName
+ const aliasedTableName = (tableName && aliases?.[tableName]) || tableName
const fullName = `${aliasedTableName}.${name}`
for (let row of results) {
if (typeof row[fullName] === "string") {