From adef1ed5ce1359e78b6c0c79575e1c105e5bba56 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 6 Jul 2023 21:49:25 +0100 Subject: [PATCH] Initial attempt to get SQS up and running within BB. --- hosting/couchdb/.gitignore | 1 + hosting/couchdb/Dockerfile | 6 +- hosting/couchdb/runner.sh | 3 +- hosting/docker-compose.dev.yaml | 4 +- packages/backend-core/src/constants/db.ts | 1 + .../backend-core/src/db/couch/DatabaseImpl.ts | 16 + .../backend-core/src/db/couch/connections.ts | 1 + packages/backend-core/src/db/couch/utils.ts | 9 +- packages/backend-core/src/environment.ts | 1 + packages/builder/src/helpers/searchFields.js | 8 +- .../api/controllers/row/ExternalRequest.ts | 344 +----------------- .../src/api/controllers/row/internal.ts | 84 ++--- .../src/api/controllers/row/internalSql.ts | 158 ++++++++ .../src/api/controllers/row/utils/basic.ts | 107 ++++++ .../src/api/controllers/row/utils/index.ts | 3 + .../src/api/controllers/row/utils/sqlUtils.ts | 224 ++++++++++++ .../api/controllers/row/{ => utils}/utils.ts | 91 ++++- .../src/api/controllers/table/sqlite.ts | 75 ++++ .../server/src/api/controllers/table/utils.ts | 2 + .../server/src/db/linkedRows/LinkDocument.ts | 11 +- packages/server/src/db/utils.ts | 15 + packages/server/src/definitions/datasource.ts | 1 + packages/server/src/integrations/base/sql.ts | 8 +- packages/server/src/integrations/utils.ts | 1 + packages/types/src/documents/app/index.ts | 1 + packages/types/src/documents/app/links.ts | 1 + packages/types/src/documents/app/sqlite.ts | 24 ++ packages/types/src/documents/app/table.ts | 1 + packages/types/src/sdk/db.ts | 1 + 29 files changed, 815 insertions(+), 387 deletions(-) create mode 100644 hosting/couchdb/.gitignore create mode 100644 packages/server/src/api/controllers/row/internalSql.ts create mode 100644 packages/server/src/api/controllers/row/utils/basic.ts create mode 100644 packages/server/src/api/controllers/row/utils/index.ts create mode 100644 packages/server/src/api/controllers/row/utils/sqlUtils.ts rename packages/server/src/api/controllers/row/{ => utils}/utils.ts (69%) create mode 100644 packages/server/src/api/controllers/table/sqlite.ts create mode 100644 packages/types/src/documents/app/sqlite.ts diff --git a/hosting/couchdb/.gitignore b/hosting/couchdb/.gitignore new file mode 100644 index 0000000000..532c6c608d --- /dev/null +++ b/hosting/couchdb/.gitignore @@ -0,0 +1 @@ +sqlite diff --git a/hosting/couchdb/Dockerfile b/hosting/couchdb/Dockerfile index 70b4413859..632f326c9b 100644 --- a/hosting/couchdb/Dockerfile +++ b/hosting/couchdb/Dockerfile @@ -3,6 +3,7 @@ FROM couchdb:3.2.1 ENV COUCHDB_USER admin ENV COUCHDB_PASSWORD admin EXPOSE 5984 +EXPOSE 4984 RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \ wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add - && \ @@ -28,8 +29,11 @@ ADD clouseau/log4j.properties clouseau/clouseau.ini ./ WORKDIR /opt/couchdb ADD couch/vm.args couch/local.ini ./etc/ +WORKDIR /opt/sqs +ADD sqlite/sqs sqlite/better_sqlite3.node ./ + WORKDIR / ADD build-target-paths.sh . ADD runner.sh ./bbcouch-runner.sh -RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau ./build-target-paths.sh +RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau ./build-target-paths.sh /opt/sqs/sqs CMD ["./bbcouch-runner.sh"] diff --git a/hosting/couchdb/runner.sh b/hosting/couchdb/runner.sh index 4102d2a751..7b04a9c14d 100644 --- a/hosting/couchdb/runner.sh +++ b/hosting/couchdb/runner.sh @@ -8,7 +8,8 @@ chown -R couchdb:couchdb ${DATA_DIR}/couch /build-target-paths.sh /opt/clouseau/bin/clouseau > /dev/stdout 2>&1 & /docker-entrypoint.sh /opt/couchdb/bin/couchdb & +/opt/sqs/sqs --server "http://localhost:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 & sleep 10 curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_users curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_replicator -sleep infinity \ No newline at end of file +sleep infinity diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index 394f5ac256..cf138ff981 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -40,15 +40,15 @@ services: - PROXY_ADDRESS=host.docker.internal couchdb-service: - # platform: linux/amd64 container_name: budi-couchdb3-dev restart: on-failure - image: budibase/couchdb + image: couch-sqs environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} ports: - "${COUCH_DB_PORT}:5984" + - "4984:4984" volumes: - couchdb_data:/data diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index be49b9f261..716acc877f 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -99,3 +99,4 @@ export const APP_PREFIX = DocumentType.APP + SEPARATOR export const APP_DEV = DocumentType.APP_DEV + SEPARATOR export const APP_DEV_PREFIX = APP_DEV export const BUDIBASE_DATASOURCE_TYPE = "budibase" +export const SQLITE_DESIGN_DOC_ID = "_design/sqlite" diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 29ca4123f5..a1f8696af2 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -16,6 +16,7 @@ import { directCouchUrlCall } from "./utils" import { getPouchDB } from "./pouchDB" import { WriteStream, ReadStream } from "fs" import { newid } from "../../docIds/newid" +import { SQLITE_DESIGN_DOC_ID } from "../../constants" function buildNano(couchInfo: { url: string; cookie: string }) { return Nano({ @@ -180,6 +181,21 @@ export class DatabaseImpl implements Database { return this.updateOutput(() => db.list(params)) } + async sql(sql: string): Promise { + const dbName = this.name + const url = `/${dbName}/${SQLITE_DESIGN_DOC_ID}` + const response = await directCouchUrlCall({ + url: `${this.couchInfo.sqlUrl}/${url}`, + method: "POST", + cookie: this.couchInfo.cookie, + body: sql, + }) + if (response.status > 300) { + throw new Error(await response.text()) + } + return (await response.json()) as T + } + async query( viewName: string, params: DatabaseQueryOpts diff --git a/packages/backend-core/src/db/couch/connections.ts b/packages/backend-core/src/db/couch/connections.ts index 4214c7cdc6..8dbbe34e3a 100644 --- a/packages/backend-core/src/db/couch/connections.ts +++ b/packages/backend-core/src/db/couch/connections.ts @@ -25,6 +25,7 @@ export const getCouchInfo = (connection?: string) => { const authCookie = Buffer.from(`${username}:${password}`).toString("base64") return { url: urlInfo.url!, + sqlUrl: env.COUCH_DB_SQL_URL, auth: { username: username, password: password, diff --git a/packages/backend-core/src/db/couch/utils.ts b/packages/backend-core/src/db/couch/utils.ts index 51b2a38998..005b02a896 100644 --- a/packages/backend-core/src/db/couch/utils.ts +++ b/packages/backend-core/src/db/couch/utils.ts @@ -30,8 +30,13 @@ export async function directCouchUrlCall({ }, } if (body && method !== "GET") { - params.body = JSON.stringify(body) - params.headers["Content-Type"] = "application/json" + if (typeof body === "string") { + params.body = body + params.headers["Content-Type"] = "text/plain" + } else { + params.body = JSON.stringify(body) + params.headers["Content-Type"] = "application/json" + } } return await fetch(checkSlashesInUrl(encodeURI(url)), params) } diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index eab8cd4c45..6e3707718e 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -89,6 +89,7 @@ const environment = { ENCRYPTION_KEY: process.env.ENCRYPTION_KEY, API_ENCRYPTION_KEY: getAPIEncryptionKey(), COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", + COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984", COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, diff --git a/packages/builder/src/helpers/searchFields.js b/packages/builder/src/helpers/searchFields.js index a9c837d570..b6b4048695 100644 --- a/packages/builder/src/helpers/searchFields.js +++ b/packages/builder/src/helpers/searchFields.js @@ -4,11 +4,11 @@ import { get } from "svelte/store" export function getTableFields(linkField) { const table = get(tables).list.find(table => table._id === linkField.tableId) - if (!table || !table.sql) { + if (!table) { return [] } const linkFields = getFields(Object.values(table.schema), { - allowLinks: false, + tableFields: true, }) return linkFields.map(field => ({ ...field, @@ -16,11 +16,11 @@ export function getTableFields(linkField) { })) } -export function getFields(fields, { allowLinks } = { allowLinks: true }) { +export function getFields(fields, { tableFields }) { let filteredFields = fields.filter( field => !BannedSearchTypes.includes(field.type) ) - if (allowLinks) { + if (!tableFields) { const linkFields = fields.filter(field => field.type === "link") for (let linkField of linkFields) { // only allow one depth of SQL relationship filtering diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 0139147e35..66ca2fe878 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -6,7 +6,6 @@ import { IncludeRelationship, Operation, PaginationJson, - RelationshipsJson, RelationshipTypes, Row, SearchFilters, @@ -18,19 +17,21 @@ import { breakExternalTableId, breakRowIdField, convertRowId, - generateRowIdField, - getPrimaryDisplay, isRowId, isSQL, } from "../../../integrations/utils" -import { getDatasourceAndQuery } from "./utils" +import { + getDatasourceAndQuery, + generateIdForRow, + buildExternalRelationships, + buildSqlFieldList, + sqlOutputProcessing, +} from "./utils" import { FieldTypes } from "../../../constants" import { processObjectSync } from "@budibase/string-templates" import { cloneDeep } from "lodash/fp" -import { processDates, processFormulas } from "../../../utilities/rowProcessor" import { db as dbCore } from "@budibase/backend-core" import sdk from "../../../sdk" -import { isEditableColumn } from "../../../sdk/app/tables/validation" export interface ManyRelationship { tableId?: string @@ -146,34 +147,6 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig { return config } -function generateIdForRow( - row: Row | undefined, - table: Table, - isLinked: boolean = false -): string { - const primary = table.primary - if (!row || !primary) { - return "" - } - // build id array - let idParts = [] - for (let field of primary) { - let fieldValue = extractFieldValue({ - row, - tableName: table.name, - fieldName: field, - isLinked, - }) - if (fieldValue) { - idParts.push(fieldValue) - } - } - if (idParts.length === 0) { - return "" - } - return generateRowIdField(idParts) -} - function getEndpoint(tableId: string | undefined, operation: string) { if (!tableId) { return {} @@ -186,74 +159,6 @@ function getEndpoint(tableId: string | undefined, operation: string) { } } -// need to handle table name + field or just field, depending on if relationships used -function extractFieldValue({ - row, - tableName, - fieldName, - isLinked, -}: { - row: Row - tableName: string - fieldName: string - isLinked: boolean -}) { - let value = row[`${tableName}.${fieldName}`] - if (value == null && !isLinked) { - value = row[fieldName] - } - return value -} - -function basicProcessing({ - row, - table, - isLinked, -}: { - row: Row - table: Table - isLinked: boolean -}): Row { - const thisRow: Row = {} - // filter the row down to what is actually the row (not joined) - for (let field of Object.values(table.schema)) { - const fieldName = field.name - - const value = extractFieldValue({ - row, - tableName: table.name, - fieldName, - isLinked, - }) - - // all responses include "select col as table.col" so that overlaps are handled - if (value != null) { - thisRow[fieldName] = value - } - } - thisRow._id = generateIdForRow(row, table, isLinked) - thisRow.tableId = table._id - thisRow._rev = "rev" - return processFormulas(table, thisRow) -} - -function fixArrayTypes(row: Row, table: Table) { - for (let [fieldName, schema] of Object.entries(table.schema)) { - if ( - schema.type === FieldTypes.ARRAY && - typeof row[fieldName] === "string" - ) { - try { - row[fieldName] = JSON.parse(row[fieldName]) - } catch (err) { - // couldn't convert back to array, ignore - delete row[fieldName] - } - } - } - return row -} - function isOneSide(field: FieldSchema) { return ( field.relationshipType && field.relationshipType.split("-")[0] === "one" @@ -372,189 +277,6 @@ export class ExternalRequest { return { row: newRow, manyRelationships } } - squashRelationshipColumns( - table: Table, - row: Row, - relationships: RelationshipsJson[] - ): Row { - for (let relationship of relationships) { - const linkedTable = this.tables[relationship.tableName] - if (!linkedTable || !row[relationship.column]) { - continue - } - const display = linkedTable.primaryDisplay - for (let key of Object.keys(row[relationship.column])) { - let relatedRow: Row = row[relationship.column][key] - // add this row as context for the relationship - for (let col of Object.values(linkedTable.schema)) { - if (col.type === FieldType.LINK && col.tableId === table._id) { - relatedRow[col.name] = [row] - } - } - relatedRow = processFormulas(linkedTable, relatedRow) - let relatedDisplay - if (display) { - relatedDisplay = getPrimaryDisplay(relatedRow[display]) - } - row[relationship.column][key] = { - primaryDisplay: relatedDisplay || "Invalid display column", - _id: relatedRow._id, - } - } - } - return row - } - - /** - * This iterates through the returned rows and works out what elements of the rows - * actually match up to another row (based on primary keys) - this is pretty specific - * to SQL and the way that SQL relationships are returned based on joins. - * This is complicated, but the idea is that when a SQL query returns all the relations - * will be separate rows, with all of the data in each row. We have to decipher what comes - * from where (which tables) and how to convert that into budibase columns. - */ - updateRelationshipColumns( - table: Table, - row: Row, - rows: { [key: string]: Row }, - relationships: RelationshipsJson[] - ) { - const columns: { [key: string]: any } = {} - for (let relationship of relationships) { - const linkedTable = this.tables[relationship.tableName] - if (!linkedTable) { - continue - } - const fromColumn = `${table.name}.${relationship.from}` - const toColumn = `${linkedTable.name}.${relationship.to}` - // this is important when working with multiple relationships - // between the same tables, don't want to overlap/multiply the relations - if ( - !relationship.through && - row[fromColumn]?.toString() !== row[toColumn]?.toString() - ) { - continue - } - - let linked = basicProcessing({ row, table: linkedTable, isLinked: true }) - if (!linked._id) { - continue - } - columns[relationship.column] = linked - } - for (let [column, related] of Object.entries(columns)) { - if (!row._id) { - continue - } - const rowId: string = row._id - if (!Array.isArray(rows[rowId][column])) { - rows[rowId][column] = [] - } - // make sure relationship hasn't been found already - if ( - !rows[rowId][column].find( - (relation: Row) => relation._id === related._id - ) - ) { - rows[rowId][column].push(related) - } - } - return rows - } - - outputProcessing( - rows: Row[] = [], - table: Table, - relationships: RelationshipsJson[] - ) { - if (!rows || rows.length === 0 || rows[0].read === true) { - return [] - } - let finalRows: { [key: string]: Row } = {} - for (let row of rows) { - const rowId = generateIdForRow(row, table) - row._id = rowId - // this is a relationship of some sort - if (finalRows[rowId]) { - finalRows = this.updateRelationshipColumns( - table, - row, - finalRows, - relationships - ) - continue - } - const thisRow = fixArrayTypes( - basicProcessing({ row, table, isLinked: false }), - table - ) - if (thisRow._id == null) { - throw "Unable to generate row ID for SQL rows" - } - finalRows[thisRow._id] = thisRow - // do this at end once its been added to the final rows - finalRows = this.updateRelationshipColumns( - table, - row, - finalRows, - relationships - ) - } - - // Process some additional data types - let finalRowArray = Object.values(finalRows) - finalRowArray = processDates(table, finalRowArray) - finalRowArray = processFormulas(table, finalRowArray) as Row[] - - return finalRowArray.map((row: Row) => - this.squashRelationshipColumns(table, row, relationships) - ) - } - - /** - * Gets the list of relationship JSON structures based on the columns in the table, - * this will be used by the underlying library to build whatever relationship mechanism - * it has (e.g. SQL joins). - */ - buildRelationships(table: Table): RelationshipsJson[] { - const relationships = [] - for (let [fieldName, field] of Object.entries(table.schema)) { - if (field.type !== FieldTypes.LINK) { - continue - } - const { tableName: linkTableName } = breakExternalTableId(field.tableId) - // no table to link to, this is not a valid relationships - if (!linkTableName || !this.tables[linkTableName]) { - continue - } - const linkTable = this.tables[linkTableName] - if (!table.primary || !linkTable.primary) { - continue - } - const definition: any = { - // if no foreign key specified then use the name of the field in other table - from: field.foreignKey || table.primary[0], - to: field.fieldName, - tableName: linkTableName, - // need to specify where to put this back into - column: fieldName, - } - if (field.through) { - const { tableName: throughTableName } = breakExternalTableId( - field.through - ) - definition.through = throughTableName - // don't support composite keys for relationships - definition.from = field.throughTo || table.primary[0] - definition.to = field.throughFrom || linkTable.primary[0] - definition.fromPrimary = table.primary[0] - definition.toPrimary = linkTable.primary[0] - } - relationships.push(definition) - } - return relationships - } - /** * This is a cached lookup, of relationship records, this is mainly for creating/deleting junction * information. @@ -704,41 +426,6 @@ export class ExternalRequest { await Promise.all(promises) } - /** - * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which - * you have column overlap in relationships, e.g. we join a few different tables and they all have the - * concept of an ID, but for some of them it will be null (if they say don't have a relationship). - * Creating the specific list of fields that we desire, and excluding the ones that are no use to us - * is more performant and has the added benefit of protecting against this scenario. - */ - buildFields(table: Table, includeRelations: boolean) { - function extractRealFields(table: Table, existing: string[] = []) { - return Object.entries(table.schema) - .filter( - column => - column[1].type !== FieldTypes.LINK && - column[1].type !== FieldTypes.FORMULA && - !existing.find((field: string) => field === column[0]) - ) - .map(column => `${table.name}.${column[0]}`) - } - let fields = extractRealFields(table) - for (let field of Object.values(table.schema)) { - if (field.type !== FieldTypes.LINK || !includeRelations) { - continue - } - const { tableName: linkTableName } = breakExternalTableId(field.tableId) - if (linkTableName) { - const linkTable = this.tables[linkTableName] - if (linkTable) { - const linkedFields = extractRealFields(linkTable, fields) - fields = fields.concat(linkedFields) - } - } - } - return fields - } - async run(config: RunConfig) { const { operation, tableId } = this let { datasourceId, tableName } = breakExternalTableId(tableId) @@ -777,9 +464,9 @@ export class ExternalRequest { } } filters = buildFilters(id, filters || {}, table) - const relationships = this.buildRelationships(table) + const relationships = buildExternalRelationships(table, this.tables) - const includeSqlRelationships = + const incRelationships = config.includeSqlRelationships === IncludeRelationship.INCLUDE // clean up row on ingress using schema @@ -799,7 +486,11 @@ export class ExternalRequest { }, resource: { // have to specify the fields to avoid column overlap (for SQL) - fields: isSql ? this.buildFields(table, includeSqlRelationships) : [], + fields: isSql + ? buildSqlFieldList(table, this.tables, { + relationships: incRelationships, + }) + : [], }, filters, sort, @@ -825,7 +516,12 @@ export class ExternalRequest { processed.manyRelationships ) } - const output = this.outputProcessing(response, table, relationships) + const output = sqlOutputProcessing( + response, + table, + this.tables, + relationships + ) // if reading it'll just be an array of rows, return whole thing return operation === Operation.READ && Array.isArray(response) ? output diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 87d8cd7e9a..4ebe9b0acb 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -14,8 +14,8 @@ import { } from "../../../utilities/rowProcessor" import { FieldTypes } from "../../../constants" import * as utils from "./utils" -import { fullSearch, paginatedSearch } from "./internalSearch" -import { getGlobalUsersFromMetadata } from "../../../utilities/global" +// import { fullSearch, paginatedSearch } from "./internalSearch" +// import { getGlobalUsersFromMetadata } from "../../../utilities/global" import * as inMemoryViews from "../../../db/inMemoryView" import env from "../../../environment" import { @@ -36,6 +36,7 @@ import { Row, Table, } from "@budibase/types" +import { sqlSearch } from "./internalSql" import { cleanExportRows } from "./utils" @@ -355,43 +356,44 @@ export async function bulkDestroy(ctx: UserCtx) { } export async function search(ctx: UserCtx) { - // Fetch the whole table when running in cypress, as search doesn't work - if (!env.COUCH_DB_URL && env.isCypress()) { - return { rows: await fetch(ctx) } - } - - const { tableId } = ctx.params - const db = context.getAppDB() - const { paginate, query, ...params } = ctx.request.body - params.version = ctx.version - params.tableId = tableId - - let table - if (params.sort && !params.sortType) { - table = await db.get(tableId) - const schema = table.schema - const sortField = schema[params.sort] - params.sortType = sortField.type == "number" ? "number" : "string" - } - - let response - if (paginate) { - response = await paginatedSearch(query, params) - } else { - response = await fullSearch(query, params) - } - - // Enrich search results with relationships - if (response.rows && response.rows.length) { - // enrich with global users if from users table - if (tableId === InternalTables.USER_METADATA) { - response.rows = await getGlobalUsersFromMetadata(response.rows) - } - table = table || (await db.get(tableId)) - response.rows = await outputProcessing(table, response.rows) - } - - return response + return sqlSearch(ctx) + // // Fetch the whole table when running in cypress, as search doesn't work + // if (!env.COUCH_DB_URL && env.isCypress()) { + // return { rows: await fetch(ctx) } + // } + // + // const { tableId } = ctx.params + // const db = context.getAppDB() + // const { paginate, query, ...params } = ctx.request.body + // params.version = ctx.version + // params.tableId = tableId + // + // let table + // if (params.sort && !params.sortType) { + // table = await db.get(tableId) + // const schema = table.schema + // const sortField = schema[params.sort] + // params.sortType = sortField.type == "number" ? "number" : "string" + // } + // + // let response + // if (paginate) { + // response = await paginatedSearch(query, params) + // } else { + // response = await fullSearch(query, params) + // } + // + // // Enrich search results with relationships + // if (response.rows && response.rows.length) { + // // enrich with global users if from users table + // if (tableId === InternalTables.USER_METADATA) { + // response.rows = await getGlobalUsersFromMetadata(response.rows) + // } + // table = table || (await db.get(tableId)) + // response.rows = await outputProcessing(table, response.rows) + // } + // + // return response } export async function exportRows(ctx: UserCtx) { @@ -404,7 +406,7 @@ export async function exportRows(ctx: UserCtx) { } const { columns, query } = ctx.request.body - let result + let result: Row[] = [] if (rowIds) { let response = ( await db.allDocs({ @@ -413,7 +415,7 @@ export async function exportRows(ctx: UserCtx) { }) ).rows.map(row => row.doc) - result = await outputProcessing(table, response) + result = (await outputProcessing(table, response)) as Row[] } else if (query) { let searchResponse = await search(ctx) result = searchResponse.rows diff --git a/packages/server/src/api/controllers/row/internalSql.ts b/packages/server/src/api/controllers/row/internalSql.ts new file mode 100644 index 0000000000..5bc8435946 --- /dev/null +++ b/packages/server/src/api/controllers/row/internalSql.ts @@ -0,0 +1,158 @@ +import { + FieldType, + Operation, + QueryJson, + Row, + SearchFilters, + SortType, + Table, + UserCtx, +} from "@budibase/types" +import SqlQueryBuilder from "../../../integrations/base/sql" +import { SqlClient } from "../../../integrations/utils" +import { buildInternalRelationships, sqlOutputProcessing } from "./utils" +import sdk from "../../../sdk" +import { context } from "@budibase/backend-core" +import { CONSTANT_INTERNAL_ROW_COLS } from "../../../db/utils" + +function buildInternalFieldList( + table: Table, + tables: Table[], + opts: { relationships: boolean } = { relationships: true } +) { + let fieldList: string[] = [] + fieldList = fieldList.concat( + CONSTANT_INTERNAL_ROW_COLS.map(col => `${table._id}.${col}`) + ) + for (let col of Object.values(table.schema)) { + const isLink = col.type === FieldType.LINK + if (isLink && !opts.relationships) { + continue + } + if (isLink) { + const relatedTable = tables.find(table => table._id === col.tableId)! + fieldList = fieldList.concat( + buildInternalFieldList(relatedTable, tables, { relationships: false }) + ) + } else { + fieldList.push(`${table._id}.${col.name}`) + } + } + return fieldList +} + +function tableInFilter(name: string) { + return `:${name}.` +} + +function cleanupFilters(filters: SearchFilters, tables: Table[]) { + for (let filter of Object.values(filters)) { + if (typeof filter !== "object") { + continue + } + for (let [key, keyFilter] of Object.entries(filter)) { + if (keyFilter === "") { + delete filter[key] + } + + // relationship, switch to table ID + const tableRelated = tables.find(table => + key.includes(tableInFilter(table.originalName!)) + ) + if (tableRelated) { + filter[ + key.replace( + tableInFilter(tableRelated.originalName!), + tableInFilter(tableRelated._id!) + ) + ] = filter[key] + delete filter[key] + } + } + } + return filters +} + +function buildTableMap(tables: Table[]) { + const tableMap: Record = {} + for (let table of tables) { + // update the table name, should never query by name for SQLite + table.originalName = table.name + table.name = table._id! + tableMap[table._id!] = table + } + return tableMap +} + +export async function sqlSearch(ctx: UserCtx) { + const { tableId } = ctx.params + const { paginate, query, ...params } = ctx.request.body + + const builder = new SqlQueryBuilder(SqlClient.SQL_LITE) + const allTables = await sdk.tables.getAllInternalTables() + const allTablesMap = buildTableMap(allTables) + const table = allTables.find(table => table._id === tableId) + if (!table) { + ctx.throw(400, "Unable to find table") + } + + const relationships = buildInternalRelationships(table) + + const request: QueryJson = { + endpoint: { + // not important, we query ourselves + datasourceId: "internal", + entityId: table._id!, + operation: Operation.READ, + }, + filters: cleanupFilters(query, allTables), + table, + meta: { + table, + tables: allTablesMap, + }, + resource: { + fields: buildInternalFieldList(table, allTables), + }, + relationships, + } + // make sure only rows returned + request.filters!.equal = { + ...request.filters?.equal, + type: "row", + } + + if (params.sort && !params.sortType) { + const sortField = table.schema[params.sort] + const sortType = sortField.type == "number" ? "number" : "string" + request.sort = { + [sortField.name]: { + direction: params.sortOrder, + type: sortType as SortType, + }, + } + } + if (paginate) { + request.paginate = { + limit: params.limit, + page: params.bookmark, + } + } + let sql = builder._query(request, { + disableReturning: true, + disablePreparedStatements: true, + }) + + // quick hack for docIds + sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`") + sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`") + + const db = context.getAppDB() + const rows = await db.sql(sql) + + return { + rows: sqlOutputProcessing(rows, table, allTablesMap, relationships, { + internal: true, + }), + } +} diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts new file mode 100644 index 0000000000..75f380fef0 --- /dev/null +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -0,0 +1,107 @@ +// need to handle table name + field or just field, depending on if relationships used +import { Row, Table } from "@budibase/types" +import { generateRowIdField } from "../../../../integrations/utils" +import { processFormulas } from "../../../../utilities/rowProcessor" +import { FieldTypes } from "../../../../constants" +import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils" + +function extractFieldValue({ + row, + tableName, + fieldName, + isLinked, +}: { + row: Row + tableName: string + fieldName: string + isLinked: boolean +}) { + let value = row[`${tableName}.${fieldName}`] + if (value == null && !isLinked) { + value = row[fieldName] + } + return value +} + +export function generateIdForRow( + row: Row | undefined, + table: Table, + isLinked: boolean = false +): string { + const primary = table.primary + if (!row || !primary) { + return "" + } + // build id array + let idParts = [] + for (let field of primary) { + let fieldValue = extractFieldValue({ + row, + tableName: table.name, + fieldName: field, + isLinked, + }) + if (fieldValue) { + idParts.push(fieldValue) + } + } + if (idParts.length === 0) { + return "" + } + return generateRowIdField(idParts) +} + +export function basicProcessing({ + row, + table, + isLinked, + internal, +}: { + row: Row + table: Table + isLinked: boolean + internal?: boolean +}): Row { + let thisRow: Row = {} + // filter the row down to what is actually the row (not joined) + let toIterate = Object.keys(table.schema) + if (internal) { + toIterate = toIterate.concat(CONSTANT_INTERNAL_ROW_COLS) + } + for (let fieldName of toIterate) { + const value = extractFieldValue({ + row, + tableName: internal ? table._id! : table.name, + fieldName, + isLinked, + }) + + // all responses include "select col as table.col" so that overlaps are handled + if (value != null) { + thisRow[fieldName] = value + } + } + if (!internal) { + thisRow._id = generateIdForRow(row, table, isLinked) + thisRow.tableId = table._id + thisRow._rev = "rev" + } + return processFormulas(table, thisRow) +} + +export function fixArrayTypes(row: Row, table: Table) { + for (let [fieldName, schema] of Object.entries(table.schema)) { + if ( + schema.type === FieldTypes.ARRAY && + typeof row[fieldName] === "string" + ) { + try { + row[fieldName] = JSON.parse(row[fieldName]) + } catch (err) { + // couldn't convert back to array, ignore + delete row[fieldName] + } + } + } + return row +} diff --git a/packages/server/src/api/controllers/row/utils/index.ts b/packages/server/src/api/controllers/row/utils/index.ts new file mode 100644 index 0000000000..ec88d01f22 --- /dev/null +++ b/packages/server/src/api/controllers/row/utils/index.ts @@ -0,0 +1,3 @@ +export * from "./basic" +export * from "./sqlUtils" +export * from "./utils" diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts new file mode 100644 index 0000000000..9211671ace --- /dev/null +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -0,0 +1,224 @@ +import { FieldType, RelationshipsJson, Row, Table } from "@budibase/types" +import { processFormulas } from "../../../../utilities/rowProcessor" +import { + breakExternalTableId, + getPrimaryDisplay, +} from "../../../../integrations/utils" +import { basicProcessing } from "./basic" +import { generateJunctionTableID } from "../../../../db/utils" + +type TableMap = Record + +export function squashRelationshipColumns( + table: Table, + tables: TableMap, + row: Row, + relationships: RelationshipsJson[] +): Row { + for (let relationship of relationships) { + const linkedTable = tables[relationship.tableName] + if (!linkedTable || !row[relationship.column]) { + continue + } + const display = linkedTable.primaryDisplay + for (let key of Object.keys(row[relationship.column])) { + let relatedRow: Row = row[relationship.column][key] + // add this row as context for the relationship + for (let col of Object.values(linkedTable.schema)) { + if (col.type === FieldType.LINK && col.tableId === table._id) { + relatedRow[col.name] = [row] + } + } + relatedRow = processFormulas(linkedTable, relatedRow) + let relatedDisplay + if (display) { + relatedDisplay = getPrimaryDisplay(relatedRow[display]) + } + row[relationship.column][key] = { + primaryDisplay: relatedDisplay || "Invalid display column", + _id: relatedRow._id, + } + } + } + return row +} + +/** + * This iterates through the returned rows and works out what elements of the rows + * actually match up to another row (based on primary keys) - this is pretty specific + * to SQL and the way that SQL relationships are returned based on joins. + * This is complicated, but the idea is that when a SQL query returns all the relations + * will be separate rows, with all of the data in each row. We have to decipher what comes + * from where (which tables) and how to convert that into budibase columns. + */ +export function updateRelationshipColumns( + table: Table, + tables: TableMap, + row: Row, + rows: { [key: string]: Row }, + relationships: RelationshipsJson[], + opts?: { internal?: boolean } +) { + const columns: { [key: string]: any } = {} + for (let relationship of relationships) { + const linkedTable = tables[relationship.tableName] + if (!linkedTable) { + continue + } + const fromColumn = `${table.name}.${relationship.from}` + const toColumn = `${linkedTable.name}.${relationship.to}` + // this is important when working with multiple relationships + // between the same tables, don't want to overlap/multiply the relations + if ( + !relationship.through && + row[fromColumn]?.toString() !== row[toColumn]?.toString() + ) { + continue + } + + let linked = basicProcessing({ + row, + table: linkedTable, + isLinked: true, + internal: opts?.internal, + }) + if (!linked._id) { + continue + } + columns[relationship.column] = linked + } + for (let [column, related] of Object.entries(columns)) { + let rowId: string = row._id! + if (opts?.internal) { + const { _id } = basicProcessing({ + row, + table, + isLinked: false, + internal: opts?.internal, + }) + rowId = _id! + } + if (!rowId) { + continue + } + if (!Array.isArray(rows[rowId][column])) { + rows[rowId][column] = [] + } + // make sure relationship hasn't been found already + if ( + !rows[rowId][column].find((relation: Row) => relation._id === related._id) + ) { + rows[rowId][column].push(related) + } + } + return rows +} + +/** + * Gets the list of relationship JSON structures based on the columns in the table, + * this will be used by the underlying library to build whatever relationship mechanism + * it has (e.g. SQL joins). + */ +export function buildExternalRelationships( + table: Table, + tables: TableMap +): RelationshipsJson[] { + const relationships = [] + for (let [fieldName, field] of Object.entries(table.schema)) { + if (field.type !== FieldType.LINK) { + continue + } + const { tableName: linkTableName } = breakExternalTableId(field.tableId) + // no table to link to, this is not a valid relationships + if (!linkTableName || !tables[linkTableName]) { + continue + } + const linkTable = tables[linkTableName] + if (!table.primary || !linkTable.primary) { + continue + } + const definition: any = { + // if no foreign key specified then use the name of the field in other table + from: field.foreignKey || table.primary[0], + to: field.fieldName, + tableName: linkTableName, + // need to specify where to put this back into + column: fieldName, + } + if (field.through) { + const { tableName: throughTableName } = breakExternalTableId( + field.through + ) + definition.through = throughTableName + // don't support composite keys for relationships + definition.from = field.throughTo || table.primary[0] + definition.to = field.throughFrom || linkTable.primary[0] + definition.fromPrimary = table.primary[0] + definition.toPrimary = linkTable.primary[0] + } + relationships.push(definition) + } + return relationships +} + +export function buildInternalRelationships(table: Table): RelationshipsJson[] { + const relationships: RelationshipsJson[] = [] + const links = Object.values(table.schema).filter( + column => column.type === FieldType.LINK + ) + const tableId = table._id! + for (let link of links) { + const linkTableId = link.tableId! + const junctionTableId = generateJunctionTableID(tableId, linkTableId) + const isFirstTable = tableId > linkTableId + relationships.push({ + through: junctionTableId, + column: link.name, + tableName: linkTableId, + fromPrimary: "_id", + to: isFirstTable ? "doc2.rowId" : "doc1.rowId", + from: isFirstTable ? "doc1.rowId" : "doc2.rowId", + toPrimary: "_id", + }) + } + return relationships +} + +/** + * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which + * you have column overlap in relationships, e.g. we join a few different tables and they all have the + * concept of an ID, but for some of them it will be null (if they say don't have a relationship). + * Creating the specific list of fields that we desire, and excluding the ones that are no use to us + * is more performant and has the added benefit of protecting against this scenario. + */ +export function buildSqlFieldList( + table: Table, + tables: TableMap, + opts?: { relationships: boolean } +) { + function extractRealFields(table: Table, existing: string[] = []) { + return Object.entries(table.schema) + .filter( + column => + column[1].type !== FieldType.LINK && + column[1].type !== FieldType.FORMULA && + !existing.find((field: string) => field === column[0]) + ) + .map(column => `${table.name}.${column[0]}`) + } + let fields = extractRealFields(table) + for (let field of Object.values(table.schema)) { + if (field.type !== FieldType.LINK || !opts?.relationships) { + continue + } + const { tableName: linkTableName } = breakExternalTableId(field.tableId) + if (linkTableName) { + const linkTable = tables[linkTableName] + if (linkTable) { + const linkedFields = extractRealFields(linkTable, fields) + fields = fields.concat(linkedFields) + } + } + } + return fields +} diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts similarity index 69% rename from packages/server/src/api/controllers/row/utils.ts rename to packages/server/src/api/controllers/row/utils/utils.ts index a213f14a08..96a87bcf5e 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -1,11 +1,26 @@ -import { InternalTables } from "../../../db/utils" -import * as userController from "../user" -import { FieldTypes } from "../../../constants" +import { InternalTables } from "../../../../db/utils" +import * as userController from "../../user" +import { FieldTypes } from "../../../../constants" import { context } from "@budibase/backend-core" -import { makeExternalQuery } from "../../../integrations/base/query" -import { FieldType, Row, Table, UserCtx } from "@budibase/types" -import { Format } from "../view/exporters" -import sdk from "../../../sdk" +import { makeExternalQuery } from "../../../../integrations/base/query" +import { + FieldType, + RelationshipsJson, + Row, + Table, + UserCtx, +} from "@budibase/types" +import { Format } from "../../view/exporters" +import sdk from "../../../../sdk" +import { + processDates, + processFormulas, +} from "../../../../utilities/rowProcessor" +import { + squashRelationshipColumns, + updateRelationshipColumns, +} from "./sqlUtils" +import { basicProcessing, generateIdForRow, fixArrayTypes } from "./basic" const validateJs = require("validate.js") const { cloneDeep } = require("lodash/fp") @@ -177,3 +192,65 @@ export function getTableId(ctx: any) { return ctx.params.viewName } } + +export function sqlOutputProcessing( + rows: Row[] = [], + table: Table, + tables: Record, + relationships: RelationshipsJson[], + opts?: { internal?: boolean } +) { + if (!rows || rows.length === 0 || rows[0].read === true) { + return [] + } + let finalRows: { [key: string]: Row } = {} + for (let row of rows) { + let rowId = row._id + if (!rowId) { + rowId = generateIdForRow(row, table) + row._id = rowId + } + // this is a relationship of some sort + if (finalRows[rowId]) { + finalRows = updateRelationshipColumns( + table, + tables, + row, + finalRows, + relationships + ) + continue + } + const thisRow = fixArrayTypes( + basicProcessing({ + row, + table, + isLinked: false, + internal: opts?.internal, + }), + table + ) + if (thisRow._id == null) { + throw "Unable to generate row ID for SQL rows" + } + finalRows[thisRow._id] = thisRow + // do this at end once its been added to the final rows + finalRows = updateRelationshipColumns( + table, + tables, + row, + finalRows, + relationships, + opts + ) + } + + // Process some additional data types + let finalRowArray = Object.values(finalRows) + finalRowArray = processDates(table, finalRowArray) + finalRowArray = processFormulas(table, finalRowArray) as Row[] + + return finalRowArray.map((row: Row) => + squashRelationshipColumns(table, tables, row, relationships) + ) +} diff --git a/packages/server/src/api/controllers/table/sqlite.ts b/packages/server/src/api/controllers/table/sqlite.ts new file mode 100644 index 0000000000..f3c156c515 --- /dev/null +++ b/packages/server/src/api/controllers/table/sqlite.ts @@ -0,0 +1,75 @@ +import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core" +import { FieldType, SQLiteDefinition, SQLiteType, Table } from "@budibase/types" +import { cloneDeep } from "lodash" +import sdk from "../../../sdk" +import { CONSTANT_INTERNAL_ROW_COLS } from "../../../db/utils" + +const BASIC_SQLITE_DOC: SQLiteDefinition = { + _id: SQLITE_DESIGN_DOC_ID, + language: "sqlite", + sql: { + tables: {}, + options: { + table_name: "tableId", + }, + }, +} + +const FieldTypeMap: Record = { + [FieldType.BOOLEAN]: SQLiteType.NUMERIC, + [FieldType.DATETIME]: SQLiteType.TEXT, + [FieldType.FORMULA]: SQLiteType.TEXT, + [FieldType.LONGFORM]: SQLiteType.TEXT, + [FieldType.NUMBER]: SQLiteType.REAL, + [FieldType.STRING]: SQLiteType.TEXT, + [FieldType.AUTO]: SQLiteType.TEXT, + [FieldType.JSON]: SQLiteType.BLOB, + [FieldType.OPTIONS]: SQLiteType.BLOB, + [FieldType.INTERNAL]: SQLiteType.BLOB, + [FieldType.BARCODEQR]: SQLiteType.BLOB, + [FieldType.ATTACHMENT]: SQLiteType.BLOB, + [FieldType.ARRAY]: SQLiteType.BLOB, + [FieldType.LINK]: SQLiteType.BLOB, +} + +function mapTable(table: Table): { [key: string]: SQLiteType } { + const fields: Record = {} + for (let [key, column] of Object.entries(table.schema)) { + fields[key] = FieldTypeMap[column.type] + } + // there are some extra columns to map - add these in + const constantMap: Record = {} + CONSTANT_INTERNAL_ROW_COLS.forEach(col => { + constantMap[col] = SQLiteType.TEXT + }) + return { + ...constantMap, + ...fields, + } +} + +// nothing exists, need to iterate though existing tables +async function buildBaseDefinition(): Promise { + const tables = await sdk.tables.getAllInternalTables() + const definition = cloneDeep(BASIC_SQLITE_DOC) + for (let table of tables) { + definition.sql.tables[table._id!] = { + fields: mapTable(table), + } + } + return definition +} + +export async function addTableToSqlite(table: Table) { + const db = context.getAppDB() + let definition: SQLiteDefinition + try { + definition = await db.get(SQLITE_DESIGN_DOC_ID) + } catch (err) { + definition = await buildBaseDefinition() + } + definition.sql.tables[table._id!] = { + fields: mapTable(table), + } + await db.put(definition) +} diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 923608e9a2..9d33a6165a 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -27,6 +27,7 @@ import { SourceName, Table, } from "@budibase/types" +import { addTableToSqlite } from "./sqlite" export async function clearColumns(table: any, columnNames: any) { const db: Database = context.getAppDB() @@ -293,6 +294,7 @@ class TableSaveFunctions { async after(table: any) { table = await handleSearchIndexes(table) table = await handleDataImport(this.user, table, this.importRows) + await addTableToSqlite(table) return table } diff --git a/packages/server/src/db/linkedRows/LinkDocument.ts b/packages/server/src/db/linkedRows/LinkDocument.ts index 9035641d5f..b902a76f26 100644 --- a/packages/server/src/db/linkedRows/LinkDocument.ts +++ b/packages/server/src/db/linkedRows/LinkDocument.ts @@ -1,4 +1,4 @@ -import { generateLinkID } from "../utils" +import { generateLinkID, generateJunctionTableID } from "../utils" import { FieldTypes } from "../../constants" import { LinkDocument } from "@budibase/types" @@ -17,6 +17,7 @@ import { LinkDocument } from "@budibase/types" class LinkDocumentImpl implements LinkDocument { _id: string type: string + tableId: string doc1: { rowId: string fieldName: string @@ -44,16 +45,20 @@ class LinkDocumentImpl implements LinkDocument { fieldName2 ) this.type = FieldTypes.LINK - this.doc1 = { + this.tableId = generateJunctionTableID(tableId1, tableId2) + const docA = { tableId: tableId1, fieldName: fieldName1, rowId: rowId1, } - this.doc2 = { + const docB = { tableId: tableId2, fieldName: fieldName2, rowId: rowId2, } + // have to determine which one will be doc1 - very important for SQL linking + this.doc1 = docA.tableId > docB.tableId ? docA : docB + this.doc2 = docA.tableId > docB.tableId ? docB : docA } } diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index e08392c3a1..8da1e0ac53 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -1,5 +1,6 @@ import newid from "./newid" import { db as dbCore } from "@budibase/backend-core" +import { SQLiteType } from "@budibase/types" type Optional = string | null @@ -43,6 +44,14 @@ export const getUserMetadataParams = dbCore.getUserMetadataParams export const generateUserMetadataID = dbCore.generateUserMetadataID export const getGlobalIDFromUserMetadataID = dbCore.getGlobalIDFromUserMetadataID +export const CONSTANT_INTERNAL_ROW_COLS = [ + "_id", + "_rev", + "type", + "createdAt", + "updatedAt", + "tableId", +] /** * Gets parameters for retrieving tables, this is a utility function for the getDocParams function. @@ -262,6 +271,12 @@ export function generatePluginID(name: string) { return `${DocumentType.PLUGIN}${SEPARATOR}${name}` } +export function generateJunctionTableID(tableId1: string, tableId2: string) { + const first = tableId1 > tableId2 ? tableId1 : tableId2 + const second = tableId1 > tableId2 ? tableId2 : tableId1 + return `${first}${SEPARATOR}${second}` +} + /** * This can be used with the db.allDocs to get a list of IDs */ diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index 1e436670a8..1c9c18ca7b 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -6,4 +6,5 @@ export interface QueryOptions { disableReturning?: boolean + disablePreparedStatements?: boolean } diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 90107dfd4e..2adad1334d 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -564,8 +564,12 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { throw `Operation type is not supported by SQL query builder` } - // @ts-ignore - return query.toSQL().toNative() + if (opts?.disablePreparedStatements) { + return query.toString() + } else { + // @ts-ignore + return query.toSQL().toNative() + } } async getReturningRow(queryFn: Function, json: QueryJson) { diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 75deaf7f30..27f3ae0589 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -74,6 +74,7 @@ export enum SqlClient { POSTGRES = "pg", MY_SQL = "mysql2", ORACLE = "oracledb", + SQL_LITE = "better-sqlite3", } export function isExternalTable(tableId: string) { diff --git a/packages/types/src/documents/app/index.ts b/packages/types/src/documents/app/index.ts index b81c9e36ac..c2c1c8eb1a 100644 --- a/packages/types/src/documents/app/index.ts +++ b/packages/types/src/documents/app/index.ts @@ -14,3 +14,4 @@ export * from "./backup" export * from "./webhook" export * from "./links" export * from "./component" +export * from "./sqlite" diff --git a/packages/types/src/documents/app/links.ts b/packages/types/src/documents/app/links.ts index d6b2adddf8..3f3a83740a 100644 --- a/packages/types/src/documents/app/links.ts +++ b/packages/types/src/documents/app/links.ts @@ -2,6 +2,7 @@ import { Document } from "../document" export interface LinkDocument extends Document { type: string + tableId: string doc1: { rowId: string fieldName: string diff --git a/packages/types/src/documents/app/sqlite.ts b/packages/types/src/documents/app/sqlite.ts new file mode 100644 index 0000000000..76c47bbd74 --- /dev/null +++ b/packages/types/src/documents/app/sqlite.ts @@ -0,0 +1,24 @@ +export enum SQLiteType { + REAL = "REAL", + TEXT = "VARCHAR", + INT = "INTEGER", + BLOB = "BLOB", + NUMERIC = "NUMERIC", +} + +export interface SQLiteDefinition { + _id: string + language: string + sql: { + tables: { + [tableName: string]: { + fields: { + [key: string]: SQLiteType | { field: string; type: SQLiteType } + } + } + } + options: { + table_name: string + } + } +} diff --git a/packages/types/src/documents/app/table.ts b/packages/types/src/documents/app/table.ts index 18b415da5f..40ec81a59a 100644 --- a/packages/types/src/documents/app/table.ts +++ b/packages/types/src/documents/app/table.ts @@ -74,6 +74,7 @@ export interface Table extends Document { type?: string views?: { [key: string]: View } name: string + originalName?: string primary?: string[] schema: TableSchema primaryDisplay?: string diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index 58b3b7e5bc..d5d3dd58b2 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -100,6 +100,7 @@ export interface Database { ): Promise bulkDocs(documents: AnyDocument[]): Promise allDocs(params: DatabaseQueryOpts): Promise> + sql(sql: string): Promise query( viewName: string, params: DatabaseQueryOpts