diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 5d0a251900..fd24c8e12c 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -34,6 +34,8 @@ import { } from "@budibase/types" import environment from "../environment" import { helpers } from "@budibase/shared-core" +import { isPlainObject } from "lodash" +import { ColumnSplitter } from "@budibase/shared-core/src/filters" type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any @@ -72,9 +74,15 @@ function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] { class InternalBuilder { private readonly client: SqlClient + private readonly query: QueryJson - constructor(client: SqlClient) { + constructor(client: SqlClient, query: QueryJson) { this.client = client + this.query = query + } + + get table(): Table { + return this.query.meta.table } // Takes a string like foo and returns a quoted string like [foo] for SQL Server @@ -101,11 +109,8 @@ class InternalBuilder { .join(".") } - private generateSelectStatement( - json: QueryJson, - knex: Knex - ): (string | Knex.Raw)[] | "*" { - const { resource, meta } = json + private generateSelectStatement(knex: Knex): (string | Knex.Raw)[] | "*" { + const { resource, meta } = this.query const client = knex.client.config.client as SqlClient if (!resource || !resource.fields || resource.fields.length === 0) { @@ -183,10 +188,10 @@ class InternalBuilder { // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses, // so when we use them we need to wrap them in to_char(). This function // converts a field name to the appropriate identifier. - private convertClobs(table: Table, field: string): string { + private convertClobs(field: string): string { const parts = field.split(".") const col = parts.pop()! - const schema = table.schema[col] + const schema = this.table.schema[col] let identifier = this.quotedIdentifier(field) if ( schema.type === FieldType.STRING || @@ -201,54 +206,60 @@ class InternalBuilder { } private parse(input: any, schema: FieldSchema) { + if (input == undefined) { + return null + } + + if (isPlainObject(input)) { + for (const [key, value] of Object.entries(input)) { + input[key] = this.parse(value, schema) + } + return input + } + if (schema.type === FieldType.DATETIME && schema.timeOnly) { if (this.client === SqlClient.ORACLE) { return new Date(`1970-01-01 ${input}`) } } - if (Array.isArray(input)) { - return JSON.stringify(input) - } - if (input == undefined) { - return null - } - if (typeof input !== "string") { - return input - } - if (isInvalidISODateString(input)) { - return null - } - if (isValidISODateString(input)) { - return new Date(input.trim()) + if (typeof input === "string") { + if (isInvalidISODateString(input)) { + return null + } + if (isValidISODateString(input)) { + return new Date(input.trim()) + } } + return input } - private parseBody(body: any, table: Table) { + private parseBody(body: any) { for (let [key, value] of Object.entries(body)) { - body[key] = this.parse(value, table.schema[key]) + body[key] = this.parse(value, this.table.schema[key]) } return body } - private parseFilters( - filters: SearchFilters | undefined, - table: Table - ): SearchFilters { + private parseFilters(filters: SearchFilters | undefined): SearchFilters { if (!filters) { return {} } - for (let [key, value] of Object.entries(filters)) { - let parsed - if (typeof value === "object") { - parsed = this.parseFilters(value, table) - } else { - parsed = this.parse(value, table.schema[key]) + + for (const [_, filter] of Object.entries(filters)) { + for (const [key, value] of Object.entries(filter)) { + const { column } = new ColumnSplitter([this.table]).run(key) + const schema = this.table.schema[column] + if (!schema) { + throw new Error( + `Column ${key} does not exist in table ${this.table._id}` + ) + } + filter[key] = this.parse(value, schema) } - // @ts-ignore - filters[key] = parsed } + return filters } @@ -256,28 +267,26 @@ class InternalBuilder { addFilters( query: Knex.QueryBuilder, filters: SearchFilters | undefined, - table: Table, - opts: { - aliases?: Record + opts?: { relationship?: boolean - columnPrefix?: string } ): Knex.QueryBuilder { if (!filters) { return query } - filters = this.parseFilters(filters, table) + filters = this.parseFilters(filters) + const aliases = this.query.tableAliases // if all or specified in filters, then everything is an or const allOr = filters.allOr - const sqlStatements = new SqlStatements(this.client, table, { + const sqlStatements = new SqlStatements(this.client, this.table, { allOr, - columnPrefix: opts.columnPrefix, + columnPrefix: this.query.meta.columnPrefix, }) const tableName = - this.client === SqlClient.SQL_LITE ? table._id! : table.name + this.client === SqlClient.SQL_LITE ? this.table._id! : this.table.name function getTableAlias(name: string) { - const alias = opts.aliases?.[name] + const alias = aliases?.[name] return alias || name } function iterate( @@ -303,10 +312,10 @@ class InternalBuilder { ), castedTypeValue.values ) - } else if (!opts.relationship && !isRelationshipField) { + } else if (!opts?.relationship && !isRelationshipField) { const alias = getTableAlias(tableName) fn(alias ? `${alias}.${updatedKey}` : updatedKey, value) - } else if (opts.relationship && isRelationshipField) { + } else if (opts?.relationship && isRelationshipField) { const [filterTableName, property] = updatedKey.split(".") const alias = getTableAlias(filterTableName) fn(alias ? `${alias}.${property}` : property, value) @@ -394,7 +403,7 @@ class InternalBuilder { filters.oneOf, (key: string, array) => { if (this.client === SqlClient.ORACLE) { - key = this.convertClobs(table, key) + key = this.convertClobs(key) array = Array.isArray(array) ? array : [array] const binding = new Array(array.length).fill("?").join(",") query = query.whereRaw(`${key} IN (${binding})`, array) @@ -460,7 +469,7 @@ class InternalBuilder { [value] ) } else if (this.client === SqlClient.ORACLE) { - const identifier = this.convertClobs(table, key) + const identifier = this.convertClobs(key) query = query[fnc]( `(${identifier} IS NOT NULL AND ${identifier} = ?)`, [value] @@ -482,7 +491,7 @@ class InternalBuilder { [value] ) } else if (this.client === SqlClient.ORACLE) { - const identifier = this.convertClobs(table, key) + const identifier = this.convertClobs(key) query = query[fnc]( `(${identifier} IS NOT NULL AND ${identifier} != ?)`, [value] @@ -517,9 +526,9 @@ class InternalBuilder { contains(filters.containsAny, true) } - const tableRef = opts?.aliases?.[table._id!] || table._id + const tableRef = aliases?.[this.table._id!] || this.table._id // when searching internal tables make sure long looking for rows - if (filters.documentType && !isExternalTable(table) && tableRef) { + if (filters.documentType && !isExternalTable(this.table) && tableRef) { // has to be its own option, must always be AND onto the search query.andWhereLike( `${tableRef}._id`, @@ -530,29 +539,26 @@ class InternalBuilder { return query } - addDistinctCount( - query: Knex.QueryBuilder, - json: QueryJson - ): Knex.QueryBuilder { - const table = json.meta.table - const primary = table.primary - const aliases = json.tableAliases + addDistinctCount(query: Knex.QueryBuilder): Knex.QueryBuilder { + const primary = this.table.primary + const aliases = this.query.tableAliases const aliased = - table.name && aliases?.[table.name] ? aliases[table.name] : table.name + this.table.name && aliases?.[this.table.name] + ? aliases[this.table.name] + : this.table.name if (!primary) { throw new Error("SQL counting requires primary key to be supplied") } return query.countDistinct(`${aliased}.${primary[0]} as total`) } - addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder { - let { sort } = json - const table = json.meta.table - const primaryKey = table.primary - const tableName = getTableName(table) - const aliases = json.tableAliases + addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder { + let { sort } = this.query + const primaryKey = this.table.primary + const tableName = getTableName(this.table) + const aliases = this.query.tableAliases const aliased = - tableName && aliases?.[tableName] ? aliases[tableName] : table?.name + tableName && aliases?.[tableName] ? aliases[tableName] : this.table?.name if (!Array.isArray(primaryKey)) { throw new Error("Sorting requires primary key to be specified for table") } @@ -667,26 +673,28 @@ class InternalBuilder { return query } - knexWithAlias( + qualifiedKnex( knex: Knex, - endpoint: QueryJson["endpoint"], - aliases?: QueryJson["tableAliases"] + opts?: { alias?: string | boolean } ): Knex.QueryBuilder { - const tableName = endpoint.entityId - const tableAlias = aliases?.[tableName] - + let alias = this.query.tableAliases?.[this.query.endpoint.entityId] + if (opts?.alias === false) { + alias = undefined + } else if (typeof opts?.alias === "string") { + alias = opts.alias + } return knex( - this.tableNameWithSchema(tableName, { - alias: tableAlias, - schema: endpoint.schema, + this.tableNameWithSchema(this.query.endpoint.entityId, { + alias, + schema: this.query.endpoint.schema, }) ) } - create(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder { - const { endpoint, body } = json - let query = this.knexWithAlias(knex, endpoint) - const parsedBody = this.parseBody(body, json.meta.table) + create(knex: Knex, opts: QueryOptions): Knex.QueryBuilder { + const { body } = this.query + let query = this.qualifiedKnex(knex, { alias: false }) + const parsedBody = this.parseBody(body) // make sure no null values in body for creation for (let [key, value] of Object.entries(parsedBody)) { if (value == null) { @@ -702,29 +710,29 @@ class InternalBuilder { } } - bulkCreate(knex: Knex, json: QueryJson): Knex.QueryBuilder { - const { endpoint, body } = json - let query = this.knexWithAlias(knex, endpoint) + bulkCreate(knex: Knex): Knex.QueryBuilder { + const { body } = this.query + let query = this.qualifiedKnex(knex, { alias: false }) if (!Array.isArray(body)) { return query } - const parsedBody = body.map(row => this.parseBody(row, json.meta.table)) + const parsedBody = body.map(row => this.parseBody(row)) return query.insert(parsedBody) } - bulkUpsert(knex: Knex, json: QueryJson): Knex.QueryBuilder { - const { endpoint, body } = json - let query = this.knexWithAlias(knex, endpoint) + bulkUpsert(knex: Knex): Knex.QueryBuilder { + const { body } = this.query + let query = this.qualifiedKnex(knex, { alias: false }) if (!Array.isArray(body)) { return query } - const parsedBody = body.map(row => this.parseBody(row, json.meta.table)) + const parsedBody = body.map(row => this.parseBody(row)) if ( this.client === SqlClient.POSTGRES || this.client === SqlClient.SQL_LITE || this.client === SqlClient.MY_SQL ) { - const primary = json.meta.table.primary + const primary = this.table.primary if (!primary) { throw new Error("Primary key is required for upsert") } @@ -743,18 +751,18 @@ class InternalBuilder { read( knex: Knex, - json: QueryJson, opts: { limits?: { base: number; query: number } } = {} ): Knex.QueryBuilder { - let { endpoint, filters, paginate, relationships, tableAliases } = json + let { endpoint, filters, paginate, relationships, tableAliases } = + this.query const { limits } = opts const counting = endpoint.operation === Operation.COUNT const tableName = endpoint.entityId // start building the query - let query = this.knexWithAlias(knex, endpoint, tableAliases) + let query = this.qualifiedKnex(knex) // handle pagination let foundOffset: number | null = null let foundLimit = limits?.query || limits?.base @@ -782,13 +790,10 @@ class InternalBuilder { } // add sorting to pre-query // no point in sorting when counting - query = this.addSorting(query, json) + query = this.addSorting(query) } // add filters to the query (where) - query = this.addFilters(query, filters, json.meta.table, { - columnPrefix: json.meta.columnPrefix, - aliases: tableAliases, - }) + query = this.addFilters(query, filters) const alias = tableAliases?.[tableName] || tableName let preQuery: Knex.QueryBuilder = knex({ @@ -800,11 +805,11 @@ class InternalBuilder { }) // if counting, use distinct count, else select preQuery = !counting - ? preQuery.select(this.generateSelectStatement(json, knex)) - : this.addDistinctCount(preQuery, json) + ? preQuery.select(this.generateSelectStatement(knex)) + : this.addDistinctCount(preQuery) // have to add after as well (this breaks MS-SQL) if (this.client !== SqlClient.MS_SQL && !counting) { - preQuery = this.addSorting(preQuery, json) + preQuery = this.addSorting(preQuery) } // handle joins query = this.addRelationships( @@ -821,21 +826,14 @@ class InternalBuilder { query = query.limit(limits.base) } - return this.addFilters(query, filters, json.meta.table, { - columnPrefix: json.meta.columnPrefix, - relationship: true, - aliases: tableAliases, - }) + return this.addFilters(query, filters, { relationship: true }) } - update(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder { - const { endpoint, body, filters, tableAliases } = json - let query = this.knexWithAlias(knex, endpoint, tableAliases) - const parsedBody = this.parseBody(body, json.meta.table) - query = this.addFilters(query, filters, json.meta.table, { - columnPrefix: json.meta.columnPrefix, - aliases: tableAliases, - }) + update(knex: Knex, opts: QueryOptions): Knex.QueryBuilder { + const { body, filters } = this.query + let query = this.qualifiedKnex(knex) + const parsedBody = this.parseBody(body) + query = this.addFilters(query, filters) // mysql can't use returning if (opts.disableReturning) { return query.update(parsedBody) @@ -844,18 +842,15 @@ class InternalBuilder { } } - delete(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder { - const { endpoint, filters, tableAliases } = json - let query = this.knexWithAlias(knex, endpoint, tableAliases) - query = this.addFilters(query, filters, json.meta.table, { - columnPrefix: json.meta.columnPrefix, - aliases: tableAliases, - }) + delete(knex: Knex, opts: QueryOptions): Knex.QueryBuilder { + const { filters } = this.query + let query = this.qualifiedKnex(knex) + query = this.addFilters(query, filters) // mysql can't use returning if (opts.disableReturning) { return query.delete() } else { - return query.delete().returning(this.generateSelectStatement(json, knex)) + return query.delete().returning(this.generateSelectStatement(knex)) } } } @@ -899,13 +894,13 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { const client = knex(config) let query: Knex.QueryBuilder - const builder = new InternalBuilder(sqlClient) + const builder = new InternalBuilder(sqlClient, json) switch (this._operation(json)) { case Operation.CREATE: - query = builder.create(client, json, opts) + query = builder.create(client, opts) break case Operation.READ: - query = builder.read(client, json, { + query = builder.read(client, { limits: { query: this.limit, base: BASE_LIMIT, @@ -914,19 +909,19 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { break case Operation.COUNT: // read without any limits to count - query = builder.read(client, json) + query = builder.read(client) break case Operation.UPDATE: - query = builder.update(client, json, opts) + query = builder.update(client, opts) break case Operation.DELETE: - query = builder.delete(client, json, opts) + query = builder.delete(client, opts) break case Operation.BULK_CREATE: - query = builder.bulkCreate(client, json) + query = builder.bulkCreate(client) break case Operation.BULK_UPSERT: - query = builder.bulkUpsert(client, json) + query = builder.bulkUpsert(client) break case Operation.CREATE_TABLE: case Operation.UPDATE_TABLE: diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index c6c5786e53..a08fac7396 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -40,14 +40,14 @@ import { structures } from "@budibase/backend-core/tests" import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default" describe.each([ - // ["in-memory", undefined], - // ["lucene", undefined], - // ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + ["in-memory", undefined], + ["lucene", undefined], + ["sqs", undefined], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" @@ -1318,7 +1318,7 @@ describe.each([ }) !isInternal && - describe.only("datetime - time only", () => { + describe("datetime - time only", () => { const T_1000 = "10:00:00" const T_1045 = "10:45:00" const T_1200 = "12:00:00" @@ -2389,9 +2389,9 @@ describe.each([ describe.each([ { low: "2024-07-03T00:00:00.000Z", high: "9999-00-00T00:00:00.000Z" }, - { low: "2024-07-03T00:00:00.000Z", high: "9998-00-00T00:00:00.000Z" }, - { low: "0000-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" }, - { low: "0001-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" }, + // { low: "2024-07-03T00:00:00.000Z", high: "9998-00-00T00:00:00.000Z" }, + // { low: "0000-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" }, + // { low: "0001-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" }, ])("date special cases", ({ low, high }) => { const earlyDate = "2024-07-03T10:00:00.000Z", laterDate = "2024-07-03T11:00:00.000Z" @@ -2405,7 +2405,7 @@ describe.each([ await createRows([{ date: earlyDate }, { date: laterDate }]) }) - it("should be able to handle a date search", async () => { + it.only("should be able to handle a date search", async () => { await expectSearch({ query: { range: { @@ -2418,25 +2418,25 @@ describe.each([ describe.each([ "名前", // Japanese for "name" - // "Benutzer-ID", // German for "user ID", includes a hyphen - // "numéro", // French for "number", includes an accent - // "år", // Swedish for "year", includes a ring above - // "naïve", // English word borrowed from French, includes an umlaut - // "الاسم", // Arabic for "name" - // "оплата", // Russian for "payment" - // "पता", // Hindi for "address" - // "用戶名", // Chinese for "username" - // "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla - // "preço", // Portuguese for "price", includes a cedilla - // "사용자명", // Korean for "username" - // "usuario_ñoño", // Spanish, uses an underscore and includes "ñ" - // "файл", // Bulgarian for "file" - // "δεδομένα", // Greek for "data" - // "geändert_am", // German for "modified on", includes an umlaut - // "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore - // "São_Paulo", // Portuguese, includes an underscore and a tilde - // "età", // Italian for "age", includes an accent - // "ชื่อผู้ใช้", // Thai for "username" + "Benutzer-ID", // German for "user ID", includes a hyphen + "numéro", // French for "number", includes an accent + "år", // Swedish for "year", includes a ring above + "naïve", // English word borrowed from French, includes an umlaut + "الاسم", // Arabic for "name" + "оплата", // Russian for "payment" + "पता", // Hindi for "address" + "用戶名", // Chinese for "username" + "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla + "preço", // Portuguese for "price", includes a cedilla + "사용자명", // Korean for "username" + "usuario_ñoño", // Spanish, uses an underscore and includes "ñ" + "файл", // Bulgarian for "file" + "δεδομένα", // Greek for "data" + "geändert_am", // German for "modified on", includes an umlaut + "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore + "São_Paulo", // Portuguese, includes an underscore and a tilde + "età", // Italian for "age", includes an accent + "ชื่อผู้ใช้", // Thai for "username" ])("non-ascii column name: %s", name => { beforeAll(async () => { table = await createTable({ diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index cd0650e4c4..815094eeeb 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -123,7 +123,8 @@ export async function search( } catch (err: any) { if (err.message && err.message.includes("does not exist")) { throw new Error( - `Table updated externally, please re-fetch - ${err.message}` + `Table updated externally, please re-fetch - ${err.message}`, + { cause: err } ) } else { throw err