From 50d1972127facb73c0c26c2e5b36b07b2d7bdb86 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 29 Jul 2024 09:57:24 +0100 Subject: [PATCH] wip --- packages/backend-core/src/sql/sql.ts | 45 ++++++++++++-- packages/server/__mocks__/oracledb.ts | 21 ------- .../src/api/routes/tests/search.spec.ts | 61 ++++++++++--------- packages/server/src/integrations/oracle.ts | 13 +++- .../src/integrations/tests/utils/oracle.ts | 39 +++++++----- .../src/utilities/rowProcessor/index.ts | 7 +++ 6 files changed, 114 insertions(+), 72 deletions(-) delete mode 100644 packages/server/__mocks__/oracledb.ts diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index a67da7bc10..621afe7f3e 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -109,6 +109,26 @@ function parseFilters(filters: SearchFilters | undefined): SearchFilters { return filters } +// OracleDB can't use character-large-objects (CLOBs) in WHERE clauses, +// so when we use them we need to wrap them in to_char(). This function +// converts a field name to the appropriate identifier. +function convertClobs(client: SqlClient, table: Table, field: string): string { + const parts = field.split(".") + const col = parts.pop()! + const schema = table.schema[col] + let identifier = quotedIdentifier(client, field) + if ( + schema.type === FieldType.STRING || + schema.type === FieldType.LONGFORM || + schema.type === FieldType.BB_REFERENCE_SINGLE || + schema.type === FieldType.OPTIONS || + schema.type === FieldType.BARCODEQR + ) { + identifier = `to_char(${identifier})` + } + return identifier +} + function generateSelectStatement( json: QueryJson, knex: Knex @@ -372,7 +392,15 @@ class InternalBuilder { iterate( filters.oneOf, (key: string, array) => { - query = query[fnc](key, Array.isArray(array) ? array : [array]) + if (this.client === SqlClient.ORACLE) { + key = convertClobs(this.client, table, key) + query = query.whereRaw( + `${key} IN (?)`, + Array.isArray(array) ? array : [array] + ) + } else { + query = query[fnc](key, Array.isArray(array) ? array : [array]) + } }, (key: string[], array) => { query = query[fnc](key, Array.isArray(array) ? array : [array]) @@ -436,8 +464,9 @@ class InternalBuilder { [value] ) } else if (this.client === SqlClient.ORACLE) { + const identifier = convertClobs(this.client, table, key) query = query[fnc]( - `COALESCE(${quotedIdentifier(this.client, key)}, -1) = ?`, + `(${identifier} IS NOT NULL AND ${identifier} = ?)`, [value] ) } else { @@ -460,8 +489,9 @@ class InternalBuilder { [value] ) } else if (this.client === SqlClient.ORACLE) { + const identifier = convertClobs(this.client, table, key) query = query[fnc]( - `COALESCE(${quotedIdentifier(this.client, key)}, -1) != ?`, + `(${identifier} IS NOT NULL AND ${identifier} != ?)`, [value] ) } else { @@ -707,8 +737,11 @@ class InternalBuilder { } const ret = query.insert(parsedBody).onConflict(primary).merge() return ret - } else if (this.client === SqlClient.MS_SQL) { - // No upsert or onConflict support in MSSQL yet, see: + } else if ( + this.client === SqlClient.MS_SQL || + this.client === SqlClient.ORACLE + ) { + // No upsert or onConflict support in MSSQL/Oracle yet, see: // https://github.com/knex/knex/pull/6050 return query.insert(parsedBody) } @@ -867,7 +900,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { const config: Knex.Config = { client: sqlClient, } - if (sqlClient === SqlClient.SQL_LITE) { + if (sqlClient === SqlClient.SQL_LITE || sqlClient === SqlClient.ORACLE) { config.useNullAsDefault = true } diff --git a/packages/server/__mocks__/oracledb.ts b/packages/server/__mocks__/oracledb.ts deleted file mode 100644 index 0172ace0e6..0000000000 --- a/packages/server/__mocks__/oracledb.ts +++ /dev/null @@ -1,21 +0,0 @@ -const executeMock = jest.fn(() => ({ - rows: [ - { - a: "string", - b: 1, - }, - ], -})) - -const closeMock = jest.fn() - -class Connection { - execute = executeMock - close = closeMock -} - -module.exports = { - getConnection: jest.fn(() => new Connection()), - executeMock, - closeMock, -} diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 6e685b13ab..57afdb4853 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -40,13 +40,13 @@ import { structures } from "@budibase/backend-core/tests" import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default" describe.each([ - //["in-memory", undefined], - //["lucene", undefined], - //["sqs", undefined], - //[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - //[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - //[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - //[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // ["in-memory", undefined], + // ["lucene", undefined], + // ["sqs", undefined], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" @@ -292,7 +292,7 @@ describe.each([ }) describe("equal", () => { - it.only("successfully finds true row", async () => { + it("successfully finds true row", async () => { await expectQuery({ equal: { isTrue: true } }).toMatchExactly([ { isTrue: true }, ]) @@ -1577,12 +1577,15 @@ describe.each([ }) }) - describe("bigints", () => { + describe.only("bigints", () => { const SMALL = "1" const MEDIUM = "10000000" // Our bigints are int64s in most datasources. - const BIG = "9223372036854775807" + let BIG = "9223372036854775807" + if (name === DatabaseName.ORACLE) { + // BIG = "9223372036854775808" + } beforeAll(async () => { table = await createTable({ @@ -2415,25 +2418,25 @@ describe.each([ describe.each([ "名前", // Japanese for "name" - "Benutzer-ID", // German for "user ID", includes a hyphen - "numéro", // French for "number", includes an accent - "år", // Swedish for "year", includes a ring above - "naïve", // English word borrowed from French, includes an umlaut - "الاسم", // Arabic for "name" - "оплата", // Russian for "payment" - "पता", // Hindi for "address" - "用戶名", // Chinese for "username" - "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla - "preço", // Portuguese for "price", includes a cedilla - "사용자명", // Korean for "username" - "usuario_ñoño", // Spanish, uses an underscore and includes "ñ" - "файл", // Bulgarian for "file" - "δεδομένα", // Greek for "data" - "geändert_am", // German for "modified on", includes an umlaut - "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore - "São_Paulo", // Portuguese, includes an underscore and a tilde - "età", // Italian for "age", includes an accent - "ชื่อผู้ใช้", // Thai for "username" + // "Benutzer-ID", // German for "user ID", includes a hyphen + // "numéro", // French for "number", includes an accent + // "år", // Swedish for "year", includes a ring above + // "naïve", // English word borrowed from French, includes an umlaut + // "الاسم", // Arabic for "name" + // "оплата", // Russian for "payment" + // "पता", // Hindi for "address" + // "用戶名", // Chinese for "username" + // "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla + // "preço", // Portuguese for "price", includes a cedilla + // "사용자명", // Korean for "username" + // "usuario_ñoño", // Spanish, uses an underscore and includes "ñ" + // "файл", // Bulgarian for "file" + // "δεδομένα", // Greek for "data" + // "geändert_am", // German for "modified on", includes an umlaut + // "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore + // "São_Paulo", // Portuguese, includes an underscore and a tilde + // "età", // Italian for "age", includes an accent + // "ชื่อผู้ใช้", // Thai for "username" ])("non-ascii column name: %s", name => { beforeAll(async () => { table = await createTable({ diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 9f40372546..b36501525b 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -360,11 +360,20 @@ class OracleIntegration extends Sql implements DatasourcePlus { this.index = 1 connection = await this.getConnection() - const options: ExecuteOptions = { autoCommit: true } + const options: ExecuteOptions = { + autoCommit: true, + fetchTypeHandler: function (metaData) { + if (metaData.dbType === oracledb.CLOB) { + return { type: oracledb.STRING } + } + return undefined + }, + } const bindings: BindParameters = query.bindings || [] this.log(query.sql, bindings) - return await connection.execute(query.sql, bindings, options) + const result = await connection.execute(query.sql, bindings, options) + return result as Result } finally { if (connection) { try { diff --git a/packages/server/src/integrations/tests/utils/oracle.ts b/packages/server/src/integrations/tests/utils/oracle.ts index a4f294d7ba..9b75a52da7 100644 --- a/packages/server/src/integrations/tests/utils/oracle.ts +++ b/packages/server/src/integrations/tests/utils/oracle.ts @@ -8,7 +8,7 @@ let ports: Promise export async function getDatasource(): Promise { if (!ports) { - let image = "oracle/database:19.3.0.0-ee" + let image = "oracle/database:19.3.0.0-ee-slim-faststart" if (process.arch.startsWith("arm")) { image = "samhuang78/oracle-database:19.3.0-ee-slim-faststart" } @@ -17,7 +17,7 @@ export async function getDatasource(): Promise { new GenericContainer(image) .withExposedPorts(1521) .withEnvironment({ ORACLE_PASSWORD: "password" }) - .withWaitStrategy(Wait.forHealthCheck().withStartupTimeout(10000)) + .withWaitStrategy(Wait.forHealthCheck().withStartupTimeout(60000)) ) } @@ -26,23 +26,25 @@ export async function getDatasource(): Promise { throw new Error("Oracle port not found") } + const host = "127.0.0.1" + const user = "SYSTEM" + const password = "password" + const datasource: Datasource = { type: "datasource_plus", source: SourceName.ORACLE, plus: true, - config: { - host: "127.0.0.1", - port, - database: "postgres", - user: "SYS", - password: "password", - }, + config: { host, port, user, password, database: "FREEPDB1" }, } - const database = generator.guid().replaceAll("-", "") + const newUser = "a" + generator.guid().replaceAll("-", "") const client = await knexClient(datasource) - await client.raw(`CREATE DATABASE "${database}"`) - datasource.config!.database = database + await client.raw(`CREATE USER ${newUser} IDENTIFIED BY password`) + await client.raw( + `GRANT CONNECT, RESOURCE, CREATE VIEW, CREATE SESSION TO ${newUser}` + ) + await client.raw(`GRANT UNLIMITED TABLESPACE TO ${newUser}`) + datasource.config!.user = newUser return datasource } @@ -55,8 +57,17 @@ export async function knexClient(ds: Datasource) { throw new Error("Datasource source is not Oracle") } - return knex({ + const db = ds.config.database || "FREEPDB1" + const connectString = `${ds.config.host}:${ds.config.port}/${db}` + + const c = knex({ client: "oracledb", - connection: ds.config, + connection: { + connectString, + user: ds.config.user, + password: ds.config.password, + }, }) + + return c } diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 71de056814..a367c6da1e 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -315,6 +315,13 @@ export async function outputProcessing( column.subtype ) } + } else if (column.type === FieldType.BIGINT) { + for (const row of enriched) { + if (row[property] == null) { + continue + } + row[property] = row[property].toString() + } } }