MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

wip

This commit is contained in:
Sam Rose 2024-07-29 09:57:24 +01:00
parent 20bad903cc
commit 50d1972127
No known key found for this signature in database
6 changed files with 114 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
packages/backend-core/src/sql/sql.ts
View File

@ -109,6 +109,26 @@ function parseFilters(filters: SearchFilters | undefined): SearchFilters {
return filters
}
// OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
// so when we use them we need to wrap them in to_char(). This function
// converts a field name to the appropriate identifier.
function convertClobs(client: SqlClient, table: Table, field: string): string {
const parts = field.split(".")
const col = parts.pop()!
const schema = table.schema[col]
let identifier = quotedIdentifier(client, field)
if (
schema.type === FieldType.STRING ||
schema.type === FieldType.LONGFORM ||
schema.type === FieldType.BB_REFERENCE_SINGLE ||
schema.type === FieldType.OPTIONS ||
schema.type === FieldType.BARCODEQR
) {
identifier = `to_char(${identifier})`
}
return identifier
}
function generateSelectStatement(
json: QueryJson,
knex: Knex
@ -372,7 +392,15 @@ class InternalBuilder {
iterate(
filters.oneOf,
(key: string, array) => {
query = query[fnc](key, Array.isArray(array) ? array : [array])
if (this.client === SqlClient.ORACLE) {
key = convertClobs(this.client, table, key)
query = query.whereRaw(
`${key} IN (?)`,
Array.isArray(array) ? array : [array]
)
} else {
query = query[fnc](key, Array.isArray(array) ? array : [array])
}
},
(key: string[], array) => {
query = query[fnc](key, Array.isArray(array) ? array : [array])
@ -436,8 +464,9 @@ class InternalBuilder {
[value]
)
} else if (this.client === SqlClient.ORACLE) {
const identifier = convertClobs(this.client, table, key)
query = query[fnc](
`COALESCE(${quotedIdentifier(this.client, key)}, -1) = ?`,
`(${identifier} IS NOT NULL AND ${identifier} = ?)`,
[value]
)
} else {
@ -460,8 +489,9 @@ class InternalBuilder {
[value]
)
} else if (this.client === SqlClient.ORACLE) {
const identifier = convertClobs(this.client, table, key)
query = query[fnc](
`COALESCE(${quotedIdentifier(this.client, key)}, -1) != ?`,
`(${identifier} IS NOT NULL AND ${identifier} != ?)`,
[value]
)
} else {
@ -707,8 +737,11 @@ class InternalBuilder {
}
const ret = query.insert(parsedBody).onConflict(primary).merge()
return ret
} else if (this.client === SqlClient.MS_SQL) {
// No upsert or onConflict support in MSSQL yet, see:
} else if (
this.client === SqlClient.MS_SQL ||
this.client === SqlClient.ORACLE
) {
// No upsert or onConflict support in MSSQL/Oracle yet, see:
// https://github.com/knex/knex/pull/6050
return query.insert(parsedBody)
}
@ -867,7 +900,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
const config: Knex.Config = {
client: sqlClient,
}
if (sqlClient === SqlClient.SQL_LITE) {
if (sqlClient === SqlClient.SQL_LITE || sqlClient === SqlClient.ORACLE) {
config.useNullAsDefault = true
}

21
packages/server/__mocks__/oracledb.ts
View File

@ -1,21 +0,0 @@
const executeMock = jest.fn(() => ({
rows: [
{
a: "string",
b: 1,
},
],
}))
const closeMock = jest.fn()
class Connection {
execute = executeMock
close = closeMock
}
module.exports = {
getConnection: jest.fn(() => new Connection()),
executeMock,
closeMock,
}

61
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -40,13 +40,13 @@ import { structures } from "@budibase/backend-core/tests"
import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
describe.each([
//["in-memory", undefined],
//["lucene", undefined],
//["sqs", undefined],
//[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
//[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
//[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
//[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
// ["in-memory", undefined],
// ["lucene", undefined],
// ["sqs", undefined],
// [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
// [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
// [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
// [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("search (%s)", (name, dsProvider) => {
const isSqs = name === "sqs"
@ -292,7 +292,7 @@ describe.each([
})
describe("equal", () => {
it.only("successfully finds true row", async () => {
it("successfully finds true row", async () => {
await expectQuery({ equal: { isTrue: true } }).toMatchExactly([
{ isTrue: true },
])
@ -1577,12 +1577,15 @@ describe.each([
})
})
describe("bigints", () => {
describe.only("bigints", () => {
const SMALL = "1"
const MEDIUM = "10000000"
// Our bigints are int64s in most datasources.
const BIG = "9223372036854775807"
let BIG = "9223372036854775807"
if (name === DatabaseName.ORACLE) {
// BIG = "9223372036854775808"
}
beforeAll(async () => {
table = await createTable({
@ -2415,25 +2418,25 @@ describe.each([
describe.each([
"名前", // Japanese for "name"
"Benutzer-ID", // German for "user ID", includes a hyphen
"numéro", // French for "number", includes an accent
"år", // Swedish for "year", includes a ring above
"naïve", // English word borrowed from French, includes an umlaut
"الاسم", // Arabic for "name"
"оплата", // Russian for "payment"
"पता", // Hindi for "address"
"用戶名", // Chinese for "username"
"çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla
"preço", // Portuguese for "price", includes a cedilla
"사용자명", // Korean for "username"
"usuario_ñoño", // Spanish, uses an underscore and includes "ñ"
"файл", // Bulgarian for "file"
"δεδομένα", // Greek for "data"
"geändert_am", // German for "modified on", includes an umlaut
"ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore
"São_Paulo", // Portuguese, includes an underscore and a tilde
"età", // Italian for "age", includes an accent
"ชื่อผู้ใช้", // Thai for "username"
// "Benutzer-ID", // German for "user ID", includes a hyphen
// "numéro", // French for "number", includes an accent
// "år", // Swedish for "year", includes a ring above
// "naïve", // English word borrowed from French, includes an umlaut
// "الاسم", // Arabic for "name"
// "оплата", // Russian for "payment"
// "पता", // Hindi for "address"
// "用戶名", // Chinese for "username"
// "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla
// "preço", // Portuguese for "price", includes a cedilla
// "사용자명", // Korean for "username"
// "usuario_ñoño", // Spanish, uses an underscore and includes "ñ"
// "файл", // Bulgarian for "file"
// "δεδομένα", // Greek for "data"
// "geändert_am", // German for "modified on", includes an umlaut
// "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore
// "São_Paulo", // Portuguese, includes an underscore and a tilde
// "età", // Italian for "age", includes an accent
// "ชื่อผู้ใช้", // Thai for "username"
])("non-ascii column name: %s", name => {
beforeAll(async () => {
table = await createTable({

13
packages/server/src/integrations/oracle.ts
View File

@ -360,11 +360,20 @@ class OracleIntegration extends Sql implements DatasourcePlus {
this.index = 1
connection = await this.getConnection()
const options: ExecuteOptions = { autoCommit: true }
const options: ExecuteOptions = {
autoCommit: true,
fetchTypeHandler: function (metaData) {
if (metaData.dbType === oracledb.CLOB) {
return { type: oracledb.STRING }
}
return undefined
},
}
const bindings: BindParameters = query.bindings || []
this.log(query.sql, bindings)
return await connection.execute<T>(query.sql, bindings, options)
const result = await connection.execute(query.sql, bindings, options)
return result as Result<T>
} finally {
if (connection) {
try {

39
packages/server/src/integrations/tests/utils/oracle.ts
View File

@ -8,7 +8,7 @@ let ports: Promise<testContainerUtils.Port[]>
export async function getDatasource(): Promise<Datasource> {
if (!ports) {
let image = "oracle/database:19.3.0.0-ee"
let image = "oracle/database:19.3.0.0-ee-slim-faststart"
if (process.arch.startsWith("arm")) {
image = "samhuang78/oracle-database:19.3.0-ee-slim-faststart"
}
@ -17,7 +17,7 @@ export async function getDatasource(): Promise<Datasource> {
new GenericContainer(image)
.withExposedPorts(1521)
.withEnvironment({ ORACLE_PASSWORD: "password" })
.withWaitStrategy(Wait.forHealthCheck().withStartupTimeout(10000))
.withWaitStrategy(Wait.forHealthCheck().withStartupTimeout(60000))
)
}
@ -26,23 +26,25 @@ export async function getDatasource(): Promise<Datasource> {
throw new Error("Oracle port not found")
}
const host = "127.0.0.1"
const user = "SYSTEM"
const password = "password"
const datasource: Datasource = {
type: "datasource_plus",
source: SourceName.ORACLE,
plus: true,
config: {
host: "127.0.0.1",
port,
database: "postgres",
user: "SYS",
password: "password",
},
config: { host, port, user, password, database: "FREEPDB1" },
}
const database = generator.guid().replaceAll("-", "")
const newUser = "a" + generator.guid().replaceAll("-", "")
const client = await knexClient(datasource)
await client.raw(`CREATE DATABASE "${database}"`)
datasource.config!.database = database
await client.raw(`CREATE USER ${newUser} IDENTIFIED BY password`)
await client.raw(
`GRANT CONNECT, RESOURCE, CREATE VIEW, CREATE SESSION TO ${newUser}`
)
await client.raw(`GRANT UNLIMITED TABLESPACE TO ${newUser}`)
datasource.config!.user = newUser
return datasource
}
@ -55,8 +57,17 @@ export async function knexClient(ds: Datasource) {
throw new Error("Datasource source is not Oracle")
}
return knex({
const db = ds.config.database || "FREEPDB1"
const connectString = `${ds.config.host}:${ds.config.port}/${db}`
const c = knex({
client: "oracledb",
connection: ds.config,
connection: {
connectString,
user: ds.config.user,
password: ds.config.password,
},
})
return c
}

7
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -315,6 +315,13 @@ export async function outputProcessing<T extends Row[] | Row>(
column.subtype
)
}
} else if (column.type === FieldType.BIGINT) {
for (const row of enriched) {
if (row[property] == null) {
continue
}
row[property] = row[property].toString()
}
}
}