MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into BUDI-8508/sql-support-for-logical-operators

This commit is contained in:
Adria Navarro 2024-08-05 17:28:58 +02:00 committed by GitHub
parent c845db966f f846507877
commit 9743aca715
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 1017 additions and 733 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/budibase_ci.yml vendored
View File

@ -175,6 +175,7 @@ jobs:
docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }} &
docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }} &
docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }} &
docker pull budibase/oracle-database:23.2-slim-faststart &
docker pull minio/minio &
docker pull redis &
docker pull testcontainers/ryuk:0.5.1 &

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.29.27",
"version": "2.29.29",
"npmClient": "yarn",
"packages": [
"packages/*",

757
packages/backend-core/src/sql/sql.ts
View File

File diff suppressed because it is too large Load Diff

87
packages/backend-core/src/sql/sqlStatements.ts
View File

@ -1,87 +0,0 @@
import { FieldType, Table, FieldSchema, SqlClient } from "@budibase/types"
import { Knex } from "knex"
export class SqlStatements {
client: string
table: Table
allOr: boolean | undefined
columnPrefix: string | undefined
constructor(
client: string,
table: Table,
{ allOr, columnPrefix }: { allOr?: boolean; columnPrefix?: string } = {}
) {
this.client = client
this.table = table
this.allOr = allOr
this.columnPrefix = columnPrefix
}
getField(key: string): FieldSchema | undefined {
const fieldName = key.split(".")[1]
let found = this.table.schema[fieldName]
if (!found && this.columnPrefix) {
const prefixRemovedFieldName = fieldName.replace(this.columnPrefix, "")
found = this.table.schema[prefixRemovedFieldName]
}
return found
}
between(
query: Knex.QueryBuilder,
key: string,
low: number | string,
high: number | string
) {
// Use a between operator if we have 2 valid range values
const field = this.getField(key)
if (
field?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(
`CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
[low, high]
)
} else {
const fnc = this.allOr ? "orWhereBetween" : "whereBetween"
query = query[fnc](key, [low, high])
}
return query
}
lte(query: Knex.QueryBuilder, key: string, low: number | string) {
// Use just a single greater than operator if we only have a low
const field = this.getField(key)
if (
field?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [
low,
])
} else {
const fnc = this.allOr ? "orWhere" : "where"
query = query[fnc](key, ">=", low)
}
return query
}
gte(query: Knex.QueryBuilder, key: string, high: number | string) {
const field = this.getField(key)
// Use just a single less than operator if we only have a high
if (
field?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [
high,
])
} else {
const fnc = this.allOr ? "orWhere" : "where"
query = query[fnc](key, "<=", high)
}
return query
}
}

1
packages/backend-core/src/sql/utils.ts
View File

@ -22,6 +22,7 @@ export function getNativeSql(
query: Knex.SchemaBuilder | Knex.QueryBuilder
): SqlQuery | SqlQuery[] {
let sql = query.toSQL()
if (Array.isArray(sql)) {
return sql as SqlQuery[]
}

21
packages/server/__mocks__/oracledb.ts
View File

@ -1,21 +0,0 @@
const executeMock = jest.fn(() => ({
rows: [
{
a: "string",
b: 1,
},
],
}))
const closeMock = jest.fn()
class Connection {
execute = executeMock
close = closeMock
}
module.exports = {
getConnection: jest.fn(() => new Connection()),
executeMock,
closeMock,
}

6
packages/server/scripts/integrations/oracle/docker-compose.yml
View File

@ -6,9 +6,9 @@ services:
db:
restart: unless-stopped
platform: linux/x86_64
image: container-registry.oracle.com/database/express:18.4.0-xe
image: gvenzl/oracle-free:23.2-slim-faststart
environment:
ORACLE_PWD: oracle
ORACLE_PWD: Password1
ports:
- 1521:1521
- 5500:5500
@ -16,4 +16,4 @@ services:
- oracle_data:/opt/oracle/oradata
volumes:
oracle_data:
oracle_data:

300
packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
View File

@ -22,9 +22,13 @@ describe.each(
DatabaseName.MYSQL,
DatabaseName.SQL_SERVER,
DatabaseName.MARIADB,
DatabaseName.ORACLE,
].map(name => [name, getDatasource(name)])
)("queries (%s)", (dbName, dsProvider) => {
const config = setup.getConfig()
const isOracle = dbName === DatabaseName.ORACLE
const isMsSQL = dbName === DatabaseName.SQL_SERVER
let rawDatasource: Datasource
let datasource: Datasource
let client: Knex
@ -97,7 +101,7 @@ describe.each(
const query = await createQuery({
name: "New Query",
fields: {
sql: "SELECT * FROM test_table",
sql: client("test_table").select("*").toString(),
},
})
@ -106,7 +110,7 @@ describe.each(
name: "New Query",
parameters: [],
fields: {
sql: "SELECT * FROM test_table",
sql: client("test_table").select("*").toString(),
},
schema: {},
queryVerb: "read",
@ -125,7 +129,7 @@ describe.each(
it("should be able to update a query", async () => {
const query = await createQuery({
fields: {
sql: "SELECT * FROM test_table",
sql: client("test_table").select("*").toString(),
},
})
@ -135,7 +139,7 @@ describe.each(
...query,
name: "Updated Query",
fields: {
sql: "SELECT * FROM test_table WHERE id = 1",
sql: client("test_table").where({ id: 1 }).toString(),
},
})
@ -144,7 +148,7 @@ describe.each(
name: "Updated Query",
parameters: [],
fields: {
sql: "SELECT * FROM test_table WHERE id = 1",
sql: client("test_table").where({ id: 1 }).toString(),
},
schema: {},
queryVerb: "read",
@ -161,7 +165,7 @@ describe.each(
it("should be able to delete a query", async () => {
const query = await createQuery({
fields: {
sql: "SELECT * FROM test_table",
sql: client("test_table").select("*").toString(),
},
})
@ -180,7 +184,7 @@ describe.each(
it("should be able to list queries", async () => {
const query = await createQuery({
fields: {
sql: "SELECT * FROM test_table",
sql: client("test_table").select("*").toString(),
},
})
@ -191,7 +195,7 @@ describe.each(
it("should strip sensitive fields for prod apps", async () => {
const query = await createQuery({
fields: {
sql: "SELECT * FROM test_table",
sql: client("test_table").select("*").toString(),
},
})
@ -212,7 +216,7 @@ describe.each(
datasourceId: datasource._id!,
queryVerb: "read",
fields: {
sql: `SELECT * FROM test_table WHERE id = 1`,
sql: client("test_table").where({ id: 1 }).toString(),
},
parameters: [],
transformer: "return data",
@ -270,7 +274,7 @@ describe.each(
name: "Test Query",
queryVerb: "read",
fields: {
sql: `SELECT * FROM ${tableName}`,
sql: client(tableName).select("*").toString(),
},
parameters: [],
transformer: "return data",
@ -284,11 +288,13 @@ describe.each(
})
)
await client(tableName).delete()
await client.schema.alterTable(tableName, table => {
table.string("data").alter()
})
await client(tableName).update({
await client(tableName).insert({
name: "test",
data: "string value",
})
@ -297,7 +303,7 @@ describe.each(
name: "Test Query",
queryVerb: "read",
fields: {
sql: `SELECT * FROM ${tableName}`,
sql: client(tableName).select("*").toString(),
},
parameters: [],
transformer: "return data",
@ -311,6 +317,7 @@ describe.each(
})
)
})
it("should work with static variables", async () => {
await config.api.datasource.update({
...datasource,
@ -326,7 +333,7 @@ describe.each(
datasourceId: datasource._id!,
queryVerb: "read",
fields: {
sql: `SELECT '{{ foo }}' as foo`,
sql: `SELECT '{{ foo }}' AS foo ${isOracle ? "FROM dual" : ""}`,
},
parameters: [],
transformer: "return data",
@ -337,16 +344,17 @@ describe.each(
const response = await config.api.query.preview(request)
let key = isOracle ? "FOO" : "foo"
expect(response.schema).toEqual({
foo: {
name: "foo",
[key]: {
name: key,
type: "string",
},
})
expect(response.rows).toEqual([
{
foo: "bar",
[key]: "bar",
},
])
})
@ -354,7 +362,7 @@ describe.each(
it("should work with dynamic variables", async () => {
const basedOnQuery = await createQuery({
fields: {
sql: "SELECT name FROM test_table WHERE id = 1",
sql: client("test_table").select("name").where({ id: 1 }).toString(),
},
})
@ -376,7 +384,7 @@ describe.each(
datasourceId: datasource._id!,
queryVerb: "read",
fields: {
sql: `SELECT '{{ foo }}' as foo`,
sql: `SELECT '{{ foo }}' AS foo ${isOracle ? "FROM dual" : ""}`,
},
parameters: [],
transformer: "return data",
@ -385,16 +393,17 @@ describe.each(
readable: true,
})
let key = isOracle ? "FOO" : "foo"
expect(preview.schema).toEqual({
foo: {
name: "foo",
[key]: {
name: key,
type: "string",
},
})
expect(preview.rows).toEqual([
{
foo: "one",
[key]: "one",
},
])
})
@ -402,7 +411,7 @@ describe.each(
it("should handle the dynamic base query being deleted", async () => {
const basedOnQuery = await createQuery({
fields: {
sql: "SELECT name FROM test_table WHERE id = 1",
sql: client("test_table").select("name").where({ id: 1 }).toString(),
},
})
@ -426,7 +435,7 @@ describe.each(
datasourceId: datasource._id!,
queryVerb: "read",
fields: {
sql: `SELECT '{{ foo }}' as foo`,
sql: `SELECT '{{ foo }}' AS foo ${isOracle ? "FROM dual" : ""}`,
},
parameters: [],
transformer: "return data",
@ -435,16 +444,17 @@ describe.each(
readable: true,
})
let key = isOracle ? "FOO" : "foo"
expect(preview.schema).toEqual({
foo: {
name: "foo",
[key]: {
name: key,
type: "string",
},
})
expect(preview.rows).toEqual([
{
foo: datasource.source === SourceName.SQL_SERVER ? "" : null,
[key]: datasource.source === SourceName.SQL_SERVER ? "" : null,
},
])
})
@ -455,7 +465,7 @@ describe.each(
it("should be able to insert with bindings", async () => {
const query = await createQuery({
fields: {
sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
sql: client("test_table").insert({ name: "{{ foo }}" }).toString(),
},
parameters: [
{
@ -488,7 +498,7 @@ describe.each(
it("should not allow handlebars as parameters", async () => {
const query = await createQuery({
fields: {
sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
sql: client("test_table").insert({ name: "{{ foo }}" }).toString(),
},
parameters: [
{
@ -516,46 +526,55 @@ describe.each(
)
})
it.each(["2021-02-05T12:01:00.000Z", "2021-02-05"])(
"should coerce %s into a date",
async datetimeStr => {
const date = new Date(datetimeStr)
const query = await createQuery({
fields: {
sql: `INSERT INTO test_table (name, birthday) VALUES ('foo', {{ birthday }})`,
},
parameters: [
{
name: "birthday",
default: "",
// Oracle doesn't automatically coerce strings into dates.
!isOracle &&
it.each(["2021-02-05T12:01:00.000Z", "2021-02-05"])(
"should coerce %s into a date",
async datetimeStr => {
const date = new Date(datetimeStr)
const query = await createQuery({
fields: {
sql: client("test_table")
.insert({
name: "foo",
birthday: client.raw("{{ birthday }}"),
})
.toString(),
},
],
queryVerb: "create",
})
parameters: [
{
name: "birthday",
default: "",
},
],
queryVerb: "create",
})
const result = await config.api.query.execute(query._id!, {
parameters: { birthday: datetimeStr },
})
const result = await config.api.query.execute(query._id!, {
parameters: { birthday: datetimeStr },
})
expect(result.data).toEqual([{ created: true }])
expect(result.data).toEqual([{ created: true }])
const rows = await client("test_table")
.where({ birthday: datetimeStr })
.select()
expect(rows).toHaveLength(1)
const rows = await client("test_table")
.where({ birthday: datetimeStr })
.select()
expect(rows).toHaveLength(1)
for (const row of rows) {
expect(new Date(row.birthday)).toEqual(date)
for (const row of rows) {
expect(new Date(row.birthday)).toEqual(date)
}
}
}
)
)
it.each(["2021,02,05", "202205-1500"])(
"should not coerce %s as a date",
async notDateStr => {
const query = await createQuery({
fields: {
sql: "INSERT INTO test_table (name) VALUES ({{ name }})",
sql: client("test_table")
.insert({ name: client.raw("{{ name }}") })
.toString(),
},
parameters: [
{
@ -586,7 +605,7 @@ describe.each(
it("should execute a query", async () => {
const query = await createQuery({
fields: {
sql: "SELECT * FROM test_table ORDER BY id",
sql: client("test_table").select("*").orderBy("id").toString(),
},
})
@ -629,7 +648,7 @@ describe.each(
it("should be able to transform a query", async () => {
const query = await createQuery({
fields: {
sql: "SELECT * FROM test_table WHERE id = 1",
sql: client("test_table").where({ id: 1 }).select("*").toString(),
},
transformer: `
data[0].id = data[0].id + 1;
@ -652,7 +671,10 @@ describe.each(
it("should coerce numeric bindings", async () => {
const query = await createQuery({
fields: {
sql: "SELECT * FROM test_table WHERE id = {{ id }}",
sql: client("test_table")
.where({ id: client.raw("{{ id }}") })
.select("*")
.toString(),
},
parameters: [
{
@ -683,7 +705,10 @@ describe.each(
it("should be able to update rows", async () => {
const query = await createQuery({
fields: {
sql: "UPDATE test_table SET name = {{ name }} WHERE id = {{ id }}",
sql: client("test_table")
.update({ name: client.raw("{{ name }}") })
.where({ id: client.raw("{{ id }}") })
.toString(),
},
parameters: [
{
@ -698,19 +723,13 @@ describe.each(
queryVerb: "update",
})
const result = await config.api.query.execute(query._id!, {
await config.api.query.execute(query._id!, {
parameters: {
id: "1",
name: "foo",
},
})
expect(result.data).toEqual([
{
updated: true,
},
])
const rows = await client("test_table").where({ id: 1 }).select()
expect(rows).toEqual([
{ id: 1, name: "foo", birthday: null, number: null },
@ -720,35 +739,34 @@ describe.each(
it("should be able to execute an update that updates no rows", async () => {
const query = await createQuery({
fields: {
sql: "UPDATE test_table SET name = 'updated' WHERE id = 100",
sql: client("test_table")
.update({ name: "updated" })
.where({ id: 100 })
.toString(),
},
queryVerb: "update",
})
const result = await config.api.query.execute(query._id!)
await config.api.query.execute(query._id!)
expect(result.data).toEqual([
{
updated: true,
},
])
const rows = await client("test_table").select()
for (const row of rows) {
expect(row.name).not.toEqual("updated")
}
})
it("should be able to execute a delete that deletes no rows", async () => {
const query = await createQuery({
fields: {
sql: "DELETE FROM test_table WHERE id = 100",
sql: client("test_table").where({ id: 100 }).delete().toString(),
},
queryVerb: "delete",
})
const result = await config.api.query.execute(query._id!)
await config.api.query.execute(query._id!)
expect(result.data).toEqual([
{
deleted: true,
},
])
const rows = await client("test_table").select()
expect(rows).toHaveLength(5)
})
})
@ -756,7 +774,10 @@ describe.each(
it("should be able to delete rows", async () => {
const query = await createQuery({
fields: {
sql: "DELETE FROM test_table WHERE id = {{ id }}",
sql: client("test_table")
.where({ id: client.raw("{{ id }}") })
.delete()
.toString(),
},
parameters: [
{
@ -767,18 +788,12 @@ describe.each(
queryVerb: "delete",
})
const result = await config.api.query.execute(query._id!, {
await config.api.query.execute(query._id!, {
parameters: {
id: "1",
},
})
expect(result.data).toEqual([
{
deleted: true,
},
])
const rows = await client("test_table").where({ id: 1 }).select()
expect(rows).toHaveLength(0)
})
@ -823,72 +838,63 @@ describe.each(
})
})
it("should be able to execute an update that updates no rows", async () => {
const query = await createQuery({
fields: {
sql: "UPDATE test_table SET name = 'updated' WHERE id = 100",
},
queryVerb: "update",
// this parameter really only impacts SQL queries
describe("confirm nullDefaultSupport", () => {
let queryParams: Partial<Query>
beforeAll(async () => {
queryParams = {
fields: {
sql: client("test_table")
.insert({
name: client.raw("{{ bindingName }}"),
number: client.raw("{{ bindingNumber }}"),
})
.toString(),
},
parameters: [
{
name: "bindingName",
default: "",
},
{
name: "bindingNumber",
default: "",
},
],
queryVerb: "create",
}
})
const result = await config.api.query.execute(query._id!, {})
it("should error for old queries", async () => {
const query = await createQuery(queryParams)
await config.api.query.save({ ...query, nullDefaultSupport: false })
let error: string | undefined
try {
await config.api.query.execute(query._id!, {
parameters: {
bindingName: "testing",
},
})
} catch (err: any) {
error = err.message
}
if (isMsSQL || isOracle) {
expect(error).toBeUndefined()
} else {
expect(error).toBeDefined()
expect(error).toContain("integer")
}
})
expect(result.data).toEqual([
{
updated: true,
},
])
})
})
// this parameter really only impacts SQL queries
describe("confirm nullDefaultSupport", () => {
const queryParams = {
fields: {
sql: "INSERT INTO test_table (name, number) VALUES ({{ bindingName }}, {{ bindingNumber }})",
},
parameters: [
{
name: "bindingName",
default: "",
},
{
name: "bindingNumber",
default: "",
},
],
queryVerb: "create",
}
it("should error for old queries", async () => {
const query = await createQuery(queryParams)
await config.api.query.save({ ...query, nullDefaultSupport: false })
let error: string | undefined
try {
await config.api.query.execute(query._id!, {
it("should not error for new queries", async () => {
const query = await createQuery(queryParams)
const results = await config.api.query.execute(query._id!, {
parameters: {
bindingName: "testing",
},
})
} catch (err: any) {
error = err.message
}
if (dbName === "mssql") {
expect(error).toBeUndefined()
} else {
expect(error).toBeDefined()
expect(error).toContain("integer")
}
})
it("should not error for new queries", async () => {
const query = await createQuery(queryParams)
const results = await config.api.query.execute(query._id!, {
parameters: {
bindingName: "testing",
},
expect(results).toEqual({ data: [{ created: true }] })
})
expect(results).toEqual({ data: [{ created: true }] })
})
})
})

14
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -72,9 +72,11 @@ describe.each([
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("/rows (%s)", (providerType, dsProvider) => {
const isInternal = dsProvider === undefined
const isMSSQL = providerType === DatabaseName.SQL_SERVER
const isOracle = providerType === DatabaseName.ORACLE
const config = setup.getConfig()
let table: Table
@ -129,7 +131,8 @@ describe.each([
primary: ["id"],
schema: defaultSchema,
}
return merge(req, ...overrides)
const merged = merge(req, ...overrides)
return merged
}
function defaultTable(
@ -1406,9 +1409,10 @@ describe.each([
await assertRowUsage(rowUsage + 3)
})
// Upserting isn't yet supported in MSSQL, see:
// Upserting isn't yet supported in MSSQL / Oracle, see:
// https://github.com/knex/knex/pull/6050
!isMSSQL &&
!isOracle &&
it("should be able to update existing rows with bulkImport", async () => {
const table = await config.api.table.save(
saveTableRequest({
@ -1478,9 +1482,10 @@ describe.each([
expect(rows[2].description).toEqual("Row 3 description")
})
// Upserting isn't yet supported in MSSQL, see:
// Upserting isn't yet supported in MSSQL or Oracle, see:
// https://github.com/knex/knex/pull/6050
!isMSSQL &&
!isOracle &&
!isInternal &&
it("should be able to update existing rows with composite primary keys with bulkImport", async () => {
const tableName = uuid.v4()
@ -1547,9 +1552,10 @@ describe.each([
expect(rows[2].description).toEqual("Row 3 description")
})
// Upserting isn't yet supported in MSSQL, see:
// Upserting isn't yet supported in MSSQL/Oracle, see:
// https://github.com/knex/knex/pull/6050
!isMSSQL &&
!isOracle &&
!isInternal &&
it("should be able to update existing rows an autoID primary key", async () => {
const tableName = uuid.v4()

6
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -48,6 +48,7 @@ describe.each([
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("search (%s)", (name, dsProvider) => {
const isSqs = name === "sqs"
const isLucene = name === "lucene"
@ -1594,7 +1595,10 @@ describe.each([
const MEDIUM = "10000000"
// Our bigints are int64s in most datasources.
const BIG = "9223372036854775807"
let BIG = "9223372036854775807"
if (name === DatabaseName.ORACLE) {
// BIG = "9223372036854775808"
}
beforeAll(async () => {
table = await createTable({

16
packages/server/src/api/routes/tests/table.spec.ts
View File

@ -40,7 +40,8 @@ describe.each([
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("/tables (%s)", (_, dsProvider) => {
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("/tables (%s)", (name, dsProvider) => {
const isInternal: boolean = !dsProvider
let datasource: Datasource | undefined
let config = setup.getConfig()
@ -59,15 +60,20 @@ describe.each([
jest.clearAllMocks()
})
it.each([
let names = [
"alphanum",
"with spaces",
"with-dashes",
"with_underscores",
'with "double quotes"',
"with 'single quotes'",
"with `backticks`",
])("creates a table with name: %s", async name => {
]
if (name !== DatabaseName.ORACLE) {
names.push(`with "double quotes"`)
names.push(`with 'single quotes'`)
}
it.each(names)("creates a table with name: %s", async name => {
const table = await config.api.table.save(
tableForDatasource(datasource, { name })
)

1
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -33,6 +33,7 @@ describe.each([
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("/v2/views (%s)", (name, dsProvider) => {
const config = setup.getConfig()
const isSqs = name === "sqs"

28
packages/server/src/automations/tests/executeQuery.spec.ts
View File

@ -1,27 +1,20 @@
import { Datasource, Query, SourceName } from "@budibase/types"
import { Datasource, Query } from "@budibase/types"
import * as setup from "./utilities"
import { DatabaseName, getDatasource } from "../../integrations/tests/utils"
import knex, { Knex } from "knex"
import {
DatabaseName,
getDatasource,
knexClient,
} from "../../integrations/tests/utils"
import { Knex } from "knex"
import { generator } from "@budibase/backend-core/tests"
function getKnexClientName(source: SourceName) {
switch (source) {
case SourceName.MYSQL:
return "mysql2"
case SourceName.SQL_SERVER:
return "mssql"
case SourceName.POSTGRES:
return "pg"
}
throw new Error(`Unsupported source: ${source}`)
}
describe.each(
[
DatabaseName.POSTGRES,
DatabaseName.MYSQL,
DatabaseName.SQL_SERVER,
DatabaseName.MARIADB,
DatabaseName.ORACLE,
].map(name => [name, getDatasource(name)])
)("execute query action (%s)", (_, dsProvider) => {
let tableName: string
@ -35,10 +28,7 @@ describe.each(
const ds = await dsProvider
datasource = await config.api.datasource.create(ds)
client = knex({
client: getKnexClientName(ds.source),
connection: ds.config,
})
client = await knexClient(ds)
})
beforeEach(async () => {

31
packages/server/src/integrations/base/types.ts
View File

@ -104,6 +104,37 @@ export interface OracleColumnsResponse {
SEARCH_CONDITION: null | string
}
export enum TriggeringEvent {
INSERT = "INSERT",
DELETE = "DELETE",
UPDATE = "UPDATE",
LOGON = "LOGON",
LOGOFF = "LOGOFF",
STARTUP = "STARTUP",
SHUTDOWN = "SHUTDOWN",
SERVERERROR = "SERVERERROR",
SCHEMA = "SCHEMA",
ALTER = "ALTER",
DROP = "DROP",
}
export enum TriggerType {
BEFORE_EACH_ROW = "BEFORE EACH ROW",
AFTER_EACH_ROW = "AFTER EACH ROW",
BEFORE_STATEMENT = "BEFORE STATEMENT",
AFTER_STATEMENT = "AFTER STATEMENT",
INSTEAD_OF = "INSTEAD OF",
COMPOUND = "COMPOUND",
}
export interface OracleTriggersResponse {
TABLE_NAME: string
TRIGGER_NAME: string
TRIGGER_TYPE: TriggerType
TRIGGERING_EVENT: TriggeringEvent
TRIGGER_BODY: string
}
/**
* An oracle constraint
*/

135
packages/server/src/integrations/oracle.ts
View File

@ -31,7 +31,14 @@ import oracledb, {
ExecuteOptions,
Result,
} from "oracledb"
import { OracleTable, OracleColumn, OracleColumnsResponse } from "./base/types"
import {
OracleTable,
OracleColumn,
OracleColumnsResponse,
OracleTriggersResponse,
TriggeringEvent,
TriggerType,
} from "./base/types"
import { sql } from "@budibase/backend-core"
const Sql = sql.Sql
@ -98,7 +105,7 @@ const SCHEMA: Integration = {
},
}
const UNSUPPORTED_TYPES = ["BLOB", "CLOB", "NCLOB"]
const UNSUPPORTED_TYPES = ["BLOB", "NCLOB"]
const OracleContraintTypes = {
PRIMARY: "P",
@ -111,7 +118,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
private readonly config: OracleConfig
private index: number = 1
private readonly COLUMNS_SQL = `
private static readonly COLUMNS_SQL = `
SELECT
tabs.table_name,
cols.column_name,
@ -139,6 +146,19 @@ class OracleIntegration extends Sql implements DatasourcePlus {
(cons.status = 'ENABLED'
OR cons.status IS NULL)
`
private static readonly TRIGGERS_SQL = `
SELECT
table_name,
trigger_name,
trigger_type,
triggering_event,
trigger_body
FROM
all_triggers
WHERE status = 'ENABLED'
`
constructor(config: OracleConfig) {
super(SqlClient.ORACLE)
this.config = config
@ -211,6 +231,75 @@ class OracleIntegration extends Sql implements DatasourcePlus {
return oracleTables
}
private getTriggersFor(
tableName: string,
triggersResponse: Result<OracleTriggersResponse>,
opts?: { event?: TriggeringEvent; type?: TriggerType }
): OracleTriggersResponse[] {
const triggers: OracleTriggersResponse[] = []
for (const trigger of triggersResponse.rows || []) {
if (trigger.TABLE_NAME !== tableName) {
continue
}
if (opts?.event && opts.event !== trigger.TRIGGERING_EVENT) {
continue
}
if (opts?.type && opts.type !== trigger.TRIGGER_TYPE) {
continue
}
triggers.push(trigger)
}
return triggers
}
private markAutoIncrementColumns(
triggersResponse: Result<OracleTriggersResponse>,
tables: Record<string, Table>
) {
for (const table of Object.values(tables)) {
const triggers = this.getTriggersFor(table.name, triggersResponse, {
type: TriggerType.BEFORE_EACH_ROW,
event: TriggeringEvent.INSERT,
})
// This is the trigger body Knex generates for an auto increment column
// called "id" on a table called "foo":
//
// declare checking number := 1;
// begin if (:new. "id" is null) then while checking >= 1 loop
// select
// "foo_seq".nextval into :new. "id"
// from
// dual;
// select
// count("id") into checking
// from
// "foo"
// where
// "id" = :new. "id";
// end loop;
// end if;
// end;
for (const [columnName, schema] of Object.entries(table.schema)) {
const autoIncrementTriggers = triggers.filter(
trigger =>
// This is a bit heuristic, but I think it's the best we can do with
// the information we have. We're looking for triggers that run
// before each row is inserted, and that have a body that contains a
// call to a function that generates a new value for the column. We
// also check that the column name is in the trigger body, to make
// sure we're not picking up triggers that don't affect the column.
trigger.TRIGGER_BODY.includes(`"${columnName}"`) &&
trigger.TRIGGER_BODY.includes(`.nextval`)
)
if (autoIncrementTriggers.length > 0) {
schema.autocolumn = true
}
}
}
}
private static isSupportedColumn(column: OracleColumn) {
return !UNSUPPORTED_TYPES.includes(column.type)
}
@ -255,7 +344,10 @@ class OracleIntegration extends Sql implements DatasourcePlus {
entities: Record<string, Table>
): Promise<Schema> {
const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
sql: this.COLUMNS_SQL,
sql: OracleIntegration.COLUMNS_SQL,
})
const triggersResponse = await this.internalQuery<OracleTriggersResponse>({
sql: OracleIntegration.TRIGGERS_SQL,
})
const oracleTables = this.mapColumns(columnsResponse)
@ -318,6 +410,8 @@ class OracleIntegration extends Sql implements DatasourcePlus {
})
})
this.markAutoIncrementColumns(triggersResponse, tables)
let externalTables = finaliseExternalTables(tables, entities)
let errors = checkExternalTables(externalTables)
return { tables: externalTables, errors }
@ -325,7 +419,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
async getTableNames() {
const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
sql: this.COLUMNS_SQL,
sql: OracleIntegration.COLUMNS_SQL,
})
return (columnsResponse.rows || []).map(row => row.TABLE_NAME)
}
@ -360,11 +454,32 @@ class OracleIntegration extends Sql implements DatasourcePlus {
this.index = 1
connection = await this.getConnection()
const options: ExecuteOptions = { autoCommit: true }
const options: ExecuteOptions = {
autoCommit: true,
fetchTypeHandler: function (metaData) {
if (metaData.dbType === oracledb.CLOB) {
return { type: oracledb.STRING }
} else if (
// When we create a new table in OracleDB from Budibase, bigints get
// created as NUMBER(20,0). Budibase expects bigints to be returned
// as strings, which is what we're doing here. However, this is
// likely to be brittle if we connect to externally created
// databases that have used different precisions and scales.
// We shold find a way to do better.
metaData.dbType === oracledb.NUMBER &&
metaData.precision === 20 &&
metaData.scale === 0
) {
return { type: oracledb.STRING }
}
return undefined
},
}
const bindings: BindParameters = query.bindings || []
this.log(query.sql, bindings)
return await connection.execute<T>(query.sql, bindings, options)
const result = await connection.execute(query.sql, bindings, options)
return result as Result<T>
} finally {
if (connection) {
try {
@ -377,7 +492,6 @@ class OracleIntegration extends Sql implements DatasourcePlus {
}
private getConnection = async (): Promise<Connection> => {
//connectString : "(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = localhost)(PORT = 1521))(CONNECT_DATA =(SID= ORCL)))"
const connectString = `${this.config.host}:${this.config.port || 1521}/${
this.config.database
}`
@ -386,7 +500,10 @@ class OracleIntegration extends Sql implements DatasourcePlus {
password: this.config.password,
connectString,
}
return oracledb.getConnection(attributes)
const tz = Intl.DateTimeFormat().resolvedOptions().timeZone
const connection = await oracledb.getConnection(attributes)
await connection.execute(`ALTER SESSION SET TIME_ZONE = '${tz}'`)
return connection
}
async create(query: SqlQuery | string): Promise<any[]> {

100
packages/server/src/integrations/tests/oracle.spec.ts
View File

@ -1,100 +0,0 @@
const oracledb = require("oracledb")
import { default as OracleIntegration } from "../oracle"
jest.mock("oracledb")
class TestConfiguration {
integration: any
constructor(config: any = {}) {
this.integration = new OracleIntegration.integration(config)
}
}
const options = { autoCommit: true }
describe("Oracle Integration", () => {
let config: any
beforeEach(() => {
jest.clearAllMocks()
config = new TestConfiguration()
})
it("calls the create method with the correct params", async () => {
const sql = "insert into users (name, age) values ('Joe', 123);"
await config.integration.create({
sql,
})
expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
})
it("calls the read method with the correct params", async () => {
const sql = "select * from users;"
await config.integration.read({
sql,
})
expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
})
it("calls the update method with the correct params", async () => {
const sql = "update table users set name = 'test';"
await config.integration.update({
sql,
})
expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
})
it("calls the delete method with the correct params", async () => {
const sql = "delete from users where name = 'todelete';"
await config.integration.delete({
sql,
})
expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
})
describe("no rows returned", () => {
beforeEach(() => {
oracledb.executeMock.mockImplementation(() => ({ rows: [] }))
})
it("returns the correct response when the create response has no rows", async () => {
const sql = "insert into users (name, age) values ('Joe', 123);"
const response = await config.integration.create({
sql,
})
expect(response).toEqual([{ created: true }])
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
})
it("returns the correct response when the update response has no rows", async () => {
const sql = "update table users set name = 'test';"
const response = await config.integration.update({
sql,
})
expect(response).toEqual([{ updated: true }])
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
})
it("returns the correct response when the delete response has no rows", async () => {
const sql = "delete from users where name = 'todelete';"
const response = await config.integration.delete({
sql,
})
expect(response).toEqual([{ deleted: true }])
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
})
})
})

41
packages/server/src/integrations/tests/sql.spec.ts
View File

@ -1,12 +1,16 @@
import {
FieldType,
Operation,
PaginationJson,
QueryJson,
SearchFilters,
SortJson,
SqlClient,
Table,
TableSourceType,
SqlClient,
} from "@budibase/types"
import { sql } from "@budibase/backend-core"
import { merge } from "lodash"
const Sql = sql.Sql
@ -25,7 +29,16 @@ const TABLE: Table = {
primary: ["id"],
}
function endpoint(table: any, operation: any) {
const ORACLE_TABLE: Partial<Table> = {
schema: {
name: {
name: "name",
type: FieldType.STRING,
},
},
}
function endpoint(table: string, operation: Operation) {
return {
datasourceId: "Postgres",
operation: operation,
@ -39,19 +52,25 @@ function generateReadJson({
filters,
sort,
paginate,
}: any = {}): QueryJson {
const tableObj = { ...TABLE }
}: {
table?: Partial<Table>
fields?: string[]
filters?: SearchFilters
sort?: SortJson
paginate?: PaginationJson
} = {}): QueryJson {
let tableObj: Table = { ...TABLE }
if (table) {
tableObj.name = table
tableObj = merge(TABLE, table)
}
return {
endpoint: endpoint(table || TABLE_NAME, "READ"),
endpoint: endpoint(tableObj.name || TABLE_NAME, Operation.READ),
resource: {
fields: fields || [],
},
filters: filters || {},
sort: sort || {},
paginate: paginate || {},
paginate: paginate || undefined,
meta: {
table: tableObj,
},
@ -191,7 +210,7 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: ["%20%", "%25%", `%"john"%`, `%"mary"%`, limit, 5000],
sql: `select * from (select * from (select * from (select * from "test" where (COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2) and (COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4) order by "test"."id" asc) where rownum <= :5) "test" order by "test"."id" asc) where rownum <= :6`,
sql: `select * from (select * from (select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5) "test" order by "test"."id" asc) where rownum <= :6`,
})
query = new Sql(SqlClient.ORACLE, limit)._query(
@ -212,6 +231,7 @@ describe("SQL query builder", () => {
it("should use an oracle compatible coalesce query for oracle when using the equals filter", () => {
let query = new Sql(SqlClient.ORACLE, limit)._query(
generateReadJson({
table: ORACLE_TABLE,
filters: {
equal: {
name: "John",
@ -222,13 +242,14 @@ describe("SQL query builder", () => {
expect(query).toEqual({
bindings: ["John", limit, 5000],
sql: `select * from (select * from (select * from (select * from "test" where COALESCE("test"."name", -1) = :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
})
})
it("should use an oracle compatible coalesce query for oracle when using the not equals filter", () => {
let query = new Sql(SqlClient.ORACLE, limit)._query(
generateReadJson({
table: ORACLE_TABLE,
filters: {
notEqual: {
name: "John",
@ -239,7 +260,7 @@ describe("SQL query builder", () => {
expect(query).toEqual({
bindings: ["John", limit, 5000],
sql: `select * from (select * from (select * from (select * from "test" where COALESCE("test"."name", -1) != :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
})
})
})

6
packages/server/src/integrations/tests/utils/index.ts
View File

@ -5,6 +5,7 @@ import * as mongodb from "./mongodb"
import * as mysql from "./mysql"
import * as mssql from "./mssql"
import * as mariadb from "./mariadb"
import * as oracle from "./oracle"
import { GenericContainer, StartedTestContainer } from "testcontainers"
import { testContainerUtils } from "@budibase/backend-core/tests"
import cloneDeep from "lodash/cloneDeep"
@ -17,6 +18,7 @@ export enum DatabaseName {
MYSQL = "mysql",
SQL_SERVER = "mssql",
MARIADB = "mariadb",
ORACLE = "oracle",
}
const providers: Record<DatabaseName, DatasourceProvider> = {
@ -25,6 +27,7 @@ const providers: Record<DatabaseName, DatasourceProvider> = {
[DatabaseName.MYSQL]: mysql.getDatasource,
[DatabaseName.SQL_SERVER]: mssql.getDatasource,
[DatabaseName.MARIADB]: mariadb.getDatasource,
[DatabaseName.ORACLE]: oracle.getDatasource,
}
export function getDatasourceProviders(
@ -60,6 +63,9 @@ export async function knexClient(ds: Datasource) {
case SourceName.SQL_SERVER: {
return mssql.knexClient(ds)
}
case SourceName.ORACLE: {
return oracle.knexClient(ds)
}
default: {
throw new Error(`Unsupported source: ${ds.source}`)
}

78
packages/server/src/integrations/tests/utils/oracle.ts Normal file
View File

@ -0,0 +1,78 @@
import { Datasource, SourceName } from "@budibase/types"
import { GenericContainer, Wait } from "testcontainers"
import { generator, testContainerUtils } from "@budibase/backend-core/tests"
import { startContainer } from "."
import knex from "knex"
let ports: Promise<testContainerUtils.Port[]>
export async function getDatasource(): Promise<Datasource> {
// password needs to conform to Oracle standards
const password = "password"
if (!ports) {
// couldn't build 19.3.0 for X64
let image = "budibase/oracle-database:23.2-slim-faststart"
if (process.arch.startsWith("arm")) {
// there isn't an ARM compatible 23.2 build
image = "budibase/oracle-database:19.3.0-ee-slim-faststart"
}
ports = startContainer(
new GenericContainer(image)
.withExposedPorts(1521)
.withEnvironment({
ORACLE_PASSWORD: password,
})
.withWaitStrategy(Wait.forLogMessage("DATABASE IS READY TO USE!"))
)
}
const port = (await ports).find(x => x.container === 1521)?.host
if (!port) {
throw new Error("Oracle port not found")
}
const host = "127.0.0.1"
const user = "SYSTEM"
const datasource: Datasource = {
type: "datasource_plus",
source: SourceName.ORACLE,
plus: true,
config: { host, port, user, password, database: "FREEPDB1" },
}
const newUser = "a" + generator.guid().replaceAll("-", "")
const client = await knexClient(datasource)
await client.raw(`CREATE USER ${newUser} IDENTIFIED BY password`)
await client.raw(
`GRANT CONNECT, RESOURCE, CREATE VIEW, CREATE SESSION TO ${newUser}`
)
await client.raw(`GRANT UNLIMITED TABLESPACE TO ${newUser}`)
datasource.config!.user = newUser
return datasource
}
export async function knexClient(ds: Datasource) {
if (!ds.config) {
throw new Error("Datasource config is missing")
}
if (ds.source !== SourceName.ORACLE) {
throw new Error("Datasource source is not Oracle")
}
const db = ds.config.database || "FREEPDB1"
const connectString = `${ds.config.host}:${ds.config.port}/${db}`
const c = knex({
client: "oracledb",
connection: {
connectString,
user: ds.config.user,
password: ds.config.password,
},
})
return c
}

3
packages/server/src/sdk/app/rows/search/external.ts
View File

@ -147,7 +147,8 @@ export async function search(
} catch (err: any) {
if (err.message && err.message.includes("does not exist")) {
throw new Error(
`Table updated externally, please re-fetch - ${err.message}`
`Table updated externally, please re-fetch - ${err.message}`,
{ cause: err }
)
} else {
throw err

11
packages/server/src/sdk/app/rows/search/internal/sqs.ts
View File

@ -20,12 +20,11 @@ import {
buildInternalRelationships,
sqlOutputProcessing,
} from "../../../../../api/controllers/row/utils"
import sdk from "../../../../index"
import {
decodeNonAscii,
mapToUserColumn,
USER_COLUMN_PREFIX,
} from "../../../tables/internal/sqs"
import sdk from "../../../../index"
import {
context,
sql,
@ -42,7 +41,11 @@ import {
getRelationshipColumns,
getTableIDList,
} from "../filters"
import { dataFilters, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
import {
dataFilters,
helpers,
PROTECTED_INTERNAL_COLUMNS,
} from "@budibase/shared-core"
import { isSearchingByRowID } from "../utils"
import tracer from "dd-trace"
@ -191,7 +194,7 @@ function reverseUserColumnMapping(rows: Row[]) {
if (index !== -1) {
// cut out the prefix
const newKey = key.slice(0, index) + key.slice(index + prefixLength)
const decoded = decodeNonAscii(newKey)
const decoded = helpers.schema.decodeNonAscii(newKey)
finalRow[decoded] = row[key]
} else {
finalRow[key] = row[key]

23
packages/server/src/sdk/app/tables/internal/sqs.ts
View File

@ -13,7 +13,7 @@ import tablesSdk from "../"
import { generateJunctionTableID } from "../../../../db/utils"
import { isEqual } from "lodash"
import { DEFAULT_TABLES } from "../../../../db/defaultData/datasource_bb_default"
import { PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
const FieldTypeMap: Record<FieldType, SQLiteType> = {
[FieldType.BOOLEAN]: SQLiteType.NUMERIC,
@ -63,29 +63,10 @@ function buildRelationshipDefinitions(
export const USER_COLUMN_PREFIX = "data_"
// SQS does not support non-ASCII characters in column names, so we need to
// replace them with unicode escape sequences.
function encodeNonAscii(str: string): string {
return str
.split("")
.map(char => {
return char.charCodeAt(0) > 127
? "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0")
: char
})
.join("")
}
export function decodeNonAscii(str: string): string {
return str.replace(/\\u([0-9a-fA-F]{4})/g, (match, p1) =>
String.fromCharCode(parseInt(p1, 16))
)
}
// utility function to denote that columns in SQLite are mapped to avoid overlap issues
// the overlaps can occur due to case insensitivity and some of the columns which Budibase requires
export function mapToUserColumn(key: string) {
return `${USER_COLUMN_PREFIX}${encodeNonAscii(key)}`
return `${USER_COLUMN_PREFIX}${helpers.schema.encodeNonAscii(key)}`
}
// this can generate relationship tables as part of the mapping

15
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -315,6 +315,21 @@ export async function outputProcessing<T extends Row[] | Row>(
column.subtype
)
}
} else if (column.type === FieldType.DATETIME && column.timeOnly) {
for (let row of enriched) {
if (row[property] instanceof Date) {
const hours = row[property].getUTCHours().toString().padStart(2, "0")
const minutes = row[property]
.getUTCMinutes()
.toString()
.padStart(2, "0")
const seconds = row[property]
.getUTCSeconds()
.toString()
.padStart(2, "0")
row[property] = `${hours}:${minutes}:${seconds}`
}
}
}
}

43
packages/shared-core/src/filters.ts
View File

@ -22,6 +22,7 @@ import dayjs from "dayjs"
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
import { deepGet, schema } from "./helpers"
import { isPlainObject, isEmpty } from "lodash"
import { decodeNonAscii } from "./helpers/schema"
const HBS_REGEX = /{{([^{].*?)}}/g
@ -181,8 +182,16 @@ export class ColumnSplitter {
tableIds: string[]
relationshipColumnNames: string[]
relationships: string[]
aliases?: Record<string, string>
columnPrefix?: string
constructor(tables: Table[]) {
constructor(
tables: Table[],
opts?: {
aliases?: Record<string, string>
columnPrefix?: string
}
) {
this.tableNames = tables.map(table => table.name)
this.tableIds = tables.map(table => table._id!)
this.relationshipColumnNames = tables.flatMap(table =>
@ -195,16 +204,38 @@ export class ColumnSplitter {
.concat(this.relationshipColumnNames)
// sort by length - makes sure there's no mis-matches due to similarities (sub column names)
.sort((a, b) => b.length - a.length)
if (opts?.aliases) {
this.aliases = {}
for (const [key, value] of Object.entries(opts.aliases)) {
this.aliases[value] = key
}
}
this.columnPrefix = opts?.columnPrefix
}
run(key: string): {
numberPrefix?: string
relationshipPrefix?: string
tableName?: string
column: string
} {
let { prefix, key: splitKey } = getKeyNumbering(key)
let tableName: string | undefined = undefined
if (this.aliases) {
for (const possibleAlias of Object.keys(this.aliases || {})) {
const withDot = `${possibleAlias}.`
if (splitKey.startsWith(withDot)) {
tableName = this.aliases[possibleAlias]!
splitKey = splitKey.slice(withDot.length)
}
}
}
let relationship: string | undefined
for (let possibleRelationship of this.relationships) {
for (const possibleRelationship of this.relationships) {
const withDot = `${possibleRelationship}.`
if (splitKey.startsWith(withDot)) {
const finalKeyParts = splitKey.split(withDot)
@ -214,7 +245,15 @@ export class ColumnSplitter {
break
}
}
if (this.columnPrefix) {
if (splitKey.startsWith(this.columnPrefix)) {
splitKey = decodeNonAscii(splitKey.slice(this.columnPrefix.length))
}
}
return {
tableName,
numberPrefix: prefix,
relationshipPrefix: relationship,
column: splitKey,

19
packages/shared-core/src/helpers/schema.ts
View File

@ -26,3 +26,22 @@ export function isRequired(constraints: FieldConstraints | undefined) {
constraints.presence === true)
return isRequired
}
// SQS does not support non-ASCII characters in column names, so we need to
// replace them with unicode escape sequences.
export function encodeNonAscii(str: string): string {
return str
.split("")
.map(char => {
return char.charCodeAt(0) > 127
? "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0")
: char
})
.join("")
}
export function decodeNonAscii(str: string): string {
return str.replace(/\\u([0-9a-fA-F]{4})/g, (match, p1) =>
String.fromCharCode(parseInt(p1, 16))
)
}

5
packages/worker/src/sdk/tenants/tenants.ts
View File

@ -48,10 +48,13 @@ async function removeTenantUsers(tenantId: string) {
try {
const allUsers = await getTenantUsers(tenantId)
const allEmails = allUsers.rows.map((row: any) => row.doc.email)
const allSsoIds = allUsers.rows
.map((row: any) => row.doc.ssoId)
.filter(id => !!id)
// get the id and email doc ids
let keys = allUsers.rows.map((row: any) => row.id)
keys = keys.concat(allEmails)
keys = keys.concat(allEmails).concat(allSsoIds)
const platformDb = platform.getPlatformDB()