MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into dean-fixes

This commit is contained in:
Dean 2024-08-05 17:05:12 +01:00
parent e3f021aea1 f846507877
commit 8279a0b31c
26 changed files with 1017 additions and 733 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/budibase_ci.yml vendored
View File

@ -175,6 +175,7 @@ jobs:
docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }} &
docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }} &
docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }} &
docker pull budibase/oracle-database:23.2-slim-faststart &
docker pull minio/minio &
docker pull redis &
docker pull testcontainers/ryuk:0.5.1 &

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.29.28",
"version": "2.29.29",
"npmClient": "yarn",
"packages": [
"packages/*",

619
packages/backend-core/src/sql/sql.ts
View File

@ -3,15 +3,16 @@ import * as dbCore from "../db"
import {
getNativeSql,
isExternalTable,
isValidISODateString,
isValidFilter,
sqlLog,
isInvalidISODateString,
isValidFilter,
isValidISODateString,
sqlLog,
} from "./utils"
import { SqlStatements } from "./sqlStatements"
import SqlTableQueryBuilder from "./sqlTable"
import {
AnySearchFilter,
ArrayOperator,
BasicOperator,
BBReferenceFieldMetadata,
FieldSchema,
FieldType,
@ -23,6 +24,7 @@ import {
prefixed,
QueryJson,
QueryOptions,
RangeOperator,
RelationshipsJson,
SearchFilters,
SortOrder,
@ -33,7 +35,7 @@ import {
TableSourceType,
} from "@budibase/types"
import environment from "../environment"
import { helpers } from "@budibase/shared-core"
import { dataFilters, helpers } from "@budibase/shared-core"
type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
@ -44,10 +46,64 @@ function getBaseLimit() {
return envLimit || 5000
}
function getTableName(table?: Table): string | undefined {
// SQS uses the table ID rather than the table name
if (
table?.sourceType === TableSourceType.INTERNAL ||
table?.sourceId === INTERNAL_TABLE_SOURCE_ID
) {
return table?._id
} else {
return table?.name
}
}
function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] {
if (Array.isArray(query)) {
return query.map((q: SqlQuery) => convertBooleans(q) as SqlQuery)
} else {
if (query.bindings) {
query.bindings = query.bindings.map(binding => {
if (typeof binding === "boolean") {
return binding ? 1 : 0
}
return binding
})
}
}
return query
}
class InternalBuilder {
private readonly client: SqlClient
private readonly query: QueryJson
private readonly splitter: dataFilters.ColumnSplitter
private readonly knex: Knex
constructor(client: SqlClient, knex: Knex, query: QueryJson) {
this.client = client
this.query = query
this.knex = knex
this.splitter = new dataFilters.ColumnSplitter([this.table], {
aliases: this.query.tableAliases,
columnPrefix: this.query.meta.columnPrefix,
})
}
get table(): Table {
return this.query.meta.table
}
getFieldSchema(key: string): FieldSchema | undefined {
const { column } = this.splitter.run(key)
return this.table.schema[column]
}
// Takes a string like foo and returns a quoted string like [foo] for SQL Server
// and "foo" for Postgres.
function quote(client: SqlClient, str: string): string {
switch (client) {
private quote(str: string): string {
switch (this.client) {
case SqlClient.SQL_LITE:
case SqlClient.ORACLE:
case SqlClient.POSTGRES:
@ -61,62 +117,15 @@ function quote(client: SqlClient, str: string): string {
// Takes a string like a.b.c and returns a quoted identifier like [a].[b].[c]
// for SQL Server and `a`.`b`.`c` for MySQL.
function quotedIdentifier(client: SqlClient, key: string): string {
private quotedIdentifier(key: string): string {
return key
.split(".")
.map(part => quote(client, part))
.map(part => this.quote(part))
.join(".")
}
function parse(input: any) {
if (Array.isArray(input)) {
return JSON.stringify(input)
}
if (input == undefined) {
return null
}
if (typeof input !== "string") {
return input
}
if (isInvalidISODateString(input)) {
return null
}
if (isValidISODateString(input)) {
return new Date(input.trim())
}
return input
}
function parseBody(body: any) {
for (let [key, value] of Object.entries(body)) {
body[key] = parse(value)
}
return body
}
function parseFilters(filters: SearchFilters | undefined): SearchFilters {
if (!filters) {
return {}
}
for (let [key, value] of Object.entries(filters)) {
let parsed
if (typeof value === "object") {
parsed = parseFilters(value)
} else {
parsed = parse(value)
}
// @ts-ignore
filters[key] = parsed
}
return filters
}
function generateSelectStatement(
json: QueryJson,
knex: Knex
): (string | Knex.Raw)[] | "*" {
const { resource, meta } = json
const client = knex.client.config.client as SqlClient
private generateSelectStatement(): (string | Knex.Raw)[] | "*" {
const { resource, meta } = this.query
if (!resource || !resource.fields || resource.fields.length === 0) {
return "*"
@ -152,25 +161,24 @@ function generateSelectStatement(
const columnSchema = schema[column]
if (
client === SqlClient.POSTGRES &&
this.client === SqlClient.POSTGRES &&
columnSchema?.externalType?.includes("money")
) {
return knex.raw(
`${quotedIdentifier(
client,
return this.knex.raw(
`${this.quotedIdentifier(
[table, column].join(".")
)}::money::numeric as ${quote(client, field)}`
)}::money::numeric as ${this.quote(field)}`
)
}
if (
client === SqlClient.MS_SQL &&
this.client === SqlClient.MS_SQL &&
columnSchema?.type === FieldType.DATETIME &&
columnSchema.timeOnly
) {
// Time gets returned as timestamp from mssql, not matching the expected
// HH:mm format
return knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
}
// There's at least two edge cases being handled in the expression below.
@ -182,79 +190,162 @@ function generateSelectStatement(
// aren't actually clear to me, but `table`.`doc1` breaks things with the
// sample data tests.
if (table) {
return knex.raw(
`${quote(client, table)}.${quote(client, column)} as ${quote(
client,
field
)}`
return this.knex.raw(
`${this.quote(table)}.${this.quote(column)} as ${this.quote(field)}`
)
} else {
return knex.raw(`${quote(client, field)} as ${quote(client, field)}`)
return this.knex.raw(`${this.quote(field)} as ${this.quote(field)}`)
}
})
}
function getTableName(table?: Table): string | undefined {
// SQS uses the table ID rather than the table name
// OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
// so when we use them we need to wrap them in to_char(). This function
// converts a field name to the appropriate identifier.
private convertClobs(field: string): string {
const parts = field.split(".")
const col = parts.pop()!
const schema = this.table.schema[col]
let identifier = this.quotedIdentifier(field)
if (
table?.sourceType === TableSourceType.INTERNAL ||
table?.sourceId === INTERNAL_TABLE_SOURCE_ID
schema.type === FieldType.STRING ||
schema.type === FieldType.LONGFORM ||
schema.type === FieldType.BB_REFERENCE_SINGLE ||
schema.type === FieldType.BB_REFERENCE ||
schema.type === FieldType.OPTIONS ||
schema.type === FieldType.BARCODEQR
) {
return table?._id
} else {
return table?.name
identifier = `to_char(${identifier})`
}
return identifier
}
private parse(input: any, schema: FieldSchema) {
if (Array.isArray(input)) {
return JSON.stringify(input)
}
if (input == undefined) {
return null
}
if (
this.client === SqlClient.ORACLE &&
schema.type === FieldType.DATETIME &&
schema.timeOnly
) {
if (input instanceof Date) {
const hours = input.getHours().toString().padStart(2, "0")
const minutes = input.getMinutes().toString().padStart(2, "0")
const seconds = input.getSeconds().toString().padStart(2, "0")
return `${hours}:${minutes}:${seconds}`
}
if (typeof input === "string") {
return new Date(`1970-01-01T${input}Z`)
}
}
function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] {
if (Array.isArray(query)) {
return query.map((q: SqlQuery) => convertBooleans(q) as SqlQuery)
} else {
if (query.bindings) {
query.bindings = query.bindings.map(binding => {
if (typeof binding === "boolean") {
return binding ? 1 : 0
if (typeof input === "string") {
if (isInvalidISODateString(input)) {
return null
}
return binding
})
if (isValidISODateString(input)) {
return new Date(input.trim())
}
}
return query
return input
}
class InternalBuilder {
private readonly client: SqlClient
private parseBody(body: any) {
for (let [key, value] of Object.entries(body)) {
const { column } = this.splitter.run(key)
const schema = this.table.schema[column]
if (!schema) {
continue
}
body[key] = this.parse(value, schema)
}
return body
}
constructor(client: SqlClient) {
this.client = client
private parseFilters(filters: SearchFilters): SearchFilters {
for (const op of Object.values(BasicOperator)) {
const filter = filters[op]
if (!filter) {
continue
}
for (const key of Object.keys(filter)) {
if (Array.isArray(filter[key])) {
filter[key] = JSON.stringify(filter[key])
continue
}
const { column } = this.splitter.run(key)
const schema = this.table.schema[column]
if (!schema) {
continue
}
filter[key] = this.parse(filter[key], schema)
}
}
for (const op of Object.values(ArrayOperator)) {
const filter = filters[op]
if (!filter) {
continue
}
for (const key of Object.keys(filter)) {
const { column } = this.splitter.run(key)
const schema = this.table.schema[column]
if (!schema) {
continue
}
filter[key] = filter[key].map(v => this.parse(v, schema))
}
}
for (const op of Object.values(RangeOperator)) {
const filter = filters[op]
if (!filter) {
continue
}
for (const key of Object.keys(filter)) {
const { column } = this.splitter.run(key)
const schema = this.table.schema[column]
if (!schema) {
continue
}
const value = filter[key]
if ("low" in value) {
value.low = this.parse(value.low, schema)
}
if ("high" in value) {
value.high = this.parse(value.high, schema)
}
}
}
return filters
}
// right now we only do filters on the specific table being queried
addFilters(
query: Knex.QueryBuilder,
filters: SearchFilters | undefined,
table: Table,
opts: {
aliases?: Record<string, string>
opts?: {
relationship?: boolean
columnPrefix?: string
}
): Knex.QueryBuilder {
if (!filters) {
return query
}
filters = parseFilters(filters)
filters = this.parseFilters(filters)
const aliases = this.query.tableAliases
// if all or specified in filters, then everything is an or
const allOr = filters.allOr
const sqlStatements = new SqlStatements(this.client, table, {
allOr,
columnPrefix: opts.columnPrefix,
})
const tableName =
this.client === SqlClient.SQL_LITE ? table._id! : table.name
this.client === SqlClient.SQL_LITE ? this.table._id! : this.table.name
function getTableAlias(name: string) {
const alias = opts.aliases?.[name]
const alias = aliases?.[name]
return alias || name
}
function iterate(
@ -280,10 +371,10 @@ class InternalBuilder {
),
castedTypeValue.values
)
} else if (!opts.relationship && !isRelationshipField) {
} else if (!opts?.relationship && !isRelationshipField) {
const alias = getTableAlias(tableName)
fn(alias ? `${alias}.${updatedKey}` : updatedKey, value)
} else if (opts.relationship && isRelationshipField) {
} else if (opts?.relationship && isRelationshipField) {
const [filterTableName, property] = updatedKey.split(".")
const alias = getTableAlias(filterTableName)
fn(alias ? `${alias}.${property}` : property, value)
@ -300,10 +391,9 @@ class InternalBuilder {
} else {
const rawFnc = `${fnc}Raw`
// @ts-ignore
query = query[rawFnc](
`LOWER(${quotedIdentifier(this.client, key)}) LIKE ?`,
[`%${value.toLowerCase()}%`]
)
query = query[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [
`%${value.toLowerCase()}%`,
])
}
}
@ -345,26 +435,30 @@ class InternalBuilder {
const andOr = mode === filters?.containsAny ? " OR " : " AND "
iterate(mode, (key, value) => {
let statement = ""
const identifier = this.quotedIdentifier(key)
for (let i in value) {
if (typeof value[i] === "string") {
value[i] = `%"${value[i].toLowerCase()}"%`
} else {
value[i] = `%${value[i]}%`
}
statement +=
(statement ? andOr : "") +
`COALESCE(LOWER(${quotedIdentifier(
this.client,
key
)}), '') LIKE ?`
statement += `${
statement ? andOr : ""
}COALESCE(LOWER(${identifier}), '') LIKE ?`
}
if (statement === "") {
return
}
// @ts-ignore
query = query[rawFnc](`${not}(${statement})`, value)
if (not) {
query = query[rawFnc](
`(NOT (${statement}) OR ${identifier} IS NULL)`,
value
)
} else {
query = query[rawFnc](statement, value)
}
})
}
}
@ -374,11 +468,26 @@ class InternalBuilder {
iterate(
filters.oneOf,
(key: string, array) => {
if (this.client === SqlClient.ORACLE) {
key = this.convertClobs(key)
array = Array.isArray(array) ? array : [array]
const binding = new Array(array.length).fill("?").join(",")
query = query.whereRaw(`${key} IN (${binding})`, array)
} else {
query = query[fnc](key, Array.isArray(array) ? array : [array])
}
},
(key: string[], array) => {
if (this.client === SqlClient.ORACLE) {
const keyStr = `(${key.map(k => this.convertClobs(k)).join(",")})`
const binding = `(${array
.map((a: any) => `(${new Array(a.length).fill("?").join(",")})`)
.join(",")})`
query = query.whereRaw(`${keyStr} IN ${binding}`, array.flat())
} else {
query = query[fnc](key, Array.isArray(array) ? array : [array])
}
}
)
}
if (filters.string) {
@ -390,10 +499,9 @@ class InternalBuilder {
} else {
const rawFnc = `${fnc}Raw`
// @ts-ignore
query = query[rawFnc](
`LOWER(${quotedIdentifier(this.client, key)}) LIKE ?`,
[`${value.toLowerCase()}%`]
)
query = query[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [
`${value.toLowerCase()}%`,
])
}
})
}
@ -417,12 +525,53 @@ class InternalBuilder {
}
const lowValid = isValidFilter(value.low),
highValid = isValidFilter(value.high)
const schema = this.getFieldSchema(key)
if (this.client === SqlClient.ORACLE) {
// @ts-ignore
key = this.knex.raw(this.convertClobs(key))
}
if (lowValid && highValid) {
query = sqlStatements.between(query, key, value.low, value.high)
if (
schema?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(
`CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
[value.low, value.high]
)
} else {
const fnc = allOr ? "orWhereBetween" : "whereBetween"
query = query[fnc](key, [value.low, value.high])
}
} else if (lowValid) {
query = sqlStatements.lte(query, key, value.low)
if (
schema?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(
`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`,
[value.low]
)
} else {
const fnc = allOr ? "orWhere" : "where"
query = query[fnc](key, ">=", value.low)
}
} else if (highValid) {
query = sqlStatements.gte(query, key, value.high)
if (
schema?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(
`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`,
[value.high]
)
} else {
const fnc = allOr ? "orWhere" : "where"
query = query[fnc](key, "<=", value.high)
}
}
})
}
@ -431,20 +580,18 @@ class InternalBuilder {
const fnc = allOr ? "orWhereRaw" : "whereRaw"
if (this.client === SqlClient.MS_SQL) {
query = query[fnc](
`CASE WHEN ${quotedIdentifier(
this.client,
key
)} = ? THEN 1 ELSE 0 END = 1`,
`CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 1`,
[value]
)
} else if (this.client === SqlClient.ORACLE) {
const identifier = this.convertClobs(key)
query = query[fnc](
`COALESCE(${quotedIdentifier(this.client, key)}, -1) = ?`,
`(${identifier} IS NOT NULL AND ${identifier} = ?)`,
[value]
)
} else {
query = query[fnc](
`COALESCE(${quotedIdentifier(this.client, key)} = ?, FALSE)`,
`COALESCE(${this.quotedIdentifier(key)} = ?, FALSE)`,
[value]
)
}
@ -455,20 +602,18 @@ class InternalBuilder {
const fnc = allOr ? "orWhereRaw" : "whereRaw"
if (this.client === SqlClient.MS_SQL) {
query = query[fnc](
`CASE WHEN ${quotedIdentifier(
this.client,
key
)} = ? THEN 1 ELSE 0 END = 0`,
`CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 0`,
[value]
)
} else if (this.client === SqlClient.ORACLE) {
const identifier = this.convertClobs(key)
query = query[fnc](
`COALESCE(${quotedIdentifier(this.client, key)}, -1) != ?`,
`(${identifier} IS NOT NULL AND ${identifier} != ?) OR ${identifier} IS NULL`,
[value]
)
} else {
query = query[fnc](
`COALESCE(${quotedIdentifier(this.client, key)} != ?, TRUE)`,
`COALESCE(${this.quotedIdentifier(key)} != ?, TRUE)`,
[value]
)
}
@ -496,9 +641,9 @@ class InternalBuilder {
contains(filters.containsAny, true)
}
const tableRef = opts?.aliases?.[table._id!] || table._id
const tableRef = aliases?.[this.table._id!] || this.table._id
// when searching internal tables make sure long looking for rows
if (filters.documentType && !isExternalTable(table) && tableRef) {
if (filters.documentType && !isExternalTable(this.table) && tableRef) {
// has to be its own option, must always be AND onto the search
query.andWhereLike(
`${tableRef}._id`,
@ -509,29 +654,26 @@ class InternalBuilder {
return query
}
addDistinctCount(
query: Knex.QueryBuilder,
json: QueryJson
): Knex.QueryBuilder {
const table = json.meta.table
const primary = table.primary
const aliases = json.tableAliases
addDistinctCount(query: Knex.QueryBuilder): Knex.QueryBuilder {
const primary = this.table.primary
const aliases = this.query.tableAliases
const aliased =
table.name && aliases?.[table.name] ? aliases[table.name] : table.name
this.table.name && aliases?.[this.table.name]
? aliases[this.table.name]
: this.table.name
if (!primary) {
throw new Error("SQL counting requires primary key to be supplied")
}
return query.countDistinct(`${aliased}.${primary[0]} as total`)
}
addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder {
let { sort } = json
const table = json.meta.table
const primaryKey = table.primary
const tableName = getTableName(table)
const aliases = json.tableAliases
addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder {
let { sort } = this.query
const primaryKey = this.table.primary
const tableName = getTableName(this.table)
const aliases = this.query.tableAliases
const aliased =
tableName && aliases?.[tableName] ? aliases[tableName] : table?.name
tableName && aliases?.[tableName] ? aliases[tableName] : this.table?.name
if (!Array.isArray(primaryKey)) {
throw new Error("Sorting requires primary key to be specified for table")
}
@ -539,13 +681,23 @@ class InternalBuilder {
for (let [key, value] of Object.entries(sort)) {
const direction =
value.direction === SortOrder.ASCENDING ? "asc" : "desc"
let nulls
if (this.client === SqlClient.POSTGRES) {
// All other clients already sort this as expected by default, and adding this to the rest of the clients is causing issues
let nulls: "first" | "last" | undefined = undefined
if (
this.client === SqlClient.POSTGRES ||
this.client === SqlClient.ORACLE
) {
nulls = value.direction === SortOrder.ASCENDING ? "first" : "last"
}
query = query.orderBy(`${aliased}.${key}`, direction, nulls)
let composite = `${aliased}.${key}`
if (this.client === SqlClient.ORACLE) {
query = query.orderByRaw(
`${this.convertClobs(composite)} ${direction} nulls ${nulls}`
)
} else {
query = query.orderBy(composite, direction, nulls)
}
}
}
@ -646,32 +798,54 @@ class InternalBuilder {
return query
}
knexWithAlias(
knex: Knex,
endpoint: QueryJson["endpoint"],
aliases?: QueryJson["tableAliases"]
): Knex.QueryBuilder {
const tableName = endpoint.entityId
const tableAlias = aliases?.[tableName]
return knex(
this.tableNameWithSchema(tableName, {
alias: tableAlias,
schema: endpoint.schema,
qualifiedKnex(opts?: { alias?: string | boolean }): Knex.QueryBuilder {
let alias = this.query.tableAliases?.[this.query.endpoint.entityId]
if (opts?.alias === false) {
alias = undefined
} else if (typeof opts?.alias === "string") {
alias = opts.alias
}
return this.knex(
this.tableNameWithSchema(this.query.endpoint.entityId, {
alias,
schema: this.query.endpoint.schema,
})
)
}
create(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
const { endpoint, body } = json
let query = this.knexWithAlias(knex, endpoint)
const parsedBody = parseBody(body)
create(opts: QueryOptions): Knex.QueryBuilder {
const { body } = this.query
let query = this.qualifiedKnex({ alias: false })
const parsedBody = this.parseBody(body)
if (this.client === SqlClient.ORACLE) {
// Oracle doesn't seem to automatically insert nulls
// if we don't specify them, so we need to do that here
for (const [column, schema] of Object.entries(
this.query.meta.table.schema
)) {
if (
schema.constraints?.presence === true ||
schema.type === FieldType.FORMULA ||
schema.type === FieldType.AUTO ||
schema.type === FieldType.LINK
) {
continue
}
const value = parsedBody[column]
if (value == null) {
parsedBody[column] = null
}
}
} else {
// make sure no null values in body for creation
for (let [key, value] of Object.entries(parsedBody)) {
if (value == null) {
delete parsedBody[key]
}
}
}
// mysql can't use returning
if (opts.disableReturning) {
@ -681,36 +855,39 @@ class InternalBuilder {
}
}
bulkCreate(knex: Knex, json: QueryJson): Knex.QueryBuilder {
const { endpoint, body } = json
let query = this.knexWithAlias(knex, endpoint)
bulkCreate(): Knex.QueryBuilder {
const { body } = this.query
let query = this.qualifiedKnex({ alias: false })
if (!Array.isArray(body)) {
return query
}
const parsedBody = body.map(row => parseBody(row))
const parsedBody = body.map(row => this.parseBody(row))
return query.insert(parsedBody)
}
bulkUpsert(knex: Knex, json: QueryJson): Knex.QueryBuilder {
const { endpoint, body } = json
let query = this.knexWithAlias(knex, endpoint)
bulkUpsert(): Knex.QueryBuilder {
const { body } = this.query
let query = this.qualifiedKnex({ alias: false })
if (!Array.isArray(body)) {
return query
}
const parsedBody = body.map(row => parseBody(row))
const parsedBody = body.map(row => this.parseBody(row))
if (
this.client === SqlClient.POSTGRES ||
this.client === SqlClient.SQL_LITE ||
this.client === SqlClient.MY_SQL
) {
const primary = json.meta.table.primary
const primary = this.table.primary
if (!primary) {
throw new Error("Primary key is required for upsert")
}
const ret = query.insert(parsedBody).onConflict(primary).merge()
return ret
} else if (this.client === SqlClient.MS_SQL) {
// No upsert or onConflict support in MSSQL yet, see:
} else if (
this.client === SqlClient.MS_SQL ||
this.client === SqlClient.ORACLE
) {
// No upsert or onConflict support in MSSQL/Oracle yet, see:
// https://github.com/knex/knex/pull/6050
return query.insert(parsedBody)
}
@ -718,19 +895,18 @@ class InternalBuilder {
}
read(
knex: Knex,
json: QueryJson,
opts: {
limits?: { base: number; query: number }
} = {}
): Knex.QueryBuilder {
let { endpoint, filters, paginate, relationships, tableAliases } = json
let { endpoint, filters, paginate, relationships, tableAliases } =
this.query
const { limits } = opts
const counting = endpoint.operation === Operation.COUNT
const tableName = endpoint.entityId
// start building the query
let query = this.knexWithAlias(knex, endpoint, tableAliases)
let query = this.qualifiedKnex()
// handle pagination
let foundOffset: number | null = null
let foundLimit = limits?.query || limits?.base
@ -758,16 +934,13 @@ class InternalBuilder {
}
// add sorting to pre-query
// no point in sorting when counting
query = this.addSorting(query, json)
query = this.addSorting(query)
}
// add filters to the query (where)
query = this.addFilters(query, filters, json.meta.table, {
columnPrefix: json.meta.columnPrefix,
aliases: tableAliases,
})
query = this.addFilters(query, filters)
const alias = tableAliases?.[tableName] || tableName
let preQuery: Knex.QueryBuilder = knex({
let preQuery: Knex.QueryBuilder = this.knex({
// the typescript definition for the knex constructor doesn't support this
// syntax, but it is the only way to alias a pre-query result as part of
// a query - there is an alias dictionary type, but it assumes it can only
@ -776,11 +949,11 @@ class InternalBuilder {
})
// if counting, use distinct count, else select
preQuery = !counting
? preQuery.select(generateSelectStatement(json, knex))
: this.addDistinctCount(preQuery, json)
? preQuery.select(this.generateSelectStatement())
: this.addDistinctCount(preQuery)
// have to add after as well (this breaks MS-SQL)
if (this.client !== SqlClient.MS_SQL && !counting) {
preQuery = this.addSorting(preQuery, json)
preQuery = this.addSorting(preQuery)
}
// handle joins
query = this.addRelationships(
@ -797,21 +970,14 @@ class InternalBuilder {
query = query.limit(limits.base)
}
return this.addFilters(query, filters, json.meta.table, {
columnPrefix: json.meta.columnPrefix,
relationship: true,
aliases: tableAliases,
})
return this.addFilters(query, filters, { relationship: true })
}
update(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
const { endpoint, body, filters, tableAliases } = json
let query = this.knexWithAlias(knex, endpoint, tableAliases)
const parsedBody = parseBody(body)
query = this.addFilters(query, filters, json.meta.table, {
columnPrefix: json.meta.columnPrefix,
aliases: tableAliases,
})
update(opts: QueryOptions): Knex.QueryBuilder {
const { body, filters } = this.query
let query = this.qualifiedKnex()
const parsedBody = this.parseBody(body)
query = this.addFilters(query, filters)
// mysql can't use returning
if (opts.disableReturning) {
return query.update(parsedBody)
@ -820,18 +986,15 @@ class InternalBuilder {
}
}
delete(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
const { endpoint, filters, tableAliases } = json
let query = this.knexWithAlias(knex, endpoint, tableAliases)
query = this.addFilters(query, filters, json.meta.table, {
columnPrefix: json.meta.columnPrefix,
aliases: tableAliases,
})
delete(opts: QueryOptions): Knex.QueryBuilder {
const { filters } = this.query
let query = this.qualifiedKnex()
query = this.addFilters(query, filters)
// mysql can't use returning
if (opts.disableReturning) {
return query.delete()
} else {
return query.delete().returning(generateSelectStatement(json, knex))
return query.delete().returning(this.generateSelectStatement())
}
}
}
@ -869,19 +1032,19 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
const config: Knex.Config = {
client: sqlClient,
}
if (sqlClient === SqlClient.SQL_LITE) {
if (sqlClient === SqlClient.SQL_LITE || sqlClient === SqlClient.ORACLE) {
config.useNullAsDefault = true
}
const client = knex(config)
let query: Knex.QueryBuilder
const builder = new InternalBuilder(sqlClient)
const builder = new InternalBuilder(sqlClient, client, json)
switch (this._operation(json)) {
case Operation.CREATE:
query = builder.create(client, json, opts)
query = builder.create(opts)
break
case Operation.READ:
query = builder.read(client, json, {
query = builder.read({
limits: {
query: this.limit,
base: getBaseLimit(),
@ -890,19 +1053,19 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
break
case Operation.COUNT:
// read without any limits to count
query = builder.read(client, json)
query = builder.read()
break
case Operation.UPDATE:
query = builder.update(client, json, opts)
query = builder.update(opts)
break
case Operation.DELETE:
query = builder.delete(client, json, opts)
query = builder.delete(opts)
break
case Operation.BULK_CREATE:
query = builder.bulkCreate(client, json)
query = builder.bulkCreate()
break
case Operation.BULK_UPSERT:
query = builder.bulkUpsert(client, json)
query = builder.bulkUpsert()
break
case Operation.CREATE_TABLE:
case Operation.UPDATE_TABLE:

87
packages/backend-core/src/sql/sqlStatements.ts
View File

@ -1,87 +0,0 @@
import { FieldType, Table, FieldSchema, SqlClient } from "@budibase/types"
import { Knex } from "knex"
export class SqlStatements {
client: string
table: Table
allOr: boolean | undefined
columnPrefix: string | undefined
constructor(
client: string,
table: Table,
{ allOr, columnPrefix }: { allOr?: boolean; columnPrefix?: string } = {}
) {
this.client = client
this.table = table
this.allOr = allOr
this.columnPrefix = columnPrefix
}
getField(key: string): FieldSchema | undefined {
const fieldName = key.split(".")[1]
let found = this.table.schema[fieldName]
if (!found && this.columnPrefix) {
const prefixRemovedFieldName = fieldName.replace(this.columnPrefix, "")
found = this.table.schema[prefixRemovedFieldName]
}
return found
}
between(
query: Knex.QueryBuilder,
key: string,
low: number | string,
high: number | string
) {
// Use a between operator if we have 2 valid range values
const field = this.getField(key)
if (
field?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(
`CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
[low, high]
)
} else {
const fnc = this.allOr ? "orWhereBetween" : "whereBetween"
query = query[fnc](key, [low, high])
}
return query
}
lte(query: Knex.QueryBuilder, key: string, low: number | string) {
// Use just a single greater than operator if we only have a low
const field = this.getField(key)
if (
field?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [
low,
])
} else {
const fnc = this.allOr ? "orWhere" : "where"
query = query[fnc](key, ">=", low)
}
return query
}
gte(query: Knex.QueryBuilder, key: string, high: number | string) {
const field = this.getField(key)
// Use just a single less than operator if we only have a high
if (
field?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [
high,
])
} else {
const fnc = this.allOr ? "orWhere" : "where"
query = query[fnc](key, "<=", high)
}
return query
}
}

1
packages/backend-core/src/sql/utils.ts
View File

@ -22,6 +22,7 @@ export function getNativeSql(
query: Knex.SchemaBuilder | Knex.QueryBuilder
): SqlQuery | SqlQuery[] {
let sql = query.toSQL()
if (Array.isArray(sql)) {
return sql as SqlQuery[]
}

21
packages/server/__mocks__/oracledb.ts
View File

@ -1,21 +0,0 @@
const executeMock = jest.fn(() => ({
rows: [
{
a: "string",
b: 1,
},
],
}))
const closeMock = jest.fn()
class Connection {
execute = executeMock
close = closeMock
}
module.exports = {
getConnection: jest.fn(() => new Connection()),
executeMock,
closeMock,
}

4
packages/server/scripts/integrations/oracle/docker-compose.yml
View File

@ -6,9 +6,9 @@ services:
db:
restart: unless-stopped
platform: linux/x86_64
image: container-registry.oracle.com/database/express:18.4.0-xe
image: gvenzl/oracle-free:23.2-slim-faststart
environment:
ORACLE_PWD: oracle
ORACLE_PWD: Password1
ports:
- 1521:1521
- 5500:5500

174
packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
View File

@ -22,9 +22,13 @@ describe.each(
DatabaseName.MYSQL,
DatabaseName.SQL_SERVER,
DatabaseName.MARIADB,
DatabaseName.ORACLE,
].map(name => [name, getDatasource(name)])
)("queries (%s)", (dbName, dsProvider) => {
const config = setup.getConfig()
const isOracle = dbName === DatabaseName.ORACLE
const isMsSQL = dbName === DatabaseName.SQL_SERVER
let rawDatasource: Datasource
let datasource: Datasource
let client: Knex
@ -97,7 +101,7 @@ describe.each(
const query = await createQuery({
name: "New Query",
fields: {
sql: "SELECT * FROM test_table",
sql: client("test_table").select("*").toString(),
},
})
@ -106,7 +110,7 @@ describe.each(
name: "New Query",
parameters: [],
fields: {
sql: "SELECT * FROM test_table",
sql: client("test_table").select("*").toString(),
},
schema: {},
queryVerb: "read",
@ -125,7 +129,7 @@ describe.each(
it("should be able to update a query", async () => {
const query = await createQuery({
fields: {
sql: "SELECT * FROM test_table",
sql: client("test_table").select("*").toString(),
},
})
@ -135,7 +139,7 @@ describe.each(
...query,
name: "Updated Query",
fields: {
sql: "SELECT * FROM test_table WHERE id = 1",
sql: client("test_table").where({ id: 1 }).toString(),
},
})
@ -144,7 +148,7 @@ describe.each(
name: "Updated Query",
parameters: [],
fields: {
sql: "SELECT * FROM test_table WHERE id = 1",
sql: client("test_table").where({ id: 1 }).toString(),
},
schema: {},
queryVerb: "read",
@ -161,7 +165,7 @@ describe.each(
it("should be able to delete a query", async () => {
const query = await createQuery({
fields: {
sql: "SELECT * FROM test_table",
sql: client("test_table").select("*").toString(),
},
})
@ -180,7 +184,7 @@ describe.each(
it("should be able to list queries", async () => {
const query = await createQuery({
fields: {
sql: "SELECT * FROM test_table",
sql: client("test_table").select("*").toString(),
},
})
@ -191,7 +195,7 @@ describe.each(
it("should strip sensitive fields for prod apps", async () => {
const query = await createQuery({
fields: {
sql: "SELECT * FROM test_table",
sql: client("test_table").select("*").toString(),
},
})
@ -212,7 +216,7 @@ describe.each(
datasourceId: datasource._id!,
queryVerb: "read",
fields: {
sql: `SELECT * FROM test_table WHERE id = 1`,
sql: client("test_table").where({ id: 1 }).toString(),
},
parameters: [],
transformer: "return data",
@ -270,7 +274,7 @@ describe.each(
name: "Test Query",
queryVerb: "read",
fields: {
sql: `SELECT * FROM ${tableName}`,
sql: client(tableName).select("*").toString(),
},
parameters: [],
transformer: "return data",
@ -284,11 +288,13 @@ describe.each(
})
)
await client(tableName).delete()
await client.schema.alterTable(tableName, table => {
table.string("data").alter()
})
await client(tableName).update({
await client(tableName).insert({
name: "test",
data: "string value",
})
@ -297,7 +303,7 @@ describe.each(
name: "Test Query",
queryVerb: "read",
fields: {
sql: `SELECT * FROM ${tableName}`,
sql: client(tableName).select("*").toString(),
},
parameters: [],
transformer: "return data",
@ -311,6 +317,7 @@ describe.each(
})
)
})
it("should work with static variables", async () => {
await config.api.datasource.update({
...datasource,
@ -326,7 +333,7 @@ describe.each(
datasourceId: datasource._id!,
queryVerb: "read",
fields: {
sql: `SELECT '{{ foo }}' as foo`,
sql: `SELECT '{{ foo }}' AS foo ${isOracle ? "FROM dual" : ""}`,
},
parameters: [],
transformer: "return data",
@ -337,16 +344,17 @@ describe.each(
const response = await config.api.query.preview(request)
let key = isOracle ? "FOO" : "foo"
expect(response.schema).toEqual({
foo: {
name: "foo",
[key]: {
name: key,
type: "string",
},
})
expect(response.rows).toEqual([
{
foo: "bar",
[key]: "bar",
},
])
})
@ -354,7 +362,7 @@ describe.each(
it("should work with dynamic variables", async () => {
const basedOnQuery = await createQuery({
fields: {
sql: "SELECT name FROM test_table WHERE id = 1",
sql: client("test_table").select("name").where({ id: 1 }).toString(),
},
})
@ -376,7 +384,7 @@ describe.each(
datasourceId: datasource._id!,
queryVerb: "read",
fields: {
sql: `SELECT '{{ foo }}' as foo`,
sql: `SELECT '{{ foo }}' AS foo ${isOracle ? "FROM dual" : ""}`,
},
parameters: [],
transformer: "return data",
@ -385,16 +393,17 @@ describe.each(
readable: true,
})
let key = isOracle ? "FOO" : "foo"
expect(preview.schema).toEqual({
foo: {
name: "foo",
[key]: {
name: key,
type: "string",
},
})
expect(preview.rows).toEqual([
{
foo: "one",
[key]: "one",
},
])
})
@ -402,7 +411,7 @@ describe.each(
it("should handle the dynamic base query being deleted", async () => {
const basedOnQuery = await createQuery({
fields: {
sql: "SELECT name FROM test_table WHERE id = 1",
sql: client("test_table").select("name").where({ id: 1 }).toString(),
},
})
@ -426,7 +435,7 @@ describe.each(
datasourceId: datasource._id!,
queryVerb: "read",
fields: {
sql: `SELECT '{{ foo }}' as foo`,
sql: `SELECT '{{ foo }}' AS foo ${isOracle ? "FROM dual" : ""}`,
},
parameters: [],
transformer: "return data",
@ -435,16 +444,17 @@ describe.each(
readable: true,
})
let key = isOracle ? "FOO" : "foo"
expect(preview.schema).toEqual({
foo: {
name: "foo",
[key]: {
name: key,
type: "string",
},
})
expect(preview.rows).toEqual([
{
foo: datasource.source === SourceName.SQL_SERVER ? "" : null,
[key]: datasource.source === SourceName.SQL_SERVER ? "" : null,
},
])
})
@ -455,7 +465,7 @@ describe.each(
it("should be able to insert with bindings", async () => {
const query = await createQuery({
fields: {
sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
sql: client("test_table").insert({ name: "{{ foo }}" }).toString(),
},
parameters: [
{
@ -488,7 +498,7 @@ describe.each(
it("should not allow handlebars as parameters", async () => {
const query = await createQuery({
fields: {
sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
sql: client("test_table").insert({ name: "{{ foo }}" }).toString(),
},
parameters: [
{
@ -516,13 +526,20 @@ describe.each(
)
})
// Oracle doesn't automatically coerce strings into dates.
!isOracle &&
it.each(["2021-02-05T12:01:00.000Z", "2021-02-05"])(
"should coerce %s into a date",
async datetimeStr => {
const date = new Date(datetimeStr)
const query = await createQuery({
fields: {
sql: `INSERT INTO test_table (name, birthday) VALUES ('foo', {{ birthday }})`,
sql: client("test_table")
.insert({
name: "foo",
birthday: client.raw("{{ birthday }}"),
})
.toString(),
},
parameters: [
{
@ -555,7 +572,9 @@ describe.each(
async notDateStr => {
const query = await createQuery({
fields: {
sql: "INSERT INTO test_table (name) VALUES ({{ name }})",
sql: client("test_table")
.insert({ name: client.raw("{{ name }}") })
.toString(),
},
parameters: [
{
@ -586,7 +605,7 @@ describe.each(
it("should execute a query", async () => {
const query = await createQuery({
fields: {
sql: "SELECT * FROM test_table ORDER BY id",
sql: client("test_table").select("*").orderBy("id").toString(),
},
})
@ -629,7 +648,7 @@ describe.each(
it("should be able to transform a query", async () => {
const query = await createQuery({
fields: {
sql: "SELECT * FROM test_table WHERE id = 1",
sql: client("test_table").where({ id: 1 }).select("*").toString(),
},
transformer: `
data[0].id = data[0].id + 1;
@ -652,7 +671,10 @@ describe.each(
it("should coerce numeric bindings", async () => {
const query = await createQuery({
fields: {
sql: "SELECT * FROM test_table WHERE id = {{ id }}",
sql: client("test_table")
.where({ id: client.raw("{{ id }}") })
.select("*")
.toString(),
},
parameters: [
{
@ -683,7 +705,10 @@ describe.each(
it("should be able to update rows", async () => {
const query = await createQuery({
fields: {
sql: "UPDATE test_table SET name = {{ name }} WHERE id = {{ id }}",
sql: client("test_table")
.update({ name: client.raw("{{ name }}") })
.where({ id: client.raw("{{ id }}") })
.toString(),
},
parameters: [
{
@ -698,19 +723,13 @@ describe.each(
queryVerb: "update",
})
const result = await config.api.query.execute(query._id!, {
await config.api.query.execute(query._id!, {
parameters: {
id: "1",
name: "foo",
},
})
expect(result.data).toEqual([
{
updated: true,
},
])
const rows = await client("test_table").where({ id: 1 }).select()
expect(rows).toEqual([
{ id: 1, name: "foo", birthday: null, number: null },
@ -720,35 +739,34 @@ describe.each(
it("should be able to execute an update that updates no rows", async () => {
const query = await createQuery({
fields: {
sql: "UPDATE test_table SET name = 'updated' WHERE id = 100",
sql: client("test_table")
.update({ name: "updated" })
.where({ id: 100 })
.toString(),
},
queryVerb: "update",
})
const result = await config.api.query.execute(query._id!)
await config.api.query.execute(query._id!)
expect(result.data).toEqual([
{
updated: true,
},
])
const rows = await client("test_table").select()
for (const row of rows) {
expect(row.name).not.toEqual("updated")
}
})
it("should be able to execute a delete that deletes no rows", async () => {
const query = await createQuery({
fields: {
sql: "DELETE FROM test_table WHERE id = 100",
sql: client("test_table").where({ id: 100 }).delete().toString(),
},
queryVerb: "delete",
})
const result = await config.api.query.execute(query._id!)
await config.api.query.execute(query._id!)
expect(result.data).toEqual([
{
deleted: true,
},
])
const rows = await client("test_table").select()
expect(rows).toHaveLength(5)
})
})
@ -756,7 +774,10 @@ describe.each(
it("should be able to delete rows", async () => {
const query = await createQuery({
fields: {
sql: "DELETE FROM test_table WHERE id = {{ id }}",
sql: client("test_table")
.where({ id: client.raw("{{ id }}") })
.delete()
.toString(),
},
parameters: [
{
@ -767,18 +788,12 @@ describe.each(
queryVerb: "delete",
})
const result = await config.api.query.execute(query._id!, {
await config.api.query.execute(query._id!, {
parameters: {
id: "1",
},
})
expect(result.data).toEqual([
{
deleted: true,
},
])
const rows = await client("test_table").where({ id: 1 }).select()
expect(rows).toHaveLength(0)
})
@ -823,29 +838,18 @@ describe.each(
})
})
it("should be able to execute an update that updates no rows", async () => {
const query = await createQuery({
fields: {
sql: "UPDATE test_table SET name = 'updated' WHERE id = 100",
},
queryVerb: "update",
})
const result = await config.api.query.execute(query._id!, {})
expect(result.data).toEqual([
{
updated: true,
},
])
})
})
// this parameter really only impacts SQL queries
describe("confirm nullDefaultSupport", () => {
const queryParams = {
let queryParams: Partial<Query>
beforeAll(async () => {
queryParams = {
fields: {
sql: "INSERT INTO test_table (name, number) VALUES ({{ bindingName }}, {{ bindingNumber }})",
sql: client("test_table")
.insert({
name: client.raw("{{ bindingName }}"),
number: client.raw("{{ bindingNumber }}"),
})
.toString(),
},
parameters: [
{
@ -859,6 +863,7 @@ describe.each(
],
queryVerb: "create",
}
})
it("should error for old queries", async () => {
const query = await createQuery(queryParams)
@ -873,7 +878,7 @@ describe.each(
} catch (err: any) {
error = err.message
}
if (dbName === "mssql") {
if (isMsSQL || isOracle) {
expect(error).toBeUndefined()
} else {
expect(error).toBeDefined()
@ -892,3 +897,4 @@ describe.each(
})
})
})
})

14
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -72,9 +72,11 @@ describe.each([
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("/rows (%s)", (providerType, dsProvider) => {
const isInternal = dsProvider === undefined
const isMSSQL = providerType === DatabaseName.SQL_SERVER
const isOracle = providerType === DatabaseName.ORACLE
const config = setup.getConfig()
let table: Table
@ -129,7 +131,8 @@ describe.each([
primary: ["id"],
schema: defaultSchema,
}
return merge(req, ...overrides)
const merged = merge(req, ...overrides)
return merged
}
function defaultTable(
@ -1406,9 +1409,10 @@ describe.each([
await assertRowUsage(rowUsage + 3)
})
// Upserting isn't yet supported in MSSQL, see:
// Upserting isn't yet supported in MSSQL / Oracle, see:
// https://github.com/knex/knex/pull/6050
!isMSSQL &&
!isOracle &&
it("should be able to update existing rows with bulkImport", async () => {
const table = await config.api.table.save(
saveTableRequest({
@ -1478,9 +1482,10 @@ describe.each([
expect(rows[2].description).toEqual("Row 3 description")
})
// Upserting isn't yet supported in MSSQL, see:
// Upserting isn't yet supported in MSSQL or Oracle, see:
// https://github.com/knex/knex/pull/6050
!isMSSQL &&
!isOracle &&
!isInternal &&
it("should be able to update existing rows with composite primary keys with bulkImport", async () => {
const tableName = uuid.v4()
@ -1547,9 +1552,10 @@ describe.each([
expect(rows[2].description).toEqual("Row 3 description")
})
// Upserting isn't yet supported in MSSQL, see:
// Upserting isn't yet supported in MSSQL/Oracle, see:
// https://github.com/knex/knex/pull/6050
!isMSSQL &&
!isOracle &&
!isInternal &&
it("should be able to update existing rows an autoID primary key", async () => {
const tableName = uuid.v4()

6
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -48,6 +48,7 @@ describe.each([
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("search (%s)", (name, dsProvider) => {
const isSqs = name === "sqs"
const isLucene = name === "lucene"
@ -1594,7 +1595,10 @@ describe.each([
const MEDIUM = "10000000"
// Our bigints are int64s in most datasources.
const BIG = "9223372036854775807"
let BIG = "9223372036854775807"
if (name === DatabaseName.ORACLE) {
// BIG = "9223372036854775808"
}
beforeAll(async () => {
table = await createTable({

16
packages/server/src/api/routes/tests/table.spec.ts
View File

@ -40,7 +40,8 @@ describe.each([
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("/tables (%s)", (_, dsProvider) => {
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("/tables (%s)", (name, dsProvider) => {
const isInternal: boolean = !dsProvider
let datasource: Datasource | undefined
let config = setup.getConfig()
@ -59,15 +60,20 @@ describe.each([
jest.clearAllMocks()
})
it.each([
let names = [
"alphanum",
"with spaces",
"with-dashes",
"with_underscores",
'with "double quotes"',
"with 'single quotes'",
"with `backticks`",
])("creates a table with name: %s", async name => {
]
if (name !== DatabaseName.ORACLE) {
names.push(`with "double quotes"`)
names.push(`with 'single quotes'`)
}
it.each(names)("creates a table with name: %s", async name => {
const table = await config.api.table.save(
tableForDatasource(datasource, { name })
)

1
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -33,6 +33,7 @@ describe.each([
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("/v2/views (%s)", (name, dsProvider) => {
const config = setup.getConfig()
const isSqs = name === "sqs"

28
packages/server/src/automations/tests/executeQuery.spec.ts
View File

@ -1,27 +1,20 @@
import { Datasource, Query, SourceName } from "@budibase/types"
import { Datasource, Query } from "@budibase/types"
import * as setup from "./utilities"
import { DatabaseName, getDatasource } from "../../integrations/tests/utils"
import knex, { Knex } from "knex"
import {
DatabaseName,
getDatasource,
knexClient,
} from "../../integrations/tests/utils"
import { Knex } from "knex"
import { generator } from "@budibase/backend-core/tests"
function getKnexClientName(source: SourceName) {
switch (source) {
case SourceName.MYSQL:
return "mysql2"
case SourceName.SQL_SERVER:
return "mssql"
case SourceName.POSTGRES:
return "pg"
}
throw new Error(`Unsupported source: ${source}`)
}
describe.each(
[
DatabaseName.POSTGRES,
DatabaseName.MYSQL,
DatabaseName.SQL_SERVER,
DatabaseName.MARIADB,
DatabaseName.ORACLE,
].map(name => [name, getDatasource(name)])
)("execute query action (%s)", (_, dsProvider) => {
let tableName: string
@ -35,10 +28,7 @@ describe.each(
const ds = await dsProvider
datasource = await config.api.datasource.create(ds)
client = knex({
client: getKnexClientName(ds.source),
connection: ds.config,
})
client = await knexClient(ds)
})
beforeEach(async () => {

31
packages/server/src/integrations/base/types.ts
View File

@ -104,6 +104,37 @@ export interface OracleColumnsResponse {
SEARCH_CONDITION: null | string
}
export enum TriggeringEvent {
INSERT = "INSERT",
DELETE = "DELETE",
UPDATE = "UPDATE",
LOGON = "LOGON",
LOGOFF = "LOGOFF",
STARTUP = "STARTUP",
SHUTDOWN = "SHUTDOWN",
SERVERERROR = "SERVERERROR",
SCHEMA = "SCHEMA",
ALTER = "ALTER",
DROP = "DROP",
}
export enum TriggerType {
BEFORE_EACH_ROW = "BEFORE EACH ROW",
AFTER_EACH_ROW = "AFTER EACH ROW",
BEFORE_STATEMENT = "BEFORE STATEMENT",
AFTER_STATEMENT = "AFTER STATEMENT",
INSTEAD_OF = "INSTEAD OF",
COMPOUND = "COMPOUND",
}
export interface OracleTriggersResponse {
TABLE_NAME: string
TRIGGER_NAME: string
TRIGGER_TYPE: TriggerType
TRIGGERING_EVENT: TriggeringEvent
TRIGGER_BODY: string
}
/**
* An oracle constraint
*/

135
packages/server/src/integrations/oracle.ts
View File

@ -31,7 +31,14 @@ import oracledb, {
ExecuteOptions,
Result,
} from "oracledb"
import { OracleTable, OracleColumn, OracleColumnsResponse } from "./base/types"
import {
OracleTable,
OracleColumn,
OracleColumnsResponse,
OracleTriggersResponse,
TriggeringEvent,
TriggerType,
} from "./base/types"
import { sql } from "@budibase/backend-core"
const Sql = sql.Sql
@ -98,7 +105,7 @@ const SCHEMA: Integration = {
},
}
const UNSUPPORTED_TYPES = ["BLOB", "CLOB", "NCLOB"]
const UNSUPPORTED_TYPES = ["BLOB", "NCLOB"]
const OracleContraintTypes = {
PRIMARY: "P",
@ -111,7 +118,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
private readonly config: OracleConfig
private index: number = 1
private readonly COLUMNS_SQL = `
private static readonly COLUMNS_SQL = `
SELECT
tabs.table_name,
cols.column_name,
@ -139,6 +146,19 @@ class OracleIntegration extends Sql implements DatasourcePlus {
(cons.status = 'ENABLED'
OR cons.status IS NULL)
`
private static readonly TRIGGERS_SQL = `
SELECT
table_name,
trigger_name,
trigger_type,
triggering_event,
trigger_body
FROM
all_triggers
WHERE status = 'ENABLED'
`
constructor(config: OracleConfig) {
super(SqlClient.ORACLE)
this.config = config
@ -211,6 +231,75 @@ class OracleIntegration extends Sql implements DatasourcePlus {
return oracleTables
}
private getTriggersFor(
tableName: string,
triggersResponse: Result<OracleTriggersResponse>,
opts?: { event?: TriggeringEvent; type?: TriggerType }
): OracleTriggersResponse[] {
const triggers: OracleTriggersResponse[] = []
for (const trigger of triggersResponse.rows || []) {
if (trigger.TABLE_NAME !== tableName) {
continue
}
if (opts?.event && opts.event !== trigger.TRIGGERING_EVENT) {
continue
}
if (opts?.type && opts.type !== trigger.TRIGGER_TYPE) {
continue
}
triggers.push(trigger)
}
return triggers
}
private markAutoIncrementColumns(
triggersResponse: Result<OracleTriggersResponse>,
tables: Record<string, Table>
) {
for (const table of Object.values(tables)) {
const triggers = this.getTriggersFor(table.name, triggersResponse, {
type: TriggerType.BEFORE_EACH_ROW,
event: TriggeringEvent.INSERT,
})
// This is the trigger body Knex generates for an auto increment column
// called "id" on a table called "foo":
//
// declare checking number := 1;
// begin if (:new. "id" is null) then while checking >= 1 loop
// select
// "foo_seq".nextval into :new. "id"
// from
// dual;
// select
// count("id") into checking
// from
// "foo"
// where
// "id" = :new. "id";
// end loop;
// end if;
// end;
for (const [columnName, schema] of Object.entries(table.schema)) {
const autoIncrementTriggers = triggers.filter(
trigger =>
// This is a bit heuristic, but I think it's the best we can do with
// the information we have. We're looking for triggers that run
// before each row is inserted, and that have a body that contains a
// call to a function that generates a new value for the column. We
// also check that the column name is in the trigger body, to make
// sure we're not picking up triggers that don't affect the column.
trigger.TRIGGER_BODY.includes(`"${columnName}"`) &&
trigger.TRIGGER_BODY.includes(`.nextval`)
)
if (autoIncrementTriggers.length > 0) {
schema.autocolumn = true
}
}
}
}
private static isSupportedColumn(column: OracleColumn) {
return !UNSUPPORTED_TYPES.includes(column.type)
}
@ -255,7 +344,10 @@ class OracleIntegration extends Sql implements DatasourcePlus {
entities: Record<string, Table>
): Promise<Schema> {
const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
sql: this.COLUMNS_SQL,
sql: OracleIntegration.COLUMNS_SQL,
})
const triggersResponse = await this.internalQuery<OracleTriggersResponse>({
sql: OracleIntegration.TRIGGERS_SQL,
})
const oracleTables = this.mapColumns(columnsResponse)
@ -318,6 +410,8 @@ class OracleIntegration extends Sql implements DatasourcePlus {
})
})
this.markAutoIncrementColumns(triggersResponse, tables)
let externalTables = finaliseExternalTables(tables, entities)
let errors = checkExternalTables(externalTables)
return { tables: externalTables, errors }
@ -325,7 +419,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
async getTableNames() {
const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
sql: this.COLUMNS_SQL,
sql: OracleIntegration.COLUMNS_SQL,
})
return (columnsResponse.rows || []).map(row => row.TABLE_NAME)
}
@ -360,11 +454,32 @@ class OracleIntegration extends Sql implements DatasourcePlus {
this.index = 1
connection = await this.getConnection()
const options: ExecuteOptions = { autoCommit: true }
const options: ExecuteOptions = {
autoCommit: true,
fetchTypeHandler: function (metaData) {
if (metaData.dbType === oracledb.CLOB) {
return { type: oracledb.STRING }
} else if (
// When we create a new table in OracleDB from Budibase, bigints get
// created as NUMBER(20,0). Budibase expects bigints to be returned
// as strings, which is what we're doing here. However, this is
// likely to be brittle if we connect to externally created
// databases that have used different precisions and scales.
// We shold find a way to do better.
metaData.dbType === oracledb.NUMBER &&
metaData.precision === 20 &&
metaData.scale === 0
) {
return { type: oracledb.STRING }
}
return undefined
},
}
const bindings: BindParameters = query.bindings || []
this.log(query.sql, bindings)
return await connection.execute<T>(query.sql, bindings, options)
const result = await connection.execute(query.sql, bindings, options)
return result as Result<T>
} finally {
if (connection) {
try {
@ -377,7 +492,6 @@ class OracleIntegration extends Sql implements DatasourcePlus {
}
private getConnection = async (): Promise<Connection> => {
//connectString : "(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = localhost)(PORT = 1521))(CONNECT_DATA =(SID= ORCL)))"
const connectString = `${this.config.host}:${this.config.port || 1521}/${
this.config.database
}`
@ -386,7 +500,10 @@ class OracleIntegration extends Sql implements DatasourcePlus {
password: this.config.password,
connectString,
}
return oracledb.getConnection(attributes)
const tz = Intl.DateTimeFormat().resolvedOptions().timeZone
const connection = await oracledb.getConnection(attributes)
await connection.execute(`ALTER SESSION SET TIME_ZONE = '${tz}'`)
return connection
}
async create(query: SqlQuery | string): Promise<any[]> {

100
packages/server/src/integrations/tests/oracle.spec.ts
View File

@ -1,100 +0,0 @@
const oracledb = require("oracledb")
import { default as OracleIntegration } from "../oracle"
jest.mock("oracledb")
class TestConfiguration {
integration: any
constructor(config: any = {}) {
this.integration = new OracleIntegration.integration(config)
}
}
const options = { autoCommit: true }
describe("Oracle Integration", () => {
let config: any
beforeEach(() => {
jest.clearAllMocks()
config = new TestConfiguration()
})
it("calls the create method with the correct params", async () => {
const sql = "insert into users (name, age) values ('Joe', 123);"
await config.integration.create({
sql,
})
expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
})
it("calls the read method with the correct params", async () => {
const sql = "select * from users;"
await config.integration.read({
sql,
})
expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
})
it("calls the update method with the correct params", async () => {
const sql = "update table users set name = 'test';"
await config.integration.update({
sql,
})
expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
})
it("calls the delete method with the correct params", async () => {
const sql = "delete from users where name = 'todelete';"
await config.integration.delete({
sql,
})
expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
})
describe("no rows returned", () => {
beforeEach(() => {
oracledb.executeMock.mockImplementation(() => ({ rows: [] }))
})
it("returns the correct response when the create response has no rows", async () => {
const sql = "insert into users (name, age) values ('Joe', 123);"
const response = await config.integration.create({
sql,
})
expect(response).toEqual([{ created: true }])
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
})
it("returns the correct response when the update response has no rows", async () => {
const sql = "update table users set name = 'test';"
const response = await config.integration.update({
sql,
})
expect(response).toEqual([{ updated: true }])
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
})
it("returns the correct response when the delete response has no rows", async () => {
const sql = "delete from users where name = 'todelete';"
const response = await config.integration.delete({
sql,
})
expect(response).toEqual([{ deleted: true }])
expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
})
})
})

41
packages/server/src/integrations/tests/sql.spec.ts
View File

@ -1,12 +1,16 @@
import {
FieldType,
Operation,
PaginationJson,
QueryJson,
SearchFilters,
SortJson,
SqlClient,
Table,
TableSourceType,
SqlClient,
} from "@budibase/types"
import { sql } from "@budibase/backend-core"
import { merge } from "lodash"
const Sql = sql.Sql
@ -25,7 +29,16 @@ const TABLE: Table = {
primary: ["id"],
}
function endpoint(table: any, operation: any) {
const ORACLE_TABLE: Partial<Table> = {
schema: {
name: {
name: "name",
type: FieldType.STRING,
},
},
}
function endpoint(table: string, operation: Operation) {
return {
datasourceId: "Postgres",
operation: operation,
@ -39,19 +52,25 @@ function generateReadJson({
filters,
sort,
paginate,
}: any = {}): QueryJson {
const tableObj = { ...TABLE }
}: {
table?: Partial<Table>
fields?: string[]
filters?: SearchFilters
sort?: SortJson
paginate?: PaginationJson
} = {}): QueryJson {
let tableObj: Table = { ...TABLE }
if (table) {
tableObj.name = table
tableObj = merge(TABLE, table)
}
return {
endpoint: endpoint(table || TABLE_NAME, "READ"),
endpoint: endpoint(tableObj.name || TABLE_NAME, Operation.READ),
resource: {
fields: fields || [],
},
filters: filters || {},
sort: sort || {},
paginate: paginate || {},
paginate: paginate || undefined,
meta: {
table: tableObj,
},
@ -191,7 +210,7 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: ["%20%", "%25%", `%"john"%`, `%"mary"%`, limit, 5000],
sql: `select * from (select * from (select * from (select * from "test" where (COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2) and (COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4) order by "test"."id" asc) where rownum <= :5) "test" order by "test"."id" asc) where rownum <= :6`,
sql: `select * from (select * from (select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5) "test" order by "test"."id" asc) where rownum <= :6`,
})
query = new Sql(SqlClient.ORACLE, limit)._query(
@ -212,6 +231,7 @@ describe("SQL query builder", () => {
it("should use an oracle compatible coalesce query for oracle when using the equals filter", () => {
let query = new Sql(SqlClient.ORACLE, limit)._query(
generateReadJson({
table: ORACLE_TABLE,
filters: {
equal: {
name: "John",
@ -222,13 +242,14 @@ describe("SQL query builder", () => {
expect(query).toEqual({
bindings: ["John", limit, 5000],
sql: `select * from (select * from (select * from (select * from "test" where COALESCE("test"."name", -1) = :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
})
})
it("should use an oracle compatible coalesce query for oracle when using the not equals filter", () => {
let query = new Sql(SqlClient.ORACLE, limit)._query(
generateReadJson({
table: ORACLE_TABLE,
filters: {
notEqual: {
name: "John",
@ -239,7 +260,7 @@ describe("SQL query builder", () => {
expect(query).toEqual({
bindings: ["John", limit, 5000],
sql: `select * from (select * from (select * from (select * from "test" where COALESCE("test"."name", -1) != :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
})
})
})

6
packages/server/src/integrations/tests/utils/index.ts
View File

@ -5,6 +5,7 @@ import * as mongodb from "./mongodb"
import * as mysql from "./mysql"
import * as mssql from "./mssql"
import * as mariadb from "./mariadb"
import * as oracle from "./oracle"
import { GenericContainer, StartedTestContainer } from "testcontainers"
import { testContainerUtils } from "@budibase/backend-core/tests"
import cloneDeep from "lodash/cloneDeep"
@ -17,6 +18,7 @@ export enum DatabaseName {
MYSQL = "mysql",
SQL_SERVER = "mssql",
MARIADB = "mariadb",
ORACLE = "oracle",
}
const providers: Record<DatabaseName, DatasourceProvider> = {
@ -25,6 +27,7 @@ const providers: Record<DatabaseName, DatasourceProvider> = {
[DatabaseName.MYSQL]: mysql.getDatasource,
[DatabaseName.SQL_SERVER]: mssql.getDatasource,
[DatabaseName.MARIADB]: mariadb.getDatasource,
[DatabaseName.ORACLE]: oracle.getDatasource,
}
export function getDatasourceProviders(
@ -60,6 +63,9 @@ export async function knexClient(ds: Datasource) {
case SourceName.SQL_SERVER: {
return mssql.knexClient(ds)
}
case SourceName.ORACLE: {
return oracle.knexClient(ds)
}
default: {
throw new Error(`Unsupported source: ${ds.source}`)
}

78
packages/server/src/integrations/tests/utils/oracle.ts Normal file
View File

@ -0,0 +1,78 @@
import { Datasource, SourceName } from "@budibase/types"
import { GenericContainer, Wait } from "testcontainers"
import { generator, testContainerUtils } from "@budibase/backend-core/tests"
import { startContainer } from "."
import knex from "knex"
let ports: Promise<testContainerUtils.Port[]>
export async function getDatasource(): Promise<Datasource> {
// password needs to conform to Oracle standards
const password = "password"
if (!ports) {
// couldn't build 19.3.0 for X64
let image = "budibase/oracle-database:23.2-slim-faststart"
if (process.arch.startsWith("arm")) {
// there isn't an ARM compatible 23.2 build
image = "budibase/oracle-database:19.3.0-ee-slim-faststart"
}
ports = startContainer(
new GenericContainer(image)
.withExposedPorts(1521)
.withEnvironment({
ORACLE_PASSWORD: password,
})
.withWaitStrategy(Wait.forLogMessage("DATABASE IS READY TO USE!"))
)
}
const port = (await ports).find(x => x.container === 1521)?.host
if (!port) {
throw new Error("Oracle port not found")
}
const host = "127.0.0.1"
const user = "SYSTEM"
const datasource: Datasource = {
type: "datasource_plus",
source: SourceName.ORACLE,
plus: true,
config: { host, port, user, password, database: "FREEPDB1" },
}
const newUser = "a" + generator.guid().replaceAll("-", "")
const client = await knexClient(datasource)
await client.raw(`CREATE USER ${newUser} IDENTIFIED BY password`)
await client.raw(
`GRANT CONNECT, RESOURCE, CREATE VIEW, CREATE SESSION TO ${newUser}`
)
await client.raw(`GRANT UNLIMITED TABLESPACE TO ${newUser}`)
datasource.config!.user = newUser
return datasource
}
export async function knexClient(ds: Datasource) {
if (!ds.config) {
throw new Error("Datasource config is missing")
}
if (ds.source !== SourceName.ORACLE) {
throw new Error("Datasource source is not Oracle")
}
const db = ds.config.database || "FREEPDB1"
const connectString = `${ds.config.host}:${ds.config.port}/${db}`
const c = knex({
client: "oracledb",
connection: {
connectString,
user: ds.config.user,
password: ds.config.password,
},
})
return c
}

3
packages/server/src/sdk/app/rows/search/external.ts
View File

@ -147,7 +147,8 @@ export async function search(
} catch (err: any) {
if (err.message && err.message.includes("does not exist")) {
throw new Error(
`Table updated externally, please re-fetch - ${err.message}`
`Table updated externally, please re-fetch - ${err.message}`,
{ cause: err }
)
} else {
throw err

11
packages/server/src/sdk/app/rows/search/internal/sqs.ts
View File

@ -19,12 +19,11 @@ import {
buildInternalRelationships,
sqlOutputProcessing,
} from "../../../../../api/controllers/row/utils"
import sdk from "../../../../index"
import {
decodeNonAscii,
mapToUserColumn,
USER_COLUMN_PREFIX,
} from "../../../tables/internal/sqs"
import sdk from "../../../../index"
import {
context,
sql,
@ -41,7 +40,11 @@ import {
getRelationshipColumns,
getTableIDList,
} from "../filters"
import { dataFilters, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
import {
dataFilters,
helpers,
PROTECTED_INTERNAL_COLUMNS,
} from "@budibase/shared-core"
import { isSearchingByRowID } from "../utils"
import tracer from "dd-trace"
@ -174,7 +177,7 @@ function reverseUserColumnMapping(rows: Row[]) {
if (index !== -1) {
// cut out the prefix
const newKey = key.slice(0, index) + key.slice(index + prefixLength)
const decoded = decodeNonAscii(newKey)
const decoded = helpers.schema.decodeNonAscii(newKey)
finalRow[decoded] = row[key]
} else {
finalRow[key] = row[key]

23
packages/server/src/sdk/app/tables/internal/sqs.ts
View File

@ -13,7 +13,7 @@ import tablesSdk from "../"
import { generateJunctionTableID } from "../../../../db/utils"
import { isEqual } from "lodash"
import { DEFAULT_TABLES } from "../../../../db/defaultData/datasource_bb_default"
import { PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
const FieldTypeMap: Record<FieldType, SQLiteType> = {
[FieldType.BOOLEAN]: SQLiteType.NUMERIC,
@ -63,29 +63,10 @@ function buildRelationshipDefinitions(
export const USER_COLUMN_PREFIX = "data_"
// SQS does not support non-ASCII characters in column names, so we need to
// replace them with unicode escape sequences.
function encodeNonAscii(str: string): string {
return str
.split("")
.map(char => {
return char.charCodeAt(0) > 127
? "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0")
: char
})
.join("")
}
export function decodeNonAscii(str: string): string {
return str.replace(/\\u([0-9a-fA-F]{4})/g, (match, p1) =>
String.fromCharCode(parseInt(p1, 16))
)
}
// utility function to denote that columns in SQLite are mapped to avoid overlap issues
// the overlaps can occur due to case insensitivity and some of the columns which Budibase requires
export function mapToUserColumn(key: string) {
return `${USER_COLUMN_PREFIX}${encodeNonAscii(key)}`
return `${USER_COLUMN_PREFIX}${helpers.schema.encodeNonAscii(key)}`
}
// this can generate relationship tables as part of the mapping

15
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -315,6 +315,21 @@ export async function outputProcessing<T extends Row[] | Row>(
column.subtype
)
}
} else if (column.type === FieldType.DATETIME && column.timeOnly) {
for (let row of enriched) {
if (row[property] instanceof Date) {
const hours = row[property].getUTCHours().toString().padStart(2, "0")
const minutes = row[property]
.getUTCMinutes()
.toString()
.padStart(2, "0")
const seconds = row[property]
.getUTCSeconds()
.toString()
.padStart(2, "0")
row[property] = `${hours}:${minutes}:${seconds}`
}
}
}
}

43
packages/shared-core/src/filters.ts
View File

@ -22,6 +22,7 @@ import dayjs from "dayjs"
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
import { deepGet, schema } from "./helpers"
import { isPlainObject, isEmpty } from "lodash"
import { decodeNonAscii } from "./helpers/schema"
const HBS_REGEX = /{{([^{].*?)}}/g
@ -181,8 +182,16 @@ export class ColumnSplitter {
tableIds: string[]
relationshipColumnNames: string[]
relationships: string[]
aliases?: Record<string, string>
columnPrefix?: string
constructor(tables: Table[]) {
constructor(
tables: Table[],
opts?: {
aliases?: Record<string, string>
columnPrefix?: string
}
) {
this.tableNames = tables.map(table => table.name)
this.tableIds = tables.map(table => table._id!)
this.relationshipColumnNames = tables.flatMap(table =>
@ -195,16 +204,38 @@ export class ColumnSplitter {
.concat(this.relationshipColumnNames)
// sort by length - makes sure there's no mis-matches due to similarities (sub column names)
.sort((a, b) => b.length - a.length)
if (opts?.aliases) {
this.aliases = {}
for (const [key, value] of Object.entries(opts.aliases)) {
this.aliases[value] = key
}
}
this.columnPrefix = opts?.columnPrefix
}
run(key: string): {
numberPrefix?: string
relationshipPrefix?: string
tableName?: string
column: string
} {
let { prefix, key: splitKey } = getKeyNumbering(key)
let tableName: string | undefined = undefined
if (this.aliases) {
for (const possibleAlias of Object.keys(this.aliases || {})) {
const withDot = `${possibleAlias}.`
if (splitKey.startsWith(withDot)) {
tableName = this.aliases[possibleAlias]!
splitKey = splitKey.slice(withDot.length)
}
}
}
let relationship: string | undefined
for (let possibleRelationship of this.relationships) {
for (const possibleRelationship of this.relationships) {
const withDot = `${possibleRelationship}.`
if (splitKey.startsWith(withDot)) {
const finalKeyParts = splitKey.split(withDot)
@ -214,7 +245,15 @@ export class ColumnSplitter {
break
}
}
if (this.columnPrefix) {
if (splitKey.startsWith(this.columnPrefix)) {
splitKey = decodeNonAscii(splitKey.slice(this.columnPrefix.length))
}
}
return {
tableName,
numberPrefix: prefix,
relationshipPrefix: relationship,
column: splitKey,

19
packages/shared-core/src/helpers/schema.ts
View File

@ -26,3 +26,22 @@ export function isRequired(constraints: FieldConstraints | undefined) {
constraints.presence === true)
return isRequired
}
// SQS does not support non-ASCII characters in column names, so we need to
// replace them with unicode escape sequences.
export function encodeNonAscii(str: string): string {
return str
.split("")
.map(char => {
return char.charCodeAt(0) > 127
? "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0")
: char
})
.join("")
}
export function decodeNonAscii(str: string): string {
return str.replace(/\\u([0-9a-fA-F]{4})/g, (match, p1) =>
String.fromCharCode(parseInt(p1, 16))
)
}

5
packages/worker/src/sdk/tenants/tenants.ts
View File

@ -48,10 +48,13 @@ async function removeTenantUsers(tenantId: string) {
try {
const allUsers = await getTenantUsers(tenantId)
const allEmails = allUsers.rows.map((row: any) => row.doc.email)
const allSsoIds = allUsers.rows
.map((row: any) => row.doc.ssoId)
.filter(id => !!id)
// get the id and email doc ids
let keys = allUsers.rows.map((row: any) => row.id)
keys = keys.concat(allEmails)
keys = keys.concat(allEmails).concat(allSsoIds)
const platformDb = platform.getPlatformDB()