Merge pull request #14532 from Budibase/fix/sql-many-relationships
SQL - utilise JSON aggregations for relationships
This commit is contained in:
commit
31f8691708
|
@ -171,6 +171,7 @@ const environment = {
|
||||||
// Couch/search
|
// Couch/search
|
||||||
SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE,
|
SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE,
|
||||||
SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
|
SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
|
||||||
|
SQL_MAX_RELATED_ROWS: process.env.MAX_RELATED_ROWS,
|
||||||
// smtp
|
// smtp
|
||||||
SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,
|
SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,
|
||||||
SMTP_USER: process.env.SMTP_USER,
|
SMTP_USER: process.env.SMTP_USER,
|
||||||
|
|
|
@ -7,6 +7,7 @@ import {
|
||||||
isValidFilter,
|
isValidFilter,
|
||||||
isValidISODateString,
|
isValidISODateString,
|
||||||
sqlLog,
|
sqlLog,
|
||||||
|
validateManyToMany,
|
||||||
} from "./utils"
|
} from "./utils"
|
||||||
import SqlTableQueryBuilder from "./sqlTable"
|
import SqlTableQueryBuilder from "./sqlTable"
|
||||||
import {
|
import {
|
||||||
|
@ -39,6 +40,7 @@ import { dataFilters, helpers } from "@budibase/shared-core"
|
||||||
import { cloneDeep } from "lodash"
|
import { cloneDeep } from "lodash"
|
||||||
|
|
||||||
type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
|
type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
|
||||||
|
const MAX_SQS_RELATIONSHIP_FIELDS = 63
|
||||||
|
|
||||||
function getBaseLimit() {
|
function getBaseLimit() {
|
||||||
const envLimit = environment.SQL_MAX_ROWS
|
const envLimit = environment.SQL_MAX_ROWS
|
||||||
|
@ -47,6 +49,13 @@ function getBaseLimit() {
|
||||||
return envLimit || 5000
|
return envLimit || 5000
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getRelationshipLimit() {
|
||||||
|
const envLimit = environment.SQL_MAX_RELATED_ROWS
|
||||||
|
? parseInt(environment.SQL_MAX_RELATED_ROWS)
|
||||||
|
: null
|
||||||
|
return envLimit || 500
|
||||||
|
}
|
||||||
|
|
||||||
function getTableName(table?: Table): string | undefined {
|
function getTableName(table?: Table): string | undefined {
|
||||||
// SQS uses the table ID rather than the table name
|
// SQS uses the table ID rather than the table name
|
||||||
if (
|
if (
|
||||||
|
@ -92,6 +101,23 @@ class InternalBuilder {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// states the various situations in which we need a full mapped select statement
|
||||||
|
private readonly SPECIAL_SELECT_CASES = {
|
||||||
|
POSTGRES_MONEY: (field: FieldSchema | undefined) => {
|
||||||
|
return (
|
||||||
|
this.client === SqlClient.POSTGRES &&
|
||||||
|
field?.externalType?.includes("money")
|
||||||
|
)
|
||||||
|
},
|
||||||
|
MSSQL_DATES: (field: FieldSchema | undefined) => {
|
||||||
|
return (
|
||||||
|
this.client === SqlClient.MS_SQL &&
|
||||||
|
field?.type === FieldType.DATETIME &&
|
||||||
|
field.timeOnly
|
||||||
|
)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
get table(): Table {
|
get table(): Table {
|
||||||
return this.query.meta.table
|
return this.query.meta.table
|
||||||
}
|
}
|
||||||
|
@ -125,46 +151,52 @@ class InternalBuilder {
|
||||||
.join(".")
|
.join(".")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private isFullSelectStatementRequired(): boolean {
|
||||||
|
const { meta } = this.query
|
||||||
|
for (let column of Object.values(meta.table.schema)) {
|
||||||
|
if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) {
|
||||||
|
return true
|
||||||
|
} else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
private generateSelectStatement(): (string | Knex.Raw)[] | "*" {
|
private generateSelectStatement(): (string | Knex.Raw)[] | "*" {
|
||||||
const { resource, meta } = this.query
|
const { meta, endpoint, resource, tableAliases } = this.query
|
||||||
|
|
||||||
if (!resource || !resource.fields || resource.fields.length === 0) {
|
if (!resource || !resource.fields || resource.fields.length === 0) {
|
||||||
return "*"
|
return "*"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const alias = tableAliases?.[endpoint.entityId]
|
||||||
|
? tableAliases?.[endpoint.entityId]
|
||||||
|
: endpoint.entityId
|
||||||
const schema = meta.table.schema
|
const schema = meta.table.schema
|
||||||
return resource.fields.map(field => {
|
if (!this.isFullSelectStatementRequired()) {
|
||||||
|
return [this.knex.raw(`${this.quote(alias)}.*`)]
|
||||||
|
}
|
||||||
|
// get just the fields for this table
|
||||||
|
return resource.fields
|
||||||
|
.map(field => {
|
||||||
const parts = field.split(/\./g)
|
const parts = field.split(/\./g)
|
||||||
let table: string | undefined = undefined
|
let table: string | undefined = undefined
|
||||||
let column: string | undefined = undefined
|
let column = parts[0]
|
||||||
|
|
||||||
// Just a column name, e.g.: "column"
|
// Just a column name, e.g.: "column"
|
||||||
if (parts.length === 1) {
|
if (parts.length > 1) {
|
||||||
column = parts[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
// A table name and a column name, e.g.: "table.column"
|
|
||||||
if (parts.length === 2) {
|
|
||||||
table = parts[0]
|
|
||||||
column = parts[1]
|
|
||||||
}
|
|
||||||
|
|
||||||
// A link doc, e.g.: "table.doc1.fieldName"
|
|
||||||
if (parts.length > 2) {
|
|
||||||
table = parts[0]
|
table = parts[0]
|
||||||
column = parts.slice(1).join(".")
|
column = parts.slice(1).join(".")
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!column) {
|
return { table, column, field }
|
||||||
throw new Error(`Invalid field name: ${field}`)
|
})
|
||||||
}
|
.filter(({ table }) => !table || table === alias)
|
||||||
|
.map(({ table, column, field }) => {
|
||||||
const columnSchema = schema[column]
|
const columnSchema = schema[column]
|
||||||
|
|
||||||
if (
|
if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) {
|
||||||
this.client === SqlClient.POSTGRES &&
|
|
||||||
columnSchema?.externalType?.includes("money")
|
|
||||||
) {
|
|
||||||
return this.knex.raw(
|
return this.knex.raw(
|
||||||
`${this.quotedIdentifier(
|
`${this.quotedIdentifier(
|
||||||
[table, column].join(".")
|
[table, column].join(".")
|
||||||
|
@ -172,31 +204,16 @@ class InternalBuilder {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(columnSchema)) {
|
||||||
this.client === SqlClient.MS_SQL &&
|
|
||||||
columnSchema?.type === FieldType.DATETIME &&
|
|
||||||
columnSchema.timeOnly
|
|
||||||
) {
|
|
||||||
// Time gets returned as timestamp from mssql, not matching the expected
|
// Time gets returned as timestamp from mssql, not matching the expected
|
||||||
// HH:mm format
|
// HH:mm format
|
||||||
return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
|
return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
|
||||||
}
|
}
|
||||||
|
|
||||||
// There's at least two edge cases being handled in the expression below.
|
const quoted = table
|
||||||
// 1. The column name could start/end with a space, and in that case we
|
? `${this.quote(table)}.${this.quote(column)}`
|
||||||
// want to preseve that space.
|
: this.quote(field)
|
||||||
// 2. Almost all column names are specified in the form table.column, except
|
return this.knex.raw(quoted)
|
||||||
// in the case of relationships, where it's table.doc1.column. In that
|
|
||||||
// case, we want to split it into `table`.`doc1.column` for reasons that
|
|
||||||
// aren't actually clear to me, but `table`.`doc1` breaks things with the
|
|
||||||
// sample data tests.
|
|
||||||
if (table) {
|
|
||||||
return this.knex.raw(
|
|
||||||
`${this.quote(table)}.${this.quote(column)} as ${this.quote(field)}`
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
return this.knex.raw(`${this.quote(field)} as ${this.quote(field)}`)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -328,6 +345,85 @@ class InternalBuilder {
|
||||||
return filters
|
return filters
|
||||||
}
|
}
|
||||||
|
|
||||||
|
addJoinFieldCheck(query: Knex.QueryBuilder, relationship: RelationshipsJson) {
|
||||||
|
const document = relationship.from?.split(".")[0] || ""
|
||||||
|
return query.andWhere(`${document}.fieldName`, "=", relationship.column)
|
||||||
|
}
|
||||||
|
|
||||||
|
addRelationshipForFilter(
|
||||||
|
query: Knex.QueryBuilder,
|
||||||
|
filterKey: string,
|
||||||
|
whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder
|
||||||
|
): Knex.QueryBuilder {
|
||||||
|
const mainKnex = this.knex
|
||||||
|
const { relationships, endpoint, tableAliases: aliases } = this.query
|
||||||
|
const tableName = endpoint.entityId
|
||||||
|
const fromAlias = aliases?.[tableName] || tableName
|
||||||
|
const matches = (possibleTable: string) =>
|
||||||
|
filterKey.startsWith(`${possibleTable}`)
|
||||||
|
if (!relationships) {
|
||||||
|
return query
|
||||||
|
}
|
||||||
|
for (const relationship of relationships) {
|
||||||
|
const relatedTableName = relationship.tableName
|
||||||
|
const toAlias = aliases?.[relatedTableName] || relatedTableName
|
||||||
|
// this is the relationship which is being filtered
|
||||||
|
if (
|
||||||
|
(matches(relatedTableName) || matches(toAlias)) &&
|
||||||
|
relationship.to &&
|
||||||
|
relationship.tableName
|
||||||
|
) {
|
||||||
|
let subQuery = mainKnex
|
||||||
|
.select(mainKnex.raw(1))
|
||||||
|
.from({ [toAlias]: relatedTableName })
|
||||||
|
const manyToMany = validateManyToMany(relationship)
|
||||||
|
if (manyToMany) {
|
||||||
|
const throughAlias =
|
||||||
|
aliases?.[manyToMany.through] || relationship.through
|
||||||
|
let throughTable = this.tableNameWithSchema(manyToMany.through, {
|
||||||
|
alias: throughAlias,
|
||||||
|
schema: endpoint.schema,
|
||||||
|
})
|
||||||
|
subQuery = subQuery
|
||||||
|
// add a join through the junction table
|
||||||
|
.innerJoin(throughTable, function () {
|
||||||
|
// @ts-ignore
|
||||||
|
this.on(
|
||||||
|
`${toAlias}.${manyToMany.toPrimary}`,
|
||||||
|
"=",
|
||||||
|
`${throughAlias}.${manyToMany.to}`
|
||||||
|
)
|
||||||
|
})
|
||||||
|
// check the document in the junction table points to the main table
|
||||||
|
.where(
|
||||||
|
`${throughAlias}.${manyToMany.from}`,
|
||||||
|
"=",
|
||||||
|
mainKnex.raw(
|
||||||
|
this.quotedIdentifier(`${fromAlias}.${manyToMany.fromPrimary}`)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
// in SQS the same junction table is used for different many-to-many relationships between the
|
||||||
|
// two same tables, this is needed to avoid rows ending up in all columns
|
||||||
|
if (this.client === SqlClient.SQL_LITE) {
|
||||||
|
subQuery = this.addJoinFieldCheck(subQuery, manyToMany)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// "join" to the main table, making sure the ID matches that of the main
|
||||||
|
subQuery = subQuery.where(
|
||||||
|
`${toAlias}.${relationship.to}`,
|
||||||
|
"=",
|
||||||
|
mainKnex.raw(
|
||||||
|
this.quotedIdentifier(`${fromAlias}.${relationship.from}`)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
query = query.whereExists(whereCb(subQuery))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return query
|
||||||
|
}
|
||||||
|
|
||||||
// right now we only do filters on the specific table being queried
|
// right now we only do filters on the specific table being queried
|
||||||
addFilters(
|
addFilters(
|
||||||
query: Knex.QueryBuilder,
|
query: Knex.QueryBuilder,
|
||||||
|
@ -339,12 +435,13 @@ class InternalBuilder {
|
||||||
if (!filters) {
|
if (!filters) {
|
||||||
return query
|
return query
|
||||||
}
|
}
|
||||||
|
const builder = this
|
||||||
filters = this.parseFilters({ ...filters })
|
filters = this.parseFilters({ ...filters })
|
||||||
const aliases = this.query.tableAliases
|
const aliases = this.query.tableAliases
|
||||||
// if all or specified in filters, then everything is an or
|
// if all or specified in filters, then everything is an or
|
||||||
const allOr = filters.allOr
|
const allOr = filters.allOr
|
||||||
const tableName =
|
const isSqlite = this.client === SqlClient.SQL_LITE
|
||||||
this.client === SqlClient.SQL_LITE ? this.table._id! : this.table.name
|
const tableName = isSqlite ? this.table._id! : this.table.name
|
||||||
|
|
||||||
function getTableAlias(name: string) {
|
function getTableAlias(name: string) {
|
||||||
const alias = aliases?.[name]
|
const alias = aliases?.[name]
|
||||||
|
@ -352,13 +449,33 @@ class InternalBuilder {
|
||||||
}
|
}
|
||||||
function iterate(
|
function iterate(
|
||||||
structure: AnySearchFilter,
|
structure: AnySearchFilter,
|
||||||
fn: (key: string, value: any) => void,
|
fn: (
|
||||||
complexKeyFn?: (key: string[], value: any) => void
|
query: Knex.QueryBuilder,
|
||||||
|
key: string,
|
||||||
|
value: any
|
||||||
|
) => Knex.QueryBuilder,
|
||||||
|
complexKeyFn?: (
|
||||||
|
query: Knex.QueryBuilder,
|
||||||
|
key: string[],
|
||||||
|
value: any
|
||||||
|
) => Knex.QueryBuilder
|
||||||
) {
|
) {
|
||||||
|
const handleRelationship = (
|
||||||
|
q: Knex.QueryBuilder,
|
||||||
|
key: string,
|
||||||
|
value: any
|
||||||
|
) => {
|
||||||
|
const [filterTableName, ...otherProperties] = key.split(".")
|
||||||
|
const property = otherProperties.join(".")
|
||||||
|
const alias = getTableAlias(filterTableName)
|
||||||
|
return fn(q, alias ? `${alias}.${property}` : property, value)
|
||||||
|
}
|
||||||
for (const key in structure) {
|
for (const key in structure) {
|
||||||
const value = structure[key]
|
const value = structure[key]
|
||||||
const updatedKey = dbCore.removeKeyNumbering(key)
|
const updatedKey = dbCore.removeKeyNumbering(key)
|
||||||
const isRelationshipField = updatedKey.includes(".")
|
const isRelationshipField = updatedKey.includes(".")
|
||||||
|
const shouldProcessRelationship =
|
||||||
|
opts?.relationship && isRelationshipField
|
||||||
|
|
||||||
let castedTypeValue
|
let castedTypeValue
|
||||||
if (
|
if (
|
||||||
|
@ -367,7 +484,8 @@ class InternalBuilder {
|
||||||
complexKeyFn
|
complexKeyFn
|
||||||
) {
|
) {
|
||||||
const alias = getTableAlias(tableName)
|
const alias = getTableAlias(tableName)
|
||||||
complexKeyFn(
|
query = complexKeyFn(
|
||||||
|
query,
|
||||||
castedTypeValue.id.map((x: string) =>
|
castedTypeValue.id.map((x: string) =>
|
||||||
alias ? `${alias}.${x}` : x
|
alias ? `${alias}.${x}` : x
|
||||||
),
|
),
|
||||||
|
@ -375,26 +493,29 @@ class InternalBuilder {
|
||||||
)
|
)
|
||||||
} else if (!isRelationshipField) {
|
} else if (!isRelationshipField) {
|
||||||
const alias = getTableAlias(tableName)
|
const alias = getTableAlias(tableName)
|
||||||
fn(alias ? `${alias}.${updatedKey}` : updatedKey, value)
|
query = fn(
|
||||||
}
|
query,
|
||||||
if (opts?.relationship && isRelationshipField) {
|
alias ? `${alias}.${updatedKey}` : updatedKey,
|
||||||
const [filterTableName, property] = updatedKey.split(".")
|
value
|
||||||
const alias = getTableAlias(filterTableName)
|
)
|
||||||
fn(alias ? `${alias}.${property}` : property, value)
|
} else if (shouldProcessRelationship) {
|
||||||
|
query = builder.addRelationshipForFilter(query, updatedKey, q => {
|
||||||
|
return handleRelationship(q, updatedKey, value)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const like = (key: string, value: any) => {
|
const like = (q: Knex.QueryBuilder, key: string, value: any) => {
|
||||||
const fuzzyOr = filters?.fuzzyOr
|
const fuzzyOr = filters?.fuzzyOr
|
||||||
const fnc = fuzzyOr || allOr ? "orWhere" : "where"
|
const fnc = fuzzyOr || allOr ? "orWhere" : "where"
|
||||||
// postgres supports ilike, nothing else does
|
// postgres supports ilike, nothing else does
|
||||||
if (this.client === SqlClient.POSTGRES) {
|
if (this.client === SqlClient.POSTGRES) {
|
||||||
query = query[fnc](key, "ilike", `%${value}%`)
|
return q[fnc](key, "ilike", `%${value}%`)
|
||||||
} else {
|
} else {
|
||||||
const rawFnc = `${fnc}Raw`
|
const rawFnc = `${fnc}Raw`
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
query = query[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [
|
return q[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [
|
||||||
`%${value.toLowerCase()}%`,
|
`%${value.toLowerCase()}%`,
|
||||||
])
|
])
|
||||||
}
|
}
|
||||||
|
@ -412,13 +533,13 @@ class InternalBuilder {
|
||||||
return `[${value.join(",")}]`
|
return `[${value.join(",")}]`
|
||||||
}
|
}
|
||||||
if (this.client === SqlClient.POSTGRES) {
|
if (this.client === SqlClient.POSTGRES) {
|
||||||
iterate(mode, (key, value) => {
|
iterate(mode, (q, key, value) => {
|
||||||
const wrap = any ? "" : "'"
|
const wrap = any ? "" : "'"
|
||||||
const op = any ? "\\?| array" : "@>"
|
const op = any ? "\\?| array" : "@>"
|
||||||
const fieldNames = key.split(/\./g)
|
const fieldNames = key.split(/\./g)
|
||||||
const table = fieldNames[0]
|
const table = fieldNames[0]
|
||||||
const col = fieldNames[1]
|
const col = fieldNames[1]
|
||||||
query = query[rawFnc](
|
return q[rawFnc](
|
||||||
`${not}COALESCE("${table}"."${col}"::jsonb ${op} ${wrap}${stringifyArray(
|
`${not}COALESCE("${table}"."${col}"::jsonb ${op} ${wrap}${stringifyArray(
|
||||||
value,
|
value,
|
||||||
any ? "'" : '"'
|
any ? "'" : '"'
|
||||||
|
@ -427,8 +548,8 @@ class InternalBuilder {
|
||||||
})
|
})
|
||||||
} else if (this.client === SqlClient.MY_SQL) {
|
} else if (this.client === SqlClient.MY_SQL) {
|
||||||
const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS"
|
const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS"
|
||||||
iterate(mode, (key, value) => {
|
iterate(mode, (q, key, value) => {
|
||||||
query = query[rawFnc](
|
return q[rawFnc](
|
||||||
`${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray(
|
`${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray(
|
||||||
value
|
value
|
||||||
)}'), FALSE)`
|
)}'), FALSE)`
|
||||||
|
@ -436,7 +557,7 @@ class InternalBuilder {
|
||||||
})
|
})
|
||||||
} else {
|
} else {
|
||||||
const andOr = mode === filters?.containsAny ? " OR " : " AND "
|
const andOr = mode === filters?.containsAny ? " OR " : " AND "
|
||||||
iterate(mode, (key, value) => {
|
iterate(mode, (q, key, value) => {
|
||||||
let statement = ""
|
let statement = ""
|
||||||
const identifier = this.quotedIdentifier(key)
|
const identifier = this.quotedIdentifier(key)
|
||||||
for (let i in value) {
|
for (let i in value) {
|
||||||
|
@ -451,16 +572,16 @@ class InternalBuilder {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (statement === "") {
|
if (statement === "") {
|
||||||
return
|
return q
|
||||||
}
|
}
|
||||||
|
|
||||||
if (not) {
|
if (not) {
|
||||||
query = query[rawFnc](
|
return q[rawFnc](
|
||||||
`(NOT (${statement}) OR ${identifier} IS NULL)`,
|
`(NOT (${statement}) OR ${identifier} IS NULL)`,
|
||||||
value
|
value
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
query = query[rawFnc](statement, value)
|
return q[rawFnc](statement, value)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
@ -490,39 +611,39 @@ class InternalBuilder {
|
||||||
const fnc = allOr ? "orWhereIn" : "whereIn"
|
const fnc = allOr ? "orWhereIn" : "whereIn"
|
||||||
iterate(
|
iterate(
|
||||||
filters.oneOf,
|
filters.oneOf,
|
||||||
(key: string, array) => {
|
(q, key: string, array) => {
|
||||||
if (this.client === SqlClient.ORACLE) {
|
if (this.client === SqlClient.ORACLE) {
|
||||||
key = this.convertClobs(key)
|
key = this.convertClobs(key)
|
||||||
array = Array.isArray(array) ? array : [array]
|
array = Array.isArray(array) ? array : [array]
|
||||||
const binding = new Array(array.length).fill("?").join(",")
|
const binding = new Array(array.length).fill("?").join(",")
|
||||||
query = query.whereRaw(`${key} IN (${binding})`, array)
|
return q.whereRaw(`${key} IN (${binding})`, array)
|
||||||
} else {
|
} else {
|
||||||
query = query[fnc](key, Array.isArray(array) ? array : [array])
|
return q[fnc](key, Array.isArray(array) ? array : [array])
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
(key: string[], array) => {
|
(q, key: string[], array) => {
|
||||||
if (this.client === SqlClient.ORACLE) {
|
if (this.client === SqlClient.ORACLE) {
|
||||||
const keyStr = `(${key.map(k => this.convertClobs(k)).join(",")})`
|
const keyStr = `(${key.map(k => this.convertClobs(k)).join(",")})`
|
||||||
const binding = `(${array
|
const binding = `(${array
|
||||||
.map((a: any) => `(${new Array(a.length).fill("?").join(",")})`)
|
.map((a: any) => `(${new Array(a.length).fill("?").join(",")})`)
|
||||||
.join(",")})`
|
.join(",")})`
|
||||||
query = query.whereRaw(`${keyStr} IN ${binding}`, array.flat())
|
return q.whereRaw(`${keyStr} IN ${binding}`, array.flat())
|
||||||
} else {
|
} else {
|
||||||
query = query[fnc](key, Array.isArray(array) ? array : [array])
|
return q[fnc](key, Array.isArray(array) ? array : [array])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
if (filters.string) {
|
if (filters.string) {
|
||||||
iterate(filters.string, (key, value) => {
|
iterate(filters.string, (q, key, value) => {
|
||||||
const fnc = allOr ? "orWhere" : "where"
|
const fnc = allOr ? "orWhere" : "where"
|
||||||
// postgres supports ilike, nothing else does
|
// postgres supports ilike, nothing else does
|
||||||
if (this.client === SqlClient.POSTGRES) {
|
if (this.client === SqlClient.POSTGRES) {
|
||||||
query = query[fnc](key, "ilike", `${value}%`)
|
return q[fnc](key, "ilike", `${value}%`)
|
||||||
} else {
|
} else {
|
||||||
const rawFnc = `${fnc}Raw`
|
const rawFnc = `${fnc}Raw`
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
query = query[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [
|
return q[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [
|
||||||
`${value.toLowerCase()}%`,
|
`${value.toLowerCase()}%`,
|
||||||
])
|
])
|
||||||
}
|
}
|
||||||
|
@ -532,7 +653,7 @@ class InternalBuilder {
|
||||||
iterate(filters.fuzzy, like)
|
iterate(filters.fuzzy, like)
|
||||||
}
|
}
|
||||||
if (filters.range) {
|
if (filters.range) {
|
||||||
iterate(filters.range, (key, value) => {
|
iterate(filters.range, (q, key, value) => {
|
||||||
const isEmptyObject = (val: any) => {
|
const isEmptyObject = (val: any) => {
|
||||||
return (
|
return (
|
||||||
val &&
|
val &&
|
||||||
|
@ -561,97 +682,93 @@ class InternalBuilder {
|
||||||
schema?.type === FieldType.BIGINT &&
|
schema?.type === FieldType.BIGINT &&
|
||||||
this.client === SqlClient.SQL_LITE
|
this.client === SqlClient.SQL_LITE
|
||||||
) {
|
) {
|
||||||
query = query.whereRaw(
|
return q.whereRaw(
|
||||||
`CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
|
`CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
|
||||||
[value.low, value.high]
|
[value.low, value.high]
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
const fnc = allOr ? "orWhereBetween" : "whereBetween"
|
const fnc = allOr ? "orWhereBetween" : "whereBetween"
|
||||||
query = query[fnc](key, [value.low, value.high])
|
return q[fnc](key, [value.low, value.high])
|
||||||
}
|
}
|
||||||
} else if (lowValid) {
|
} else if (lowValid) {
|
||||||
if (
|
if (
|
||||||
schema?.type === FieldType.BIGINT &&
|
schema?.type === FieldType.BIGINT &&
|
||||||
this.client === SqlClient.SQL_LITE
|
this.client === SqlClient.SQL_LITE
|
||||||
) {
|
) {
|
||||||
query = query.whereRaw(
|
return q.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [
|
||||||
`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`,
|
value.low,
|
||||||
[value.low]
|
])
|
||||||
)
|
|
||||||
} else {
|
} else {
|
||||||
const fnc = allOr ? "orWhere" : "where"
|
const fnc = allOr ? "orWhere" : "where"
|
||||||
query = query[fnc](key, ">=", value.low)
|
return q[fnc](key, ">=", value.low)
|
||||||
}
|
}
|
||||||
} else if (highValid) {
|
} else if (highValid) {
|
||||||
if (
|
if (
|
||||||
schema?.type === FieldType.BIGINT &&
|
schema?.type === FieldType.BIGINT &&
|
||||||
this.client === SqlClient.SQL_LITE
|
this.client === SqlClient.SQL_LITE
|
||||||
) {
|
) {
|
||||||
query = query.whereRaw(
|
return q.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [
|
||||||
`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`,
|
value.high,
|
||||||
[value.high]
|
])
|
||||||
)
|
|
||||||
} else {
|
} else {
|
||||||
const fnc = allOr ? "orWhere" : "where"
|
const fnc = allOr ? "orWhere" : "where"
|
||||||
query = query[fnc](key, "<=", value.high)
|
return q[fnc](key, "<=", value.high)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return q
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if (filters.equal) {
|
if (filters.equal) {
|
||||||
iterate(filters.equal, (key, value) => {
|
iterate(filters.equal, (q, key, value) => {
|
||||||
const fnc = allOr ? "orWhereRaw" : "whereRaw"
|
const fnc = allOr ? "orWhereRaw" : "whereRaw"
|
||||||
if (this.client === SqlClient.MS_SQL) {
|
if (this.client === SqlClient.MS_SQL) {
|
||||||
query = query[fnc](
|
return q[fnc](
|
||||||
`CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 1`,
|
`CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 1`,
|
||||||
[value]
|
[value]
|
||||||
)
|
)
|
||||||
} else if (this.client === SqlClient.ORACLE) {
|
} else if (this.client === SqlClient.ORACLE) {
|
||||||
const identifier = this.convertClobs(key)
|
const identifier = this.convertClobs(key)
|
||||||
query = query[fnc](
|
return q[fnc](`(${identifier} IS NOT NULL AND ${identifier} = ?)`, [
|
||||||
`(${identifier} IS NOT NULL AND ${identifier} = ?)`,
|
value,
|
||||||
[value]
|
])
|
||||||
)
|
|
||||||
} else {
|
} else {
|
||||||
query = query[fnc](
|
return q[fnc](`COALESCE(${this.quotedIdentifier(key)} = ?, FALSE)`, [
|
||||||
`COALESCE(${this.quotedIdentifier(key)} = ?, FALSE)`,
|
value,
|
||||||
[value]
|
])
|
||||||
)
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if (filters.notEqual) {
|
if (filters.notEqual) {
|
||||||
iterate(filters.notEqual, (key, value) => {
|
iterate(filters.notEqual, (q, key, value) => {
|
||||||
const fnc = allOr ? "orWhereRaw" : "whereRaw"
|
const fnc = allOr ? "orWhereRaw" : "whereRaw"
|
||||||
if (this.client === SqlClient.MS_SQL) {
|
if (this.client === SqlClient.MS_SQL) {
|
||||||
query = query[fnc](
|
return q[fnc](
|
||||||
`CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 0`,
|
`CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 0`,
|
||||||
[value]
|
[value]
|
||||||
)
|
)
|
||||||
} else if (this.client === SqlClient.ORACLE) {
|
} else if (this.client === SqlClient.ORACLE) {
|
||||||
const identifier = this.convertClobs(key)
|
const identifier = this.convertClobs(key)
|
||||||
query = query[fnc](
|
return q[fnc](
|
||||||
`(${identifier} IS NOT NULL AND ${identifier} != ?) OR ${identifier} IS NULL`,
|
`(${identifier} IS NOT NULL AND ${identifier} != ?) OR ${identifier} IS NULL`,
|
||||||
[value]
|
[value]
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
query = query[fnc](
|
return q[fnc](`COALESCE(${this.quotedIdentifier(key)} != ?, TRUE)`, [
|
||||||
`COALESCE(${this.quotedIdentifier(key)} != ?, TRUE)`,
|
value,
|
||||||
[value]
|
])
|
||||||
)
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if (filters.empty) {
|
if (filters.empty) {
|
||||||
iterate(filters.empty, key => {
|
iterate(filters.empty, (q, key) => {
|
||||||
const fnc = allOr ? "orWhereNull" : "whereNull"
|
const fnc = allOr ? "orWhereNull" : "whereNull"
|
||||||
query = query[fnc](key)
|
return q[fnc](key)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if (filters.notEmpty) {
|
if (filters.notEmpty) {
|
||||||
iterate(filters.notEmpty, key => {
|
iterate(filters.notEmpty, (q, key) => {
|
||||||
const fnc = allOr ? "orWhereNotNull" : "whereNotNull"
|
const fnc = allOr ? "orWhereNotNull" : "whereNotNull"
|
||||||
query = query[fnc](key)
|
return q[fnc](key)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if (filters.contains) {
|
if (filters.contains) {
|
||||||
|
@ -745,16 +862,222 @@ class InternalBuilder {
|
||||||
return withSchema
|
return withSchema
|
||||||
}
|
}
|
||||||
|
|
||||||
|
addJsonRelationships(
|
||||||
|
query: Knex.QueryBuilder,
|
||||||
|
fromTable: string,
|
||||||
|
relationships: RelationshipsJson[]
|
||||||
|
): Knex.QueryBuilder {
|
||||||
|
const sqlClient = this.client
|
||||||
|
const knex = this.knex
|
||||||
|
const { resource, tableAliases: aliases, endpoint } = this.query
|
||||||
|
const fields = resource?.fields || []
|
||||||
|
const jsonField = (field: string) => {
|
||||||
|
const parts = field.split(".")
|
||||||
|
let tableField: string, unaliased: string
|
||||||
|
if (parts.length > 1) {
|
||||||
|
const alias = parts.shift()!
|
||||||
|
unaliased = parts.join(".")
|
||||||
|
tableField = `${this.quote(alias)}.${this.quote(unaliased)}`
|
||||||
|
} else {
|
||||||
|
unaliased = parts.join(".")
|
||||||
|
tableField = this.quote(unaliased)
|
||||||
|
}
|
||||||
|
let separator = ","
|
||||||
|
switch (sqlClient) {
|
||||||
|
case SqlClient.ORACLE:
|
||||||
|
separator = " VALUE "
|
||||||
|
break
|
||||||
|
case SqlClient.MS_SQL:
|
||||||
|
separator = ":"
|
||||||
|
}
|
||||||
|
return `'${unaliased}'${separator}${tableField}`
|
||||||
|
}
|
||||||
|
for (let relationship of relationships) {
|
||||||
|
const {
|
||||||
|
tableName: toTable,
|
||||||
|
through: throughTable,
|
||||||
|
to: toKey,
|
||||||
|
from: fromKey,
|
||||||
|
fromPrimary,
|
||||||
|
toPrimary,
|
||||||
|
} = relationship
|
||||||
|
// skip invalid relationships
|
||||||
|
if (!toTable || !fromTable) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
const toAlias = aliases?.[toTable] || toTable,
|
||||||
|
fromAlias = aliases?.[fromTable] || fromTable
|
||||||
|
let toTableWithSchema = this.tableNameWithSchema(toTable, {
|
||||||
|
alias: toAlias,
|
||||||
|
schema: endpoint.schema,
|
||||||
|
})
|
||||||
|
let relationshipFields = fields.filter(
|
||||||
|
field => field.split(".")[0] === toAlias
|
||||||
|
)
|
||||||
|
if (this.client === SqlClient.SQL_LITE) {
|
||||||
|
relationshipFields = relationshipFields.slice(
|
||||||
|
0,
|
||||||
|
MAX_SQS_RELATIONSHIP_FIELDS
|
||||||
|
)
|
||||||
|
}
|
||||||
|
const fieldList: string = relationshipFields
|
||||||
|
.map(field => jsonField(field))
|
||||||
|
.join(",")
|
||||||
|
// SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax
|
||||||
|
// it reduces the result set rather than limiting how much data it filters over
|
||||||
|
const primaryKey = `${toAlias}.${toPrimary || toKey}`
|
||||||
|
let subQuery: Knex.QueryBuilder = knex
|
||||||
|
.from(toTableWithSchema)
|
||||||
|
.limit(getRelationshipLimit())
|
||||||
|
// add sorting to get consistent order
|
||||||
|
.orderBy(primaryKey)
|
||||||
|
|
||||||
|
// many-to-many relationship with junction table
|
||||||
|
if (throughTable && toPrimary && fromPrimary) {
|
||||||
|
const throughAlias = aliases?.[throughTable] || throughTable
|
||||||
|
let throughTableWithSchema = this.tableNameWithSchema(throughTable, {
|
||||||
|
alias: throughAlias,
|
||||||
|
schema: endpoint.schema,
|
||||||
|
})
|
||||||
|
subQuery = subQuery
|
||||||
|
.join(throughTableWithSchema, function () {
|
||||||
|
this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`)
|
||||||
|
})
|
||||||
|
.where(
|
||||||
|
`${throughAlias}.${fromKey}`,
|
||||||
|
"=",
|
||||||
|
knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`))
|
||||||
|
)
|
||||||
|
}
|
||||||
|
// one-to-many relationship with foreign key
|
||||||
|
else {
|
||||||
|
subQuery = subQuery.where(
|
||||||
|
`${toAlias}.${toKey}`,
|
||||||
|
"=",
|
||||||
|
knex.raw(this.quotedIdentifier(`${fromAlias}.${fromKey}`))
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const standardWrap = (select: string): Knex.QueryBuilder => {
|
||||||
|
subQuery = subQuery.select(`${toAlias}.*`)
|
||||||
|
// @ts-ignore - the from alias syntax isn't in Knex typing
|
||||||
|
return knex.select(knex.raw(select)).from({
|
||||||
|
[toAlias]: subQuery,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
let wrapperQuery: Knex.QueryBuilder | Knex.Raw
|
||||||
|
switch (sqlClient) {
|
||||||
|
case SqlClient.SQL_LITE:
|
||||||
|
// need to check the junction table document is to the right column, this is just for SQS
|
||||||
|
subQuery = this.addJoinFieldCheck(subQuery, relationship)
|
||||||
|
wrapperQuery = standardWrap(
|
||||||
|
`json_group_array(json_object(${fieldList}))`
|
||||||
|
)
|
||||||
|
break
|
||||||
|
case SqlClient.POSTGRES:
|
||||||
|
wrapperQuery = standardWrap(
|
||||||
|
`json_agg(json_build_object(${fieldList}))`
|
||||||
|
)
|
||||||
|
break
|
||||||
|
case SqlClient.MY_SQL:
|
||||||
|
wrapperQuery = subQuery.select(
|
||||||
|
knex.raw(`json_arrayagg(json_object(${fieldList}))`)
|
||||||
|
)
|
||||||
|
break
|
||||||
|
case SqlClient.ORACLE:
|
||||||
|
wrapperQuery = standardWrap(
|
||||||
|
`json_arrayagg(json_object(${fieldList}))`
|
||||||
|
)
|
||||||
|
break
|
||||||
|
case SqlClient.MS_SQL:
|
||||||
|
wrapperQuery = knex.raw(
|
||||||
|
`(SELECT ${this.quote(toAlias)} = (${knex
|
||||||
|
.select(`${fromAlias}.*`)
|
||||||
|
// @ts-ignore - from alias syntax not TS supported
|
||||||
|
.from({
|
||||||
|
[fromAlias]: subQuery.select(`${toAlias}.*`),
|
||||||
|
})} FOR JSON PATH))`
|
||||||
|
)
|
||||||
|
break
|
||||||
|
default:
|
||||||
|
throw new Error(`JSON relationships not implement for ${sqlClient}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
query = query.select({ [relationship.column]: wrapperQuery })
|
||||||
|
}
|
||||||
|
return query
|
||||||
|
}
|
||||||
|
|
||||||
|
addJoin(
|
||||||
|
query: Knex.QueryBuilder,
|
||||||
|
tables: { from: string; to: string; through?: string },
|
||||||
|
columns: {
|
||||||
|
from?: string
|
||||||
|
to?: string
|
||||||
|
fromPrimary?: string
|
||||||
|
toPrimary?: string
|
||||||
|
}[]
|
||||||
|
): Knex.QueryBuilder {
|
||||||
|
const { tableAliases: aliases, endpoint } = this.query
|
||||||
|
const schema = endpoint.schema
|
||||||
|
const toTable = tables.to,
|
||||||
|
fromTable = tables.from,
|
||||||
|
throughTable = tables.through
|
||||||
|
const toAlias = aliases?.[toTable] || toTable,
|
||||||
|
throughAlias = (throughTable && aliases?.[throughTable]) || throughTable,
|
||||||
|
fromAlias = aliases?.[fromTable] || fromTable
|
||||||
|
let toTableWithSchema = this.tableNameWithSchema(toTable, {
|
||||||
|
alias: toAlias,
|
||||||
|
schema,
|
||||||
|
})
|
||||||
|
let throughTableWithSchema = throughTable
|
||||||
|
? this.tableNameWithSchema(throughTable, {
|
||||||
|
alias: throughAlias,
|
||||||
|
schema,
|
||||||
|
})
|
||||||
|
: undefined
|
||||||
|
if (!throughTable) {
|
||||||
|
// @ts-ignore
|
||||||
|
query = query.leftJoin(toTableWithSchema, function () {
|
||||||
|
for (let relationship of columns) {
|
||||||
|
const from = relationship.from,
|
||||||
|
to = relationship.to
|
||||||
|
// @ts-ignore
|
||||||
|
this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
query = query
|
||||||
|
// @ts-ignore
|
||||||
|
.leftJoin(throughTableWithSchema, function () {
|
||||||
|
for (let relationship of columns) {
|
||||||
|
const fromPrimary = relationship.fromPrimary
|
||||||
|
const from = relationship.from
|
||||||
|
// @ts-ignore
|
||||||
|
this.orOn(
|
||||||
|
`${fromAlias}.${fromPrimary}`,
|
||||||
|
"=",
|
||||||
|
`${throughAlias}.${from}`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.leftJoin(toTableWithSchema, function () {
|
||||||
|
for (let relationship of columns) {
|
||||||
|
const toPrimary = relationship.toPrimary
|
||||||
|
const to = relationship.to
|
||||||
|
// @ts-ignore
|
||||||
|
this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return query
|
||||||
|
}
|
||||||
|
|
||||||
addRelationships(
|
addRelationships(
|
||||||
query: Knex.QueryBuilder,
|
query: Knex.QueryBuilder,
|
||||||
fromTable: string,
|
fromTable: string,
|
||||||
relationships: RelationshipsJson[] | undefined,
|
relationships: RelationshipsJson[]
|
||||||
schema: string | undefined,
|
|
||||||
aliases?: Record<string, string>
|
|
||||||
): Knex.QueryBuilder {
|
): Knex.QueryBuilder {
|
||||||
if (!relationships) {
|
|
||||||
return query
|
|
||||||
}
|
|
||||||
const tableSets: Record<string, [RelationshipsJson]> = {}
|
const tableSets: Record<string, [RelationshipsJson]> = {}
|
||||||
// aggregate into table sets (all the same to tables)
|
// aggregate into table sets (all the same to tables)
|
||||||
for (let relationship of relationships) {
|
for (let relationship of relationships) {
|
||||||
|
@ -774,52 +1097,16 @@ class InternalBuilder {
|
||||||
}
|
}
|
||||||
for (let [key, relationships] of Object.entries(tableSets)) {
|
for (let [key, relationships] of Object.entries(tableSets)) {
|
||||||
const { toTable, throughTable } = JSON.parse(key)
|
const { toTable, throughTable } = JSON.parse(key)
|
||||||
const toAlias = aliases?.[toTable] || toTable,
|
query = this.addJoin(
|
||||||
throughAlias = aliases?.[throughTable] || throughTable,
|
query,
|
||||||
fromAlias = aliases?.[fromTable] || fromTable
|
{
|
||||||
let toTableWithSchema = this.tableNameWithSchema(toTable, {
|
from: fromTable,
|
||||||
alias: toAlias,
|
to: toTable,
|
||||||
schema,
|
through: throughTable,
|
||||||
})
|
},
|
||||||
let throughTableWithSchema = this.tableNameWithSchema(throughTable, {
|
relationships
|
||||||
alias: throughAlias,
|
|
||||||
schema,
|
|
||||||
})
|
|
||||||
if (!throughTable) {
|
|
||||||
// @ts-ignore
|
|
||||||
query = query.leftJoin(toTableWithSchema, function () {
|
|
||||||
for (let relationship of relationships) {
|
|
||||||
const from = relationship.from,
|
|
||||||
to = relationship.to
|
|
||||||
// @ts-ignore
|
|
||||||
this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
query = query
|
|
||||||
// @ts-ignore
|
|
||||||
.leftJoin(throughTableWithSchema, function () {
|
|
||||||
for (let relationship of relationships) {
|
|
||||||
const fromPrimary = relationship.fromPrimary
|
|
||||||
const from = relationship.from
|
|
||||||
// @ts-ignore
|
|
||||||
this.orOn(
|
|
||||||
`${fromAlias}.${fromPrimary}`,
|
|
||||||
"=",
|
|
||||||
`${throughAlias}.${from}`
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
})
|
|
||||||
.leftJoin(toTableWithSchema, function () {
|
|
||||||
for (let relationship of relationships) {
|
|
||||||
const toPrimary = relationship.toPrimary
|
|
||||||
const to = relationship.to
|
|
||||||
// @ts-ignore
|
|
||||||
this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return query
|
return query
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -924,8 +1211,7 @@ class InternalBuilder {
|
||||||
limits?: { base: number; query: number }
|
limits?: { base: number; query: number }
|
||||||
} = {}
|
} = {}
|
||||||
): Knex.QueryBuilder {
|
): Knex.QueryBuilder {
|
||||||
let { endpoint, filters, paginate, relationships, tableAliases } =
|
let { endpoint, filters, paginate, relationships } = this.query
|
||||||
this.query
|
|
||||||
const { limits } = opts
|
const { limits } = opts
|
||||||
const counting = endpoint.operation === Operation.COUNT
|
const counting = endpoint.operation === Operation.COUNT
|
||||||
|
|
||||||
|
@ -957,42 +1243,19 @@ class InternalBuilder {
|
||||||
if (foundOffset != null) {
|
if (foundOffset != null) {
|
||||||
query = query.offset(foundOffset)
|
query = query.offset(foundOffset)
|
||||||
}
|
}
|
||||||
// add sorting to pre-query
|
}
|
||||||
// no point in sorting when counting
|
|
||||||
|
// if counting, use distinct count, else select
|
||||||
|
query = !counting
|
||||||
|
? query.select(this.generateSelectStatement())
|
||||||
|
: this.addDistinctCount(query)
|
||||||
|
// have to add after as well (this breaks MS-SQL)
|
||||||
|
if (!counting) {
|
||||||
query = this.addSorting(query)
|
query = this.addSorting(query)
|
||||||
}
|
}
|
||||||
// add filters to the query (where)
|
|
||||||
query = this.addFilters(query, filters)
|
|
||||||
|
|
||||||
const alias = tableAliases?.[tableName] || tableName
|
|
||||||
let preQuery: Knex.QueryBuilder = this.knex({
|
|
||||||
// the typescript definition for the knex constructor doesn't support this
|
|
||||||
// syntax, but it is the only way to alias a pre-query result as part of
|
|
||||||
// a query - there is an alias dictionary type, but it assumes it can only
|
|
||||||
// be a table name, not a pre-query
|
|
||||||
[alias]: query as any,
|
|
||||||
})
|
|
||||||
// if counting, use distinct count, else select
|
|
||||||
preQuery = !counting
|
|
||||||
? preQuery.select(this.generateSelectStatement())
|
|
||||||
: this.addDistinctCount(preQuery)
|
|
||||||
// have to add after as well (this breaks MS-SQL)
|
|
||||||
if (this.client !== SqlClient.MS_SQL && !counting) {
|
|
||||||
preQuery = this.addSorting(preQuery)
|
|
||||||
}
|
|
||||||
// handle joins
|
// handle joins
|
||||||
query = this.addRelationships(
|
if (relationships) {
|
||||||
preQuery,
|
query = this.addJsonRelationships(query, tableName, relationships)
|
||||||
tableName,
|
|
||||||
relationships,
|
|
||||||
endpoint.schema,
|
|
||||||
tableAliases
|
|
||||||
)
|
|
||||||
|
|
||||||
// add a base limit over the whole query
|
|
||||||
// if counting we can't set this limit
|
|
||||||
if (limits?.base) {
|
|
||||||
query = query.limit(limits.base)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return this.addFilters(query, filters, { relationship: true })
|
return this.addFilters(query, filters, { relationship: true })
|
||||||
|
|
|
@ -1,4 +1,11 @@
|
||||||
import { DocumentType, SqlQuery, Table, TableSourceType } from "@budibase/types"
|
import {
|
||||||
|
DocumentType,
|
||||||
|
ManyToManyRelationshipJson,
|
||||||
|
RelationshipsJson,
|
||||||
|
SqlQuery,
|
||||||
|
Table,
|
||||||
|
TableSourceType,
|
||||||
|
} from "@budibase/types"
|
||||||
import { DEFAULT_BB_DATASOURCE_ID } from "../constants"
|
import { DEFAULT_BB_DATASOURCE_ID } from "../constants"
|
||||||
import { Knex } from "knex"
|
import { Knex } from "knex"
|
||||||
import { SEPARATOR } from "../db"
|
import { SEPARATOR } from "../db"
|
||||||
|
@ -163,3 +170,24 @@ export function sqlLog(client: string, query: string, values?: any[]) {
|
||||||
}
|
}
|
||||||
console.log(string)
|
console.log(string)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isValidManyToManyRelationship(
|
||||||
|
relationship: RelationshipsJson
|
||||||
|
): relationship is ManyToManyRelationshipJson {
|
||||||
|
return (
|
||||||
|
!!relationship.through &&
|
||||||
|
!!relationship.fromPrimary &&
|
||||||
|
!!relationship.from &&
|
||||||
|
!!relationship.toPrimary &&
|
||||||
|
!!relationship.to
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function validateManyToMany(
|
||||||
|
relationship: RelationshipsJson
|
||||||
|
): ManyToManyRelationshipJson | undefined {
|
||||||
|
if (isValidManyToManyRelationship(relationship)) {
|
||||||
|
return relationship
|
||||||
|
}
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
MSSQL_SHA=sha256:c4369c38385eba011c10906dc8892425831275bb035d5ce69656da8e29de50d8
|
MSSQL_SHA=sha256:3b913841850a4d57fcfcb798be06acc88ea0f2acc5418bc0c140a43e91c4a545
|
||||||
MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe
|
MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe
|
||||||
POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e
|
POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e
|
||||||
MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d
|
MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d
|
||||||
|
|
|
@ -1,6 +1,10 @@
|
||||||
// need to handle table name + field or just field, depending on if relationships used
|
// need to handle table name + field or just field, depending on if relationships used
|
||||||
import { FieldType, Row, Table } from "@budibase/types"
|
import { FieldSchema, FieldType, Row, Table, JsonTypes } from "@budibase/types"
|
||||||
import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
|
import {
|
||||||
|
helpers,
|
||||||
|
PROTECTED_EXTERNAL_COLUMNS,
|
||||||
|
PROTECTED_INTERNAL_COLUMNS,
|
||||||
|
} from "@budibase/shared-core"
|
||||||
import { generateRowIdField } from "../../../../integrations/utils"
|
import { generateRowIdField } from "../../../../integrations/utils"
|
||||||
|
|
||||||
function extractFieldValue({
|
function extractFieldValue({
|
||||||
|
@ -58,14 +62,32 @@ export function generateIdForRow(
|
||||||
return generateRowIdField(idParts)
|
return generateRowIdField(idParts)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function fixJsonTypes(row: Row, table: Table) {
|
||||||
|
for (let [fieldName, schema] of Object.entries(table.schema)) {
|
||||||
|
if (JsonTypes.includes(schema.type) && typeof row[fieldName] === "string") {
|
||||||
|
try {
|
||||||
|
row[fieldName] = JSON.parse(row[fieldName])
|
||||||
|
} catch (err) {
|
||||||
|
if (!helpers.schema.isDeprecatedSingleUserColumn(schema)) {
|
||||||
|
// couldn't convert back to array, ignore
|
||||||
|
delete row[fieldName]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return row
|
||||||
|
}
|
||||||
|
|
||||||
export function basicProcessing({
|
export function basicProcessing({
|
||||||
row,
|
row,
|
||||||
table,
|
table,
|
||||||
|
tables,
|
||||||
isLinked,
|
isLinked,
|
||||||
sqs,
|
sqs,
|
||||||
}: {
|
}: {
|
||||||
row: Row
|
row: Row
|
||||||
table: Table
|
table: Table
|
||||||
|
tables: Table[]
|
||||||
isLinked: boolean
|
isLinked: boolean
|
||||||
sqs?: boolean
|
sqs?: boolean
|
||||||
}): Row {
|
}): Row {
|
||||||
|
@ -82,16 +104,18 @@ export function basicProcessing({
|
||||||
value = value.toString()
|
value = value.toString()
|
||||||
}
|
}
|
||||||
// all responses include "select col as table.col" so that overlaps are handled
|
// all responses include "select col as table.col" so that overlaps are handled
|
||||||
if (value != null) {
|
else if (value != null) {
|
||||||
thisRow[fieldName] = value
|
thisRow[fieldName] = value
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
let columns: string[] = Object.keys(table.schema)
|
||||||
if (!sqs) {
|
if (!sqs) {
|
||||||
thisRow._id = generateIdForRow(row, table, isLinked)
|
thisRow._id = generateIdForRow(row, table, isLinked)
|
||||||
thisRow.tableId = table._id
|
thisRow.tableId = table._id
|
||||||
thisRow._rev = "rev"
|
thisRow._rev = "rev"
|
||||||
|
columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS)
|
||||||
} else {
|
} else {
|
||||||
const columns = Object.keys(table.schema)
|
columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS)
|
||||||
for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) {
|
for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) {
|
||||||
thisRow[internalColumn] = extractFieldValue({
|
thisRow[internalColumn] = extractFieldValue({
|
||||||
row,
|
row,
|
||||||
|
@ -101,24 +125,56 @@ export function basicProcessing({
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return thisRow
|
for (let col of columns) {
|
||||||
|
const schema: FieldSchema | undefined = table.schema[col]
|
||||||
|
if (schema?.type !== FieldType.LINK) {
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
const relatedTable = tables.find(tbl => tbl._id === schema.tableId)
|
||||||
export function fixArrayTypes(row: Row, table: Table) {
|
if (!relatedTable) {
|
||||||
for (let [fieldName, schema] of Object.entries(table.schema)) {
|
continue
|
||||||
if (
|
}
|
||||||
[FieldType.ARRAY, FieldType.BB_REFERENCE].includes(schema.type) &&
|
const value = extractFieldValue({
|
||||||
typeof row[fieldName] === "string"
|
row,
|
||||||
) {
|
tableName: table._id!,
|
||||||
try {
|
fieldName: col,
|
||||||
row[fieldName] = JSON.parse(row[fieldName])
|
isLinked,
|
||||||
} catch (err) {
|
})
|
||||||
if (!helpers.schema.isDeprecatedSingleUserColumn(schema)) {
|
const array: Row[] = Array.isArray(value)
|
||||||
// couldn't convert back to array, ignore
|
? value
|
||||||
delete row[fieldName]
|
: typeof value === "string"
|
||||||
|
? JSON.parse(value)
|
||||||
|
: undefined
|
||||||
|
if (array) {
|
||||||
|
thisRow[col] = array
|
||||||
|
// make sure all of them have an _id
|
||||||
|
if (Array.isArray(thisRow[col])) {
|
||||||
|
const sortField =
|
||||||
|
relatedTable.primaryDisplay || relatedTable.primary![0]!
|
||||||
|
thisRow[col] = (thisRow[col] as Row[])
|
||||||
|
.map(relatedRow =>
|
||||||
|
basicProcessing({
|
||||||
|
row: relatedRow,
|
||||||
|
table: relatedTable,
|
||||||
|
tables,
|
||||||
|
isLinked: false,
|
||||||
|
sqs,
|
||||||
|
})
|
||||||
|
)
|
||||||
|
.sort((a, b) => {
|
||||||
|
const aField = a?.[sortField],
|
||||||
|
bField = b?.[sortField]
|
||||||
|
if (!aField) {
|
||||||
|
return 1
|
||||||
|
} else if (!bField) {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
return aField.localeCompare
|
||||||
|
? aField.localeCompare(bField)
|
||||||
|
: aField - bField
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
return fixJsonTypes(thisRow, table)
|
||||||
return row
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,11 +7,9 @@ import {
|
||||||
ManyToManyRelationshipFieldMetadata,
|
ManyToManyRelationshipFieldMetadata,
|
||||||
RelationshipFieldMetadata,
|
RelationshipFieldMetadata,
|
||||||
RelationshipsJson,
|
RelationshipsJson,
|
||||||
Row,
|
|
||||||
Table,
|
Table,
|
||||||
} from "@budibase/types"
|
} from "@budibase/types"
|
||||||
import { breakExternalTableId } from "../../../../integrations/utils"
|
import { breakExternalTableId } from "../../../../integrations/utils"
|
||||||
import { basicProcessing } from "./basic"
|
|
||||||
import { generateJunctionTableID } from "../../../../db/utils"
|
import { generateJunctionTableID } from "../../../../db/utils"
|
||||||
|
|
||||||
type TableMap = Record<string, Table>
|
type TableMap = Record<string, Table>
|
||||||
|
@ -22,87 +20,6 @@ export function isManyToMany(
|
||||||
return !!(field as ManyToManyRelationshipFieldMetadata).through
|
return !!(field as ManyToManyRelationshipFieldMetadata).through
|
||||||
}
|
}
|
||||||
|
|
||||||
function isCorrectRelationship(
|
|
||||||
relationship: RelationshipsJson,
|
|
||||||
table1: Table,
|
|
||||||
table2: Table,
|
|
||||||
row: Row
|
|
||||||
): boolean {
|
|
||||||
const junctionTableId = generateJunctionTableID(table1._id!, table2._id!)
|
|
||||||
const possibleColumns = [
|
|
||||||
`${junctionTableId}.doc1.fieldName`,
|
|
||||||
`${junctionTableId}.doc2.fieldName`,
|
|
||||||
]
|
|
||||||
return !!possibleColumns.find(col => row[col] === relationship.column)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This iterates through the returned rows and works out what elements of the rows
|
|
||||||
* actually match up to another row (based on primary keys) - this is pretty specific
|
|
||||||
* to SQL and the way that SQL relationships are returned based on joins.
|
|
||||||
* This is complicated, but the idea is that when a SQL query returns all the relations
|
|
||||||
* will be separate rows, with all of the data in each row. We have to decipher what comes
|
|
||||||
* from where (which tables) and how to convert that into budibase columns.
|
|
||||||
*/
|
|
||||||
export async function updateRelationshipColumns(
|
|
||||||
table: Table,
|
|
||||||
tables: TableMap,
|
|
||||||
row: Row,
|
|
||||||
rows: { [key: string]: Row },
|
|
||||||
relationships: RelationshipsJson[],
|
|
||||||
opts?: { sqs?: boolean }
|
|
||||||
) {
|
|
||||||
const columns: { [key: string]: any } = {}
|
|
||||||
for (let relationship of relationships) {
|
|
||||||
const linkedTable = tables[relationship.tableName]
|
|
||||||
if (!linkedTable) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
const fromColumn = `${table.name}.${relationship.from}`
|
|
||||||
const toColumn = `${linkedTable.name}.${relationship.to}`
|
|
||||||
// this is important when working with multiple relationships
|
|
||||||
// between the same tables, don't want to overlap/multiply the relations
|
|
||||||
if (
|
|
||||||
!relationship.through &&
|
|
||||||
row[fromColumn]?.toString() !== row[toColumn]?.toString()
|
|
||||||
) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
let linked = basicProcessing({
|
|
||||||
row,
|
|
||||||
table: linkedTable,
|
|
||||||
isLinked: true,
|
|
||||||
sqs: opts?.sqs,
|
|
||||||
})
|
|
||||||
if (!linked._id) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
!opts?.sqs ||
|
|
||||||
isCorrectRelationship(relationship, table, linkedTable, row)
|
|
||||||
) {
|
|
||||||
columns[relationship.column] = linked
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (let [column, related] of Object.entries(columns)) {
|
|
||||||
if (!row._id) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
const rowId: string = row._id
|
|
||||||
if (!Array.isArray(rows[rowId][column])) {
|
|
||||||
rows[rowId][column] = []
|
|
||||||
}
|
|
||||||
// make sure relationship hasn't been found already
|
|
||||||
if (
|
|
||||||
!rows[rowId][column].find((relation: Row) => relation._id === related._id)
|
|
||||||
) {
|
|
||||||
rows[rowId][column].push(related)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return rows
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Gets the list of relationship JSON structures based on the columns in the table,
|
* Gets the list of relationship JSON structures based on the columns in the table,
|
||||||
* this will be used by the underlying library to build whatever relationship mechanism
|
* this will be used by the underlying library to build whatever relationship mechanism
|
||||||
|
|
|
@ -13,13 +13,8 @@ import {
|
||||||
processDates,
|
processDates,
|
||||||
processFormulas,
|
processFormulas,
|
||||||
} from "../../../../utilities/rowProcessor"
|
} from "../../../../utilities/rowProcessor"
|
||||||
import { isKnexEmptyReadResponse, updateRelationshipColumns } from "./sqlUtils"
|
import { isKnexEmptyReadResponse } from "./sqlUtils"
|
||||||
import {
|
import { basicProcessing, generateIdForRow, getInternalRowId } from "./basic"
|
||||||
basicProcessing,
|
|
||||||
generateIdForRow,
|
|
||||||
fixArrayTypes,
|
|
||||||
getInternalRowId,
|
|
||||||
} from "./basic"
|
|
||||||
import sdk from "../../../../sdk"
|
import sdk from "../../../../sdk"
|
||||||
import { processStringSync } from "@budibase/string-templates"
|
import { processStringSync } from "@budibase/string-templates"
|
||||||
import validateJs from "validate.js"
|
import validateJs from "validate.js"
|
||||||
|
@ -149,42 +144,18 @@ export async function sqlOutputProcessing(
|
||||||
rowId = generateIdForRow(row, table)
|
rowId = generateIdForRow(row, table)
|
||||||
row._id = rowId
|
row._id = rowId
|
||||||
}
|
}
|
||||||
// this is a relationship of some sort
|
const thisRow = basicProcessing({
|
||||||
if (finalRows[rowId]) {
|
|
||||||
finalRows = await updateRelationshipColumns(
|
|
||||||
table,
|
|
||||||
tables,
|
|
||||||
row,
|
|
||||||
finalRows,
|
|
||||||
relationships,
|
|
||||||
opts
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
const thisRow = fixArrayTypes(
|
|
||||||
basicProcessing({
|
|
||||||
row,
|
row,
|
||||||
table,
|
table,
|
||||||
|
tables: Object.values(tables),
|
||||||
isLinked: false,
|
isLinked: false,
|
||||||
sqs: opts?.sqs,
|
sqs: opts?.sqs,
|
||||||
}),
|
})
|
||||||
table
|
|
||||||
)
|
|
||||||
if (thisRow._id == null) {
|
if (thisRow._id == null) {
|
||||||
throw new Error("Unable to generate row ID for SQL rows")
|
throw new Error("Unable to generate row ID for SQL rows")
|
||||||
}
|
}
|
||||||
|
|
||||||
finalRows[thisRow._id] = fixBooleanFields({ row: thisRow, table })
|
finalRows[thisRow._id] = fixBooleanFields({ row: thisRow, table })
|
||||||
|
|
||||||
// do this at end once its been added to the final rows
|
|
||||||
finalRows = await updateRelationshipColumns(
|
|
||||||
table,
|
|
||||||
tables,
|
|
||||||
row,
|
|
||||||
finalRows,
|
|
||||||
relationships,
|
|
||||||
opts
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// make sure all related rows are correct
|
// make sure all related rows are correct
|
||||||
|
|
|
@ -832,10 +832,12 @@ describe.each(
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
expect(res).toHaveLength(1)
|
expect(res).toHaveLength(1)
|
||||||
expect(res[0]).toEqual({
|
expect(res[0]).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
id: 2,
|
id: 2,
|
||||||
name: "two",
|
name: "two",
|
||||||
})
|
})
|
||||||
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
// this parameter really only impacts SQL queries
|
// this parameter really only impacts SQL queries
|
||||||
|
|
|
@ -9,10 +9,10 @@ import {
|
||||||
db as dbCore,
|
db as dbCore,
|
||||||
MAX_VALID_DATE,
|
MAX_VALID_DATE,
|
||||||
MIN_VALID_DATE,
|
MIN_VALID_DATE,
|
||||||
|
setEnv as setCoreEnv,
|
||||||
SQLITE_DESIGN_DOC_ID,
|
SQLITE_DESIGN_DOC_ID,
|
||||||
utils,
|
utils,
|
||||||
withEnv as withCoreEnv,
|
withEnv as withCoreEnv,
|
||||||
setEnv as setCoreEnv,
|
|
||||||
} from "@budibase/backend-core"
|
} from "@budibase/backend-core"
|
||||||
|
|
||||||
import * as setup from "./utilities"
|
import * as setup from "./utilities"
|
||||||
|
@ -1937,6 +1937,67 @@ describe.each([
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
isSql &&
|
||||||
|
describe("related formulas", () => {
|
||||||
|
beforeAll(async () => {
|
||||||
|
const arrayTable = await createTable(
|
||||||
|
{
|
||||||
|
name: { name: "name", type: FieldType.STRING },
|
||||||
|
array: {
|
||||||
|
name: "array",
|
||||||
|
type: FieldType.ARRAY,
|
||||||
|
constraints: {
|
||||||
|
type: JsonFieldSubType.ARRAY,
|
||||||
|
inclusion: ["option 1", "option 2"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"array"
|
||||||
|
)
|
||||||
|
table = await createTable(
|
||||||
|
{
|
||||||
|
relationship: {
|
||||||
|
type: FieldType.LINK,
|
||||||
|
relationshipType: RelationshipType.MANY_TO_ONE,
|
||||||
|
name: "relationship",
|
||||||
|
fieldName: "relate",
|
||||||
|
tableId: arrayTable._id!,
|
||||||
|
constraints: {
|
||||||
|
type: "array",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
formula: {
|
||||||
|
type: FieldType.FORMULA,
|
||||||
|
name: "formula",
|
||||||
|
formula: encodeJSBinding(
|
||||||
|
`let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.sort().join(",")`
|
||||||
|
),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"main"
|
||||||
|
)
|
||||||
|
const arrayRows = await Promise.all([
|
||||||
|
config.api.row.save(arrayTable._id!, {
|
||||||
|
name: "foo",
|
||||||
|
array: ["option 1"],
|
||||||
|
}),
|
||||||
|
config.api.row.save(arrayTable._id!, {
|
||||||
|
name: "bar",
|
||||||
|
array: ["option 2"],
|
||||||
|
}),
|
||||||
|
])
|
||||||
|
await Promise.all([
|
||||||
|
config.api.row.save(table._id!, {
|
||||||
|
relationship: [arrayRows[0]._id, arrayRows[1]._id],
|
||||||
|
}),
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("formula is correct with relationship arrays", async () => {
|
||||||
|
await expectQuery({}).toContain([{ formula: "option 1,option 2" }])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
describe("user", () => {
|
describe("user", () => {
|
||||||
let user1: User
|
let user1: User
|
||||||
let user2: User
|
let user2: User
|
||||||
|
@ -2690,81 +2751,6 @@ describe.each([
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
isSql &&
|
|
||||||
describe("pagination edge case with relationships", () => {
|
|
||||||
let mainRows: Row[] = []
|
|
||||||
|
|
||||||
beforeAll(async () => {
|
|
||||||
const toRelateTable = await createTable({
|
|
||||||
name: {
|
|
||||||
name: "name",
|
|
||||||
type: FieldType.STRING,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
table = await createTable({
|
|
||||||
name: {
|
|
||||||
name: "name",
|
|
||||||
type: FieldType.STRING,
|
|
||||||
},
|
|
||||||
rel: {
|
|
||||||
name: "rel",
|
|
||||||
type: FieldType.LINK,
|
|
||||||
relationshipType: RelationshipType.MANY_TO_ONE,
|
|
||||||
tableId: toRelateTable._id!,
|
|
||||||
fieldName: "rel",
|
|
||||||
},
|
|
||||||
})
|
|
||||||
const relatedRows = await Promise.all([
|
|
||||||
config.api.row.save(toRelateTable._id!, { name: "tag 1" }),
|
|
||||||
config.api.row.save(toRelateTable._id!, { name: "tag 2" }),
|
|
||||||
config.api.row.save(toRelateTable._id!, { name: "tag 3" }),
|
|
||||||
config.api.row.save(toRelateTable._id!, { name: "tag 4" }),
|
|
||||||
config.api.row.save(toRelateTable._id!, { name: "tag 5" }),
|
|
||||||
config.api.row.save(toRelateTable._id!, { name: "tag 6" }),
|
|
||||||
])
|
|
||||||
mainRows = await Promise.all([
|
|
||||||
config.api.row.save(table._id!, {
|
|
||||||
name: "product 1",
|
|
||||||
rel: relatedRows.map(row => row._id),
|
|
||||||
}),
|
|
||||||
config.api.row.save(table._id!, {
|
|
||||||
name: "product 2",
|
|
||||||
rel: [],
|
|
||||||
}),
|
|
||||||
config.api.row.save(table._id!, {
|
|
||||||
name: "product 3",
|
|
||||||
rel: [],
|
|
||||||
}),
|
|
||||||
])
|
|
||||||
})
|
|
||||||
|
|
||||||
it("can still page when the hard limit is hit", async () => {
|
|
||||||
await withCoreEnv(
|
|
||||||
{
|
|
||||||
SQL_MAX_ROWS: "6",
|
|
||||||
},
|
|
||||||
async () => {
|
|
||||||
const params: Omit<RowSearchParams, "tableId"> = {
|
|
||||||
query: {},
|
|
||||||
paginate: true,
|
|
||||||
limit: 3,
|
|
||||||
sort: "name",
|
|
||||||
sortType: SortType.STRING,
|
|
||||||
sortOrder: SortOrder.ASCENDING,
|
|
||||||
}
|
|
||||||
const page1 = await expectSearch(params).toContain([mainRows[0]])
|
|
||||||
expect(page1.hasNextPage).toBe(true)
|
|
||||||
expect(page1.bookmark).toBeDefined()
|
|
||||||
const page2 = await expectSearch({
|
|
||||||
...params,
|
|
||||||
bookmark: page1.bookmark,
|
|
||||||
}).toContain([mainRows[1], mainRows[2]])
|
|
||||||
expect(page2.hasNextPage).toBe(false)
|
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
isSql &&
|
isSql &&
|
||||||
describe("primaryDisplay", () => {
|
describe("primaryDisplay", () => {
|
||||||
beforeAll(async () => {
|
beforeAll(async () => {
|
||||||
|
|
|
@ -112,6 +112,7 @@ const environment = {
|
||||||
parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) ||
|
parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) ||
|
||||||
DEFAULTS.JS_RUNNER_MEMORY_LIMIT,
|
DEFAULTS.JS_RUNNER_MEMORY_LIMIT,
|
||||||
LOG_JS_ERRORS: process.env.LOG_JS_ERRORS,
|
LOG_JS_ERRORS: process.env.LOG_JS_ERRORS,
|
||||||
|
DISABLE_USER_SYNC: process.env.DISABLE_USER_SYNC,
|
||||||
// old
|
// old
|
||||||
CLIENT_ID: process.env.CLIENT_ID,
|
CLIENT_ID: process.env.CLIENT_ID,
|
||||||
_set(key: string, value: any) {
|
_set(key: string, value: any) {
|
||||||
|
|
|
@ -343,9 +343,9 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
|
||||||
err.number
|
err.number
|
||||||
)
|
)
|
||||||
if (readableMessage) {
|
if (readableMessage) {
|
||||||
throw new Error(readableMessage)
|
throw new Error(readableMessage, { cause: err })
|
||||||
} else {
|
} else {
|
||||||
throw new Error(err.message as string)
|
throw new Error(err.message as string, { cause: err })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -149,6 +149,7 @@ function generateManyRelationshipJson(config: { schema?: string } = {}) {
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("SQL query builder", () => {
|
describe("SQL query builder", () => {
|
||||||
|
const relationshipLimit = 500
|
||||||
const limit = 500
|
const limit = 500
|
||||||
const client = SqlClient.POSTGRES
|
const client = SqlClient.POSTGRES
|
||||||
let sql: any
|
let sql: any
|
||||||
|
@ -160,16 +161,16 @@ describe("SQL query builder", () => {
|
||||||
it("should add the schema to the LEFT JOIN", () => {
|
it("should add the schema to the LEFT JOIN", () => {
|
||||||
const query = sql._query(generateRelationshipJson({ schema: "production" }))
|
const query = sql._query(generateRelationshipJson({ schema: "production" }))
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: [500, 5000],
|
bindings: [relationshipLimit, limit],
|
||||||
sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "production"."brands" order by "test"."id" asc limit $1) as "brands" left join "production"."products" as "products" on "brands"."brand_id" = "products"."brand_id" order by "test"."id" asc limit $2`,
|
sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "production"."brands" order by "test"."id" asc limit $2`,
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
it("should handle if the schema is not present when doing a LEFT JOIN", () => {
|
it("should handle if the schema is not present when doing a LEFT JOIN", () => {
|
||||||
const query = sql._query(generateRelationshipJson())
|
const query = sql._query(generateRelationshipJson())
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: [500, 5000],
|
bindings: [relationshipLimit, limit],
|
||||||
sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "brands" order by "test"."id" asc limit $1) as "brands" left join "products" as "products" on "brands"."brand_id" = "products"."brand_id" order by "test"."id" asc limit $2`,
|
sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "brands" order by "test"."id" asc limit $2`,
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -178,8 +179,8 @@ describe("SQL query builder", () => {
|
||||||
generateManyRelationshipJson({ schema: "production" })
|
generateManyRelationshipJson({ schema: "production" })
|
||||||
)
|
)
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: [500, 5000],
|
bindings: [relationshipLimit, limit],
|
||||||
sql: `select "stores"."store_id" as "stores.store_id", "stores"."store_name" as "stores.store_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name" from (select * from "production"."stores" order by "test"."id" asc limit $1) as "stores" left join "production"."stocks" as "stocks" on "stores"."store_id" = "stocks"."store_id" left join "production"."products" as "products" on "products"."product_id" = "stocks"."product_id" order by "test"."id" asc limit $2`,
|
sql: `select "stores".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" inner join "production"."stocks" as "stocks" on "products"."product_id" = "stocks"."product_id" where "stocks"."store_id" = "stores"."store_id" order by "products"."product_id" asc limit $1) as "products") as "products" from "production"."stores" order by "test"."id" asc limit $2`,
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -194,8 +195,8 @@ describe("SQL query builder", () => {
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: ["john%", limit, "john%", 5000],
|
bindings: ["john%", limit],
|
||||||
sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" where LOWER("test"."name") LIKE :3 order by "test"."id" asc) where rownum <= :4`,
|
sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`,
|
||||||
})
|
})
|
||||||
|
|
||||||
query = new Sql(SqlClient.ORACLE, limit)._query(
|
query = new Sql(SqlClient.ORACLE, limit)._query(
|
||||||
|
@ -210,8 +211,8 @@ describe("SQL query builder", () => {
|
||||||
)
|
)
|
||||||
const filterSet = [`%20%`, `%25%`, `%"john"%`, `%"mary"%`]
|
const filterSet = [`%20%`, `%25%`, `%"john"%`, `%"mary"%`]
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: [...filterSet, limit, ...filterSet, 5000],
|
bindings: [...filterSet, limit],
|
||||||
sql: `select * from (select * from (select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5) "test" where COALESCE(LOWER("test"."age"), '') LIKE :6 AND COALESCE(LOWER("test"."age"), '') LIKE :7 and COALESCE(LOWER("test"."name"), '') LIKE :8 AND COALESCE(LOWER("test"."name"), '') LIKE :9 order by "test"."id" asc) where rownum <= :10`,
|
sql: `select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5`,
|
||||||
})
|
})
|
||||||
|
|
||||||
query = new Sql(SqlClient.ORACLE, limit)._query(
|
query = new Sql(SqlClient.ORACLE, limit)._query(
|
||||||
|
@ -224,8 +225,8 @@ describe("SQL query builder", () => {
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: [`%jo%`, limit, `%jo%`, 5000],
|
bindings: [`%jo%`, limit],
|
||||||
sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" where LOWER("test"."name") LIKE :3 order by "test"."id" asc) where rownum <= :4`,
|
sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`,
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -242,8 +243,8 @@ describe("SQL query builder", () => {
|
||||||
)
|
)
|
||||||
|
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: ["John", limit, "John", 5000],
|
bindings: ["John", limit],
|
||||||
sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2) "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :3) order by "test"."id" asc) where rownum <= :4`,
|
sql: `select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2`,
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -260,8 +261,8 @@ describe("SQL query builder", () => {
|
||||||
)
|
)
|
||||||
|
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: ["John", limit, "John", 5000],
|
bindings: ["John", limit],
|
||||||
sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2) "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :3) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :4`,
|
sql: `select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2`,
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
|
@ -32,8 +32,8 @@ function multiline(sql: string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("Captures of real examples", () => {
|
describe("Captures of real examples", () => {
|
||||||
const limit = 5000
|
const relationshipLimit = 500
|
||||||
const relationshipLimit = 100
|
const primaryLimit = 100
|
||||||
|
|
||||||
function getJson(name: string): QueryJson {
|
function getJson(name: string): QueryJson {
|
||||||
return require(join(__dirname, "sqlQueryJson", name)) as QueryJson
|
return require(join(__dirname, "sqlQueryJson", name)) as QueryJson
|
||||||
|
@ -42,7 +42,9 @@ describe("Captures of real examples", () => {
|
||||||
describe("create", () => {
|
describe("create", () => {
|
||||||
it("should create a row with relationships", () => {
|
it("should create a row with relationships", () => {
|
||||||
const queryJson = getJson("createWithRelationships.json")
|
const queryJson = getJson("createWithRelationships.json")
|
||||||
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
|
let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
|
||||||
|
queryJson
|
||||||
|
)
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: ["A Street", 34, "London", "A", "B", "designer", 1990],
|
bindings: ["A Street", 34, "London", "A", "B", "designer", 1990],
|
||||||
sql: multiline(`insert into "persons" ("address", "age", "city", "firstname", "lastname", "type", "year")
|
sql: multiline(`insert into "persons" ("address", "age", "city", "firstname", "lastname", "type", "year")
|
||||||
|
@ -54,40 +56,48 @@ describe("Captures of real examples", () => {
|
||||||
describe("read", () => {
|
describe("read", () => {
|
||||||
it("should handle basic retrieval with relationships", () => {
|
it("should handle basic retrieval with relationships", () => {
|
||||||
const queryJson = getJson("basicFetchWithRelationships.json")
|
const queryJson = getJson("basicFetchWithRelationships.json")
|
||||||
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
|
let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
|
||||||
|
queryJson
|
||||||
|
)
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: [relationshipLimit, limit],
|
bindings: [relationshipLimit, relationshipLimit, primaryLimit],
|
||||||
sql: expect.stringContaining(
|
sql: expect.stringContaining(
|
||||||
multiline(`select "a"."year" as "a.year", "a"."firstname" as "a.firstname", "a"."personid" as "a.personid",
|
multiline(
|
||||||
"a"."address" as "a.address", "a"."age" as "a.age", "a"."type" as "a.type", "a"."city" as "a.city",
|
`select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid",'executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")`
|
||||||
"a"."lastname" as "a.lastname", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname",
|
)
|
||||||
"b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid",
|
|
||||||
"b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid",
|
|
||||||
"b"."completed" as "b.completed", "b"."qaid" as "b.qaid"`)
|
|
||||||
),
|
),
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
it("should handle filtering by relationship", () => {
|
it("should handle filtering by relationship", () => {
|
||||||
const queryJson = getJson("filterByRelationship.json")
|
const queryJson = getJson("filterByRelationship.json")
|
||||||
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
|
let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
|
||||||
|
queryJson
|
||||||
|
)
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: [relationshipLimit, "assembling", limit],
|
bindings: [relationshipLimit, "assembling", primaryLimit],
|
||||||
sql: expect.stringContaining(
|
sql: expect.stringContaining(
|
||||||
multiline(`where COALESCE("b"."taskname" = $2, FALSE)
|
multiline(
|
||||||
order by "a"."productname" asc nulls first, "a"."productid" asc limit $3`)
|
`where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid"
|
||||||
|
where "c"."productid" = "a"."productid" and COALESCE("b"."taskname" = $2, FALSE)`
|
||||||
|
)
|
||||||
),
|
),
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
it("should handle fetching many to many relationships", () => {
|
it("should handle fetching many to many relationships", () => {
|
||||||
const queryJson = getJson("fetchManyToMany.json")
|
const queryJson = getJson("fetchManyToMany.json")
|
||||||
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
|
let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
|
||||||
|
queryJson
|
||||||
|
)
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: [relationshipLimit, limit],
|
bindings: [relationshipLimit, primaryLimit],
|
||||||
sql: expect.stringContaining(
|
sql: expect.stringContaining(
|
||||||
multiline(`left join "products_tasks" as "c" on "a"."productid" = "c"."productid"
|
multiline(
|
||||||
left join "tasks" as "b" on "b"."taskid" = "c"."taskid" `)
|
`select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid"))
|
||||||
|
from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid"
|
||||||
|
where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $1`
|
||||||
|
)
|
||||||
),
|
),
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
@ -95,22 +105,25 @@ describe("Captures of real examples", () => {
|
||||||
it("should handle enrichment of rows", () => {
|
it("should handle enrichment of rows", () => {
|
||||||
const queryJson = getJson("enrichRelationship.json")
|
const queryJson = getJson("enrichRelationship.json")
|
||||||
const filters = queryJson.filters?.oneOf?.taskid as number[]
|
const filters = queryJson.filters?.oneOf?.taskid as number[]
|
||||||
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
|
let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
|
||||||
|
queryJson
|
||||||
|
)
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: [...filters, limit, ...filters, limit],
|
bindings: [relationshipLimit, ...filters, relationshipLimit],
|
||||||
sql: multiline(
|
sql: multiline(
|
||||||
`select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", "a"."taskid" as "a.taskid",
|
`select "a".*, (select json_agg(json_build_object('productname',"b"."productname",'productid',"b"."productid"))
|
||||||
"a"."completed" as "a.completed", "a"."qaid" as "a.qaid", "b"."productname" as "b.productname", "b"."productid" as "b.productid"
|
from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid"
|
||||||
from (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) as "a"
|
where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $1) as "b") as "products"
|
||||||
left join "products_tasks" as "c" on "a"."taskid" = "c"."taskid" left join "products" as "b" on "b"."productid" = "c"."productid"
|
from "tasks" as "a" where "a"."taskid" in ($2, $3) order by "a"."taskid" asc limit $4`
|
||||||
where "a"."taskid" in ($4, $5) order by "a"."taskid" asc limit $6`
|
|
||||||
),
|
),
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
it("should manage query with many relationship filters", () => {
|
it("should manage query with many relationship filters", () => {
|
||||||
const queryJson = getJson("manyRelationshipFilters.json")
|
const queryJson = getJson("manyRelationshipFilters.json")
|
||||||
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
|
let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
|
||||||
|
queryJson
|
||||||
|
)
|
||||||
const filters = queryJson.filters
|
const filters = queryJson.filters
|
||||||
const notEqualsValue = Object.values(filters?.notEqual!)[0]
|
const notEqualsValue = Object.values(filters?.notEqual!)[0]
|
||||||
const rangeValue: { high?: string | number; low?: string | number } =
|
const rangeValue: { high?: string | number; low?: string | number } =
|
||||||
|
@ -119,17 +132,18 @@ describe("Captures of real examples", () => {
|
||||||
|
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: [
|
bindings: [
|
||||||
notEqualsValue,
|
relationshipLimit,
|
||||||
|
relationshipLimit,
|
||||||
relationshipLimit,
|
relationshipLimit,
|
||||||
rangeValue.low,
|
rangeValue.low,
|
||||||
rangeValue.high,
|
rangeValue.high,
|
||||||
equalValue,
|
equalValue,
|
||||||
true,
|
notEqualsValue,
|
||||||
limit,
|
primaryLimit,
|
||||||
],
|
],
|
||||||
sql: expect.stringContaining(
|
sql: expect.stringContaining(
|
||||||
multiline(
|
multiline(
|
||||||
`where "c"."year" between $3 and $4 and COALESCE("b"."productname" = $5, FALSE)`
|
`where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and "c"."year" between $4 and $5)`
|
||||||
)
|
)
|
||||||
),
|
),
|
||||||
})
|
})
|
||||||
|
@ -139,17 +153,23 @@ describe("Captures of real examples", () => {
|
||||||
describe("update", () => {
|
describe("update", () => {
|
||||||
it("should handle performing a simple update", () => {
|
it("should handle performing a simple update", () => {
|
||||||
const queryJson = getJson("updateSimple.json")
|
const queryJson = getJson("updateSimple.json")
|
||||||
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
|
let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
|
||||||
|
queryJson
|
||||||
|
)
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
|
bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
|
||||||
sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4,
|
sql: multiline(
|
||||||
"type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`),
|
`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4,
|
||||||
|
"type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`
|
||||||
|
),
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
it("should handle performing an update of relationships", () => {
|
it("should handle performing an update of relationships", () => {
|
||||||
const queryJson = getJson("updateRelationship.json")
|
const queryJson = getJson("updateRelationship.json")
|
||||||
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
|
let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
|
||||||
|
queryJson
|
||||||
|
)
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
|
bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
|
||||||
sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4,
|
sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4,
|
||||||
|
@ -161,12 +181,14 @@ describe("Captures of real examples", () => {
|
||||||
describe("delete", () => {
|
describe("delete", () => {
|
||||||
it("should handle deleting with relationships", () => {
|
it("should handle deleting with relationships", () => {
|
||||||
const queryJson = getJson("deleteSimple.json")
|
const queryJson = getJson("deleteSimple.json")
|
||||||
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
|
let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
|
||||||
|
queryJson
|
||||||
|
)
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
bindings: ["ddd", ""],
|
bindings: ["ddd", ""],
|
||||||
sql: multiline(`delete from "compositetable" as "a"
|
sql: multiline(`delete from "compositetable" as "a"
|
||||||
where COALESCE("a"."keypartone" = $1, FALSE) and COALESCE("a"."keyparttwo" = $2, FALSE)
|
where COALESCE("a"."keypartone" = $1, FALSE) and COALESCE("a"."keyparttwo" = $2, FALSE)
|
||||||
returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`),
|
returning "a".*`),
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
@ -174,7 +196,7 @@ describe("Captures of real examples", () => {
|
||||||
describe("returning (everything bar Postgres)", () => {
|
describe("returning (everything bar Postgres)", () => {
|
||||||
it("should be able to handle row returning", () => {
|
it("should be able to handle row returning", () => {
|
||||||
const queryJson = getJson("createSimple.json")
|
const queryJson = getJson("createSimple.json")
|
||||||
const SQL = new Sql(SqlClient.MS_SQL, limit)
|
const SQL = new Sql(SqlClient.MS_SQL, relationshipLimit)
|
||||||
let query = SQL._query(queryJson, { disableReturning: true })
|
let query = SQL._query(queryJson, { disableReturning: true })
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
sql: "insert into [people] ([age], [name]) values (@p0, @p1)",
|
sql: "insert into [people] ([age], [name]) values (@p0, @p1)",
|
||||||
|
@ -187,10 +209,11 @@ describe("Captures of real examples", () => {
|
||||||
returningQuery = input
|
returningQuery = input
|
||||||
}, queryJson)
|
}, queryJson)
|
||||||
expect(returningQuery).toEqual({
|
expect(returningQuery).toEqual({
|
||||||
sql: multiline(`select top (@p0) * from (select top (@p1) * from [people] where CASE WHEN [people].[name] = @p2
|
sql: multiline(
|
||||||
THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p3 THEN 1 ELSE 0 END = 1 order by [people].[name] asc) as [people]
|
`select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1
|
||||||
where CASE WHEN [people].[name] = @p4 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p5 THEN 1 ELSE 0 END = 1`),
|
and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc`
|
||||||
bindings: [5000, 1, "Test", 22, "Test", 22],
|
),
|
||||||
|
bindings: [1, "Test", 22],
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
|
@ -8,6 +8,10 @@ import { generateUserMetadataID, InternalTables } from "../../../db/utils"
|
||||||
|
|
||||||
type DeletedUser = { _id: string; deleted: boolean }
|
type DeletedUser = { _id: string; deleted: boolean }
|
||||||
|
|
||||||
|
function userSyncEnabled() {
|
||||||
|
return !env.DISABLE_USER_SYNC
|
||||||
|
}
|
||||||
|
|
||||||
async function syncUsersToApp(
|
async function syncUsersToApp(
|
||||||
appId: string,
|
appId: string,
|
||||||
users: (User | DeletedUser)[],
|
users: (User | DeletedUser)[],
|
||||||
|
@ -56,7 +60,7 @@ async function syncUsersToApp(
|
||||||
|
|
||||||
// the user doesn't exist, or doesn't have a role anymore
|
// the user doesn't exist, or doesn't have a role anymore
|
||||||
// get rid of their metadata
|
// get rid of their metadata
|
||||||
if (deletedUser || !roleId) {
|
if (userSyncEnabled() && (deletedUser || !roleId)) {
|
||||||
await db.remove(metadata)
|
await db.remove(metadata)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
@ -149,7 +153,9 @@ export async function syncApp(
|
||||||
}
|
}
|
||||||
|
|
||||||
// sync the users - kept for safe keeping
|
// sync the users - kept for safe keeping
|
||||||
|
if (userSyncEnabled()) {
|
||||||
await sdk.users.syncGlobalUsers()
|
await sdk.users.syncGlobalUsers()
|
||||||
|
}
|
||||||
|
|
||||||
if (error) {
|
if (error) {
|
||||||
throw error
|
throw error
|
||||||
|
|
|
@ -37,9 +37,9 @@ import { outputProcessing } from "../../../../../utilities/rowProcessor"
|
||||||
import pick from "lodash/pick"
|
import pick from "lodash/pick"
|
||||||
import { processRowCountResponse } from "../../utils"
|
import { processRowCountResponse } from "../../utils"
|
||||||
import {
|
import {
|
||||||
updateFilterKeys,
|
|
||||||
getRelationshipColumns,
|
getRelationshipColumns,
|
||||||
getTableIDList,
|
getTableIDList,
|
||||||
|
updateFilterKeys,
|
||||||
} from "../filters"
|
} from "../filters"
|
||||||
import {
|
import {
|
||||||
dataFilters,
|
dataFilters,
|
||||||
|
@ -182,11 +182,20 @@ function buildTableMap(tables: Table[]) {
|
||||||
return tableMap
|
return tableMap
|
||||||
}
|
}
|
||||||
|
|
||||||
function reverseUserColumnMapping(rows: Row[]) {
|
// table is only needed to handle relationships
|
||||||
|
function reverseUserColumnMapping(rows: Row[], table?: Table) {
|
||||||
const prefixLength = USER_COLUMN_PREFIX.length
|
const prefixLength = USER_COLUMN_PREFIX.length
|
||||||
return rows.map(row => {
|
return rows.map(row => {
|
||||||
const finalRow: Row = {}
|
const finalRow: Row = {}
|
||||||
for (let key of Object.keys(row)) {
|
for (let key of Object.keys(row)) {
|
||||||
|
// handle relationships
|
||||||
|
if (
|
||||||
|
table?.schema[key]?.type === FieldType.LINK &&
|
||||||
|
typeof row[key] === "string"
|
||||||
|
) {
|
||||||
|
// no table required, relationship rows don't contain relationships
|
||||||
|
row[key] = reverseUserColumnMapping(JSON.parse(row[key]))
|
||||||
|
}
|
||||||
// it should be the first prefix
|
// it should be the first prefix
|
||||||
const index = key.indexOf(USER_COLUMN_PREFIX)
|
const index = key.indexOf(USER_COLUMN_PREFIX)
|
||||||
if (index !== -1) {
|
if (index !== -1) {
|
||||||
|
@ -261,7 +270,7 @@ async function runSqlQuery(
|
||||||
if (opts?.countTotalRows) {
|
if (opts?.countTotalRows) {
|
||||||
return processRowCountResponse(response)
|
return processRowCountResponse(response)
|
||||||
} else if (Array.isArray(response)) {
|
} else if (Array.isArray(response)) {
|
||||||
return reverseUserColumnMapping(response)
|
return reverseUserColumnMapping(response, json.meta.table)
|
||||||
}
|
}
|
||||||
return response
|
return response
|
||||||
}
|
}
|
||||||
|
|
|
@ -337,6 +337,13 @@ export async function outputProcessing<T extends Row[] | Row>(
|
||||||
row[property] = `${hours}:${minutes}:${seconds}`
|
row[property] = `${hours}:${minutes}:${seconds}`
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} else if (column.type === FieldType.LINK) {
|
||||||
|
for (let row of enriched) {
|
||||||
|
// if relationship is empty - remove the array, this has been part of the API for some time
|
||||||
|
if (Array.isArray(row[property]) && row[property].length === 0) {
|
||||||
|
delete row[property]
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -134,6 +134,17 @@ export interface RelationshipsJson {
|
||||||
column: string
|
column: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO - this can be combined with the above type
|
||||||
|
export interface ManyToManyRelationshipJson {
|
||||||
|
through: string
|
||||||
|
from: string
|
||||||
|
to: string
|
||||||
|
fromPrimary: string
|
||||||
|
toPrimary: string
|
||||||
|
tableName: string
|
||||||
|
column: string
|
||||||
|
}
|
||||||
|
|
||||||
export interface QueryJson {
|
export interface QueryJson {
|
||||||
endpoint: {
|
endpoint: {
|
||||||
datasourceId: string
|
datasourceId: string
|
||||||
|
|
Loading…
Reference in New Issue