MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14509 from Budibase/aggregate-all-sql-dbs 

Apply new relationship retrieval to all SQL DBs
This commit is contained in:
Michael Drury 2024-09-06 10:38:08 +01:00 committed by GitHub
parent ac7838f80d f7d9b8a9b3
commit aac1d7699e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 364 additions and 387 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

320
packages/backend-core/src/sql/sql.ts
View File

@ -7,6 +7,7 @@ import {
isValidFilter,
isValidISODateString,
sqlLog,
validateManyToMany,
} from "./utils"
import SqlTableQueryBuilder from "./sqlTable"
import {
@ -93,6 +94,23 @@ class InternalBuilder {
})
}
// states the various situations in which we need a full mapped select statement
private readonly SPECIAL_SELECT_CASES = {
POSTGRES_MONEY: (field: FieldSchema | undefined) => {
return (
this.client === SqlClient.POSTGRES &&
field?.externalType?.includes("money")
)
},
MSSQL_DATES: (field: FieldSchema | undefined) => {
return (
this.client === SqlClient.MS_SQL &&
field?.type === FieldType.DATETIME &&
field.timeOnly
)
},
}
get table(): Table {
return this.query.meta.table
}
@ -126,87 +144,70 @@ class InternalBuilder {
.join(".")
}
private isFullSelectStatementRequired(): boolean {
const { meta } = this.query
for (let column of Object.values(meta.table.schema)) {
if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) {
return true
} else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) {
return true
}
}
return false
}
private generateSelectStatement(): (string | Knex.Raw)[] | "*" {
const { endpoint, resource, meta, tableAliases } = this.query
const { meta, endpoint, resource, tableAliases } = this.query
if (!resource || !resource.fields || resource.fields.length === 0) {
return "*"
}
// no relationships - select everything in SQLite
if (this.client === SqlClient.SQL_LITE) {
const alias = tableAliases?.[endpoint.entityId]
? tableAliases?.[endpoint.entityId]
: endpoint.entityId
const alias = tableAliases?.[endpoint.entityId]
? tableAliases?.[endpoint.entityId]
: endpoint.entityId
const schema = meta.table.schema
if (!this.isFullSelectStatementRequired()) {
return [this.knex.raw(`${this.quote(alias)}.*`)]
}
// get just the fields for this table
return resource.fields
.map(field => {
const parts = field.split(/\./g)
let table: string | undefined = undefined
let column = parts[0]
const schema = meta.table.schema
return resource.fields.map(field => {
const parts = field.split(/\./g)
let table: string | undefined = undefined
let column: string | undefined = undefined
// Just a column name, e.g.: "column"
if (parts.length > 1) {
table = parts[0]
column = parts.slice(1).join(".")
}
// Just a column name, e.g.: "column"
if (parts.length === 1) {
column = parts[0]
}
return { table, column, field }
})
.filter(({ table }) => !table || table === alias)
.map(({ table, column, field }) => {
const columnSchema = schema[column]
// A table name and a column name, e.g.: "table.column"
if (parts.length === 2) {
table = parts[0]
column = parts[1]
}
if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) {
return this.knex.raw(
`${this.quotedIdentifier(
[table, column].join(".")
)}::money::numeric as ${this.quote(field)}`
)
}
// A link doc, e.g.: "table.doc1.fieldName"
if (parts.length > 2) {
table = parts[0]
column = parts.slice(1).join(".")
}
if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(columnSchema)) {
// Time gets returned as timestamp from mssql, not matching the expected
// HH:mm format
return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
}
if (!column) {
throw new Error(`Invalid field name: ${field}`)
}
const columnSchema = schema[column]
if (
this.client === SqlClient.POSTGRES &&
columnSchema?.externalType?.includes("money")
) {
return this.knex.raw(
`${this.quotedIdentifier(
[table, column].join(".")
)}::money::numeric as ${this.quote(field)}`
)
}
if (
this.client === SqlClient.MS_SQL &&
columnSchema?.type === FieldType.DATETIME &&
columnSchema.timeOnly
) {
// Time gets returned as timestamp from mssql, not matching the expected
// HH:mm format
return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
}
// There's at least two edge cases being handled in the expression below.
// 1. The column name could start/end with a space, and in that case we
// want to preseve that space.
// 2. Almost all column names are specified in the form table.column, except
// in the case of relationships, where it's table.doc1.column. In that
// case, we want to split it into `table`.`doc1.column` for reasons that
// aren't actually clear to me, but `table`.`doc1` breaks things with the
// sample data tests.
if (table) {
return this.knex.raw(
`${this.quote(table)}.${this.quote(column)} as ${this.quote(field)}`
)
} else {
return this.knex.raw(`${this.quote(field)} as ${this.quote(field)}`)
}
})
const quoted = table
? `${this.quote(table)}.${this.quote(column)}`
: this.quote(field)
return this.knex.raw(quoted)
})
}
// OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
@ -368,35 +369,47 @@ class InternalBuilder {
let subQuery = mainKnex
.select(mainKnex.raw(1))
.from({ [toAlias]: relatedTableName })
let mainTableRelatesTo = toAlias
if (relationship.through) {
const manyToMany = validateManyToMany(relationship)
if (manyToMany) {
const throughAlias =
aliases?.[relationship.through] || relationship.through
let throughTable = this.tableNameWithSchema(relationship.through, {
aliases?.[manyToMany.through] || relationship.through
let throughTable = this.tableNameWithSchema(manyToMany.through, {
alias: throughAlias,
schema: endpoint.schema,
})
subQuery = subQuery.innerJoin(throughTable, function () {
// @ts-ignore
this.on(
`${toAlias}.${relationship.toPrimary}`,
subQuery = subQuery
// add a join through the junction table
.innerJoin(throughTable, function () {
// @ts-ignore
this.on(
`${toAlias}.${manyToMany.toPrimary}`,
"=",
`${throughAlias}.${manyToMany.to}`
)
})
// check the document in the junction table points to the main table
.where(
`${throughAlias}.${manyToMany.from}`,
"=",
`${throughAlias}.${relationship.to}`
mainKnex.raw(
this.quotedIdentifier(`${fromAlias}.${manyToMany.fromPrimary}`)
)
)
})
// in SQS the same junction table is used for different many-to-many relationships between the
// two same tables, this is needed to avoid rows ending up in all columns
if (this.client === SqlClient.SQL_LITE) {
subQuery = this.addJoinFieldCheck(subQuery, relationship)
subQuery = this.addJoinFieldCheck(subQuery, manyToMany)
}
mainTableRelatesTo = throughAlias
}
// "join" to the main table, making sure the ID matches that of the main
subQuery = subQuery.where(
`${mainTableRelatesTo}.${relationship.from}`,
"=",
mainKnex.raw(
this.quotedIdentifier(`${fromAlias}.${relationship.fromPrimary}`)
} else {
// "join" to the main table, making sure the ID matches that of the main
subQuery = subQuery.where(
`${toAlias}.${relationship.to}`,
"=",
mainKnex.raw(
this.quotedIdentifier(`${fromAlias}.${relationship.from}`)
)
)
)
}
query = query.whereExists(whereCb(subQuery))
break
}
@ -478,12 +491,10 @@ class InternalBuilder {
alias ? `${alias}.${updatedKey}` : updatedKey,
value
)
} else if (isSqlite && shouldProcessRelationship) {
} else if (shouldProcessRelationship) {
query = builder.addRelationshipForFilter(query, updatedKey, q => {
return handleRelationship(q, updatedKey, value)
})
} else if (shouldProcessRelationship) {
query = handleRelationship(query, updatedKey, value)
}
}
}
@ -849,6 +860,8 @@ class InternalBuilder {
fromTable: string,
relationships: RelationshipsJson[]
): Knex.QueryBuilder {
const sqlClient = this.client
const knex = this.knex
const { resource, tableAliases: aliases, endpoint } = this.query
const fields = resource?.fields || []
const jsonField = (field: string) => {
@ -862,7 +875,15 @@ class InternalBuilder {
unaliased = parts.join(".")
tableField = this.quote(unaliased)
}
return `'${unaliased}',${tableField}`
let separator = ","
switch (sqlClient) {
case SqlClient.ORACLE:
separator = " VALUE "
break
case SqlClient.MS_SQL:
separator = ":"
}
return `'${unaliased}'${separator}${tableField}`
}
for (let relationship of relationships) {
const {
@ -874,23 +895,15 @@ class InternalBuilder {
toPrimary,
} = relationship
// skip invalid relationships
if (!toTable || !fromTable || !fromPrimary || !toPrimary) {
if (!toTable || !fromTable) {
continue
}
if (!throughTable) {
throw new Error("Only many-to-many implemented for JSON relationships")
}
const toAlias = aliases?.[toTable] || toTable,
throughAlias = aliases?.[throughTable] || throughTable,
fromAlias = aliases?.[fromTable] || fromTable
let toTableWithSchema = this.tableNameWithSchema(toTable, {
alias: toAlias,
schema: endpoint.schema,
})
let throughTableWithSchema = this.tableNameWithSchema(throughTable, {
alias: throughAlias,
schema: endpoint.schema,
})
let relationshipFields = fields.filter(
field => field.split(".")[0] === toAlias
)
@ -903,36 +916,87 @@ class InternalBuilder {
const fieldList: string = relationshipFields
.map(field => jsonField(field))
.join(",")
let rawJsonArray: Knex.Raw
switch (this.client) {
// SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax
// it reduces the result set rather than limiting how much data it filters over
const primaryKey = `${toAlias}.${toPrimary || toKey}`
let subQuery: Knex.QueryBuilder = knex
.from(toTableWithSchema)
.limit(getBaseLimit())
// add sorting to get consistent order
.orderBy(primaryKey)
// many-to-many relationship with junction table
if (throughTable && toPrimary && fromPrimary) {
const throughAlias = aliases?.[throughTable] || throughTable
let throughTableWithSchema = this.tableNameWithSchema(throughTable, {
alias: throughAlias,
schema: endpoint.schema,
})
subQuery = subQuery
.join(throughTableWithSchema, function () {
this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`)
})
.where(
`${throughAlias}.${fromKey}`,
"=",
knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`))
)
}
// one-to-many relationship with foreign key
else {
subQuery = subQuery.where(
`${toAlias}.${toKey}`,
"=",
knex.raw(this.quotedIdentifier(`${fromAlias}.${fromKey}`))
)
}
const standardWrap = (select: string): Knex.QueryBuilder => {
subQuery = subQuery.select(`${toAlias}.*`)
// @ts-ignore - the from alias syntax isn't in Knex typing
return knex.select(knex.raw(select)).from({
[toAlias]: subQuery,
})
}
let wrapperQuery: Knex.QueryBuilder | Knex.Raw
switch (sqlClient) {
case SqlClient.SQL_LITE:
rawJsonArray = this.knex.raw(
// need to check the junction table document is to the right column, this is just for SQS
subQuery = this.addJoinFieldCheck(subQuery, relationship)
wrapperQuery = standardWrap(
`json_group_array(json_object(${fieldList}))`
)
break
case SqlClient.POSTGRES:
wrapperQuery = standardWrap(
`json_agg(json_build_object(${fieldList}))`
)
break
case SqlClient.MY_SQL:
wrapperQuery = subQuery.select(
knex.raw(`json_arrayagg(json_object(${fieldList}))`)
)
break
case SqlClient.ORACLE:
wrapperQuery = standardWrap(
`json_arrayagg(json_object(${fieldList}))`
)
break
case SqlClient.MS_SQL:
wrapperQuery = knex.raw(
`(SELECT ${this.quote(toAlias)} = (${knex
.select(`${fromAlias}.*`)
// @ts-ignore - from alias syntax not TS supported
.from({
[fromAlias]: subQuery.select(`${toAlias}.*`),
})} FOR JSON PATH))`
)
break
default:
throw new Error(`JSON relationships not implement for ${this.client}`)
throw new Error(`JSON relationships not implement for ${sqlClient}`)
}
let subQuery = this.knex
.select(rawJsonArray)
.from(toTableWithSchema)
.join(throughTableWithSchema, function () {
this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`)
})
.where(
`${throughAlias}.${fromKey}`,
"=",
this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`))
)
// relationships should never have more than the base limit
.limit(getBaseLimit())
// add sorting to get consistent order
.orderBy(`${toAlias}.${toPrimary}`)
// need to check the junction table document is to the right column
if (this.client === SqlClient.SQL_LITE) {
subQuery = this.addJoinFieldCheck(subQuery, relationship)
}
query = query.select({ [relationship.column]: subQuery })
query = query.select({ [relationship.column]: wrapperQuery })
}
return query
}
@ -1179,14 +1243,12 @@ class InternalBuilder {
? query.select(this.generateSelectStatement())
: this.addDistinctCount(query)
// have to add after as well (this breaks MS-SQL)
if (this.client !== SqlClient.MS_SQL && !counting) {
if (!counting) {
query = this.addSorting(query)
}
// handle joins
if (relationships && this.client === SqlClient.SQL_LITE) {
if (relationships) {
query = this.addJsonRelationships(query, tableName, relationships)
} else if (relationships) {
query = this.addRelationships(query, tableName, relationships)
}
return this.addFilters(query, filters, { relationship: true })

30
packages/backend-core/src/sql/utils.ts
View File

@ -1,4 +1,11 @@
import { DocumentType, SqlQuery, Table, TableSourceType } from "@budibase/types"
import {
DocumentType,
ManyToManyRelationshipJson,
RelationshipsJson,
SqlQuery,
Table,
TableSourceType,
} from "@budibase/types"
import { DEFAULT_BB_DATASOURCE_ID } from "../constants"
import { Knex } from "knex"
import { SEPARATOR } from "../db"
@ -163,3 +170,24 @@ export function sqlLog(client: string, query: string, values?: any[]) {
}
console.log(string)
}
function isValidManyToManyRelationship(
relationship: RelationshipsJson
): relationship is ManyToManyRelationshipJson {
return (
!!relationship.through &&
!!relationship.fromPrimary &&
!!relationship.from &&
!!relationship.toPrimary &&
!!relationship.to
)
}
export function validateManyToMany(
relationship: RelationshipsJson
): ManyToManyRelationshipJson | undefined {
if (isValidManyToManyRelationship(relationship)) {
return relationship
}
return undefined
}

2
packages/server/datasource-sha.env
View File

@ -1,4 +1,4 @@
MSSQL_SHA=sha256:c4369c38385eba011c10906dc8892425831275bb035d5ce69656da8e29de50d8
MSSQL_SHA=sha256:3b913841850a4d57fcfcb798be06acc88ea0f2acc5418bc0c140a43e91c4a545
MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe
POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e
MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d

65
packages/server/src/api/controllers/row/utils/basic.ts
View File

@ -1,6 +1,10 @@
// need to handle table name + field or just field, depending on if relationships used
import { FieldSchema, FieldType, Row, Table } from "@budibase/types"
import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
import {
helpers,
PROTECTED_EXTERNAL_COLUMNS,
PROTECTED_INTERNAL_COLUMNS,
} from "@budibase/shared-core"
import { generateRowIdField } from "../../../../integrations/utils"
function extractFieldValue({
@ -61,11 +65,13 @@ export function generateIdForRow(
export function basicProcessing({
row,
table,
tables,
isLinked,
sqs,
}: {
row: Row
table: Table
tables: Table[]
isLinked: boolean
sqs?: boolean
}): Row {
@ -86,24 +92,69 @@ export function basicProcessing({
thisRow[fieldName] = value
}
}
let columns: string[] = Object.keys(table.schema)
if (!sqs) {
thisRow._id = generateIdForRow(row, table, isLinked)
thisRow.tableId = table._id
thisRow._rev = "rev"
columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS)
} else {
const columns = Object.keys(table.schema)
columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS)
for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) {
const schema: FieldSchema | undefined = table.schema[internalColumn]
let value = extractFieldValue({
thisRow[internalColumn] = extractFieldValue({
row,
tableName: table._id!,
fieldName: internalColumn,
isLinked,
})
if (sqs && schema?.type === FieldType.LINK && typeof value === "string") {
value = JSON.parse(value)
}
}
for (let col of columns) {
const schema: FieldSchema | undefined = table.schema[col]
if (schema?.type !== FieldType.LINK) {
continue
}
const relatedTable = tables.find(tbl => tbl._id === schema.tableId)
if (!relatedTable) {
continue
}
const value = extractFieldValue({
row,
tableName: table._id!,
fieldName: col,
isLinked,
})
const array: Row[] = Array.isArray(value)
? value
: typeof value === "string"
? JSON.parse(value)
: undefined
if (array) {
thisRow[col] = array
// make sure all of them have an _id
if (Array.isArray(thisRow[col])) {
const sortField =
relatedTable.primaryDisplay || relatedTable.primary![0]!
thisRow[col] = (thisRow[col] as Row[])
.map(relatedRow => {
relatedRow._id = relatedRow._id
? relatedRow._id
: generateIdForRow(relatedRow, relatedTable)
return relatedRow
})
.sort((a, b) => {
const aField = a?.[sortField],
bField = b?.[sortField]
if (!aField) {
return 1
} else if (!bField) {
return -1
}
return aField.localeCompare
? aField.localeCompare(bField)
: aField - bField
})
}
thisRow[internalColumn] = value
}
}
return thisRow

83
packages/server/src/api/controllers/row/utils/sqlUtils.ts
View File

@ -7,11 +7,9 @@ import {
ManyToManyRelationshipFieldMetadata,
RelationshipFieldMetadata,
RelationshipsJson,
Row,
Table,
} from "@budibase/types"
import { breakExternalTableId } from "../../../../integrations/utils"
import { basicProcessing } from "./basic"
import { generateJunctionTableID } from "../../../../db/utils"
type TableMap = Record<string, Table>
@ -22,87 +20,6 @@ export function isManyToMany(
return !!(field as ManyToManyRelationshipFieldMetadata).through
}
function isCorrectRelationship(
relationship: RelationshipsJson,
table1: Table,
table2: Table,
row: Row
): boolean {
const junctionTableId = generateJunctionTableID(table1._id!, table2._id!)
const possibleColumns = [
`${junctionTableId}.doc1.fieldName`,
`${junctionTableId}.doc2.fieldName`,
]
return !!possibleColumns.find(col => row[col] === relationship.column)
}
/**
* This iterates through the returned rows and works out what elements of the rows
* actually match up to another row (based on primary keys) - this is pretty specific
* to SQL and the way that SQL relationships are returned based on joins.
* This is complicated, but the idea is that when a SQL query returns all the relations
* will be separate rows, with all of the data in each row. We have to decipher what comes
* from where (which tables) and how to convert that into budibase columns.
*/
export async function updateRelationshipColumns(
table: Table,
tables: TableMap,
row: Row,
rows: { [key: string]: Row },
relationships: RelationshipsJson[],
opts?: { sqs?: boolean }
) {
const columns: { [key: string]: any } = {}
for (let relationship of relationships) {
const linkedTable = tables[relationship.tableName]
if (!linkedTable) {
continue
}
const fromColumn = `${table.name}.${relationship.from}`
const toColumn = `${linkedTable.name}.${relationship.to}`
// this is important when working with multiple relationships
// between the same tables, don't want to overlap/multiply the relations
if (
!relationship.through &&
row[fromColumn]?.toString() !== row[toColumn]?.toString()
) {
continue
}
let linked = basicProcessing({
row,
table: linkedTable,
isLinked: true,
sqs: opts?.sqs,
})
if (!linked._id) {
continue
}
if (
!opts?.sqs ||
isCorrectRelationship(relationship, table, linkedTable, row)
) {
columns[relationship.column] = linked
}
}
for (let [column, related] of Object.entries(columns)) {
if (!row._id) {
continue
}
const rowId: string = row._id
if (!Array.isArray(rows[rowId][column])) {
rows[rowId][column] = []
}
// make sure relationship hasn't been found already
if (
!rows[rowId][column].find((relation: Row) => relation._id === related._id)
) {
rows[rowId][column].push(related)
}
}
return rows
}
/**
* Gets the list of relationship JSON structures based on the columns in the table,
* this will be used by the underlying library to build whatever relationship mechanism

27
packages/server/src/api/controllers/row/utils/utils.ts
View File

@ -13,7 +13,7 @@ import {
processDates,
processFormulas,
} from "../../../../utilities/rowProcessor"
import { isKnexEmptyReadResponse, updateRelationshipColumns } from "./sqlUtils"
import { isKnexEmptyReadResponse } from "./sqlUtils"
import {
basicProcessing,
generateIdForRow,
@ -149,22 +149,11 @@ export async function sqlOutputProcessing(
rowId = generateIdForRow(row, table)
row._id = rowId
}
// this is a relationship of some sort
if (!opts?.sqs && finalRows[rowId]) {
finalRows = await updateRelationshipColumns(
table,
tables,
row,
finalRows,
relationships,
opts
)
continue
}
const thisRow = fixArrayTypes(
basicProcessing({
row,
table,
tables: Object.values(tables),
isLinked: false,
sqs: opts?.sqs,
}),
@ -175,18 +164,6 @@ export async function sqlOutputProcessing(
}
finalRows[thisRow._id] = fixBooleanFields({ row: thisRow, table })
// do this at end once its been added to the final rows
if (!opts?.sqs) {
finalRows = await updateRelationshipColumns(
table,
tables,
row,
finalRows,
relationships,
opts
)
}
}
// make sure all related rows are correct

10
packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
View File

@ -832,10 +832,12 @@ describe.each(
},
})
expect(res).toHaveLength(1)
expect(res[0]).toEqual({
id: 2,
name: "two",
})
expect(res[0]).toEqual(
expect.objectContaining({
id: 2,
name: "two",
})
)
})
// this parameter really only impacts SQL queries

76
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -2690,82 +2690,6 @@ describe.each([
})
})
// TODO: when all SQL databases use the same mechanism - remove this test, new relationship system doesn't have this problem
!isInternal &&
describe("pagination edge case with relationships", () => {
let mainRows: Row[] = []
beforeAll(async () => {
const toRelateTable = await createTable({
name: {
name: "name",
type: FieldType.STRING,
},
})
table = await createTable({
name: {
name: "name",
type: FieldType.STRING,
},
rel: {
name: "rel",
type: FieldType.LINK,
relationshipType: RelationshipType.MANY_TO_ONE,
tableId: toRelateTable._id!,
fieldName: "rel",
},
})
const relatedRows = await Promise.all([
config.api.row.save(toRelateTable._id!, { name: "tag 1" }),
config.api.row.save(toRelateTable._id!, { name: "tag 2" }),
config.api.row.save(toRelateTable._id!, { name: "tag 3" }),
config.api.row.save(toRelateTable._id!, { name: "tag 4" }),
config.api.row.save(toRelateTable._id!, { name: "tag 5" }),
config.api.row.save(toRelateTable._id!, { name: "tag 6" }),
])
mainRows = await Promise.all([
config.api.row.save(table._id!, {
name: "product 1",
rel: relatedRows.map(row => row._id),
}),
config.api.row.save(table._id!, {
name: "product 2",
rel: [],
}),
config.api.row.save(table._id!, {
name: "product 3",
rel: [],
}),
])
})
it("can still page when the hard limit is hit", async () => {
await withCoreEnv(
{
SQL_MAX_ROWS: "6",
},
async () => {
const params: Omit<RowSearchParams, "tableId"> = {
query: {},
paginate: true,
limit: 3,
sort: "name",
sortType: SortType.STRING,
sortOrder: SortOrder.ASCENDING,
}
const page1 = await expectSearch(params).toContain([mainRows[0]])
expect(page1.hasNextPage).toBe(true)
expect(page1.bookmark).toBeDefined()
const page2 = await expectSearch({
...params,
bookmark: page1.bookmark,
}).toContain([mainRows[1], mainRows[2]])
expect(page2.hasNextPage).toBe(false)
}
)
})
})
isSql &&
describe("primaryDisplay", () => {
beforeAll(async () => {

4
packages/server/src/integrations/microsoftSqlServer.ts
View File

@ -343,9 +343,9 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
err.number
)
if (readableMessage) {
throw new Error(readableMessage)
throw new Error(readableMessage, { cause: err })
} else {
throw new Error(err.message as string)
throw new Error(err.message as string, { cause: err })
}
}
}

32
packages/server/src/integrations/tests/sql.spec.ts
View File

@ -160,16 +160,16 @@ describe("SQL query builder", () => {
it("should add the schema to the LEFT JOIN", () => {
const query = sql._query(generateRelationshipJson({ schema: "production" }))
expect(query).toEqual({
bindings: [500, 5000],
sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "production"."brands" order by "test"."id" asc limit $1) as "brands" left join "production"."products" as "products" on "brands"."brand_id" = "products"."brand_id" order by "test"."id" asc limit $2`,
bindings: [5000, limit],
sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "production"."brands" order by "test"."id" asc limit $2`,
})
})
it("should handle if the schema is not present when doing a LEFT JOIN", () => {
const query = sql._query(generateRelationshipJson())
expect(query).toEqual({
bindings: [500, 5000],
sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "brands" order by "test"."id" asc limit $1) as "brands" left join "products" as "products" on "brands"."brand_id" = "products"."brand_id" order by "test"."id" asc limit $2`,
bindings: [5000, limit],
sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "brands" order by "test"."id" asc limit $2`,
})
})
@ -178,8 +178,8 @@ describe("SQL query builder", () => {
generateManyRelationshipJson({ schema: "production" })
)
expect(query).toEqual({
bindings: [500, 5000],
sql: `select "stores"."store_id" as "stores.store_id", "stores"."store_name" as "stores.store_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name" from (select * from "production"."stores" order by "test"."id" asc limit $1) as "stores" left join "production"."stocks" as "stocks" on "stores"."store_id" = "stocks"."store_id" left join "production"."products" as "products" on "products"."product_id" = "stocks"."product_id" order by "test"."id" asc limit $2`,
bindings: [5000, limit],
sql: `select "stores".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" inner join "production"."stocks" as "stocks" on "products"."product_id" = "stocks"."product_id" where "stocks"."store_id" = "stores"."store_id" order by "products"."product_id" asc limit $1) as "products") as "products" from "production"."stores" order by "test"."id" asc limit $2`,
})
})
@ -194,8 +194,8 @@ describe("SQL query builder", () => {
})
)
expect(query).toEqual({
bindings: ["john%", limit, "john%", 5000],
sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" where LOWER("test"."name") LIKE :3 order by "test"."id" asc) where rownum <= :4`,
bindings: ["john%", limit],
sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`,
})
query = new Sql(SqlClient.ORACLE, limit)._query(
@ -210,8 +210,8 @@ describe("SQL query builder", () => {
)
const filterSet = [`%20%`, `%25%`, `%"john"%`, `%"mary"%`]
expect(query).toEqual({
bindings: [...filterSet, limit, ...filterSet, 5000],
sql: `select * from (select * from (select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5) "test" where COALESCE(LOWER("test"."age"), '') LIKE :6 AND COALESCE(LOWER("test"."age"), '') LIKE :7 and COALESCE(LOWER("test"."name"), '') LIKE :8 AND COALESCE(LOWER("test"."name"), '') LIKE :9 order by "test"."id" asc) where rownum <= :10`,
bindings: [...filterSet, limit],
sql: `select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5`,
})
query = new Sql(SqlClient.ORACLE, limit)._query(
@ -224,8 +224,8 @@ describe("SQL query builder", () => {
})
)
expect(query).toEqual({
bindings: [`%jo%`, limit, `%jo%`, 5000],
sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" where LOWER("test"."name") LIKE :3 order by "test"."id" asc) where rownum <= :4`,
bindings: [`%jo%`, limit],
sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`,
})
})
@ -242,8 +242,8 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: ["John", limit, "John", 5000],
sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2) "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :3) order by "test"."id" asc) where rownum <= :4`,
bindings: ["John", limit],
sql: `select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2`,
})
})
@ -260,8 +260,8 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: ["John", limit, "John", 5000],
sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2) "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :3) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :4`,
bindings: ["John", limit],
sql: `select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2`,
})
})
})

91
packages/server/src/integrations/tests/sqlAlias.spec.ts
View File

@ -32,8 +32,8 @@ function multiline(sql: string) {
}
describe("Captures of real examples", () => {
const limit = 5000
const relationshipLimit = 100
const baseLimit = 5000
const primaryLimit = 100
function getJson(name: string): QueryJson {
return require(join(__dirname, "sqlQueryJson", name)) as QueryJson
@ -42,7 +42,7 @@ describe("Captures of real examples", () => {
describe("create", () => {
it("should create a row with relationships", () => {
const queryJson = getJson("createWithRelationships.json")
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
expect(query).toEqual({
bindings: ["A Street", 34, "London", "A", "B", "designer", 1990],
sql: multiline(`insert into "persons" ("address", "age", "city", "firstname", "lastname", "type", "year")
@ -54,40 +54,42 @@ describe("Captures of real examples", () => {
describe("read", () => {
it("should handle basic retrieval with relationships", () => {
const queryJson = getJson("basicFetchWithRelationships.json")
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
expect(query).toEqual({
bindings: [relationshipLimit, limit],
bindings: [baseLimit, baseLimit, primaryLimit],
sql: expect.stringContaining(
multiline(`select "a"."year" as "a.year", "a"."firstname" as "a.firstname", "a"."personid" as "a.personid",
"a"."address" as "a.address", "a"."age" as "a.age", "a"."type" as "a.type", "a"."city" as "a.city",
"a"."lastname" as "a.lastname", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname",
"b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid",
"b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid",
"b"."completed" as "b.completed", "b"."qaid" as "b.qaid"`)
multiline(
`select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid",'executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")`
)
),
})
})
it("should handle filtering by relationship", () => {
const queryJson = getJson("filterByRelationship.json")
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
expect(query).toEqual({
bindings: [relationshipLimit, "assembling", limit],
bindings: [baseLimit, "assembling", primaryLimit],
sql: expect.stringContaining(
multiline(`where COALESCE("b"."taskname" = $2, FALSE)
order by "a"."productname" asc nulls first, "a"."productid" asc limit $3`)
multiline(
`where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid"
where "c"."productid" = "a"."productid" and COALESCE("b"."taskname" = $2, FALSE)`
)
),
})
})
it("should handle fetching many to many relationships", () => {
const queryJson = getJson("fetchManyToMany.json")
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
expect(query).toEqual({
bindings: [relationshipLimit, limit],
bindings: [baseLimit, primaryLimit],
sql: expect.stringContaining(
multiline(`left join "products_tasks" as "c" on "a"."productid" = "c"."productid"
left join "tasks" as "b" on "b"."taskid" = "c"."taskid" `)
multiline(
`select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid"))
from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid"
where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $1`
)
),
})
})
@ -95,22 +97,21 @@ describe("Captures of real examples", () => {
it("should handle enrichment of rows", () => {
const queryJson = getJson("enrichRelationship.json")
const filters = queryJson.filters?.oneOf?.taskid as number[]
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
expect(query).toEqual({
bindings: [...filters, limit, ...filters, limit],
bindings: [baseLimit, ...filters, baseLimit],
sql: multiline(
`select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", "a"."taskid" as "a.taskid",
"a"."completed" as "a.completed", "a"."qaid" as "a.qaid", "b"."productname" as "b.productname", "b"."productid" as "b.productid"
from (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) as "a"
left join "products_tasks" as "c" on "a"."taskid" = "c"."taskid" left join "products" as "b" on "b"."productid" = "c"."productid"
where "a"."taskid" in ($4, $5) order by "a"."taskid" asc limit $6`
`select "a".*, (select json_agg(json_build_object('productname',"b"."productname",'productid',"b"."productid"))
from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid"
where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $1) as "b") as "products"
from "tasks" as "a" where "a"."taskid" in ($2, $3) order by "a"."taskid" asc limit $4`
),
})
})
it("should manage query with many relationship filters", () => {
const queryJson = getJson("manyRelationshipFilters.json")
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
const filters = queryJson.filters
const notEqualsValue = Object.values(filters?.notEqual!)[0]
const rangeValue: { high?: string | number; low?: string | number } =
@ -119,17 +120,18 @@ describe("Captures of real examples", () => {
expect(query).toEqual({
bindings: [
notEqualsValue,
relationshipLimit,
baseLimit,
baseLimit,
baseLimit,
rangeValue.low,
rangeValue.high,
equalValue,
true,
limit,
notEqualsValue,
primaryLimit,
],
sql: expect.stringContaining(
multiline(
`where "c"."year" between $3 and $4 and COALESCE("b"."productname" = $5, FALSE)`
`where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and "c"."year" between $4 and $5)`
)
),
})
@ -139,17 +141,19 @@ describe("Captures of real examples", () => {
describe("update", () => {
it("should handle performing a simple update", () => {
const queryJson = getJson("updateSimple.json")
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
expect(query).toEqual({
bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4,
"type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`),
sql: multiline(
`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4,
"type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`
),
})
})
it("should handle performing an update of relationships", () => {
const queryJson = getJson("updateRelationship.json")
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
expect(query).toEqual({
bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4,
@ -161,12 +165,12 @@ describe("Captures of real examples", () => {
describe("delete", () => {
it("should handle deleting with relationships", () => {
const queryJson = getJson("deleteSimple.json")
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
expect(query).toEqual({
bindings: ["ddd", ""],
sql: multiline(`delete from "compositetable" as "a"
where COALESCE("a"."keypartone" = $1, FALSE) and COALESCE("a"."keyparttwo" = $2, FALSE)
returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`),
returning "a".*`),
})
})
})
@ -174,7 +178,7 @@ describe("Captures of real examples", () => {
describe("returning (everything bar Postgres)", () => {
it("should be able to handle row returning", () => {
const queryJson = getJson("createSimple.json")
const SQL = new Sql(SqlClient.MS_SQL, limit)
const SQL = new Sql(SqlClient.MS_SQL, baseLimit)
let query = SQL._query(queryJson, { disableReturning: true })
expect(query).toEqual({
sql: "insert into [people] ([age], [name]) values (@p0, @p1)",
@ -187,10 +191,11 @@ describe("Captures of real examples", () => {
returningQuery = input
}, queryJson)
expect(returningQuery).toEqual({
sql: multiline(`select top (@p0) * from (select top (@p1) * from [people] where CASE WHEN [people].[name] = @p2
THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p3 THEN 1 ELSE 0 END = 1 order by [people].[name] asc) as [people]
where CASE WHEN [people].[name] = @p4 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p5 THEN 1 ELSE 0 END = 1`),
bindings: [5000, 1, "Test", 22, "Test", 22],
sql: multiline(
`select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1
and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc`
),
bindings: [1, "Test", 22],
})
})
})

11
packages/types/src/sdk/search.ts
View File

@ -134,6 +134,17 @@ export interface RelationshipsJson {
column: string
}
// TODO - this can be combined with the above type
export interface ManyToManyRelationshipJson {
through: string
from: string
to: string
fromPrimary: string
toPrimary: string
tableName: string
column: string
}
export interface QueryJson {
endpoint: {
datasourceId: string