diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 626ab3bf8e..b1a2bc060a 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -7,6 +7,7 @@ import { isValidFilter, isValidISODateString, sqlLog, + validateManyToMany, } from "./utils" import SqlTableQueryBuilder from "./sqlTable" import { @@ -93,6 +94,23 @@ class InternalBuilder { }) } + // states the various situations in which we need a full mapped select statement + private readonly SPECIAL_SELECT_CASES = { + POSTGRES_MONEY: (field: FieldSchema | undefined) => { + return ( + this.client === SqlClient.POSTGRES && + field?.externalType?.includes("money") + ) + }, + MSSQL_DATES: (field: FieldSchema | undefined) => { + return ( + this.client === SqlClient.MS_SQL && + field?.type === FieldType.DATETIME && + field.timeOnly + ) + }, + } + get table(): Table { return this.query.meta.table } @@ -126,87 +144,70 @@ class InternalBuilder { .join(".") } + private isFullSelectStatementRequired(): boolean { + const { meta } = this.query + for (let column of Object.values(meta.table.schema)) { + if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) { + return true + } else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) { + return true + } + } + return false + } + private generateSelectStatement(): (string | Knex.Raw)[] | "*" { - const { endpoint, resource, meta, tableAliases } = this.query + const { meta, endpoint, resource, tableAliases } = this.query if (!resource || !resource.fields || resource.fields.length === 0) { return "*" } - // no relationships - select everything in SQLite - if (this.client === SqlClient.SQL_LITE) { - const alias = tableAliases?.[endpoint.entityId] - ? tableAliases?.[endpoint.entityId] - : endpoint.entityId + const alias = tableAliases?.[endpoint.entityId] + ? tableAliases?.[endpoint.entityId] + : endpoint.entityId + const schema = meta.table.schema + if (!this.isFullSelectStatementRequired()) { return [this.knex.raw(`${this.quote(alias)}.*`)] } + // get just the fields for this table + return resource.fields + .map(field => { + const parts = field.split(/\./g) + let table: string | undefined = undefined + let column = parts[0] - const schema = meta.table.schema - return resource.fields.map(field => { - const parts = field.split(/\./g) - let table: string | undefined = undefined - let column: string | undefined = undefined + // Just a column name, e.g.: "column" + if (parts.length > 1) { + table = parts[0] + column = parts.slice(1).join(".") + } - // Just a column name, e.g.: "column" - if (parts.length === 1) { - column = parts[0] - } + return { table, column, field } + }) + .filter(({ table }) => !table || table === alias) + .map(({ table, column, field }) => { + const columnSchema = schema[column] - // A table name and a column name, e.g.: "table.column" - if (parts.length === 2) { - table = parts[0] - column = parts[1] - } + if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) { + return this.knex.raw( + `${this.quotedIdentifier( + [table, column].join(".") + )}::money::numeric as ${this.quote(field)}` + ) + } - // A link doc, e.g.: "table.doc1.fieldName" - if (parts.length > 2) { - table = parts[0] - column = parts.slice(1).join(".") - } + if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(columnSchema)) { + // Time gets returned as timestamp from mssql, not matching the expected + // HH:mm format + return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`) + } - if (!column) { - throw new Error(`Invalid field name: ${field}`) - } - - const columnSchema = schema[column] - - if ( - this.client === SqlClient.POSTGRES && - columnSchema?.externalType?.includes("money") - ) { - return this.knex.raw( - `${this.quotedIdentifier( - [table, column].join(".") - )}::money::numeric as ${this.quote(field)}` - ) - } - - if ( - this.client === SqlClient.MS_SQL && - columnSchema?.type === FieldType.DATETIME && - columnSchema.timeOnly - ) { - // Time gets returned as timestamp from mssql, not matching the expected - // HH:mm format - return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`) - } - - // There's at least two edge cases being handled in the expression below. - // 1. The column name could start/end with a space, and in that case we - // want to preseve that space. - // 2. Almost all column names are specified in the form table.column, except - // in the case of relationships, where it's table.doc1.column. In that - // case, we want to split it into `table`.`doc1.column` for reasons that - // aren't actually clear to me, but `table`.`doc1` breaks things with the - // sample data tests. - if (table) { - return this.knex.raw( - `${this.quote(table)}.${this.quote(column)} as ${this.quote(field)}` - ) - } else { - return this.knex.raw(`${this.quote(field)} as ${this.quote(field)}`) - } - }) + const quoted = table + ? `${this.quote(table)}.${this.quote(column)}` + : this.quote(field) + return this.knex.raw(quoted) + }) } // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses, @@ -368,35 +369,47 @@ class InternalBuilder { let subQuery = mainKnex .select(mainKnex.raw(1)) .from({ [toAlias]: relatedTableName }) - let mainTableRelatesTo = toAlias - if (relationship.through) { + const manyToMany = validateManyToMany(relationship) + if (manyToMany) { const throughAlias = - aliases?.[relationship.through] || relationship.through - let throughTable = this.tableNameWithSchema(relationship.through, { + aliases?.[manyToMany.through] || relationship.through + let throughTable = this.tableNameWithSchema(manyToMany.through, { alias: throughAlias, schema: endpoint.schema, }) - subQuery = subQuery.innerJoin(throughTable, function () { - // @ts-ignore - this.on( - `${toAlias}.${relationship.toPrimary}`, + subQuery = subQuery + // add a join through the junction table + .innerJoin(throughTable, function () { + // @ts-ignore + this.on( + `${toAlias}.${manyToMany.toPrimary}`, + "=", + `${throughAlias}.${manyToMany.to}` + ) + }) + // check the document in the junction table points to the main table + .where( + `${throughAlias}.${manyToMany.from}`, "=", - `${throughAlias}.${relationship.to}` + mainKnex.raw( + this.quotedIdentifier(`${fromAlias}.${manyToMany.fromPrimary}`) + ) ) - }) + // in SQS the same junction table is used for different many-to-many relationships between the + // two same tables, this is needed to avoid rows ending up in all columns if (this.client === SqlClient.SQL_LITE) { - subQuery = this.addJoinFieldCheck(subQuery, relationship) + subQuery = this.addJoinFieldCheck(subQuery, manyToMany) } - mainTableRelatesTo = throughAlias - } - // "join" to the main table, making sure the ID matches that of the main - subQuery = subQuery.where( - `${mainTableRelatesTo}.${relationship.from}`, - "=", - mainKnex.raw( - this.quotedIdentifier(`${fromAlias}.${relationship.fromPrimary}`) + } else { + // "join" to the main table, making sure the ID matches that of the main + subQuery = subQuery.where( + `${toAlias}.${relationship.to}`, + "=", + mainKnex.raw( + this.quotedIdentifier(`${fromAlias}.${relationship.from}`) + ) ) - ) + } query = query.whereExists(whereCb(subQuery)) break } @@ -478,12 +491,10 @@ class InternalBuilder { alias ? `${alias}.${updatedKey}` : updatedKey, value ) - } else if (isSqlite && shouldProcessRelationship) { + } else if (shouldProcessRelationship) { query = builder.addRelationshipForFilter(query, updatedKey, q => { return handleRelationship(q, updatedKey, value) }) - } else if (shouldProcessRelationship) { - query = handleRelationship(query, updatedKey, value) } } } @@ -849,6 +860,8 @@ class InternalBuilder { fromTable: string, relationships: RelationshipsJson[] ): Knex.QueryBuilder { + const sqlClient = this.client + const knex = this.knex const { resource, tableAliases: aliases, endpoint } = this.query const fields = resource?.fields || [] const jsonField = (field: string) => { @@ -862,7 +875,15 @@ class InternalBuilder { unaliased = parts.join(".") tableField = this.quote(unaliased) } - return `'${unaliased}',${tableField}` + let separator = "," + switch (sqlClient) { + case SqlClient.ORACLE: + separator = " VALUE " + break + case SqlClient.MS_SQL: + separator = ":" + } + return `'${unaliased}'${separator}${tableField}` } for (let relationship of relationships) { const { @@ -874,23 +895,15 @@ class InternalBuilder { toPrimary, } = relationship // skip invalid relationships - if (!toTable || !fromTable || !fromPrimary || !toPrimary) { + if (!toTable || !fromTable) { continue } - if (!throughTable) { - throw new Error("Only many-to-many implemented for JSON relationships") - } const toAlias = aliases?.[toTable] || toTable, - throughAlias = aliases?.[throughTable] || throughTable, fromAlias = aliases?.[fromTable] || fromTable let toTableWithSchema = this.tableNameWithSchema(toTable, { alias: toAlias, schema: endpoint.schema, }) - let throughTableWithSchema = this.tableNameWithSchema(throughTable, { - alias: throughAlias, - schema: endpoint.schema, - }) let relationshipFields = fields.filter( field => field.split(".")[0] === toAlias ) @@ -903,36 +916,87 @@ class InternalBuilder { const fieldList: string = relationshipFields .map(field => jsonField(field)) .join(",") - let rawJsonArray: Knex.Raw - switch (this.client) { + // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax + // it reduces the result set rather than limiting how much data it filters over + const primaryKey = `${toAlias}.${toPrimary || toKey}` + let subQuery: Knex.QueryBuilder = knex + .from(toTableWithSchema) + .limit(getBaseLimit()) + // add sorting to get consistent order + .orderBy(primaryKey) + + // many-to-many relationship with junction table + if (throughTable && toPrimary && fromPrimary) { + const throughAlias = aliases?.[throughTable] || throughTable + let throughTableWithSchema = this.tableNameWithSchema(throughTable, { + alias: throughAlias, + schema: endpoint.schema, + }) + subQuery = subQuery + .join(throughTableWithSchema, function () { + this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`) + }) + .where( + `${throughAlias}.${fromKey}`, + "=", + knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`)) + ) + } + // one-to-many relationship with foreign key + else { + subQuery = subQuery.where( + `${toAlias}.${toKey}`, + "=", + knex.raw(this.quotedIdentifier(`${fromAlias}.${fromKey}`)) + ) + } + + const standardWrap = (select: string): Knex.QueryBuilder => { + subQuery = subQuery.select(`${toAlias}.*`) + // @ts-ignore - the from alias syntax isn't in Knex typing + return knex.select(knex.raw(select)).from({ + [toAlias]: subQuery, + }) + } + let wrapperQuery: Knex.QueryBuilder | Knex.Raw + switch (sqlClient) { case SqlClient.SQL_LITE: - rawJsonArray = this.knex.raw( + // need to check the junction table document is to the right column, this is just for SQS + subQuery = this.addJoinFieldCheck(subQuery, relationship) + wrapperQuery = standardWrap( `json_group_array(json_object(${fieldList}))` ) break + case SqlClient.POSTGRES: + wrapperQuery = standardWrap( + `json_agg(json_build_object(${fieldList}))` + ) + break + case SqlClient.MY_SQL: + wrapperQuery = subQuery.select( + knex.raw(`json_arrayagg(json_object(${fieldList}))`) + ) + break + case SqlClient.ORACLE: + wrapperQuery = standardWrap( + `json_arrayagg(json_object(${fieldList}))` + ) + break + case SqlClient.MS_SQL: + wrapperQuery = knex.raw( + `(SELECT ${this.quote(toAlias)} = (${knex + .select(`${fromAlias}.*`) + // @ts-ignore - from alias syntax not TS supported + .from({ + [fromAlias]: subQuery.select(`${toAlias}.*`), + })} FOR JSON PATH))` + ) + break default: - throw new Error(`JSON relationships not implement for ${this.client}`) + throw new Error(`JSON relationships not implement for ${sqlClient}`) } - let subQuery = this.knex - .select(rawJsonArray) - .from(toTableWithSchema) - .join(throughTableWithSchema, function () { - this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`) - }) - .where( - `${throughAlias}.${fromKey}`, - "=", - this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`)) - ) - // relationships should never have more than the base limit - .limit(getBaseLimit()) - // add sorting to get consistent order - .orderBy(`${toAlias}.${toPrimary}`) - // need to check the junction table document is to the right column - if (this.client === SqlClient.SQL_LITE) { - subQuery = this.addJoinFieldCheck(subQuery, relationship) - } - query = query.select({ [relationship.column]: subQuery }) + + query = query.select({ [relationship.column]: wrapperQuery }) } return query } @@ -1179,14 +1243,12 @@ class InternalBuilder { ? query.select(this.generateSelectStatement()) : this.addDistinctCount(query) // have to add after as well (this breaks MS-SQL) - if (this.client !== SqlClient.MS_SQL && !counting) { + if (!counting) { query = this.addSorting(query) } // handle joins - if (relationships && this.client === SqlClient.SQL_LITE) { + if (relationships) { query = this.addJsonRelationships(query, tableName, relationships) - } else if (relationships) { - query = this.addRelationships(query, tableName, relationships) } return this.addFilters(query, filters, { relationship: true }) diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts index 1b32cc6da7..1b80ff337d 100644 --- a/packages/backend-core/src/sql/utils.ts +++ b/packages/backend-core/src/sql/utils.ts @@ -1,4 +1,11 @@ -import { DocumentType, SqlQuery, Table, TableSourceType } from "@budibase/types" +import { + DocumentType, + ManyToManyRelationshipJson, + RelationshipsJson, + SqlQuery, + Table, + TableSourceType, +} from "@budibase/types" import { DEFAULT_BB_DATASOURCE_ID } from "../constants" import { Knex } from "knex" import { SEPARATOR } from "../db" @@ -163,3 +170,24 @@ export function sqlLog(client: string, query: string, values?: any[]) { } console.log(string) } + +function isValidManyToManyRelationship( + relationship: RelationshipsJson +): relationship is ManyToManyRelationshipJson { + return ( + !!relationship.through && + !!relationship.fromPrimary && + !!relationship.from && + !!relationship.toPrimary && + !!relationship.to + ) +} + +export function validateManyToMany( + relationship: RelationshipsJson +): ManyToManyRelationshipJson | undefined { + if (isValidManyToManyRelationship(relationship)) { + return relationship + } + return undefined +} diff --git a/packages/server/datasource-sha.env b/packages/server/datasource-sha.env index 9b935ed8eb..61249d530c 100644 --- a/packages/server/datasource-sha.env +++ b/packages/server/datasource-sha.env @@ -1,4 +1,4 @@ -MSSQL_SHA=sha256:c4369c38385eba011c10906dc8892425831275bb035d5ce69656da8e29de50d8 +MSSQL_SHA=sha256:3b913841850a4d57fcfcb798be06acc88ea0f2acc5418bc0c140a43e91c4a545 MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index 8f3607bc73..b754e288ed 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -1,6 +1,10 @@ // need to handle table name + field or just field, depending on if relationships used import { FieldSchema, FieldType, Row, Table } from "@budibase/types" -import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core" +import { + helpers, + PROTECTED_EXTERNAL_COLUMNS, + PROTECTED_INTERNAL_COLUMNS, +} from "@budibase/shared-core" import { generateRowIdField } from "../../../../integrations/utils" function extractFieldValue({ @@ -61,11 +65,13 @@ export function generateIdForRow( export function basicProcessing({ row, table, + tables, isLinked, sqs, }: { row: Row table: Table + tables: Table[] isLinked: boolean sqs?: boolean }): Row { @@ -86,24 +92,69 @@ export function basicProcessing({ thisRow[fieldName] = value } } + let columns: string[] = Object.keys(table.schema) if (!sqs) { thisRow._id = generateIdForRow(row, table, isLinked) thisRow.tableId = table._id thisRow._rev = "rev" + columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) } else { - const columns = Object.keys(table.schema) + columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) { - const schema: FieldSchema | undefined = table.schema[internalColumn] - let value = extractFieldValue({ + thisRow[internalColumn] = extractFieldValue({ row, tableName: table._id!, fieldName: internalColumn, isLinked, }) - if (sqs && schema?.type === FieldType.LINK && typeof value === "string") { - value = JSON.parse(value) + } + } + for (let col of columns) { + const schema: FieldSchema | undefined = table.schema[col] + if (schema?.type !== FieldType.LINK) { + continue + } + const relatedTable = tables.find(tbl => tbl._id === schema.tableId) + if (!relatedTable) { + continue + } + const value = extractFieldValue({ + row, + tableName: table._id!, + fieldName: col, + isLinked, + }) + const array: Row[] = Array.isArray(value) + ? value + : typeof value === "string" + ? JSON.parse(value) + : undefined + if (array) { + thisRow[col] = array + // make sure all of them have an _id + if (Array.isArray(thisRow[col])) { + const sortField = + relatedTable.primaryDisplay || relatedTable.primary![0]! + thisRow[col] = (thisRow[col] as Row[]) + .map(relatedRow => { + relatedRow._id = relatedRow._id + ? relatedRow._id + : generateIdForRow(relatedRow, relatedTable) + return relatedRow + }) + .sort((a, b) => { + const aField = a?.[sortField], + bField = b?.[sortField] + if (!aField) { + return 1 + } else if (!bField) { + return -1 + } + return aField.localeCompare + ? aField.localeCompare(bField) + : aField - bField + }) } - thisRow[internalColumn] = value } } return thisRow diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index a24ec17c26..249bb43bbc 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -7,11 +7,9 @@ import { ManyToManyRelationshipFieldMetadata, RelationshipFieldMetadata, RelationshipsJson, - Row, Table, } from "@budibase/types" import { breakExternalTableId } from "../../../../integrations/utils" -import { basicProcessing } from "./basic" import { generateJunctionTableID } from "../../../../db/utils" type TableMap = Record @@ -22,87 +20,6 @@ export function isManyToMany( return !!(field as ManyToManyRelationshipFieldMetadata).through } -function isCorrectRelationship( - relationship: RelationshipsJson, - table1: Table, - table2: Table, - row: Row -): boolean { - const junctionTableId = generateJunctionTableID(table1._id!, table2._id!) - const possibleColumns = [ - `${junctionTableId}.doc1.fieldName`, - `${junctionTableId}.doc2.fieldName`, - ] - return !!possibleColumns.find(col => row[col] === relationship.column) -} - -/** - * This iterates through the returned rows and works out what elements of the rows - * actually match up to another row (based on primary keys) - this is pretty specific - * to SQL and the way that SQL relationships are returned based on joins. - * This is complicated, but the idea is that when a SQL query returns all the relations - * will be separate rows, with all of the data in each row. We have to decipher what comes - * from where (which tables) and how to convert that into budibase columns. - */ -export async function updateRelationshipColumns( - table: Table, - tables: TableMap, - row: Row, - rows: { [key: string]: Row }, - relationships: RelationshipsJson[], - opts?: { sqs?: boolean } -) { - const columns: { [key: string]: any } = {} - for (let relationship of relationships) { - const linkedTable = tables[relationship.tableName] - if (!linkedTable) { - continue - } - const fromColumn = `${table.name}.${relationship.from}` - const toColumn = `${linkedTable.name}.${relationship.to}` - // this is important when working with multiple relationships - // between the same tables, don't want to overlap/multiply the relations - if ( - !relationship.through && - row[fromColumn]?.toString() !== row[toColumn]?.toString() - ) { - continue - } - - let linked = basicProcessing({ - row, - table: linkedTable, - isLinked: true, - sqs: opts?.sqs, - }) - if (!linked._id) { - continue - } - if ( - !opts?.sqs || - isCorrectRelationship(relationship, table, linkedTable, row) - ) { - columns[relationship.column] = linked - } - } - for (let [column, related] of Object.entries(columns)) { - if (!row._id) { - continue - } - const rowId: string = row._id - if (!Array.isArray(rows[rowId][column])) { - rows[rowId][column] = [] - } - // make sure relationship hasn't been found already - if ( - !rows[rowId][column].find((relation: Row) => relation._id === related._id) - ) { - rows[rowId][column].push(related) - } - } - return rows -} - /** * Gets the list of relationship JSON structures based on the columns in the table, * this will be used by the underlying library to build whatever relationship mechanism diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index e52ee63e42..45f0cef085 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -13,7 +13,7 @@ import { processDates, processFormulas, } from "../../../../utilities/rowProcessor" -import { isKnexEmptyReadResponse, updateRelationshipColumns } from "./sqlUtils" +import { isKnexEmptyReadResponse } from "./sqlUtils" import { basicProcessing, generateIdForRow, @@ -149,22 +149,11 @@ export async function sqlOutputProcessing( rowId = generateIdForRow(row, table) row._id = rowId } - // this is a relationship of some sort - if (!opts?.sqs && finalRows[rowId]) { - finalRows = await updateRelationshipColumns( - table, - tables, - row, - finalRows, - relationships, - opts - ) - continue - } const thisRow = fixArrayTypes( basicProcessing({ row, table, + tables: Object.values(tables), isLinked: false, sqs: opts?.sqs, }), @@ -175,18 +164,6 @@ export async function sqlOutputProcessing( } finalRows[thisRow._id] = fixBooleanFields({ row: thisRow, table }) - - // do this at end once its been added to the final rows - if (!opts?.sqs) { - finalRows = await updateRelationshipColumns( - table, - tables, - row, - finalRows, - relationships, - opts - ) - } } // make sure all related rows are correct diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index a84b243e2d..4bd1951d67 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -832,10 +832,12 @@ describe.each( }, }) expect(res).toHaveLength(1) - expect(res[0]).toEqual({ - id: 2, - name: "two", - }) + expect(res[0]).toEqual( + expect.objectContaining({ + id: 2, + name: "two", + }) + ) }) // this parameter really only impacts SQL queries diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index de7fac4f1b..f5c5ade2f8 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2690,82 +2690,6 @@ describe.each([ }) }) - // TODO: when all SQL databases use the same mechanism - remove this test, new relationship system doesn't have this problem - !isInternal && - describe("pagination edge case with relationships", () => { - let mainRows: Row[] = [] - - beforeAll(async () => { - const toRelateTable = await createTable({ - name: { - name: "name", - type: FieldType.STRING, - }, - }) - table = await createTable({ - name: { - name: "name", - type: FieldType.STRING, - }, - rel: { - name: "rel", - type: FieldType.LINK, - relationshipType: RelationshipType.MANY_TO_ONE, - tableId: toRelateTable._id!, - fieldName: "rel", - }, - }) - const relatedRows = await Promise.all([ - config.api.row.save(toRelateTable._id!, { name: "tag 1" }), - config.api.row.save(toRelateTable._id!, { name: "tag 2" }), - config.api.row.save(toRelateTable._id!, { name: "tag 3" }), - config.api.row.save(toRelateTable._id!, { name: "tag 4" }), - config.api.row.save(toRelateTable._id!, { name: "tag 5" }), - config.api.row.save(toRelateTable._id!, { name: "tag 6" }), - ]) - mainRows = await Promise.all([ - config.api.row.save(table._id!, { - name: "product 1", - rel: relatedRows.map(row => row._id), - }), - config.api.row.save(table._id!, { - name: "product 2", - rel: [], - }), - config.api.row.save(table._id!, { - name: "product 3", - rel: [], - }), - ]) - }) - - it("can still page when the hard limit is hit", async () => { - await withCoreEnv( - { - SQL_MAX_ROWS: "6", - }, - async () => { - const params: Omit = { - query: {}, - paginate: true, - limit: 3, - sort: "name", - sortType: SortType.STRING, - sortOrder: SortOrder.ASCENDING, - } - const page1 = await expectSearch(params).toContain([mainRows[0]]) - expect(page1.hasNextPage).toBe(true) - expect(page1.bookmark).toBeDefined() - const page2 = await expectSearch({ - ...params, - bookmark: page1.bookmark, - }).toContain([mainRows[1], mainRows[2]]) - expect(page2.hasNextPage).toBe(false) - } - ) - }) - }) - isSql && describe("primaryDisplay", () => { beforeAll(async () => { diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 88c75891e6..0a07371cd3 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -343,9 +343,9 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { err.number ) if (readableMessage) { - throw new Error(readableMessage) + throw new Error(readableMessage, { cause: err }) } else { - throw new Error(err.message as string) + throw new Error(err.message as string, { cause: err }) } } } diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index a6e63c434d..c434ec42cb 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -160,16 +160,16 @@ describe("SQL query builder", () => { it("should add the schema to the LEFT JOIN", () => { const query = sql._query(generateRelationshipJson({ schema: "production" })) expect(query).toEqual({ - bindings: [500, 5000], - sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "production"."brands" order by "test"."id" asc limit $1) as "brands" left join "production"."products" as "products" on "brands"."brand_id" = "products"."brand_id" order by "test"."id" asc limit $2`, + bindings: [5000, limit], + sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "production"."brands" order by "test"."id" asc limit $2`, }) }) it("should handle if the schema is not present when doing a LEFT JOIN", () => { const query = sql._query(generateRelationshipJson()) expect(query).toEqual({ - bindings: [500, 5000], - sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "brands" order by "test"."id" asc limit $1) as "brands" left join "products" as "products" on "brands"."brand_id" = "products"."brand_id" order by "test"."id" asc limit $2`, + bindings: [5000, limit], + sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "brands" order by "test"."id" asc limit $2`, }) }) @@ -178,8 +178,8 @@ describe("SQL query builder", () => { generateManyRelationshipJson({ schema: "production" }) ) expect(query).toEqual({ - bindings: [500, 5000], - sql: `select "stores"."store_id" as "stores.store_id", "stores"."store_name" as "stores.store_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name" from (select * from "production"."stores" order by "test"."id" asc limit $1) as "stores" left join "production"."stocks" as "stocks" on "stores"."store_id" = "stocks"."store_id" left join "production"."products" as "products" on "products"."product_id" = "stocks"."product_id" order by "test"."id" asc limit $2`, + bindings: [5000, limit], + sql: `select "stores".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" inner join "production"."stocks" as "stocks" on "products"."product_id" = "stocks"."product_id" where "stocks"."store_id" = "stores"."store_id" order by "products"."product_id" asc limit $1) as "products") as "products" from "production"."stores" order by "test"."id" asc limit $2`, }) }) @@ -194,8 +194,8 @@ describe("SQL query builder", () => { }) ) expect(query).toEqual({ - bindings: ["john%", limit, "john%", 5000], - sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" where LOWER("test"."name") LIKE :3 order by "test"."id" asc) where rownum <= :4`, + bindings: ["john%", limit], + sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`, }) query = new Sql(SqlClient.ORACLE, limit)._query( @@ -210,8 +210,8 @@ describe("SQL query builder", () => { ) const filterSet = [`%20%`, `%25%`, `%"john"%`, `%"mary"%`] expect(query).toEqual({ - bindings: [...filterSet, limit, ...filterSet, 5000], - sql: `select * from (select * from (select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5) "test" where COALESCE(LOWER("test"."age"), '') LIKE :6 AND COALESCE(LOWER("test"."age"), '') LIKE :7 and COALESCE(LOWER("test"."name"), '') LIKE :8 AND COALESCE(LOWER("test"."name"), '') LIKE :9 order by "test"."id" asc) where rownum <= :10`, + bindings: [...filterSet, limit], + sql: `select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5`, }) query = new Sql(SqlClient.ORACLE, limit)._query( @@ -224,8 +224,8 @@ describe("SQL query builder", () => { }) ) expect(query).toEqual({ - bindings: [`%jo%`, limit, `%jo%`, 5000], - sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" where LOWER("test"."name") LIKE :3 order by "test"."id" asc) where rownum <= :4`, + bindings: [`%jo%`, limit], + sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`, }) }) @@ -242,8 +242,8 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ - bindings: ["John", limit, "John", 5000], - sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2) "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :3) order by "test"."id" asc) where rownum <= :4`, + bindings: ["John", limit], + sql: `select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2`, }) }) @@ -260,8 +260,8 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ - bindings: ["John", limit, "John", 5000], - sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2) "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :3) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :4`, + bindings: ["John", limit], + sql: `select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2`, }) }) }) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 6f34f4eb89..1ba37018dc 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -32,8 +32,8 @@ function multiline(sql: string) { } describe("Captures of real examples", () => { - const limit = 5000 - const relationshipLimit = 100 + const baseLimit = 5000 + const primaryLimit = 100 function getJson(name: string): QueryJson { return require(join(__dirname, "sqlQueryJson", name)) as QueryJson @@ -42,7 +42,7 @@ describe("Captures of real examples", () => { describe("create", () => { it("should create a row with relationships", () => { const queryJson = getJson("createWithRelationships.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ bindings: ["A Street", 34, "London", "A", "B", "designer", 1990], sql: multiline(`insert into "persons" ("address", "age", "city", "firstname", "lastname", "type", "year") @@ -54,40 +54,42 @@ describe("Captures of real examples", () => { describe("read", () => { it("should handle basic retrieval with relationships", () => { const queryJson = getJson("basicFetchWithRelationships.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ - bindings: [relationshipLimit, limit], + bindings: [baseLimit, baseLimit, primaryLimit], sql: expect.stringContaining( - multiline(`select "a"."year" as "a.year", "a"."firstname" as "a.firstname", "a"."personid" as "a.personid", - "a"."address" as "a.address", "a"."age" as "a.age", "a"."type" as "a.type", "a"."city" as "a.city", - "a"."lastname" as "a.lastname", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", - "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid", - "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", - "b"."completed" as "b.completed", "b"."qaid" as "b.qaid"`) + multiline( + `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid",'executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")` + ) ), }) }) it("should handle filtering by relationship", () => { const queryJson = getJson("filterByRelationship.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ - bindings: [relationshipLimit, "assembling", limit], + bindings: [baseLimit, "assembling", primaryLimit], sql: expect.stringContaining( - multiline(`where COALESCE("b"."taskname" = $2, FALSE) - order by "a"."productname" asc nulls first, "a"."productid" asc limit $3`) + multiline( + `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" + where "c"."productid" = "a"."productid" and COALESCE("b"."taskname" = $2, FALSE)` + ) ), }) }) it("should handle fetching many to many relationships", () => { const queryJson = getJson("fetchManyToMany.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ - bindings: [relationshipLimit, limit], + bindings: [baseLimit, primaryLimit], sql: expect.stringContaining( - multiline(`left join "products_tasks" as "c" on "a"."productid" = "c"."productid" - left join "tasks" as "b" on "b"."taskid" = "c"."taskid" `) + multiline( + `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")) + from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" + where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $1` + ) ), }) }) @@ -95,22 +97,21 @@ describe("Captures of real examples", () => { it("should handle enrichment of rows", () => { const queryJson = getJson("enrichRelationship.json") const filters = queryJson.filters?.oneOf?.taskid as number[] - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ - bindings: [...filters, limit, ...filters, limit], + bindings: [baseLimit, ...filters, baseLimit], sql: multiline( - `select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", "a"."taskid" as "a.taskid", - "a"."completed" as "a.completed", "a"."qaid" as "a.qaid", "b"."productname" as "b.productname", "b"."productid" as "b.productid" - from (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) as "a" - left join "products_tasks" as "c" on "a"."taskid" = "c"."taskid" left join "products" as "b" on "b"."productid" = "c"."productid" - where "a"."taskid" in ($4, $5) order by "a"."taskid" asc limit $6` + `select "a".*, (select json_agg(json_build_object('productname',"b"."productname",'productid',"b"."productid")) + from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" + where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $1) as "b") as "products" + from "tasks" as "a" where "a"."taskid" in ($2, $3) order by "a"."taskid" asc limit $4` ), }) }) it("should manage query with many relationship filters", () => { const queryJson = getJson("manyRelationshipFilters.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) const filters = queryJson.filters const notEqualsValue = Object.values(filters?.notEqual!)[0] const rangeValue: { high?: string | number; low?: string | number } = @@ -119,17 +120,18 @@ describe("Captures of real examples", () => { expect(query).toEqual({ bindings: [ - notEqualsValue, - relationshipLimit, + baseLimit, + baseLimit, + baseLimit, rangeValue.low, rangeValue.high, equalValue, - true, - limit, + notEqualsValue, + primaryLimit, ], sql: expect.stringContaining( multiline( - `where "c"."year" between $3 and $4 and COALESCE("b"."productname" = $5, FALSE)` + `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and "c"."year" between $4 and $5)` ) ), }) @@ -139,17 +141,19 @@ describe("Captures of real examples", () => { describe("update", () => { it("should handle performing a simple update", () => { const queryJson = getJson("updateSimple.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5], - sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, - "type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`), + sql: multiline( + `update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, + "type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *` + ), }) }) it("should handle performing an update of relationships", () => { const queryJson = getJson("updateRelationship.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5], sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, @@ -161,12 +165,12 @@ describe("Captures of real examples", () => { describe("delete", () => { it("should handle deleting with relationships", () => { const queryJson = getJson("deleteSimple.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ bindings: ["ddd", ""], sql: multiline(`delete from "compositetable" as "a" where COALESCE("a"."keypartone" = $1, FALSE) and COALESCE("a"."keyparttwo" = $2, FALSE) - returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`), + returning "a".*`), }) }) }) @@ -174,7 +178,7 @@ describe("Captures of real examples", () => { describe("returning (everything bar Postgres)", () => { it("should be able to handle row returning", () => { const queryJson = getJson("createSimple.json") - const SQL = new Sql(SqlClient.MS_SQL, limit) + const SQL = new Sql(SqlClient.MS_SQL, baseLimit) let query = SQL._query(queryJson, { disableReturning: true }) expect(query).toEqual({ sql: "insert into [people] ([age], [name]) values (@p0, @p1)", @@ -187,10 +191,11 @@ describe("Captures of real examples", () => { returningQuery = input }, queryJson) expect(returningQuery).toEqual({ - sql: multiline(`select top (@p0) * from (select top (@p1) * from [people] where CASE WHEN [people].[name] = @p2 - THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p3 THEN 1 ELSE 0 END = 1 order by [people].[name] asc) as [people] - where CASE WHEN [people].[name] = @p4 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p5 THEN 1 ELSE 0 END = 1`), - bindings: [5000, 1, "Test", 22, "Test", 22], + sql: multiline( + `select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1 + and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc` + ), + bindings: [1, "Test", 22], }) }) }) diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 6feea40766..7d61aebdfb 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -134,6 +134,17 @@ export interface RelationshipsJson { column: string } +// TODO - this can be combined with the above type +export interface ManyToManyRelationshipJson { + through: string + from: string + to: string + fromPrimary: string + toPrimary: string + tableName: string + column: string +} + export interface QueryJson { endpoint: { datasourceId: string