From f6ecd423ce614ef473dcb266e2bfabd42d634132 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 24 Nov 2021 20:55:03 +0000 Subject: [PATCH] Fixing some issues discovered with many to many relationships in SQL, as well as problems uncovered by #3531. --- .../api/controllers/row/ExternalRequest.ts | 64 +++++++++++++------ packages/server/src/integrations/base/sql.ts | 30 +++++---- packages/server/src/integrations/index.ts | 4 +- packages/server/src/integrations/mysql.ts | 4 +- packages/server/src/integrations/oracle.ts | 29 ++++++--- packages/server/src/integrations/utils.ts | 8 ++- 6 files changed, 93 insertions(+), 46 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 95710cd688..2226dc99be 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -197,9 +197,9 @@ module External { return row } - function isMany(field: FieldSchema) { + function isOneSide(field: FieldSchema) { return ( - field.relationshipType && field.relationshipType.split("-")[0] === "many" + field.relationshipType && field.relationshipType.split("-")[0] === "one" ) } @@ -274,25 +274,37 @@ module External { const linkTable = this.tables[linkTableName] // @ts-ignore const linkTablePrimary = linkTable.primary[0] - if (!isMany(field)) { + // one to many + if (isOneSide(field)) { newRow[field.foreignKey || linkTablePrimary] = breakRowIdField( row[key][0] )[0] - } else { + } + // many to many + else if (field.through) { // we're not inserting a doc, will be a bunch of update calls - const isUpdate = !field.through - const thisKey: string = isUpdate - ? "id" - : field.throughTo || linkTablePrimary - // @ts-ignore - const otherKey: string = isUpdate - ? field.fieldName - : field.throughFrom || tablePrimary + const otherKey: string = field.throughFrom || linkTablePrimary + const thisKey: string = field.throughTo || tablePrimary row[key].map((relationship: any) => { - // we don't really support composite keys for relationships, this is why [0] is used manyRelationships.push({ tableId: field.through || field.tableId, - isUpdate, + isUpdate: false, + key: otherKey, + [otherKey]: breakRowIdField(relationship)[0], + // leave the ID for enrichment later + [thisKey]: `{{ literal ${tablePrimary} }}`, + }) + }) + } + // many to one + else { + const thisKey: string = "id" + // @ts-ignore + const otherKey: string = field.fieldName + row[key].map((relationship: any) => { + manyRelationships.push({ + tableId: field.tableId, + isUpdate: true, key: otherKey, [thisKey]: breakRowIdField(relationship)[0], // leave the ID for enrichment later @@ -425,8 +437,8 @@ module External { ) definition.through = throughTableName // don't support composite keys for relationships - definition.from = field.throughFrom || table.primary[0] - definition.to = field.throughTo || linkTable.primary[0] + definition.from = field.throughTo || table.primary[0] + definition.to = field.throughFrom || linkTable.primary[0] definition.fromPrimary = table.primary[0] definition.toPrimary = linkTable.primary[0] } @@ -448,24 +460,36 @@ module External { // make a new request to get the row with all its relationships // we need this to work out if any relationships need removed for (let field of Object.values(table.schema)) { - if (field.type !== FieldTypes.LINK || !field.fieldName) { + if ( + field.type !== FieldTypes.LINK || + !field.fieldName || + isOneSide(field) + ) { continue } const isMany = field.relationshipType === RelationshipTypes.MANY_TO_MANY const tableId = isMany ? field.through : field.tableId - const manyKey = field.throughFrom || primaryKey + const { tableName: relatedTableName } = breakExternalTableId(tableId) + // @ts-ignore + const linkPrimaryKey = this.tables[relatedTableName].primary[0] + const manyKey = field.throughTo || primaryKey + const lookupField = isMany ? primaryKey : field.foreignKey const fieldName = isMany ? manyKey : field.fieldName + if (!lookupField || !row[lookupField]) { + continue + } const response = await getDatasourceAndQuery(this.appId, { endpoint: getEndpoint(tableId, DataSourceOperation.READ), filters: { equal: { - [fieldName]: row[primaryKey], + [fieldName]: row[lookupField], }, }, }) // this is the response from knex if no rows found const rows = !response[0].read ? response : [] - related[fieldName] = { rows, isMany, tableId } + const storeTo = isMany ? field.throughFrom || linkPrimaryKey : manyKey + related[storeTo] = { rows, isMany, tableId } } return related } diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index f4d95f48a1..471774db3d 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -1,11 +1,11 @@ import { Knex, knex } from "knex" import { - Operation, PaginationJson, + Operation, QueryJson, QueryOptions, RelationshipsJson, SearchFilters, - SortDirection, SortJson, + SortDirection, } from "../../definitions/datasource" import { isIsoDateString, SqlClients } from "../utils" import SqlTableQueryBuilder from "./sqlTable" @@ -21,6 +21,9 @@ function parse(input: any) { if (Array.isArray(input)) { return JSON.stringify(input) } + if (input == undefined) { + return null + } if (typeof input !== "string") { return input } @@ -43,7 +46,10 @@ function parseBody(body: any) { return body } -function parseFilters(filters: SearchFilters): SearchFilters { +function parseFilters(filters: SearchFilters | undefined): SearchFilters { + if (!filters) { + return {} + } for (let [key, value] of Object.entries(filters)) { let parsed if (typeof value === "object") { @@ -152,21 +158,19 @@ class InternalBuilder { return query } - addSorting( - query: KnexQuery, - sort: SortJson | undefined, - paginate: PaginationJson | undefined - ): KnexQuery { + addSorting(query: KnexQuery, json: QueryJson): KnexQuery { + let { sort, paginate } = json if (!sort) { return query } + const table = json.meta?.table for (let [key, value] of Object.entries(sort)) { const direction = value === SortDirection.ASCENDING ? "asc" : "desc" - query = query.orderBy(key, direction) + query = query.orderBy(`${table?.name}.${key}`, direction) } if (this.client === SqlClients.MS_SQL && !sort && paginate?.limit) { // @ts-ignore - query = query.orderBy(json.meta?.table?.primary[0]) + query = query.orderBy(`${table?.name}.${table?.primary[0]}`) } return query } @@ -270,15 +274,15 @@ class InternalBuilder { } query = this.addFilters(tableName, query, filters) // add sorting to pre-query - query = this.addSorting(query, sort, paginate) + query = this.addSorting(query, json) // @ts-ignore - let preQuery : KnexQuery = knex({ + let preQuery: KnexQuery = knex({ // @ts-ignore [tableName]: query, }).select(selectStatement) // have to add after as well (this breaks MS-SQL) if (this.client !== SqlClients.MS_SQL) { - preQuery = this.addSorting(preQuery, sort, paginate) + preQuery = this.addSorting(preQuery, json) } // handle joins return this.addRelationships( diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index eff499f978..8f2f083fc5 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -40,9 +40,9 @@ const INTEGRATIONS = { } // optionally add oracle integration if the oracle binary can be installed -if (!(process.arch === 'arm64' && process.platform === 'darwin')) { +if (!(process.arch === "arm64" && process.platform === "darwin")) { const oracle = require("./oracle") - DEFINITIONS[SourceNames.ORACLE] = oracle.schema + DEFINITIONS[SourceNames.ORACLE] = oracle.schema INTEGRATIONS[SourceNames.ORACLE] = oracle.integration } diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index a870d2a3ae..98584e04d2 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -134,7 +134,9 @@ module MySQLModule { false ) const tableNames = tablesResp.map( - (obj: any) => obj[`Tables_in_${database}`] || obj[`Tables_in_${database.toLowerCase()}`] + (obj: any) => + obj[`Tables_in_${database}`] || + obj[`Tables_in_${database.toLowerCase()}`] ) for (let tableName of tableNames) { const primaryKeys = [] diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index c74977bece..13658399db 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -352,14 +352,23 @@ module OracleModule { * Knex default returning behaviour does not work with oracle * Manually add the behaviour for the return column */ - private addReturning(query: SqlQuery, bindings: BindParameters, returnColumn: string) { + private addReturning( + query: SqlQuery, + bindings: BindParameters, + returnColumn: string + ) { if (bindings instanceof Array) { bindings.push({ dir: oracledb.BIND_OUT }) - query.sql = query.sql + ` returning \"${returnColumn}\" into :${bindings.length}` + query.sql = + query.sql + ` returning \"${returnColumn}\" into :${bindings.length}` } } - private async internalQuery(query: SqlQuery, returnColum?: string, operation?: string): Promise> { + private async internalQuery( + query: SqlQuery, + returnColum?: string, + operation?: string + ): Promise> { let connection try { connection = await this.getConnection() @@ -367,7 +376,10 @@ module OracleModule { const options: ExecuteOptions = { autoCommit: true } const bindings: BindParameters = query.bindings || [] - if (returnColum && (operation === Operation.CREATE || operation === Operation.UPDATE)) { + if ( + returnColum && + (operation === Operation.CREATE || operation === Operation.UPDATE) + ) { this.addReturning(query, bindings, returnColum) } @@ -414,14 +426,14 @@ module OracleModule { return response.rows ? response.rows : [] } - async update(query: SqlQuery | string): Promise { + async update(query: SqlQuery | string): Promise { const response = await this.internalQuery(getSqlQuery(query)) return response.rows && response.rows.length ? response.rows : [{ updated: true }] } - async delete(query: SqlQuery | string): Promise { + async delete(query: SqlQuery | string): Promise { const response = await this.internalQuery(getSqlQuery(query)) return response.rows && response.rows.length ? response.rows @@ -431,8 +443,9 @@ module OracleModule { async query(json: QueryJson) { const primaryKeys = json.meta!.table!.primary const primaryKey = primaryKeys ? primaryKeys[0] : undefined - const queryFn = (query: any, operation: string) => this.internalQuery(query, primaryKey, operation) - const processFn = (response: any) => response.rows ? response.rows : [] + const queryFn = (query: any, operation: string) => + this.internalQuery(query, primaryKey, operation) + const processFn = (response: any) => (response.rows ? response.rows : []) const output = await this.queryWithReturning(json, queryFn, processFn) return output } diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 07cf0505bf..97380b1b5b 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -2,7 +2,11 @@ import { SqlQuery } from "../definitions/datasource" import { Datasource, Table } from "../definitions/common" import { SourceNames } from "../definitions/datasource" const { DocumentTypes, SEPARATOR } = require("../db/utils") -const { FieldTypes, BuildSchemaErrors, InvalidColumns } = require("../constants") +const { + FieldTypes, + BuildSchemaErrors, + InvalidColumns, +} = require("../constants") const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` const ROW_ID_REGEX = /^\[.*]$/g @@ -42,7 +46,7 @@ export enum SqlClients { MS_SQL = "mssql", POSTGRES = "pg", MY_SQL = "mysql", - ORACLE = "oracledb" + ORACLE = "oracledb", } export function isExternalTable(tableId: string) {