From ebf82a0ef5dc24f42c4a735652c997cb3ca86636 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 21 Nov 2024 17:33:47 +0000 Subject: [PATCH 01/30] wip --- .../server/src/api/routes/tests/row.spec.ts | 53 +++++++++++++++++++ packages/server/src/integrations/postgres.ts | 4 ++ 2 files changed, 57 insertions(+) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 02995e6d0a..f0dc27708c 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3495,6 +3495,59 @@ if (descriptions.length) { ) }) }) + + !isInternal && + describe("bigint ids", () => { + let table: Table + let relatedTable: Table + + beforeAll(async () => { + const tableName = generator.guid().substring(0, 10) + await client!.schema.createTable(tableName, table => { + table.bigIncrements("id").primary() + }) + + const relatedTableName = generator.guid().substring(0, 10) + await client!.schema.createTable(relatedTableName, table => { + table.increments("id").primary() + table + .bigInteger("tableid") + .unsigned() + .references("id") + .inTable(tableName) + }) + + const resp = await config.api.datasource.fetchSchema({ + datasourceId: datasource!._id!, + }) + + const tables = Object.values(resp.datasource.entities || {}) + table = tables.find(t => t.name === tableName)! + relatedTable = tables.find(t => t.name === relatedTableName)! + + await config.api.table.save({ + ...table, + schema: { + ...table.schema, + related: { + name: "related", + type: FieldType.LINK, + tableId: relatedTable._id!, + fieldName: "tableid", + relationshipType: RelationshipType.ONE_TO_MANY, + }, + }, + }) + }) + + it.only("should be able to fetch rows with related bigint ids", async () => { + const row = await config.api.row.save(table._id!, {}) + await config.api.row.save(relatedTable._id!, { tableid: row.id }) + + const { rows } = await config.api.row.search(table._id!) + expect(rows).toEqual([]) + }) + }) } ) } diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 28400f616f..881017c24e 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -39,6 +39,10 @@ if (types) { types.setTypeParser(1114, (val: any) => val) // timestamp types.setTypeParser(1082, (val: any) => val) // date types.setTypeParser(1184, (val: any) => val) // timestampz + // types.setTypeParser(114, JSON.parse) // json + // types.setTypeParser(3802, JSON.parse) // jsonb + // types.setTypeParser(199, parseJsonArray) // json[] + // types.setTypeParser(3807, parseJsonArray) // jsonb[] } const JSON_REGEX = /'{\s*.*?\s*}'::json/gs From f464a8048461c1952102950655f056409c2e5452 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 21 Nov 2024 18:09:21 +0000 Subject: [PATCH 02/30] Adding helper to create relationship to datasource API in tests. --- .../src/tests/utilities/api/datasource.ts | 66 +++++++++++++++++-- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts index 67484a688a..03ff6ac8d0 100644 --- a/packages/server/src/tests/utilities/api/datasource.ts +++ b/packages/server/src/tests/utilities/api/datasource.ts @@ -1,14 +1,17 @@ import { - Datasource, - VerifyDatasourceRequest, - CreateDatasourceResponse, - UpdateDatasourceResponse, - UpdateDatasourceRequest, - QueryJson, BuildSchemaFromSourceResponse, + CreateDatasourceResponse, + Datasource, FetchDatasourceInfoResponse, + FieldType, + QueryJson, + RelationshipType, + UpdateDatasourceRequest, + UpdateDatasourceResponse, + VerifyDatasourceRequest, } from "@budibase/types" import { Expectations, TestAPI } from "./base" +import { sql } from "@budibase/backend-core" export class DatasourceAPI extends TestAPI { create = async ( @@ -103,4 +106,55 @@ export class DatasourceAPI extends TestAPI { } ) } + + addExistingRelationship = async ( + tableId1: string, + tableId2: string, + relationshipNameInTable1: string, + relationshipNameInTable2: string, + primaryKey: string, + foreignKey: string, + expectations?: Expectations + ) => { + const tableInfo1 = sql.utils.breakExternalTableId(tableId1), + tableInfo2 = sql.utils.breakExternalTableId(tableId2) + if (tableInfo1.datasourceId !== tableInfo2.datasourceId) { + throw new Error( + "Tables are in different datasources, cannot create relationship." + ) + } + const datasource = await this.get(tableInfo1.datasourceId) + const table1 = datasource.entities?.[tableInfo1.tableName], + table2 = datasource.entities?.[tableInfo2.tableName] + if (!table1 || !table2) { + throw new Error( + "Both tables not found in datasource, cannot create relationship." + ) + } + + const table1HasPrimary = table1.primary!.includes(primaryKey) + table1.schema[relationshipNameInTable1] = { + type: FieldType.LINK, + name: relationshipNameInTable1, + tableId: tableId2, + relationshipType: table1HasPrimary + ? RelationshipType.MANY_TO_ONE + : RelationshipType.ONE_TO_MANY, + fieldName: table1HasPrimary ? foreignKey : primaryKey, + foreignKey: table1HasPrimary ? primaryKey : foreignKey, + main: table1HasPrimary, + } + table2.schema[relationshipNameInTable2] = { + type: FieldType.LINK, + name: relationshipNameInTable2, + tableId: tableId1, + relationshipType: table1HasPrimary + ? RelationshipType.ONE_TO_MANY + : RelationshipType.MANY_TO_ONE, + fieldName: table1HasPrimary ? primaryKey : foreignKey, + foreignKey: table1HasPrimary ? foreignKey : primaryKey, + main: !table1HasPrimary, + } + return await this.update(datasource, expectations) + } } From 23f29f9381e21e47f55cd073787c9f74d8686b98 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 21 Nov 2024 18:41:04 +0000 Subject: [PATCH 03/30] Adding relationship helper for defining existing relationships. --- .../server/src/api/routes/tests/row.spec.ts | 44 ++++++++------ .../src/tests/utilities/api/datasource.ts | 59 +++++++++---------- 2 files changed, 54 insertions(+), 49 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index f0dc27708c..a7712a7137 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3496,18 +3496,18 @@ if (descriptions.length) { }) }) - !isInternal && + if (!isInternal) { describe("bigint ids", () => { - let table: Table - let relatedTable: Table + let table: Table, relatedTable: Table + let tableName: string, relatedTableName: string beforeAll(async () => { - const tableName = generator.guid().substring(0, 10) + tableName = generator.guid().substring(0, 10) await client!.schema.createTable(tableName, table => { table.bigIncrements("id").primary() }) - const relatedTableName = generator.guid().substring(0, 10) + relatedTableName = generator.guid().substring(0, 10) await client!.schema.createTable(relatedTableName, table => { table.increments("id").primary() table @@ -3525,17 +3525,16 @@ if (descriptions.length) { table = tables.find(t => t.name === tableName)! relatedTable = tables.find(t => t.name === relatedTableName)! - await config.api.table.save({ - ...table, - schema: { - ...table.schema, - related: { - name: "related", - type: FieldType.LINK, - tableId: relatedTable._id!, - fieldName: "tableid", - relationshipType: RelationshipType.ONE_TO_MANY, - }, + await config.api.datasource.addExistingRelationship({ + one: { + tableId: relatedTable._id!, + relationshipName: "one", + foreignKey: "tableid", + }, + many: { + tableId: table._id!, + relationshipName: "many", + primaryKey: "id", }, }) }) @@ -3545,9 +3544,20 @@ if (descriptions.length) { await config.api.row.save(relatedTable._id!, { tableid: row.id }) const { rows } = await config.api.row.search(table._id!) - expect(rows).toEqual([]) + expect(rows.length).toEqual(1) + expect(rows[0]).toEqual( + expect.objectContaining({ + many: [ + { + _id: "%5B1%5D", + primaryDisplay: 1, + }, + ], + }) + ) }) }) + } } ) } diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts index 03ff6ac8d0..87f03c8a6f 100644 --- a/packages/server/src/tests/utilities/api/datasource.ts +++ b/packages/server/src/tests/utilities/api/datasource.ts @@ -108,52 +108,47 @@ export class DatasourceAPI extends TestAPI { } addExistingRelationship = async ( - tableId1: string, - tableId2: string, - relationshipNameInTable1: string, - relationshipNameInTable2: string, - primaryKey: string, - foreignKey: string, + { + one, + many, + }: { + one: { tableId: string; relationshipName: string; foreignKey: string } + many: { tableId: string; relationshipName: string; primaryKey: string } + }, expectations?: Expectations ) => { - const tableInfo1 = sql.utils.breakExternalTableId(tableId1), - tableInfo2 = sql.utils.breakExternalTableId(tableId2) - if (tableInfo1.datasourceId !== tableInfo2.datasourceId) { + const oneTableInfo = sql.utils.breakExternalTableId(one.tableId), + manyTableInfo = sql.utils.breakExternalTableId(many.tableId) + if (oneTableInfo.datasourceId !== manyTableInfo.datasourceId) { throw new Error( "Tables are in different datasources, cannot create relationship." ) } - const datasource = await this.get(tableInfo1.datasourceId) - const table1 = datasource.entities?.[tableInfo1.tableName], - table2 = datasource.entities?.[tableInfo2.tableName] - if (!table1 || !table2) { + const datasource = await this.get(oneTableInfo.datasourceId) + const oneTable = datasource.entities?.[oneTableInfo.tableName], + manyTable = datasource.entities?.[manyTableInfo.tableName] + if (!oneTable || !manyTable) { throw new Error( "Both tables not found in datasource, cannot create relationship." ) } - const table1HasPrimary = table1.primary!.includes(primaryKey) - table1.schema[relationshipNameInTable1] = { + manyTable.schema[many.relationshipName] = { type: FieldType.LINK, - name: relationshipNameInTable1, - tableId: tableId2, - relationshipType: table1HasPrimary - ? RelationshipType.MANY_TO_ONE - : RelationshipType.ONE_TO_MANY, - fieldName: table1HasPrimary ? foreignKey : primaryKey, - foreignKey: table1HasPrimary ? primaryKey : foreignKey, - main: table1HasPrimary, + name: many.relationshipName, + tableId: oneTable._id!, + relationshipType: RelationshipType.MANY_TO_ONE, + fieldName: one.foreignKey, + foreignKey: many.primaryKey, + main: true, } - table2.schema[relationshipNameInTable2] = { + oneTable.schema[one.relationshipName] = { type: FieldType.LINK, - name: relationshipNameInTable2, - tableId: tableId1, - relationshipType: table1HasPrimary - ? RelationshipType.ONE_TO_MANY - : RelationshipType.MANY_TO_ONE, - fieldName: table1HasPrimary ? primaryKey : foreignKey, - foreignKey: table1HasPrimary ? foreignKey : primaryKey, - main: !table1HasPrimary, + name: one.relationshipName, + tableId: manyTable._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: many.primaryKey, + foreignKey: one.foreignKey, } return await this.update(datasource, expectations) } From e25d92a49e759e06e20e8611463a20f994323d95 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 17:18:12 +0000 Subject: [PATCH 04/30] wip --- packages/backend-core/src/sql/sql.ts | 23 +++++++++++++++---- .../server/src/api/routes/tests/row.spec.ts | 14 ++++++++++- packages/server/src/integrations/postgres.ts | 4 ---- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index e4b2b843af..4ec9246cbb 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1191,8 +1191,9 @@ class InternalBuilder { return withSchema } - private buildJsonField(field: string): string { + private buildJsonField(table: Table, field: string): string { const parts = field.split(".") + const baseName = parts[parts.length - 1] let unaliased: string let tableField: string @@ -1205,10 +1206,22 @@ class InternalBuilder { tableField = unaliased } + const schema = table.schema[baseName] + const separator = this.client === SqlClient.ORACLE ? " VALUE " : "," - return this.knex - .raw(`?${separator}??`, [unaliased, this.rawQuotedIdentifier(tableField)]) - .toString() + let identifier = this.rawQuotedIdentifier(tableField) + if (schema.type === FieldType.BIGINT) { + identifier = this.castIntToString(identifier) + } else if (schema.type === FieldType.LINK) { + const otherTable = this.query.meta.tables![schema.tableId] + const otherField = otherTable.schema[schema.fieldName] + if (otherField.type === FieldType.BIGINT) { + identifier = this.castIntToString(identifier) + } + } else if (schema.autocolumn && schema.autoReason === "foreign_key") + return this.knex + .raw(`?${separator}??`, [unaliased, identifier]) + .toString() } maxFunctionParameters() { @@ -1272,7 +1285,7 @@ class InternalBuilder { Math.floor(this.maxFunctionParameters() / 2) ) const fieldList: string = relationshipFields - .map(field => this.buildJsonField(field)) + .map(field => this.buildJsonField(relatedTable!, field)) .join(",") // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax // it reduces the result set rather than limiting how much data it filters over diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index f0dc27708c..0ad5439653 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3545,7 +3545,19 @@ if (descriptions.length) { await config.api.row.save(relatedTable._id!, { tableid: row.id }) const { rows } = await config.api.row.search(table._id!) - expect(rows).toEqual([]) + expect(rows).toEqual([ + expect.objectContaining({ + _id: "%5B'1'%5D", + _rev: "rev", + id: "1", + related: [ + { + _id: "%5B'1'%5D", + primaryDisplay: 1, + }, + ], + }), + ]) }) }) } diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 881017c24e..28400f616f 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -39,10 +39,6 @@ if (types) { types.setTypeParser(1114, (val: any) => val) // timestamp types.setTypeParser(1082, (val: any) => val) // date types.setTypeParser(1184, (val: any) => val) // timestampz - // types.setTypeParser(114, JSON.parse) // json - // types.setTypeParser(3802, JSON.parse) // jsonb - // types.setTypeParser(199, parseJsonArray) // json[] - // types.setTypeParser(3807, parseJsonArray) // jsonb[] } const JSON_REGEX = /'{\s*.*?\s*}'::json/gs From 93d8bd792c158fdaf6ed2630edb03088135fb483 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 17:36:18 +0000 Subject: [PATCH 05/30] Trying to make it easier to understand my tests. --- packages/backend-core/src/sql/sql.ts | 6 +-- .../server/src/api/routes/tests/row.spec.ts | 44 +++++++++---------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 4ec9246cbb..b8aaca0f71 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1218,10 +1218,8 @@ class InternalBuilder { if (otherField.type === FieldType.BIGINT) { identifier = this.castIntToString(identifier) } - } else if (schema.autocolumn && schema.autoReason === "foreign_key") - return this.knex - .raw(`?${separator}??`, [unaliased, identifier]) - .toString() + } + return this.knex.raw(`?${separator}??`, [unaliased, identifier]).toString() } maxFunctionParameters() { diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 0752d38f17..a9d56c4577 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3498,23 +3498,23 @@ if (descriptions.length) { if (!isInternal) { describe("bigint ids", () => { - let table: Table, relatedTable: Table - let tableName: string, relatedTableName: string + let table1: Table, table2: Table + let table1Name: string, table2Name: string beforeAll(async () => { - tableName = generator.guid().substring(0, 10) - await client!.schema.createTable(tableName, table => { - table.bigIncrements("id").primary() + table1Name = `table1-${generator.guid().substring(0, 5)}` + await client!.schema.createTable(table1Name, table => { + table.bigIncrements("table1Id").primary() }) - relatedTableName = generator.guid().substring(0, 10) - await client!.schema.createTable(relatedTableName, table => { - table.increments("id").primary() + table2Name = `table2-${generator.guid().substring(0, 5)}` + await client!.schema.createTable(table2Name, table => { + table.increments("table2Id").primary() table - .bigInteger("tableid") + .bigInteger("table1Ref") .unsigned() - .references("id") - .inTable(tableName) + .references("table1Id") + .inTable(table1Name) }) const resp = await config.api.datasource.fetchSchema({ @@ -3522,34 +3522,34 @@ if (descriptions.length) { }) const tables = Object.values(resp.datasource.entities || {}) - table = tables.find(t => t.name === tableName)! - relatedTable = tables.find(t => t.name === relatedTableName)! + table1 = tables.find(t => t.name === table1Name)! + table2 = tables.find(t => t.name === table2Name)! await config.api.datasource.addExistingRelationship({ one: { - tableId: relatedTable._id!, + tableId: table2._id!, relationshipName: "one", - foreignKey: "tableid", + foreignKey: "table1Ref", }, many: { - tableId: table._id!, + tableId: table1._id!, relationshipName: "many", - primaryKey: "id", + primaryKey: "table1Id", }, }) }) it.only("should be able to fetch rows with related bigint ids", async () => { - const row = await config.api.row.save(table._id!, {}) - await config.api.row.save(relatedTable._id!, { tableid: row.id }) + const row = await config.api.row.save(table1._id!, {}) + await config.api.row.save(table2._id!, { table1Ref: row.id }) - const { rows } = await config.api.row.search(table._id!) + const { rows } = await config.api.row.search(table1._id!) expect(rows).toEqual([ expect.objectContaining({ _id: "%5B'1'%5D", _rev: "rev", - id: "1", - related: [ + table1Id: "1", + many: [ { _id: "%5B'1'%5D", primaryDisplay: 1, From d1b0bf965b15cee6e57c770e13a065c94496258a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 17:51:47 +0000 Subject: [PATCH 06/30] wip --- packages/backend-core/src/sql/sql.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b8aaca0f71..d7d7235f3a 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1210,15 +1210,15 @@ class InternalBuilder { const separator = this.client === SqlClient.ORACLE ? " VALUE " : "," let identifier = this.rawQuotedIdentifier(tableField) - if (schema.type === FieldType.BIGINT) { - identifier = this.castIntToString(identifier) - } else if (schema.type === FieldType.LINK) { - const otherTable = this.query.meta.tables![schema.tableId] - const otherField = otherTable.schema[schema.fieldName] - if (otherField.type === FieldType.BIGINT) { - identifier = this.castIntToString(identifier) - } - } + // if (schema.type === FieldType.BIGINT) { + // identifier = this.castIntToString(identifier) + // } else if (schema.type === FieldType.LINK) { + // const otherTable = this.query.meta.tables![schema.tableId] + // const otherField = otherTable.schema[schema.fieldName] + // if (otherField.type === FieldType.BIGINT) { + // identifier = this.castIntToString(identifier) + // } + // } return this.knex.raw(`?${separator}??`, [unaliased, identifier]).toString() } From 50a7539e2589481cc35a7043327403a5b63ebf18 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 17:53:03 +0000 Subject: [PATCH 07/30] Revert "wip" This reverts commit d1b0bf965b15cee6e57c770e13a065c94496258a. --- packages/backend-core/src/sql/sql.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index d7d7235f3a..b8aaca0f71 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1210,15 +1210,15 @@ class InternalBuilder { const separator = this.client === SqlClient.ORACLE ? " VALUE " : "," let identifier = this.rawQuotedIdentifier(tableField) - // if (schema.type === FieldType.BIGINT) { - // identifier = this.castIntToString(identifier) - // } else if (schema.type === FieldType.LINK) { - // const otherTable = this.query.meta.tables![schema.tableId] - // const otherField = otherTable.schema[schema.fieldName] - // if (otherField.type === FieldType.BIGINT) { - // identifier = this.castIntToString(identifier) - // } - // } + if (schema.type === FieldType.BIGINT) { + identifier = this.castIntToString(identifier) + } else if (schema.type === FieldType.LINK) { + const otherTable = this.query.meta.tables![schema.tableId] + const otherField = otherTable.schema[schema.fieldName] + if (otherField.type === FieldType.BIGINT) { + identifier = this.castIntToString(identifier) + } + } return this.knex.raw(`?${separator}??`, [unaliased, identifier]).toString() } From c3b2c2e1c7400b9c43f1ab1d252aa42e81425136 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 17:53:11 +0000 Subject: [PATCH 08/30] Revert "Trying to make it easier to understand my tests." This reverts commit 93d8bd792c158fdaf6ed2630edb03088135fb483. --- packages/backend-core/src/sql/sql.ts | 6 ++- .../server/src/api/routes/tests/row.spec.ts | 44 +++++++++---------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b8aaca0f71..4ec9246cbb 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1218,8 +1218,10 @@ class InternalBuilder { if (otherField.type === FieldType.BIGINT) { identifier = this.castIntToString(identifier) } - } - return this.knex.raw(`?${separator}??`, [unaliased, identifier]).toString() + } else if (schema.autocolumn && schema.autoReason === "foreign_key") + return this.knex + .raw(`?${separator}??`, [unaliased, identifier]) + .toString() } maxFunctionParameters() { diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index a9d56c4577..0752d38f17 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3498,23 +3498,23 @@ if (descriptions.length) { if (!isInternal) { describe("bigint ids", () => { - let table1: Table, table2: Table - let table1Name: string, table2Name: string + let table: Table, relatedTable: Table + let tableName: string, relatedTableName: string beforeAll(async () => { - table1Name = `table1-${generator.guid().substring(0, 5)}` - await client!.schema.createTable(table1Name, table => { - table.bigIncrements("table1Id").primary() + tableName = generator.guid().substring(0, 10) + await client!.schema.createTable(tableName, table => { + table.bigIncrements("id").primary() }) - table2Name = `table2-${generator.guid().substring(0, 5)}` - await client!.schema.createTable(table2Name, table => { - table.increments("table2Id").primary() + relatedTableName = generator.guid().substring(0, 10) + await client!.schema.createTable(relatedTableName, table => { + table.increments("id").primary() table - .bigInteger("table1Ref") + .bigInteger("tableid") .unsigned() - .references("table1Id") - .inTable(table1Name) + .references("id") + .inTable(tableName) }) const resp = await config.api.datasource.fetchSchema({ @@ -3522,34 +3522,34 @@ if (descriptions.length) { }) const tables = Object.values(resp.datasource.entities || {}) - table1 = tables.find(t => t.name === table1Name)! - table2 = tables.find(t => t.name === table2Name)! + table = tables.find(t => t.name === tableName)! + relatedTable = tables.find(t => t.name === relatedTableName)! await config.api.datasource.addExistingRelationship({ one: { - tableId: table2._id!, + tableId: relatedTable._id!, relationshipName: "one", - foreignKey: "table1Ref", + foreignKey: "tableid", }, many: { - tableId: table1._id!, + tableId: table._id!, relationshipName: "many", - primaryKey: "table1Id", + primaryKey: "id", }, }) }) it.only("should be able to fetch rows with related bigint ids", async () => { - const row = await config.api.row.save(table1._id!, {}) - await config.api.row.save(table2._id!, { table1Ref: row.id }) + const row = await config.api.row.save(table._id!, {}) + await config.api.row.save(relatedTable._id!, { tableid: row.id }) - const { rows } = await config.api.row.search(table1._id!) + const { rows } = await config.api.row.search(table._id!) expect(rows).toEqual([ expect.objectContaining({ _id: "%5B'1'%5D", _rev: "rev", - table1Id: "1", - many: [ + id: "1", + related: [ { _id: "%5B'1'%5D", primaryDisplay: 1, From 7138c02f2703b719a738de47db15fb4b75b34207 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 17:53:58 +0000 Subject: [PATCH 09/30] Put things back how they were. --- packages/backend-core/src/sql/sql.ts | 22 +++++++++---------- .../server/src/api/routes/tests/row.spec.ts | 2 +- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 4ec9246cbb..d7d7235f3a 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1210,18 +1210,16 @@ class InternalBuilder { const separator = this.client === SqlClient.ORACLE ? " VALUE " : "," let identifier = this.rawQuotedIdentifier(tableField) - if (schema.type === FieldType.BIGINT) { - identifier = this.castIntToString(identifier) - } else if (schema.type === FieldType.LINK) { - const otherTable = this.query.meta.tables![schema.tableId] - const otherField = otherTable.schema[schema.fieldName] - if (otherField.type === FieldType.BIGINT) { - identifier = this.castIntToString(identifier) - } - } else if (schema.autocolumn && schema.autoReason === "foreign_key") - return this.knex - .raw(`?${separator}??`, [unaliased, identifier]) - .toString() + // if (schema.type === FieldType.BIGINT) { + // identifier = this.castIntToString(identifier) + // } else if (schema.type === FieldType.LINK) { + // const otherTable = this.query.meta.tables![schema.tableId] + // const otherField = otherTable.schema[schema.fieldName] + // if (otherField.type === FieldType.BIGINT) { + // identifier = this.castIntToString(identifier) + // } + // } + return this.knex.raw(`?${separator}??`, [unaliased, identifier]).toString() } maxFunctionParameters() { diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 0752d38f17..28542a8701 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3549,7 +3549,7 @@ if (descriptions.length) { _id: "%5B'1'%5D", _rev: "rev", id: "1", - related: [ + many: [ { _id: "%5B'1'%5D", primaryDisplay: 1, From ac3e0750bc5f55fc7d5e09b302a4207975e43315 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 17:56:16 +0000 Subject: [PATCH 10/30] Rename table variables. --- .../server/src/api/routes/tests/row.spec.ts | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 28542a8701..56efb2980d 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3498,23 +3498,23 @@ if (descriptions.length) { if (!isInternal) { describe("bigint ids", () => { - let table: Table, relatedTable: Table - let tableName: string, relatedTableName: string + let table1: Table, table2: Table + let table1Name: string, table2Name: string beforeAll(async () => { - tableName = generator.guid().substring(0, 10) - await client!.schema.createTable(tableName, table => { + table1Name = `table1-${generator.guid().substring(0, 5)}` + await client!.schema.createTable(table1Name, table => { table.bigIncrements("id").primary() }) - relatedTableName = generator.guid().substring(0, 10) - await client!.schema.createTable(relatedTableName, table => { + table2Name = `table2-${generator.guid().substring(0, 5)}` + await client!.schema.createTable(table2Name, table => { table.increments("id").primary() table .bigInteger("tableid") .unsigned() .references("id") - .inTable(tableName) + .inTable(table1Name) }) const resp = await config.api.datasource.fetchSchema({ @@ -3522,17 +3522,17 @@ if (descriptions.length) { }) const tables = Object.values(resp.datasource.entities || {}) - table = tables.find(t => t.name === tableName)! - relatedTable = tables.find(t => t.name === relatedTableName)! + table1 = tables.find(t => t.name === table1Name)! + table2 = tables.find(t => t.name === table2Name)! await config.api.datasource.addExistingRelationship({ one: { - tableId: relatedTable._id!, + tableId: table2._id!, relationshipName: "one", foreignKey: "tableid", }, many: { - tableId: table._id!, + tableId: table1._id!, relationshipName: "many", primaryKey: "id", }, @@ -3540,10 +3540,10 @@ if (descriptions.length) { }) it.only("should be able to fetch rows with related bigint ids", async () => { - const row = await config.api.row.save(table._id!, {}) - await config.api.row.save(relatedTable._id!, { tableid: row.id }) + const row = await config.api.row.save(table1._id!, {}) + await config.api.row.save(table2._id!, { tableid: row.id }) - const { rows } = await config.api.row.search(table._id!) + const { rows } = await config.api.row.search(table1._id!) expect(rows).toEqual([ expect.objectContaining({ _id: "%5B'1'%5D", From 1c27ddcb8abbbc9247746552f6deab7c28a9254d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 17:58:40 +0000 Subject: [PATCH 11/30] rename table fields --- packages/server/src/api/routes/tests/row.spec.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 56efb2980d..8809ae5cbc 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3504,16 +3504,16 @@ if (descriptions.length) { beforeAll(async () => { table1Name = `table1-${generator.guid().substring(0, 5)}` await client!.schema.createTable(table1Name, table => { - table.bigIncrements("id").primary() + table.bigIncrements("table1Id").primary() }) table2Name = `table2-${generator.guid().substring(0, 5)}` await client!.schema.createTable(table2Name, table => { - table.increments("id").primary() + table.increments("table2Id").primary() table - .bigInteger("tableid") + .bigInteger("table1Ref") .unsigned() - .references("id") + .references("table1Id") .inTable(table1Name) }) @@ -3529,26 +3529,26 @@ if (descriptions.length) { one: { tableId: table2._id!, relationshipName: "one", - foreignKey: "tableid", + foreignKey: "table1Ref", }, many: { tableId: table1._id!, relationshipName: "many", - primaryKey: "id", + primaryKey: "table1Id", }, }) }) it.only("should be able to fetch rows with related bigint ids", async () => { const row = await config.api.row.save(table1._id!, {}) - await config.api.row.save(table2._id!, { tableid: row.id }) + await config.api.row.save(table2._id!, { table1Ref: row.table1Id }) const { rows } = await config.api.row.search(table1._id!) expect(rows).toEqual([ expect.objectContaining({ _id: "%5B'1'%5D", _rev: "rev", - id: "1", + table1Id: "1", many: [ { _id: "%5B'1'%5D", From 26fa0222a18d70978f7cf51404974d0b1fdfd791 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Nov 2024 12:00:30 +0000 Subject: [PATCH 12/30] Get all bigint tests passing. --- packages/backend-core/src/sql/sql.ts | 46 +++++++++++-------- .../server/src/api/routes/tests/row.spec.ts | 42 ++++++++++++----- .../src/integrations/microsoftSqlServer.ts | 3 +- 3 files changed, 59 insertions(+), 32 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index d7d7235f3a..e97e31869c 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1191,7 +1191,7 @@ class InternalBuilder { return withSchema } - private buildJsonField(table: Table, field: string): string { + private buildJsonField(table: Table, field: string): [string, Knex.Raw] { const parts = field.split(".") const baseName = parts[parts.length - 1] let unaliased: string @@ -1208,18 +1208,11 @@ class InternalBuilder { const schema = table.schema[baseName] - const separator = this.client === SqlClient.ORACLE ? " VALUE " : "," let identifier = this.rawQuotedIdentifier(tableField) - // if (schema.type === FieldType.BIGINT) { - // identifier = this.castIntToString(identifier) - // } else if (schema.type === FieldType.LINK) { - // const otherTable = this.query.meta.tables![schema.tableId] - // const otherField = otherTable.schema[schema.fieldName] - // if (otherField.type === FieldType.BIGINT) { - // identifier = this.castIntToString(identifier) - // } - // } - return this.knex.raw(`?${separator}??`, [unaliased, identifier]).toString() + if (schema.type === FieldType.BIGINT) { + identifier = this.castIntToString(identifier) + } + return [unaliased, identifier] } maxFunctionParameters() { @@ -1282,8 +1275,14 @@ class InternalBuilder { 0, Math.floor(this.maxFunctionParameters() / 2) ) - const fieldList: string = relationshipFields - .map(field => this.buildJsonField(relatedTable!, field)) + const fieldList = relationshipFields.map(field => + this.buildJsonField(relatedTable!, field) + ) + const fieldListFormatted = fieldList + .map(f => { + const separator = this.client === SqlClient.ORACLE ? " VALUE " : "," + return this.knex.raw(`?${separator}??`, [f[0], f[1]]).toString() + }) .join(",") // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax // it reduces the result set rather than limiting how much data it filters over @@ -1331,35 +1330,42 @@ class InternalBuilder { // need to check the junction table document is to the right column, this is just for SQS subQuery = this.addJoinFieldCheck(subQuery, relationship) wrapperQuery = standardWrap( - this.knex.raw(`json_group_array(json_object(${fieldList}))`) + this.knex.raw( + `json_group_array(json_object(${fieldListFormatted}))` + ) ) break case SqlClient.POSTGRES: wrapperQuery = standardWrap( - this.knex.raw(`json_agg(json_build_object(${fieldList}))`) + this.knex.raw(`json_agg(json_build_object(${fieldListFormatted}))`) ) break case SqlClient.MARIADB: // can't use the standard wrap due to correlated sub-query limitations in MariaDB wrapperQuery = subQuery.select( knex.raw( - `json_arrayagg(json_object(${fieldList}) LIMIT ${getRelationshipLimit()})` + `json_arrayagg(json_object(${fieldListFormatted}) LIMIT ${getRelationshipLimit()})` ) ) break case SqlClient.MY_SQL: case SqlClient.ORACLE: wrapperQuery = standardWrap( - this.knex.raw(`json_arrayagg(json_object(${fieldList}))`) + this.knex.raw(`json_arrayagg(json_object(${fieldListFormatted}))`) ) break case SqlClient.MS_SQL: { const comparatorQuery = knex - .select(`${fromAlias}.*`) + .select(`*`) // @ts-ignore - from alias syntax not TS supported .from({ [fromAlias]: subQuery - .select(`${toAlias}.*`) + .select( + fieldList.map(f => { + // @ts-expect-error raw is fine here, knex types are wrong + return knex.ref(f[1]).as(f[0]) + }) + ) .limit(getRelationshipLimit()), }) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 8809ae5cbc..3b1b151f1f 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3496,7 +3496,7 @@ if (descriptions.length) { }) }) - if (!isInternal) { + if (!isInternal && !isOracle) { describe("bigint ids", () => { let table1: Table, table2: Table let table1Name: string, table2Name: string @@ -3504,15 +3504,14 @@ if (descriptions.length) { beforeAll(async () => { table1Name = `table1-${generator.guid().substring(0, 5)}` await client!.schema.createTable(table1Name, table => { - table.bigIncrements("table1Id").primary() + table.bigInteger("table1Id").primary() }) table2Name = `table2-${generator.guid().substring(0, 5)}` await client!.schema.createTable(table2Name, table => { - table.increments("table2Id").primary() + table.bigInteger("table2Id").primary() table .bigInteger("table1Ref") - .unsigned() .references("table1Id") .inTable(table1Name) }) @@ -3540,21 +3539,42 @@ if (descriptions.length) { }) it.only("should be able to fetch rows with related bigint ids", async () => { - const row = await config.api.row.save(table1._id!, {}) - await config.api.row.save(table2._id!, { table1Ref: row.table1Id }) + const row = await config.api.row.save(table1._id!, { + table1Id: "1", + }) + await config.api.row.save(table2._id!, { + table2Id: "2", + table1Ref: row.table1Id, + }) - const { rows } = await config.api.row.search(table1._id!) - expect(rows).toEqual([ + let resp = await config.api.row.search(table1._id!) + expect(resp.rows).toEqual([ expect.objectContaining({ _id: "%5B'1'%5D", - _rev: "rev", table1Id: "1", many: [ { - _id: "%5B'1'%5D", - primaryDisplay: 1, + _id: "%5B'2'%5D", + primaryDisplay: "2", }, ], + tableId: table1._id, + }), + ]) + + resp = await config.api.row.search(table2._id!) + expect(resp.rows).toEqual([ + expect.objectContaining({ + _id: "%5B'2'%5D", + table2Id: "2", + table1Ref: "1", + one: [ + { + _id: "%5B'1'%5D", + primaryDisplay: "1", + }, + ], + tableId: table2._id, }), ]) }) diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 1c74e6b1ff..4997f3256d 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -342,7 +342,8 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { ? `${query.sql}; SELECT SCOPE_IDENTITY() AS id;` : query.sql this.log(sql, query.bindings) - return await request.query(sql) + const resp = await request.query(sql) + return resp } catch (err: any) { let readableMessage = getReadableErrorMessage( SourceName.SQL_SERVER, From f7c81d742c0c261ec089a5b76d915e4ecf063c8c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Nov 2024 12:00:42 +0000 Subject: [PATCH 13/30] Remove .only --- packages/server/src/api/routes/tests/row.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 3b1b151f1f..9e84e74d05 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3538,7 +3538,7 @@ if (descriptions.length) { }) }) - it.only("should be able to fetch rows with related bigint ids", async () => { + it("should be able to fetch rows with related bigint ids", async () => { const row = await config.api.row.save(table1._id!, { table1Id: "1", }) From 594af57339a9cac886f766abc694475f18fc45e5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Nov 2024 16:28:22 +0000 Subject: [PATCH 14/30] wip --- packages/backend-core/src/sql/sql.ts | 60 +++-- packages/backend-core/src/sql/sqlTable.ts | 24 +- .../api/controllers/row/ExternalRequest.ts | 56 ++--- .../server/src/integrations/base/query.ts | 40 ++-- .../src/integrations/microsoftSqlServer.ts | 10 +- packages/server/src/integrations/mysql.ts | 8 +- packages/server/src/integrations/oracle.ts | 4 +- packages/server/src/integrations/postgres.ts | 4 +- .../basicFetchWithRelationships.json | 208 +++++++++-------- .../tests/sqlQueryJson/createSimple.json | 64 +++--- .../sqlQueryJson/createWithRelationships.json | 215 +++++++++-------- .../tests/sqlQueryJson/deleteSimple.json | 83 +++---- .../sqlQueryJson/enrichRelationship.json | 143 ++++++------ .../tests/sqlQueryJson/fetchManyToMany.json | 78 +++---- .../sqlQueryJson/filterByRelationship.json | 78 +++---- .../sqlQueryJson/manyRelationshipFilters.json | 168 +++++++------- .../sqlQueryJson/updateRelationship.json | 217 +++++++++--------- .../tests/sqlQueryJson/updateSimple.json | 217 +++++++++--------- .../src/sdk/app/rows/search/internal/sqs.ts | 23 +- packages/server/src/sdk/app/rows/sqlAlias.ts | 28 ++- packages/server/src/sdk/app/rows/utils.ts | 36 ++- packages/types/src/sdk/datasources.ts | 4 +- packages/types/src/sdk/search.ts | 13 +- 23 files changed, 854 insertions(+), 927 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index e97e31869c..c897c106b0 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -18,6 +18,7 @@ import { BasicOperator, BBReferenceFieldMetadata, CalculationType, + EnrichedQueryJson, FieldSchema, FieldType, INTERNAL_TABLE_SOURCE_ID, @@ -27,7 +28,6 @@ import { LogicalOperator, Operation, prefixed, - QueryJson, QueryOptions, RangeOperator, RelationshipsJson, @@ -134,18 +134,18 @@ const allowEmptyRelationships: Record = { class InternalBuilder { private readonly client: SqlClient - private readonly query: QueryJson + private readonly query: EnrichedQueryJson private readonly splitter: dataFilters.ColumnSplitter private readonly knex: Knex - constructor(client: SqlClient, knex: Knex, query: QueryJson) { + constructor(client: SqlClient, knex: Knex, query: EnrichedQueryJson) { this.client = client this.query = query this.knex = knex this.splitter = new dataFilters.ColumnSplitter([this.table], { aliases: this.query.tableAliases, - columnPrefix: this.query.meta.columnPrefix, + columnPrefix: this.query.meta?.columnPrefix, }) } @@ -167,7 +167,7 @@ class InternalBuilder { } get table(): Table { - return this.query.meta.table + return this.query.table } get knexClient(): Knex.Client { @@ -273,8 +273,7 @@ class InternalBuilder { } private isFullSelectStatementRequired(): boolean { - const { meta } = this.query - for (let column of Object.values(meta.table.schema)) { + for (let column of Object.values(this.table.schema)) { if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) { return true } else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) { @@ -285,14 +284,14 @@ class InternalBuilder { } private generateSelectStatement(): (string | Knex.Raw)[] | "*" { - const { meta, endpoint, resource } = this.query + const { endpoint, resource } = this.query if (!resource || !resource.fields || resource.fields.length === 0) { return "*" } const alias = this.getTableName(endpoint.entityId) - const schema = meta.table.schema + const schema = this.table.schema if (!this.isFullSelectStatementRequired()) { return [this.knex.raw("??", [`${alias}.*`])] } @@ -1018,15 +1017,15 @@ class InternalBuilder { const name = tableOrName if (this.query.table?.name === name) { table = this.query.table - } else if (this.query.meta.table?.name === name) { - table = this.query.meta.table - } else if (!this.query.meta.tables?.[name]) { + } else if (this.query.table.name === name) { + table = this.query.table + } else if (!this.query.tables[name]) { // This can legitimately happen in custom queries, where the user is // querying against a table that may not have been imported into // Budibase. return name } else { - table = this.query.meta.tables[name] + table = this.query.tables[name] } } else if (tableOrName) { table = tableOrName @@ -1238,7 +1237,7 @@ class InternalBuilder { ): Knex.QueryBuilder { const sqlClient = this.client const knex = this.knex - const { resource, tableAliases: aliases, endpoint, meta } = this.query + const { resource, tableAliases: aliases, endpoint, tables } = this.query const fields = resource?.fields || [] for (let relationship of relationships) { const { @@ -1253,7 +1252,7 @@ class InternalBuilder { if (!toTable || !fromTable) { continue } - const relatedTable = meta.tables?.[toTable] + const relatedTable = tables[toTable] const toAlias = aliases?.[toTable] || toTable, fromAlias = aliases?.[fromTable] || fromTable, throughAlias = (throughTable && aliases?.[throughTable]) || throughTable @@ -1276,7 +1275,7 @@ class InternalBuilder { Math.floor(this.maxFunctionParameters() / 2) ) const fieldList = relationshipFields.map(field => - this.buildJsonField(relatedTable!, field) + this.buildJsonField(relatedTable, field) ) const fieldListFormatted = fieldList .map(f => { @@ -1472,9 +1471,7 @@ class InternalBuilder { if (this.client === SqlClient.ORACLE) { // Oracle doesn't seem to automatically insert nulls // if we don't specify them, so we need to do that here - for (const [column, schema] of Object.entries( - this.query.meta.table.schema - )) { + for (const [column, schema] of Object.entries(this.query.table.schema)) { if ( schema.constraints?.presence === true || schema.type === FieldType.FORMULA || @@ -1678,7 +1675,10 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { * which for the sake of mySQL stops adding the returning statement to inserts, updates and deletes. * @return the query ready to be passed to the driver. */ - _query(json: QueryJson, opts: QueryOptions = {}): SqlQuery | SqlQuery[] { + _query( + json: EnrichedQueryJson, + opts: QueryOptions = {} + ): SqlQuery | SqlQuery[] { const sqlClient = this.getSqlClient() const config: Knex.Config = { client: this.getBaseSqlClient(), @@ -1728,34 +1728,30 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { return this.convertToNative(query, opts) } - async getReturningRow(queryFn: QueryFunction, json: QueryJson) { + async getReturningRow(queryFn: QueryFunction, json: EnrichedQueryJson) { if (!json.extra || !json.extra.idFilter) { return {} } const input = this._query({ + ...json, endpoint: { ...json.endpoint, operation: Operation.READ, }, - resource: { - fields: [], - }, + resource: { fields: [] }, filters: json.extra?.idFilter, - paginate: { - limit: 1, - }, - meta: json.meta, + paginate: { limit: 1 }, }) return queryFn(input, Operation.READ) } // when creating if an ID has been inserted need to make sure // the id filter is enriched with it before trying to retrieve the row - checkLookupKeys(id: any, json: QueryJson) { - if (!id || !json.meta.table || !json.meta.table.primary) { + checkLookupKeys(id: any, json: EnrichedQueryJson) { + if (!id || !json.table.primary) { return json } - const primaryKey = json.meta.table.primary?.[0] + const primaryKey = json.table.primary[0] json.extra = { idFilter: { equal: { @@ -1768,7 +1764,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { // this function recreates the returning functionality of postgres async queryWithReturning( - json: QueryJson, + json: EnrichedQueryJson, queryFn: QueryFunction, processFn: Function = (result: any) => result ) { diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index 84f4e290aa..ee5dfbf453 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -3,13 +3,13 @@ import { FieldType, NumberFieldMetadata, Operation, - QueryJson, RelationshipType, RenameColumn, SqlQuery, Table, TableSourceType, SqlClient, + EnrichedQueryJson, } from "@budibase/types" import { breakExternalTableId, getNativeSql } from "./utils" import { helpers, utils } from "@budibase/shared-core" @@ -200,7 +200,7 @@ function buildUpdateTable( table: Table, tables: Record, oldTable: Table, - renamed: RenameColumn + renamed?: RenameColumn ): SchemaBuilder { return knex.alterTable(table.name, schema => { generateSchema(schema, table, tables, oldTable, renamed) @@ -238,11 +238,11 @@ class SqlTableQueryBuilder { * @param json the input JSON structure from which an SQL query will be built. * @return the operation that was found in the JSON. */ - _operation(json: QueryJson): Operation { + _operation(json: EnrichedQueryJson): Operation { return json.endpoint.operation } - _tableQuery(json: QueryJson): SqlQuery | SqlQuery[] { + _tableQuery(json: EnrichedQueryJson): SqlQuery | SqlQuery[] { let client = knex({ client: this.sqlClient }).schema let schemaName = json?.endpoint?.schema if (schemaName) { @@ -250,7 +250,7 @@ class SqlTableQueryBuilder { } let query: Knex.SchemaBuilder - if (!json.table || !json.meta || !json.meta.tables) { + if (!json.table || !json.tables) { throw new Error("Cannot execute without table being specified") } if (json.table.sourceType === TableSourceType.INTERNAL) { @@ -259,14 +259,14 @@ class SqlTableQueryBuilder { switch (this._operation(json)) { case Operation.CREATE_TABLE: - query = buildCreateTable(client, json.table, json.meta.tables) + query = buildCreateTable(client, json.table, json.tables) break case Operation.UPDATE_TABLE: - if (!json.meta || !json.meta.table) { + if (!json.table) { throw new Error("Must specify old table for update") } // renameColumn does not work for MySQL, so return a raw query - if (this.sqlClient === SqlClient.MY_SQL && json.meta.renamed) { + if (this.sqlClient === SqlClient.MY_SQL && json.meta?.renamed) { const updatedColumn = json.meta.renamed.updated const tableName = schemaName ? `\`${schemaName}\`.\`${json.table.name}\`` @@ -280,14 +280,14 @@ class SqlTableQueryBuilder { query = buildUpdateTable( client, json.table, - json.meta.tables, - json.meta.table, - json.meta.renamed! + json.tables, + json.table, + json.meta?.renamed ) // renameColumn for SQL Server returns a parameterised `sp_rename` query, // which is not supported by SQL Server and gives a syntax error. - if (this.sqlClient === SqlClient.MS_SQL && json.meta.renamed) { + if (this.sqlClient === SqlClient.MS_SQL && json.meta?.renamed) { const oldColumn = json.meta.renamed.old const updatedColumn = json.meta.renamed.updated const tableName = schemaName diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 939ea399d9..baf1665cbe 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -42,7 +42,7 @@ import { sqlOutputProcessing, } from "./utils" import { - getDatasourceAndQuery, + enrichQueryJson, processRowCountResponse, } from "../../../sdk/app/rows/utils" import { processObjectSync } from "@budibase/string-templates" @@ -135,16 +135,9 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig { return config } -function getEndpoint(tableId: string | undefined, operation: string) { - if (!tableId) { - throw new Error("Cannot get endpoint information - no table ID specified") - } +function getEndpoint(tableId: string, operation: Operation) { const { datasourceId, tableName } = breakExternalTableId(tableId) - return { - datasourceId: datasourceId, - entityId: tableName, - operation: operation as Operation, - } + return { datasourceId, entityId: tableName, operation } } function isOneSide( @@ -268,12 +261,9 @@ export class ExternalRequest { const filters = this.prepareFilters(rowId, {}, table) // safety check, if there are no filters on deletion bad things happen if (Object.keys(filters).length !== 0) { - return getDatasourceAndQuery({ + return makeExternalQuery({ endpoint: getEndpoint(tableId, Operation.DELETE), filters, - meta: { - table, - }, }) } else { return [] @@ -289,13 +279,10 @@ export class ExternalRequest { const filters = this.prepareFilters(rowId, {}, table) // safety check, if there are no filters on deletion bad things happen if (Object.keys(filters).length !== 0) { - return getDatasourceAndQuery({ + return makeExternalQuery({ endpoint: getEndpoint(tableId, Operation.UPDATE), body: { [colName]: null }, filters, - meta: { - table, - }, }) } else { return [] @@ -311,12 +298,9 @@ export class ExternalRequest { } async getRow(table: Table, rowId: string): Promise { - const response = await getDatasourceAndQuery({ + const response = await makeExternalQuery({ endpoint: getEndpoint(table._id!, Operation.READ), filters: this.prepareFilters(rowId, {}, table), - meta: { - table, - }, }) if (Array.isArray(response) && response.length > 0) { return response[0] @@ -490,16 +474,13 @@ export class ExternalRequest { if (!relatedTable) { throw new Error("unable to find related table") } - const response = await getDatasourceAndQuery({ - endpoint: endpoint, + const response = await makeExternalQuery({ + endpoint, filters: { equal: { [fieldName]: row[lookupField], }, }, - meta: { - table: relatedTable, - }, }) // this is the response from knex if no rows found const rows: Row[] = @@ -537,6 +518,11 @@ export class ExternalRequest { for (let relationship of relationships) { const { key, tableId, isUpdate, id, relationshipType, ...rest } = relationship + + if (!tableId) { + throw new Error("Table ID is unknown, cannot find table") + } + const body: { [key: string]: any } = processObjectSync(rest, row, {}) const linkTable = this.getTable(tableId) const relationshipPrimary = linkTable?.primary || [] @@ -583,14 +569,11 @@ export class ExternalRequest { const operation = isUpdate ? Operation.UPDATE : Operation.CREATE if (!existingRelationship) { promises.push( - getDatasourceAndQuery({ + makeExternalQuery({ endpoint: getEndpoint(tableId, operation), // if we're doing many relationships then we're writing, only one response body, filters: this.prepareFilters(id, {}, linkTable), - meta: { - table: linkTable, - }, }) ) } else { @@ -749,10 +732,6 @@ export class ExternalRequest { table ), }, - meta: { - table, - tables: this.tables, - }, } // remove any relationships that could block deletion @@ -773,8 +752,11 @@ export class ExternalRequest { response = [unprocessedRow] } else { response = env.SQL_ALIASING_DISABLE - ? await getDatasourceAndQuery(json) - : await aliasing.queryWithAliasing(json, makeExternalQuery) + ? await makeExternalQuery(json) + : await aliasing.queryWithAliasing( + await enrichQueryJson(json), + makeExternalQuery + ) } // if it's a counting operation there will be no more processing, just return the number diff --git a/packages/server/src/integrations/base/query.ts b/packages/server/src/integrations/base/query.ts index 55886cd20f..2185eace2c 100644 --- a/packages/server/src/integrations/base/query.ts +++ b/packages/server/src/integrations/base/query.ts @@ -1,37 +1,27 @@ -import { - QueryJson, - Datasource, - DatasourcePlusQueryResponse, - RowOperations, -} from "@budibase/types" +import { DatasourcePlusQueryResponse, QueryJson } from "@budibase/types" import { getIntegration } from "../index" import sdk from "../../sdk" +import { enrichQueryJson } from "../../sdk/app/rows/utils" export async function makeExternalQuery( - datasource: Datasource, json: QueryJson ): Promise { - const entityId = json.endpoint.entityId, - tableName = json.meta.table.name, - tableId = json.meta.table._id - // case found during testing - make sure this doesn't happen again - if ( - RowOperations.includes(json.endpoint.operation) && - entityId !== tableId && - entityId !== tableName - ) { - throw new Error("Entity ID and table metadata do not align") - } - if (!datasource) { + const enrichedJson = await enrichQueryJson(json) + if (!enrichedJson.datasource) { throw new Error("No datasource provided for external query") } - datasource = await sdk.datasources.enrich(datasource) - const Integration = await getIntegration(datasource.source) + + enrichedJson.datasource = await sdk.datasources.enrich( + enrichedJson.datasource + ) + + const Integration = await getIntegration(enrichedJson.datasource.source) + // query is the opinionated function - if (Integration.prototype.query) { - const integration = new Integration(datasource.config) - return integration.query(json) - } else { + if (!Integration.prototype.query) { throw "Datasource does not support query." } + + const integration = new Integration(enrichedJson.datasource.config) + return integration.query(json) } diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 4997f3256d..a7a0b53312 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -4,9 +4,9 @@ import { DatasourceFieldType, DatasourcePlus, DatasourcePlusQueryResponse, + EnrichedQueryJson, Integration, Operation, - QueryJson, QueryType, Schema, SourceName, @@ -506,7 +506,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { return response.recordset || [{ deleted: true }] } - async query(json: QueryJson): Promise { + async query(json: EnrichedQueryJson): Promise { const schema = this.config.schema await this.connect() if (schema && schema !== DEFAULT_SCHEMA && json?.endpoint) { @@ -515,14 +515,12 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { const operation = this._operation(json) const queryFn = (query: any, op: string) => this.internalQuery(query, op) const processFn = (result: any) => { - if (json?.meta?.table && result.recordset) { + if (result.recordset) { return this.convertJsonStringColumns( - json.meta.table, + json.table, result.recordset, json.tableAliases ) - } else if (result.recordset) { - return result.recordset } return [{ [operation]: true }] } diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index f78915fb49..9236409375 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -2,7 +2,6 @@ import { Integration, DatasourceFieldType, QueryType, - QueryJson, SqlQuery, Table, TableSchema, @@ -15,6 +14,7 @@ import { DatasourcePlusQueryResponse, SqlQueryBinding, SqlClient, + EnrichedQueryJson, } from "@budibase/types" import { getSqlQuery, @@ -390,15 +390,15 @@ class MySQLIntegration extends Sql implements DatasourcePlus { return results.length ? results : [{ deleted: true }] } - async query(json: QueryJson): Promise { + async query(json: EnrichedQueryJson): Promise { await this.connect() try { const queryFn = (query: any) => this.internalQuery(query, { connect: false, disableCoercion: true }) const processFn = (result: any) => { - if (json?.meta?.table && Array.isArray(result)) { + if (Array.isArray(result)) { return this.convertJsonStringColumns( - json.meta.table, + json.table, result, json.tableAliases ) diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index fcbc0731ea..b48a95e3e6 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -3,7 +3,6 @@ import { DatasourceFieldType, Integration, Operation, - QueryJson, QueryType, SqlQuery, Table, @@ -15,6 +14,7 @@ import { Row, DatasourcePlusQueryResponse, SqlClient, + EnrichedQueryJson, } from "@budibase/types" import { buildExternalTableId, @@ -545,7 +545,7 @@ class OracleIntegration extends Sql implements DatasourcePlus { : [{ deleted: true }] } - async query(json: QueryJson): Promise { + async query(json: EnrichedQueryJson): Promise { const operation = this._operation(json) const input = this._query(json, { disableReturning: true }) as SqlQuery if (Array.isArray(input)) { diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 28400f616f..164468b5ae 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -3,7 +3,6 @@ import { Integration, DatasourceFieldType, QueryType, - QueryJson, SqlQuery, Table, DatasourcePlus, @@ -14,6 +13,7 @@ import { TableSourceType, DatasourcePlusQueryResponse, SqlClient, + EnrichedQueryJson, } from "@budibase/types" import { getSqlQuery, @@ -419,7 +419,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus { return response.rows.length ? response.rows : [{ deleted: true }] } - async query(json: QueryJson): Promise { + async query(json: EnrichedQueryJson): Promise { const operation = this._operation(json).toLowerCase() const input = this._query(json) as SqlQuery if (Array.isArray(input)) { diff --git a/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json b/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json index aa2cab3afd..27d6efec29 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json @@ -61,113 +61,111 @@ "extra": { "idFilter": {} }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", - "primary": ["personid"], - "name": "persons", - "schema": { - "year": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "year", - "constraints": { - "presence": false - } - }, - "firstname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "firstname", - "constraints": { - "presence": false - } - }, - "personid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "personid", - "constraints": { - "presence": false - } - }, - "address": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "address", - "constraints": { - "presence": false - } - }, - "age": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "age", - "constraints": { - "presence": false - } - }, - "type": { - "type": "options", - "externalType": "USER-DEFINED", - "autocolumn": false, - "name": "type", - "constraints": { - "presence": false, - "inclusion": ["support", "designer", "programmer", "qa"] - } - }, - "city": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "city", - "constraints": { - "presence": false - } - }, - "lastname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "lastname", - "constraints": { - "presence": false - } - }, - "QA": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "QA", - "relationshipType": "many-to-one", - "fieldName": "qaid", - "type": "link", - "main": true, - "_id": "ccb68481c80c34217a4540a2c6c27fe46", - "foreignKey": "personid" - }, - "executor": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "executor", - "relationshipType": "many-to-one", - "fieldName": "executorid", - "type": "link", - "main": true, - "_id": "c89530b9770d94bec851e062b5cff3001", - "foreignKey": "personid", - "tableName": "persons" + "table": { + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", + "primary": ["personid"], + "name": "persons", + "schema": { + "year": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "year", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "firstname", - "views": {} - } + "firstname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "firstname", + "constraints": { + "presence": false + } + }, + "personid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "personid", + "constraints": { + "presence": false + } + }, + "address": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "address", + "constraints": { + "presence": false + } + }, + "age": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + }, + "type": { + "type": "options", + "externalType": "USER-DEFINED", + "autocolumn": false, + "name": "type", + "constraints": { + "presence": false, + "inclusion": ["support", "designer", "programmer", "qa"] + } + }, + "city": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "city", + "constraints": { + "presence": false + } + }, + "lastname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "lastname", + "constraints": { + "presence": false + } + }, + "QA": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "QA", + "relationshipType": "many-to-one", + "fieldName": "qaid", + "type": "link", + "main": true, + "_id": "ccb68481c80c34217a4540a2c6c27fe46", + "foreignKey": "personid" + }, + "executor": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "executor", + "relationshipType": "many-to-one", + "fieldName": "executorid", + "type": "link", + "main": true, + "_id": "c89530b9770d94bec851e062b5cff3001", + "foreignKey": "personid", + "tableName": "persons" + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "firstname", + "views": {} }, "tableAliases": { "persons": "a", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json index 33a88d30e1..474be32cf2 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json @@ -5,10 +5,7 @@ "operation": "CREATE" }, "resource": { - "fields": [ - "a.name", - "a.age" - ] + "fields": ["a.name", "a.age"] }, "filters": {}, "relationships": [], @@ -24,41 +21,36 @@ } } }, - "meta": { - "table": { - "_id": "datasource_plus_0ed5835e5552496285df546030f7c4ae__people", - "type": "table", - "sourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae", - "sourceType": "external", - "primary": [ - "name", - "age" - ], - "name": "people", - "schema": { - "name": { - "type": "string", - "externalType": "varchar", - "autocolumn": false, - "name": "name", - "constraints": { - "presence": true - } - }, - "age": { - "type": "number", - "externalType": "int", - "autocolumn": false, - "name": "age", - "constraints": { - "presence": false - } + "table": { + "_id": "datasource_plus_0ed5835e5552496285df546030f7c4ae__people", + "type": "table", + "sourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae", + "sourceType": "external", + "primary": ["name", "age"], + "name": "people", + "schema": { + "name": { + "type": "string", + "externalType": "varchar", + "autocolumn": false, + "name": "name", + "constraints": { + "presence": true } }, - "primaryDisplay": "name" - } + "age": { + "type": "number", + "externalType": "int", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + } + }, + "primaryDisplay": "name" }, "tableAliases": { "people": "a" } -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json index 82d85c417b..fbe0b468e9 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json @@ -52,122 +52,113 @@ "idFilter": {} }, "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", - "primary": [ - "personid" - ], - "name": "persons", - "schema": { - "year": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "year", - "constraints": { - "presence": false - } - }, - "firstname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "firstname", - "constraints": { - "presence": false - } - }, - "personid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "personid", - "constraints": { - "presence": false - } - }, - "address": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "address", - "constraints": { - "presence": false - } - }, - "age": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "age", - "constraints": { - "presence": false - } - }, - "type": { - "type": "options", - "externalType": "USER-DEFINED", - "autocolumn": false, - "name": "type", - "constraints": { - "presence": false, - "inclusion": [ - "support", - "designer", - "programmer", - "qa" - ] - } - }, - "city": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "city", - "constraints": { - "presence": false - } - }, - "lastname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "lastname", - "constraints": { - "presence": false - } - }, - "QA": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "QA", - "relationshipType": "many-to-one", - "fieldName": "qaid", - "type": "link", - "main": true, - "_id": "ccb68481c80c34217a4540a2c6c27fe46", - "foreignKey": "personid" - }, - "executor": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "executor", - "relationshipType": "many-to-one", - "fieldName": "executorid", - "type": "link", - "main": true, - "_id": "c89530b9770d94bec851e062b5cff3001", - "foreignKey": "personid", - "tableName": "persons" + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", + "primary": ["personid"], + "name": "persons", + "schema": { + "year": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "year", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "firstname", - "views": {} - } + "firstname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "firstname", + "constraints": { + "presence": false + } + }, + "personid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "personid", + "constraints": { + "presence": false + } + }, + "address": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "address", + "constraints": { + "presence": false + } + }, + "age": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + }, + "type": { + "type": "options", + "externalType": "USER-DEFINED", + "autocolumn": false, + "name": "type", + "constraints": { + "presence": false, + "inclusion": ["support", "designer", "programmer", "qa"] + } + }, + "city": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "city", + "constraints": { + "presence": false + } + }, + "lastname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "lastname", + "constraints": { + "presence": false + } + }, + "QA": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "QA", + "relationshipType": "many-to-one", + "fieldName": "qaid", + "type": "link", + "main": true, + "_id": "ccb68481c80c34217a4540a2c6c27fe46", + "foreignKey": "personid" + }, + "executor": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "executor", + "relationshipType": "many-to-one", + "fieldName": "executorid", + "type": "link", + "main": true, + "_id": "c89530b9770d94bec851e062b5cff3001", + "foreignKey": "personid", + "tableName": "persons" + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "firstname", + "views": {} }, "tableAliases": { "persons": "a", "tasks": "b" } -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json index d6e099c4b6..d9fc34165f 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json @@ -5,11 +5,7 @@ "operation": "DELETE" }, "resource": { - "fields": [ - "a.keyparttwo", - "a.keypartone", - "a.name" - ] + "fields": ["a.keyparttwo", "a.keypartone", "a.name"] }, "filters": { "equal": { @@ -26,50 +22,45 @@ } } }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__compositetable", - "primary": [ - "keypartone", - "keyparttwo" - ], - "name": "compositetable", - "schema": { - "keyparttwo": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "keyparttwo", - "constraints": { - "presence": true - } - }, - "keypartone": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "keypartone", - "constraints": { - "presence": true - } - }, - "name": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "name", - "constraints": { - "presence": false - } + "table": { + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__compositetable", + "primary": ["keypartone", "keyparttwo"], + "name": "compositetable", + "schema": { + "keyparttwo": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "keyparttwo", + "constraints": { + "presence": true } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "keypartone" - } + "keypartone": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "keypartone", + "constraints": { + "presence": true + } + }, + "name": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "name", + "constraints": { + "presence": false + } + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "keypartone" }, "tableAliases": { "compositetable": "a" } -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json index d71f0552c6..f24b46859f 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json @@ -17,10 +17,7 @@ }, "filters": { "oneOf": { - "taskid": [ - 1, - 2 - ] + "taskid": [1, 2] } }, "relationships": [ @@ -42,82 +39,78 @@ "extra": { "idFilter": {} }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", - "primary": [ - "taskid" - ], - "name": "tasks", - "schema": { - "executorid": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "executorid", - "constraints": { - "presence": false - } - }, - "taskname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "taskname", - "constraints": { - "presence": false - } - }, - "taskid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "taskid", - "constraints": { - "presence": false - } - }, - "completed": { - "type": "boolean", - "externalType": "boolean", - "autocolumn": false, - "name": "completed", - "constraints": { - "presence": false - } - }, - "qaid": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "qaid", - "constraints": { - "presence": false - } - }, - "products": { - "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", - "name": "products", - "relationshipType": "many-to-many", - "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", - "type": "link", - "_id": "c3b91d00cd36c4cc1a347794725b9adbd", - "fieldName": "productid", - "throughFrom": "productid", - "throughTo": "taskid" + "table": { + "type": "table", + "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", + "primary": ["taskid"], + "name": "tasks", + "schema": { + "executorid": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "executorid", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", - "sourceType": "external", - "primaryDisplay": "taskname", - "sql": true, - "views": {} - } + "taskname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "taskname", + "constraints": { + "presence": false + } + }, + "taskid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "taskid", + "constraints": { + "presence": false + } + }, + "completed": { + "type": "boolean", + "externalType": "boolean", + "autocolumn": false, + "name": "completed", + "constraints": { + "presence": false + } + }, + "qaid": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "qaid", + "constraints": { + "presence": false + } + }, + "products": { + "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", + "name": "products", + "relationshipType": "many-to-many", + "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", + "type": "link", + "_id": "c3b91d00cd36c4cc1a347794725b9adbd", + "fieldName": "productid", + "throughFrom": "productid", + "throughTo": "taskid" + } + }, + "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", + "sourceType": "external", + "primaryDisplay": "taskname", + "sql": true, + "views": {} }, "tableAliases": { "tasks": "a", "products": "b", "products_tasks": "c" } -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json b/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json index d81c8cf6d0..b746e05c5e 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json @@ -56,48 +56,46 @@ "extra": { "idFilter": {} }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", - "primary": ["productid"], - "name": "products", - "schema": { - "productname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "productname", - "constraints": { - "presence": false - } - }, - "productid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "productid", - "constraints": { - "presence": false - } - }, - "tasks": { - "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", - "name": "tasks", - "relationshipType": "many-to-many", - "fieldName": "taskid", - "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", - "throughFrom": "taskid", - "throughTo": "productid", - "type": "link", - "main": true, - "_id": "c3b91d00cd36c4cc1a347794725b9adbd" + "table": { + "type": "table", + "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", + "primary": ["productid"], + "name": "products", + "schema": { + "productname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "productname", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", - "sourceType": "external", - "primaryDisplay": "productname" - } + "productid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "productid", + "constraints": { + "presence": false + } + }, + "tasks": { + "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", + "name": "tasks", + "relationshipType": "many-to-many", + "fieldName": "taskid", + "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", + "throughFrom": "taskid", + "throughTo": "productid", + "type": "link", + "main": true, + "_id": "c3b91d00cd36c4cc1a347794725b9adbd" + } + }, + "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", + "sourceType": "external", + "primaryDisplay": "productname" }, "tableAliases": { "products": "a", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json index 36010bf61d..6a9717f5dd 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json @@ -46,47 +46,45 @@ "tasks": "b", "products": "a" }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products", - "primary": ["productid"], - "name": "products", - "schema": { - "productname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "productname", - "constraints": { - "presence": false - } - }, - "productid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "productid", - "constraints": { - "presence": false - } - }, - "tasks": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "tasks", - "relationshipType": "many-to-many", - "fieldName": "taskid", - "through": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products_tasks", - "throughFrom": "taskid", - "throughTo": "productid", - "type": "link", - "main": true, - "_id": "ca6862d9ba09146dd8a68e3b5b7055a09" + "table": { + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products", + "primary": ["productid"], + "name": "products", + "schema": { + "productname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "productname", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "productname" - } + "productid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "productid", + "constraints": { + "presence": false + } + }, + "tasks": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "tasks", + "relationshipType": "many-to-many", + "fieldName": "taskid", + "through": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products_tasks", + "throughFrom": "taskid", + "throughTo": "productid", + "type": "link", + "main": true, + "_id": "ca6862d9ba09146dd8a68e3b5b7055a09" + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "productname" } } diff --git a/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json b/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json index 01b85f433f..23f56796a9 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json @@ -102,94 +102,92 @@ "extra": { "idFilter": {} }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", - "primary": ["taskid"], - "name": "tasks", - "schema": { - "executorid": { - "type": "number", - "externalType": "integer", - "name": "executorid", - "constraints": { - "presence": false - }, - "autocolumn": true, - "autoReason": "foreign_key" + "table": { + "type": "table", + "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", + "primary": ["taskid"], + "name": "tasks", + "schema": { + "executorid": { + "type": "number", + "externalType": "integer", + "name": "executorid", + "constraints": { + "presence": false }, - "taskname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "taskname", - "constraints": { - "presence": false - } - }, - "taskid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "taskid", - "constraints": { - "presence": false - } - }, - "completed": { - "type": "boolean", - "externalType": "boolean", - "autocolumn": false, - "name": "completed", - "constraints": { - "presence": false - } - }, - "qaid": { - "type": "number", - "externalType": "integer", - "name": "qaid", - "constraints": { - "presence": false - } - }, - "products": { - "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", - "name": "products", - "relationshipType": "many-to-many", - "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", - "type": "link", - "_id": "c3b91d00cd36c4cc1a347794725b9adbd", - "fieldName": "productid", - "throughFrom": "productid", - "throughTo": "taskid" - }, - "tasksToExecute": { - "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons", - "name": "tasksToExecute", - "relationshipType": "one-to-many", - "type": "link", - "_id": "c0f440590bda04f28846242156c1dd60b", - "foreignKey": "executorid", - "fieldName": "personid" - }, - "tasksToQA": { - "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons", - "name": "tasksToQA", - "relationshipType": "one-to-many", - "type": "link", - "_id": "c5fdf453a0ba743d58e29491d174c974b", - "foreignKey": "qaid", - "fieldName": "personid" + "autocolumn": true, + "autoReason": "foreign_key" + }, + "taskname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "taskname", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", - "sourceType": "external", - "primaryDisplay": "taskname", - "sql": true, - "views": {} - } + "taskid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "taskid", + "constraints": { + "presence": false + } + }, + "completed": { + "type": "boolean", + "externalType": "boolean", + "autocolumn": false, + "name": "completed", + "constraints": { + "presence": false + } + }, + "qaid": { + "type": "number", + "externalType": "integer", + "name": "qaid", + "constraints": { + "presence": false + } + }, + "products": { + "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", + "name": "products", + "relationshipType": "many-to-many", + "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", + "type": "link", + "_id": "c3b91d00cd36c4cc1a347794725b9adbd", + "fieldName": "productid", + "throughFrom": "productid", + "throughTo": "taskid" + }, + "tasksToExecute": { + "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons", + "name": "tasksToExecute", + "relationshipType": "one-to-many", + "type": "link", + "_id": "c0f440590bda04f28846242156c1dd60b", + "foreignKey": "executorid", + "fieldName": "personid" + }, + "tasksToQA": { + "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons", + "name": "tasksToQA", + "relationshipType": "one-to-many", + "type": "link", + "_id": "c5fdf453a0ba743d58e29491d174c974b", + "foreignKey": "qaid", + "fieldName": "personid" + } + }, + "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", + "sourceType": "external", + "primaryDisplay": "taskname", + "sql": true, + "views": {} }, "tableAliases": { "tasks": "a", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json index 42c2a44335..8171e22c64 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json @@ -59,123 +59,114 @@ } } }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", - "primary": [ - "personid" - ], - "name": "persons", - "schema": { - "year": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "year", - "constraints": { - "presence": false - } - }, - "firstname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "firstname", - "constraints": { - "presence": false - } - }, - "personid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "personid", - "constraints": { - "presence": false - } - }, - "address": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "address", - "constraints": { - "presence": false - } - }, - "age": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "age", - "constraints": { - "presence": false - } - }, - "type": { - "type": "options", - "externalType": "USER-DEFINED", - "autocolumn": false, - "name": "type", - "constraints": { - "presence": false, - "inclusion": [ - "support", - "designer", - "programmer", - "qa" - ] - } - }, - "city": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "city", - "constraints": { - "presence": false - } - }, - "lastname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "lastname", - "constraints": { - "presence": false - } - }, - "QA": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "QA", - "relationshipType": "many-to-one", - "fieldName": "qaid", - "type": "link", - "main": true, - "_id": "ccb68481c80c34217a4540a2c6c27fe46", - "foreignKey": "personid" - }, - "executor": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "executor", - "relationshipType": "many-to-one", - "fieldName": "executorid", - "type": "link", - "main": true, - "_id": "c89530b9770d94bec851e062b5cff3001", - "foreignKey": "personid", - "tableName": "persons" + "table": { + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", + "primary": ["personid"], + "name": "persons", + "schema": { + "year": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "year", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "firstname", - "views": {} - } + "firstname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "firstname", + "constraints": { + "presence": false + } + }, + "personid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "personid", + "constraints": { + "presence": false + } + }, + "address": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "address", + "constraints": { + "presence": false + } + }, + "age": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + }, + "type": { + "type": "options", + "externalType": "USER-DEFINED", + "autocolumn": false, + "name": "type", + "constraints": { + "presence": false, + "inclusion": ["support", "designer", "programmer", "qa"] + } + }, + "city": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "city", + "constraints": { + "presence": false + } + }, + "lastname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "lastname", + "constraints": { + "presence": false + } + }, + "QA": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "QA", + "relationshipType": "many-to-one", + "fieldName": "qaid", + "type": "link", + "main": true, + "_id": "ccb68481c80c34217a4540a2c6c27fe46", + "foreignKey": "personid" + }, + "executor": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "executor", + "relationshipType": "many-to-one", + "fieldName": "executorid", + "type": "link", + "main": true, + "_id": "c89530b9770d94bec851e062b5cff3001", + "foreignKey": "personid", + "tableName": "persons" + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "firstname", + "views": {} }, "tableAliases": { "persons": "a", "tasks": "b" } -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json index 42c2a44335..8171e22c64 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json @@ -59,123 +59,114 @@ } } }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", - "primary": [ - "personid" - ], - "name": "persons", - "schema": { - "year": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "year", - "constraints": { - "presence": false - } - }, - "firstname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "firstname", - "constraints": { - "presence": false - } - }, - "personid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "personid", - "constraints": { - "presence": false - } - }, - "address": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "address", - "constraints": { - "presence": false - } - }, - "age": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "age", - "constraints": { - "presence": false - } - }, - "type": { - "type": "options", - "externalType": "USER-DEFINED", - "autocolumn": false, - "name": "type", - "constraints": { - "presence": false, - "inclusion": [ - "support", - "designer", - "programmer", - "qa" - ] - } - }, - "city": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "city", - "constraints": { - "presence": false - } - }, - "lastname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "lastname", - "constraints": { - "presence": false - } - }, - "QA": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "QA", - "relationshipType": "many-to-one", - "fieldName": "qaid", - "type": "link", - "main": true, - "_id": "ccb68481c80c34217a4540a2c6c27fe46", - "foreignKey": "personid" - }, - "executor": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "executor", - "relationshipType": "many-to-one", - "fieldName": "executorid", - "type": "link", - "main": true, - "_id": "c89530b9770d94bec851e062b5cff3001", - "foreignKey": "personid", - "tableName": "persons" + "table": { + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", + "primary": ["personid"], + "name": "persons", + "schema": { + "year": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "year", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "firstname", - "views": {} - } + "firstname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "firstname", + "constraints": { + "presence": false + } + }, + "personid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "personid", + "constraints": { + "presence": false + } + }, + "address": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "address", + "constraints": { + "presence": false + } + }, + "age": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + }, + "type": { + "type": "options", + "externalType": "USER-DEFINED", + "autocolumn": false, + "name": "type", + "constraints": { + "presence": false, + "inclusion": ["support", "designer", "programmer", "qa"] + } + }, + "city": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "city", + "constraints": { + "presence": false + } + }, + "lastname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "lastname", + "constraints": { + "presence": false + } + }, + "QA": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "QA", + "relationshipType": "many-to-one", + "fieldName": "qaid", + "type": "link", + "main": true, + "_id": "ccb68481c80c34217a4540a2c6c27fe46", + "foreignKey": "personid" + }, + "executor": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "executor", + "relationshipType": "many-to-one", + "fieldName": "executorid", + "type": "link", + "main": true, + "_id": "c89530b9770d94bec851e062b5cff3001", + "foreignKey": "personid", + "tableName": "persons" + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "firstname", + "views": {} }, "tableAliases": { "persons": "a", "tasks": "b" } -} \ No newline at end of file +} diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index a160bfbb3e..a12b842afc 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -1,8 +1,8 @@ import { Aggregation, CalculationType, - Datasource, DocumentType, + EnrichedQueryJson, FieldType, isLogicalSearchOperator, Operation, @@ -38,7 +38,7 @@ import { generateJunctionTableID } from "../../../../../db/utils" import AliasTables from "../../sqlAlias" import { outputProcessing } from "../../../../../utilities/rowProcessor" import pick from "lodash/pick" -import { processRowCountResponse } from "../../utils" +import { enrichQueryJson, processRowCountResponse } from "../../utils" import { dataFilters, helpers, @@ -223,18 +223,18 @@ function reverseUserColumnMapping(rows: Row[], table?: Table) { } function runSqlQuery( - json: QueryJson, + json: EnrichedQueryJson, tables: Table[], relationships: RelationshipsJson[] ): Promise function runSqlQuery( - json: QueryJson, + json: EnrichedQueryJson, tables: Table[], relationships: RelationshipsJson[], opts: { countTotalRows: true } ): Promise async function runSqlQuery( - json: QueryJson, + json: EnrichedQueryJson, tables: Table[], relationships: RelationshipsJson[], opts?: { countTotalRows?: boolean } @@ -246,7 +246,7 @@ async function runSqlQuery( if (opts?.countTotalRows) { json.endpoint.operation = Operation.COUNT } - const processSQLQuery = async (_: Datasource, json: QueryJson) => { + const processSQLQuery = async (json: EnrichedQueryJson) => { const query = builder._query(json, { disableReturning: true, }) @@ -281,7 +281,7 @@ async function runSqlQuery( if (opts?.countTotalRows) { return processRowCountResponse(response) } else if (Array.isArray(response)) { - return reverseUserColumnMapping(response, json.meta.table) + return reverseUserColumnMapping(response, json.table) } return response } @@ -372,10 +372,7 @@ export async function search( operation: Operation.READ, }, filters: searchFilters, - table, meta: { - table, - tables: allTablesMap, columnPrefix: USER_COLUMN_PREFIX, }, resource: { @@ -427,11 +424,13 @@ export async function search( } } + const enrichedRequest = await enrichQueryJson(request) + try { const [rows, totalRows] = await Promise.all([ - runSqlQuery(request, allTables, relationships), + runSqlQuery(enrichedRequest, allTables, relationships), options.countRows - ? runSqlQuery(request, allTables, relationships, { + ? runSqlQuery(enrichedRequest, allTables, relationships, { countTotalRows: true, }) : Promise.resolve(undefined), diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index 535709791d..c748940add 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -1,11 +1,13 @@ import { Datasource, DatasourcePlusQueryResponse, + EnrichedQueryJson, Operation, QueryJson, Row, SearchFilters, SqlClient, + Table, } from "@budibase/types" import { SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getSQLClient } from "./utils" @@ -15,7 +17,6 @@ import { BudibaseInternalDB } from "../../../db/utils" import { dataFilters } from "@budibase/shared-core" type PerformQueryFunction = ( - datasource: Datasource, json: QueryJson ) => Promise @@ -99,7 +100,11 @@ export default class AliasTables { return true } - getAlias(tableName: string) { + getAlias(tableName: string | Table) { + if (typeof tableName === "object") { + tableName = tableName.name + } + if (this.aliases[tableName]) { return this.aliases[tableName] } @@ -177,7 +182,7 @@ export default class AliasTables { } async queryWithAliasing( - json: QueryJson, + json: EnrichedQueryJson, queryFn: PerformQueryFunction ): Promise { const datasourceId = json.endpoint.datasourceId @@ -215,14 +220,7 @@ export default class AliasTables { } json.filters = aliasFilters(json.filters) } - if (json.meta?.table) { - this.getAlias(json.meta.table.name) - } - if (json.meta?.tables) { - Object.keys(json.meta.tables).forEach(tableName => - this.getAlias(tableName) - ) - } + if (json.relationships) { json.relationships = json.relationships.map(relationship => ({ ...relationship, @@ -233,6 +231,12 @@ export default class AliasTables { ]), })) } + + this.getAlias(json.table) + for (const tableName of Object.keys(json.tables)) { + this.getAlias(tableName) + } + // invert and return const invertedTableAliases: Record = {} for (let [key, value] of Object.entries(this.tableAliases)) { @@ -241,7 +245,7 @@ export default class AliasTables { json.tableAliases = invertedTableAliases } - let response: DatasourcePlusQueryResponse = await queryFn(datasource, json) + let response = await queryFn(json) if (Array.isArray(response) && aliasingEnabled) { return this.reverse(response) } else { diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index bded6a7a18..f7646a3a85 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -14,13 +14,13 @@ import { SqlClient, ArrayOperator, ViewV2, + EnrichedQueryJson, } from "@budibase/types" -import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" import sdk from "../.." import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils" import { isSQL } from "../../../integrations/utils" -import { docIds, sql } from "@budibase/backend-core" +import { docIds, sql, SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getTableFromSource } from "../../../api/controllers/row/utils" import env from "../../../environment" @@ -73,18 +73,32 @@ export function processRowCountResponse( } } -export async function getDatasourceAndQuery( +export async function enrichQueryJson( json: QueryJson -): Promise { - const datasourceId = json.endpoint.datasourceId - const datasource = await sdk.datasources.get(datasourceId) - const table = datasource.entities?.[json.endpoint.entityId] - if (!json.meta && table) { - json.meta = { - table, +): Promise { + let datasource: Datasource | undefined = undefined + if (json.endpoint.datasourceId !== SQS_DATASOURCE_INTERNAL) { + datasource = await sdk.datasources.get(json.endpoint.datasourceId) + } + + let tables: Record + if (datasource) { + tables = datasource.entities || {} + } else { + tables = {} + for (const table of await sdk.tables.getAllInternalTables()) { + tables[table._id!] = table } } - return makeExternalQuery(datasource, json) + + const table = tables[json.endpoint.entityId] + + return { + table, + tables, + datasource, + ...json, + } } export function cleanExportRows( diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index bdad9c4825..8b0574bd7c 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -1,5 +1,5 @@ import { Table, Row } from "../documents" -import { QueryJson } from "./search" +import { EnrichedQueryJson } from "./search" export const PASSWORD_REPLACEMENT = "--secret-value--" @@ -207,7 +207,7 @@ export interface DatasourcePlus extends IntegrationBase { // this returns the format of the identifier getBindingIdentifier(): string getStringConcat(parts: string[]): string - query(json: QueryJson): Promise + query(json: EnrichedQueryJson): Promise buildSchema( datasourceId: string, entities: Record diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index d6dcb0a2fa..d87472cca9 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -1,5 +1,5 @@ import { Operation } from "./datasources" -import { Row, Table, DocumentType } from "../documents" +import { Row, DocumentType, Table, Datasource } from "../documents" import { SortOrder, SortType } from "../api" import { Knex } from "knex" import { Aggregation } from "./row" @@ -171,10 +171,7 @@ export interface QueryJson { sort?: SortJson paginate?: PaginationJson body?: Row | Row[] - table?: Table - meta: { - table: Table - tables?: Record + meta?: { renamed?: RenameColumn // can specify something that columns could be prefixed with columnPrefix?: string @@ -186,6 +183,12 @@ export interface QueryJson { tableAliases?: Record } +export interface EnrichedQueryJson extends QueryJson { + table: Table + tables: Record + datasource?: Datasource +} + export interface QueryOptions { disableReturning?: boolean disableBindings?: boolean From e4ae16cc3954875a0b127398108f48b004020102 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Nov 2024 16:28:55 +0000 Subject: [PATCH 15/30] wip --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 25dd40ee12..749b3861b5 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 25dd40ee12b048307b558ebcedb36548d6e042cd +Subproject commit 749b3861b579b391496bd8cbce1c473e2f978cc9 From c35e443225d519202d6b65c01f25a0d3b6e7940d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Nov 2024 17:05:47 +0000 Subject: [PATCH 16/30] Making progress toward type checking successfully. --- packages/backend-core/src/sql/sql.ts | 3 + .../server/src/api/controllers/datasource.ts | 6 +- .../api/controllers/table/ExternalRequest.ts | 17 +- .../server/src/integrations/googlesheets.ts | 6 +- .../server/src/integrations/tests/sql.spec.ts | 268 ------------------ packages/server/src/sdk/app/rows/sqlAlias.ts | 2 +- 6 files changed, 16 insertions(+), 286 deletions(-) delete mode 100644 packages/server/src/integrations/tests/sql.spec.ts diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index c897c106b0..5333bfb591 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1253,6 +1253,9 @@ class InternalBuilder { continue } const relatedTable = tables[toTable] + if (!relatedTable) { + throw new Error(`related table "${toTable}" not found in datasource`) + } const toAlias = aliases?.[toTable] || toTable, fromAlias = aliases?.[fromTable] || fromTable, throughAlias = (throughTable && aliases?.[throughTable]) || throughTable diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 794e2dfddd..9f4fff4c46 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -23,11 +23,13 @@ import { Table, RowValue, DynamicVariable, + QueryJson, } from "@budibase/types" import sdk from "../../sdk" import { builderSocket } from "../../websockets" import { isEqual } from "lodash" import { processTable } from "../../sdk/app/tables/getters" +import { makeExternalQuery } from "src/integrations/base/query" export async function fetch(ctx: UserCtx) { ctx.body = await sdk.datasources.fetch() @@ -297,10 +299,10 @@ export async function find(ctx: UserCtx) { } // dynamic query functionality -export async function query(ctx: UserCtx) { +export async function query(ctx: UserCtx) { const queryJson = ctx.request.body try { - ctx.body = await sdk.rows.utils.getDatasourceAndQuery(queryJson) + ctx.body = await makeExternalQuery(queryJson) } catch (err: any) { ctx.throw(400, err) } diff --git a/packages/server/src/api/controllers/table/ExternalRequest.ts b/packages/server/src/api/controllers/table/ExternalRequest.ts index 1e57ea3294..dc32c79c5a 100644 --- a/packages/server/src/api/controllers/table/ExternalRequest.ts +++ b/packages/server/src/api/controllers/table/ExternalRequest.ts @@ -11,8 +11,6 @@ export async function makeTableRequest( datasource: Datasource, operation: Operation, table: Table, - tables: Record, - oldTable?: Table, renamed?: RenameColumn ) { const json: QueryJson = { @@ -21,17 +19,12 @@ export async function makeTableRequest( entityId: table._id!, operation, }, - meta: { - table, - tables, - }, - table, - } - if (oldTable) { - json.meta!.table = oldTable } if (renamed) { - json.meta!.renamed = renamed + if (!json.meta) { + json.meta = {} + } + json.meta.renamed = renamed } - return makeExternalQuery(datasource, json) + return makeExternalQuery(json) } diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 5f61791683..d8bf0e376d 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -7,7 +7,6 @@ import { Integration, Operation, PaginationJson, - QueryJson, QueryType, Row, Schema, @@ -18,6 +17,7 @@ import { TableSourceType, DatasourcePlusQueryResponse, BBReferenceFieldSubType, + EnrichedQueryJson, } from "@budibase/types" import { OAuth2Client } from "google-auth-library" import { @@ -381,7 +381,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus { return { tables: externalTables, errors } } - async query(json: QueryJson): Promise { + async query(json: EnrichedQueryJson): Promise { const sheet = json.endpoint.entityId switch (json.endpoint.operation) { case Operation.CREATE: @@ -400,7 +400,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus { rowIndex: json.extra?.idFilter?.equal?.rowNumber, sheet, row: json.body, - table: json.meta.table, + table: json.table, }) case Operation.DELETE: return this.delete({ diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts deleted file mode 100644 index 2b3649161c..0000000000 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ /dev/null @@ -1,268 +0,0 @@ -import { - FieldType, - Operation, - PaginationJson, - QueryJson, - SearchFilters, - SortJson, - SqlClient, - Table, - TableSourceType, -} from "@budibase/types" -import { sql } from "@budibase/backend-core" -import { merge } from "lodash" - -const Sql = sql.Sql - -const TABLE_NAME = "test" -const TABLE: Table = { - type: "table", - sourceType: TableSourceType.EXTERNAL, - sourceId: "SOURCE_ID", - schema: { - id: { - name: "id", - type: FieldType.NUMBER, - }, - }, - name: TABLE_NAME, - primary: ["id"], -} - -const ORACLE_TABLE: Partial = { - schema: { - name: { - name: "name", - type: FieldType.STRING, - }, - }, -} - -function endpoint(table: string, operation: Operation) { - return { - datasourceId: "Postgres", - operation: operation, - entityId: table || TABLE_NAME, - } -} - -function generateReadJson({ - table, - fields, - filters, - sort, - paginate, -}: { - table?: Partial
- fields?: string[] - filters?: SearchFilters - sort?: SortJson - paginate?: PaginationJson -} = {}): QueryJson { - let tableObj: Table = { ...TABLE } - if (table) { - tableObj = merge(TABLE, table) - } - return { - endpoint: endpoint(tableObj.name || TABLE_NAME, Operation.READ), - resource: { - fields: fields || [], - }, - filters: filters || {}, - sort: sort || {}, - paginate: paginate || undefined, - meta: { - table: tableObj, - }, - } -} - -function generateRelationshipJson(config: { schema?: string } = {}): QueryJson { - return { - endpoint: { - datasourceId: "Postgres", - entityId: "brands", - operation: Operation.READ, - schema: config.schema, - }, - resource: { - fields: [ - "brands.brand_id", - "brands.brand_name", - "products.product_id", - "products.product_name", - "products.brand_id", - ], - }, - filters: {}, - sort: {}, - relationships: [ - { - from: "brand_id", - to: "brand_id", - tableName: "products", - column: "products", - }, - ], - extra: { idFilter: {} }, - meta: { - table: TABLE, - }, - } -} - -function generateManyRelationshipJson(config: { schema?: string } = {}) { - return { - endpoint: { - datasourceId: "Postgres", - entityId: "stores", - operation: "READ", - schema: config.schema, - }, - resource: { - fields: [ - "stores.store_id", - "stores.store_name", - "products.product_id", - "products.product_name", - ], - }, - filters: {}, - sort: {}, - paginate: {}, - relationships: [ - { - from: "store_id", - to: "product_id", - tableName: "products", - column: "products", - through: "stocks", - fromPrimary: "store_id", - toPrimary: "product_id", - }, - ], - extra: { idFilter: {} }, - meta: { - table: TABLE, - }, - } -} - -describe("SQL query builder", () => { - const relationshipLimit = 500 - const limit = 500 - const client = SqlClient.POSTGRES - let sql: any - - beforeEach(() => { - sql = new Sql(client, limit) - }) - - it("should add the schema to the LEFT JOIN", () => { - const query = sql._query(generateRelationshipJson({ schema: "production" })) - expect(query).toEqual({ - bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, - }) - }) - - it("should handle if the schema is not present when doing a LEFT JOIN", () => { - const query = sql._query(generateRelationshipJson()) - expect(query).toEqual({ - bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, - }) - }) - - it("should add the schema to both the toTable and throughTable in many-to-many join", () => { - const query = sql._query( - generateManyRelationshipJson({ schema: "production" }) - ) - expect(query).toEqual({ - bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "stores".* from "production"."stores" order by "test"."id" asc limit $1) select "stores".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" inner join "production"."stocks" as "stocks" on "products"."product_id" = "stocks"."product_id" where "stocks"."store_id" = "stores"."store_id" order by "products"."product_id" asc limit $2) as "products") as "products" from "paginated" as "stores" order by "test"."id" asc`, - }) - }) - - it("should lowercase the values for Oracle LIKE statements", () => { - let query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - filters: { - string: { - name: "John", - }, - }, - }) - ) - expect(query).toEqual({ - bindings: ["john%", limit], - sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`, - }) - - query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - filters: { - contains: { - age: [20, 25], - name: ["John", "Mary"], - }, - }, - }) - ) - const filterSet = [`%20%`, `%25%`, `%"john"%`, `%"mary"%`] - expect(query).toEqual({ - bindings: [...filterSet, limit], - sql: `select * from (select * from "test" where ((COALESCE(LOWER("test"."age"), '') like :1 and COALESCE(LOWER("test"."age"), '') like :2)) and ((COALESCE(LOWER("test"."name"), '') like :3 and COALESCE(LOWER("test"."name"), '') like :4)) order by "test"."id" asc) where rownum <= :5`, - }) - - query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - filters: { - fuzzy: { - name: "Jo", - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [`%jo%`, limit], - sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`, - }) - }) - - it("should use an oracle compatible coalesce query for oracle when using the equals filter", () => { - let query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - table: ORACLE_TABLE, - filters: { - equal: { - name: "John", - }, - }, - }) - ) - - expect(query).toEqual({ - bindings: ["John", limit], - sql: `select * from (select * from "test" where (to_char("test"."name") is not null and to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2`, - }) - }) - - it("should use an oracle compatible coalesce query for oracle when using the not equals filter", () => { - let query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - table: ORACLE_TABLE, - filters: { - notEqual: { - name: "John", - }, - }, - }) - ) - - expect(query).toEqual({ - bindings: ["John", limit], - sql: `select * from (select * from "test" where (to_char("test"."name") is not null and to_char("test"."name") != :1) or to_char("test"."name") is null order by "test"."id" asc) where rownum <= :2`, - }) - }) -}) diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index c748940add..c14307360b 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -17,7 +17,7 @@ import { BudibaseInternalDB } from "../../../db/utils" import { dataFilters } from "@budibase/shared-core" type PerformQueryFunction = ( - json: QueryJson + json: EnrichedQueryJson ) => Promise const WRITE_OPERATIONS: Operation[] = [ From 8e756ccf848943a07526f929fc288ab9775507bb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Nov 2024 17:09:32 +0000 Subject: [PATCH 17/30] Type checks pass. --- .../server/src/integrations/tests/sqlAlias.spec.ts | 8 +++----- .../server/src/sdk/app/tables/external/index.ts | 13 +++---------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 739d3a4aee..66340bc794 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -7,6 +7,7 @@ import { Table, TableSourceType, SqlClient, + EnrichedQueryJson, } from "@budibase/types" import { sql } from "@budibase/backend-core" import { join } from "path" @@ -35,8 +36,8 @@ describe("Captures of real examples", () => { const relationshipLimit = 500 const primaryLimit = 100 - function getJson(name: string): QueryJson { - return require(join(__dirname, "sqlQueryJson", name)) as QueryJson + function getJson(name: string) { + return require(join(__dirname, "sqlQueryJson", name)) as EnrichedQueryJson } describe("create", () => { @@ -252,9 +253,6 @@ describe("Captures of real examples", () => { resource: { fields, }, - meta: { - table: TABLE, - }, } } diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 941d193b94..5d8f03a7a6 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -241,19 +241,12 @@ export async function save( } const operation = tableId ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE - await makeTableRequest( - datasource, - operation, - tableToSave, - tables, - oldTable, - opts?.renaming - ) + await makeTableRequest(datasource, operation, tableToSave, opts?.renaming) // update any extra tables (like foreign keys in other tables) for (let extraTable of extraTablesToUpdate) { const oldExtraTable = oldTables[extraTable.name] let op = oldExtraTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE - await makeTableRequest(datasource, op, extraTable, tables, oldExtraTable) + await makeTableRequest(datasource, op, extraTable) } // make sure the constrained list, all still exist @@ -292,7 +285,7 @@ export async function destroy(datasourceId: string, table: Table) { const operation = Operation.DELETE_TABLE if (tables) { - await makeTableRequest(datasource, operation, table, tables) + await makeTableRequest(datasource, operation, table) cleanupRelationships(table, tables, { deleting: true }) delete tables[table.name] datasource.entities = tables From f21199011788c2f6e69e77d5ea983f3f8662961b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Nov 2024 17:36:17 +0000 Subject: [PATCH 18/30] Some sqs fixes. --- packages/backend-core/src/sql/sql.ts | 13 ++++++++++--- packages/server/src/api/controllers/datasource.ts | 2 +- packages/server/src/api/routes/tests/row.spec.ts | 2 +- .../server/src/sdk/app/rows/search/internal/sqs.ts | 3 ++- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 5333bfb591..baeeefbb81 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -3,6 +3,7 @@ import * as dbCore from "../db" import { getNativeSql, isExternalTable, + isInternalTableID, isInvalidISODateString, isValidFilter, isValidISODateString, @@ -1192,7 +1193,7 @@ class InternalBuilder { private buildJsonField(table: Table, field: string): [string, Knex.Raw] { const parts = field.split(".") - const baseName = parts[parts.length - 1] + let baseName = parts[parts.length - 1] let unaliased: string let tableField: string @@ -1205,10 +1206,16 @@ class InternalBuilder { tableField = unaliased } - const schema = table.schema[baseName] + if (this.query.meta?.columnPrefix) { + baseName = baseName.replace(this.query.meta.columnPrefix, "") + } let identifier = this.rawQuotedIdentifier(tableField) - if (schema.type === FieldType.BIGINT) { + // Internal tables have special _id, _rev, createdAt, and updatedAt fields + // that do not appear in the schema, meaning schema could actually be + // undefined. + const schema: FieldSchema | undefined = table.schema[baseName] + if (schema && schema.type === FieldType.BIGINT) { identifier = this.castIntToString(identifier) } return [unaliased, identifier] diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 9f4fff4c46..01d81425b6 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -29,7 +29,7 @@ import sdk from "../../sdk" import { builderSocket } from "../../websockets" import { isEqual } from "lodash" import { processTable } from "../../sdk/app/tables/getters" -import { makeExternalQuery } from "src/integrations/base/query" +import { makeExternalQuery } from "../../integrations/base/query" export async function fetch(ctx: UserCtx) { ctx.body = await sdk.datasources.fetch() diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 9e84e74d05..1ab12feebb 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3333,7 +3333,7 @@ if (descriptions.length) { }) isInternal && - it("should coerce a static handlebars formula", async () => { + it.only("should coerce a static handlebars formula", async () => { await updateFormulaColumn(encodeJS("return 1"), { responseType: FieldType.NUMBER, formulaType: FormulaType.STATIC, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index a12b842afc..aec934ae4e 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -494,6 +494,7 @@ export async function search( if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) { return { rows: [] } } - throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) + // throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) + throw err } } From 92e92c8b7f5e012993b8c7dd875585caed305d0f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 26 Nov 2024 18:15:08 +0000 Subject: [PATCH 19/30] Updating sql alias test case. --- packages/backend-core/src/sql/sql.ts | 2 +- .../src/integrations/tests/sqlAlias.spec.ts | 59 +++++++++++++------ .../sqlQueryJson/createWithRelationships.json | 2 +- 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index c897c106b0..f6acfbffe2 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1208,7 +1208,7 @@ class InternalBuilder { const schema = table.schema[baseName] let identifier = this.rawQuotedIdentifier(tableField) - if (schema.type === FieldType.BIGINT) { + if (schema?.type === FieldType.BIGINT) { identifier = this.castIntToString(identifier) } return [unaliased, identifier] diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 739d3a4aee..951c24a65f 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -1,12 +1,13 @@ import { Datasource, Operation, - QueryJson, SourceName, SqlQuery, Table, TableSourceType, SqlClient, + EnrichedQueryJson, + TableSchema, } from "@budibase/types" import { sql } from "@budibase/backend-core" import { join } from "path" @@ -16,17 +17,21 @@ import sdk from "../../sdk" const Sql = sql.Sql // this doesn't exist strictly -const TABLE: Table = { - type: "table", - sourceType: TableSourceType.EXTERNAL, - sourceId: "SOURCE_ID", - schema: {}, - name: "tableName", - primary: ["id"], -} +const TABLE = buildTable("tableName", {}) const AliasTables = sdk.rows.AliasTables +function buildTable(name: string, schema: TableSchema): Table { + return { + type: "table", + sourceType: TableSourceType.EXTERNAL, + sourceId: "SOURCE_ID", + schema: schema, + name: name, + primary: ["id"], + } +} + function multiline(sql: string) { return sql.replace(/\n/g, "").replace(/ +/g, " ") } @@ -35,8 +40,22 @@ describe("Captures of real examples", () => { const relationshipLimit = 500 const primaryLimit = 100 - function getJson(name: string): QueryJson { - return require(join(__dirname, "sqlQueryJson", name)) as QueryJson + function getJson(name: string): EnrichedQueryJson { + // tables aren't fully specified in the test JSON + const base = require(join(__dirname, "sqlQueryJson", name)) as Omit< + EnrichedQueryJson, + "tables" + > + const tables: Record = { [base.table.name]: base.table } + if (base.relationships) { + for (let { tableName } of base.relationships) { + tables[tableName] = buildTable(tableName, {}) + } + } + return { + ...base, + tables: tables, + } } describe("create", () => { @@ -63,7 +82,7 @@ describe("Captures of real examples", () => { bindings: [primaryLimit, relationshipLimit, relationshipLimit], sql: expect.stringContaining( multiline( - `select json_agg(json_build_object('completed',"b"."completed",'completed',"b"."completed",'executorid',"b"."executorid",'executorid',"b"."executorid",'qaid',"b"."qaid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskid',"b"."taskid",'taskname',"b"."taskname",'taskname',"b"."taskname")` + `select json_agg(json_build_object('executorid',"b"."executorid",'executorid',"b"."executorid",'qaid',"b"."qaid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskid',"b"."taskid",'completed',"b"."completed",'completed',"b"."completed",'taskname',"b"."taskname",'taskname',"b"."taskname"` ) ), }) @@ -94,7 +113,7 @@ describe("Captures of real examples", () => { sql: expect.stringContaining( multiline( `with "paginated" as (select "a".* from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) - select "a".*, (select json_agg(json_build_object('completed',"b"."completed",'executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskname',"b"."taskname")) + select "a".*, (select json_agg(json_build_object('executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'completed',"b"."completed",'taskname',"b"."taskname")) from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc` ) @@ -212,8 +231,8 @@ describe("Captures of real examples", () => { }, queryJson) expect(returningQuery).toEqual({ sql: multiline( - `select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1 - and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc` + `select top (@p0) * from [people] as [a] where CASE WHEN [a].[name] = @p1 THEN 1 ELSE 0 END = 1 and + CASE WHEN [a].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [a].[name] asc` ), bindings: [1, "Test", 22], }) @@ -246,15 +265,17 @@ describe("Captures of real examples", () => { } } - function getQuery(op: Operation, fields: string[] = ["a"]): QueryJson { + function getQuery( + op: Operation, + fields: string[] = ["a"] + ): EnrichedQueryJson { return { endpoint: { datasourceId: "", entityId: "", operation: op }, resource: { fields, }, - meta: { - table: TABLE, - }, + table: TABLE, + tables: { [TABLE.name]: TABLE }, } } diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json index fbe0b468e9..060197dd01 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json @@ -51,7 +51,7 @@ "extra": { "idFilter": {} }, - "meta": { + "table": { "type": "table", "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", "primary": ["personid"], From bd17770e4ec3cc9414ca97e4ce3de3dc923eb9b4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 27 Nov 2024 12:21:27 +0000 Subject: [PATCH 20/30] Fix _some_ sqs problems. --- .../server/src/api/routes/tests/row.spec.ts | 6 +++--- .../src/sdk/app/rows/search/internal/sqs.ts | 19 +++++-------------- packages/server/src/sdk/app/rows/utils.ts | 16 ++++++++++++---- packages/server/src/sdk/app/tables/getters.ts | 1 + 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 1ab12feebb..6756d2e462 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3268,7 +3268,7 @@ if (descriptions.length) { formula: { name: "formula", type: FieldType.FORMULA, - formula: formula, + formula, responseType: opts?.responseType, formulaType: opts?.formulaType || FormulaType.DYNAMIC, }, @@ -3276,7 +3276,7 @@ if (descriptions.length) { }) } - it("should be able to search for rows containing formulas", async () => { + it.only("should be able to search for rows containing formulas", async () => { const { rows } = await config.api.row.search(table._id!) expect(rows.length).toBe(1) expect(rows[0].links.length).toBe(1) @@ -3333,7 +3333,7 @@ if (descriptions.length) { }) isInternal && - it.only("should coerce a static handlebars formula", async () => { + it("should coerce a static handlebars formula", async () => { await updateFormulaColumn(encodeJS("return 1"), { responseType: FieldType.NUMBER, formulaType: FormulaType.STATIC, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index aec934ae4e..1a55fdf99c 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -180,19 +180,6 @@ function cleanupFilters(filters: SearchFilters, allTables: Table[]) { return filters } -function buildTableMap(tables: Table[]) { - const tableMap: Record = {} - for (let table of tables) { - // update the table name, should never query by name for SQLite - table.originalName = table.name - table.name = table._id! - // need a primary for sorting, lookups etc - table.primary = ["_id"] - tableMap[table._id!] = table - } - return tableMap -} - // table is only needed to handle relationships function reverseUserColumnMapping(rows: Row[], table?: Table) { const prefixLength = USER_COLUMN_PREFIX.length @@ -315,7 +302,11 @@ export async function search( } const allTables = await sdk.tables.getAllInternalTables() - const allTablesMap = buildTableMap(allTables) + const allTablesMap = allTables.reduce((acc, table) => { + acc[table._id!] = table + return acc + }, {} as Record) + // make sure we have the mapped/latest table if (table._id) { table = allTablesMap[table._id] diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index f7646a3a85..eb6859c58b 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -73,6 +73,17 @@ export function processRowCountResponse( } } +function processInternalTables(tables: Table[]) { + const tableMap: Record = {} + for (let table of tables) { + // update the table name, should never query by name for SQLite + table.originalName = table.name + table.name = table._id! + tableMap[table._id!] = table + } + return tableMap +} + export async function enrichQueryJson( json: QueryJson ): Promise { @@ -85,10 +96,7 @@ export async function enrichQueryJson( if (datasource) { tables = datasource.entities || {} } else { - tables = {} - for (const table of await sdk.tables.getAllInternalTables()) { - tables[table._id!] = table - } + tables = processInternalTables(await sdk.tables.getAllInternalTables()) } const table = tables[json.endpoint.entityId] diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 1ad82b8e42..79091ea660 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -46,6 +46,7 @@ export async function processTable(table: Table): Promise
{ const processed: Table = { ...table, type: "table", + primary: ["_id"], // internal tables must always use _id as primary key sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID, sourceType: TableSourceType.INTERNAL, sql: true, From 504ddba9b98599e973f35ca73e47cae271eadd9c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 27 Nov 2024 12:40:34 +0000 Subject: [PATCH 21/30] SQS row tests passing. --- packages/server/src/api/routes/tests/row.spec.ts | 2 +- packages/server/src/sdk/app/rows/search/internal/sqs.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 6756d2e462..b371ab1f8a 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3276,7 +3276,7 @@ if (descriptions.length) { }) } - it.only("should be able to search for rows containing formulas", async () => { + it("should be able to search for rows containing formulas", async () => { const { rows } = await config.api.row.search(table._id!) expect(rows.length).toBe(1) expect(rows[0].links.length).toBe(1) diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 1a55fdf99c..ba5614edca 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -228,7 +228,7 @@ async function runSqlQuery( ) { const relationshipJunctionTableIds = relationships.map(rel => rel.through!) const alias = new AliasTables( - tables.map(table => table.name).concat(relationshipJunctionTableIds) + tables.map(table => table._id!).concat(relationshipJunctionTableIds) ) if (opts?.countTotalRows) { json.endpoint.operation = Operation.COUNT From 332505388dba3b2a35de2e32c120d04ead4ee427 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 27 Nov 2024 13:14:42 +0000 Subject: [PATCH 22/30] Fix postgres tests. --- .../server/src/api/controllers/datasource.ts | 4 ++-- .../src/api/controllers/row/ExternalRequest.ts | 4 ++-- .../api/controllers/table/ExternalRequest.ts | 2 +- .../routes/tests/queries/generic-sql.spec.ts | 2 +- packages/server/src/integrations/base/query.ts | 2 +- .../src/integrations/tests/sqlAlias.spec.ts | 15 +-------------- .../src/sdk/app/rows/search/internal/sqs.ts | 2 +- packages/server/src/sdk/app/rows/sqlAlias.ts | 11 +++-------- packages/server/src/sdk/app/rows/utils.ts | 18 +++++++++++++++--- .../src/tests/utilities/api/datasource.ts | 7 ++----- packages/types/src/sdk/search.ts | 6 +++++- 11 files changed, 34 insertions(+), 39 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 01d81425b6..82eadca312 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -23,7 +23,7 @@ import { Table, RowValue, DynamicVariable, - QueryJson, + QueryJsonRequest, } from "@budibase/types" import sdk from "../../sdk" import { builderSocket } from "../../websockets" @@ -299,7 +299,7 @@ export async function find(ctx: UserCtx) { } // dynamic query functionality -export async function query(ctx: UserCtx) { +export async function query(ctx: UserCtx) { const queryJson = ctx.request.body try { ctx.body = await makeExternalQuery(queryJson) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index baf1665cbe..422896cd52 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -137,7 +137,7 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig { function getEndpoint(tableId: string, operation: Operation) { const { datasourceId, tableName } = breakExternalTableId(tableId) - return { datasourceId, entityId: tableName, operation } + return { datasource: datasourceId, entityId: tableName, operation } } function isOneSide( @@ -706,7 +706,7 @@ export class ExternalRequest { let json: QueryJson = { endpoint: { - datasourceId: this.datasource._id!, + datasource: this.datasource, entityId: table.name, operation, }, diff --git a/packages/server/src/api/controllers/table/ExternalRequest.ts b/packages/server/src/api/controllers/table/ExternalRequest.ts index dc32c79c5a..ca4fcb1d16 100644 --- a/packages/server/src/api/controllers/table/ExternalRequest.ts +++ b/packages/server/src/api/controllers/table/ExternalRequest.ts @@ -15,7 +15,7 @@ export async function makeTableRequest( ) { const json: QueryJson = { endpoint: { - datasourceId: datasource._id!, + datasource, entityId: table._id!, operation, }, diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 44b21e0350..9130034501 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -837,7 +837,7 @@ if (descriptions.length) { const res = await config.api.datasource.query({ endpoint: { - datasourceId: datasource._id!, + datasource: datasource._id!, operation: Operation.READ, entityId, }, diff --git a/packages/server/src/integrations/base/query.ts b/packages/server/src/integrations/base/query.ts index 2185eace2c..6555a5de51 100644 --- a/packages/server/src/integrations/base/query.ts +++ b/packages/server/src/integrations/base/query.ts @@ -23,5 +23,5 @@ export async function makeExternalQuery( } const integration = new Integration(enrichedJson.datasource.config) - return integration.query(json) + return integration.query(enrichedJson) } diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 66340bc794..73a9c9fe8d 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -4,8 +4,6 @@ import { QueryJson, SourceName, SqlQuery, - Table, - TableSourceType, SqlClient, EnrichedQueryJson, } from "@budibase/types" @@ -15,17 +13,6 @@ import { generator } from "@budibase/backend-core/tests" import sdk from "../../sdk" const Sql = sql.Sql - -// this doesn't exist strictly -const TABLE: Table = { - type: "table", - sourceType: TableSourceType.EXTERNAL, - sourceId: "SOURCE_ID", - schema: {}, - name: "tableName", - primary: ["id"], -} - const AliasTables = sdk.rows.AliasTables function multiline(sql: string) { @@ -249,7 +236,7 @@ describe("Captures of real examples", () => { function getQuery(op: Operation, fields: string[] = ["a"]): QueryJson { return { - endpoint: { datasourceId: "", entityId: "", operation: op }, + endpoint: { datasource: "", entityId: "", operation: op }, resource: { fields, }, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index ba5614edca..d6ac2a5862 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -358,7 +358,7 @@ export async function search( const request: QueryJson = { endpoint: { // not important, we query ourselves - datasourceId: SQS_DATASOURCE_INTERNAL, + datasource: SQS_DATASOURCE_INTERNAL, entityId: table._id!, operation: Operation.READ, }, diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index c14307360b..2f9bea3c91 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -9,11 +9,8 @@ import { SqlClient, Table, } from "@budibase/types" -import { SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getSQLClient } from "./utils" import { cloneDeep } from "lodash" -import datasources from "../datasources" -import { BudibaseInternalDB } from "../../../db/utils" import { dataFilters } from "@budibase/shared-core" type PerformQueryFunction = ( @@ -185,14 +182,12 @@ export default class AliasTables { json: EnrichedQueryJson, queryFn: PerformQueryFunction ): Promise { - const datasourceId = json.endpoint.datasourceId - const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL - let aliasingEnabled: boolean, datasource: Datasource + const datasource = json.datasource + const isSqs = datasource === undefined + let aliasingEnabled: boolean if (isSqs) { aliasingEnabled = this.isAliasingEnabled(json) - datasource = BudibaseInternalDB } else { - datasource = await datasources.get(datasourceId) aliasingEnabled = this.isAliasingEnabled(json, datasource) } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index eb6859c58b..17bbebb251 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -23,6 +23,8 @@ import { isSQL } from "../../../integrations/utils" import { docIds, sql, SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getTableFromSource } from "../../../api/controllers/row/utils" import env from "../../../environment" +import { breakExternalTableId } from "@budibase/backend-core/src/sql/utils" +import { isDatasourceId } from "@budibase/backend-core/src/docIds" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, @@ -88,8 +90,18 @@ export async function enrichQueryJson( json: QueryJson ): Promise { let datasource: Datasource | undefined = undefined - if (json.endpoint.datasourceId !== SQS_DATASOURCE_INTERNAL) { - datasource = await sdk.datasources.get(json.endpoint.datasourceId) + let entityId = json.endpoint.entityId + if (typeof json.endpoint.datasource === "string") { + if (json.endpoint.datasource !== SQS_DATASOURCE_INTERNAL) { + datasource = await sdk.datasources.get(json.endpoint.datasource, { + enriched: true, + }) + } + } else { + datasource = json.endpoint.datasource + if (isDatasourceId(entityId)) { + entityId = breakExternalTableId(entityId).tableName + } } let tables: Record @@ -99,7 +111,7 @@ export async function enrichQueryJson( tables = processInternalTables(await sdk.tables.getAllInternalTables()) } - const table = tables[json.endpoint.entityId] + const table = tables[entityId] return { table, diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts index 87f03c8a6f..f4adaefa24 100644 --- a/packages/server/src/tests/utilities/api/datasource.ts +++ b/packages/server/src/tests/utilities/api/datasource.ts @@ -4,7 +4,7 @@ import { Datasource, FetchDatasourceInfoResponse, FieldType, - QueryJson, + QueryJsonRequest, RelationshipType, UpdateDatasourceRequest, UpdateDatasourceResponse, @@ -69,10 +69,7 @@ export class DatasourceAPI extends TestAPI { return await this._get(`/api/datasources`, { expectations }) } - query = async ( - query: Omit & Partial>, - expectations?: Expectations - ) => { + query = async (query: QueryJsonRequest, expectations?: Expectations) => { return await this._post(`/api/datasources/query`, { body: query, expectations, diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index d87472cca9..72f7de455a 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -158,7 +158,7 @@ export interface ManyToManyRelationshipJson { export interface QueryJson { endpoint: { - datasourceId: string + datasource: string | Datasource entityId: string operation: Operation schema?: string @@ -189,6 +189,10 @@ export interface EnrichedQueryJson extends QueryJson { datasource?: Datasource } +export interface QueryJsonRequest extends Omit { + endpoint: QueryJson["endpoint"] & { datasource: string } +} + export interface QueryOptions { disableReturning?: boolean disableBindings?: boolean From 525cf2ab09085c48e189d04cb3653998da2ee442 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 27 Nov 2024 13:16:21 +0000 Subject: [PATCH 23/30] Update pro reference. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 749b3861b5..06e4506cef 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 749b3861b579b391496bd8cbce1c473e2f978cc9 +Subproject commit 06e4506cef37110e509a0ef0b9811ac07f8b2844 From 4feda79903c592e0f58815ee982207940f60d856 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 27 Nov 2024 15:52:39 +0000 Subject: [PATCH 24/30] Fix Oracle tests. --- packages/backend-core/src/sql/sql.ts | 4 +++- packages/server/src/integrations/oracle.ts | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index baeeefbb81..c21a9b3fe5 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1743,7 +1743,9 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { return {} } const input = this._query({ - ...json, + datasource: json.datasource, + table: json.table, + tables: json.tables, endpoint: { ...json.endpoint, operation: Operation.READ, diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index b48a95e3e6..4343593d1e 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -486,6 +486,8 @@ class OracleIntegration extends Sql implements DatasourcePlus { this.log(query.sql, bindings) const result = await connection.execute(query.sql, bindings, options) return result as Result + } catch (err) { + throw err } finally { if (connection) { try { From 78bbf90f170435158f0220f27d6c5a4031e68570 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 27 Nov 2024 16:37:29 +0000 Subject: [PATCH 25/30] Fix row.spec.ts --- .../server/src/api/routes/tests/row.spec.ts | 35 +++++-------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index b371ab1f8a..b5fff83b43 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3548,35 +3548,16 @@ if (descriptions.length) { }) let resp = await config.api.row.search(table1._id!) - expect(resp.rows).toEqual([ - expect.objectContaining({ - _id: "%5B'1'%5D", - table1Id: "1", - many: [ - { - _id: "%5B'2'%5D", - primaryDisplay: "2", - }, - ], - tableId: table1._id, - }), - ]) + expect(resp.rows).toHaveLength(1) + expect(resp.rows[0]._id).toBe("%5B'1'%5D") + expect(resp.rows[0].many).toHaveLength(1) + expect(resp.rows[0].many[0]._id).toBe("%5B'2'%5D") resp = await config.api.row.search(table2._id!) - expect(resp.rows).toEqual([ - expect.objectContaining({ - _id: "%5B'2'%5D", - table2Id: "2", - table1Ref: "1", - one: [ - { - _id: "%5B'1'%5D", - primaryDisplay: "1", - }, - ], - tableId: table2._id, - }), - ]) + expect(resp.rows).toHaveLength(1) + expect(resp.rows[0]._id).toBe("%5B'2'%5D") + expect(resp.rows[0].one).toHaveLength(1) + expect(resp.rows[0].one[0]._id).toBe("%5B'1'%5D") }) }) } From dbe9eedcbadc6d164447078039801a53a2dcfa65 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 27 Nov 2024 17:28:20 +0000 Subject: [PATCH 26/30] Fix table.spec.ts. --- packages/backend-core/src/sql/sql.ts | 19 +++++--------- packages/backend-core/src/sql/sqlTable.ts | 8 +++--- packages/pro | 2 +- .../api/controllers/row/ExternalRequest.ts | 6 ++--- .../api/controllers/table/ExternalRequest.ts | 14 ++++++---- .../src/api/controllers/table/external.ts | 2 +- .../routes/tests/queries/generic-sql.spec.ts | 2 +- .../server/src/integrations/googlesheets.ts | 2 +- packages/server/src/integrations/oracle.ts | 2 +- .../src/integrations/tests/sqlAlias.spec.ts | 5 ++-- .../src/sdk/app/rows/search/internal/sqs.ts | 2 +- packages/server/src/sdk/app/rows/sqlAlias.ts | 2 +- packages/server/src/sdk/app/rows/utils.ts | 26 +++++++++++-------- .../src/sdk/app/tables/external/index.ts | 8 +++++- packages/types/src/sdk/search.ts | 7 ++--- 15 files changed, 58 insertions(+), 49 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index c21a9b3fe5..d251ff672d 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -3,7 +3,6 @@ import * as dbCore from "../db" import { getNativeSql, isExternalTable, - isInternalTableID, isInvalidISODateString, isValidFilter, isValidISODateString, @@ -497,9 +496,8 @@ class InternalBuilder { filterKey: string, whereCb: (filterKey: string, query: Knex.QueryBuilder) => Knex.QueryBuilder ): Knex.QueryBuilder { - const { relationships, endpoint, tableAliases: aliases } = this.query - const tableName = endpoint.entityId - const fromAlias = aliases?.[tableName] || tableName + const { relationships, endpoint, tableAliases: aliases, table } = this.query + const fromAlias = aliases?.[table.name] || table.name const matches = (value: string) => filterKey.match(new RegExp(`^${value}\\.`)) if (!relationships) { @@ -1455,14 +1453,14 @@ class InternalBuilder { } qualifiedKnex(opts?: { alias?: string | boolean }): Knex.QueryBuilder { - let alias = this.query.tableAliases?.[this.query.endpoint.entityId] + let alias = this.query.tableAliases?.[this.query.table.name] if (opts?.alias === false) { alias = undefined } else if (typeof opts?.alias === "string") { alias = opts.alias } return this.knex( - this.tableNameWithSchema(this.query.endpoint.entityId, { + this.tableNameWithSchema(this.query.table.name, { alias, schema: this.query.endpoint.schema, }) @@ -1558,11 +1556,10 @@ class InternalBuilder { limits?: { base: number; query: number } } = {} ): Knex.QueryBuilder { - let { endpoint, filters, paginate, relationships } = this.query + let { endpoint, filters, paginate, relationships, table } = this.query const { limits } = opts const counting = endpoint.operation === Operation.COUNT - const tableName = endpoint.entityId // start building the query let query = this.qualifiedKnex() // handle pagination @@ -1610,9 +1607,7 @@ class InternalBuilder { // handle relationships with a CTE for all others if (relationships?.length && aggregations.length === 0) { - const mainTable = - this.query.tableAliases?.[this.query.endpoint.entityId] || - this.query.endpoint.entityId + const mainTable = this.query.tableAliases?.[table.name] || table.name const cte = this.addSorting( this.knex .with("paginated", query) @@ -1622,7 +1617,7 @@ class InternalBuilder { }) ) // add JSON aggregations attached to the CTE - return this.addJsonRelationships(cte, tableName, relationships) + return this.addJsonRelationships(cte, table.name, relationships) } return query diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index ee5dfbf453..5b76e155f9 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -25,7 +25,7 @@ function generateSchema( schema: CreateTableBuilder, table: Table, tables: Record, - oldTable: null | Table = null, + oldTable?: Table, renamed?: RenameColumn ) { let primaryKeys = table && table.primary ? table.primary : [] @@ -55,7 +55,7 @@ function generateSchema( ) for (let [key, column] of Object.entries(table.schema)) { // skip things that are already correct - const oldColumn = oldTable ? oldTable.schema[key] : null + const oldColumn = oldTable?.schema[key] if ( (oldColumn && oldColumn.type) || columnTypeSet.includes(key) || @@ -199,7 +199,7 @@ function buildUpdateTable( knex: SchemaBuilder, table: Table, tables: Record, - oldTable: Table, + oldTable?: Table, renamed?: RenameColumn ): SchemaBuilder { return knex.alterTable(table.name, schema => { @@ -281,7 +281,7 @@ class SqlTableQueryBuilder { client, json.table, json.tables, - json.table, + json.meta?.oldTable, json.meta?.renamed ) diff --git a/packages/pro b/packages/pro index 06e4506cef..8c5125b4f6 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 06e4506cef37110e509a0ef0b9811ac07f8b2844 +Subproject commit 8c5125b4f6e53d0a43d4de90aecbf53d9f7da8ff diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 422896cd52..5f01f323e6 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -137,7 +137,7 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig { function getEndpoint(tableId: string, operation: Operation) { const { datasourceId, tableName } = breakExternalTableId(tableId) - return { datasource: datasourceId, entityId: tableName, operation } + return { datasourceId, entityId: tableName, operation } } function isOneSide( @@ -706,8 +706,8 @@ export class ExternalRequest { let json: QueryJson = { endpoint: { - datasource: this.datasource, - entityId: table.name, + datasourceId: this.datasource, + entityId: table, operation, }, resource: { diff --git a/packages/server/src/api/controllers/table/ExternalRequest.ts b/packages/server/src/api/controllers/table/ExternalRequest.ts index ca4fcb1d16..df5707b072 100644 --- a/packages/server/src/api/controllers/table/ExternalRequest.ts +++ b/packages/server/src/api/controllers/table/ExternalRequest.ts @@ -11,19 +11,23 @@ export async function makeTableRequest( datasource: Datasource, operation: Operation, table: Table, + oldTable?: Table, renamed?: RenameColumn ) { const json: QueryJson = { endpoint: { - datasource, - entityId: table._id!, + datasourceId: datasource, + entityId: table, operation, }, } + if (!json.meta) { + json.meta = {} + } + if (oldTable) { + json.meta.oldTable = oldTable + } if (renamed) { - if (!json.meta) { - json.meta = {} - } json.meta.renamed = renamed } return makeExternalQuery(json) diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 6f09bf4a61..5e77dc1dc6 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -54,7 +54,7 @@ export async function updateTable( return table } catch (err: any) { if (err instanceof Error) { - ctx.throw(400, err.message) + throw err } else { ctx.throw(err.status || 500, err?.message || err) } diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 9130034501..44b21e0350 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -837,7 +837,7 @@ if (descriptions.length) { const res = await config.api.datasource.query({ endpoint: { - datasource: datasource._id!, + datasourceId: datasource._id!, operation: Operation.READ, entityId, }, diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index d8bf0e376d..6492a651e1 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -382,7 +382,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus { } async query(json: EnrichedQueryJson): Promise { - const sheet = json.endpoint.entityId + const sheet = json.table.name switch (json.endpoint.operation) { case Operation.CREATE: return this.create({ sheet, row: json.body as Row }) diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 4343593d1e..d690a0e17e 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -580,7 +580,7 @@ class OracleIntegration extends Sql implements DatasourcePlus { operation !== Operation.DELETE ) { const lastRow = await this.internalQuery({ - sql: `SELECT * FROM "${json.endpoint.entityId}" WHERE ROWID = '${response.lastRowid}'`, + sql: `SELECT * FROM "${json.table.name}" WHERE ROWID = '${response.lastRowid}'`, }) return lastRow.rows as Row[] } else { diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index c06ba2d35c..4de84f689e 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -231,8 +231,7 @@ describe("Captures of real examples", () => { }, queryJson) expect(returningQuery).toEqual({ sql: multiline( - `select top (@p0) * from [people] as [a] where CASE WHEN [a].[name] = @p1 THEN 1 ELSE 0 END = 1 and - CASE WHEN [a].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [a].[name] asc` + `select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc` ), bindings: [1, "Test", 22], }) @@ -270,7 +269,7 @@ describe("Captures of real examples", () => { fields: string[] = ["a"] ): EnrichedQueryJson { return { - endpoint: { datasource: "", entityId: "", operation: op }, + endpoint: { datasourceId: "", entityId: "", operation: op }, resource: { fields, }, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index d6ac2a5862..ba5614edca 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -358,7 +358,7 @@ export async function search( const request: QueryJson = { endpoint: { // not important, we query ourselves - datasource: SQS_DATASOURCE_INTERNAL, + datasourceId: SQS_DATASOURCE_INTERNAL, entityId: table._id!, operation: Operation.READ, }, diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index 2f9bea3c91..62e0db9aad 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -222,7 +222,7 @@ export default class AliasTables { aliases: this.aliasMap([ relationship.through, relationship.tableName, - json.endpoint.entityId, + json.table.name, ]), })) } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 17bbebb251..977cdb9bd2 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -23,8 +23,6 @@ import { isSQL } from "../../../integrations/utils" import { docIds, sql, SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getTableFromSource } from "../../../api/controllers/row/utils" import env from "../../../environment" -import { breakExternalTableId } from "@budibase/backend-core/src/sql/utils" -import { isDatasourceId } from "@budibase/backend-core/src/docIds" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, @@ -90,18 +88,15 @@ export async function enrichQueryJson( json: QueryJson ): Promise { let datasource: Datasource | undefined = undefined - let entityId = json.endpoint.entityId - if (typeof json.endpoint.datasource === "string") { - if (json.endpoint.datasource !== SQS_DATASOURCE_INTERNAL) { - datasource = await sdk.datasources.get(json.endpoint.datasource, { + + if (typeof json.endpoint.datasourceId === "string") { + if (json.endpoint.datasourceId !== SQS_DATASOURCE_INTERNAL) { + datasource = await sdk.datasources.get(json.endpoint.datasourceId, { enriched: true, }) } } else { - datasource = json.endpoint.datasource - if (isDatasourceId(entityId)) { - entityId = breakExternalTableId(entityId).tableName - } + datasource = json.endpoint.datasourceId } let tables: Record @@ -111,7 +106,16 @@ export async function enrichQueryJson( tables = processInternalTables(await sdk.tables.getAllInternalTables()) } - const table = tables[entityId] + let table: Table + if (typeof json.endpoint.entityId === "string") { + let entityId = json.endpoint.entityId + if (docIds.isDatasourceId(entityId)) { + entityId = sql.utils.breakExternalTableId(entityId).tableName + } + table = tables[entityId] + } else { + table = json.endpoint.entityId + } return { table, diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 5d8f03a7a6..3511222678 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -241,7 +241,13 @@ export async function save( } const operation = tableId ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE - await makeTableRequest(datasource, operation, tableToSave, opts?.renaming) + await makeTableRequest( + datasource, + operation, + tableToSave, + oldTable, + opts?.renaming + ) // update any extra tables (like foreign keys in other tables) for (let extraTable of extraTablesToUpdate) { const oldExtraTable = oldTables[extraTable.name] diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 72f7de455a..e25390bdd0 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -158,8 +158,8 @@ export interface ManyToManyRelationshipJson { export interface QueryJson { endpoint: { - datasource: string | Datasource - entityId: string + datasourceId: string | Datasource + entityId: string | Table operation: Operation schema?: string } @@ -173,6 +173,7 @@ export interface QueryJson { body?: Row | Row[] meta?: { renamed?: RenameColumn + oldTable?: Table // can specify something that columns could be prefixed with columnPrefix?: string } @@ -190,7 +191,7 @@ export interface EnrichedQueryJson extends QueryJson { } export interface QueryJsonRequest extends Omit { - endpoint: QueryJson["endpoint"] & { datasource: string } + endpoint: QueryJson["endpoint"] & { datasourceId: string; entityId: string } } export interface QueryOptions { From f49f9280d9c676956439449884dec0c9dcf5b5e8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 09:42:01 +0000 Subject: [PATCH 27/30] Working on search tests. --- packages/server/src/api/routes/tests/search.spec.ts | 2 +- packages/server/src/integrations/oracle.ts | 2 -- packages/server/src/sdk/app/tables/external/index.ts | 6 +++++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5384444067..6a56ec37c6 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -3329,7 +3329,7 @@ if (descriptions.length) { }) isSql && - describe("primaryDisplay", () => { + describe.only("primaryDisplay", () => { beforeAll(async () => { let toRelateTableId = await createTable({ name: { diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index d690a0e17e..ded21487f6 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -486,8 +486,6 @@ class OracleIntegration extends Sql implements DatasourcePlus { this.log(query.sql, bindings) const result = await connection.execute(query.sql, bindings, options) return result as Result - } catch (err) { - throw err } finally { if (connection) { try { diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 3511222678..a8fad40a8b 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -252,7 +252,11 @@ export async function save( for (let extraTable of extraTablesToUpdate) { const oldExtraTable = oldTables[extraTable.name] let op = oldExtraTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE - await makeTableRequest(datasource, op, extraTable) + try { + await makeTableRequest(datasource, op, extraTable) + } catch (err: any) { + throw err + } } // make sure the constrained list, all still exist From b700f16a3a6f1ed00b412ea2ff88a96598332f24 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 10:12:16 +0000 Subject: [PATCH 28/30] Fix search.spec.ts. --- packages/server/src/api/routes/tests/search.spec.ts | 2 +- packages/server/src/sdk/app/tables/external/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 6a56ec37c6..5384444067 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -3329,7 +3329,7 @@ if (descriptions.length) { }) isSql && - describe.only("primaryDisplay", () => { + describe("primaryDisplay", () => { beforeAll(async () => { let toRelateTableId = await createTable({ name: { diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index a8fad40a8b..df3f287842 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -253,7 +253,7 @@ export async function save( const oldExtraTable = oldTables[extraTable.name] let op = oldExtraTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE try { - await makeTableRequest(datasource, op, extraTable) + await makeTableRequest(datasource, op, extraTable, oldExtraTable) } catch (err: any) { throw err } From 8e2b916343ca2340f26a5dca8519fcd4f28cbf2c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 14:31:22 +0000 Subject: [PATCH 29/30] Update yarn.lock --- yarn.lock | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/yarn.lock b/yarn.lock index 85d8b160f5..edf81d1f1e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4075,7 +4075,7 @@ resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.4.tgz#af85eb080f6934580e4d3b58046026b6c2b18717" integrity sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng== -"@rollup/plugin-commonjs@16.0.0", "@rollup/plugin-commonjs@^16.0.0": +"@rollup/plugin-commonjs@^16.0.0": version "16.0.0" resolved "https://registry.yarnpkg.com/@rollup/plugin-commonjs/-/plugin-commonjs-16.0.0.tgz#169004d56cd0f0a1d0f35915d31a036b0efe281f" integrity sha512-LuNyypCP3msCGVQJ7ki8PqYdpjfEkE/xtFa5DqlF+7IBD0JsfMZ87C58heSwIMint58sAUZbt3ITqOmdQv/dXw== @@ -4153,7 +4153,7 @@ is-module "^1.0.0" resolve "^1.22.1" -"@rollup/plugin-replace@^5.0.2", "@rollup/plugin-replace@^5.0.3": +"@rollup/plugin-replace@^5.0.3": version "5.0.7" resolved "https://registry.yarnpkg.com/@rollup/plugin-replace/-/plugin-replace-5.0.7.tgz#150c9ee9db8031d9e4580a61a0edeaaed3d37687" integrity sha512-PqxSfuorkHz/SPpyngLyg5GCEkOcee9M1bkxiVDr41Pd61mqP1PLOoDPbpl44SB2mQGKwV/In74gqQmGITOhEQ== @@ -12615,11 +12615,6 @@ husky@^8.0.3: resolved "https://registry.yarnpkg.com/husky/-/husky-8.0.3.tgz#4936d7212e46d1dea28fef29bb3a108872cd9184" integrity sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg== -husky@^9.1.4: - version "9.1.7" - resolved "https://registry.yarnpkg.com/husky/-/husky-9.1.7.tgz#d46a38035d101b46a70456a850ff4201344c0b2d" - integrity sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA== - ical-generator@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/ical-generator/-/ical-generator-4.1.0.tgz#2a336c951864c5583a2aa715d16f2edcdfd2d90b" @@ -20231,7 +20226,16 @@ string-range@~1.2, string-range@~1.2.1: resolved "https://registry.yarnpkg.com/string-range/-/string-range-1.2.2.tgz#a893ed347e72299bc83befbbf2a692a8d239d5dd" integrity sha512-tYft6IFi8SjplJpxCUxyqisD3b+R2CSkomrtJYCkvuf1KuCAWgz7YXt4O0jip7efpfCemwHEzTEAO8EuOYgh3w== -"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -20323,7 +20327,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -20337,6 +20341,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -22158,7 +22169,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -22176,6 +22187,15 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From 22d6b951013304cb003dbdc2684c902645f604e9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 15:39:16 +0000 Subject: [PATCH 30/30] Put errors back how they were. --- packages/server/src/api/controllers/table/external.ts | 2 +- packages/server/src/sdk/app/rows/search/internal/sqs.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 5e77dc1dc6..6f09bf4a61 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -54,7 +54,7 @@ export async function updateTable( return table } catch (err: any) { if (err instanceof Error) { - throw err + ctx.throw(400, err.message) } else { ctx.throw(err.status || 500, err?.message || err) } diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index ba5614edca..9bfa0ecd27 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -485,7 +485,6 @@ export async function search( if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) { return { rows: [] } } - // throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) - throw err + throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) } }