diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index c897c106b0..5333bfb591 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1253,6 +1253,9 @@ class InternalBuilder { continue } const relatedTable = tables[toTable] + if (!relatedTable) { + throw new Error(`related table "${toTable}" not found in datasource`) + } const toAlias = aliases?.[toTable] || toTable, fromAlias = aliases?.[fromTable] || fromTable, throughAlias = (throughTable && aliases?.[throughTable]) || throughTable diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 794e2dfddd..9f4fff4c46 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -23,11 +23,13 @@ import { Table, RowValue, DynamicVariable, + QueryJson, } from "@budibase/types" import sdk from "../../sdk" import { builderSocket } from "../../websockets" import { isEqual } from "lodash" import { processTable } from "../../sdk/app/tables/getters" +import { makeExternalQuery } from "src/integrations/base/query" export async function fetch(ctx: UserCtx) { ctx.body = await sdk.datasources.fetch() @@ -297,10 +299,10 @@ export async function find(ctx: UserCtx) { } // dynamic query functionality -export async function query(ctx: UserCtx) { +export async function query(ctx: UserCtx) { const queryJson = ctx.request.body try { - ctx.body = await sdk.rows.utils.getDatasourceAndQuery(queryJson) + ctx.body = await makeExternalQuery(queryJson) } catch (err: any) { ctx.throw(400, err) } diff --git a/packages/server/src/api/controllers/table/ExternalRequest.ts b/packages/server/src/api/controllers/table/ExternalRequest.ts index 1e57ea3294..dc32c79c5a 100644 --- a/packages/server/src/api/controllers/table/ExternalRequest.ts +++ b/packages/server/src/api/controllers/table/ExternalRequest.ts @@ -11,8 +11,6 @@ export async function makeTableRequest( datasource: Datasource, operation: Operation, table: Table, - tables: Record, - oldTable?: Table, renamed?: RenameColumn ) { const json: QueryJson = { @@ -21,17 +19,12 @@ export async function makeTableRequest( entityId: table._id!, operation, }, - meta: { - table, - tables, - }, - table, - } - if (oldTable) { - json.meta!.table = oldTable } if (renamed) { - json.meta!.renamed = renamed + if (!json.meta) { + json.meta = {} + } + json.meta.renamed = renamed } - return makeExternalQuery(datasource, json) + return makeExternalQuery(json) } diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 5f61791683..d8bf0e376d 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -7,7 +7,6 @@ import { Integration, Operation, PaginationJson, - QueryJson, QueryType, Row, Schema, @@ -18,6 +17,7 @@ import { TableSourceType, DatasourcePlusQueryResponse, BBReferenceFieldSubType, + EnrichedQueryJson, } from "@budibase/types" import { OAuth2Client } from "google-auth-library" import { @@ -381,7 +381,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus { return { tables: externalTables, errors } } - async query(json: QueryJson): Promise { + async query(json: EnrichedQueryJson): Promise { const sheet = json.endpoint.entityId switch (json.endpoint.operation) { case Operation.CREATE: @@ -400,7 +400,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus { rowIndex: json.extra?.idFilter?.equal?.rowNumber, sheet, row: json.body, - table: json.meta.table, + table: json.table, }) case Operation.DELETE: return this.delete({ diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts deleted file mode 100644 index 2b3649161c..0000000000 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ /dev/null @@ -1,268 +0,0 @@ -import { - FieldType, - Operation, - PaginationJson, - QueryJson, - SearchFilters, - SortJson, - SqlClient, - Table, - TableSourceType, -} from "@budibase/types" -import { sql } from "@budibase/backend-core" -import { merge } from "lodash" - -const Sql = sql.Sql - -const TABLE_NAME = "test" -const TABLE: Table = { - type: "table", - sourceType: TableSourceType.EXTERNAL, - sourceId: "SOURCE_ID", - schema: { - id: { - name: "id", - type: FieldType.NUMBER, - }, - }, - name: TABLE_NAME, - primary: ["id"], -} - -const ORACLE_TABLE: Partial = { - schema: { - name: { - name: "name", - type: FieldType.STRING, - }, - }, -} - -function endpoint(table: string, operation: Operation) { - return { - datasourceId: "Postgres", - operation: operation, - entityId: table || TABLE_NAME, - } -} - -function generateReadJson({ - table, - fields, - filters, - sort, - paginate, -}: { - table?: Partial
- fields?: string[] - filters?: SearchFilters - sort?: SortJson - paginate?: PaginationJson -} = {}): QueryJson { - let tableObj: Table = { ...TABLE } - if (table) { - tableObj = merge(TABLE, table) - } - return { - endpoint: endpoint(tableObj.name || TABLE_NAME, Operation.READ), - resource: { - fields: fields || [], - }, - filters: filters || {}, - sort: sort || {}, - paginate: paginate || undefined, - meta: { - table: tableObj, - }, - } -} - -function generateRelationshipJson(config: { schema?: string } = {}): QueryJson { - return { - endpoint: { - datasourceId: "Postgres", - entityId: "brands", - operation: Operation.READ, - schema: config.schema, - }, - resource: { - fields: [ - "brands.brand_id", - "brands.brand_name", - "products.product_id", - "products.product_name", - "products.brand_id", - ], - }, - filters: {}, - sort: {}, - relationships: [ - { - from: "brand_id", - to: "brand_id", - tableName: "products", - column: "products", - }, - ], - extra: { idFilter: {} }, - meta: { - table: TABLE, - }, - } -} - -function generateManyRelationshipJson(config: { schema?: string } = {}) { - return { - endpoint: { - datasourceId: "Postgres", - entityId: "stores", - operation: "READ", - schema: config.schema, - }, - resource: { - fields: [ - "stores.store_id", - "stores.store_name", - "products.product_id", - "products.product_name", - ], - }, - filters: {}, - sort: {}, - paginate: {}, - relationships: [ - { - from: "store_id", - to: "product_id", - tableName: "products", - column: "products", - through: "stocks", - fromPrimary: "store_id", - toPrimary: "product_id", - }, - ], - extra: { idFilter: {} }, - meta: { - table: TABLE, - }, - } -} - -describe("SQL query builder", () => { - const relationshipLimit = 500 - const limit = 500 - const client = SqlClient.POSTGRES - let sql: any - - beforeEach(() => { - sql = new Sql(client, limit) - }) - - it("should add the schema to the LEFT JOIN", () => { - const query = sql._query(generateRelationshipJson({ schema: "production" })) - expect(query).toEqual({ - bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, - }) - }) - - it("should handle if the schema is not present when doing a LEFT JOIN", () => { - const query = sql._query(generateRelationshipJson()) - expect(query).toEqual({ - bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, - }) - }) - - it("should add the schema to both the toTable and throughTable in many-to-many join", () => { - const query = sql._query( - generateManyRelationshipJson({ schema: "production" }) - ) - expect(query).toEqual({ - bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "stores".* from "production"."stores" order by "test"."id" asc limit $1) select "stores".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" inner join "production"."stocks" as "stocks" on "products"."product_id" = "stocks"."product_id" where "stocks"."store_id" = "stores"."store_id" order by "products"."product_id" asc limit $2) as "products") as "products" from "paginated" as "stores" order by "test"."id" asc`, - }) - }) - - it("should lowercase the values for Oracle LIKE statements", () => { - let query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - filters: { - string: { - name: "John", - }, - }, - }) - ) - expect(query).toEqual({ - bindings: ["john%", limit], - sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`, - }) - - query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - filters: { - contains: { - age: [20, 25], - name: ["John", "Mary"], - }, - }, - }) - ) - const filterSet = [`%20%`, `%25%`, `%"john"%`, `%"mary"%`] - expect(query).toEqual({ - bindings: [...filterSet, limit], - sql: `select * from (select * from "test" where ((COALESCE(LOWER("test"."age"), '') like :1 and COALESCE(LOWER("test"."age"), '') like :2)) and ((COALESCE(LOWER("test"."name"), '') like :3 and COALESCE(LOWER("test"."name"), '') like :4)) order by "test"."id" asc) where rownum <= :5`, - }) - - query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - filters: { - fuzzy: { - name: "Jo", - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [`%jo%`, limit], - sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`, - }) - }) - - it("should use an oracle compatible coalesce query for oracle when using the equals filter", () => { - let query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - table: ORACLE_TABLE, - filters: { - equal: { - name: "John", - }, - }, - }) - ) - - expect(query).toEqual({ - bindings: ["John", limit], - sql: `select * from (select * from "test" where (to_char("test"."name") is not null and to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2`, - }) - }) - - it("should use an oracle compatible coalesce query for oracle when using the not equals filter", () => { - let query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - table: ORACLE_TABLE, - filters: { - notEqual: { - name: "John", - }, - }, - }) - ) - - expect(query).toEqual({ - bindings: ["John", limit], - sql: `select * from (select * from "test" where (to_char("test"."name") is not null and to_char("test"."name") != :1) or to_char("test"."name") is null order by "test"."id" asc) where rownum <= :2`, - }) - }) -}) diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index c748940add..c14307360b 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -17,7 +17,7 @@ import { BudibaseInternalDB } from "../../../db/utils" import { dataFilters } from "@budibase/shared-core" type PerformQueryFunction = ( - json: QueryJson + json: EnrichedQueryJson ) => Promise const WRITE_OPERATIONS: Operation[] = [