Merge branch 'bigint-relationship-fix' of github.com:Budibase/budibase into bigint-relationship-fix

packages/backend-core/src/sql/sql.ts

@@ -1253,6 +1253,9 @@ class InternalBuilder {
         continue
       }
       const relatedTable = tables[toTable]
+      if (!relatedTable) {
+        throw new Error(`related table "${toTable}" not found in datasource`)
+      }
       const toAlias = aliases?.[toTable] || toTable,
         fromAlias = aliases?.[fromTable] || fromTable,
         throughAlias = (throughTable && aliases?.[throughTable]) || throughTable

packages/server/src/api/controllers/datasource.ts

@@ -23,11 +23,13 @@ import {
   Table,
   RowValue,
   DynamicVariable,
+  QueryJson,
 } from "@budibase/types"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
 import { isEqual } from "lodash"
 import { processTable } from "../../sdk/app/tables/getters"
+import { makeExternalQuery } from "../../integrations/base/query"
 
 export async function fetch(ctx: UserCtx) {
   ctx.body = await sdk.datasources.fetch()
@@ -297,10 +299,10 @@ export async function find(ctx: UserCtx) {
 }
 
 // dynamic query functionality
-export async function query(ctx: UserCtx) {
+export async function query(ctx: UserCtx<QueryJson>) {
   const queryJson = ctx.request.body
   try {
-    ctx.body = await sdk.rows.utils.getDatasourceAndQuery(queryJson)
+    ctx.body = await makeExternalQuery(queryJson)
   } catch (err: any) {
     ctx.throw(400, err)
   }

packages/server/src/api/controllers/table/ExternalRequest.ts

@@ -11,8 +11,6 @@ export async function makeTableRequest(
   datasource: Datasource,
   operation: Operation,
   table: Table,
-  tables: Record<string, Table>,
-  oldTable?: Table,
   renamed?: RenameColumn
 ) {
   const json: QueryJson = {
@@ -21,17 +19,12 @@ export async function makeTableRequest(
       entityId: table._id!,
       operation,
     },
-    meta: {
-      table,
-      tables,
-    },
-    table,
-  }
-  if (oldTable) {
-    json.meta!.table = oldTable
   }
   if (renamed) {
-    json.meta!.renamed = renamed
+    if (!json.meta) {
+      json.meta = {}
     }
-  return makeExternalQuery(datasource, json)
+    json.meta.renamed = renamed
+  }
+  return makeExternalQuery(json)
 }

packages/server/src/integrations/googlesheets.ts

@@ -7,7 +7,6 @@ import {
   Integration,
   Operation,
   PaginationJson,
-  QueryJson,
   QueryType,
   Row,
   Schema,
@@ -18,6 +17,7 @@ import {
   TableSourceType,
   DatasourcePlusQueryResponse,
   BBReferenceFieldSubType,
+  EnrichedQueryJson,
 } from "@budibase/types"
 import { OAuth2Client } from "google-auth-library"
 import {
@@ -381,7 +381,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
     return { tables: externalTables, errors }
   }
 
-  async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
+  async query(json: EnrichedQueryJson): Promise<DatasourcePlusQueryResponse> {
     const sheet = json.endpoint.entityId
     switch (json.endpoint.operation) {
       case Operation.CREATE:
@@ -400,7 +400,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
           rowIndex: json.extra?.idFilter?.equal?.rowNumber,
           sheet,
           row: json.body,
-          table: json.meta.table,
+          table: json.table,
         })
       case Operation.DELETE:
         return this.delete({

packages/server/src/integrations/tests/sql.spec.ts

@@ -1,268 +0,0 @@
-import {
-  FieldType,
-  Operation,
-  PaginationJson,
-  QueryJson,
-  SearchFilters,
-  SortJson,
-  SqlClient,
-  Table,
-  TableSourceType,
-} from "@budibase/types"
-import { sql } from "@budibase/backend-core"
-import { merge } from "lodash"
-
-const Sql = sql.Sql
-
-const TABLE_NAME = "test"
-const TABLE: Table = {
-  type: "table",
-  sourceType: TableSourceType.EXTERNAL,
-  sourceId: "SOURCE_ID",
-  schema: {
-    id: {
-      name: "id",
-      type: FieldType.NUMBER,
-    },
-  },
-  name: TABLE_NAME,
-  primary: ["id"],
-}
-
-const ORACLE_TABLE: Partial<Table> = {
-  schema: {
-    name: {
-      name: "name",
-      type: FieldType.STRING,
-    },
-  },
-}
-
-function endpoint(table: string, operation: Operation) {
-  return {
-    datasourceId: "Postgres",
-    operation: operation,
-    entityId: table || TABLE_NAME,
-  }
-}
-
-function generateReadJson({
-  table,
-  fields,
-  filters,
-  sort,
-  paginate,
-}: {
-  table?: Partial<Table>
-  fields?: string[]
-  filters?: SearchFilters
-  sort?: SortJson
-  paginate?: PaginationJson
-} = {}): QueryJson {
-  let tableObj: Table = { ...TABLE }
-  if (table) {
-    tableObj = merge(TABLE, table)
-  }
-  return {
-    endpoint: endpoint(tableObj.name || TABLE_NAME, Operation.READ),
-    resource: {
-      fields: fields || [],
-    },
-    filters: filters || {},
-    sort: sort || {},
-    paginate: paginate || undefined,
-    meta: {
-      table: tableObj,
-    },
-  }
-}
-
-function generateRelationshipJson(config: { schema?: string } = {}): QueryJson {
-  return {
-    endpoint: {
-      datasourceId: "Postgres",
-      entityId: "brands",
-      operation: Operation.READ,
-      schema: config.schema,
-    },
-    resource: {
-      fields: [
-        "brands.brand_id",
-        "brands.brand_name",
-        "products.product_id",
-        "products.product_name",
-        "products.brand_id",
-      ],
-    },
-    filters: {},
-    sort: {},
-    relationships: [
-      {
-        from: "brand_id",
-        to: "brand_id",
-        tableName: "products",
-        column: "products",
-      },
-    ],
-    extra: { idFilter: {} },
-    meta: {
-      table: TABLE,
-    },
-  }
-}
-
-function generateManyRelationshipJson(config: { schema?: string } = {}) {
-  return {
-    endpoint: {
-      datasourceId: "Postgres",
-      entityId: "stores",
-      operation: "READ",
-      schema: config.schema,
-    },
-    resource: {
-      fields: [
-        "stores.store_id",
-        "stores.store_name",
-        "products.product_id",
-        "products.product_name",
-      ],
-    },
-    filters: {},
-    sort: {},
-    paginate: {},
-    relationships: [
-      {
-        from: "store_id",
-        to: "product_id",
-        tableName: "products",
-        column: "products",
-        through: "stocks",
-        fromPrimary: "store_id",
-        toPrimary: "product_id",
-      },
-    ],
-    extra: { idFilter: {} },
-    meta: {
-      table: TABLE,
-    },
-  }
-}
-
-describe("SQL query builder", () => {
-  const relationshipLimit = 500
-  const limit = 500
-  const client = SqlClient.POSTGRES
-  let sql: any
-
-  beforeEach(() => {
-    sql = new Sql(client, limit)
-  })
-
-  it("should add the schema to the LEFT JOIN", () => {
-    const query = sql._query(generateRelationshipJson({ schema: "production" }))
-    expect(query).toEqual({
-      bindings: [limit, relationshipLimit],
-      sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`,
-    })
-  })
-
-  it("should handle if the schema is not present when doing a LEFT JOIN", () => {
-    const query = sql._query(generateRelationshipJson())
-    expect(query).toEqual({
-      bindings: [limit, relationshipLimit],
-      sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`,
-    })
-  })
-
-  it("should add the schema to both the toTable and throughTable in many-to-many join", () => {
-    const query = sql._query(
-      generateManyRelationshipJson({ schema: "production" })
-    )
-    expect(query).toEqual({
-      bindings: [limit, relationshipLimit],
-      sql: `with "paginated" as (select "stores".* from "production"."stores" order by "test"."id" asc limit $1) select "stores".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" inner join "production"."stocks" as "stocks" on "products"."product_id" = "stocks"."product_id" where "stocks"."store_id" = "stores"."store_id" order by "products"."product_id" asc limit $2) as "products") as "products" from "paginated" as "stores" order by "test"."id" asc`,
-    })
-  })
-
-  it("should lowercase the values for Oracle LIKE statements", () => {
-    let query = new Sql(SqlClient.ORACLE, limit)._query(
-      generateReadJson({
-        filters: {
-          string: {
-            name: "John",
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: ["john%", limit],
-      sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`,
-    })
-
-    query = new Sql(SqlClient.ORACLE, limit)._query(
-      generateReadJson({
-        filters: {
-          contains: {
-            age: [20, 25],
-            name: ["John", "Mary"],
-          },
-        },
-      })
-    )
-    const filterSet = [`%20%`, `%25%`, `%"john"%`, `%"mary"%`]
-    expect(query).toEqual({
-      bindings: [...filterSet, limit],
-      sql: `select * from (select * from "test" where ((COALESCE(LOWER("test"."age"), '') like :1 and COALESCE(LOWER("test"."age"), '') like :2)) and ((COALESCE(LOWER("test"."name"), '') like :3 and COALESCE(LOWER("test"."name"), '') like :4)) order by "test"."id" asc) where rownum <= :5`,
-    })
-
-    query = new Sql(SqlClient.ORACLE, limit)._query(
-      generateReadJson({
-        filters: {
-          fuzzy: {
-            name: "Jo",
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [`%jo%`, limit],
-      sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`,
-    })
-  })
-
-  it("should use an oracle compatible coalesce query for oracle when using the equals filter", () => {
-    let query = new Sql(SqlClient.ORACLE, limit)._query(
-      generateReadJson({
-        table: ORACLE_TABLE,
-        filters: {
-          equal: {
-            name: "John",
-          },
-        },
-      })
-    )
-
-    expect(query).toEqual({
-      bindings: ["John", limit],
-      sql: `select * from (select * from "test" where (to_char("test"."name") is not null and to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2`,
-    })
-  })
-
-  it("should use an oracle compatible coalesce query for oracle when using the not equals filter", () => {
-    let query = new Sql(SqlClient.ORACLE, limit)._query(
-      generateReadJson({
-        table: ORACLE_TABLE,
-        filters: {
-          notEqual: {
-            name: "John",
-          },
-        },
-      })
-    )
-
-    expect(query).toEqual({
-      bindings: ["John", limit],
-      sql: `select * from (select * from "test" where (to_char("test"."name") is not null and to_char("test"."name") != :1) or to_char("test"."name") is null order by "test"."id" asc) where rownum <= :2`,
-    })
-  })
-})

packages/server/src/sdk/app/rows/sqlAlias.ts

@@ -17,7 +17,7 @@ import { BudibaseInternalDB } from "../../../db/utils"
 import { dataFilters } from "@budibase/shared-core"
 
 type PerformQueryFunction = (
-  json: QueryJson
+  json: EnrichedQueryJson
 ) => Promise<DatasourcePlusQueryResponse>
 
 const WRITE_OPERATIONS: Operation[] = [

packages/server/src/sdk/app/tables/external/index.ts

@@ -241,19 +241,12 @@ export async function save(
   }
 
   const operation = tableId ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE
-  await makeTableRequest(
+  await makeTableRequest(datasource, operation, tableToSave, opts?.renaming)
-    datasource,
-    operation,
-    tableToSave,
-    tables,
-    oldTable,
-    opts?.renaming
-  )
   // update any extra tables (like foreign keys in other tables)
   for (let extraTable of extraTablesToUpdate) {
     const oldExtraTable = oldTables[extraTable.name]
     let op = oldExtraTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE
-    await makeTableRequest(datasource, op, extraTable, tables, oldExtraTable)
+    await makeTableRequest(datasource, op, extraTable)
   }
 
   // make sure the constrained list, all still exist
@@ -292,7 +285,7 @@ export async function destroy(datasourceId: string, table: Table) {
 
   const operation = Operation.DELETE_TABLE
   if (tables) {
-    await makeTableRequest(datasource, operation, table, tables)
+    await makeTableRequest(datasource, operation, table)
     cleanupRelationships(table, tables, { deleting: true })
     delete tables[table.name]
     datasource.entities = tables

yarn.lock

@@ -12635,6 +12635,11 @@ husky@^8.0.3:
   resolved "https://registry.yarnpkg.com/husky/-/husky-8.0.3.tgz#4936d7212e46d1dea28fef29bb3a108872cd9184"
   integrity sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg==
 
+husky@^9.1.4:
+  version "9.1.7"
+  resolved "https://registry.yarnpkg.com/husky/-/husky-9.1.7.tgz#d46a38035d101b46a70456a850ff4201344c0b2d"
+  integrity sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==
+
 ical-generator@4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/ical-generator/-/ical-generator-4.1.0.tgz#2a336c951864c5583a2aa715d16f2edcdfd2d90b"