Merge master.

packages/backend-core/src/sql/sqlTable.ts

@@ -28,16 +28,25 @@ function generateSchema(
   oldTable: null | Table = null,
   renamed?: RenameColumn
 ) {
-  let primaryKey = table && table.primary ? table.primary[0] : null
+  let primaryKeys = table && table.primary ? table.primary : []
   const columns = Object.values(table.schema)
   // all columns in a junction table will be meta
   let metaCols = columns.filter(col => (col as NumberFieldMetadata).meta)
   let isJunction = metaCols.length === columns.length
+  let columnTypeSet: string[] = []
+
   // can't change primary once its set for now
-  if (primaryKey && !oldTable && !isJunction) {
+  if (!oldTable) {
-    schema.increments(primaryKey).primary()
+    // junction tables are special - we have an expected format
-  } else if (!oldTable && isJunction) {
+    if (isJunction) {
-    schema.primary(metaCols.map(col => col.name))
+      schema.primary(metaCols.map(col => col.name))
+    } else if (primaryKeys.length === 1) {
+      schema.increments(primaryKeys[0]).primary()
+      // note that we've set its type
+      columnTypeSet.push(primaryKeys[0])
+    } else {
+      schema.primary(primaryKeys)
+    }
   }
 
   // check if any columns need added
@@ -49,7 +58,7 @@ function generateSchema(
     const oldColumn = oldTable ? oldTable.schema[key] : null
     if (
       (oldColumn && oldColumn.type) ||
-      (primaryKey === key && !isJunction) ||
+      columnTypeSet.includes(key) ||
       renamed?.updated === key
     ) {
       continue
@@ -61,7 +70,12 @@ function generateSchema(
       case FieldType.LONGFORM:
       case FieldType.BARCODEQR:
       case FieldType.BB_REFERENCE_SINGLE:
-        schema.text(key)
+        // primary key strings have to have a length in some DBs
+        if (primaryKeys.includes(key)) {
+          schema.string(key, 255)
+        } else {
+          schema.text(key)
+        }
         break
       case FieldType.NUMBER:
         // if meta is specified then this is a junction table entry

packages/server/package.json

@@ -144,6 +144,7 @@
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",
     "jest": "29.7.0",
+    "jest-extended": "^4.0.2",
     "jest-openapi": "0.14.2",
     "nock": "13.5.4",
     "nodemon": "2.0.15",

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -195,12 +195,13 @@ export class ExternalRequest<T extends Operation> {
     if (filters) {
       // need to map over the filters and make sure the _id field isn't present
       let prefix = 1
-      for (const operator of Object.values(filters)) {
+      for (const [operatorType, operator] of Object.entries(filters)) {
+        const isArrayOp = sdk.rows.utils.isArrayFilter(operatorType)
         for (const field of Object.keys(operator || {})) {
           if (dbCore.removeKeyNumbering(field) === "_id") {
             if (primary) {
               const parts = breakRowIdField(operator[field])
-              if (primary.length > 1) {
+              if (primary.length > 1 && isArrayOp) {
                 operator[InternalSearchFilterOperator.COMPLEX_ID_OPERATOR] = {
                   id: primary,
                   values: parts[0],

packages/server/src/api/controllers/row/utils/basic.ts

@@ -71,8 +71,7 @@ export function basicProcessing({
 }): Row {
   const thisRow: Row = {}
   // filter the row down to what is actually the row (not joined)
-  for (let field of Object.values(table.schema)) {
+  for (let fieldName of Object.keys(table.schema)) {
-    const fieldName = field.name
     let value = extractFieldValue({
       row,
       tableName: table.name,

packages/server/src/api/routes/tests/datasource.spec.ts

@@ -1,5 +1,5 @@
 import * as setup from "./utilities"
-import { checkBuilderEndpoint } from "./utilities/TestFunctions"
+import { checkBuilderEndpoint, allowUndefined } from "./utilities/TestFunctions"
 import { getCachedVariable } from "../../../threads/utils"
 import { context, events } from "@budibase/backend-core"
 import sdk from "../../../sdk"
@@ -380,21 +380,24 @@ describe("/datasources", () => {
             persisted?.entities &&
             Object.entries(persisted.entities).reduce<Record<string, Table>>(
               (acc, [tableName, table]) => {
-                acc[tableName] = {
+                acc[tableName] = expect.objectContaining({
                   ...table,
                   primaryDisplay: expect.not.stringMatching(
                     new RegExp(`^${table.primaryDisplay || ""}$`)
                   ),
                   schema: Object.entries(table.schema).reduce<TableSchema>(
                     (acc, [fieldName, field]) => {
-                      acc[fieldName] = expect.objectContaining({
+                      acc[fieldName] = {
                         ...field,
-                      })
+                        externalType: allowUndefined(expect.any(String)),
+                        constraints: allowUndefined(expect.any(Object)),
+                        autocolumn: allowUndefined(expect.any(Boolean)),
+                      }
                       return acc
                     },
                     {}
                   ),
-                }
+                })
                 return acc
               },
               {}

packages/server/src/api/routes/tests/search.spec.ts

@@ -41,6 +41,7 @@ import { dataFilters } from "@budibase/shared-core"
 import { Knex } from "knex"
 import { structures } from "@budibase/backend-core/tests"
 import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
+import { generateRowIdField } from "../../../integrations/utils"
 
 describe.each([
   ["in-memory", undefined],
@@ -2355,6 +2356,35 @@ describe.each([
       })
     })
 
+  describe("Invalid column definitions", () => {
+    beforeAll(async () => {
+      // need to create an invalid table - means ignoring typescript
+      table = await createTable({
+        // @ts-ignore
+        invalid: {
+          type: FieldType.STRING,
+        },
+        name: {
+          name: "name",
+          type: FieldType.STRING,
+        },
+      })
+      await createRows([
+        { name: "foo", invalid: "id1" },
+        { name: "bar", invalid: "id2" },
+      ])
+    })
+
+    it("can get rows with all table data", async () => {
+      await expectSearch({
+        query: {},
+      }).toContain([
+        { name: "foo", invalid: "id1" },
+        { name: "bar", invalid: "id2" },
+      ])
+    })
+  })
+
   describe.each(["data_name_test", "name_data_test", "name_test_data_"])(
     "special (%s) case",
     column => {
@@ -2621,6 +2651,42 @@ describe.each([
       })
     })
 
+  !isInternal &&
+    describe("search by composite key", () => {
+      beforeAll(async () => {
+        table = await config.api.table.save(
+          tableForDatasource(datasource, {
+            schema: {
+              idColumn1: {
+                name: "idColumn1",
+                type: FieldType.NUMBER,
+              },
+              idColumn2: {
+                name: "idColumn2",
+                type: FieldType.NUMBER,
+              },
+            },
+            primary: ["idColumn1", "idColumn2"],
+          })
+        )
+        await createRows([{ idColumn1: 1, idColumn2: 2 }])
+      })
+
+      it("can filter by the row ID with limit 1", async () => {
+        await expectSearch({
+          query: {
+            equal: { _id: generateRowIdField([1, 2]) },
+          },
+          limit: 1,
+        }).toContain([
+          {
+            idColumn1: 1,
+            idColumn2: 2,
+          },
+        ])
+      })
+    })
+
   isSql &&
     describe("pagination edge case with relationships", () => {
       let mainRows: Row[] = []

packages/server/src/api/routes/tests/utilities/TestFunctions.ts

@@ -184,3 +184,7 @@ export const runInProd = async (func: any) => {
   env._set("NODE_ENV", nodeEnv)
   env._set("JEST_WORKER_ID", workerId)
 }
+
+export function allowUndefined(expectation: jest.Expect) {
+  return expect.toBeOneOf([expectation, undefined, null])
+}

packages/server/src/integrations/utils/utils.ts

@@ -15,6 +15,7 @@ import { helpers, utils } from "@budibase/shared-core"
 import { pipeline } from "stream/promises"
 import tmp from "tmp"
 import fs from "fs"
+import { merge, cloneDeep } from "lodash"
 
 type PrimitiveTypes =
   | FieldType.STRING
@@ -291,10 +292,16 @@ function copyExistingPropsOver(
         const fetchedColumnDefinition: FieldSchema | undefined =
           table.schema[key]
         table.schema[key] = {
-          ...existingTableSchema[key],
+          // merge the properties - anything missing will be filled in, old definition preferred
+          // have to clone due to the way merge works
+          ...merge(
+            cloneDeep(fetchedColumnDefinition),
+            existingTableSchema[key]
+          ),
+          // always take externalType and autocolumn from the fetched definition
           externalType:
             existingTableSchema[key].externalType ||
-            table.schema[key]?.externalType,
+            fetchedColumnDefinition?.externalType,
           autocolumn: fetchedColumnDefinition?.autocolumn,
         } as FieldSchema
         // check constraints which can be fetched from the DB (they could be updated)

packages/server/src/sdk/app/rows/search/internal/sqs.ts

@@ -73,13 +73,14 @@ function buildInternalFieldList(
   fieldList = fieldList.concat(
     PROTECTED_INTERNAL_COLUMNS.map(col => `${table._id}.${col}`)
   )
-  for (let col of Object.values(table.schema)) {
+  for (let key of Object.keys(table.schema)) {
+    const col = table.schema[key]
     const isRelationship = col.type === FieldType.LINK
     if (!opts?.relationships && isRelationship) {
       continue
     }
     if (!isRelationship) {
-      fieldList.push(`${table._id}.${mapToUserColumn(col.name)}`)
+      fieldList.push(`${table._id}.${mapToUserColumn(key)}`)
     } else {
       const linkCol = col as RelationshipFieldMetadata
       const relatedTable = tables.find(table => table._id === linkCol.tableId)

packages/server/src/sdk/app/rows/utils.ts

@@ -12,6 +12,7 @@ import {
   Table,
   TableSchema,
   SqlClient,
+  ArrayOperator,
 } from "@budibase/types"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Format } from "../../../api/controllers/view/exporters"
@@ -311,3 +312,8 @@ function validateTimeOnlyField(
 
   return res
 }
+
+// type-guard check
+export function isArrayFilter(operator: any): operator is ArrayOperator {
+  return Object.values(ArrayOperator).includes(operator)
+}

packages/server/src/tests/jestSetup.ts

@@ -1,8 +1,10 @@
 import env from "../environment"
+import * as matchers from "jest-extended"
 import { env as coreEnv, timers } from "@budibase/backend-core"
 import { testContainerUtils } from "@budibase/backend-core/tests"
 import nock from "nock"
 
+expect.extend(matchers)
 if (!process.env.CI) {
   // set a longer timeout in dev for debugging 100 seconds
   jest.setTimeout(100 * 1000)

packages/server/src/utilities/schema.ts

@@ -146,7 +146,8 @@ export function parse(rows: Rows, table: Table): Rows {
   return rows.map(row => {
     const parsedRow: Row = {}
 
-    Object.entries(row).forEach(([columnName, columnData]) => {
+    Object.keys(row).forEach(columnName => {
+      const columnData = row[columnName]
       const schema = table.schema
       if (!(columnName in schema)) {
         // Objects can be present in the row data but not in the schema, so make sure we don't proceed in such a case

yarn.lock

@@ -13614,7 +13614,7 @@ jest-config@^29.7.0:
     slash "^3.0.0"
     strip-json-comments "^3.1.1"
 
-"jest-diff@>=29.4.3 < 30", jest-diff@^29.4.1, jest-diff@^29.7.0:
+"jest-diff@>=29.4.3 < 30", jest-diff@^29.0.0, jest-diff@^29.4.1, jest-diff@^29.7.0:
   version "29.7.0"
   resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.7.0.tgz#017934a66ebb7ecf6f205e84699be10afd70458a"
   integrity sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==
@@ -13673,12 +13673,20 @@ jest-environment-node@^29.7.0:
     jest-mock "^29.7.0"
     jest-util "^29.7.0"
 
+jest-extended@^4.0.2:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/jest-extended/-/jest-extended-4.0.2.tgz#d23b52e687cedf66694e6b2d77f65e211e99e021"
+  integrity sha512-FH7aaPgtGYHc9mRjriS0ZEHYM5/W69tLrFTIdzm+yJgeoCmmrSB/luSfMSqWP9O29QWHPEmJ4qmU6EwsZideog==
+  dependencies:
+    jest-diff "^29.0.0"
+    jest-get-type "^29.0.0"
+
 jest-get-type@^26.3.0:
   version "26.3.0"
   resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-26.3.0.tgz#e97dc3c3f53c2b406ca7afaed4493b1d099199e0"
   integrity sha512-TpfaviN1R2pQWkIihlfEanwOXK0zcxrKEE4MlU6Tn7keoXdN6/3gK/xl0yEh8DOunn5pOVGKf8hB4R9gVh04ig==
 
-jest-get-type@^29.6.3:
+jest-get-type@^29.0.0, jest-get-type@^29.6.3:
   version "29.6.3"
   resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-29.6.3.tgz#36f499fdcea197c1045a127319c0481723908fd1"
   integrity sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==