Merge remote-tracking branch 'origin/master' into dean-bug-fixes

.github/workflows/budibase_ci.yml

@@ -92,7 +92,6 @@ jobs:
   test-libraries:
     runs-on: ubuntu-latest
     env:
-      DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull
       REUSE_CONTAINERS: true
     steps:
       - name: Checkout repo
@@ -110,7 +109,7 @@ jobs:
       - name: Pull testcontainers images
         run: |
           docker pull testcontainers/ryuk:0.5.1 &
-          docker pull budibase/couchdb &
+          docker pull budibase/couchdb:v3.2.1-sql &
           docker pull redis &
 
           wait $(jobs -p)
@@ -151,7 +150,6 @@ jobs:
   test-server:
     runs-on: budi-tubby-tornado-quad-core-150gb
     env:
-      DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull
       REUSE_CONTAINERS: true
     steps:
       - name: Checkout repo
@@ -175,7 +173,7 @@ jobs:
           docker pull mongo:7.0-jammy &
           docker pull mariadb:lts &
           docker pull testcontainers/ryuk:0.5.1 &
-          docker pull budibase/couchdb &
+          docker pull budibase/couchdb:v3.2.1-sql &
           docker pull redis &
 
           wait $(jobs -p)

globalSetup.ts

@@ -13,8 +13,8 @@ export default async function setup() {
   }
 
   try {
-    let couchdb = new GenericContainer("budibase/couchdb")
+    let couchdb = new GenericContainer("budibase/couchdb:v3.2.1-sqs")
-      .withExposedPorts(5984)
+      .withExposedPorts(5984, 4984)
       .withEnvironment({
         COUCHDB_PASSWORD: "budibase",
         COUCHDB_USER: "budibase",

package.json

@@ -59,7 +59,7 @@
     "dev:all": "yarn run kill-all && lerna run --stream dev",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
     "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
-    "test": "lerna run --stream test --stream",
+    "test": "REUSE_CONTAINERS=1 lerna run --concurrency 1 --stream test --stream",
     "lint:eslint": "eslint packages --max-warnings=0",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"",
     "lint": "yarn run lint:eslint && yarn run lint:prettier",

packages/backend-core/tests/core/utilities/testContainerUtils.ts

@@ -77,9 +77,15 @@ export function setupEnv(...envs: any[]) {
     throw new Error("CouchDB port not found")
   }
 
+  const couchSqlPort = getExposedV4Port(couch, 4984)
+  if (!couchSqlPort) {
+    throw new Error("CouchDB SQL port not found")
+  }
+
   const configs = [
     { key: "COUCH_DB_PORT", value: `${couchPort}` },
     { key: "COUCH_DB_URL", value: `http://127.0.0.1:${couchPort}` },
+    { key: "COUCH_DB_SQL_URL", value: `http://127.0.0.1:${couchSqlPort}` },
   ]
 
   for (const config of configs.filter(x => !!x.value)) {

packages/server/src/api/controllers/row/utils/basic.ts

@@ -1,6 +1,7 @@
 // need to handle table name + field or just field, depending on if relationships used
 import { FieldType, Row, Table } from "@budibase/types"
 import { generateRowIdField } from "../../../../integrations/utils"
+import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
 
 function extractFieldValue({
   row,
@@ -20,6 +21,15 @@ function extractFieldValue({
   return value
 }
 
+export function getInternalRowId(row: Row, table: Table): string {
+  return extractFieldValue({
+    row,
+    tableName: table._id!,
+    fieldName: "_id",
+    isLinked: false,
+  })
+}
+
 export function generateIdForRow(
   row: Row | undefined,
   table: Table,
@@ -78,6 +88,15 @@ export function basicProcessing({
     thisRow._id = generateIdForRow(row, table, isLinked)
     thisRow.tableId = table._id
     thisRow._rev = "rev"
+  } else {
+    for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) {
+      thisRow[internalColumn] = extractFieldValue({
+        row,
+        tableName: table._id!,
+        fieldName: internalColumn,
+        isLinked: false,
+      })
+    }
   }
   return thisRow
 }

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -32,7 +32,7 @@ export async function updateRelationshipColumns(
   row: Row,
   rows: { [key: string]: Row },
   relationships: RelationshipsJson[],
-  opts?: { internal?: boolean }
+  opts?: { sqs?: boolean }
 ) {
   const columns: { [key: string]: any } = {}
   for (let relationship of relationships) {
@@ -55,7 +55,7 @@ export async function updateRelationshipColumns(
       row,
       table: linkedTable,
       isLinked: true,
-      internal: opts?.internal,
+      internal: opts?.sqs,
     })
     if (!linked._id) {
       continue

packages/server/src/api/controllers/row/utils/utils.ts

@@ -15,7 +15,12 @@ import {
   processFormulas,
 } from "../../../../utilities/rowProcessor"
 import { updateRelationshipColumns } from "./sqlUtils"
-import { basicProcessing, generateIdForRow, fixArrayTypes } from "./basic"
+import {
+  basicProcessing,
+  generateIdForRow,
+  fixArrayTypes,
+  getInternalRowId,
+} from "./basic"
 import sdk from "../../../../sdk"
 
 import validateJs from "validate.js"
@@ -117,7 +122,7 @@ export async function sqlOutputProcessing(
   table: Table,
   tables: Record<string, Table>,
   relationships: RelationshipsJson[],
-  opts?: { internal?: boolean }
+  opts?: { sqs?: boolean }
 ): Promise<Row[]> {
   if (!Array.isArray(rows) || rows.length === 0 || rows[0].read === true) {
     return []
@@ -125,7 +130,9 @@ export async function sqlOutputProcessing(
   let finalRows: { [key: string]: Row } = {}
   for (let row of rows as Row[]) {
     let rowId = row._id
-    if (!rowId) {
+    if (opts?.sqs) {
+      rowId = getInternalRowId(row, table)
+    } else if (!rowId) {
       rowId = generateIdForRow(row, table)
       row._id = rowId
     }
@@ -146,7 +153,7 @@ export async function sqlOutputProcessing(
         row,
         table,
         isLinked: false,
-        internal: opts?.internal,
+        internal: opts?.sqs,
       }),
       table
     )

packages/server/src/api/routes/tests/search.spec.ts

@@ -0,0 +1,74 @@
+import { tableForDatasource } from "../../../tests/utilities/structures"
+import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
+
+import * as setup from "./utilities"
+import { Datasource, FieldType, Table } from "@budibase/types"
+
+jest.unmock("mssql")
+
+describe.each([
+  ["internal", undefined],
+  ["internal-sqs", undefined],
+  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+])("/api/:sourceId/search (%s)", (name, dsProvider) => {
+  const isSqs = name === "internal-sqs"
+  const config = setup.getConfig()
+
+  let envCleanup: (() => void) | undefined
+  let table: Table
+  let datasource: Datasource | undefined
+
+  beforeAll(async () => {
+    if (isSqs) {
+      envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" })
+    }
+    await config.init()
+    if (dsProvider) {
+      datasource = await config.createDatasource({
+        datasource: await dsProvider,
+      })
+    }
+  })
+
+  afterAll(async () => {
+    setup.afterAll()
+    if (envCleanup) {
+      envCleanup()
+    }
+  })
+
+  beforeEach(async () => {
+    table = await config.api.table.save(
+      tableForDatasource(datasource, {
+        schema: {
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+          },
+        },
+      })
+    )
+  })
+
+  it("should return rows", async () => {
+    const rows = await Promise.all([
+      config.api.row.save(table._id!, { name: "foo" }),
+      config.api.row.save(table._id!, { name: "bar" }),
+    ])
+
+    const result = await config.api.row.search(table._id!, {
+      tableId: table._id!,
+      query: {},
+    })
+
+    expect(result.rows).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ _id: rows[0]._id }),
+        expect.objectContaining({ _id: rows[1]._id }),
+      ])
+    )
+  })
+})

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -179,7 +179,7 @@ export async function search(
         allTablesMap,
         relationships,
         {
-          internal: true,
+          sqs: true,
         }
       ),
     }