From 8c3c341c558645dfcbdd678ed47fc5450d5650c3 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 9 Apr 2024 15:09:56 +0100 Subject: [PATCH 1/7] Set up first search test that hits SQS et al --- globalSetup.ts | 2 +- .../src/api/routes/tests/search.spec.ts | 74 +++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/api/routes/tests/search.spec.ts diff --git a/globalSetup.ts b/globalSetup.ts index 7bf5e2152c..2cfb22f715 100644 --- a/globalSetup.ts +++ b/globalSetup.ts @@ -13,7 +13,7 @@ export default async function setup() { } try { - let couchdb = new GenericContainer("budibase/couchdb") + let couchdb = new GenericContainer("budibase/couchdb:v3.2.1-sqs") .withExposedPorts(5984) .withEnvironment({ COUCHDB_PASSWORD: "budibase", diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts new file mode 100644 index 0000000000..a281d624dc --- /dev/null +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -0,0 +1,74 @@ +import { tableForDatasource } from "../../../tests/utilities/structures" +import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" + +import * as setup from "./utilities" +import { Datasource, FieldType, Table } from "@budibase/types" + +jest.unmock("mssql") + +describe.each([ + ["internal", undefined], + ["internal-sqs", undefined], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], +])("/api/:sourceId/search (%s)", (name, dsProvider) => { + const isSqs = name === "internal-sqs" + const config = setup.getConfig() + + let envCleanup: (() => void) | undefined + let table: Table + let datasource: Datasource | undefined + + beforeAll(async () => { + if (isSqs) { + envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" }) + } + await config.init() + if (dsProvider) { + datasource = await config.createDatasource({ + datasource: await dsProvider, + }) + } + }) + + afterAll(async () => { + setup.afterAll() + if (envCleanup) { + envCleanup() + } + }) + + beforeEach(async () => { + table = await config.api.table.save( + tableForDatasource(datasource, { + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + }, + }) + ) + }) + + it("should return rows", async () => { + const rows = await Promise.all([ + config.api.row.save(table._id!, { name: "foo" }), + config.api.row.save(table._id!, { name: "bar" }), + ]) + + const result = await config.api.row.search(table._id!, { + tableId: table._id!, + query: {}, + }) + + expect(result.rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ _id: rows[0]._id }), + expect.objectContaining({ _id: rows[1]._id }), + ]) + ) + }) +}) From 8880920c9797a46bff75673e27f7811d88bfcb97 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 9 Apr 2024 15:22:02 +0100 Subject: [PATCH 2/7] Remove the test container logs. --- .github/workflows/budibase_ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 42d73ba8bb..0bbed77458 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -92,7 +92,6 @@ jobs: test-libraries: runs-on: ubuntu-latest env: - DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull REUSE_CONTAINERS: true steps: - name: Checkout repo @@ -151,7 +150,6 @@ jobs: test-server: runs-on: budi-tubby-tornado-quad-core-150gb env: - DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull REUSE_CONTAINERS: true steps: - name: Checkout repo From 16bf48586daaeabd536dbc5c8ad67864b96a722d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 9 Apr 2024 15:22:31 +0100 Subject: [PATCH 3/7] Run top level 'test' sequentially, rather than in parallel, which can cause problems, also use REUSE_CONTAINERS by default. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d99304423a..2816247939 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "dev:all": "yarn run kill-all && lerna run --stream dev", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", - "test": "lerna run --stream test --stream", + "test": "REUSE_CONTAINERS=1 lerna run --concurrency 1 --stream test --stream", "lint:eslint": "eslint packages --max-warnings=0", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"", "lint": "yarn run lint:eslint && yarn run lint:prettier", From c5580b476784d02f010faf494d3b7ef343e4df71 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 9 Apr 2024 15:31:32 +0100 Subject: [PATCH 4/7] Get SQS plumbing working in tests. --- globalSetup.ts | 2 +- .../backend-core/tests/core/utilities/testContainerUtils.ts | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/globalSetup.ts b/globalSetup.ts index 2cfb22f715..115796c395 100644 --- a/globalSetup.ts +++ b/globalSetup.ts @@ -14,7 +14,7 @@ export default async function setup() { try { let couchdb = new GenericContainer("budibase/couchdb:v3.2.1-sqs") - .withExposedPorts(5984) + .withExposedPorts(5984, 4984) .withEnvironment({ COUCHDB_PASSWORD: "budibase", COUCHDB_USER: "budibase", diff --git a/packages/backend-core/tests/core/utilities/testContainerUtils.ts b/packages/backend-core/tests/core/utilities/testContainerUtils.ts index 951a6f0517..d0dd2c9b4d 100644 --- a/packages/backend-core/tests/core/utilities/testContainerUtils.ts +++ b/packages/backend-core/tests/core/utilities/testContainerUtils.ts @@ -77,9 +77,15 @@ export function setupEnv(...envs: any[]) { throw new Error("CouchDB port not found") } + const couchSqlPort = getExposedV4Port(couch, 4984) + if (!couchSqlPort) { + throw new Error("CouchDB SQL port not found") + } + const configs = [ { key: "COUCH_DB_PORT", value: `${couchPort}` }, { key: "COUCH_DB_URL", value: `http://127.0.0.1:${couchPort}` }, + { key: "COUCH_DB_SQL_URL", value: `http://127.0.0.1:${couchSqlPort}` }, ] for (const config of configs.filter(x => !!x.value)) { From 2d36cf6c6a29f313f0a4c94b768253cb89d0afe3 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 9 Apr 2024 15:55:44 +0100 Subject: [PATCH 5/7] Working towards getting first SQS test passing. --- .../server/src/api/controllers/row/utils/sqlUtils.ts | 4 ++-- packages/server/src/api/controllers/row/utils/utils.ts | 4 ++-- packages/server/src/api/routes/tests/search.spec.ts | 10 +++++----- packages/server/src/sdk/app/rows/search/sqs.ts | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index 2df829154f..6f9837e0ab 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -32,7 +32,7 @@ export async function updateRelationshipColumns( row: Row, rows: { [key: string]: Row }, relationships: RelationshipsJson[], - opts?: { internal?: boolean } + opts?: { sqs?: boolean } ) { const columns: { [key: string]: any } = {} for (let relationship of relationships) { @@ -55,7 +55,7 @@ export async function updateRelationshipColumns( row, table: linkedTable, isLinked: true, - internal: opts?.internal, + internal: opts?.sqs, }) if (!linked._id) { continue diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index c77ccf63a9..49b6aff85b 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -117,7 +117,7 @@ export async function sqlOutputProcessing( table: Table, tables: Record, relationships: RelationshipsJson[], - opts?: { internal?: boolean } + opts?: { sqs?: boolean } ): Promise { if (!Array.isArray(rows) || rows.length === 0 || rows[0].read === true) { return [] @@ -146,7 +146,7 @@ export async function sqlOutputProcessing( row, table, isLinked: false, - internal: opts?.internal, + internal: opts?.sqs, }), table ) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index a281d624dc..b1fe273e27 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -7,12 +7,12 @@ import { Datasource, FieldType, Table } from "@budibase/types" jest.unmock("mssql") describe.each([ - ["internal", undefined], + // ["internal", undefined], ["internal-sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + //[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + //[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + //[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + //[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/api/:sourceId/search (%s)", (name, dsProvider) => { const isSqs = name === "internal-sqs" const config = setup.getConfig() diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 8aba94c886..89dae7628f 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -179,7 +179,7 @@ export async function search( allTablesMap, relationships, { - internal: true, + sqs: true, } ), } From 678c429f64cc189c6c7a0e09fd3efc985c4cf8c3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 9 Apr 2024 16:05:00 +0100 Subject: [PATCH 6/7] Fixing test case. --- .../src/api/controllers/row/utils/basic.ts | 19 +++++++++++++++++++ .../src/api/controllers/row/utils/utils.ts | 11 +++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index 2fb67b07d5..1fc84de9c7 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -1,6 +1,7 @@ // need to handle table name + field or just field, depending on if relationships used import { FieldType, Row, Table } from "@budibase/types" import { generateRowIdField } from "../../../../integrations/utils" +import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils" function extractFieldValue({ row, @@ -20,6 +21,15 @@ function extractFieldValue({ return value } +export function getInternalRowId(row: Row, table: Table): string { + return extractFieldValue({ + row, + tableName: table._id!, + fieldName: "_id", + isLinked: false, + }) +} + export function generateIdForRow( row: Row | undefined, table: Table, @@ -78,6 +88,15 @@ export function basicProcessing({ thisRow._id = generateIdForRow(row, table, isLinked) thisRow.tableId = table._id thisRow._rev = "rev" + } else { + for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) { + thisRow[internalColumn] = extractFieldValue({ + row, + tableName: table._id!, + fieldName: internalColumn, + isLinked: false, + }) + } } return thisRow } diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index 49b6aff85b..f387a468cf 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -15,7 +15,12 @@ import { processFormulas, } from "../../../../utilities/rowProcessor" import { updateRelationshipColumns } from "./sqlUtils" -import { basicProcessing, generateIdForRow, fixArrayTypes } from "./basic" +import { + basicProcessing, + generateIdForRow, + fixArrayTypes, + getInternalRowId, +} from "./basic" import sdk from "../../../../sdk" import validateJs from "validate.js" @@ -125,7 +130,9 @@ export async function sqlOutputProcessing( let finalRows: { [key: string]: Row } = {} for (let row of rows as Row[]) { let rowId = row._id - if (!rowId) { + if (opts?.sqs) { + rowId = getInternalRowId(row, table) + } else if (!rowId) { rowId = generateIdForRow(row, table) row._id = rowId } From 06e01be5f92a340df53a19a47a03f6de98be41fe Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 9 Apr 2024 16:48:54 +0100 Subject: [PATCH 7/7] Enable all search.spec.ts tests, pull the sqs CouchDB image in CI. --- .github/workflows/budibase_ci.yml | 4 ++-- packages/server/src/api/routes/tests/search.spec.ts | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 42d73ba8bb..9a5bbb3058 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -110,7 +110,7 @@ jobs: - name: Pull testcontainers images run: | docker pull testcontainers/ryuk:0.5.1 & - docker pull budibase/couchdb & + docker pull budibase/couchdb:v3.2.1-sql & docker pull redis & wait $(jobs -p) @@ -175,7 +175,7 @@ jobs: docker pull mongo:7.0-jammy & docker pull mariadb:lts & docker pull testcontainers/ryuk:0.5.1 & - docker pull budibase/couchdb & + docker pull budibase/couchdb:v3.2.1-sql & docker pull redis & wait $(jobs -p) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index b1fe273e27..a281d624dc 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -7,12 +7,12 @@ import { Datasource, FieldType, Table } from "@budibase/types" jest.unmock("mssql") describe.each([ - // ["internal", undefined], + ["internal", undefined], ["internal-sqs", undefined], - //[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - //[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - //[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - //[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/api/:sourceId/search (%s)", (name, dsProvider) => { const isSqs = name === "internal-sqs" const config = setup.getConfig()