From f4846736675ac9fe7cb549c75366931d57e62fa1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Feb 2025 12:43:05 +0000 Subject: [PATCH 1/7] Remove elasticsearch global mock, fix tests. --- .../__mocks__/@elastic/elasticsearch.ts | 24 ---- packages/server/datasource-sha.env | 1 + .../server/src/integrations/elasticsearch.ts | 42 ++++--- .../integrations/tests/elasticsearch.spec.ts | 119 +++++++++--------- .../integrations/tests/utils/elasticsearch.ts | 43 +++++++ .../src/integrations/tests/utils/images.ts | 1 + 6 files changed, 126 insertions(+), 104 deletions(-) delete mode 100644 packages/server/__mocks__/@elastic/elasticsearch.ts create mode 100644 packages/server/src/integrations/tests/utils/elasticsearch.ts diff --git a/packages/server/__mocks__/@elastic/elasticsearch.ts b/packages/server/__mocks__/@elastic/elasticsearch.ts deleted file mode 100644 index 5e13437f29..0000000000 --- a/packages/server/__mocks__/@elastic/elasticsearch.ts +++ /dev/null @@ -1,24 +0,0 @@ -const elastic: any = {} - -elastic.Client = function () { - this.index = jest.fn().mockResolvedValue({ body: [] }) - this.search = jest.fn().mockResolvedValue({ - body: { - hits: { - hits: [ - { - _source: { - name: "test", - }, - }, - ], - }, - }, - }) - this.update = jest.fn().mockResolvedValue({ body: [] }) - this.delete = jest.fn().mockResolvedValue({ body: [] }) - - this.close = jest.fn() -} - -module.exports = elastic diff --git a/packages/server/datasource-sha.env b/packages/server/datasource-sha.env index 61249d530c..383fb191f4 100644 --- a/packages/server/datasource-sha.env +++ b/packages/server/datasource-sha.env @@ -3,3 +3,4 @@ MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588eb POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d MARIADB_SHA=sha256:e59ba8783bf7bc02a4779f103bb0d8751ac0e10f9471089709608377eded7aa8 +ELASTICSEARCH_SHA=sha256:9a6443f55243f6acbfeb4a112d15eb3b9aac74bf25e0e39fa19b3ddd3a6879d0 \ No newline at end of file diff --git a/packages/server/src/integrations/elasticsearch.ts b/packages/server/src/integrations/elasticsearch.ts index af03baaef1..10f9d1e697 100644 --- a/packages/server/src/integrations/elasticsearch.ts +++ b/packages/server/src/integrations/elasticsearch.ts @@ -10,7 +10,7 @@ import { import { Client, ClientOptions } from "@elastic/elasticsearch" import { HOST_ADDRESS } from "./utils" -interface ElasticsearchConfig { +export interface ElasticsearchConfig { url: string ssl?: boolean ca?: string @@ -99,9 +99,9 @@ const SCHEMA: Integration = { }, } -class ElasticSearchIntegration implements IntegrationBase { +export class ElasticSearchIntegration implements IntegrationBase { private config: ElasticsearchConfig - private client + private client: Client constructor(config: ElasticsearchConfig) { this.config = config @@ -132,20 +132,23 @@ class ElasticSearchIntegration implements IntegrationBase { } } - async create(query: { index: string; json: object }) { - const { index, json } = query + async create(query: { + index: string + json: object + extra?: Record + }) { + const { index, json, extra } = query try { const result = await this.client.index({ index, body: json, + ...extra, }) return result.body } catch (err) { console.error("Error writing to elasticsearch", err) throw err - } finally { - await this.client.close() } } @@ -160,41 +163,46 @@ class ElasticSearchIntegration implements IntegrationBase { } catch (err) { console.error("Error querying elasticsearch", err) throw err - } finally { - await this.client.close() } } - async update(query: { id: string; index: string; json: object }) { - const { id, index, json } = query + async update(query: { + id: string + index: string + json: object + extra?: Record + }) { + const { id, index, json, extra } = query try { const result = await this.client.update({ id, index, body: json, + ...extra, }) return result.body } catch (err) { console.error("Error querying elasticsearch", err) throw err - } finally { - await this.client.close() } } - async delete(query: { id: string; index: string }) { - const { id, index } = query + async delete(query: { + id: string + index: string + extra?: Record + }) { + const { id, index, extra } = query try { const result = await this.client.delete({ id, index, + ...extra, }) return result.body } catch (err) { console.error("Error deleting from elasticsearch", err) throw err - } finally { - await this.client.close() } } } diff --git a/packages/server/src/integrations/tests/elasticsearch.spec.ts b/packages/server/src/integrations/tests/elasticsearch.spec.ts index f8a1dd8013..da9c5600ad 100644 --- a/packages/server/src/integrations/tests/elasticsearch.spec.ts +++ b/packages/server/src/integrations/tests/elasticsearch.spec.ts @@ -1,83 +1,76 @@ -import { default as ElasticSearchIntegration } from "../elasticsearch" - -jest.mock("@elastic/elasticsearch") - -class TestConfiguration { - integration: any - - constructor(config: any = {}) { - this.integration = new ElasticSearchIntegration.integration(config) - } -} +import { Datasource } from "@budibase/types" +import { ElasticsearchConfig, ElasticSearchIntegration } from "../elasticsearch" +import * as elasticsearch from "../tests/utils/elasticsearch" +import { generator } from "@budibase/backend-core/tests" describe("Elasticsearch Integration", () => { - let config: any - let indexName = "Users" + let datasource: Datasource + let integration: ElasticSearchIntegration + + let index: string + + beforeAll(async () => { + datasource = await elasticsearch.getDatasource() + }) beforeEach(() => { - config = new TestConfiguration() + index = generator.guid() + integration = new ElasticSearchIntegration( + datasource.config! as ElasticsearchConfig + ) }) - it("calls the create method with the correct params", async () => { - const body = { - name: "Hello", - } - await config.integration.create({ - index: indexName, - json: body, + it("can create a record", async () => { + await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, }) - expect(config.integration.client.index).toHaveBeenCalledWith({ - index: indexName, - body, + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, }) + expect(records).toEqual([{ name: "Hello" }]) }) - it("calls the read method with the correct params", async () => { - const body = { - query: { - term: { - name: "kimchy", - }, - }, - } - const response = await config.integration.read({ - index: indexName, - json: body, + it("can update a record", async () => { + const create = await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, }) - expect(config.integration.client.search).toHaveBeenCalledWith({ - index: indexName, - body, + + await integration.update({ + id: create._id, + index, + json: { doc: { name: "World" } }, + extra: { refresh: "true" }, }) - expect(response).toEqual(expect.any(Array)) + + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([{ name: "World" }]) }) - it("calls the update method with the correct params", async () => { - const body = { - name: "updated", - } - - const response = await config.integration.update({ - id: "1234", - index: indexName, - json: body, + it("can delete a record", async () => { + const create = await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, }) - expect(config.integration.client.update).toHaveBeenCalledWith({ - id: "1234", - index: indexName, - body, + await integration.delete({ + id: create._id, + index, + extra: { refresh: "true" }, }) - expect(response).toEqual(expect.any(Array)) - }) - it("calls the delete method with the correct params", async () => { - const body = { - id: "1234", - } - - const response = await config.integration.delete(body) - - expect(config.integration.client.delete).toHaveBeenCalledWith(body) - expect(response).toEqual(expect.any(Array)) + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([]) }) }) diff --git a/packages/server/src/integrations/tests/utils/elasticsearch.ts b/packages/server/src/integrations/tests/utils/elasticsearch.ts new file mode 100644 index 0000000000..a7229c3f60 --- /dev/null +++ b/packages/server/src/integrations/tests/utils/elasticsearch.ts @@ -0,0 +1,43 @@ +import { Datasource, SourceName } from "@budibase/types" +import { GenericContainer, Wait } from "testcontainers" +import { testContainerUtils } from "@budibase/backend-core/tests" +import { startContainer } from "." +import { ELASTICSEARCH_IMAGE } from "./images" +import { ElasticsearchConfig } from "../../elasticsearch" + +let ports: Promise + +export async function getDatasource(): Promise { + if (!ports) { + ports = startContainer( + new GenericContainer(ELASTICSEARCH_IMAGE) + .withExposedPorts(9200) + .withEnvironment({ + "discovery.type": "single-node", + "xpack.security.enabled": "false", + }) + .withWaitStrategy( + Wait.forHttp( + "/_cluster/health?wait_for_status=yellow&timeout=10s", + 9200 + ).withStartupTimeout(60000) + ) + .withTmpFs({ "/usr/share/elasticsearch/data": "rw" }) + ) + } + + const port = (await ports).find(x => x.container === 9200)?.host + if (!port) { + throw new Error("Elasticsearch port not found") + } + + const config: ElasticsearchConfig = { + url: `http://127.0.0.1:${port}`, + } + + return { + type: "datasource", + source: SourceName.ELASTICSEARCH, + config, + } +} diff --git a/packages/server/src/integrations/tests/utils/images.ts b/packages/server/src/integrations/tests/utils/images.ts index 00686412c6..c09b130ea5 100644 --- a/packages/server/src/integrations/tests/utils/images.ts +++ b/packages/server/src/integrations/tests/utils/images.ts @@ -12,3 +12,4 @@ export const POSTGRES_IMAGE = `postgres@${process.env.POSTGRES_SHA}` export const POSTGRES_LEGACY_IMAGE = `postgres:9.5.25` export const MONGODB_IMAGE = `mongo@${process.env.MONGODB_SHA}` export const MARIADB_IMAGE = `mariadb@${process.env.MARIADB_SHA}` +export const ELASTICSEARCH_IMAGE = `elasticsearch@${process.env.ELASTICSEARCH_SHA}` From 1c23509839517910e1b44a66bdb184ffd8f02c27 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Feb 2025 14:58:52 +0000 Subject: [PATCH 2/7] Integrate Elasticsearch as a test datasource. --- .github/workflows/budibase_ci.yml | 15 +- .../src/api/routes/tests/datasource.spec.ts | 6 +- .../routes/tests/queries/generic-sql.spec.ts | 3 +- .../server/src/api/routes/tests/row.spec.ts | 7 +- .../src/api/routes/tests/search.spec.ts | 7 +- .../server/src/api/routes/tests/table.spec.ts | 7 +- .../src/api/routes/tests/viewV2.spec.ts | 7 +- .../tests/steps/executeQuery.spec.ts | 3 +- .../integrations/tests/elasticsearch.spec.ts | 141 +++++++++--------- .../src/integrations/tests/utils/index.ts | 56 +++++-- .../sdk/app/rows/search/tests/search.spec.ts | 7 +- 11 files changed, 145 insertions(+), 114 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index e294e50e8d..6a9aa3e8d0 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -155,7 +155,18 @@ jobs: strategy: matrix: datasource: - [mssql, mysql, postgres, postgres_legacy, mongodb, mariadb, oracle, sqs, none] + [ + mssql, + mysql, + postgres, + postgres_legacy, + mongodb, + mariadb, + oracle, + sqs, + elasticsearch, + none, + ] steps: - name: Checkout repo uses: actions/checkout@v4 @@ -192,6 +203,8 @@ jobs: docker pull budibase/oracle-database:23.2-slim-faststart elif [ "${{ matrix.datasource }}" == "postgres_legacy" ]; then docker pull postgres:9.5.25 + elif [ "${{ matrix.datasource }}" == "elasticsearch" ]; then + docker pull elasticsearch@${{ steps.dotenv.outputs.ELASTICSEARCH_SHA }} fi docker pull minio/minio & docker pull redis & diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts index 21e9effa77..12029c39c4 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.ts +++ b/packages/server/src/api/routes/tests/datasource.spec.ts @@ -165,7 +165,8 @@ describe("/datasources", () => { }) const descriptions = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS], + plus: true, + exclude: [DatabaseName.SQS], }) if (descriptions.length) { @@ -590,7 +591,8 @@ if (descriptions.length) { } const datasources = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS, DatabaseName.ORACLE], + plus: true, + exclude: [DatabaseName.SQS, DatabaseName.ORACLE], }) if (datasources.length) { diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 863f5b65e0..4a545b253e 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -9,7 +9,8 @@ import { Knex } from "knex" import { generator } from "@budibase/backend-core/tests" const descriptions = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS], + plus: true, + exclude: [DatabaseName.SQS], }) if (descriptions.length) { diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 87002670b7..b349a1df8a 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1,9 +1,6 @@ import * as setup from "./utilities" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import tk from "timekeeper" import emitter from "../../../../src/events" @@ -80,7 +77,7 @@ function encodeJS(binding: string) { return `{{ js "${Buffer.from(binding).toString("base64")}"}}` } -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index ee372914d7..a904c0c469 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1,8 +1,5 @@ import { tableForDatasource } from "../../../tests/utilities/structures" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import { context, db as dbCore, @@ -60,7 +57,7 @@ jest.mock("@budibase/pro", () => ({ }, })) -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 2a7f039ff5..29b576d16a 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -28,17 +28,14 @@ import * as setup from "./utilities" import * as uuid from "uuid" import { generator } from "@budibase/backend-core/tests" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import { tableForDatasource } from "../../../tests/utilities/structures" import timekeeper from "timekeeper" const { basicTable } = setup.structures const ISO_REGEX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/ -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 9531737d30..7eed1811d9 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -37,17 +37,14 @@ import { ViewV2Type, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import merge from "lodash/merge" import { quotas } from "@budibase/pro" import { context, db, events, roles, setEnv } from "@budibase/backend-core" import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai" import nock from "nock" -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/automations/tests/steps/executeQuery.spec.ts b/packages/server/src/automations/tests/steps/executeQuery.spec.ts index dff3580b7e..a51d335902 100644 --- a/packages/server/src/automations/tests/steps/executeQuery.spec.ts +++ b/packages/server/src/automations/tests/steps/executeQuery.spec.ts @@ -9,7 +9,8 @@ import { generator } from "@budibase/backend-core/tests" import { createAutomationBuilder } from "../utilities/AutomationTestBuilder" const descriptions = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS], + plus: true, + exclude: [DatabaseName.SQS], }) if (descriptions.length) { diff --git a/packages/server/src/integrations/tests/elasticsearch.spec.ts b/packages/server/src/integrations/tests/elasticsearch.spec.ts index da9c5600ad..14a272d610 100644 --- a/packages/server/src/integrations/tests/elasticsearch.spec.ts +++ b/packages/server/src/integrations/tests/elasticsearch.spec.ts @@ -2,75 +2,80 @@ import { Datasource } from "@budibase/types" import { ElasticsearchConfig, ElasticSearchIntegration } from "../elasticsearch" import * as elasticsearch from "../tests/utils/elasticsearch" import { generator } from "@budibase/backend-core/tests" +import { DatabaseName, datasourceDescribe } from "./utils" -describe("Elasticsearch Integration", () => { - let datasource: Datasource - let integration: ElasticSearchIntegration +const describes = datasourceDescribe({ only: [DatabaseName.ELASTICSEARCH] }) - let index: string +if (describes.length) { + describe.each(describes)("Elasticsearch Integration", () => { + let datasource: Datasource + let integration: ElasticSearchIntegration - beforeAll(async () => { - datasource = await elasticsearch.getDatasource() + let index: string + + beforeAll(async () => { + datasource = await elasticsearch.getDatasource() + }) + + beforeEach(() => { + index = generator.guid() + integration = new ElasticSearchIntegration( + datasource.config! as ElasticsearchConfig + ) + }) + + it("can create a record", async () => { + await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, + }) + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([{ name: "Hello" }]) + }) + + it("can update a record", async () => { + const create = await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, + }) + + await integration.update({ + id: create._id, + index, + json: { doc: { name: "World" } }, + extra: { refresh: "true" }, + }) + + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([{ name: "World" }]) + }) + + it("can delete a record", async () => { + const create = await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, + }) + + await integration.delete({ + id: create._id, + index, + extra: { refresh: "true" }, + }) + + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([]) + }) }) - - beforeEach(() => { - index = generator.guid() - integration = new ElasticSearchIntegration( - datasource.config! as ElasticsearchConfig - ) - }) - - it("can create a record", async () => { - await integration.create({ - index, - json: { name: "Hello" }, - extra: { refresh: "true" }, - }) - const records = await integration.read({ - index, - json: { query: { match_all: {} } }, - }) - expect(records).toEqual([{ name: "Hello" }]) - }) - - it("can update a record", async () => { - const create = await integration.create({ - index, - json: { name: "Hello" }, - extra: { refresh: "true" }, - }) - - await integration.update({ - id: create._id, - index, - json: { doc: { name: "World" } }, - extra: { refresh: "true" }, - }) - - const records = await integration.read({ - index, - json: { query: { match_all: {} } }, - }) - expect(records).toEqual([{ name: "World" }]) - }) - - it("can delete a record", async () => { - const create = await integration.create({ - index, - json: { name: "Hello" }, - extra: { refresh: "true" }, - }) - - await integration.delete({ - id: create._id, - index, - extra: { refresh: "true" }, - }) - - const records = await integration.read({ - index, - json: { query: { match_all: {} } }, - }) - expect(records).toEqual([]) - }) -}) +} diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index 9e2c4f7e70..08777cab89 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -6,6 +6,7 @@ import * as mysql from "./mysql" import * as mssql from "./mssql" import * as mariadb from "./mariadb" import * as oracle from "./oracle" +import * as elasticsearch from "./elasticsearch" import { testContainerUtils } from "@budibase/backend-core/tests" import { Knex } from "knex" import TestConfiguration from "../../../tests/utilities/TestConfiguration" @@ -23,22 +24,32 @@ export enum DatabaseName { MARIADB = "mariadb", ORACLE = "oracle", SQS = "sqs", + ELASTICSEARCH = "elasticsearch", } +const DATASOURCE_PLUS = [ + DatabaseName.POSTGRES, + DatabaseName.POSTGRES_LEGACY, + DatabaseName.MYSQL, + DatabaseName.SQL_SERVER, + DatabaseName.MARIADB, + DatabaseName.ORACLE, + DatabaseName.SQS, +] + const providers: Record = { + // datasource_plus entries [DatabaseName.POSTGRES]: postgres.getDatasource, [DatabaseName.POSTGRES_LEGACY]: postgres.getLegacyDatasource, - [DatabaseName.MONGODB]: mongodb.getDatasource, [DatabaseName.MYSQL]: mysql.getDatasource, [DatabaseName.SQL_SERVER]: mssql.getDatasource, [DatabaseName.MARIADB]: mariadb.getDatasource, [DatabaseName.ORACLE]: oracle.getDatasource, [DatabaseName.SQS]: async () => undefined, -} -export interface DatasourceDescribeOpts { - only?: DatabaseName[] - exclude?: DatabaseName[] + // rest + [DatabaseName.ELASTICSEARCH]: elasticsearch.getDatasource, + [DatabaseName.MONGODB]: mongodb.getDatasource, } export interface DatasourceDescribeReturnPromise { @@ -103,6 +114,20 @@ function createDummyTest() { }) } +interface OnlyOpts { + only: DatabaseName[] +} + +interface PlusOpts { + plus: true + exclude?: DatabaseName[] +} + +export type DatasourceDescribeOpts = OnlyOpts | PlusOpts + +// If you ever want to rename this function, be mindful that you will also need +// to modify src/tests/filters/index.js to make sure that we're correctly +// filtering datasource/non-datasource tests in CI. export function datasourceDescribe(opts: DatasourceDescribeOpts) { // tests that call this need a lot longer timeouts jest.setTimeout(120000) @@ -111,17 +136,15 @@ export function datasourceDescribe(opts: DatasourceDescribeOpts) { createDummyTest() } - const { only, exclude } = opts - - if (only && exclude) { - throw new Error("you can only supply one of 'only' or 'exclude'") - } - - let databases = Object.values(DatabaseName) - if (only) { - databases = only - } else if (exclude) { - databases = databases.filter(db => !exclude.includes(db)) + let databases: DatabaseName[] = [] + if ("only" in opts) { + databases = opts.only + } else if ("plus" in opts) { + databases = Object.values(DatabaseName) + .filter(db => DATASOURCE_PLUS.includes(db)) + .filter(db => !opts.exclude?.includes(db)) + } else { + throw new Error("invalid options") } if (process.env.DATASOURCE) { @@ -156,6 +179,7 @@ export function datasourceDescribe(opts: DatasourceDescribeOpts) { isMSSQL: dbName === DatabaseName.SQL_SERVER, isOracle: dbName === DatabaseName.ORACLE, isMariaDB: dbName === DatabaseName.MARIADB, + isElasticsearch: dbName === DatabaseName.ELASTICSEARCH, })) } diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index b424c3707d..eaf495e25f 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -10,16 +10,13 @@ import { import { search } from "../../../../../sdk/app/rows/search" import { generator } from "@budibase/backend-core/tests" -import { - DatabaseName, - datasourceDescribe, -} from "../../../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../../../integrations/tests/utils" import { tableForDatasource } from "../../../../../tests/utilities/structures" // These test cases are only for things that cannot be tested through the API // (e.g. limiting searches to returning specific fields). If it's possible to // test through the API, it should be done there instead. -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( From 48674833fcca04598f3549b580e1c511a3670541 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Feb 2025 15:09:39 +0000 Subject: [PATCH 3/7] Correct use of dsProvider in elasticsearch.spec.ts. --- .../server/src/integrations/tests/elasticsearch.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/integrations/tests/elasticsearch.spec.ts b/packages/server/src/integrations/tests/elasticsearch.spec.ts index 14a272d610..bcf8def1e9 100644 --- a/packages/server/src/integrations/tests/elasticsearch.spec.ts +++ b/packages/server/src/integrations/tests/elasticsearch.spec.ts @@ -1,20 +1,20 @@ import { Datasource } from "@budibase/types" import { ElasticsearchConfig, ElasticSearchIntegration } from "../elasticsearch" -import * as elasticsearch from "../tests/utils/elasticsearch" import { generator } from "@budibase/backend-core/tests" import { DatabaseName, datasourceDescribe } from "./utils" const describes = datasourceDescribe({ only: [DatabaseName.ELASTICSEARCH] }) if (describes.length) { - describe.each(describes)("Elasticsearch Integration", () => { + describe.each(describes)("Elasticsearch Integration", ({ dsProvider }) => { let datasource: Datasource let integration: ElasticSearchIntegration let index: string beforeAll(async () => { - datasource = await elasticsearch.getDatasource() + const ds = await dsProvider() + datasource = ds.datasource! }) beforeEach(() => { From 9f0da2a65a1252a9754afa2806d07fac30cd5e60 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Feb 2025 15:12:35 +0000 Subject: [PATCH 4/7] Document elasticsearch container params. --- .../src/integrations/tests/utils/elasticsearch.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/server/src/integrations/tests/utils/elasticsearch.ts b/packages/server/src/integrations/tests/utils/elasticsearch.ts index a7229c3f60..a2ea22f73b 100644 --- a/packages/server/src/integrations/tests/utils/elasticsearch.ts +++ b/packages/server/src/integrations/tests/utils/elasticsearch.ts @@ -13,15 +13,26 @@ export async function getDatasource(): Promise { new GenericContainer(ELASTICSEARCH_IMAGE) .withExposedPorts(9200) .withEnvironment({ + // We need to set the discovery type to single-node to avoid the + // cluster waiting for other nodes to join before starting up. "discovery.type": "single-node", + // We disable security to avoid having to do any auth against the + // container, and to disable SSL. With SSL enabled it uses a self + // signed certificate that we'd have to ignore anyway. "xpack.security.enabled": "false", }) .withWaitStrategy( Wait.forHttp( + // Single node clusters never reach status green, so we wait for + // yellow instead. "/_cluster/health?wait_for_status=yellow&timeout=10s", 9200 ).withStartupTimeout(60000) ) + // We gave the container a tmpfs data directory. Without this, I found + // that the default data directory was very small and the container + // easily filled it up. This caused the cluster to go into a red status + // and stop responding to requests. .withTmpFs({ "/usr/share/elasticsearch/data": "rw" }) ) } From 3be31652524b9849e62b650cf1bd26ccf6769fa1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Feb 2025 15:22:32 +0000 Subject: [PATCH 5/7] Remove mocks from static.spec.ts --- .../src/api/routes/tests/static.spec.ts | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/routes/tests/static.spec.ts b/packages/server/src/api/routes/tests/static.spec.ts index 872085c382..6ec7b54a7f 100644 --- a/packages/server/src/api/routes/tests/static.spec.ts +++ b/packages/server/src/api/routes/tests/static.spec.ts @@ -1,11 +1,3 @@ -// Directly mock the AWS SDK -jest.mock("@aws-sdk/s3-request-presigner", () => ({ - getSignedUrl: jest.fn(() => { - return `http://example.com` - }), -})) -jest.mock("@aws-sdk/client-s3") - import { Datasource, SourceName } from "@budibase/types" import { setEnv } from "../../../environment" import { getRequest, getConfig, afterAll as _afterAll } from "./utilities" @@ -92,7 +84,17 @@ describe("/static", () => { .set(config.defaultHeaders()) .expect("Content-Type", /json/) .expect(200) - expect(res.body.signedUrl).toEqual("http://example.com") + + expect(res.body.signedUrl).toStartWith( + "https://foo.s3.eu-west-1.amazonaws.com/bar?" + ) + expect(res.body.signedUrl).toContain("X-Amz-Algorithm=AWS4-HMAC-SHA256") + expect(res.body.signedUrl).toContain("X-Amz-Credential=bb") + expect(res.body.signedUrl).toContain("X-Amz-Date=") + expect(res.body.signedUrl).toContain("X-Amz-Signature=") + expect(res.body.signedUrl).toContain("X-Amz-Expires=900") + expect(res.body.signedUrl).toContain("X-Amz-SignedHeaders=host") + expect(res.body.publicUrl).toEqual( `https://${bucket}.s3.eu-west-1.amazonaws.com/${key}` ) From 7a63890c1f94a13839a76370066c765b651afadd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 25 Feb 2025 15:32:29 +0000 Subject: [PATCH 6/7] Adding zoom capabilities to QR-code reader to help with smaller qr codes. --- packages/client/manifest.json | 6 ++++++ packages/client/package.json | 2 +- .../client/src/components/app/forms/CodeScanner.svelte | 8 ++++++++ .../src/components/app/forms/CodeScannerField.svelte | 2 ++ yarn.lock | 8 ++++---- 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/client/manifest.json b/packages/client/manifest.json index a2d29d3020..886ad49650 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -4492,6 +4492,12 @@ } ] }, + { + "type": "text", + "label": "Zoom level", + "key": "defaultZoom", + "defaultValue": "1" + }, { "type": "event", "label": "On change", diff --git a/packages/client/package.json b/packages/client/package.json index 72be403698..a7ea04fd0b 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -28,7 +28,7 @@ "apexcharts": "^3.48.0", "dayjs": "^1.10.8", "downloadjs": "1.4.7", - "html5-qrcode": "^2.2.1", + "html5-qrcode": "^2.3.8", "leaflet": "^1.7.1", "sanitize-html": "^2.13.0", "screenfull": "^6.0.1", diff --git a/packages/client/src/components/app/forms/CodeScanner.svelte b/packages/client/src/components/app/forms/CodeScanner.svelte index 008c0f5727..5c5d6ff73b 100644 --- a/packages/client/src/components/app/forms/CodeScanner.svelte +++ b/packages/client/src/components/app/forms/CodeScanner.svelte @@ -20,6 +20,7 @@ export let beepFrequency = 2637 export let customFrequency = 1046 export let preferredCamera = "environment" + export let defaultZoom = 1 export let validator const dispatch = createEventDispatcher() @@ -58,6 +59,13 @@ html5QrCode .start(cameraSetting, cameraConfig, onScanSuccess) .then(() => { + if (defaultZoom > 1) { + const cameraOptions = html5QrCode.getRunningTrackCameraCapabilities() + const zoom = cameraOptions.zoomFeature() + if (zoom.isSupported()) { + zoom.apply(defaultZoom) + } + } resolve({ initialised: true }) }) .catch(err => { diff --git a/packages/client/src/components/app/forms/CodeScannerField.svelte b/packages/client/src/components/app/forms/CodeScannerField.svelte index 7c9948554a..dd9e986804 100644 --- a/packages/client/src/components/app/forms/CodeScannerField.svelte +++ b/packages/client/src/components/app/forms/CodeScannerField.svelte @@ -17,6 +17,7 @@ export let beepFrequency export let customFrequency export let preferredCamera + export let defaultZoom export let helpText = null let fieldState @@ -56,6 +57,7 @@ {beepFrequency} {customFrequency} {preferredCamera} + {defaultZoom} validator={fieldState.validator} /> {/if} diff --git a/yarn.lock b/yarn.lock index ceae41458c..64810291c4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12768,10 +12768,10 @@ html-tag@^2.0.0: is-self-closing "^1.0.1" kind-of "^6.0.0" -html5-qrcode@^2.2.1: - version "2.3.7" - resolved "https://registry.yarnpkg.com/html5-qrcode/-/html5-qrcode-2.3.7.tgz#09ed2ca7473a47bd551088c15fcfcb7cb409a5be" - integrity sha512-Jmlok9Ynm49hgVXkdupWryf8o430proIFoQsRl1LmTg4Rq461W72omylR9yw9tsEMtswMEw3wacUM5y0agOBQA== +html5-qrcode@^2.3.8: + version "2.3.8" + resolved "https://registry.yarnpkg.com/html5-qrcode/-/html5-qrcode-2.3.8.tgz#0b0cdf7a9926cfd4be530e13a51db47592adfa0d" + integrity sha512-jsr4vafJhwoLVEDW3n1KvPnCCXWaQfRng0/EEYk1vNcQGcG/htAdhJX0be8YyqMoSz7+hZvOZSTAepsabiuhiQ== htmlparser2@^8.0.0: version "8.0.1" From a1670c309f53a95f3e890cdfd6298c3db08ab778 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 25 Feb 2025 15:42:50 +0000 Subject: [PATCH 7/7] Linting. --- packages/client/src/components/app/forms/CodeScanner.svelte | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/client/src/components/app/forms/CodeScanner.svelte b/packages/client/src/components/app/forms/CodeScanner.svelte index 5c5d6ff73b..1a046d7168 100644 --- a/packages/client/src/components/app/forms/CodeScanner.svelte +++ b/packages/client/src/components/app/forms/CodeScanner.svelte @@ -60,7 +60,8 @@ .start(cameraSetting, cameraConfig, onScanSuccess) .then(() => { if (defaultZoom > 1) { - const cameraOptions = html5QrCode.getRunningTrackCameraCapabilities() + const cameraOptions = + html5QrCode.getRunningTrackCameraCapabilities() const zoom = cameraOptions.zoomFeature() if (zoom.isSupported()) { zoom.apply(defaultZoom)