diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index e294e50e8d..6a9aa3e8d0 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -155,7 +155,18 @@ jobs: strategy: matrix: datasource: - [mssql, mysql, postgres, postgres_legacy, mongodb, mariadb, oracle, sqs, none] + [ + mssql, + mysql, + postgres, + postgres_legacy, + mongodb, + mariadb, + oracle, + sqs, + elasticsearch, + none, + ] steps: - name: Checkout repo uses: actions/checkout@v4 @@ -192,6 +203,8 @@ jobs: docker pull budibase/oracle-database:23.2-slim-faststart elif [ "${{ matrix.datasource }}" == "postgres_legacy" ]; then docker pull postgres:9.5.25 + elif [ "${{ matrix.datasource }}" == "elasticsearch" ]; then + docker pull elasticsearch@${{ steps.dotenv.outputs.ELASTICSEARCH_SHA }} fi docker pull minio/minio & docker pull redis & diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts index 21e9effa77..12029c39c4 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.ts +++ b/packages/server/src/api/routes/tests/datasource.spec.ts @@ -165,7 +165,8 @@ describe("/datasources", () => { }) const descriptions = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS], + plus: true, + exclude: [DatabaseName.SQS], }) if (descriptions.length) { @@ -590,7 +591,8 @@ if (descriptions.length) { } const datasources = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS, DatabaseName.ORACLE], + plus: true, + exclude: [DatabaseName.SQS, DatabaseName.ORACLE], }) if (datasources.length) { diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 863f5b65e0..4a545b253e 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -9,7 +9,8 @@ import { Knex } from "knex" import { generator } from "@budibase/backend-core/tests" const descriptions = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS], + plus: true, + exclude: [DatabaseName.SQS], }) if (descriptions.length) { diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 87002670b7..b349a1df8a 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1,9 +1,6 @@ import * as setup from "./utilities" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import tk from "timekeeper" import emitter from "../../../../src/events" @@ -80,7 +77,7 @@ function encodeJS(binding: string) { return `{{ js "${Buffer.from(binding).toString("base64")}"}}` } -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index ee372914d7..a904c0c469 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1,8 +1,5 @@ import { tableForDatasource } from "../../../tests/utilities/structures" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import { context, db as dbCore, @@ -60,7 +57,7 @@ jest.mock("@budibase/pro", () => ({ }, })) -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 2a7f039ff5..29b576d16a 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -28,17 +28,14 @@ import * as setup from "./utilities" import * as uuid from "uuid" import { generator } from "@budibase/backend-core/tests" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import { tableForDatasource } from "../../../tests/utilities/structures" import timekeeper from "timekeeper" const { basicTable } = setup.structures const ISO_REGEX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/ -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 9531737d30..7eed1811d9 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -37,17 +37,14 @@ import { ViewV2Type, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import merge from "lodash/merge" import { quotas } from "@budibase/pro" import { context, db, events, roles, setEnv } from "@budibase/backend-core" import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai" import nock from "nock" -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/automations/tests/steps/executeQuery.spec.ts b/packages/server/src/automations/tests/steps/executeQuery.spec.ts index dff3580b7e..a51d335902 100644 --- a/packages/server/src/automations/tests/steps/executeQuery.spec.ts +++ b/packages/server/src/automations/tests/steps/executeQuery.spec.ts @@ -9,7 +9,8 @@ import { generator } from "@budibase/backend-core/tests" import { createAutomationBuilder } from "../utilities/AutomationTestBuilder" const descriptions = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS], + plus: true, + exclude: [DatabaseName.SQS], }) if (descriptions.length) { diff --git a/packages/server/src/integrations/tests/elasticsearch.spec.ts b/packages/server/src/integrations/tests/elasticsearch.spec.ts index da9c5600ad..14a272d610 100644 --- a/packages/server/src/integrations/tests/elasticsearch.spec.ts +++ b/packages/server/src/integrations/tests/elasticsearch.spec.ts @@ -2,75 +2,80 @@ import { Datasource } from "@budibase/types" import { ElasticsearchConfig, ElasticSearchIntegration } from "../elasticsearch" import * as elasticsearch from "../tests/utils/elasticsearch" import { generator } from "@budibase/backend-core/tests" +import { DatabaseName, datasourceDescribe } from "./utils" -describe("Elasticsearch Integration", () => { - let datasource: Datasource - let integration: ElasticSearchIntegration +const describes = datasourceDescribe({ only: [DatabaseName.ELASTICSEARCH] }) - let index: string +if (describes.length) { + describe.each(describes)("Elasticsearch Integration", () => { + let datasource: Datasource + let integration: ElasticSearchIntegration - beforeAll(async () => { - datasource = await elasticsearch.getDatasource() + let index: string + + beforeAll(async () => { + datasource = await elasticsearch.getDatasource() + }) + + beforeEach(() => { + index = generator.guid() + integration = new ElasticSearchIntegration( + datasource.config! as ElasticsearchConfig + ) + }) + + it("can create a record", async () => { + await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, + }) + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([{ name: "Hello" }]) + }) + + it("can update a record", async () => { + const create = await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, + }) + + await integration.update({ + id: create._id, + index, + json: { doc: { name: "World" } }, + extra: { refresh: "true" }, + }) + + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([{ name: "World" }]) + }) + + it("can delete a record", async () => { + const create = await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, + }) + + await integration.delete({ + id: create._id, + index, + extra: { refresh: "true" }, + }) + + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([]) + }) }) - - beforeEach(() => { - index = generator.guid() - integration = new ElasticSearchIntegration( - datasource.config! as ElasticsearchConfig - ) - }) - - it("can create a record", async () => { - await integration.create({ - index, - json: { name: "Hello" }, - extra: { refresh: "true" }, - }) - const records = await integration.read({ - index, - json: { query: { match_all: {} } }, - }) - expect(records).toEqual([{ name: "Hello" }]) - }) - - it("can update a record", async () => { - const create = await integration.create({ - index, - json: { name: "Hello" }, - extra: { refresh: "true" }, - }) - - await integration.update({ - id: create._id, - index, - json: { doc: { name: "World" } }, - extra: { refresh: "true" }, - }) - - const records = await integration.read({ - index, - json: { query: { match_all: {} } }, - }) - expect(records).toEqual([{ name: "World" }]) - }) - - it("can delete a record", async () => { - const create = await integration.create({ - index, - json: { name: "Hello" }, - extra: { refresh: "true" }, - }) - - await integration.delete({ - id: create._id, - index, - extra: { refresh: "true" }, - }) - - const records = await integration.read({ - index, - json: { query: { match_all: {} } }, - }) - expect(records).toEqual([]) - }) -}) +} diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index 9e2c4f7e70..08777cab89 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -6,6 +6,7 @@ import * as mysql from "./mysql" import * as mssql from "./mssql" import * as mariadb from "./mariadb" import * as oracle from "./oracle" +import * as elasticsearch from "./elasticsearch" import { testContainerUtils } from "@budibase/backend-core/tests" import { Knex } from "knex" import TestConfiguration from "../../../tests/utilities/TestConfiguration" @@ -23,22 +24,32 @@ export enum DatabaseName { MARIADB = "mariadb", ORACLE = "oracle", SQS = "sqs", + ELASTICSEARCH = "elasticsearch", } +const DATASOURCE_PLUS = [ + DatabaseName.POSTGRES, + DatabaseName.POSTGRES_LEGACY, + DatabaseName.MYSQL, + DatabaseName.SQL_SERVER, + DatabaseName.MARIADB, + DatabaseName.ORACLE, + DatabaseName.SQS, +] + const providers: Record = { + // datasource_plus entries [DatabaseName.POSTGRES]: postgres.getDatasource, [DatabaseName.POSTGRES_LEGACY]: postgres.getLegacyDatasource, - [DatabaseName.MONGODB]: mongodb.getDatasource, [DatabaseName.MYSQL]: mysql.getDatasource, [DatabaseName.SQL_SERVER]: mssql.getDatasource, [DatabaseName.MARIADB]: mariadb.getDatasource, [DatabaseName.ORACLE]: oracle.getDatasource, [DatabaseName.SQS]: async () => undefined, -} -export interface DatasourceDescribeOpts { - only?: DatabaseName[] - exclude?: DatabaseName[] + // rest + [DatabaseName.ELASTICSEARCH]: elasticsearch.getDatasource, + [DatabaseName.MONGODB]: mongodb.getDatasource, } export interface DatasourceDescribeReturnPromise { @@ -103,6 +114,20 @@ function createDummyTest() { }) } +interface OnlyOpts { + only: DatabaseName[] +} + +interface PlusOpts { + plus: true + exclude?: DatabaseName[] +} + +export type DatasourceDescribeOpts = OnlyOpts | PlusOpts + +// If you ever want to rename this function, be mindful that you will also need +// to modify src/tests/filters/index.js to make sure that we're correctly +// filtering datasource/non-datasource tests in CI. export function datasourceDescribe(opts: DatasourceDescribeOpts) { // tests that call this need a lot longer timeouts jest.setTimeout(120000) @@ -111,17 +136,15 @@ export function datasourceDescribe(opts: DatasourceDescribeOpts) { createDummyTest() } - const { only, exclude } = opts - - if (only && exclude) { - throw new Error("you can only supply one of 'only' or 'exclude'") - } - - let databases = Object.values(DatabaseName) - if (only) { - databases = only - } else if (exclude) { - databases = databases.filter(db => !exclude.includes(db)) + let databases: DatabaseName[] = [] + if ("only" in opts) { + databases = opts.only + } else if ("plus" in opts) { + databases = Object.values(DatabaseName) + .filter(db => DATASOURCE_PLUS.includes(db)) + .filter(db => !opts.exclude?.includes(db)) + } else { + throw new Error("invalid options") } if (process.env.DATASOURCE) { @@ -156,6 +179,7 @@ export function datasourceDescribe(opts: DatasourceDescribeOpts) { isMSSQL: dbName === DatabaseName.SQL_SERVER, isOracle: dbName === DatabaseName.ORACLE, isMariaDB: dbName === DatabaseName.MARIADB, + isElasticsearch: dbName === DatabaseName.ELASTICSEARCH, })) } diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index b424c3707d..eaf495e25f 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -10,16 +10,13 @@ import { import { search } from "../../../../../sdk/app/rows/search" import { generator } from "@budibase/backend-core/tests" -import { - DatabaseName, - datasourceDescribe, -} from "../../../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../../../integrations/tests/utils" import { tableForDatasource } from "../../../../../tests/utilities/structures" // These test cases are only for things that cannot be tested through the API // (e.g. limiting searches to returning specific fields). If it's possible to // test through the API, it should be done there instead. -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)(