diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index e294e50e8d..2e7851b338 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -30,7 +30,7 @@ env: jobs: lint: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -47,7 +47,7 @@ jobs: - run: yarn lint build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -76,7 +76,7 @@ jobs: fi helm-lint: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -88,7 +88,7 @@ jobs: - run: cd charts/budibase && helm lint . test-libraries: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -122,7 +122,7 @@ jobs: fi test-worker: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -151,11 +151,22 @@ jobs: yarn test --verbose --reporters=default --reporters=github-actions test-server: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: datasource: - [mssql, mysql, postgres, postgres_legacy, mongodb, mariadb, oracle, sqs, none] + [ + mssql, + mysql, + postgres, + postgres_legacy, + mongodb, + mariadb, + oracle, + sqs, + elasticsearch, + none, + ] steps: - name: Checkout repo uses: actions/checkout@v4 @@ -192,6 +203,8 @@ jobs: docker pull budibase/oracle-database:23.2-slim-faststart elif [ "${{ matrix.datasource }}" == "postgres_legacy" ]; then docker pull postgres:9.5.25 + elif [ "${{ matrix.datasource }}" == "elasticsearch" ]; then + docker pull elasticsearch@${{ steps.dotenv.outputs.ELASTICSEARCH_SHA }} fi docker pull minio/minio & docker pull redis & @@ -240,7 +253,7 @@ jobs: yarn test --filter $FILTER --verbose --reporters=default --reporters=github-actions check-pro-submodule: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') steps: - name: Checkout repo and submodules @@ -299,7 +312,7 @@ jobs: fi check-lockfile: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') steps: - name: Checkout repo diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 1f449e7376..75b1a12d12 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -92,7 +92,7 @@ COPY hosting/single/ssh/sshd_config /etc/ COPY hosting/single/ssh/ssh_setup.sh /tmp # setup letsencrypt certificate -RUN apt-get install -y certbot python3-certbot-nginx +RUN apt-get update && apt-get install -y certbot python3-certbot-nginx COPY hosting/letsencrypt /app/letsencrypt RUN chmod +x /app/letsencrypt/certificate-request.sh /app/letsencrypt/certificate-renew.sh diff --git a/lerna.json b/lerna.json index 8ea860e3c4..b391c8e05c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.4.17", + "version": "3.4.20", "npmClient": "yarn", "concurrency": 20, "command": { diff --git a/packages/builder/src/components/common/bindings/BindingPanel.svelte b/packages/builder/src/components/common/bindings/BindingPanel.svelte index 9ebbc91309..3715719565 100644 --- a/packages/builder/src/components/common/bindings/BindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/BindingPanel.svelte @@ -90,7 +90,7 @@ $: requestEval(runtimeExpression, context, snippets) $: bindingHelpers = new BindingHelpers(getCaretPosition, insertAtPos) - $: bindingOptions = bindingsToCompletions(bindings, editorMode) + $: bindingOptions = bindingsToCompletions(enrichedBindings, editorMode) $: helperOptions = allowHelpers ? getHelperCompletions(editorMode) : [] $: snippetsOptions = usingJS && useSnippets && snippets?.length ? snippets : [] diff --git a/packages/client/manifest.json b/packages/client/manifest.json index a2d29d3020..886ad49650 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -4492,6 +4492,12 @@ } ] }, + { + "type": "text", + "label": "Zoom level", + "key": "defaultZoom", + "defaultValue": "1" + }, { "type": "event", "label": "On change", diff --git a/packages/client/package.json b/packages/client/package.json index 72be403698..a7ea04fd0b 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -28,7 +28,7 @@ "apexcharts": "^3.48.0", "dayjs": "^1.10.8", "downloadjs": "1.4.7", - "html5-qrcode": "^2.2.1", + "html5-qrcode": "^2.3.8", "leaflet": "^1.7.1", "sanitize-html": "^2.13.0", "screenfull": "^6.0.1", diff --git a/packages/client/src/components/app/forms/CodeScanner.svelte b/packages/client/src/components/app/forms/CodeScanner.svelte index 008c0f5727..1a046d7168 100644 --- a/packages/client/src/components/app/forms/CodeScanner.svelte +++ b/packages/client/src/components/app/forms/CodeScanner.svelte @@ -20,6 +20,7 @@ export let beepFrequency = 2637 export let customFrequency = 1046 export let preferredCamera = "environment" + export let defaultZoom = 1 export let validator const dispatch = createEventDispatcher() @@ -58,6 +59,14 @@ html5QrCode .start(cameraSetting, cameraConfig, onScanSuccess) .then(() => { + if (defaultZoom > 1) { + const cameraOptions = + html5QrCode.getRunningTrackCameraCapabilities() + const zoom = cameraOptions.zoomFeature() + if (zoom.isSupported()) { + zoom.apply(defaultZoom) + } + } resolve({ initialised: true }) }) .catch(err => { diff --git a/packages/client/src/components/app/forms/CodeScannerField.svelte b/packages/client/src/components/app/forms/CodeScannerField.svelte index 7c9948554a..dd9e986804 100644 --- a/packages/client/src/components/app/forms/CodeScannerField.svelte +++ b/packages/client/src/components/app/forms/CodeScannerField.svelte @@ -17,6 +17,7 @@ export let beepFrequency export let customFrequency export let preferredCamera + export let defaultZoom export let helpText = null let fieldState @@ -56,6 +57,7 @@ {beepFrequency} {customFrequency} {preferredCamera} + {defaultZoom} validator={fieldState.validator} /> {/if} diff --git a/packages/server/__mocks__/@elastic/elasticsearch.ts b/packages/server/__mocks__/@elastic/elasticsearch.ts deleted file mode 100644 index 5e13437f29..0000000000 --- a/packages/server/__mocks__/@elastic/elasticsearch.ts +++ /dev/null @@ -1,24 +0,0 @@ -const elastic: any = {} - -elastic.Client = function () { - this.index = jest.fn().mockResolvedValue({ body: [] }) - this.search = jest.fn().mockResolvedValue({ - body: { - hits: { - hits: [ - { - _source: { - name: "test", - }, - }, - ], - }, - }, - }) - this.update = jest.fn().mockResolvedValue({ body: [] }) - this.delete = jest.fn().mockResolvedValue({ body: [] }) - - this.close = jest.fn() -} - -module.exports = elastic diff --git a/packages/server/datasource-sha.env b/packages/server/datasource-sha.env index 61249d530c..69750793ce 100644 --- a/packages/server/datasource-sha.env +++ b/packages/server/datasource-sha.env @@ -1,5 +1,6 @@ -MSSQL_SHA=sha256:3b913841850a4d57fcfcb798be06acc88ea0f2acc5418bc0c140a43e91c4a545 +MSSQL_SHA=sha256:d252932ef839c24c61c1139cc98f69c85ca774fa7c6bfaaa0015b7eb02b9dc87 MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d MARIADB_SHA=sha256:e59ba8783bf7bc02a4779f103bb0d8751ac0e10f9471089709608377eded7aa8 +ELASTICSEARCH_SHA=sha256:9a6443f55243f6acbfeb4a112d15eb3b9aac74bf25e0e39fa19b3ddd3a6879d0 \ No newline at end of file diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts index 21e9effa77..12029c39c4 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.ts +++ b/packages/server/src/api/routes/tests/datasource.spec.ts @@ -165,7 +165,8 @@ describe("/datasources", () => { }) const descriptions = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS], + plus: true, + exclude: [DatabaseName.SQS], }) if (descriptions.length) { @@ -590,7 +591,8 @@ if (descriptions.length) { } const datasources = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS, DatabaseName.ORACLE], + plus: true, + exclude: [DatabaseName.SQS, DatabaseName.ORACLE], }) if (datasources.length) { diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 863f5b65e0..4a545b253e 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -9,7 +9,8 @@ import { Knex } from "knex" import { generator } from "@budibase/backend-core/tests" const descriptions = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS], + plus: true, + exclude: [DatabaseName.SQS], }) if (descriptions.length) { diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 87002670b7..b349a1df8a 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1,9 +1,6 @@ import * as setup from "./utilities" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import tk from "timekeeper" import emitter from "../../../../src/events" @@ -80,7 +77,7 @@ function encodeJS(binding: string) { return `{{ js "${Buffer.from(binding).toString("base64")}"}}` } -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 76ce4a0243..caa651f3bb 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1,8 +1,5 @@ import { tableForDatasource } from "../../../tests/utilities/structures" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import { context, db as dbCore, @@ -60,7 +57,7 @@ jest.mock("@budibase/pro", () => ({ }, })) -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/api/routes/tests/static.spec.ts b/packages/server/src/api/routes/tests/static.spec.ts index 872085c382..6ec7b54a7f 100644 --- a/packages/server/src/api/routes/tests/static.spec.ts +++ b/packages/server/src/api/routes/tests/static.spec.ts @@ -1,11 +1,3 @@ -// Directly mock the AWS SDK -jest.mock("@aws-sdk/s3-request-presigner", () => ({ - getSignedUrl: jest.fn(() => { - return `http://example.com` - }), -})) -jest.mock("@aws-sdk/client-s3") - import { Datasource, SourceName } from "@budibase/types" import { setEnv } from "../../../environment" import { getRequest, getConfig, afterAll as _afterAll } from "./utilities" @@ -92,7 +84,17 @@ describe("/static", () => { .set(config.defaultHeaders()) .expect("Content-Type", /json/) .expect(200) - expect(res.body.signedUrl).toEqual("http://example.com") + + expect(res.body.signedUrl).toStartWith( + "https://foo.s3.eu-west-1.amazonaws.com/bar?" + ) + expect(res.body.signedUrl).toContain("X-Amz-Algorithm=AWS4-HMAC-SHA256") + expect(res.body.signedUrl).toContain("X-Amz-Credential=bb") + expect(res.body.signedUrl).toContain("X-Amz-Date=") + expect(res.body.signedUrl).toContain("X-Amz-Signature=") + expect(res.body.signedUrl).toContain("X-Amz-Expires=900") + expect(res.body.signedUrl).toContain("X-Amz-SignedHeaders=host") + expect(res.body.publicUrl).toEqual( `https://${bucket}.s3.eu-west-1.amazonaws.com/${key}` ) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 2a7f039ff5..29b576d16a 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -28,17 +28,14 @@ import * as setup from "./utilities" import * as uuid from "uuid" import { generator } from "@budibase/backend-core/tests" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import { tableForDatasource } from "../../../tests/utilities/structures" import timekeeper from "timekeeper" const { basicTable } = setup.structures const ISO_REGEX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/ -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 9531737d30..7eed1811d9 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -37,17 +37,14 @@ import { ViewV2Type, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import merge from "lodash/merge" import { quotas } from "@budibase/pro" import { context, db, events, roles, setEnv } from "@budibase/backend-core" import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai" import nock from "nock" -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/automations/tests/steps/executeQuery.spec.ts b/packages/server/src/automations/tests/steps/executeQuery.spec.ts index dff3580b7e..a51d335902 100644 --- a/packages/server/src/automations/tests/steps/executeQuery.spec.ts +++ b/packages/server/src/automations/tests/steps/executeQuery.spec.ts @@ -9,7 +9,8 @@ import { generator } from "@budibase/backend-core/tests" import { createAutomationBuilder } from "../utilities/AutomationTestBuilder" const descriptions = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS], + plus: true, + exclude: [DatabaseName.SQS], }) if (descriptions.length) { diff --git a/packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts b/packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts index 7aff612a97..7452239dfa 100644 --- a/packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts +++ b/packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts @@ -1,4 +1,3 @@ -import { SendEmailResponse } from "@budibase/types" import TestConfiguration from "../../../tests/utilities/TestConfiguration" import * as workerRequests from "../../../utilities/workerRequests" @@ -6,18 +5,17 @@ jest.mock("../../../utilities/workerRequests", () => ({ sendSmtpEmail: jest.fn(), })) -function generateResponse(to: string, from: string): SendEmailResponse { +function generateResponse(to: string, from: string) { return { - message: `Email sent to ${to}.`, - accepted: [to], - envelope: { - from: from, - to: [to], + success: true, + response: { + accepted: [to], + envelope: { + from: from, + to: [to], + }, + message: `Email sent to ${to}.`, }, - messageId: "messageId", - pending: [], - rejected: [], - response: "response", } } diff --git a/packages/server/src/integrations/elasticsearch.ts b/packages/server/src/integrations/elasticsearch.ts index af03baaef1..10f9d1e697 100644 --- a/packages/server/src/integrations/elasticsearch.ts +++ b/packages/server/src/integrations/elasticsearch.ts @@ -10,7 +10,7 @@ import { import { Client, ClientOptions } from "@elastic/elasticsearch" import { HOST_ADDRESS } from "./utils" -interface ElasticsearchConfig { +export interface ElasticsearchConfig { url: string ssl?: boolean ca?: string @@ -99,9 +99,9 @@ const SCHEMA: Integration = { }, } -class ElasticSearchIntegration implements IntegrationBase { +export class ElasticSearchIntegration implements IntegrationBase { private config: ElasticsearchConfig - private client + private client: Client constructor(config: ElasticsearchConfig) { this.config = config @@ -132,20 +132,23 @@ class ElasticSearchIntegration implements IntegrationBase { } } - async create(query: { index: string; json: object }) { - const { index, json } = query + async create(query: { + index: string + json: object + extra?: Record + }) { + const { index, json, extra } = query try { const result = await this.client.index({ index, body: json, + ...extra, }) return result.body } catch (err) { console.error("Error writing to elasticsearch", err) throw err - } finally { - await this.client.close() } } @@ -160,41 +163,46 @@ class ElasticSearchIntegration implements IntegrationBase { } catch (err) { console.error("Error querying elasticsearch", err) throw err - } finally { - await this.client.close() } } - async update(query: { id: string; index: string; json: object }) { - const { id, index, json } = query + async update(query: { + id: string + index: string + json: object + extra?: Record + }) { + const { id, index, json, extra } = query try { const result = await this.client.update({ id, index, body: json, + ...extra, }) return result.body } catch (err) { console.error("Error querying elasticsearch", err) throw err - } finally { - await this.client.close() } } - async delete(query: { id: string; index: string }) { - const { id, index } = query + async delete(query: { + id: string + index: string + extra?: Record + }) { + const { id, index, extra } = query try { const result = await this.client.delete({ id, index, + ...extra, }) return result.body } catch (err) { console.error("Error deleting from elasticsearch", err) throw err - } finally { - await this.client.close() } } } diff --git a/packages/server/src/integrations/tests/elasticsearch.spec.ts b/packages/server/src/integrations/tests/elasticsearch.spec.ts index f8a1dd8013..bcf8def1e9 100644 --- a/packages/server/src/integrations/tests/elasticsearch.spec.ts +++ b/packages/server/src/integrations/tests/elasticsearch.spec.ts @@ -1,83 +1,81 @@ -import { default as ElasticSearchIntegration } from "../elasticsearch" +import { Datasource } from "@budibase/types" +import { ElasticsearchConfig, ElasticSearchIntegration } from "../elasticsearch" +import { generator } from "@budibase/backend-core/tests" +import { DatabaseName, datasourceDescribe } from "./utils" -jest.mock("@elastic/elasticsearch") +const describes = datasourceDescribe({ only: [DatabaseName.ELASTICSEARCH] }) -class TestConfiguration { - integration: any +if (describes.length) { + describe.each(describes)("Elasticsearch Integration", ({ dsProvider }) => { + let datasource: Datasource + let integration: ElasticSearchIntegration - constructor(config: any = {}) { - this.integration = new ElasticSearchIntegration.integration(config) - } + let index: string + + beforeAll(async () => { + const ds = await dsProvider() + datasource = ds.datasource! + }) + + beforeEach(() => { + index = generator.guid() + integration = new ElasticSearchIntegration( + datasource.config! as ElasticsearchConfig + ) + }) + + it("can create a record", async () => { + await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, + }) + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([{ name: "Hello" }]) + }) + + it("can update a record", async () => { + const create = await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, + }) + + await integration.update({ + id: create._id, + index, + json: { doc: { name: "World" } }, + extra: { refresh: "true" }, + }) + + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([{ name: "World" }]) + }) + + it("can delete a record", async () => { + const create = await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, + }) + + await integration.delete({ + id: create._id, + index, + extra: { refresh: "true" }, + }) + + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([]) + }) + }) } - -describe("Elasticsearch Integration", () => { - let config: any - let indexName = "Users" - - beforeEach(() => { - config = new TestConfiguration() - }) - - it("calls the create method with the correct params", async () => { - const body = { - name: "Hello", - } - await config.integration.create({ - index: indexName, - json: body, - }) - expect(config.integration.client.index).toHaveBeenCalledWith({ - index: indexName, - body, - }) - }) - - it("calls the read method with the correct params", async () => { - const body = { - query: { - term: { - name: "kimchy", - }, - }, - } - const response = await config.integration.read({ - index: indexName, - json: body, - }) - expect(config.integration.client.search).toHaveBeenCalledWith({ - index: indexName, - body, - }) - expect(response).toEqual(expect.any(Array)) - }) - - it("calls the update method with the correct params", async () => { - const body = { - name: "updated", - } - - const response = await config.integration.update({ - id: "1234", - index: indexName, - json: body, - }) - - expect(config.integration.client.update).toHaveBeenCalledWith({ - id: "1234", - index: indexName, - body, - }) - expect(response).toEqual(expect.any(Array)) - }) - - it("calls the delete method with the correct params", async () => { - const body = { - id: "1234", - } - - const response = await config.integration.delete(body) - - expect(config.integration.client.delete).toHaveBeenCalledWith(body) - expect(response).toEqual(expect.any(Array)) - }) -}) diff --git a/packages/server/src/integrations/tests/utils/elasticsearch.ts b/packages/server/src/integrations/tests/utils/elasticsearch.ts new file mode 100644 index 0000000000..a2ea22f73b --- /dev/null +++ b/packages/server/src/integrations/tests/utils/elasticsearch.ts @@ -0,0 +1,54 @@ +import { Datasource, SourceName } from "@budibase/types" +import { GenericContainer, Wait } from "testcontainers" +import { testContainerUtils } from "@budibase/backend-core/tests" +import { startContainer } from "." +import { ELASTICSEARCH_IMAGE } from "./images" +import { ElasticsearchConfig } from "../../elasticsearch" + +let ports: Promise + +export async function getDatasource(): Promise { + if (!ports) { + ports = startContainer( + new GenericContainer(ELASTICSEARCH_IMAGE) + .withExposedPorts(9200) + .withEnvironment({ + // We need to set the discovery type to single-node to avoid the + // cluster waiting for other nodes to join before starting up. + "discovery.type": "single-node", + // We disable security to avoid having to do any auth against the + // container, and to disable SSL. With SSL enabled it uses a self + // signed certificate that we'd have to ignore anyway. + "xpack.security.enabled": "false", + }) + .withWaitStrategy( + Wait.forHttp( + // Single node clusters never reach status green, so we wait for + // yellow instead. + "/_cluster/health?wait_for_status=yellow&timeout=10s", + 9200 + ).withStartupTimeout(60000) + ) + // We gave the container a tmpfs data directory. Without this, I found + // that the default data directory was very small and the container + // easily filled it up. This caused the cluster to go into a red status + // and stop responding to requests. + .withTmpFs({ "/usr/share/elasticsearch/data": "rw" }) + ) + } + + const port = (await ports).find(x => x.container === 9200)?.host + if (!port) { + throw new Error("Elasticsearch port not found") + } + + const config: ElasticsearchConfig = { + url: `http://127.0.0.1:${port}`, + } + + return { + type: "datasource", + source: SourceName.ELASTICSEARCH, + config, + } +} diff --git a/packages/server/src/integrations/tests/utils/images.ts b/packages/server/src/integrations/tests/utils/images.ts index 00686412c6..c09b130ea5 100644 --- a/packages/server/src/integrations/tests/utils/images.ts +++ b/packages/server/src/integrations/tests/utils/images.ts @@ -12,3 +12,4 @@ export const POSTGRES_IMAGE = `postgres@${process.env.POSTGRES_SHA}` export const POSTGRES_LEGACY_IMAGE = `postgres:9.5.25` export const MONGODB_IMAGE = `mongo@${process.env.MONGODB_SHA}` export const MARIADB_IMAGE = `mariadb@${process.env.MARIADB_SHA}` +export const ELASTICSEARCH_IMAGE = `elasticsearch@${process.env.ELASTICSEARCH_SHA}` diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index 9e2c4f7e70..08777cab89 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -6,6 +6,7 @@ import * as mysql from "./mysql" import * as mssql from "./mssql" import * as mariadb from "./mariadb" import * as oracle from "./oracle" +import * as elasticsearch from "./elasticsearch" import { testContainerUtils } from "@budibase/backend-core/tests" import { Knex } from "knex" import TestConfiguration from "../../../tests/utilities/TestConfiguration" @@ -23,22 +24,32 @@ export enum DatabaseName { MARIADB = "mariadb", ORACLE = "oracle", SQS = "sqs", + ELASTICSEARCH = "elasticsearch", } +const DATASOURCE_PLUS = [ + DatabaseName.POSTGRES, + DatabaseName.POSTGRES_LEGACY, + DatabaseName.MYSQL, + DatabaseName.SQL_SERVER, + DatabaseName.MARIADB, + DatabaseName.ORACLE, + DatabaseName.SQS, +] + const providers: Record = { + // datasource_plus entries [DatabaseName.POSTGRES]: postgres.getDatasource, [DatabaseName.POSTGRES_LEGACY]: postgres.getLegacyDatasource, - [DatabaseName.MONGODB]: mongodb.getDatasource, [DatabaseName.MYSQL]: mysql.getDatasource, [DatabaseName.SQL_SERVER]: mssql.getDatasource, [DatabaseName.MARIADB]: mariadb.getDatasource, [DatabaseName.ORACLE]: oracle.getDatasource, [DatabaseName.SQS]: async () => undefined, -} -export interface DatasourceDescribeOpts { - only?: DatabaseName[] - exclude?: DatabaseName[] + // rest + [DatabaseName.ELASTICSEARCH]: elasticsearch.getDatasource, + [DatabaseName.MONGODB]: mongodb.getDatasource, } export interface DatasourceDescribeReturnPromise { @@ -103,6 +114,20 @@ function createDummyTest() { }) } +interface OnlyOpts { + only: DatabaseName[] +} + +interface PlusOpts { + plus: true + exclude?: DatabaseName[] +} + +export type DatasourceDescribeOpts = OnlyOpts | PlusOpts + +// If you ever want to rename this function, be mindful that you will also need +// to modify src/tests/filters/index.js to make sure that we're correctly +// filtering datasource/non-datasource tests in CI. export function datasourceDescribe(opts: DatasourceDescribeOpts) { // tests that call this need a lot longer timeouts jest.setTimeout(120000) @@ -111,17 +136,15 @@ export function datasourceDescribe(opts: DatasourceDescribeOpts) { createDummyTest() } - const { only, exclude } = opts - - if (only && exclude) { - throw new Error("you can only supply one of 'only' or 'exclude'") - } - - let databases = Object.values(DatabaseName) - if (only) { - databases = only - } else if (exclude) { - databases = databases.filter(db => !exclude.includes(db)) + let databases: DatabaseName[] = [] + if ("only" in opts) { + databases = opts.only + } else if ("plus" in opts) { + databases = Object.values(DatabaseName) + .filter(db => DATASOURCE_PLUS.includes(db)) + .filter(db => !opts.exclude?.includes(db)) + } else { + throw new Error("invalid options") } if (process.env.DATASOURCE) { @@ -156,6 +179,7 @@ export function datasourceDescribe(opts: DatasourceDescribeOpts) { isMSSQL: dbName === DatabaseName.SQL_SERVER, isOracle: dbName === DatabaseName.ORACLE, isMariaDB: dbName === DatabaseName.MARIADB, + isElasticsearch: dbName === DatabaseName.ELASTICSEARCH, })) } diff --git a/packages/server/src/integrations/tests/utils/mssql.ts b/packages/server/src/integrations/tests/utils/mssql.ts index 548631a987..453c2b8bc8 100644 --- a/packages/server/src/integrations/tests/utils/mssql.ts +++ b/packages/server/src/integrations/tests/utils/mssql.ts @@ -23,7 +23,7 @@ export async function getDatasource(): Promise { }) .withWaitStrategy( Wait.forSuccessfulCommand( - "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Password_123 -q 'SELECT 1'" + "/opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P Password_123 -q 'SELECT 1'" ).withStartupTimeout(20000) ) ) @@ -44,7 +44,8 @@ export async function getDatasource(): Promise { user: "sa", password: "Password_123", options: { - encrypt: false, + encrypt: true, + trustServerCertificate: true, }, }, } diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index b424c3707d..eaf495e25f 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -10,16 +10,13 @@ import { import { search } from "../../../../../sdk/app/rows/search" import { generator } from "@budibase/backend-core/tests" -import { - DatabaseName, - datasourceDescribe, -} from "../../../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../../../integrations/tests/utils" import { tableForDatasource } from "../../../../../tests/utilities/structures" // These test cases are only for things that cannot be tested through the API // (e.g. limiting searches to returning specific fields). If it's possible to // test through the API, it should be done there instead. -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/utilities/workerRequests.ts b/packages/server/src/utilities/workerRequests.ts index dd1493b82f..0f487d9f31 100644 --- a/packages/server/src/utilities/workerRequests.ts +++ b/packages/server/src/utilities/workerRequests.ts @@ -8,15 +8,7 @@ import { logging, env as coreEnv, } from "@budibase/backend-core" -import { - Ctx, - User, - EmailInvite, - EmailAttachment, - SendEmailResponse, - SendEmailRequest, - EmailTemplatePurpose, -} from "@budibase/types" +import { Ctx, User, EmailInvite, EmailAttachment } from "@budibase/types" interface Request { ctx?: Ctx @@ -118,23 +110,25 @@ export async function sendSmtpEmail({ invite?: EmailInvite }) { // tenant ID will be set in header - const request: SendEmailRequest = { - email: to, - from, - contents, - subject, - cc, - bcc, - purpose: EmailTemplatePurpose.CUSTOM, - automation, - invite, - attachments, - } const response = await fetch( checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`), - createRequest({ method: "POST", body: request }) + createRequest({ + method: "POST", + body: { + email: to, + from, + contents, + subject, + cc, + bcc, + purpose: "custom", + automation, + invite, + attachments, + }, + }) ) - return (await checkResponse(response, "send email")) as SendEmailResponse + return checkResponse(response, "send email") } export async function removeAppFromUserRoles(ctx: Ctx, appId: string) { diff --git a/packages/types/package.json b/packages/types/package.json index a6e08ab84c..ee3c059bc9 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -17,7 +17,6 @@ "@budibase/nano": "10.1.5", "@types/json-schema": "^7.0.15", "@types/koa": "2.13.4", - "@types/nodemailer": "^6.4.17", "@types/redlock": "4.0.7", "koa-useragent": "^4.1.0", "rimraf": "3.0.2", diff --git a/packages/types/src/api/web/global/email.ts b/packages/types/src/api/web/global/email.ts index 3d2e007231..a0ca0e8485 100644 --- a/packages/types/src/api/web/global/email.ts +++ b/packages/types/src/api/web/global/email.ts @@ -1,5 +1,4 @@ import { EmailAttachment, EmailInvite } from "../../../documents" -import SMTPTransport from "nodemailer/lib/smtp-transport" export enum EmailTemplatePurpose { CORE = "core", @@ -13,17 +12,17 @@ export enum EmailTemplatePurpose { export interface SendEmailRequest { workspaceId?: string email: string - userId?: string + userId: string purpose: EmailTemplatePurpose contents?: string from?: string subject: string - cc?: string - bcc?: string + cc?: boolean + bcc?: boolean automation?: boolean invite?: EmailInvite attachments?: EmailAttachment[] } -export interface SendEmailResponse extends SMTPTransport.SentMessageInfo { +export interface SendEmailResponse extends Record { message: string } diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts index cfe2ba5147..d5ef35d059 100644 --- a/packages/types/src/documents/app/automation/automation.ts +++ b/packages/types/src/documents/app/automation/automation.ts @@ -1,10 +1,10 @@ import { Document } from "../../document" import { User } from "../../global" +import { ReadStream } from "fs" import { Row } from "../row" import { Table } from "../table" import { AutomationStep, AutomationTrigger } from "./schema" import { ContextEmitter } from "../../../sdk" -import { Readable } from "stream" export enum AutomationIOType { OBJECT = "object", @@ -108,8 +108,8 @@ export interface SendEmailOpts { subject: string // info Pass in a structure of information to be stored alongside the invitation. info?: any - cc?: string - bcc?: string + cc?: boolean + bcc?: boolean automation?: boolean invite?: EmailInvite attachments?: EmailAttachment[] @@ -269,7 +269,7 @@ export type AutomationAttachment = { export type AutomationAttachmentContent = { filename: string - content: Readable + content: ReadStream | NodeJS.ReadableStream } export type BucketedContent = AutomationAttachmentContent & { diff --git a/packages/worker/package.json b/packages/worker/package.json index edaab50d78..28728272ca 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -86,7 +86,6 @@ "@types/koa__router": "12.0.4", "@types/lodash": "4.14.200", "@types/node-fetch": "2.6.4", - "@types/nodemailer": "^6.4.17", "@types/server-destroy": "1.0.1", "@types/supertest": "2.0.14", "@types/uuid": "8.3.4", diff --git a/packages/worker/src/api/controllers/global/email.ts b/packages/worker/src/api/controllers/global/email.ts index ed2d9b5125..ad0fc3fa32 100644 --- a/packages/worker/src/api/controllers/global/email.ts +++ b/packages/worker/src/api/controllers/global/email.ts @@ -24,13 +24,10 @@ export async function sendEmail( invite, attachments, } = ctx.request.body - let user: User | undefined = undefined + let user: any if (userId) { const db = tenancy.getGlobalDB() - user = await db.tryGet(userId) - } - if (!user) { - ctx.throw(404, "User not found.") + user = await db.get(userId) } const response = await sendEmailFn(email, purpose, { workspaceId, diff --git a/packages/worker/src/utilities/email.ts b/packages/worker/src/utilities/email.ts index a2b9c3bfc2..fa9dd7a6fa 100644 --- a/packages/worker/src/utilities/email.ts +++ b/packages/worker/src/utilities/email.ts @@ -13,8 +13,7 @@ import { configs, cache, objectStore } from "@budibase/backend-core" import ical from "ical-generator" import _ from "lodash" -import nodemailer from "nodemailer" -import SMTPTransport from "nodemailer/lib/smtp-transport" +const nodemailer = require("nodemailer") const TEST_MODE = env.ENABLE_EMAIL_TEST_MODE && env.isDev() const TYPE = TemplateType.EMAIL @@ -27,7 +26,7 @@ const FULL_EMAIL_PURPOSES = [ ] function createSMTPTransport(config?: SMTPInnerConfig) { - let options: SMTPTransport.Options + let options: any let secure = config?.secure // default it if not specified if (secure == null) { @@ -162,7 +161,7 @@ export async function sendEmail( const code = await getLinkCode(purpose, email, opts.user, opts?.info) let context = await getSettingsTemplateContext(purpose, code) - let message: Parameters[0] = { + let message: any = { from: opts?.from || config?.from, html: await buildEmail(purpose, email, context, { user: opts?.user, diff --git a/yarn.lock b/yarn.lock index 3e2bd6140b..48eaaf4ecd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2785,9 +2785,9 @@ through2 "^2.0.0" "@budibase/pro@npm:@budibase/pro@latest": - version "3.4.16" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.16.tgz#c482a400e27b7e89ca73092c4c81bdeac1d24581" - integrity sha512-8ECnqOh9jQ10KlQEwmKPFcoVGE+2gGgSybj+vbshwDp1zAW76doyMR2DMNjEatNpWVnpoMnTkDWtE9aqQ5v0vQ== + version "3.4.12" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.12.tgz#60e630944de4e2de970a04179d8f0f57d48ce75e" + integrity sha512-msUBmcWxRDg+ugjZvd27XudERQqtQRdiARsO8MaDVTcp5ejIXgshEIVVshHOCj3hcbRblw9pXvBIMI53iTMUsA== dependencies: "@anthropic-ai/sdk" "^0.27.3" "@budibase/backend-core" "*" @@ -6775,13 +6775,6 @@ dependencies: undici-types "~6.19.2" -"@types/nodemailer@^6.4.17": - version "6.4.17" - resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.17.tgz#5c82a42aee16a3dd6ea31446a1bd6a447f1ac1a4" - integrity sha512-I9CCaIp6DTldEg7vyUTZi8+9Vo0hi1/T8gv3C89yk1rSAAzoKQ8H8ki/jBYJSFoH/BisgLP8tkZMlQ91CIquww== - dependencies: - "@types/node" "*" - "@types/normalize-package-data@^2.4.0": version "2.4.1" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" @@ -12789,10 +12782,10 @@ html-tag@^2.0.0: is-self-closing "^1.0.1" kind-of "^6.0.0" -html5-qrcode@^2.2.1: - version "2.3.7" - resolved "https://registry.yarnpkg.com/html5-qrcode/-/html5-qrcode-2.3.7.tgz#09ed2ca7473a47bd551088c15fcfcb7cb409a5be" - integrity sha512-Jmlok9Ynm49hgVXkdupWryf8o430proIFoQsRl1LmTg4Rq461W72omylR9yw9tsEMtswMEw3wacUM5y0agOBQA== +html5-qrcode@^2.3.8: + version "2.3.8" + resolved "https://registry.yarnpkg.com/html5-qrcode/-/html5-qrcode-2.3.8.tgz#0b0cdf7a9926cfd4be530e13a51db47592adfa0d" + integrity sha512-jsr4vafJhwoLVEDW3n1KvPnCCXWaQfRng0/EEYk1vNcQGcG/htAdhJX0be8YyqMoSz7+hZvOZSTAepsabiuhiQ== htmlparser2@^8.0.0: version "8.0.1"