Merge branch 'master' of github.com:budibase/budibase into plugin-typing

.github/workflows/budibase_ci.yml

@@ -30,7 +30,7 @@ env:
 
 jobs:
   lint:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -47,7 +47,7 @@ jobs:
       - run: yarn lint
 
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -76,7 +76,7 @@ jobs:
           fi
 
   helm-lint:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -88,7 +88,7 @@ jobs:
       - run: cd charts/budibase && helm lint .
 
   test-libraries:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -122,7 +122,7 @@ jobs:
           fi
 
   test-worker:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -151,11 +151,22 @@ jobs:
           yarn test --verbose --reporters=default --reporters=github-actions
 
   test-server:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         datasource:
-          [mssql, mysql, postgres, postgres_legacy, mongodb, mariadb, oracle, sqs, none]
+          [
+            mssql,
+            mysql,
+            postgres,
+            postgres_legacy,
+            mongodb,
+            mariadb,
+            oracle,
+            sqs,
+            elasticsearch,
+            none,
+          ]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -192,6 +203,8 @@ jobs:
             docker pull budibase/oracle-database:23.2-slim-faststart
           elif [ "${{ matrix.datasource }}" == "postgres_legacy" ]; then
             docker pull postgres:9.5.25
+          elif [ "${{ matrix.datasource }}" == "elasticsearch" ]; then
+            docker pull elasticsearch@${{ steps.dotenv.outputs.ELASTICSEARCH_SHA }}
           fi
           docker pull minio/minio &
           docker pull redis &
@@ -240,7 +253,7 @@ jobs:
           yarn test --filter $FILTER --verbose --reporters=default --reporters=github-actions
 
   check-pro-submodule:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
     steps:
       - name: Checkout repo and submodules
@@ -299,7 +312,7 @@ jobs:
           fi
 
   check-lockfile:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
     steps:
       - name: Checkout repo

hosting/single/Dockerfile

@@ -92,7 +92,7 @@ COPY hosting/single/ssh/sshd_config /etc/
 COPY hosting/single/ssh/ssh_setup.sh /tmp
 
 #  setup letsencrypt certificate
-RUN apt-get install -y certbot python3-certbot-nginx
+RUN apt-get update && apt-get install -y certbot python3-certbot-nginx
 COPY hosting/letsencrypt /app/letsencrypt
 RUN chmod +x /app/letsencrypt/certificate-request.sh /app/letsencrypt/certificate-renew.sh
 

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.4.17",
+  "version": "3.4.20",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -90,7 +90,7 @@
   $: requestEval(runtimeExpression, context, snippets)
   $: bindingHelpers = new BindingHelpers(getCaretPosition, insertAtPos)
 
-  $: bindingOptions = bindingsToCompletions(bindings, editorMode)
+  $: bindingOptions = bindingsToCompletions(enrichedBindings, editorMode)
   $: helperOptions = allowHelpers ? getHelperCompletions(editorMode) : []
   $: snippetsOptions =
     usingJS && useSnippets && snippets?.length ? snippets : []

packages/client/manifest.json

@@ -4492,6 +4492,12 @@
           }
         ]
       },
+      {
+        "type": "text",
+        "label": "Zoom level",
+        "key": "defaultZoom",
+        "defaultValue": "1"
+      },
       {
         "type": "event",
         "label": "On change",

packages/client/package.json

@@ -28,7 +28,7 @@
     "apexcharts": "^3.48.0",
     "dayjs": "^1.10.8",
     "downloadjs": "1.4.7",
-    "html5-qrcode": "^2.2.1",
+    "html5-qrcode": "^2.3.8",
     "leaflet": "^1.7.1",
     "sanitize-html": "^2.13.0",
     "screenfull": "^6.0.1",

packages/client/src/components/app/forms/CodeScanner.svelte

@@ -20,6 +20,7 @@
   export let beepFrequency = 2637
   export let customFrequency = 1046
   export let preferredCamera = "environment"
+  export let defaultZoom = 1
   export let validator
 
   const dispatch = createEventDispatcher()
@@ -58,6 +59,14 @@
       html5QrCode
         .start(cameraSetting, cameraConfig, onScanSuccess)
         .then(() => {
+          if (defaultZoom > 1) {
+            const cameraOptions =
+              html5QrCode.getRunningTrackCameraCapabilities()
+            const zoom = cameraOptions.zoomFeature()
+            if (zoom.isSupported()) {
+              zoom.apply(defaultZoom)
+            }
+          }
           resolve({ initialised: true })
         })
         .catch(err => {

packages/client/src/components/app/forms/CodeScannerField.svelte

@@ -17,6 +17,7 @@
   export let beepFrequency
   export let customFrequency
   export let preferredCamera
+  export let defaultZoom
   export let helpText = null
 
   let fieldState
@@ -56,6 +57,7 @@
       {beepFrequency}
       {customFrequency}
       {preferredCamera}
+      {defaultZoom}
       validator={fieldState.validator}
     />
   {/if}

packages/server/__mocks__/@elastic/elasticsearch.ts

@@ -1,24 +0,0 @@
-const elastic: any = {}
-
-elastic.Client = function () {
-  this.index = jest.fn().mockResolvedValue({ body: [] })
-  this.search = jest.fn().mockResolvedValue({
-    body: {
-      hits: {
-        hits: [
-          {
-            _source: {
-              name: "test",
-            },
-          },
-        ],
-      },
-    },
-  })
-  this.update = jest.fn().mockResolvedValue({ body: [] })
-  this.delete = jest.fn().mockResolvedValue({ body: [] })
-
-  this.close = jest.fn()
-}
-
-module.exports = elastic

packages/server/datasource-sha.env

@@ -1,5 +1,6 @@
-MSSQL_SHA=sha256:3b913841850a4d57fcfcb798be06acc88ea0f2acc5418bc0c140a43e91c4a545
+MSSQL_SHA=sha256:d252932ef839c24c61c1139cc98f69c85ca774fa7c6bfaaa0015b7eb02b9dc87
 MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe
 POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e
 MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d
 MARIADB_SHA=sha256:e59ba8783bf7bc02a4779f103bb0d8751ac0e10f9471089709608377eded7aa8
+ELASTICSEARCH_SHA=sha256:9a6443f55243f6acbfeb4a112d15eb3b9aac74bf25e0e39fa19b3ddd3a6879d0

packages/server/src/api/routes/tests/datasource.spec.ts

@@ -165,7 +165,8 @@ describe("/datasources", () => {
 })
 
 const descriptions = datasourceDescribe({
-  exclude: [DatabaseName.MONGODB, DatabaseName.SQS],
+  plus: true,
+  exclude: [DatabaseName.SQS],
 })
 
 if (descriptions.length) {
@@ -590,7 +591,8 @@ if (descriptions.length) {
 }
 
 const datasources = datasourceDescribe({
-  exclude: [DatabaseName.MONGODB, DatabaseName.SQS, DatabaseName.ORACLE],
+  plus: true,
+  exclude: [DatabaseName.SQS, DatabaseName.ORACLE],
 })
 
 if (datasources.length) {

packages/server/src/api/routes/tests/queries/generic-sql.spec.ts

@@ -9,7 +9,8 @@ import { Knex } from "knex"
 import { generator } from "@budibase/backend-core/tests"
 
 const descriptions = datasourceDescribe({
-  exclude: [DatabaseName.MONGODB, DatabaseName.SQS],
+  plus: true,
+  exclude: [DatabaseName.SQS],
 })
 
 if (descriptions.length) {

packages/server/src/api/routes/tests/row.spec.ts

@@ -1,9 +1,6 @@
 import * as setup from "./utilities"
 
-import {
-  DatabaseName,
-  datasourceDescribe,
-} from "../../../integrations/tests/utils"
+import { datasourceDescribe } from "../../../integrations/tests/utils"
 
 import tk from "timekeeper"
 import emitter from "../../../../src/events"
@@ -80,7 +77,7 @@ function encodeJS(binding: string) {
   return `{{ js "${Buffer.from(binding).toString("base64")}"}}`
 }
 
-const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
+const descriptions = datasourceDescribe({ plus: true })
 
 if (descriptions.length) {
   describe.each(descriptions)(

packages/server/src/api/routes/tests/search.spec.ts

@@ -1,8 +1,5 @@
 import { tableForDatasource } from "../../../tests/utilities/structures"
-import {
-  DatabaseName,
-  datasourceDescribe,
-} from "../../../integrations/tests/utils"
+import { datasourceDescribe } from "../../../integrations/tests/utils"
 import {
   context,
   db as dbCore,
@@ -60,7 +57,7 @@ jest.mock("@budibase/pro", () => ({
   },
 }))
 
-const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
+const descriptions = datasourceDescribe({ plus: true })
 
 if (descriptions.length) {
   describe.each(descriptions)(

packages/server/src/api/routes/tests/static.spec.ts

@@ -1,11 +1,3 @@
-// Directly mock the AWS SDK
-jest.mock("@aws-sdk/s3-request-presigner", () => ({
-  getSignedUrl: jest.fn(() => {
-    return `http://example.com`
-  }),
-}))
-jest.mock("@aws-sdk/client-s3")
-
 import { Datasource, SourceName } from "@budibase/types"
 import { setEnv } from "../../../environment"
 import { getRequest, getConfig, afterAll as _afterAll } from "./utilities"
@@ -92,7 +84,17 @@ describe("/static", () => {
           .set(config.defaultHeaders())
           .expect("Content-Type", /json/)
           .expect(200)
-        expect(res.body.signedUrl).toEqual("http://example.com")
+
+        expect(res.body.signedUrl).toStartWith(
+          "https://foo.s3.eu-west-1.amazonaws.com/bar?"
+        )
+        expect(res.body.signedUrl).toContain("X-Amz-Algorithm=AWS4-HMAC-SHA256")
+        expect(res.body.signedUrl).toContain("X-Amz-Credential=bb")
+        expect(res.body.signedUrl).toContain("X-Amz-Date=")
+        expect(res.body.signedUrl).toContain("X-Amz-Signature=")
+        expect(res.body.signedUrl).toContain("X-Amz-Expires=900")
+        expect(res.body.signedUrl).toContain("X-Amz-SignedHeaders=host")
+
         expect(res.body.publicUrl).toEqual(
           `https://${bucket}.s3.eu-west-1.amazonaws.com/${key}`
         )

packages/server/src/api/routes/tests/table.spec.ts

@@ -28,17 +28,14 @@ import * as setup from "./utilities"
 import * as uuid from "uuid"
 
 import { generator } from "@budibase/backend-core/tests"
-import {
-  DatabaseName,
-  datasourceDescribe,
-} from "../../../integrations/tests/utils"
+import { datasourceDescribe } from "../../../integrations/tests/utils"
 import { tableForDatasource } from "../../../tests/utilities/structures"
 import timekeeper from "timekeeper"
 
 const { basicTable } = setup.structures
 const ISO_REGEX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/
 
-const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
+const descriptions = datasourceDescribe({ plus: true })
 
 if (descriptions.length) {
   describe.each(descriptions)(

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -37,17 +37,14 @@ import {
   ViewV2Type,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
-import {
-  DatabaseName,
-  datasourceDescribe,
-} from "../../../integrations/tests/utils"
+import { datasourceDescribe } from "../../../integrations/tests/utils"
 import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
 import { context, db, events, roles, setEnv } from "@budibase/backend-core"
 import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai"
 import nock from "nock"
 
-const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
+const descriptions = datasourceDescribe({ plus: true })
 
 if (descriptions.length) {
   describe.each(descriptions)(

packages/server/src/automations/tests/steps/executeQuery.spec.ts

@@ -9,7 +9,8 @@ import { generator } from "@budibase/backend-core/tests"
 import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
 
 const descriptions = datasourceDescribe({
-  exclude: [DatabaseName.MONGODB, DatabaseName.SQS],
+  plus: true,
+  exclude: [DatabaseName.SQS],
 })
 
 if (descriptions.length) {

packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts

@@ -1,4 +1,3 @@
-import { SendEmailResponse } from "@budibase/types"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 import * as workerRequests from "../../../utilities/workerRequests"
 
@@ -6,18 +5,17 @@ jest.mock("../../../utilities/workerRequests", () => ({
   sendSmtpEmail: jest.fn(),
 }))
 
-function generateResponse(to: string, from: string): SendEmailResponse {
+function generateResponse(to: string, from: string) {
   return {
-    message: `Email sent to ${to}.`,
-    accepted: [to],
-    envelope: {
-      from: from,
-      to: [to],
+    success: true,
+    response: {
+      accepted: [to],
+      envelope: {
+        from: from,
+        to: [to],
+      },
+      message: `Email sent to ${to}.`,
     },
-    messageId: "messageId",
-    pending: [],
-    rejected: [],
-    response: "response",
   }
 }
 

packages/server/src/integrations/elasticsearch.ts

@@ -10,7 +10,7 @@ import {
 import { Client, ClientOptions } from "@elastic/elasticsearch"
 import { HOST_ADDRESS } from "./utils"
 
-interface ElasticsearchConfig {
+export interface ElasticsearchConfig {
   url: string
   ssl?: boolean
   ca?: string
@@ -99,9 +99,9 @@ const SCHEMA: Integration = {
   },
 }
 
-class ElasticSearchIntegration implements IntegrationBase {
+export class ElasticSearchIntegration implements IntegrationBase {
   private config: ElasticsearchConfig
-  private client
+  private client: Client
 
   constructor(config: ElasticsearchConfig) {
     this.config = config
@@ -132,20 +132,23 @@ class ElasticSearchIntegration implements IntegrationBase {
     }
   }
 
-  async create(query: { index: string; json: object }) {
-    const { index, json } = query
+  async create(query: {
+    index: string
+    json: object
+    extra?: Record<string, string>
+  }) {
+    const { index, json, extra } = query
 
     try {
       const result = await this.client.index({
         index,
         body: json,
+        ...extra,
       })
       return result.body
     } catch (err) {
       console.error("Error writing to elasticsearch", err)
       throw err
-    } finally {
-      await this.client.close()
     }
   }
 
@@ -160,41 +163,46 @@ class ElasticSearchIntegration implements IntegrationBase {
     } catch (err) {
       console.error("Error querying elasticsearch", err)
       throw err
-    } finally {
-      await this.client.close()
     }
   }
 
-  async update(query: { id: string; index: string; json: object }) {
-    const { id, index, json } = query
+  async update(query: {
+    id: string
+    index: string
+    json: object
+    extra?: Record<string, string>
+  }) {
+    const { id, index, json, extra } = query
     try {
       const result = await this.client.update({
         id,
         index,
         body: json,
+        ...extra,
       })
       return result.body
     } catch (err) {
       console.error("Error querying elasticsearch", err)
       throw err
-    } finally {
-      await this.client.close()
     }
   }
 
-  async delete(query: { id: string; index: string }) {
-    const { id, index } = query
+  async delete(query: {
+    id: string
+    index: string
+    extra?: Record<string, string>
+  }) {
+    const { id, index, extra } = query
     try {
       const result = await this.client.delete({
         id,
         index,
+        ...extra,
       })
       return result.body
     } catch (err) {
       console.error("Error deleting from elasticsearch", err)
       throw err
-    } finally {
-      await this.client.close()
     }
   }
 }

packages/server/src/integrations/tests/elasticsearch.spec.ts

@@ -1,83 +1,81 @@
-import { default as ElasticSearchIntegration } from "../elasticsearch"
+import { Datasource } from "@budibase/types"
+import { ElasticsearchConfig, ElasticSearchIntegration } from "../elasticsearch"
+import { generator } from "@budibase/backend-core/tests"
+import { DatabaseName, datasourceDescribe } from "./utils"
 
-jest.mock("@elastic/elasticsearch")
+const describes = datasourceDescribe({ only: [DatabaseName.ELASTICSEARCH] })
 
-class TestConfiguration {
-  integration: any
+if (describes.length) {
+  describe.each(describes)("Elasticsearch Integration", ({ dsProvider }) => {
+    let datasource: Datasource
+    let integration: ElasticSearchIntegration
 
-  constructor(config: any = {}) {
-    this.integration = new ElasticSearchIntegration.integration(config)
-  }
+    let index: string
+
+    beforeAll(async () => {
+      const ds = await dsProvider()
+      datasource = ds.datasource!
+    })
+
+    beforeEach(() => {
+      index = generator.guid()
+      integration = new ElasticSearchIntegration(
+        datasource.config! as ElasticsearchConfig
+      )
+    })
+
+    it("can create a record", async () => {
+      await integration.create({
+        index,
+        json: { name: "Hello" },
+        extra: { refresh: "true" },
+      })
+      const records = await integration.read({
+        index,
+        json: { query: { match_all: {} } },
+      })
+      expect(records).toEqual([{ name: "Hello" }])
+    })
+
+    it("can update a record", async () => {
+      const create = await integration.create({
+        index,
+        json: { name: "Hello" },
+        extra: { refresh: "true" },
+      })
+
+      await integration.update({
+        id: create._id,
+        index,
+        json: { doc: { name: "World" } },
+        extra: { refresh: "true" },
+      })
+
+      const records = await integration.read({
+        index,
+        json: { query: { match_all: {} } },
+      })
+      expect(records).toEqual([{ name: "World" }])
+    })
+
+    it("can delete a record", async () => {
+      const create = await integration.create({
+        index,
+        json: { name: "Hello" },
+        extra: { refresh: "true" },
+      })
+
+      await integration.delete({
+        id: create._id,
+        index,
+        extra: { refresh: "true" },
+      })
+
+      const records = await integration.read({
+        index,
+        json: { query: { match_all: {} } },
+      })
+      expect(records).toEqual([])
+    })
+  })
 }
-
-describe("Elasticsearch Integration", () => {
-  let config: any
-  let indexName = "Users"
-
-  beforeEach(() => {
-    config = new TestConfiguration()
-  })
-
-  it("calls the create method with the correct params", async () => {
-    const body = {
-      name: "Hello",
-    }
-    await config.integration.create({
-      index: indexName,
-      json: body,
-    })
-    expect(config.integration.client.index).toHaveBeenCalledWith({
-      index: indexName,
-      body,
-    })
-  })
-
-  it("calls the read method with the correct params", async () => {
-    const body = {
-      query: {
-        term: {
-          name: "kimchy",
-        },
-      },
-    }
-    const response = await config.integration.read({
-      index: indexName,
-      json: body,
-    })
-    expect(config.integration.client.search).toHaveBeenCalledWith({
-      index: indexName,
-      body,
-    })
-    expect(response).toEqual(expect.any(Array))
-  })
-
-  it("calls the update method with the correct params", async () => {
-    const body = {
-      name: "updated",
-    }
-
-    const response = await config.integration.update({
-      id: "1234",
-      index: indexName,
-      json: body,
-    })
-
-    expect(config.integration.client.update).toHaveBeenCalledWith({
-      id: "1234",
-      index: indexName,
-      body,
-    })
-    expect(response).toEqual(expect.any(Array))
-  })
-
-  it("calls the delete method with the correct params", async () => {
-    const body = {
-      id: "1234",
-    }
-
-    const response = await config.integration.delete(body)
-
-    expect(config.integration.client.delete).toHaveBeenCalledWith(body)
-    expect(response).toEqual(expect.any(Array))
-  })
-})

packages/server/src/integrations/tests/utils/elasticsearch.ts

@@ -0,0 +1,54 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait } from "testcontainers"
+import { testContainerUtils } from "@budibase/backend-core/tests"
+import { startContainer } from "."
+import { ELASTICSEARCH_IMAGE } from "./images"
+import { ElasticsearchConfig } from "../../elasticsearch"
+
+let ports: Promise<testContainerUtils.Port[]>
+
+export async function getDatasource(): Promise<Datasource> {
+  if (!ports) {
+    ports = startContainer(
+      new GenericContainer(ELASTICSEARCH_IMAGE)
+        .withExposedPorts(9200)
+        .withEnvironment({
+          // We need to set the discovery type to single-node to avoid the
+          // cluster waiting for other nodes to join before starting up.
+          "discovery.type": "single-node",
+          // We disable security to avoid having to do any auth against the
+          // container, and to disable SSL. With SSL enabled it uses a self
+          // signed certificate that we'd have to ignore anyway.
+          "xpack.security.enabled": "false",
+        })
+        .withWaitStrategy(
+          Wait.forHttp(
+            // Single node clusters never reach status green, so we wait for
+            // yellow instead.
+            "/_cluster/health?wait_for_status=yellow&timeout=10s",
+            9200
+          ).withStartupTimeout(60000)
+        )
+        // We gave the container a tmpfs data directory. Without this, I found
+        // that the default data directory was very small and the container
+        // easily filled it up. This caused the cluster to go into a red status
+        // and stop responding to requests.
+        .withTmpFs({ "/usr/share/elasticsearch/data": "rw" })
+    )
+  }
+
+  const port = (await ports).find(x => x.container === 9200)?.host
+  if (!port) {
+    throw new Error("Elasticsearch port not found")
+  }
+
+  const config: ElasticsearchConfig = {
+    url: `http://127.0.0.1:${port}`,
+  }
+
+  return {
+    type: "datasource",
+    source: SourceName.ELASTICSEARCH,
+    config,
+  }
+}

packages/server/src/integrations/tests/utils/images.ts

@@ -12,3 +12,4 @@ export const POSTGRES_IMAGE = `postgres@${process.env.POSTGRES_SHA}`
 export const POSTGRES_LEGACY_IMAGE = `postgres:9.5.25`
 export const MONGODB_IMAGE = `mongo@${process.env.MONGODB_SHA}`
 export const MARIADB_IMAGE = `mariadb@${process.env.MARIADB_SHA}`
+export const ELASTICSEARCH_IMAGE = `elasticsearch@${process.env.ELASTICSEARCH_SHA}`

packages/server/src/integrations/tests/utils/index.ts

@@ -6,6 +6,7 @@ import * as mysql from "./mysql"
 import * as mssql from "./mssql"
 import * as mariadb from "./mariadb"
 import * as oracle from "./oracle"
+import * as elasticsearch from "./elasticsearch"
 import { testContainerUtils } from "@budibase/backend-core/tests"
 import { Knex } from "knex"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
@@ -23,22 +24,32 @@ export enum DatabaseName {
   MARIADB = "mariadb",
   ORACLE = "oracle",
   SQS = "sqs",
+  ELASTICSEARCH = "elasticsearch",
 }
 
+const DATASOURCE_PLUS = [
+  DatabaseName.POSTGRES,
+  DatabaseName.POSTGRES_LEGACY,
+  DatabaseName.MYSQL,
+  DatabaseName.SQL_SERVER,
+  DatabaseName.MARIADB,
+  DatabaseName.ORACLE,
+  DatabaseName.SQS,
+]
+
 const providers: Record<DatabaseName, DatasourceProvider> = {
+  // datasource_plus entries
   [DatabaseName.POSTGRES]: postgres.getDatasource,
   [DatabaseName.POSTGRES_LEGACY]: postgres.getLegacyDatasource,
-  [DatabaseName.MONGODB]: mongodb.getDatasource,
   [DatabaseName.MYSQL]: mysql.getDatasource,
   [DatabaseName.SQL_SERVER]: mssql.getDatasource,
   [DatabaseName.MARIADB]: mariadb.getDatasource,
   [DatabaseName.ORACLE]: oracle.getDatasource,
   [DatabaseName.SQS]: async () => undefined,
-}
 
-export interface DatasourceDescribeOpts {
-  only?: DatabaseName[]
-  exclude?: DatabaseName[]
+  // rest
+  [DatabaseName.ELASTICSEARCH]: elasticsearch.getDatasource,
+  [DatabaseName.MONGODB]: mongodb.getDatasource,
 }
 
 export interface DatasourceDescribeReturnPromise {
@@ -103,6 +114,20 @@ function createDummyTest() {
   })
 }
 
+interface OnlyOpts {
+  only: DatabaseName[]
+}
+
+interface PlusOpts {
+  plus: true
+  exclude?: DatabaseName[]
+}
+
+export type DatasourceDescribeOpts = OnlyOpts | PlusOpts
+
+// If you ever want to rename this function, be mindful that you will also need
+// to modify src/tests/filters/index.js to make sure that we're correctly
+// filtering datasource/non-datasource tests in CI.
 export function datasourceDescribe(opts: DatasourceDescribeOpts) {
   // tests that call this need a lot longer timeouts
   jest.setTimeout(120000)
@@ -111,17 +136,15 @@ export function datasourceDescribe(opts: DatasourceDescribeOpts) {
     createDummyTest()
   }
 
-  const { only, exclude } = opts
-
-  if (only && exclude) {
-    throw new Error("you can only supply one of 'only' or 'exclude'")
-  }
-
-  let databases = Object.values(DatabaseName)
-  if (only) {
-    databases = only
-  } else if (exclude) {
-    databases = databases.filter(db => !exclude.includes(db))
+  let databases: DatabaseName[] = []
+  if ("only" in opts) {
+    databases = opts.only
+  } else if ("plus" in opts) {
+    databases = Object.values(DatabaseName)
+      .filter(db => DATASOURCE_PLUS.includes(db))
+      .filter(db => !opts.exclude?.includes(db))
+  } else {
+    throw new Error("invalid options")
   }
 
   if (process.env.DATASOURCE) {
@@ -156,6 +179,7 @@ export function datasourceDescribe(opts: DatasourceDescribeOpts) {
     isMSSQL: dbName === DatabaseName.SQL_SERVER,
     isOracle: dbName === DatabaseName.ORACLE,
     isMariaDB: dbName === DatabaseName.MARIADB,
+    isElasticsearch: dbName === DatabaseName.ELASTICSEARCH,
   }))
 }
 

packages/server/src/integrations/tests/utils/mssql.ts

@@ -23,7 +23,7 @@ export async function getDatasource(): Promise<Datasource> {
         })
         .withWaitStrategy(
           Wait.forSuccessfulCommand(
-            "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Password_123 -q 'SELECT 1'"
+            "/opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P Password_123 -q 'SELECT 1'"
           ).withStartupTimeout(20000)
         )
     )
@@ -44,7 +44,8 @@ export async function getDatasource(): Promise<Datasource> {
       user: "sa",
       password: "Password_123",
       options: {
-        encrypt: false,
+        encrypt: true,
+        trustServerCertificate: true,
       },
     },
   }

packages/server/src/sdk/app/rows/search/tests/search.spec.ts

@@ -10,16 +10,13 @@ import {
 import { search } from "../../../../../sdk/app/rows/search"
 import { generator } from "@budibase/backend-core/tests"
 
-import {
-  DatabaseName,
-  datasourceDescribe,
-} from "../../../../../integrations/tests/utils"
+import { datasourceDescribe } from "../../../../../integrations/tests/utils"
 import { tableForDatasource } from "../../../../../tests/utilities/structures"
 
 // These test cases are only for things that cannot be tested through the API
 // (e.g. limiting searches to returning specific fields). If it's possible to
 // test through the API, it should be done there instead.
-const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
+const descriptions = datasourceDescribe({ plus: true })
 
 if (descriptions.length) {
   describe.each(descriptions)(

packages/server/src/utilities/workerRequests.ts

@@ -8,15 +8,7 @@ import {
   logging,
   env as coreEnv,
 } from "@budibase/backend-core"
-import {
-  Ctx,
-  User,
-  EmailInvite,
-  EmailAttachment,
-  SendEmailResponse,
-  SendEmailRequest,
-  EmailTemplatePurpose,
-} from "@budibase/types"
+import { Ctx, User, EmailInvite, EmailAttachment } from "@budibase/types"
 
 interface Request {
   ctx?: Ctx
@@ -118,23 +110,25 @@ export async function sendSmtpEmail({
   invite?: EmailInvite
 }) {
   // tenant ID will be set in header
-  const request: SendEmailRequest = {
-    email: to,
-    from,
-    contents,
-    subject,
-    cc,
-    bcc,
-    purpose: EmailTemplatePurpose.CUSTOM,
-    automation,
-    invite,
-    attachments,
-  }
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`),
-    createRequest({ method: "POST", body: request })
+    createRequest({
+      method: "POST",
+      body: {
+        email: to,
+        from,
+        contents,
+        subject,
+        cc,
+        bcc,
+        purpose: "custom",
+        automation,
+        invite,
+        attachments,
+      },
+    })
   )
-  return (await checkResponse(response, "send email")) as SendEmailResponse
+  return checkResponse(response, "send email")
 }
 
 export async function removeAppFromUserRoles(ctx: Ctx, appId: string) {

packages/types/package.json

@@ -17,7 +17,6 @@
     "@budibase/nano": "10.1.5",
     "@types/json-schema": "^7.0.15",
     "@types/koa": "2.13.4",
-    "@types/nodemailer": "^6.4.17",
     "@types/redlock": "4.0.7",
     "koa-useragent": "^4.1.0",
     "rimraf": "3.0.2",

packages/types/src/api/web/global/email.ts

@@ -1,5 +1,4 @@
 import { EmailAttachment, EmailInvite } from "../../../documents"
-import SMTPTransport from "nodemailer/lib/smtp-transport"
 
 export enum EmailTemplatePurpose {
   CORE = "core",
@@ -13,17 +12,17 @@ export enum EmailTemplatePurpose {
 export interface SendEmailRequest {
   workspaceId?: string
   email: string
-  userId?: string
+  userId: string
   purpose: EmailTemplatePurpose
   contents?: string
   from?: string
   subject: string
-  cc?: string
-  bcc?: string
+  cc?: boolean
+  bcc?: boolean
   automation?: boolean
   invite?: EmailInvite
   attachments?: EmailAttachment[]
 }
-export interface SendEmailResponse extends SMTPTransport.SentMessageInfo {
+export interface SendEmailResponse extends Record<string, any> {
   message: string
 }

packages/types/src/documents/app/automation/automation.ts

@@ -1,10 +1,10 @@
 import { Document } from "../../document"
 import { User } from "../../global"
+import { ReadStream } from "fs"
 import { Row } from "../row"
 import { Table } from "../table"
 import { AutomationStep, AutomationTrigger } from "./schema"
 import { ContextEmitter } from "../../../sdk"
-import { Readable } from "stream"
 
 export enum AutomationIOType {
   OBJECT = "object",
@@ -108,8 +108,8 @@ export interface SendEmailOpts {
   subject: string
   // info Pass in a structure of information to be stored alongside the invitation.
   info?: any
-  cc?: string
-  bcc?: string
+  cc?: boolean
+  bcc?: boolean
   automation?: boolean
   invite?: EmailInvite
   attachments?: EmailAttachment[]
@@ -269,7 +269,7 @@ export type AutomationAttachment = {
 
 export type AutomationAttachmentContent = {
   filename: string
-  content: Readable
+  content: ReadStream | NodeJS.ReadableStream
 }
 
 export type BucketedContent = AutomationAttachmentContent & {

packages/worker/package.json

@@ -86,7 +86,6 @@
     "@types/koa__router": "12.0.4",
     "@types/lodash": "4.14.200",
     "@types/node-fetch": "2.6.4",
-    "@types/nodemailer": "^6.4.17",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.14",
     "@types/uuid": "8.3.4",

packages/worker/src/api/controllers/global/email.ts

@@ -24,13 +24,10 @@ export async function sendEmail(
     invite,
     attachments,
   } = ctx.request.body
-  let user: User | undefined = undefined
+  let user: any
   if (userId) {
     const db = tenancy.getGlobalDB()
-    user = await db.tryGet<User>(userId)
-  }
-  if (!user) {
-    ctx.throw(404, "User not found.")
+    user = await db.get<User>(userId)
   }
   const response = await sendEmailFn(email, purpose, {
     workspaceId,

packages/worker/src/utilities/email.ts

@@ -13,8 +13,7 @@ import { configs, cache, objectStore } from "@budibase/backend-core"
 import ical from "ical-generator"
 import _ from "lodash"
 
-import nodemailer from "nodemailer"
-import SMTPTransport from "nodemailer/lib/smtp-transport"
+const nodemailer = require("nodemailer")
 
 const TEST_MODE = env.ENABLE_EMAIL_TEST_MODE && env.isDev()
 const TYPE = TemplateType.EMAIL
@@ -27,7 +26,7 @@ const FULL_EMAIL_PURPOSES = [
 ]
 
 function createSMTPTransport(config?: SMTPInnerConfig) {
-  let options: SMTPTransport.Options
+  let options: any
   let secure = config?.secure
   // default it if not specified
   if (secure == null) {
@@ -162,7 +161,7 @@ export async function sendEmail(
   const code = await getLinkCode(purpose, email, opts.user, opts?.info)
   let context = await getSettingsTemplateContext(purpose, code)
 
-  let message: Parameters<typeof transport.sendMail>[0] = {
+  let message: any = {
     from: opts?.from || config?.from,
     html: await buildEmail(purpose, email, context, {
       user: opts?.user,

yarn.lock

@@ -2785,9 +2785,9 @@
     through2 "^2.0.0"
 
 "@budibase/pro@npm:@budibase/pro@latest":
-  version "3.4.16"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.16.tgz#c482a400e27b7e89ca73092c4c81bdeac1d24581"
-  integrity sha512-8ECnqOh9jQ10KlQEwmKPFcoVGE+2gGgSybj+vbshwDp1zAW76doyMR2DMNjEatNpWVnpoMnTkDWtE9aqQ5v0vQ==
+  version "3.4.12"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.12.tgz#60e630944de4e2de970a04179d8f0f57d48ce75e"
+  integrity sha512-msUBmcWxRDg+ugjZvd27XudERQqtQRdiARsO8MaDVTcp5ejIXgshEIVVshHOCj3hcbRblw9pXvBIMI53iTMUsA==
   dependencies:
     "@anthropic-ai/sdk" "^0.27.3"
     "@budibase/backend-core" "*"
@@ -6775,13 +6775,6 @@
   dependencies:
     undici-types "~6.19.2"
 
-"@types/nodemailer@^6.4.17":
-  version "6.4.17"
-  resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.17.tgz#5c82a42aee16a3dd6ea31446a1bd6a447f1ac1a4"
-  integrity sha512-I9CCaIp6DTldEg7vyUTZi8+9Vo0hi1/T8gv3C89yk1rSAAzoKQ8H8ki/jBYJSFoH/BisgLP8tkZMlQ91CIquww==
-  dependencies:
-    "@types/node" "*"
-
 "@types/normalize-package-data@^2.4.0":
   version "2.4.1"
   resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301"
@@ -12789,10 +12782,10 @@ html-tag@^2.0.0:
     is-self-closing "^1.0.1"
     kind-of "^6.0.0"
 
-html5-qrcode@^2.2.1:
-  version "2.3.7"
-  resolved "https://registry.yarnpkg.com/html5-qrcode/-/html5-qrcode-2.3.7.tgz#09ed2ca7473a47bd551088c15fcfcb7cb409a5be"
-  integrity sha512-Jmlok9Ynm49hgVXkdupWryf8o430proIFoQsRl1LmTg4Rq461W72omylR9yw9tsEMtswMEw3wacUM5y0agOBQA==
+html5-qrcode@^2.3.8:
+  version "2.3.8"
+  resolved "https://registry.yarnpkg.com/html5-qrcode/-/html5-qrcode-2.3.8.tgz#0b0cdf7a9926cfd4be530e13a51db47592adfa0d"
+  integrity sha512-jsr4vafJhwoLVEDW3n1KvPnCCXWaQfRng0/EEYk1vNcQGcG/htAdhJX0be8YyqMoSz7+hZvOZSTAepsabiuhiQ==
 
 htmlparser2@^8.0.0:
   version "8.0.1"