diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index e294e50e8d..6a9aa3e8d0 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -155,7 +155,18 @@ jobs: strategy: matrix: datasource: - [mssql, mysql, postgres, postgres_legacy, mongodb, mariadb, oracle, sqs, none] + [ + mssql, + mysql, + postgres, + postgres_legacy, + mongodb, + mariadb, + oracle, + sqs, + elasticsearch, + none, + ] steps: - name: Checkout repo uses: actions/checkout@v4 @@ -192,6 +203,8 @@ jobs: docker pull budibase/oracle-database:23.2-slim-faststart elif [ "${{ matrix.datasource }}" == "postgres_legacy" ]; then docker pull postgres:9.5.25 + elif [ "${{ matrix.datasource }}" == "elasticsearch" ]; then + docker pull elasticsearch@${{ steps.dotenv.outputs.ELASTICSEARCH_SHA }} fi docker pull minio/minio & docker pull redis & diff --git a/lerna.json b/lerna.json index bb71d10f41..8ea860e3c4 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.4.16", + "version": "3.4.17", "npmClient": "yarn", "concurrency": 20, "command": { diff --git a/packages/client/manifest.json b/packages/client/manifest.json index a2d29d3020..886ad49650 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -4492,6 +4492,12 @@ } ] }, + { + "type": "text", + "label": "Zoom level", + "key": "defaultZoom", + "defaultValue": "1" + }, { "type": "event", "label": "On change", diff --git a/packages/client/package.json b/packages/client/package.json index 72be403698..a7ea04fd0b 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -28,7 +28,7 @@ "apexcharts": "^3.48.0", "dayjs": "^1.10.8", "downloadjs": "1.4.7", - "html5-qrcode": "^2.2.1", + "html5-qrcode": "^2.3.8", "leaflet": "^1.7.1", "sanitize-html": "^2.13.0", "screenfull": "^6.0.1", diff --git a/packages/client/src/components/app/forms/CodeScanner.svelte b/packages/client/src/components/app/forms/CodeScanner.svelte index 008c0f5727..1a046d7168 100644 --- a/packages/client/src/components/app/forms/CodeScanner.svelte +++ b/packages/client/src/components/app/forms/CodeScanner.svelte @@ -20,6 +20,7 @@ export let beepFrequency = 2637 export let customFrequency = 1046 export let preferredCamera = "environment" + export let defaultZoom = 1 export let validator const dispatch = createEventDispatcher() @@ -58,6 +59,14 @@ html5QrCode .start(cameraSetting, cameraConfig, onScanSuccess) .then(() => { + if (defaultZoom > 1) { + const cameraOptions = + html5QrCode.getRunningTrackCameraCapabilities() + const zoom = cameraOptions.zoomFeature() + if (zoom.isSupported()) { + zoom.apply(defaultZoom) + } + } resolve({ initialised: true }) }) .catch(err => { diff --git a/packages/client/src/components/app/forms/CodeScannerField.svelte b/packages/client/src/components/app/forms/CodeScannerField.svelte index 7c9948554a..dd9e986804 100644 --- a/packages/client/src/components/app/forms/CodeScannerField.svelte +++ b/packages/client/src/components/app/forms/CodeScannerField.svelte @@ -17,6 +17,7 @@ export let beepFrequency export let customFrequency export let preferredCamera + export let defaultZoom export let helpText = null let fieldState @@ -56,6 +57,7 @@ {beepFrequency} {customFrequency} {preferredCamera} + {defaultZoom} validator={fieldState.validator} /> {/if} diff --git a/packages/server/__mocks__/@elastic/elasticsearch.ts b/packages/server/__mocks__/@elastic/elasticsearch.ts deleted file mode 100644 index 5e13437f29..0000000000 --- a/packages/server/__mocks__/@elastic/elasticsearch.ts +++ /dev/null @@ -1,24 +0,0 @@ -const elastic: any = {} - -elastic.Client = function () { - this.index = jest.fn().mockResolvedValue({ body: [] }) - this.search = jest.fn().mockResolvedValue({ - body: { - hits: { - hits: [ - { - _source: { - name: "test", - }, - }, - ], - }, - }, - }) - this.update = jest.fn().mockResolvedValue({ body: [] }) - this.delete = jest.fn().mockResolvedValue({ body: [] }) - - this.close = jest.fn() -} - -module.exports = elastic diff --git a/packages/server/datasource-sha.env b/packages/server/datasource-sha.env index 61249d530c..383fb191f4 100644 --- a/packages/server/datasource-sha.env +++ b/packages/server/datasource-sha.env @@ -3,3 +3,4 @@ MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588eb POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d MARIADB_SHA=sha256:e59ba8783bf7bc02a4779f103bb0d8751ac0e10f9471089709608377eded7aa8 +ELASTICSEARCH_SHA=sha256:9a6443f55243f6acbfeb4a112d15eb3b9aac74bf25e0e39fa19b3ddd3a6879d0 \ No newline at end of file diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts index 21e9effa77..12029c39c4 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.ts +++ b/packages/server/src/api/routes/tests/datasource.spec.ts @@ -165,7 +165,8 @@ describe("/datasources", () => { }) const descriptions = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS], + plus: true, + exclude: [DatabaseName.SQS], }) if (descriptions.length) { @@ -590,7 +591,8 @@ if (descriptions.length) { } const datasources = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS, DatabaseName.ORACLE], + plus: true, + exclude: [DatabaseName.SQS, DatabaseName.ORACLE], }) if (datasources.length) { diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 863f5b65e0..4a545b253e 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -9,7 +9,8 @@ import { Knex } from "knex" import { generator } from "@budibase/backend-core/tests" const descriptions = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS], + plus: true, + exclude: [DatabaseName.SQS], }) if (descriptions.length) { diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 87002670b7..b349a1df8a 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1,9 +1,6 @@ import * as setup from "./utilities" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import tk from "timekeeper" import emitter from "../../../../src/events" @@ -80,7 +77,7 @@ function encodeJS(binding: string) { return `{{ js "${Buffer.from(binding).toString("base64")}"}}` } -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index ee372914d7..caa651f3bb 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1,8 +1,5 @@ import { tableForDatasource } from "../../../tests/utilities/structures" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import { context, db as dbCore, @@ -60,7 +57,7 @@ jest.mock("@budibase/pro", () => ({ }, })) -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( @@ -3553,6 +3550,31 @@ if (descriptions.length) { limit: 1, }).toContainExactly([row]) }) + + isInternal && + describe("search by _id for relations", () => { + it("can filter by the related _id", async () => { + await expectSearch({ + query: { + equal: { "rel._id": row.rel[0]._id }, + }, + }).toContainExactly([row]) + + await expectSearch({ + query: { + equal: { "rel._id": row.rel[1]._id }, + }, + }).toContainExactly([row]) + }) + + it("can filter by the related _id and find nothing", async () => { + await expectSearch({ + query: { + equal: { "rel._id": "rel_none" }, + }, + }).toFindNothing() + }) + }) }) !isInternal && diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 2a7f039ff5..29b576d16a 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -28,17 +28,14 @@ import * as setup from "./utilities" import * as uuid from "uuid" import { generator } from "@budibase/backend-core/tests" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import { tableForDatasource } from "../../../tests/utilities/structures" import timekeeper from "timekeeper" const { basicTable } = setup.structures const ISO_REGEX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/ -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 9531737d30..7eed1811d9 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -37,17 +37,14 @@ import { ViewV2Type, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import merge from "lodash/merge" import { quotas } from "@budibase/pro" import { context, db, events, roles, setEnv } from "@budibase/backend-core" import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai" import nock from "nock" -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/automations/tests/branching.spec.ts b/packages/server/src/automations/tests/branching.spec.ts index bf9b9ce3f8..4572871c44 100644 --- a/packages/server/src/automations/tests/branching.spec.ts +++ b/packages/server/src/automations/tests/branching.spec.ts @@ -1,5 +1,5 @@ import * as automation from "../index" -import { Table, AutomationStatus } from "@budibase/types" +import { Table, AutomationStatus, EmptyFilterOption } from "@budibase/types" import { createAutomationBuilder } from "./utilities/AutomationTestBuilder" import TestConfiguration from "../../tests/utilities/TestConfiguration" @@ -280,4 +280,23 @@ describe("Branching automations", () => { expect(results.steps[2].outputs.message).toContain("Special user") }) + + it("should not fail with empty conditions", async () => { + const results = await createAutomationBuilder(config) + .onAppAction() + .branch({ + specialBranch: { + steps: stepBuilder => stepBuilder.serverLog({ text: "Hello!" }), + condition: { + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + }, + }, + }) + .test({ fields: { test_trigger: true } }) + + expect(results.steps[0].outputs.success).toEqual(false) + expect(results.steps[0].outputs.status).toEqual( + AutomationStatus.NO_CONDITION_MET + ) + }) }) diff --git a/packages/server/src/automations/tests/steps/executeQuery.spec.ts b/packages/server/src/automations/tests/steps/executeQuery.spec.ts index dff3580b7e..a51d335902 100644 --- a/packages/server/src/automations/tests/steps/executeQuery.spec.ts +++ b/packages/server/src/automations/tests/steps/executeQuery.spec.ts @@ -9,7 +9,8 @@ import { generator } from "@budibase/backend-core/tests" import { createAutomationBuilder } from "../utilities/AutomationTestBuilder" const descriptions = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS], + plus: true, + exclude: [DatabaseName.SQS], }) if (descriptions.length) { diff --git a/packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts b/packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts index 7452239dfa..7aff612a97 100644 --- a/packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts +++ b/packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts @@ -1,3 +1,4 @@ +import { SendEmailResponse } from "@budibase/types" import TestConfiguration from "../../../tests/utilities/TestConfiguration" import * as workerRequests from "../../../utilities/workerRequests" @@ -5,17 +6,18 @@ jest.mock("../../../utilities/workerRequests", () => ({ sendSmtpEmail: jest.fn(), })) -function generateResponse(to: string, from: string) { +function generateResponse(to: string, from: string): SendEmailResponse { return { - success: true, - response: { - accepted: [to], - envelope: { - from: from, - to: [to], - }, - message: `Email sent to ${to}.`, + message: `Email sent to ${to}.`, + accepted: [to], + envelope: { + from: from, + to: [to], }, + messageId: "messageId", + pending: [], + rejected: [], + response: "response", } } diff --git a/packages/server/src/integrations/elasticsearch.ts b/packages/server/src/integrations/elasticsearch.ts index af03baaef1..10f9d1e697 100644 --- a/packages/server/src/integrations/elasticsearch.ts +++ b/packages/server/src/integrations/elasticsearch.ts @@ -10,7 +10,7 @@ import { import { Client, ClientOptions } from "@elastic/elasticsearch" import { HOST_ADDRESS } from "./utils" -interface ElasticsearchConfig { +export interface ElasticsearchConfig { url: string ssl?: boolean ca?: string @@ -99,9 +99,9 @@ const SCHEMA: Integration = { }, } -class ElasticSearchIntegration implements IntegrationBase { +export class ElasticSearchIntegration implements IntegrationBase { private config: ElasticsearchConfig - private client + private client: Client constructor(config: ElasticsearchConfig) { this.config = config @@ -132,20 +132,23 @@ class ElasticSearchIntegration implements IntegrationBase { } } - async create(query: { index: string; json: object }) { - const { index, json } = query + async create(query: { + index: string + json: object + extra?: Record + }) { + const { index, json, extra } = query try { const result = await this.client.index({ index, body: json, + ...extra, }) return result.body } catch (err) { console.error("Error writing to elasticsearch", err) throw err - } finally { - await this.client.close() } } @@ -160,41 +163,46 @@ class ElasticSearchIntegration implements IntegrationBase { } catch (err) { console.error("Error querying elasticsearch", err) throw err - } finally { - await this.client.close() } } - async update(query: { id: string; index: string; json: object }) { - const { id, index, json } = query + async update(query: { + id: string + index: string + json: object + extra?: Record + }) { + const { id, index, json, extra } = query try { const result = await this.client.update({ id, index, body: json, + ...extra, }) return result.body } catch (err) { console.error("Error querying elasticsearch", err) throw err - } finally { - await this.client.close() } } - async delete(query: { id: string; index: string }) { - const { id, index } = query + async delete(query: { + id: string + index: string + extra?: Record + }) { + const { id, index, extra } = query try { const result = await this.client.delete({ id, index, + ...extra, }) return result.body } catch (err) { console.error("Error deleting from elasticsearch", err) throw err - } finally { - await this.client.close() } } } diff --git a/packages/server/src/integrations/tests/elasticsearch.spec.ts b/packages/server/src/integrations/tests/elasticsearch.spec.ts index f8a1dd8013..bcf8def1e9 100644 --- a/packages/server/src/integrations/tests/elasticsearch.spec.ts +++ b/packages/server/src/integrations/tests/elasticsearch.spec.ts @@ -1,83 +1,81 @@ -import { default as ElasticSearchIntegration } from "../elasticsearch" +import { Datasource } from "@budibase/types" +import { ElasticsearchConfig, ElasticSearchIntegration } from "../elasticsearch" +import { generator } from "@budibase/backend-core/tests" +import { DatabaseName, datasourceDescribe } from "./utils" -jest.mock("@elastic/elasticsearch") +const describes = datasourceDescribe({ only: [DatabaseName.ELASTICSEARCH] }) -class TestConfiguration { - integration: any +if (describes.length) { + describe.each(describes)("Elasticsearch Integration", ({ dsProvider }) => { + let datasource: Datasource + let integration: ElasticSearchIntegration - constructor(config: any = {}) { - this.integration = new ElasticSearchIntegration.integration(config) - } + let index: string + + beforeAll(async () => { + const ds = await dsProvider() + datasource = ds.datasource! + }) + + beforeEach(() => { + index = generator.guid() + integration = new ElasticSearchIntegration( + datasource.config! as ElasticsearchConfig + ) + }) + + it("can create a record", async () => { + await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, + }) + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([{ name: "Hello" }]) + }) + + it("can update a record", async () => { + const create = await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, + }) + + await integration.update({ + id: create._id, + index, + json: { doc: { name: "World" } }, + extra: { refresh: "true" }, + }) + + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([{ name: "World" }]) + }) + + it("can delete a record", async () => { + const create = await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, + }) + + await integration.delete({ + id: create._id, + index, + extra: { refresh: "true" }, + }) + + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([]) + }) + }) } - -describe("Elasticsearch Integration", () => { - let config: any - let indexName = "Users" - - beforeEach(() => { - config = new TestConfiguration() - }) - - it("calls the create method with the correct params", async () => { - const body = { - name: "Hello", - } - await config.integration.create({ - index: indexName, - json: body, - }) - expect(config.integration.client.index).toHaveBeenCalledWith({ - index: indexName, - body, - }) - }) - - it("calls the read method with the correct params", async () => { - const body = { - query: { - term: { - name: "kimchy", - }, - }, - } - const response = await config.integration.read({ - index: indexName, - json: body, - }) - expect(config.integration.client.search).toHaveBeenCalledWith({ - index: indexName, - body, - }) - expect(response).toEqual(expect.any(Array)) - }) - - it("calls the update method with the correct params", async () => { - const body = { - name: "updated", - } - - const response = await config.integration.update({ - id: "1234", - index: indexName, - json: body, - }) - - expect(config.integration.client.update).toHaveBeenCalledWith({ - id: "1234", - index: indexName, - body, - }) - expect(response).toEqual(expect.any(Array)) - }) - - it("calls the delete method with the correct params", async () => { - const body = { - id: "1234", - } - - const response = await config.integration.delete(body) - - expect(config.integration.client.delete).toHaveBeenCalledWith(body) - expect(response).toEqual(expect.any(Array)) - }) -}) diff --git a/packages/server/src/integrations/tests/utils/elasticsearch.ts b/packages/server/src/integrations/tests/utils/elasticsearch.ts new file mode 100644 index 0000000000..a2ea22f73b --- /dev/null +++ b/packages/server/src/integrations/tests/utils/elasticsearch.ts @@ -0,0 +1,54 @@ +import { Datasource, SourceName } from "@budibase/types" +import { GenericContainer, Wait } from "testcontainers" +import { testContainerUtils } from "@budibase/backend-core/tests" +import { startContainer } from "." +import { ELASTICSEARCH_IMAGE } from "./images" +import { ElasticsearchConfig } from "../../elasticsearch" + +let ports: Promise + +export async function getDatasource(): Promise { + if (!ports) { + ports = startContainer( + new GenericContainer(ELASTICSEARCH_IMAGE) + .withExposedPorts(9200) + .withEnvironment({ + // We need to set the discovery type to single-node to avoid the + // cluster waiting for other nodes to join before starting up. + "discovery.type": "single-node", + // We disable security to avoid having to do any auth against the + // container, and to disable SSL. With SSL enabled it uses a self + // signed certificate that we'd have to ignore anyway. + "xpack.security.enabled": "false", + }) + .withWaitStrategy( + Wait.forHttp( + // Single node clusters never reach status green, so we wait for + // yellow instead. + "/_cluster/health?wait_for_status=yellow&timeout=10s", + 9200 + ).withStartupTimeout(60000) + ) + // We gave the container a tmpfs data directory. Without this, I found + // that the default data directory was very small and the container + // easily filled it up. This caused the cluster to go into a red status + // and stop responding to requests. + .withTmpFs({ "/usr/share/elasticsearch/data": "rw" }) + ) + } + + const port = (await ports).find(x => x.container === 9200)?.host + if (!port) { + throw new Error("Elasticsearch port not found") + } + + const config: ElasticsearchConfig = { + url: `http://127.0.0.1:${port}`, + } + + return { + type: "datasource", + source: SourceName.ELASTICSEARCH, + config, + } +} diff --git a/packages/server/src/integrations/tests/utils/images.ts b/packages/server/src/integrations/tests/utils/images.ts index 00686412c6..c09b130ea5 100644 --- a/packages/server/src/integrations/tests/utils/images.ts +++ b/packages/server/src/integrations/tests/utils/images.ts @@ -12,3 +12,4 @@ export const POSTGRES_IMAGE = `postgres@${process.env.POSTGRES_SHA}` export const POSTGRES_LEGACY_IMAGE = `postgres:9.5.25` export const MONGODB_IMAGE = `mongo@${process.env.MONGODB_SHA}` export const MARIADB_IMAGE = `mariadb@${process.env.MARIADB_SHA}` +export const ELASTICSEARCH_IMAGE = `elasticsearch@${process.env.ELASTICSEARCH_SHA}` diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index 9e2c4f7e70..08777cab89 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -6,6 +6,7 @@ import * as mysql from "./mysql" import * as mssql from "./mssql" import * as mariadb from "./mariadb" import * as oracle from "./oracle" +import * as elasticsearch from "./elasticsearch" import { testContainerUtils } from "@budibase/backend-core/tests" import { Knex } from "knex" import TestConfiguration from "../../../tests/utilities/TestConfiguration" @@ -23,22 +24,32 @@ export enum DatabaseName { MARIADB = "mariadb", ORACLE = "oracle", SQS = "sqs", + ELASTICSEARCH = "elasticsearch", } +const DATASOURCE_PLUS = [ + DatabaseName.POSTGRES, + DatabaseName.POSTGRES_LEGACY, + DatabaseName.MYSQL, + DatabaseName.SQL_SERVER, + DatabaseName.MARIADB, + DatabaseName.ORACLE, + DatabaseName.SQS, +] + const providers: Record = { + // datasource_plus entries [DatabaseName.POSTGRES]: postgres.getDatasource, [DatabaseName.POSTGRES_LEGACY]: postgres.getLegacyDatasource, - [DatabaseName.MONGODB]: mongodb.getDatasource, [DatabaseName.MYSQL]: mysql.getDatasource, [DatabaseName.SQL_SERVER]: mssql.getDatasource, [DatabaseName.MARIADB]: mariadb.getDatasource, [DatabaseName.ORACLE]: oracle.getDatasource, [DatabaseName.SQS]: async () => undefined, -} -export interface DatasourceDescribeOpts { - only?: DatabaseName[] - exclude?: DatabaseName[] + // rest + [DatabaseName.ELASTICSEARCH]: elasticsearch.getDatasource, + [DatabaseName.MONGODB]: mongodb.getDatasource, } export interface DatasourceDescribeReturnPromise { @@ -103,6 +114,20 @@ function createDummyTest() { }) } +interface OnlyOpts { + only: DatabaseName[] +} + +interface PlusOpts { + plus: true + exclude?: DatabaseName[] +} + +export type DatasourceDescribeOpts = OnlyOpts | PlusOpts + +// If you ever want to rename this function, be mindful that you will also need +// to modify src/tests/filters/index.js to make sure that we're correctly +// filtering datasource/non-datasource tests in CI. export function datasourceDescribe(opts: DatasourceDescribeOpts) { // tests that call this need a lot longer timeouts jest.setTimeout(120000) @@ -111,17 +136,15 @@ export function datasourceDescribe(opts: DatasourceDescribeOpts) { createDummyTest() } - const { only, exclude } = opts - - if (only && exclude) { - throw new Error("you can only supply one of 'only' or 'exclude'") - } - - let databases = Object.values(DatabaseName) - if (only) { - databases = only - } else if (exclude) { - databases = databases.filter(db => !exclude.includes(db)) + let databases: DatabaseName[] = [] + if ("only" in opts) { + databases = opts.only + } else if ("plus" in opts) { + databases = Object.values(DatabaseName) + .filter(db => DATASOURCE_PLUS.includes(db)) + .filter(db => !opts.exclude?.includes(db)) + } else { + throw new Error("invalid options") } if (process.env.DATASOURCE) { @@ -156,6 +179,7 @@ export function datasourceDescribe(opts: DatasourceDescribeOpts) { isMSSQL: dbName === DatabaseName.SQL_SERVER, isOracle: dbName === DatabaseName.ORACLE, isMariaDB: dbName === DatabaseName.MARIADB, + isElasticsearch: dbName === DatabaseName.ELASTICSEARCH, })) } diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index ddd32870be..8ab50ea1b7 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -7,6 +7,7 @@ import { } from "@budibase/types" import { cloneDeep } from "lodash/fp" import sdk from "../../../sdk" +import { isInternal } from "../tables/utils" export const removeInvalidFilters = ( filters: SearchFilters, @@ -70,6 +71,10 @@ export const getQueryableFields = async ( opts?: { noRelationships?: boolean } ): Promise => { const result = [] + if (isInternal({ table })) { + result.push("_id") + } + for (const field of Object.keys(table.schema).filter( f => allowedFields.includes(f) && table.schema[f].visible !== false )) { @@ -113,14 +118,13 @@ export const getQueryableFields = async ( return result } - const result = [ - "_id", // Querying by _id is always allowed, even if it's never part of the schema - ] + // Querying by _id is always allowed, even if it's never part of the schema + const result = ["_id"] if (fields == null) { fields = Object.keys(table.schema) } result.push(...(await extractTableFields(table, fields, [table._id!]))) - return result + return Array.from(new Set(result)) } diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index b424c3707d..eaf495e25f 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -10,16 +10,13 @@ import { import { search } from "../../../../../sdk/app/rows/search" import { generator } from "@budibase/backend-core/tests" -import { - DatabaseName, - datasourceDescribe, -} from "../../../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../../../integrations/tests/utils" import { tableForDatasource } from "../../../../../tests/utilities/structures" // These test cases are only for things that cannot be tested through the API // (e.g. limiting searches to returning specific fields). If it's possible to // test through the API, it should be done there instead. -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts index f399801f1e..26a8431446 100644 --- a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts @@ -250,6 +250,8 @@ describe("query utils", () => { expect(result).toEqual([ "_id", "name", + "aux._id", + "auxTable._id", "aux.title", "auxTable.title", "aux.name", @@ -284,7 +286,14 @@ describe("query utils", () => { const result = await config.doInContext(config.appId, () => { return getQueryableFields(table) }) - expect(result).toEqual(["_id", "name", "aux.name", "auxTable.name"]) + expect(result).toEqual([ + "_id", + "name", + "aux._id", + "auxTable._id", + "aux.name", + "auxTable.name", + ]) }) it("excludes all relationship fields if hidden", async () => { @@ -387,10 +396,14 @@ describe("query utils", () => { "_id", "name", // aux1 primitive props + "aux1._id", + "aux1Table._id", "aux1.name", "aux1Table.name", // aux2 primitive props + "aux2._id", + "aux2Table._id", "aux2.title", "aux2Table.title", ]) @@ -405,14 +418,18 @@ describe("query utils", () => { "name", // aux2_1 primitive props + "aux2_1._id", + "aux2Table._id", "aux2_1.title", "aux2Table.title", // aux2_2 primitive props + "aux2_2._id", "aux2_2.title", - "aux2Table.title", // table primitive props + "table._id", + "TestTable._id", "table.name", "TestTable.name", ]) @@ -427,14 +444,18 @@ describe("query utils", () => { "title", // aux1_1 primitive props + "aux1_1._id", + "aux1Table._id", "aux1_1.name", "aux1Table.name", // aux1_2 primitive props + "aux1_2._id", "aux1_2.name", - "aux1Table.name", // table primitive props + "table._id", + "TestTable._id", "table.name", "TestTable.name", ]) @@ -481,6 +502,8 @@ describe("query utils", () => { "name", // deep 1 aux primitive props + "aux._id", + "auxTable._id", "aux.title", "auxTable.title", ]) @@ -495,6 +518,8 @@ describe("query utils", () => { "title", // deep 1 dependency primitive props + "table._id", + "TestTable._id", "table.name", "TestTable.name", ]) diff --git a/packages/server/src/sdk/app/tables/tests/tables.spec.ts b/packages/server/src/sdk/app/tables/tests/tables.spec.ts deleted file mode 100644 index 41ac808f5c..0000000000 --- a/packages/server/src/sdk/app/tables/tests/tables.spec.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { - FieldType, - INTERNAL_TABLE_SOURCE_ID, - Table, - TableSourceType, - ViewV2, -} from "@budibase/types" -import { generator } from "@budibase/backend-core/tests" -import sdk from "../../.." - -jest.mock("../../views", () => ({ - ...jest.requireActual("../../views"), - enrichSchema: jest.fn().mockImplementation(v => ({ ...v, mocked: true })), -})) - -describe("table sdk", () => { - describe("enrichViewSchemas", () => { - const basicTable: Table = { - _id: generator.guid(), - name: "TestTable", - type: "table", - sourceId: INTERNAL_TABLE_SOURCE_ID, - sourceType: TableSourceType.INTERNAL, - schema: { - name: { - type: FieldType.STRING, - name: "name", - visible: true, - width: 80, - order: 2, - constraints: { - type: "string", - }, - }, - description: { - type: FieldType.STRING, - name: "description", - visible: true, - width: 200, - constraints: { - type: "string", - }, - }, - id: { - type: FieldType.NUMBER, - name: "id", - visible: true, - order: 1, - constraints: { - type: "number", - }, - }, - hiddenField: { - type: FieldType.STRING, - name: "hiddenField", - visible: false, - constraints: { - type: "string", - }, - }, - }, - } - - it("should fetch the default schema if not overriden", async () => { - const tableId = basicTable._id! - function getTable() { - const view: ViewV2 = { - version: 2, - id: generator.guid(), - name: generator.guid(), - tableId, - } - return view - } - const view1 = getTable() - const view2 = getTable() - const view3 = getTable() - const res = await sdk.tables.enrichViewSchemas({ - ...basicTable, - views: { - [view1.name]: view1, - [view2.name]: view2, - [view3.name]: view3, - }, - }) - - expect(sdk.views.enrichSchema).toHaveBeenCalledTimes(3) - - expect(res).toEqual({ - ...basicTable, - views: { - [view1.name]: { - ...view1, - mocked: true, - }, - [view2.name]: { - ...view2, - mocked: true, - }, - [view3.name]: { - ...view3, - mocked: true, - }, - }, - }) - }) - }) -}) diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 4da7dc5c91..50ac707a6b 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -367,6 +367,8 @@ class Orchestrator { if (e.errno === "ETIME") { span?.addTags({ timedOut: true }) console.warn(`Automation execution timed out after ${timeout}ms`) + } else { + throw e } } diff --git a/packages/server/src/utilities/workerRequests.ts b/packages/server/src/utilities/workerRequests.ts index 0f487d9f31..dd1493b82f 100644 --- a/packages/server/src/utilities/workerRequests.ts +++ b/packages/server/src/utilities/workerRequests.ts @@ -8,7 +8,15 @@ import { logging, env as coreEnv, } from "@budibase/backend-core" -import { Ctx, User, EmailInvite, EmailAttachment } from "@budibase/types" +import { + Ctx, + User, + EmailInvite, + EmailAttachment, + SendEmailResponse, + SendEmailRequest, + EmailTemplatePurpose, +} from "@budibase/types" interface Request { ctx?: Ctx @@ -110,25 +118,23 @@ export async function sendSmtpEmail({ invite?: EmailInvite }) { // tenant ID will be set in header + const request: SendEmailRequest = { + email: to, + from, + contents, + subject, + cc, + bcc, + purpose: EmailTemplatePurpose.CUSTOM, + automation, + invite, + attachments, + } const response = await fetch( checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`), - createRequest({ - method: "POST", - body: { - email: to, - from, - contents, - subject, - cc, - bcc, - purpose: "custom", - automation, - invite, - attachments, - }, - }) + createRequest({ method: "POST", body: request }) ) - return checkResponse(response, "send email") + return (await checkResponse(response, "send email")) as SendEmailResponse } export async function removeAppFromUserRoles(ctx: Ctx, appId: string) { diff --git a/packages/types/package.json b/packages/types/package.json index ee3c059bc9..a6e08ab84c 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -17,6 +17,7 @@ "@budibase/nano": "10.1.5", "@types/json-schema": "^7.0.15", "@types/koa": "2.13.4", + "@types/nodemailer": "^6.4.17", "@types/redlock": "4.0.7", "koa-useragent": "^4.1.0", "rimraf": "3.0.2", diff --git a/packages/types/src/api/web/global/email.ts b/packages/types/src/api/web/global/email.ts index a0ca0e8485..3d2e007231 100644 --- a/packages/types/src/api/web/global/email.ts +++ b/packages/types/src/api/web/global/email.ts @@ -1,4 +1,5 @@ import { EmailAttachment, EmailInvite } from "../../../documents" +import SMTPTransport from "nodemailer/lib/smtp-transport" export enum EmailTemplatePurpose { CORE = "core", @@ -12,17 +13,17 @@ export enum EmailTemplatePurpose { export interface SendEmailRequest { workspaceId?: string email: string - userId: string + userId?: string purpose: EmailTemplatePurpose contents?: string from?: string subject: string - cc?: boolean - bcc?: boolean + cc?: string + bcc?: string automation?: boolean invite?: EmailInvite attachments?: EmailAttachment[] } -export interface SendEmailResponse extends Record { +export interface SendEmailResponse extends SMTPTransport.SentMessageInfo { message: string } diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts index cee8fecb09..fc4caecdd1 100644 --- a/packages/types/src/documents/app/automation/automation.ts +++ b/packages/types/src/documents/app/automation/automation.ts @@ -1,10 +1,10 @@ import { Document } from "../../document" import { User } from "../../global" -import { ReadStream } from "fs" import { Row } from "../row" import { Table } from "../table" import { AutomationStep, AutomationTrigger } from "./schema" import { ContextEmitter } from "../../../sdk" +import { Readable } from "stream" export enum AutomationIOType { OBJECT = "object", @@ -109,8 +109,8 @@ export interface SendEmailOpts { subject: string // info Pass in a structure of information to be stored alongside the invitation. info?: any - cc?: boolean - bcc?: boolean + cc?: string + bcc?: string automation?: boolean invite?: EmailInvite attachments?: EmailAttachment[] @@ -262,7 +262,7 @@ export type AutomationAttachment = { export type AutomationAttachmentContent = { filename: string - content: ReadStream | NodeJS.ReadableStream + content: Readable } export type BucketedContent = AutomationAttachmentContent & { diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 992e9961d4..99c3658fa8 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -3,6 +3,7 @@ import { Row, DocumentType, Table, Datasource } from "../documents" import { SortOrder, SortType } from "../api" import { Knex } from "knex" import { Aggregation } from "./row" +import _ from "lodash" export enum BasicOperator { EQUAL = "equal", @@ -83,7 +84,7 @@ type RangeFilter = Record< type LogicalFilter = { conditions: SearchFilters[] } export function isLogicalFilter(filter: any): filter is LogicalFilter { - return "conditions" in filter + return _.isPlainObject(filter) && "conditions" in filter } export type AnySearchFilter = BasicFilter | ArrayFilter | RangeFilter diff --git a/packages/worker/package.json b/packages/worker/package.json index 28728272ca..edaab50d78 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -86,6 +86,7 @@ "@types/koa__router": "12.0.4", "@types/lodash": "4.14.200", "@types/node-fetch": "2.6.4", + "@types/nodemailer": "^6.4.17", "@types/server-destroy": "1.0.1", "@types/supertest": "2.0.14", "@types/uuid": "8.3.4", diff --git a/packages/worker/src/api/controllers/global/email.ts b/packages/worker/src/api/controllers/global/email.ts index ad0fc3fa32..ed2d9b5125 100644 --- a/packages/worker/src/api/controllers/global/email.ts +++ b/packages/worker/src/api/controllers/global/email.ts @@ -24,10 +24,13 @@ export async function sendEmail( invite, attachments, } = ctx.request.body - let user: any + let user: User | undefined = undefined if (userId) { const db = tenancy.getGlobalDB() - user = await db.get(userId) + user = await db.tryGet(userId) + } + if (!user) { + ctx.throw(404, "User not found.") } const response = await sendEmailFn(email, purpose, { workspaceId, diff --git a/packages/worker/src/utilities/email.ts b/packages/worker/src/utilities/email.ts index fa9dd7a6fa..a2b9c3bfc2 100644 --- a/packages/worker/src/utilities/email.ts +++ b/packages/worker/src/utilities/email.ts @@ -13,7 +13,8 @@ import { configs, cache, objectStore } from "@budibase/backend-core" import ical from "ical-generator" import _ from "lodash" -const nodemailer = require("nodemailer") +import nodemailer from "nodemailer" +import SMTPTransport from "nodemailer/lib/smtp-transport" const TEST_MODE = env.ENABLE_EMAIL_TEST_MODE && env.isDev() const TYPE = TemplateType.EMAIL @@ -26,7 +27,7 @@ const FULL_EMAIL_PURPOSES = [ ] function createSMTPTransport(config?: SMTPInnerConfig) { - let options: any + let options: SMTPTransport.Options let secure = config?.secure // default it if not specified if (secure == null) { @@ -161,7 +162,7 @@ export async function sendEmail( const code = await getLinkCode(purpose, email, opts.user, opts?.info) let context = await getSettingsTemplateContext(purpose, code) - let message: any = { + let message: Parameters[0] = { from: opts?.from || config?.from, html: await buildEmail(purpose, email, context, { user: opts?.user, diff --git a/yarn.lock b/yarn.lock index 8f611e224c..445e0abc18 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2785,9 +2785,9 @@ through2 "^2.0.0" "@budibase/pro@npm:@budibase/pro@latest": - version "3.4.12" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.12.tgz#60e630944de4e2de970a04179d8f0f57d48ce75e" - integrity sha512-msUBmcWxRDg+ugjZvd27XudERQqtQRdiARsO8MaDVTcp5ejIXgshEIVVshHOCj3hcbRblw9pXvBIMI53iTMUsA== + version "3.4.16" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.16.tgz#c482a400e27b7e89ca73092c4c81bdeac1d24581" + integrity sha512-8ECnqOh9jQ10KlQEwmKPFcoVGE+2gGgSybj+vbshwDp1zAW76doyMR2DMNjEatNpWVnpoMnTkDWtE9aqQ5v0vQ== dependencies: "@anthropic-ai/sdk" "^0.27.3" "@budibase/backend-core" "*" @@ -6775,6 +6775,13 @@ dependencies: undici-types "~6.19.2" +"@types/nodemailer@^6.4.17": + version "6.4.17" + resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.17.tgz#5c82a42aee16a3dd6ea31446a1bd6a447f1ac1a4" + integrity sha512-I9CCaIp6DTldEg7vyUTZi8+9Vo0hi1/T8gv3C89yk1rSAAzoKQ8H8ki/jBYJSFoH/BisgLP8tkZMlQ91CIquww== + dependencies: + "@types/node" "*" + "@types/normalize-package-data@^2.4.0": version "2.4.1" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" @@ -12782,10 +12789,10 @@ html-tag@^2.0.0: is-self-closing "^1.0.1" kind-of "^6.0.0" -html5-qrcode@^2.2.1: - version "2.3.7" - resolved "https://registry.yarnpkg.com/html5-qrcode/-/html5-qrcode-2.3.7.tgz#09ed2ca7473a47bd551088c15fcfcb7cb409a5be" - integrity sha512-Jmlok9Ynm49hgVXkdupWryf8o430proIFoQsRl1LmTg4Rq461W72omylR9yw9tsEMtswMEw3wacUM5y0agOBQA== +html5-qrcode@^2.3.8: + version "2.3.8" + resolved "https://registry.yarnpkg.com/html5-qrcode/-/html5-qrcode-2.3.8.tgz#0b0cdf7a9926cfd4be530e13a51db47592adfa0d" + integrity sha512-jsr4vafJhwoLVEDW3n1KvPnCCXWaQfRng0/EEYk1vNcQGcG/htAdhJX0be8YyqMoSz7+hZvOZSTAepsabiuhiQ== htmlparser2@^8.0.0: version "8.0.1"