From 7f779027baae6105fe8b5911a33019cefe588a68 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 18 Feb 2025 17:04:49 +0000 Subject: [PATCH 01/51] Remove mocking from sqlUtils.spec.ts. --- .../row/utils/tests/sqlUtils.spec.ts | 1009 +++++++++-------- 1 file changed, 515 insertions(+), 494 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts index 365f571fcf..cf14ba9b42 100644 --- a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts +++ b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts @@ -1,511 +1,532 @@ import { AIOperationEnum, CalculationType, + Datasource, FieldType, RelationshipType, - SourceName, Table, ViewV2, ViewV2Type, } from "@budibase/types" import { buildSqlFieldList } from "../sqlUtils" import { structures } from "../../../../routes/tests/utilities" -import { sql } from "@budibase/backend-core" import { generator } from "@budibase/backend-core/tests" import { generateViewID } from "../../../../../db/utils" -import sdk from "../../../../../sdk" -import { cloneDeep } from "lodash" import { utils } from "@budibase/shared-core" +import { + DatabaseName, + datasourceDescribe, +} from "../../../../../integrations/tests/utils" +import { context } from "@budibase/backend-core" -jest.mock("../../../../../sdk/app/views", () => ({ - ...jest.requireActual("../../../../../sdk/app/views"), - getTable: jest.fn(), -})) -const getTableMock = sdk.views.getTable as jest.MockedFunction< - typeof sdk.views.getTable -> - -describe("buildSqlFieldList", () => { - let allTables: Record - - class TableConfig { - private _table: Table & { _id: string } - - constructor(name: string) { - this._table = { - ...structures.tableForDatasource({ - type: "datasource", - source: SourceName.POSTGRES, - }), - name, - _id: sql.utils.buildExternalTableId("ds_id", name), - schema: { - name: { - name: "name", - type: FieldType.STRING, - }, - description: { - name: "description", - type: FieldType.STRING, - }, - amount: { - name: "amount", - type: FieldType.NUMBER, - }, - }, - } - - allTables[name] = this._table - } - - withHiddenField(field: string) { - this._table.schema[field].visible = false - return this - } - - withField( - name: string, - type: - | FieldType.STRING - | FieldType.NUMBER - | FieldType.FORMULA - | FieldType.AI, - options?: { visible: boolean } - ) { - switch (type) { - case FieldType.NUMBER: - case FieldType.STRING: - this._table.schema[name] = { - name, - type, - ...options, - } - break - case FieldType.FORMULA: - this._table.schema[name] = { - name, - type, - formula: "any", - ...options, - } - break - case FieldType.AI: - this._table.schema[name] = { - name, - type, - operation: AIOperationEnum.PROMPT, - ...options, - } - break - default: - utils.unreachable(type) - } - return this - } - - withRelation(name: string, toTableId: string) { - this._table.schema[name] = { - name, - type: FieldType.LINK, - relationshipType: RelationshipType.ONE_TO_MANY, - fieldName: "link", - tableId: toTableId, - } - return this - } - - withPrimary(field: string) { - this._table.primary = [field] - return this - } - - withDisplay(field: string) { - this._table.primaryDisplay = field - return this - } - - create() { - return cloneDeep(this._table) - } - } - - class ViewConfig { - private _table: Table - private _view: ViewV2 - - constructor(table: Table) { - this._table = table - this._view = { - version: 2, - id: generateViewID(table._id!), - name: generator.word(), - tableId: table._id!, - } - } - - withVisible(field: string) { - this._view.schema ??= {} - this._view.schema[field] ??= {} - this._view.schema[field].visible = true - return this - } - - withHidden(field: string) { - this._view.schema ??= {} - this._view.schema[field] ??= {} - this._view.schema[field].visible = false - return this - } - - withRelationshipColumns( - field: string, - columns: Record - ) { - this._view.schema ??= {} - this._view.schema[field] ??= {} - this._view.schema[field].columns = columns - return this - } - - withCalculation( - name: string, - field: string, - calculationType: CalculationType - ) { - this._view.type = ViewV2Type.CALCULATION - this._view.schema ??= {} - this._view.schema[name] = { - field, - calculationType, - visible: true, - } - return this - } - - create() { - getTableMock.mockResolvedValueOnce(this._table) - return cloneDeep(this._view) - } - } - - beforeEach(() => { - jest.clearAllMocks() - allTables = {} - }) - - describe("table", () => { - it("extracts fields from table schema", async () => { - const table = new TableConfig("table").create() - const result = await buildSqlFieldList(table, {}) - expect(result).toEqual([ - "table.name", - "table.description", - "table.amount", - ]) - }) - - it("excludes hidden fields", async () => { - const table = new TableConfig("table") - .withHiddenField("description") - .create() - const result = await buildSqlFieldList(table, {}) - expect(result).toEqual(["table.name", "table.amount"]) - }) - - it("excludes non-sql fields fields", async () => { - const table = new TableConfig("table") - .withField("formula", FieldType.FORMULA) - .withField("ai", FieldType.AI) - .withRelation("link", "otherTableId") - .create() - - const result = await buildSqlFieldList(table, {}) - expect(result).toEqual([ - "table.name", - "table.description", - "table.amount", - ]) - }) - - it("includes hidden fields if there is a formula column", async () => { - const table = new TableConfig("table") - .withHiddenField("description") - .withField("formula", FieldType.FORMULA) - .create() - - const result = await buildSqlFieldList(table, {}) - expect(result).toEqual([ - "table.name", - "table.description", - "table.amount", - ]) - }) - - it("includes relationships fields when flagged", async () => { - const otherTable = new TableConfig("linkedTable") - .withField("id", FieldType.NUMBER) - .withPrimary("id") - .withDisplay("name") - .create() - - const table = new TableConfig("table") - .withRelation("link", otherTable._id) - .create() - - const result = await buildSqlFieldList(table, allTables, { - relationships: true, - }) - expect(result).toEqual([ - "table.name", - "table.description", - "table.amount", - "linkedTable.id", - "linkedTable.name", - ]) - }) - - it("includes all relationship fields if there is a formula column", async () => { - const otherTable = new TableConfig("linkedTable") - .withField("hidden", FieldType.STRING, { visible: false }) - .create() - - const table = new TableConfig("table") - .withRelation("link", otherTable._id) - .withField("formula", FieldType.FORMULA) - .create() - - const result = await buildSqlFieldList(table, allTables, { - relationships: true, - }) - expect(result).toEqual([ - "table.name", - "table.description", - "table.amount", - "linkedTable.name", - "linkedTable.description", - "linkedTable.amount", - "linkedTable.hidden", - ]) - }) - - it("never includes non-sql columns from relationships", async () => { - const otherTable = new TableConfig("linkedTable") - .withField("id", FieldType.NUMBER) - .withField("hidden", FieldType.STRING, { visible: false }) - .withField("formula", FieldType.FORMULA) - .withField("ai", FieldType.AI) - .withRelation("link", "otherTableId") - .create() - - const table = new TableConfig("table") - .withRelation("link", otherTable._id) - .withField("formula", FieldType.FORMULA) - .create() - - const result = await buildSqlFieldList(table, allTables, { - relationships: true, - }) - expect(result).toEqual([ - "table.name", - "table.description", - "table.amount", - "linkedTable.name", - "linkedTable.description", - "linkedTable.amount", - "linkedTable.id", - "linkedTable.hidden", - ]) - }) - }) - - describe("view", () => { - it("extracts fields from table schema", async () => { - const view = new ViewConfig(new TableConfig("table").create()) - .withVisible("amount") - .withHidden("name") - .create() - - const result = await buildSqlFieldList(view, {}) - expect(result).toEqual(["table.amount"]) - }) - - it("includes all fields if there is a formula column", async () => { - const table = new TableConfig("table") - .withField("formula", FieldType.FORMULA) - .create() - const view = new ViewConfig(table) - .withHidden("name") - .withVisible("amount") - .withVisible("formula") - .create() - - const result = await buildSqlFieldList(view, {}) - expect(result).toEqual([ - "table.name", - "table.description", - "table.amount", - ]) - }) - - it("does not includes all fields if the formula column is not included", async () => { - const table = new TableConfig("table") - .withField("formula", FieldType.FORMULA) - .create() - const view = new ViewConfig(table) - .withHidden("name") - .withVisible("amount") - .withHidden("formula") - .create() - - const result = await buildSqlFieldList(view, {}) - expect(result).toEqual(["table.amount"]) - }) - - it("includes relationships columns", async () => { - const otherTable = new TableConfig("linkedTable") - .withField("id", FieldType.NUMBER) - .withField("formula", FieldType.FORMULA) - .withPrimary("id") - .create() - - const table = new TableConfig("table") - .withRelation("link", otherTable._id) - .create() - - const view = new ViewConfig(table) - .withVisible("name") - .withVisible("link") - .withRelationshipColumns("link", { - name: { visible: false }, - amount: { visible: true }, - formula: { visible: false }, - }) - .create() - - const result = await buildSqlFieldList(view, allTables, { - relationships: true, - }) - expect(result).toEqual([ - "table.name", - "linkedTable.id", - "linkedTable.amount", - ]) - }) - - it("excludes relationships fields when view is not included in the view", async () => { - const otherTable = new TableConfig("linkedTable") - .withField("id", FieldType.NUMBER) - .withPrimary("id") - .withDisplay("name") - .create() - - const table = new TableConfig("table") - .withRelation("link", otherTable._id) - .withField("formula", FieldType.FORMULA) - .create() - - const view = new ViewConfig(table) - .withVisible("name") - .withHidden("amount") - .create() - - const result = await buildSqlFieldList(view, allTables, { - relationships: true, - }) - expect(result).toEqual(["table.name"]) - }) - - it("does not include relationships columns for hidden links", async () => { - const otherTable = new TableConfig("linkedTable") - .withField("id", FieldType.NUMBER) - .withField("formula", FieldType.FORMULA) - .withPrimary("id") - .create() - - const table = new TableConfig("table") - .withRelation("link", otherTable._id) - .create() - - const view = new ViewConfig(table) - .withVisible("name") - .withHidden("link") - .withRelationshipColumns("link", { - name: { visible: false }, - amount: { visible: true }, - formula: { visible: false }, - }) - .create() - - const result = await buildSqlFieldList(view, allTables, { - relationships: true, - }) - expect(result).toEqual(["table.name"]) - }) - - it("includes all relationship fields if there is a formula column", async () => { - const otherTable = new TableConfig("linkedTable") - .withField("id", FieldType.NUMBER) - .withField("hidden", FieldType.STRING, { visible: false }) - .withField("formula", FieldType.FORMULA) - .withField("ai", FieldType.AI) - .withRelation("link", "otherTableId") - .withPrimary("id") - .create() - - const table = new TableConfig("table") - .withRelation("link", otherTable._id) - .withField("formula", FieldType.FORMULA) - .create() - - const view = new ViewConfig(table) - .withVisible("name") - .withVisible("formula") - .withHidden("link") - .withRelationshipColumns("link", { - name: { visible: false }, - amount: { visible: true }, - formula: { visible: false }, - }) - .create() - - const result = await buildSqlFieldList(view, allTables, { - relationships: true, - }) - expect(result).toEqual([ - "table.name", - "table.description", - "table.amount", - "linkedTable.name", - "linkedTable.description", - "linkedTable.amount", - "linkedTable.id", - "linkedTable.hidden", - ]) - }) - }) - - describe("calculation view", () => { - it("does not include calculation fields", async () => { - const view = new ViewConfig(new TableConfig("table").create()) - .withCalculation("average", "amount", CalculationType.AVG) - - .create() - - const result = await buildSqlFieldList(view, {}) - expect(result).toEqual([]) - }) - - it("includes visible fields calculation fields", async () => { - const view = new ViewConfig(new TableConfig("table").create()) - .withCalculation("average", "amount", CalculationType.AVG) - .withHidden("name") - .withVisible("amount") - - .create() - - const result = await buildSqlFieldList(view, {}) - expect(result).toEqual(["table.amount"]) - }) - }) +const descriptions = datasourceDescribe({ + only: [DatabaseName.POSTGRES], }) + +if (descriptions.length) { + describe.each(descriptions)( + "buildSqlFieldList ($dbName)", + ({ config, dsProvider }) => { + let allTables: Record + let datasource: Datasource + + beforeEach(async () => { + const ds = await dsProvider() + datasource = ds.datasource! + allTables = {} + }) + + class TableConfig { + private _table: Table + + constructor(name: string) { + this._table = { + ...structures.tableForDatasource(datasource), + name, + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + description: { + name: "description", + type: FieldType.STRING, + }, + amount: { + name: "amount", + type: FieldType.NUMBER, + }, + }, + } + } + + withHiddenField(field: string) { + this._table.schema[field].visible = false + return this + } + + withField( + name: string, + type: + | FieldType.STRING + | FieldType.NUMBER + | FieldType.FORMULA + | FieldType.AI, + options?: { visible: boolean } + ) { + switch (type) { + case FieldType.NUMBER: + case FieldType.STRING: + this._table.schema[name] = { + name, + type, + ...options, + } + break + case FieldType.FORMULA: + this._table.schema[name] = { + name, + type, + formula: "any", + ...options, + } + break + case FieldType.AI: + this._table.schema[name] = { + name, + type, + operation: AIOperationEnum.PROMPT, + ...options, + } + break + default: + utils.unreachable(type) + } + return this + } + + withRelation(name: string, toTableId: string) { + this._table.schema[name] = { + name, + type: FieldType.LINK, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "link", + foreignKey: "link", + tableId: toTableId, + } + return this + } + + withPrimary(field: string) { + this._table.primary = [field] + return this + } + + withDisplay(field: string) { + this._table.primaryDisplay = field + return this + } + + async create() { + const table = await config.api.table.save(this._table) + allTables[table.name] = table + return table + } + } + + class ViewConfig { + private _view: ViewV2 + + constructor(table: Table) { + this._view = { + version: 2, + id: generateViewID(table._id!), + name: generator.word(), + tableId: table._id!, + } + } + + withVisible(field: string) { + this._view.schema ??= {} + this._view.schema[field] ??= {} + this._view.schema[field].visible = true + return this + } + + withHidden(field: string) { + this._view.schema ??= {} + this._view.schema[field] ??= {} + this._view.schema[field].visible = false + return this + } + + withRelationshipColumns( + field: string, + columns: Record + ) { + this._view.schema ??= {} + this._view.schema[field] ??= {} + this._view.schema[field].columns = columns + return this + } + + withCalculation( + name: string, + field: string, + calculationType: CalculationType + ) { + this._view.type = ViewV2Type.CALCULATION + this._view.schema ??= {} + this._view.schema[name] = { + field, + calculationType, + visible: true, + } + return this + } + + async create() { + return await config.api.viewV2.create(this._view) + } + } + + const buildSqlFieldListInApp: typeof buildSqlFieldList = async ( + table, + allTables, + opts + ) => { + return context.doInAppContext(config.getAppId(), () => + buildSqlFieldList(table, allTables, opts) + ) + } + + describe("table", () => { + it("extracts fields from table schema", async () => { + const table = await new TableConfig("table").create() + const result = await buildSqlFieldListInApp(table, {}) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "table.id", + ]) + }) + + it("excludes hidden fields", async () => { + const table = await new TableConfig("table") + .withHiddenField("description") + .create() + const result = await buildSqlFieldListInApp(table, {}) + expect(result).toEqual(["table.name", "table.amount", "table.id"]) + }) + + it("excludes non-sql fields fields", async () => { + const table = await new TableConfig("table") + .withField("formula", FieldType.FORMULA) + .withField("ai", FieldType.AI) + .create() + + const result = await buildSqlFieldListInApp(table, {}) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "table.id", + ]) + }) + + it("includes hidden fields if there is a formula column", async () => { + const table = await new TableConfig("table") + .withHiddenField("description") + .withField("formula", FieldType.FORMULA) + .create() + + const result = await buildSqlFieldListInApp(table, {}) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "table.id", + ]) + }) + + it("includes relationships fields when flagged", async () => { + const otherTable = await new TableConfig("linkedTable") + .withField("id", FieldType.NUMBER) + .withPrimary("id") + .withDisplay("name") + .create() + + const table = await new TableConfig("table") + .withRelation("link", otherTable._id!) + .create() + + const result = await buildSqlFieldListInApp(table, allTables, { + relationships: true, + }) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "table.id", + "linkedTable.id", + "linkedTable.name", + ]) + }) + + it("includes all relationship fields if there is a formula column", async () => { + const otherTable = await new TableConfig("linkedTable") + .withField("hidden", FieldType.STRING, { visible: false }) + .create() + + const table = await new TableConfig("table") + .withRelation("link", otherTable._id!) + .withField("formula", FieldType.FORMULA) + .create() + + const result = await buildSqlFieldListInApp(table, allTables, { + relationships: true, + }) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "table.id", + "linkedTable.name", + "linkedTable.description", + "linkedTable.amount", + "linkedTable.hidden", + "linkedTable.id", + ]) + }) + + it("never includes non-sql columns from relationships", async () => { + const otherTable = await new TableConfig("linkedTable") + .withField("hidden", FieldType.STRING, { visible: false }) + .withField("formula", FieldType.FORMULA) + .withField("ai", FieldType.AI) + .create() + + const table = await new TableConfig("table") + .withRelation("link", otherTable._id!) + .withField("formula", FieldType.FORMULA) + .create() + + const result = await buildSqlFieldListInApp(table, allTables, { + relationships: true, + }) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "table.id", + "linkedTable.name", + "linkedTable.description", + "linkedTable.amount", + "linkedTable.hidden", + "linkedTable.id", + ]) + }) + }) + + describe("view", () => { + it("extracts fields from table schema", async () => { + const view = await new ViewConfig( + await new TableConfig("table").create() + ) + .withVisible("amount") + .withHidden("name") + .create() + + const result = await buildSqlFieldListInApp(view, {}) + expect(result).toEqual(["table.amount", "table.id"]) + }) + + it("includes all fields if there is a formula column", async () => { + const table = await new TableConfig("table") + .withField("formula", FieldType.FORMULA) + .create() + const view = await new ViewConfig(table) + .withHidden("name") + .withVisible("amount") + .withVisible("formula") + .create() + + const result = await buildSqlFieldListInApp(view, {}) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "table.id", + ]) + }) + + it("does not includes all fields if the formula column is not included", async () => { + const table = await new TableConfig("table") + .withField("formula", FieldType.FORMULA) + .create() + const view = await new ViewConfig(table) + .withHidden("name") + .withVisible("amount") + .withHidden("formula") + .create() + + const result = await buildSqlFieldListInApp(view, {}) + expect(result).toEqual(["table.amount", "table.id"]) + }) + + it("includes relationships columns", async () => { + const otherTable = await new TableConfig("linkedTable") + .withField("id", FieldType.NUMBER) + .withField("formula", FieldType.FORMULA) + .withPrimary("id") + .create() + + const table = await new TableConfig("table") + .withRelation("link", otherTable._id!) + .create() + + const view = await new ViewConfig(table) + .withVisible("name") + .withVisible("link") + .withRelationshipColumns("link", { + name: { visible: false }, + amount: { visible: true }, + formula: { visible: false }, + }) + .create() + + const result = await buildSqlFieldListInApp(view, allTables, { + relationships: true, + }) + expect(result).toEqual([ + "table.name", + "table.id", + "linkedTable.id", + "linkedTable.amount", + ]) + }) + + it("excludes relationships fields when view is not included in the view", async () => { + const otherTable = await new TableConfig("linkedTable") + .withField("id", FieldType.NUMBER) + .withPrimary("id") + .withDisplay("name") + .create() + + const table = await new TableConfig("table") + .withRelation("link", otherTable._id!) + .withField("formula", FieldType.FORMULA) + .create() + + const view = await new ViewConfig(table) + .withVisible("name") + .withHidden("amount") + .create() + + const result = await buildSqlFieldListInApp(view, allTables, { + relationships: true, + }) + expect(result).toEqual(["table.name", "table.id"]) + }) + + it("does not include relationships columns for hidden links", async () => { + const otherTable = await new TableConfig("linkedTable") + .withField("id", FieldType.NUMBER) + .withField("formula", FieldType.FORMULA) + .withPrimary("id") + .create() + + const table = await new TableConfig("table") + .withRelation("link", otherTable._id!) + .create() + + const view = await new ViewConfig(table) + .withVisible("name") + .withHidden("link") + .withRelationshipColumns("link", { + name: { visible: false }, + amount: { visible: true }, + formula: { visible: false }, + }) + .create() + + const result = await buildSqlFieldListInApp(view, allTables, { + relationships: true, + }) + expect(result).toEqual(["table.name", "table.id"]) + }) + + it("includes all relationship fields if there is a formula column", async () => { + const otherTable = await new TableConfig("linkedTable") + .withField("id", FieldType.NUMBER) + .withField("hidden", FieldType.STRING, { visible: false }) + .withField("formula", FieldType.FORMULA) + .withField("ai", FieldType.AI) + .withPrimary("id") + .create() + + const table = await new TableConfig("table") + .withRelation("link", otherTable._id!) + .withField("formula", FieldType.FORMULA) + .create() + + const view = await new ViewConfig(table) + .withVisible("name") + .withVisible("formula") + .withHidden("link") + .withRelationshipColumns("link", { + name: { visible: false }, + amount: { visible: true }, + formula: { visible: false }, + }) + .create() + + const result = await buildSqlFieldListInApp(view, allTables, { + relationships: true, + }) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "table.id", + "linkedTable.name", + "linkedTable.description", + "linkedTable.amount", + "linkedTable.id", + "linkedTable.hidden", + ]) + }) + }) + + describe("calculation view", () => { + it("does not include calculation fields", async () => { + const view = await new ViewConfig( + await new TableConfig("table").create() + ) + .withCalculation("average", "amount", CalculationType.AVG) + + .create() + + const result = await buildSqlFieldListInApp(view, {}) + expect(result).toEqual([]) + }) + + it("includes visible fields calculation fields", async () => { + const view = await new ViewConfig( + await new TableConfig("table").create() + ) + .withCalculation("average", "amount", CalculationType.AVG) + .withHidden("name") + .withVisible("amount") + + .create() + + const result = await buildSqlFieldListInApp(view, {}) + expect(result).toEqual(["table.amount"]) + }) + }) + } + ) +} From 28553fc684c89871d5e8aec64df9a551ddf0ca38 Mon Sep 17 00:00:00 2001 From: Christos Alexiou Date: Thu, 20 Feb 2025 03:02:34 +0200 Subject: [PATCH 02/51] formatting and add special minio version --- hosting/single/Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index e4858d4af0..043cd3dd73 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -1,5 +1,5 @@ ARG BASEIMG=budibase/couchdb:v3.3.3-sqs-v2.1.1 -FROM node:20-slim as build +FROM node:20-slim AS build # install node-gyp dependencies RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq @@ -34,7 +34,7 @@ COPY packages/worker/dist packages/worker/dist COPY packages/worker/pm2.config.js packages/worker/pm2.config.js -FROM $BASEIMG as runner +FROM $BASEIMG AS runner ARG TARGETARCH ENV TARGETARCH $TARGETARCH #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) @@ -67,6 +67,11 @@ RUN mkdir -p /var/log/nginx && \ # setup minio WORKDIR /minio + +# a 2022 version of minio that supports gateway mode +COPY scripts/resources/minio /minio + +# handles the installation of minio in non-aas environments COPY scripts/install-minio.sh ./install.sh RUN chmod +x install.sh && ./install.sh From d4a2b6d0610fd27272f8d93ca65b68fc0e20a362 Mon Sep 17 00:00:00 2001 From: Christos Alexiou Date: Thu, 20 Feb 2025 03:02:42 +0200 Subject: [PATCH 03/51] handle minio with gateway --- hosting/single/runner.sh | 47 ++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index e06a197ad5..d6b77e9e42 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -19,14 +19,7 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME [[ -z "${SERVER_TOP_LEVEL_PATH}" ]] && export SERVER_TOP_LEVEL_PATH=/app # export CUSTOM_DOMAIN=budi001.custom.com -# Azure App Service customisations -if [[ "${TARGETBUILD}" = "aas" ]]; then - export DATA_DIR="${DATA_DIR:-/home}" - WEBSITES_ENABLE_APP_SERVICE_STORAGE=true - /etc/init.d/ssh start -else - export DATA_DIR=${DATA_DIR:-/data} -fi +export DATA_DIR=${DATA_DIR:-/data} mkdir -p ${DATA_DIR} # Mount NFS or GCP Filestore if env vars exist for it if [[ ! -z ${FILESHARE_IP} && ! -z ${FILESHARE_NAME} ]]; then @@ -42,8 +35,7 @@ if [ -f "${DATA_DIR}/.env" ]; then for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done fi # randomise any unset environment variables -for ENV_VAR in "${ENV_VARS[@]}" -do +for ENV_VAR in "${ENV_VARS[@]}"; do if [[ -z "${!ENV_VAR}" ]]; then eval "export $ENV_VAR=$(uuidgen | sed -e 's/-//g')" fi @@ -58,17 +50,15 @@ fi if [ ! -f "${DATA_DIR}/.env" ]; then touch ${DATA_DIR}/.env - for ENV_VAR in "${ENV_VARS[@]}" - do + for ENV_VAR in "${ENV_VARS[@]}"; do temp=$(eval "echo \$$ENV_VAR") - echo "$ENV_VAR=$temp" >> ${DATA_DIR}/.env + echo "$ENV_VAR=$temp" >>${DATA_DIR}/.env done - for ENV_VAR in "${DOCKER_VARS[@]}" - do + for ENV_VAR in "${DOCKER_VARS[@]}"; do temp=$(eval "echo \$$ENV_VAR") - echo "$ENV_VAR=$temp" >> ${DATA_DIR}/.env + echo "$ENV_VAR=$temp" >>${DATA_DIR}/.env done - echo "COUCH_DB_URL=${COUCH_DB_URL}" >> ${DATA_DIR}/.env + echo "COUCH_DB_URL=${COUCH_DB_URL}" >>${DATA_DIR}/.env fi # Read in the .env file and export the variables @@ -79,31 +69,42 @@ ln -s ${DATA_DIR}/.env /worker/.env # make these directories in runner, incase of mount mkdir -p ${DATA_DIR}/minio mkdir -p ${DATA_DIR}/redis -chown -R couchdb:couchdb ${DATA_DIR}/couch +#mkdir -p ${DATA_DIR}/couch +#chown -R couchdb:couchdb ${DATA_DIR}/couch REDIS_CONFIG="/etc/redis/redis.conf" sed -i "s#DATA_DIR#${DATA_DIR}#g" "${REDIS_CONFIG}" if [[ -n "${USE_DEFAULT_REDIS_CONFIG}" ]]; then - REDIS_CONFIG="" + REDIS_CONFIG="" fi if [[ -n "${REDIS_PASSWORD}" ]]; then - redis-server "${REDIS_CONFIG}" --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 & + redis-server "${REDIS_CONFIG}" --requirepass $REDIS_PASSWORD >/dev/stdout 2>&1 & else - redis-server "${REDIS_CONFIG}" > /dev/stdout 2>&1 & + redis-server "${REDIS_CONFIG}" >/dev/stdout 2>&1 & fi /bbcouch-runner.sh & # only start minio if use s3 isn't passed if [[ -z "${USE_S3}" ]]; then - /minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 & + if [[ $TARGETBUILD == aas ]]; then + echo "Starting MinIO in Azure Gateway mode" + if [[ -z "${AZURE_STORAGE_ACCOUNT}" || -z "${AZURE_STORAGE_KEY}" ]]; then + echo "AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY must be set when deploying in Azure App Service mode" + exit 1 + fi + /minio/minio gateway azure --console-address ":9001" >/dev/stdout 2>&1 & + else + echo "Starting MinIO in standalone mode" + /minio/minio server --console-address ":9001" ${DATA_DIR}/minio >/dev/stdout 2>&1 & + fi fi /etc/init.d/nginx restart if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then # Add monthly cron job to renew certbot certificate - echo -n "* * 2 * * root exec /app/letsencrypt/certificate-renew.sh ${CUSTOM_DOMAIN}" >> /etc/cron.d/certificate-renew + echo -n "* * 2 * * root exec /app/letsencrypt/certificate-renew.sh ${CUSTOM_DOMAIN}" >>/etc/cron.d/certificate-renew chmod +x /etc/cron.d/certificate-renew # Request the certbot certificate /app/letsencrypt/certificate-request.sh ${CUSTOM_DOMAIN} From 1650fdc75ea9fe5e15c9983fdd6f689f2a261b51 Mon Sep 17 00:00:00 2001 From: Christos Alexiou Date: Thu, 20 Feb 2025 03:03:32 +0200 Subject: [PATCH 04/51] minio from dockerfile COPY --- scripts/install-minio.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/install-minio.sh b/scripts/install-minio.sh index fede984377..b885453f2a 100755 --- a/scripts/install-minio.sh +++ b/scripts/install-minio.sh @@ -1,10 +1,18 @@ #!/bin/bash -if [[ $TARGETARCH == arm* ]] ; -then + +if [[ $TARGETBUILD == "aas" ]]; then + echo "A aas-compatible version of Minio is already installed." + exit 0 +fi + +if [[ $TARGETARCH == arm* ]]; then echo "INSTALLING ARM64 MINIO" + rm -f minio wget https://dl.min.io/server/minio/release/linux-arm64/minio else echo "INSTALLING AMD64 MINIO" + rm -f minio wget https://dl.min.io/server/minio/release/linux-amd64/minio fi -chmod +x minio \ No newline at end of file + +chmod +x minio From 32f6fc3d32cbef84cfbe619c563a78a5d720da80 Mon Sep 17 00:00:00 2001 From: Christos Alexiou Date: Thu, 20 Feb 2025 03:09:14 +0200 Subject: [PATCH 05/51] add minio with gateway --- scripts/resources/minio | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 scripts/resources/minio diff --git a/scripts/resources/minio b/scripts/resources/minio new file mode 100644 index 0000000000..c121cc0963 --- /dev/null +++ b/scripts/resources/minio @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63db3aa3c2299ebaf13b46c64523a589bd5bf272f9e971d17f1eaa55f6f1fd79 +size 118595584 From 49dda35358abcc38d70577854491e447edae9b73 Mon Sep 17 00:00:00 2001 From: Christos Alexiou Date: Thu, 20 Feb 2025 03:09:42 +0200 Subject: [PATCH 06/51] track minio in lfs --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..85b026dd08 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +scripts/resources/minio filter=lfs diff=lfs merge=lfs -text From dba89a678c246dd2daaeb5e561452f3d9066e145 Mon Sep 17 00:00:00 2001 From: Christos Alexiou Date: Thu, 20 Feb 2025 20:35:24 +0200 Subject: [PATCH 07/51] make minio binary executable --- hosting/single/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 043cd3dd73..e6c1142ab8 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -70,6 +70,7 @@ WORKDIR /minio # a 2022 version of minio that supports gateway mode COPY scripts/resources/minio /minio +RUN chmod +x minio # handles the installation of minio in non-aas environments COPY scripts/install-minio.sh ./install.sh From afe293de5648dfa35b02e3552d5cb16d1c5ea9da Mon Sep 17 00:00:00 2001 From: Christos Alexiou Date: Thu, 20 Feb 2025 20:40:17 +0200 Subject: [PATCH 08/51] linting fix --- hosting/single/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index e6c1142ab8..1f449e7376 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -36,11 +36,11 @@ COPY packages/worker/pm2.config.js packages/worker/pm2.config.js FROM $BASEIMG AS runner ARG TARGETARCH -ENV TARGETARCH $TARGETARCH +ENV TARGETARCH=$TARGETARCH #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) # e.g. docker build --build-arg TARGETBUILD=aas .... ARG TARGETBUILD=single -ENV TARGETBUILD $TARGETBUILD +ENV TARGETBUILD=$TARGETBUILD # install base dependencies RUN apt-get update && \ From 18ddcd092ef7e77c48e4598166b69ef30a7c7e44 Mon Sep 17 00:00:00 2001 From: Christos Alexiou Date: Thu, 20 Feb 2025 21:04:44 +0200 Subject: [PATCH 09/51] simplify env setup and handle MINIO_ missing --- hosting/single/runner.sh | 80 +++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 34 deletions(-) diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index d6b77e9e42..42305cf62e 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -1,45 +1,57 @@ #!/bin/bash -declare -a ENV_VARS=("COUCHDB_USER" "COUCHDB_PASSWORD" "DATA_DIR" "MINIO_ACCESS_KEY" "MINIO_SECRET_KEY" "INTERNAL_API_KEY" "JWT_SECRET" "REDIS_PASSWORD") -declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONMENT" "CLUSTER_PORT" "DEPLOYMENT_ENVIRONMENT" "MINIO_URL" "NODE_ENV" "POSTHOG_TOKEN" "REDIS_URL" "SELF_HOSTED" "WORKER_PORT" "WORKER_URL" "TENANT_FEATURE_FLAGS" "ACCOUNT_PORTAL_URL") -# Check the env vars set in Dockerfile have come through, AAS seems to drop them -[[ -z "${APP_PORT}" ]] && export APP_PORT=4001 -[[ -z "${ARCHITECTURE}" ]] && export ARCHITECTURE=amd -[[ -z "${BUDIBASE_ENVIRONMENT}" ]] && export BUDIBASE_ENVIRONMENT=PRODUCTION -[[ -z "${CLUSTER_PORT}" ]] && export CLUSTER_PORT=80 -[[ -z "${DEPLOYMENT_ENVIRONMENT}" ]] && export DEPLOYMENT_ENVIRONMENT=docker -[[ -z "${MINIO_URL}" ]] && [[ -z "${USE_S3}" ]] && export MINIO_URL=http://127.0.0.1:9000 -[[ -z "${NODE_ENV}" ]] && export NODE_ENV=production -[[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU -[[ -z "${ACCOUNT_PORTAL_URL}" ]] && export ACCOUNT_PORTAL_URL=https://account.budibase.app -[[ -z "${REDIS_URL}" ]] && export REDIS_URL=127.0.0.1:6379 -[[ -z "${SELF_HOSTED}" ]] && export SELF_HOSTED=1 -[[ -z "${WORKER_PORT}" ]] && export WORKER_PORT=4002 -[[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://127.0.0.1:4002 -[[ -z "${APPS_URL}" ]] && export APPS_URL=http://127.0.0.1:4001 -[[ -z "${SERVER_TOP_LEVEL_PATH}" ]] && export SERVER_TOP_LEVEL_PATH=/app -# export CUSTOM_DOMAIN=budi001.custom.com -export DATA_DIR=${DATA_DIR:-/data} -mkdir -p ${DATA_DIR} -# Mount NFS or GCP Filestore if env vars exist for it -if [[ ! -z ${FILESHARE_IP} && ! -z ${FILESHARE_NAME} ]]; then +echo "Starting runner.sh" + +# set defaults for Docker-related variables +export APP_PORT="${APP_PORT:-4001}" +export ARCHITECTURE="${ARCHITECTURE:-amd}" +export BUDIBASE_ENVIRONMENT="${BUDIBASE_ENVIRONMENT:-PRODUCTION}" +export CLUSTER_PORT="${CLUSTER_PORT:-80}" +export DEPLOYMENT_ENVIRONMENT="${DEPLOYMENT_ENVIRONMENT:-docker}" + +# only set MINIO_URL if neither MINIO_URL nor USE_S3 is set +if [[ -z "${MINIO_URL}" && -z "${USE_S3}" ]]; then + export MINIO_URL="http://127.0.0.1:9000" +fi + +export NODE_ENV="${NODE_ENV:-production}" +export POSTHOG_TOKEN="${POSTHOG_TOKEN:-phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU}" +export ACCOUNT_PORTAL_URL="${ACCOUNT_PORTAL_URL:-https://account.budibase.app}" +export REDIS_URL="${REDIS_URL:-127.0.0.1:6379}" +export SELF_HOSTED="${SELF_HOSTED:-1}" +export WORKER_PORT="${WORKER_PORT:-4002}" +export WORKER_URL="${WORKER_URL:-http://127.0.0.1:4002}" +export APPS_URL="${APPS_URL:-http://127.0.0.1:4001}" +export SERVER_TOP_LEVEL_PATH="${SERVER_TOP_LEVEL_PATH:-/app}" + +# set DATA_DIR and ensure the directory exists +export DATA_DIR="${DATA_DIR:-/data}" +mkdir -p "${DATA_DIR}" + +# mount NFS or GCP Filestore if FILESHARE_IP and FILESHARE_NAME are set +if [[ -n "${FILESHARE_IP}" && -n "${FILESHARE_NAME}" ]]; then echo "Mounting NFS share" apt update && apt install -y nfs-common nfs-kernel-server echo "Mount file share ${FILESHARE_IP}:/${FILESHARE_NAME} to ${DATA_DIR}" - mount -o nolock ${FILESHARE_IP}:/${FILESHARE_NAME} ${DATA_DIR} + mount -o nolock "${FILESHARE_IP}:/${FILESHARE_NAME}" "${DATA_DIR}" echo "Mounting result: $?" fi -if [ -f "${DATA_DIR}/.env" ]; then - # Read in the .env file and export the variables - for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done +# source environment variables from a .env file if it exists in DATA_DIR +if [[ -f "${DATA_DIR}/.env" ]]; then + set -a # Automatically export all variables loaded from .env + source "${DATA_DIR}/.env" + set +a fi -# randomise any unset environment variables -for ENV_VAR in "${ENV_VARS[@]}"; do - if [[ -z "${!ENV_VAR}" ]]; then - eval "export $ENV_VAR=$(uuidgen | sed -e 's/-//g')" + +# randomize any unset sensitive environment variables using uuidgen +env_vars=(COUCHDB_USER COUCHDB_PASSWORD MINIO_ACCESS_KEY MINIO_SECRET_KEY INTERNAL_API_KEY JWT_SECRET REDIS_PASSWORD) +for var in "${env_vars[@]}"; do + if [[ -z "${!var}" ]]; then + export "$var"="$(uuidgen | tr -d '-')" fi done + if [[ -z "${COUCH_DB_URL}" ]]; then export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@127.0.0.1:5984 fi @@ -88,10 +100,10 @@ fi # only start minio if use s3 isn't passed if [[ -z "${USE_S3}" ]]; then - if [[ $TARGETBUILD == aas ]]; then + if [[ ${TARGETBUILD} == aas ]]; then echo "Starting MinIO in Azure Gateway mode" - if [[ -z "${AZURE_STORAGE_ACCOUNT}" || -z "${AZURE_STORAGE_KEY}" ]]; then - echo "AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY must be set when deploying in Azure App Service mode" + if [[ -z "${AZURE_STORAGE_ACCOUNT}" || -z "${AZURE_STORAGE_KEY}" || -z "${MINIO_ACCESS_KEY}" || -z "${MINIO_SECRET_KEY}" ]]; then + echo "The following environment variables must be set: AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_KEY, MINIO_ACCESS_KEY, MINIO_SECRET_KEY" exit 1 fi /minio/minio gateway azure --console-address ":9001" >/dev/stdout 2>&1 & From 77f5c05a5b7fd39fc1fd26ffb2372dd548ce83c6 Mon Sep 17 00:00:00 2001 From: Christos Alexiou Date: Thu, 20 Feb 2025 21:29:55 +0200 Subject: [PATCH 10/51] debug messages --- hosting/single/runner.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 42305cf62e..6f81cc2fbd 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -1,6 +1,6 @@ #!/bin/bash -echo "Starting runner.sh" +echo "Starting runner.sh..." # set defaults for Docker-related variables export APP_PORT="${APP_PORT:-4001}" @@ -96,7 +96,9 @@ if [[ -n "${REDIS_PASSWORD}" ]]; then else redis-server "${REDIS_CONFIG}" >/dev/stdout 2>&1 & fi -/bbcouch-runner.sh & + +echo "Starting callback CouchDB runner..." +./bbcouch-runner.sh & # only start minio if use s3 isn't passed if [[ -z "${USE_S3}" ]]; then From bf2fcea4338b58259623c7ea0af2d3f9c5674be2 Mon Sep 17 00:00:00 2001 From: Christos Alexiou Date: Thu, 20 Feb 2025 21:36:01 +0200 Subject: [PATCH 11/51] fix issue with couchdb startup --- hosting/single/runner.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 6f81cc2fbd..1a81515d31 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -25,7 +25,11 @@ export APPS_URL="${APPS_URL:-http://127.0.0.1:4001}" export SERVER_TOP_LEVEL_PATH="${SERVER_TOP_LEVEL_PATH:-/app}" # set DATA_DIR and ensure the directory exists -export DATA_DIR="${DATA_DIR:-/data}" +if [[ ${TARGETBUILD} == "aas" ]]; then + export DATA_DIR="/home" +else + export DATA_DIR="${DATA_DIR:-/data}" +fi mkdir -p "${DATA_DIR}" # mount NFS or GCP Filestore if FILESHARE_IP and FILESHARE_NAME are set @@ -81,8 +85,8 @@ ln -s ${DATA_DIR}/.env /worker/.env # make these directories in runner, incase of mount mkdir -p ${DATA_DIR}/minio mkdir -p ${DATA_DIR}/redis -#mkdir -p ${DATA_DIR}/couch -#chown -R couchdb:couchdb ${DATA_DIR}/couch +mkdir -p ${DATA_DIR}/couch +chown -R couchdb:couchdb ${DATA_DIR}/couch REDIS_CONFIG="/etc/redis/redis.conf" sed -i "s#DATA_DIR#${DATA_DIR}#g" "${REDIS_CONFIG}" From de62913afdf96e7c8c02d8c2c16b8f037cca0966 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 20 Feb 2025 23:16:58 +0100 Subject: [PATCH 12/51] [Revert] store koa sessions in redis instead of cookies --- hosting/nginx.dev.conf | 6 ---- packages/worker/package.json | 1 - .../src/api/routes/global/tests/auth.spec.ts | 2 +- packages/worker/src/index.ts | 26 ++--------------- packages/worker/src/koa-redis.d.ts | 1 - yarn.lock | 28 ++----------------- 6 files changed, 5 insertions(+), 59 deletions(-) delete mode 100644 packages/worker/src/koa-redis.d.ts diff --git a/hosting/nginx.dev.conf b/hosting/nginx.dev.conf index a8cefe9ccc..747235e8ef 100644 --- a/hosting/nginx.dev.conf +++ b/hosting/nginx.dev.conf @@ -62,12 +62,6 @@ http { proxy_connect_timeout 120s; proxy_send_timeout 120s; proxy_http_version 1.1; - - # Enable buffering for potentially large OIDC configs - proxy_buffering on; - proxy_buffer_size 16k; - proxy_buffers 4 32k; - proxy_set_header Host $host; proxy_set_header Connection ""; diff --git a/packages/worker/package.json b/packages/worker/package.json index 28728272ca..53d14dacee 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -62,7 +62,6 @@ "koa-body": "4.2.0", "koa-compress": "4.0.1", "koa-passport": "4.1.4", - "koa-redis": "^4.0.1", "koa-send": "5.0.1", "koa-session": "5.13.1", "koa-static": "5.0.0", diff --git a/packages/worker/src/api/routes/global/tests/auth.spec.ts b/packages/worker/src/api/routes/global/tests/auth.spec.ts index f89cb4a027..bff959469e 100644 --- a/packages/worker/src/api/routes/global/tests/auth.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auth.spec.ts @@ -311,7 +311,7 @@ describe("/api/global/auth", () => { }) }) - describe.skip("GET /api/global/auth/:tenantId/oidc/callback", () => { + describe("GET /api/global/auth/:tenantId/oidc/callback", () => { it("logs in", async () => { const email = `${generator.guid()}@example.com` diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index f382aa8a20..0547afab38 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -4,7 +4,7 @@ if (process.env.DD_APM_ENABLED) { // need to load environment first import env from "./environment" -import Application, { Middleware } from "koa" +import Application from "koa" import { bootstrap } from "global-agent" import * as db from "./db" import { sdk as proSdk } from "@budibase/pro" @@ -20,7 +20,6 @@ import { cache, features, } from "@budibase/backend-core" -import RedisStore from "koa-redis" db.init() import koaBody from "koa-body" @@ -53,28 +52,7 @@ app.proxy = true app.use(handleScimBody) app.use(koaBody({ multipart: true })) -const sessionMiddleware: Middleware = async (ctx: any, next: any) => { - const redisClient = await new redis.Client( - redis.utils.Databases.SESSIONS - ).init() - return koaSession( - { - // @ts-ignore - store: new RedisStore({ client: redisClient.getClient() }), - key: "koa:sess", - maxAge: 86400000, // one day - httpOnly: true, - secure: process.env.NODE_ENV === "production", - sameSite: "strict", - rolling: true, - renew: true, - }, - app - )(ctx, next) -} - -app.use(sessionMiddleware) - +app.use(koaSession(app)) app.use(middleware.correlation) app.use(middleware.pino) app.use(middleware.ip) diff --git a/packages/worker/src/koa-redis.d.ts b/packages/worker/src/koa-redis.d.ts deleted file mode 100644 index ad1b7a46f1..0000000000 --- a/packages/worker/src/koa-redis.d.ts +++ /dev/null @@ -1 +0,0 @@ -declare module "koa-redis" {} diff --git a/yarn.lock b/yarn.lock index 8f611e224c..ceae41458c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2695,13 +2695,6 @@ dependencies: regenerator-runtime "^0.14.0" -"@babel/runtime@^7.8.3": - version "7.26.9" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.26.9.tgz#aa4c6facc65b9cb3f87d75125ffd47781b475433" - integrity sha512-aA63XwOkcl4xxQa3HjPMqOP6LiK0ZDv3mUPYEFXkpHbaFjtGggE1A61FjFzJnB+p7/oy2gA8E+rcBNl/zC1tMg== - dependencies: - regenerator-runtime "^0.14.0" - "@babel/template@^7.22.15", "@babel/template@^7.22.5", "@babel/template@^7.25.9", "@babel/template@^7.3.3": version "7.25.9" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.25.9.tgz#ecb62d81a8a6f5dc5fe8abfc3901fc52ddf15016" @@ -9048,14 +9041,7 @@ co-body@^5.1.1: raw-body "^2.2.0" type-is "^1.6.14" -co-wrap-all@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/co-wrap-all/-/co-wrap-all-1.0.0.tgz#370ae3e8333510a53f6b2f7fdfbe4568a11b7ecf" - integrity sha512-aru6gLi2vTUazr+MxVm3Rv6ST7/EKtFj9BrfkcOrbCO2Qv6LqJdE71m88HhHiBEviKw/ucVrwoGLrq2xHpOsJA== - dependencies: - co "^4.0.0" - -co@^4.0.0, co@^4.6.0: +co@^4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" integrity sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ== @@ -13191,7 +13177,7 @@ ioredis@5.3.2: redis-parser "^3.0.0" standard-as-callback "^2.1.0" -ioredis@^4.14.1, ioredis@^4.28.5: +ioredis@^4.28.5: version "4.28.5" resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.28.5.tgz#5c149e6a8d76a7f8fa8a504ffc85b7d5b6797f9f" integrity sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A== @@ -14691,16 +14677,6 @@ koa-pino-logger@4.0.0: dependencies: pino-http "^6.5.0" -koa-redis@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/koa-redis/-/koa-redis-4.0.1.tgz#57ac1b46d9ab851221a9f4952c1e8d4bf289db40" - integrity sha512-o2eTVNo1NBnloeUGhHed5Q2ZvJSLpUEj/+E1/7oH5EmH8WuQ+QLdl/VawkshxdFQ47W1p6V09lM3hCTu7D0YnQ== - dependencies: - "@babel/runtime" "^7.8.3" - co-wrap-all "^1.0.0" - debug "^4.1.1" - ioredis "^4.14.1" - koa-router@^10.0.0: version "10.1.1" resolved "https://registry.yarnpkg.com/koa-router/-/koa-router-10.1.1.tgz#20809f82648518b84726cd445037813cd99f17ff" From add89b87d230d755b949cc4d53cb2f6227777acf Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 21 Feb 2025 14:02:45 +0000 Subject: [PATCH 13/51] Bump version to 3.4.16 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 91980e0a15..bb71d10f41 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.4.15", + "version": "3.4.16", "npmClient": "yarn", "concurrency": 20, "command": { From 17cdf079da429a8b404eb4f47d74e7d729840b77 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 24 Feb 2025 13:47:13 +0000 Subject: [PATCH 14/51] Revert "[Revert] store koa sessions in redis instead of cookies" --- hosting/nginx.dev.conf | 6 ++++ packages/worker/package.json | 1 + .../src/api/routes/global/tests/auth.spec.ts | 2 +- packages/worker/src/index.ts | 26 +++++++++++++++-- packages/worker/src/koa-redis.d.ts | 1 + yarn.lock | 28 +++++++++++++++++-- 6 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 packages/worker/src/koa-redis.d.ts diff --git a/hosting/nginx.dev.conf b/hosting/nginx.dev.conf index 747235e8ef..a8cefe9ccc 100644 --- a/hosting/nginx.dev.conf +++ b/hosting/nginx.dev.conf @@ -62,6 +62,12 @@ http { proxy_connect_timeout 120s; proxy_send_timeout 120s; proxy_http_version 1.1; + + # Enable buffering for potentially large OIDC configs + proxy_buffering on; + proxy_buffer_size 16k; + proxy_buffers 4 32k; + proxy_set_header Host $host; proxy_set_header Connection ""; diff --git a/packages/worker/package.json b/packages/worker/package.json index 53d14dacee..28728272ca 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -62,6 +62,7 @@ "koa-body": "4.2.0", "koa-compress": "4.0.1", "koa-passport": "4.1.4", + "koa-redis": "^4.0.1", "koa-send": "5.0.1", "koa-session": "5.13.1", "koa-static": "5.0.0", diff --git a/packages/worker/src/api/routes/global/tests/auth.spec.ts b/packages/worker/src/api/routes/global/tests/auth.spec.ts index bff959469e..f89cb4a027 100644 --- a/packages/worker/src/api/routes/global/tests/auth.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auth.spec.ts @@ -311,7 +311,7 @@ describe("/api/global/auth", () => { }) }) - describe("GET /api/global/auth/:tenantId/oidc/callback", () => { + describe.skip("GET /api/global/auth/:tenantId/oidc/callback", () => { it("logs in", async () => { const email = `${generator.guid()}@example.com` diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 0547afab38..f382aa8a20 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -4,7 +4,7 @@ if (process.env.DD_APM_ENABLED) { // need to load environment first import env from "./environment" -import Application from "koa" +import Application, { Middleware } from "koa" import { bootstrap } from "global-agent" import * as db from "./db" import { sdk as proSdk } from "@budibase/pro" @@ -20,6 +20,7 @@ import { cache, features, } from "@budibase/backend-core" +import RedisStore from "koa-redis" db.init() import koaBody from "koa-body" @@ -52,7 +53,28 @@ app.proxy = true app.use(handleScimBody) app.use(koaBody({ multipart: true })) -app.use(koaSession(app)) +const sessionMiddleware: Middleware = async (ctx: any, next: any) => { + const redisClient = await new redis.Client( + redis.utils.Databases.SESSIONS + ).init() + return koaSession( + { + // @ts-ignore + store: new RedisStore({ client: redisClient.getClient() }), + key: "koa:sess", + maxAge: 86400000, // one day + httpOnly: true, + secure: process.env.NODE_ENV === "production", + sameSite: "strict", + rolling: true, + renew: true, + }, + app + )(ctx, next) +} + +app.use(sessionMiddleware) + app.use(middleware.correlation) app.use(middleware.pino) app.use(middleware.ip) diff --git a/packages/worker/src/koa-redis.d.ts b/packages/worker/src/koa-redis.d.ts new file mode 100644 index 0000000000..ad1b7a46f1 --- /dev/null +++ b/packages/worker/src/koa-redis.d.ts @@ -0,0 +1 @@ +declare module "koa-redis" {} diff --git a/yarn.lock b/yarn.lock index ceae41458c..8f611e224c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2695,6 +2695,13 @@ dependencies: regenerator-runtime "^0.14.0" +"@babel/runtime@^7.8.3": + version "7.26.9" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.26.9.tgz#aa4c6facc65b9cb3f87d75125ffd47781b475433" + integrity sha512-aA63XwOkcl4xxQa3HjPMqOP6LiK0ZDv3mUPYEFXkpHbaFjtGggE1A61FjFzJnB+p7/oy2gA8E+rcBNl/zC1tMg== + dependencies: + regenerator-runtime "^0.14.0" + "@babel/template@^7.22.15", "@babel/template@^7.22.5", "@babel/template@^7.25.9", "@babel/template@^7.3.3": version "7.25.9" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.25.9.tgz#ecb62d81a8a6f5dc5fe8abfc3901fc52ddf15016" @@ -9041,7 +9048,14 @@ co-body@^5.1.1: raw-body "^2.2.0" type-is "^1.6.14" -co@^4.6.0: +co-wrap-all@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/co-wrap-all/-/co-wrap-all-1.0.0.tgz#370ae3e8333510a53f6b2f7fdfbe4568a11b7ecf" + integrity sha512-aru6gLi2vTUazr+MxVm3Rv6ST7/EKtFj9BrfkcOrbCO2Qv6LqJdE71m88HhHiBEviKw/ucVrwoGLrq2xHpOsJA== + dependencies: + co "^4.0.0" + +co@^4.0.0, co@^4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" integrity sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ== @@ -13177,7 +13191,7 @@ ioredis@5.3.2: redis-parser "^3.0.0" standard-as-callback "^2.1.0" -ioredis@^4.28.5: +ioredis@^4.14.1, ioredis@^4.28.5: version "4.28.5" resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.28.5.tgz#5c149e6a8d76a7f8fa8a504ffc85b7d5b6797f9f" integrity sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A== @@ -14677,6 +14691,16 @@ koa-pino-logger@4.0.0: dependencies: pino-http "^6.5.0" +koa-redis@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/koa-redis/-/koa-redis-4.0.1.tgz#57ac1b46d9ab851221a9f4952c1e8d4bf289db40" + integrity sha512-o2eTVNo1NBnloeUGhHed5Q2ZvJSLpUEj/+E1/7oH5EmH8WuQ+QLdl/VawkshxdFQ47W1p6V09lM3hCTu7D0YnQ== + dependencies: + "@babel/runtime" "^7.8.3" + co-wrap-all "^1.0.0" + debug "^4.1.1" + ioredis "^4.14.1" + koa-router@^10.0.0: version "10.1.1" resolved "https://registry.yarnpkg.com/koa-router/-/koa-router-10.1.1.tgz#20809f82648518b84726cd445037813cd99f17ff" From c9d31dcd0620881b33e7571813c1b702208f431f Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 24 Feb 2025 13:49:31 +0000 Subject: [PATCH 15/51] updating cookie settings for koa-session --- packages/worker/src/index.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index f382aa8a20..bfb022f213 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -63,11 +63,6 @@ const sessionMiddleware: Middleware = async (ctx: any, next: any) => { store: new RedisStore({ client: redisClient.getClient() }), key: "koa:sess", maxAge: 86400000, // one day - httpOnly: true, - secure: process.env.NODE_ENV === "production", - sameSite: "strict", - rolling: true, - renew: true, }, app )(ctx, next) From 9c0d7c027884630ec7d372f56a6f47cae5eb15ee Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 24 Feb 2025 14:47:53 +0000 Subject: [PATCH 16/51] Fix bug with branches when no conditions are set. --- .../src/automations/tests/branching.spec.ts | 26 ++++++++++++++++++- packages/server/src/threads/automation.ts | 2 ++ packages/types/src/sdk/search.ts | 3 ++- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/packages/server/src/automations/tests/branching.spec.ts b/packages/server/src/automations/tests/branching.spec.ts index bf9b9ce3f8..70aa4cbaa7 100644 --- a/packages/server/src/automations/tests/branching.spec.ts +++ b/packages/server/src/automations/tests/branching.spec.ts @@ -1,5 +1,10 @@ import * as automation from "../index" -import { Table, AutomationStatus } from "@budibase/types" +import { + Table, + AutomationStatus, + AutomationStepStatus, + EmptyFilterOption, +} from "@budibase/types" import { createAutomationBuilder } from "./utilities/AutomationTestBuilder" import TestConfiguration from "../../tests/utilities/TestConfiguration" @@ -280,4 +285,23 @@ describe("Branching automations", () => { expect(results.steps[2].outputs.message).toContain("Special user") }) + + it("should not fail with empty conditions", async () => { + const results = await createAutomationBuilder(config) + .onAppAction() + .branch({ + specialBranch: { + steps: stepBuilder => stepBuilder.serverLog({ text: "Hello!" }), + condition: { + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + }, + }, + }) + .test({ fields: { test_trigger: true } }) + + expect(results.steps[0].outputs.success).toEqual(false) + expect(results.steps[0].outputs.status).toEqual( + AutomationStatus.NO_CONDITION_MET + ) + }) }) diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 762da1cbc1..def2ab4201 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -367,6 +367,8 @@ class Orchestrator { if (e.errno === "ETIME") { span?.addTags({ timedOut: true }) console.warn(`Automation execution timed out after ${timeout}ms`) + } else { + throw e } } diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 992e9961d4..99c3658fa8 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -3,6 +3,7 @@ import { Row, DocumentType, Table, Datasource } from "../documents" import { SortOrder, SortType } from "../api" import { Knex } from "knex" import { Aggregation } from "./row" +import _ from "lodash" export enum BasicOperator { EQUAL = "equal", @@ -83,7 +84,7 @@ type RangeFilter = Record< type LogicalFilter = { conditions: SearchFilters[] } export function isLogicalFilter(filter: any): filter is LogicalFilter { - return "conditions" in filter + return _.isPlainObject(filter) && "conditions" in filter } export type AnySearchFilter = BasicFilter | ArrayFilter | RangeFilter From 68b72acb717249059bad54360794a521c8519e8d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 24 Feb 2025 15:06:12 +0000 Subject: [PATCH 17/51] Fix lint. --- packages/server/src/automations/tests/branching.spec.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/server/src/automations/tests/branching.spec.ts b/packages/server/src/automations/tests/branching.spec.ts index 70aa4cbaa7..4572871c44 100644 --- a/packages/server/src/automations/tests/branching.spec.ts +++ b/packages/server/src/automations/tests/branching.spec.ts @@ -1,10 +1,5 @@ import * as automation from "../index" -import { - Table, - AutomationStatus, - AutomationStepStatus, - EmptyFilterOption, -} from "@budibase/types" +import { Table, AutomationStatus, EmptyFilterOption } from "@budibase/types" import { createAutomationBuilder } from "./utilities/AutomationTestBuilder" import TestConfiguration from "../../tests/utilities/TestConfiguration" From a959a26d60a4fae91e939382ed447b991577c49d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 24 Feb 2025 16:15:59 +0000 Subject: [PATCH 18/51] Support filtering relationships by _id --- .../src/api/routes/tests/search.spec.ts | 22 +++++++++++++++++++ .../server/src/sdk/app/rows/queryUtils.ts | 7 +++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index ee372914d7..ba2b3f0acf 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -3553,6 +3553,28 @@ if (descriptions.length) { limit: 1, }).toContainExactly([row]) }) + + it("can filter by the related _id", async () => { + await expectSearch({ + query: { + equal: { "rel._id": row.rel[0]._id }, + }, + }).toContainExactly([row]) + + await expectSearch({ + query: { + equal: { "rel._id": row.rel[1]._id }, + }, + }).toContainExactly([row]) + }) + + it("can filter by the related _id and find nothing", async () => { + await expectSearch({ + query: { + equal: { "rel._id": "rel_none" }, + }, + }).toFindNothing() + }) }) !isInternal && diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index ddd32870be..12e724b5d8 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -69,7 +69,8 @@ export const getQueryableFields = async ( fromTables: string[], opts?: { noRelationships?: boolean } ): Promise => { - const result = [] + // Querying by _id is always allowed, even if it's never part of the schema + const result = ["_id"] for (const field of Object.keys(table.schema).filter( f => allowedFields.includes(f) && table.schema[f].visible !== false )) { @@ -113,9 +114,7 @@ export const getQueryableFields = async ( return result } - const result = [ - "_id", // Querying by _id is always allowed, even if it's never part of the schema - ] + const result = [] if (fields == null) { fields = Object.keys(table.schema) From a88530438c8be7e5ebed22e09c748ed315c12c55 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 24 Feb 2025 16:36:08 +0000 Subject: [PATCH 19/51] Remove view enrichment unit test, covered by integration tests. --- .../src/sdk/app/tables/tests/tables.spec.ts | 108 ------------------ 1 file changed, 108 deletions(-) delete mode 100644 packages/server/src/sdk/app/tables/tests/tables.spec.ts diff --git a/packages/server/src/sdk/app/tables/tests/tables.spec.ts b/packages/server/src/sdk/app/tables/tests/tables.spec.ts deleted file mode 100644 index 41ac808f5c..0000000000 --- a/packages/server/src/sdk/app/tables/tests/tables.spec.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { - FieldType, - INTERNAL_TABLE_SOURCE_ID, - Table, - TableSourceType, - ViewV2, -} from "@budibase/types" -import { generator } from "@budibase/backend-core/tests" -import sdk from "../../.." - -jest.mock("../../views", () => ({ - ...jest.requireActual("../../views"), - enrichSchema: jest.fn().mockImplementation(v => ({ ...v, mocked: true })), -})) - -describe("table sdk", () => { - describe("enrichViewSchemas", () => { - const basicTable: Table = { - _id: generator.guid(), - name: "TestTable", - type: "table", - sourceId: INTERNAL_TABLE_SOURCE_ID, - sourceType: TableSourceType.INTERNAL, - schema: { - name: { - type: FieldType.STRING, - name: "name", - visible: true, - width: 80, - order: 2, - constraints: { - type: "string", - }, - }, - description: { - type: FieldType.STRING, - name: "description", - visible: true, - width: 200, - constraints: { - type: "string", - }, - }, - id: { - type: FieldType.NUMBER, - name: "id", - visible: true, - order: 1, - constraints: { - type: "number", - }, - }, - hiddenField: { - type: FieldType.STRING, - name: "hiddenField", - visible: false, - constraints: { - type: "string", - }, - }, - }, - } - - it("should fetch the default schema if not overriden", async () => { - const tableId = basicTable._id! - function getTable() { - const view: ViewV2 = { - version: 2, - id: generator.guid(), - name: generator.guid(), - tableId, - } - return view - } - const view1 = getTable() - const view2 = getTable() - const view3 = getTable() - const res = await sdk.tables.enrichViewSchemas({ - ...basicTable, - views: { - [view1.name]: view1, - [view2.name]: view2, - [view3.name]: view3, - }, - }) - - expect(sdk.views.enrichSchema).toHaveBeenCalledTimes(3) - - expect(res).toEqual({ - ...basicTable, - views: { - [view1.name]: { - ...view1, - mocked: true, - }, - [view2.name]: { - ...view2, - mocked: true, - }, - [view3.name]: { - ...view3, - mocked: true, - }, - }, - }) - }) - }) -}) From bf486542c8cb15c55f490ee8cec4314154d9ea65 Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 24 Feb 2025 16:38:03 +0000 Subject: [PATCH 20/51] Fix for automation traversal to ensure all paths calculate their terminated condition --- packages/builder/src/stores/builder/automations.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/builder/src/stores/builder/automations.ts b/packages/builder/src/stores/builder/automations.ts index 039a057a1b..eeee973290 100644 --- a/packages/builder/src/stores/builder/automations.ts +++ b/packages/builder/src/stores/builder/automations.ts @@ -484,7 +484,7 @@ const automationActions = (store: AutomationStore) => ({ branches.forEach((branch, bIdx) => { children[branch.id].forEach( (bBlock: AutomationStep, sIdx: number, array: AutomationStep[]) => { - const ended = array.length - 1 === sIdx && !branches.length + const ended = array.length - 1 === sIdx treeTraverse(bBlock, pathToCurrentNode, sIdx, bIdx, ended) } ) @@ -505,7 +505,6 @@ const automationActions = (store: AutomationStore) => ({ blocks.forEach((block, idx, array) => { treeTraverse(block, null, idx, null, array.length - 1 === idx) }) - return blockRefs }, From 1dc52cfa75ea8ab9ce98539fa088bfc93acac792 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 24 Feb 2025 17:19:02 +0000 Subject: [PATCH 21/51] Improve typing around nodemailer. --- .../server/src/utilities/workerRequests.ts | 40 ++-- packages/types/src/api/web/global/email.ts | 6 +- .../documents/app/automation/automation.ts | 8 +- packages/worker/package.json | 1 + .../src/api/controllers/global/email.ts | 7 +- packages/worker/src/utilities/email.ts | 7 +- yarn.lock | 194 +++++++++++++----- 7 files changed, 184 insertions(+), 79 deletions(-) diff --git a/packages/server/src/utilities/workerRequests.ts b/packages/server/src/utilities/workerRequests.ts index 0f487d9f31..dd1493b82f 100644 --- a/packages/server/src/utilities/workerRequests.ts +++ b/packages/server/src/utilities/workerRequests.ts @@ -8,7 +8,15 @@ import { logging, env as coreEnv, } from "@budibase/backend-core" -import { Ctx, User, EmailInvite, EmailAttachment } from "@budibase/types" +import { + Ctx, + User, + EmailInvite, + EmailAttachment, + SendEmailResponse, + SendEmailRequest, + EmailTemplatePurpose, +} from "@budibase/types" interface Request { ctx?: Ctx @@ -110,25 +118,23 @@ export async function sendSmtpEmail({ invite?: EmailInvite }) { // tenant ID will be set in header + const request: SendEmailRequest = { + email: to, + from, + contents, + subject, + cc, + bcc, + purpose: EmailTemplatePurpose.CUSTOM, + automation, + invite, + attachments, + } const response = await fetch( checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`), - createRequest({ - method: "POST", - body: { - email: to, - from, - contents, - subject, - cc, - bcc, - purpose: "custom", - automation, - invite, - attachments, - }, - }) + createRequest({ method: "POST", body: request }) ) - return checkResponse(response, "send email") + return (await checkResponse(response, "send email")) as SendEmailResponse } export async function removeAppFromUserRoles(ctx: Ctx, appId: string) { diff --git a/packages/types/src/api/web/global/email.ts b/packages/types/src/api/web/global/email.ts index a0ca0e8485..13b22a939c 100644 --- a/packages/types/src/api/web/global/email.ts +++ b/packages/types/src/api/web/global/email.ts @@ -12,13 +12,13 @@ export enum EmailTemplatePurpose { export interface SendEmailRequest { workspaceId?: string email: string - userId: string + userId?: string purpose: EmailTemplatePurpose contents?: string from?: string subject: string - cc?: boolean - bcc?: boolean + cc?: string + bcc?: string automation?: boolean invite?: EmailInvite attachments?: EmailAttachment[] diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts index d5ef35d059..cfe2ba5147 100644 --- a/packages/types/src/documents/app/automation/automation.ts +++ b/packages/types/src/documents/app/automation/automation.ts @@ -1,10 +1,10 @@ import { Document } from "../../document" import { User } from "../../global" -import { ReadStream } from "fs" import { Row } from "../row" import { Table } from "../table" import { AutomationStep, AutomationTrigger } from "./schema" import { ContextEmitter } from "../../../sdk" +import { Readable } from "stream" export enum AutomationIOType { OBJECT = "object", @@ -108,8 +108,8 @@ export interface SendEmailOpts { subject: string // info Pass in a structure of information to be stored alongside the invitation. info?: any - cc?: boolean - bcc?: boolean + cc?: string + bcc?: string automation?: boolean invite?: EmailInvite attachments?: EmailAttachment[] @@ -269,7 +269,7 @@ export type AutomationAttachment = { export type AutomationAttachmentContent = { filename: string - content: ReadStream | NodeJS.ReadableStream + content: Readable } export type BucketedContent = AutomationAttachmentContent & { diff --git a/packages/worker/package.json b/packages/worker/package.json index 53d14dacee..ee7cf03779 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -85,6 +85,7 @@ "@types/koa__router": "12.0.4", "@types/lodash": "4.14.200", "@types/node-fetch": "2.6.4", + "@types/nodemailer": "^6.4.17", "@types/server-destroy": "1.0.1", "@types/supertest": "2.0.14", "@types/uuid": "8.3.4", diff --git a/packages/worker/src/api/controllers/global/email.ts b/packages/worker/src/api/controllers/global/email.ts index ad0fc3fa32..ed2d9b5125 100644 --- a/packages/worker/src/api/controllers/global/email.ts +++ b/packages/worker/src/api/controllers/global/email.ts @@ -24,10 +24,13 @@ export async function sendEmail( invite, attachments, } = ctx.request.body - let user: any + let user: User | undefined = undefined if (userId) { const db = tenancy.getGlobalDB() - user = await db.get(userId) + user = await db.tryGet(userId) + } + if (!user) { + ctx.throw(404, "User not found.") } const response = await sendEmailFn(email, purpose, { workspaceId, diff --git a/packages/worker/src/utilities/email.ts b/packages/worker/src/utilities/email.ts index fa9dd7a6fa..a2b9c3bfc2 100644 --- a/packages/worker/src/utilities/email.ts +++ b/packages/worker/src/utilities/email.ts @@ -13,7 +13,8 @@ import { configs, cache, objectStore } from "@budibase/backend-core" import ical from "ical-generator" import _ from "lodash" -const nodemailer = require("nodemailer") +import nodemailer from "nodemailer" +import SMTPTransport from "nodemailer/lib/smtp-transport" const TEST_MODE = env.ENABLE_EMAIL_TEST_MODE && env.isDev() const TYPE = TemplateType.EMAIL @@ -26,7 +27,7 @@ const FULL_EMAIL_PURPOSES = [ ] function createSMTPTransport(config?: SMTPInnerConfig) { - let options: any + let options: SMTPTransport.Options let secure = config?.secure // default it if not specified if (secure == null) { @@ -161,7 +162,7 @@ export async function sendEmail( const code = await getLinkCode(purpose, email, opts.user, opts?.info) let context = await getSettingsTemplateContext(purpose, code) - let message: any = { + let message: Parameters[0] = { from: opts?.from || config?.from, html: await buildEmail(purpose, email, context, { user: opts?.user, diff --git a/yarn.lock b/yarn.lock index ceae41458c..8df30c905b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1457,7 +1457,7 @@ "@azure/abort-controller" "^2.0.0" tslib "^2.6.2" -"@azure/identity@4.2.1", "@azure/identity@^4.2.1": +"@azure/identity@^4.2.1": version "4.2.1" resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-4.2.1.tgz#22b366201e989b7b41c0e1690e103bd579c31e4c" integrity sha512-U8hsyC9YPcEIzoaObJlRDvp7KiF0MGS7xcWbyJSVvXRkC/HXo1f0oYeBYmEvVgRfacw7GHf6D6yAoh9JHz6A5Q== @@ -2777,28 +2777,6 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@npm:@budibase/pro@latest": - version "3.4.12" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.12.tgz#60e630944de4e2de970a04179d8f0f57d48ce75e" - integrity sha512-msUBmcWxRDg+ugjZvd27XudERQqtQRdiARsO8MaDVTcp5ejIXgshEIVVshHOCj3hcbRblw9pXvBIMI53iTMUsA== - dependencies: - "@anthropic-ai/sdk" "^0.27.3" - "@budibase/backend-core" "*" - "@budibase/shared-core" "*" - "@budibase/string-templates" "*" - "@budibase/types" "*" - "@koa/router" "13.1.0" - bull "4.10.1" - dd-trace "5.26.0" - joi "17.6.0" - jsonwebtoken "9.0.2" - lru-cache "^7.14.1" - memorystream "^0.3.1" - node-fetch "2.6.7" - openai "4.59.0" - scim-patch "^0.8.1" - scim2-parse-filter "^0.2.8" - "@budibase/vm-browserify@^1.1.4": version "1.1.4" resolved "https://registry.yarnpkg.com/@budibase/vm-browserify/-/vm-browserify-1.1.4.tgz#eecb001bd9521cb7647e26fb4d2d29d0a4dce262" @@ -6768,6 +6746,13 @@ dependencies: undici-types "~6.19.2" +"@types/nodemailer@^6.4.17": + version "6.4.17" + resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.17.tgz#5c82a42aee16a3dd6ea31446a1bd6a447f1ac1a4" + integrity sha512-I9CCaIp6DTldEg7vyUTZi8+9Vo0hi1/T8gv3C89yk1rSAAzoKQ8H8ki/jBYJSFoH/BisgLP8tkZMlQ91CIquww== + dependencies: + "@types/node" "*" + "@types/normalize-package-data@^2.4.0": version "2.4.1" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" @@ -8129,7 +8114,23 @@ aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.13.2.tgz#0aa167216965ac9474ccfa83892cfb6b3e1e52ef" integrity sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw== -axios@1.1.3, axios@1.7.7, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.6.2, axios@^1.6.8: +axios@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" + integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + +axios@^0.21.1: + version "0.21.4" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" + integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== + dependencies: + follow-redirects "^1.14.0" + +axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.6.2, axios@^1.6.8: version "1.7.7" resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.7.tgz#2f554296f9892a72ac8d8e4c5b79c14a91d0a47f" integrity sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q== @@ -11763,7 +11764,7 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== -follow-redirects@^1.15.6: +follow-redirects@^1.14.0, follow-redirects@^1.15.0, follow-redirects@^1.15.6: version "1.15.9" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== @@ -12378,10 +12379,22 @@ global@~4.4.0: min-document "^2.19.0" process "^0.11.10" -globals@15.13.0, globals@^11.1.0, globals@^13.19.0, globals@^14.0.0: - version "15.13.0" - resolved "https://registry.yarnpkg.com/globals/-/globals-15.13.0.tgz#bbec719d69aafef188ecd67954aae76a696010fc" - integrity sha512-49TewVEz0UxZjr1WYYsWpPrhyC/B/pA8Bq0fUmet2n+eR7yn0IvNzNaoBwnK6mdkzcN+se7Ez9zUgULTz2QH4g== +globals@^11.1.0: + version "11.12.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" + integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== + +globals@^13.19.0: + version "13.24.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-13.24.0.tgz#8432a19d78ce0c1e833949c36adb345400bb1171" + integrity sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ== + dependencies: + type-fest "^0.20.2" + +globals@^14.0.0: + version "14.0.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-14.0.0.tgz#898d7413c29babcf6bafe56fcadded858ada724e" + integrity sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ== globalthis@^1.0.1, globalthis@^1.0.4: version "1.0.4" @@ -12791,7 +12804,12 @@ http-assert@^1.3.0: deep-equal "~1.0.1" http-errors "~1.8.0" -http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: +http-cache-semantics@3.8.1: + version "3.8.1" + resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2" + integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w== + +http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -13247,6 +13265,11 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" +is-buffer@^1.1.5: + version "1.1.6" + resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" + integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== + is-buffer@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-2.0.5.tgz#ebc252e400d22ff8d77fa09888821a24a658c191" @@ -13686,11 +13709,6 @@ isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== -isobject@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" - integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== - isolated-vm@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65" @@ -14570,7 +14588,14 @@ kill-port@^1.6.1: get-them-args "1.3.2" shell-exec "1.0.2" -kind-of@6.0.3, kind-of@^3.0.2, kind-of@^3.1.0, kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: +kind-of@^3.0.2, kind-of@^3.1.0: + version "3.2.2" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" + integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ== + dependencies: + is-buffer "^1.1.5" + +kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: version "6.0.3" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== @@ -15953,7 +15978,7 @@ msgpackr-extract@^3.0.2: "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2" "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2" -msgpackr@1.10.1, msgpackr@^1.5.2: +msgpackr@^1.5.2: version "1.10.1" resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555" integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ== @@ -16146,13 +16171,27 @@ node-domexception@1.0.0: resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== -node-fetch@2.6.7, node-fetch@2.6.9, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9: +node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== dependencies: whatwg-url "^5.0.0" +node-fetch@2.6.9: + version "2.6.9" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.9.tgz#7c7f744b5cc6eb5fd404e0c7a9fec630a55657e6" + integrity sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg== + dependencies: + whatwg-url "^5.0.0" + +node-fetch@^2.6.9: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-forge@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -17181,7 +17220,15 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0: resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4" integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA== -passport@0.6.0, passport@^0.4.0, passport@^0.6.0: +passport@^0.4.0: + version "0.4.1" + resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270" + integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg== + dependencies: + passport-strategy "1.x.x" + pause "0.0.1" + +passport@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d" integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug== @@ -18166,6 +18213,13 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== +psl@^1.1.28: + version "1.15.0" + resolved "https://registry.yarnpkg.com/psl/-/psl-1.15.0.tgz#bdace31896f1d97cec6a79e8224898ce93d974c6" + integrity sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w== + dependencies: + punycode "^2.3.1" + psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" @@ -18199,6 +18253,11 @@ punycode@^2.1.0, punycode@^2.1.1, punycode@^2.3.0: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.0.tgz#f67fa67c94da8f4d0cfff981aee4118064199b8f" integrity sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA== +punycode@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" + integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg== + pupa@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/pupa/-/pupa-2.1.1.tgz#f5e8fd4afc2c5d97828faa523549ed8744a20d62" @@ -19030,6 +19089,11 @@ sax@1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== +sax@>=0.1.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" + integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== + sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -19102,13 +19166,28 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.2: +"semver@2 || 3 || 4 || 5", semver@^5.6.0, semver@^5.7.1: + version "5.7.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" + integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== + +semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" +semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + +semver@^7.5.4, semver@^7.6.0, semver@^7.6.2: + version "7.7.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.1.tgz#abd5098d82b18c6c81f6074ff2647fd3e7220c9f" + integrity sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA== + seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -20590,7 +20669,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: +"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -20600,6 +20679,14 @@ tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0 universalify "^0.2.0" url-parse "^1.5.3" +tough-cookie@~2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" + integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== + dependencies: + psl "^1.1.28" + punycode "^2.1.1" + tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -21067,14 +21154,6 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== -unset-value@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" - integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== - dependencies: - has-value "^2.0.2" - isobject "^4.0.0" - untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -21778,7 +21857,14 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x, xml2js@0.6.2, xml2js@^0.5.0: +xml2js@0.1.x: + version "0.1.14" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" + integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== + dependencies: + sax ">=0.1.1" + +xml2js@0.6.2: version "0.6.2" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== @@ -21786,6 +21872,14 @@ xml2js@0.1.x, xml2js@0.6.2, xml2js@^0.5.0: sax ">=0.6.0" xmlbuilder "~11.0.0" +xml2js@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" + integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + xmlbuilder@~11.0.0: version "11.0.1" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" From 81c404f88d3fd14a1ed1a89f87bc36cbf7494d0a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 24 Feb 2025 17:27:27 +0000 Subject: [PATCH 22/51] Correctly type things that interact with nodemailer. --- .../tests/steps/sendSmtpEmail.spec.ts | 20 ++++++++++--------- packages/types/package.json | 1 + packages/types/src/api/web/global/email.ts | 3 ++- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts b/packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts index 7452239dfa..7aff612a97 100644 --- a/packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts +++ b/packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts @@ -1,3 +1,4 @@ +import { SendEmailResponse } from "@budibase/types" import TestConfiguration from "../../../tests/utilities/TestConfiguration" import * as workerRequests from "../../../utilities/workerRequests" @@ -5,17 +6,18 @@ jest.mock("../../../utilities/workerRequests", () => ({ sendSmtpEmail: jest.fn(), })) -function generateResponse(to: string, from: string) { +function generateResponse(to: string, from: string): SendEmailResponse { return { - success: true, - response: { - accepted: [to], - envelope: { - from: from, - to: [to], - }, - message: `Email sent to ${to}.`, + message: `Email sent to ${to}.`, + accepted: [to], + envelope: { + from: from, + to: [to], }, + messageId: "messageId", + pending: [], + rejected: [], + response: "response", } } diff --git a/packages/types/package.json b/packages/types/package.json index ee3c059bc9..a6e08ab84c 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -17,6 +17,7 @@ "@budibase/nano": "10.1.5", "@types/json-schema": "^7.0.15", "@types/koa": "2.13.4", + "@types/nodemailer": "^6.4.17", "@types/redlock": "4.0.7", "koa-useragent": "^4.1.0", "rimraf": "3.0.2", diff --git a/packages/types/src/api/web/global/email.ts b/packages/types/src/api/web/global/email.ts index 13b22a939c..3d2e007231 100644 --- a/packages/types/src/api/web/global/email.ts +++ b/packages/types/src/api/web/global/email.ts @@ -1,4 +1,5 @@ import { EmailAttachment, EmailInvite } from "../../../documents" +import SMTPTransport from "nodemailer/lib/smtp-transport" export enum EmailTemplatePurpose { CORE = "core", @@ -23,6 +24,6 @@ export interface SendEmailRequest { invite?: EmailInvite attachments?: EmailAttachment[] } -export interface SendEmailResponse extends Record { +export interface SendEmailResponse extends SMTPTransport.SentMessageInfo { message: string } From ce7b2e69ab187470e8f367b76e67c66314a88d2c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 24 Feb 2025 17:30:14 +0000 Subject: [PATCH 23/51] Update yarn.lock. --- yarn.lock | 187 +++++++++++++++--------------------------------------- 1 file changed, 50 insertions(+), 137 deletions(-) diff --git a/yarn.lock b/yarn.lock index 8df30c905b..5a51409fd9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1457,7 +1457,7 @@ "@azure/abort-controller" "^2.0.0" tslib "^2.6.2" -"@azure/identity@^4.2.1": +"@azure/identity@4.2.1", "@azure/identity@^4.2.1": version "4.2.1" resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-4.2.1.tgz#22b366201e989b7b41c0e1690e103bd579c31e4c" integrity sha512-U8hsyC9YPcEIzoaObJlRDvp7KiF0MGS7xcWbyJSVvXRkC/HXo1f0oYeBYmEvVgRfacw7GHf6D6yAoh9JHz6A5Q== @@ -2777,6 +2777,28 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" +"@budibase/pro@npm:@budibase/pro@latest": + version "3.4.16" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.16.tgz#c482a400e27b7e89ca73092c4c81bdeac1d24581" + integrity sha512-8ECnqOh9jQ10KlQEwmKPFcoVGE+2gGgSybj+vbshwDp1zAW76doyMR2DMNjEatNpWVnpoMnTkDWtE9aqQ5v0vQ== + dependencies: + "@anthropic-ai/sdk" "^0.27.3" + "@budibase/backend-core" "*" + "@budibase/shared-core" "*" + "@budibase/string-templates" "*" + "@budibase/types" "*" + "@koa/router" "13.1.0" + bull "4.10.1" + dd-trace "5.26.0" + joi "17.6.0" + jsonwebtoken "9.0.2" + lru-cache "^7.14.1" + memorystream "^0.3.1" + node-fetch "2.6.7" + openai "4.59.0" + scim-patch "^0.8.1" + scim2-parse-filter "^0.2.8" + "@budibase/vm-browserify@^1.1.4": version "1.1.4" resolved "https://registry.yarnpkg.com/@budibase/vm-browserify/-/vm-browserify-1.1.4.tgz#eecb001bd9521cb7647e26fb4d2d29d0a4dce262" @@ -8114,23 +8136,7 @@ aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.13.2.tgz#0aa167216965ac9474ccfa83892cfb6b3e1e52ef" integrity sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw== -axios@1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" - integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== - dependencies: - follow-redirects "^1.15.0" - form-data "^4.0.0" - proxy-from-env "^1.1.0" - -axios@^0.21.1: - version "0.21.4" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" - integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== - dependencies: - follow-redirects "^1.14.0" - -axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.6.2, axios@^1.6.8: +axios@1.1.3, axios@1.7.7, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.6.2, axios@^1.6.8: version "1.7.7" resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.7.tgz#2f554296f9892a72ac8d8e4c5b79c14a91d0a47f" integrity sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q== @@ -11764,7 +11770,7 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== -follow-redirects@^1.14.0, follow-redirects@^1.15.0, follow-redirects@^1.15.6: +follow-redirects@^1.15.6: version "1.15.9" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== @@ -12379,22 +12385,10 @@ global@~4.4.0: min-document "^2.19.0" process "^0.11.10" -globals@^11.1.0: - version "11.12.0" - resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" - integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== - -globals@^13.19.0: - version "13.24.0" - resolved "https://registry.yarnpkg.com/globals/-/globals-13.24.0.tgz#8432a19d78ce0c1e833949c36adb345400bb1171" - integrity sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ== - dependencies: - type-fest "^0.20.2" - -globals@^14.0.0: - version "14.0.0" - resolved "https://registry.yarnpkg.com/globals/-/globals-14.0.0.tgz#898d7413c29babcf6bafe56fcadded858ada724e" - integrity sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ== +globals@15.13.0, globals@^11.1.0, globals@^13.19.0, globals@^14.0.0: + version "15.13.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-15.13.0.tgz#bbec719d69aafef188ecd67954aae76a696010fc" + integrity sha512-49TewVEz0UxZjr1WYYsWpPrhyC/B/pA8Bq0fUmet2n+eR7yn0IvNzNaoBwnK6mdkzcN+se7Ez9zUgULTz2QH4g== globalthis@^1.0.1, globalthis@^1.0.4: version "1.0.4" @@ -12804,12 +12798,7 @@ http-assert@^1.3.0: deep-equal "~1.0.1" http-errors "~1.8.0" -http-cache-semantics@3.8.1: - version "3.8.1" - resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2" - integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w== - -http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: +http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -13265,11 +13254,6 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-buffer@^1.1.5: - version "1.1.6" - resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" - integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== - is-buffer@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-2.0.5.tgz#ebc252e400d22ff8d77fa09888821a24a658c191" @@ -13709,6 +13693,11 @@ isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== +isobject@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" + integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== + isolated-vm@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65" @@ -14588,14 +14577,7 @@ kill-port@^1.6.1: get-them-args "1.3.2" shell-exec "1.0.2" -kind-of@^3.0.2, kind-of@^3.1.0: - version "3.2.2" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" - integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ== - dependencies: - is-buffer "^1.1.5" - -kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: +kind-of@6.0.3, kind-of@^3.0.2, kind-of@^3.1.0, kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: version "6.0.3" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== @@ -15978,7 +15960,7 @@ msgpackr-extract@^3.0.2: "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2" "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2" -msgpackr@^1.5.2: +msgpackr@1.10.1, msgpackr@^1.5.2: version "1.10.1" resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555" integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ== @@ -16171,27 +16153,13 @@ node-domexception@1.0.0: resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== -node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: +node-fetch@2.6.7, node-fetch@2.6.9, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== dependencies: whatwg-url "^5.0.0" -node-fetch@2.6.9: - version "2.6.9" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.9.tgz#7c7f744b5cc6eb5fd404e0c7a9fec630a55657e6" - integrity sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg== - dependencies: - whatwg-url "^5.0.0" - -node-fetch@^2.6.9: - version "2.7.0" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" - integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== - dependencies: - whatwg-url "^5.0.0" - node-forge@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -17220,15 +17188,7 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0: resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4" integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA== -passport@^0.4.0: - version "0.4.1" - resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270" - integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg== - dependencies: - passport-strategy "1.x.x" - pause "0.0.1" - -passport@^0.6.0: +passport@0.6.0, passport@^0.4.0, passport@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d" integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug== @@ -18213,13 +18173,6 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== -psl@^1.1.28: - version "1.15.0" - resolved "https://registry.yarnpkg.com/psl/-/psl-1.15.0.tgz#bdace31896f1d97cec6a79e8224898ce93d974c6" - integrity sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w== - dependencies: - punycode "^2.3.1" - psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" @@ -18253,11 +18206,6 @@ punycode@^2.1.0, punycode@^2.1.1, punycode@^2.3.0: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.0.tgz#f67fa67c94da8f4d0cfff981aee4118064199b8f" integrity sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA== -punycode@^2.3.1: - version "2.3.1" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" - integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg== - pupa@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/pupa/-/pupa-2.1.1.tgz#f5e8fd4afc2c5d97828faa523549ed8744a20d62" @@ -19089,11 +19037,6 @@ sax@1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== -sax@>=0.1.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" - integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== - sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -19166,28 +19109,13 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@^5.6.0, semver@^5.7.1: - version "5.7.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" - integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== - -semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: +"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.2: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" -semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: - version "6.3.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" - integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== - -semver@^7.5.4, semver@^7.6.0, semver@^7.6.2: - version "7.7.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.1.tgz#abd5098d82b18c6c81f6074ff2647fd3e7220c9f" - integrity sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA== - seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -20669,7 +20597,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: +tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -20679,14 +20607,6 @@ touch@^3.1.0: universalify "^0.2.0" url-parse "^1.5.3" -tough-cookie@~2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" - integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== - dependencies: - psl "^1.1.28" - punycode "^2.1.1" - tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -21154,6 +21074,14 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== +unset-value@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" + integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== + dependencies: + has-value "^2.0.2" + isobject "^4.0.0" + untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -21857,14 +21785,7 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x: - version "0.1.14" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" - integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== - dependencies: - sax ">=0.1.1" - -xml2js@0.6.2: +xml2js@0.1.x, xml2js@0.6.2, xml2js@^0.5.0: version "0.6.2" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== @@ -21872,14 +21793,6 @@ xml2js@0.6.2: sax ">=0.6.0" xmlbuilder "~11.0.0" -xml2js@^0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" - integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== - dependencies: - sax ">=0.6.0" - xmlbuilder "~11.0.0" - xmlbuilder@~11.0.0: version "11.0.1" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" From e6fc3dbff42922285b197f271845528f73684225 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Feb 2025 09:19:25 +0000 Subject: [PATCH 24/51] Fix queryUtils tests. --- .../src/sdk/app/rows/tests/queryUtils.spec.ts | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts index f399801f1e..100581cb54 100644 --- a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts @@ -250,6 +250,8 @@ describe("query utils", () => { expect(result).toEqual([ "_id", "name", + "aux._id", + "auxTable._id", "aux.title", "auxTable.title", "aux.name", @@ -284,7 +286,14 @@ describe("query utils", () => { const result = await config.doInContext(config.appId, () => { return getQueryableFields(table) }) - expect(result).toEqual(["_id", "name", "aux.name", "auxTable.name"]) + expect(result).toEqual([ + "_id", + "name", + "aux._id", + "auxTable._id", + "aux.name", + "auxTable.name", + ]) }) it("excludes all relationship fields if hidden", async () => { @@ -387,10 +396,14 @@ describe("query utils", () => { "_id", "name", // aux1 primitive props + "aux1._id", + "aux1Table._id", "aux1.name", "aux1Table.name", // aux2 primitive props + "aux2._id", + "aux2Table._id", "aux2.title", "aux2Table.title", ]) @@ -405,14 +418,20 @@ describe("query utils", () => { "name", // aux2_1 primitive props + "aux2_1._id", + "aux2Table._id", "aux2_1.title", "aux2Table.title", // aux2_2 primitive props + "aux2_2._id", + "aux2Table._id", "aux2_2.title", "aux2Table.title", // table primitive props + "table._id", + "TestTable._id", "table.name", "TestTable.name", ]) @@ -427,14 +446,20 @@ describe("query utils", () => { "title", // aux1_1 primitive props + "aux1_1._id", + "aux1Table._id", "aux1_1.name", "aux1Table.name", // aux1_2 primitive props + "aux1_2._id", + "aux1Table._id", "aux1_2.name", "aux1Table.name", // table primitive props + "table._id", + "TestTable._id", "table.name", "TestTable.name", ]) @@ -481,6 +506,8 @@ describe("query utils", () => { "name", // deep 1 aux primitive props + "aux._id", + "auxTable._id", "aux.title", "auxTable.title", ]) @@ -495,6 +522,8 @@ describe("query utils", () => { "title", // deep 1 dependency primitive props + "table._id", + "TestTable._id", "table.name", "TestTable.name", ]) From 3aeead4a7bfb0513314566cc5530ed04bb942d46 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 25 Feb 2025 09:27:02 +0000 Subject: [PATCH 25/51] Bump version to 3.4.17 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index bb71d10f41..8ea860e3c4 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.4.16", + "version": "3.4.17", "npmClient": "yarn", "concurrency": 20, "command": { From 3c6e2ff2d784cd79fc42df2c9955ebdba1897e2c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Feb 2025 09:54:10 +0000 Subject: [PATCH 26/51] Fix search tests. --- .../src/api/routes/tests/search.spec.ts | 41 ++++++++++--------- .../server/src/sdk/app/rows/queryUtils.ts | 11 +++-- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index ba2b3f0acf..76ce4a0243 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -3554,27 +3554,30 @@ if (descriptions.length) { }).toContainExactly([row]) }) - it("can filter by the related _id", async () => { - await expectSearch({ - query: { - equal: { "rel._id": row.rel[0]._id }, - }, - }).toContainExactly([row]) + isInternal && + describe("search by _id for relations", () => { + it("can filter by the related _id", async () => { + await expectSearch({ + query: { + equal: { "rel._id": row.rel[0]._id }, + }, + }).toContainExactly([row]) - await expectSearch({ - query: { - equal: { "rel._id": row.rel[1]._id }, - }, - }).toContainExactly([row]) - }) + await expectSearch({ + query: { + equal: { "rel._id": row.rel[1]._id }, + }, + }).toContainExactly([row]) + }) - it("can filter by the related _id and find nothing", async () => { - await expectSearch({ - query: { - equal: { "rel._id": "rel_none" }, - }, - }).toFindNothing() - }) + it("can filter by the related _id and find nothing", async () => { + await expectSearch({ + query: { + equal: { "rel._id": "rel_none" }, + }, + }).toFindNothing() + }) + }) }) !isInternal && diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index 12e724b5d8..629fb50545 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -7,6 +7,7 @@ import { } from "@budibase/types" import { cloneDeep } from "lodash/fp" import sdk from "../../../sdk" +import { isInternal } from "../tables/utils" export const removeInvalidFilters = ( filters: SearchFilters, @@ -69,8 +70,11 @@ export const getQueryableFields = async ( fromTables: string[], opts?: { noRelationships?: boolean } ): Promise => { - // Querying by _id is always allowed, even if it's never part of the schema - const result = ["_id"] + const result = [] + if (isInternal({ table })) { + result.push("_id") + } + for (const field of Object.keys(table.schema).filter( f => allowedFields.includes(f) && table.schema[f].visible !== false )) { @@ -114,7 +118,8 @@ export const getQueryableFields = async ( return result } - const result = [] + // Querying by _id is always allowed, even if it's never part of the schema + const result = ["_id"] if (fields == null) { fields = Object.keys(table.schema) From c93d413d67c04e57ab9936b8aa6bb2d30eeba7cb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Feb 2025 11:03:04 +0000 Subject: [PATCH 27/51] Improve typing around plugins. --- packages/pro | 2 +- .../src/api/controllers/plugin/index.ts | 14 ++++--- packages/server/src/sdk/index.ts | 2 +- packages/server/src/sdk/plugins/index.ts | 42 +++++++++++++++++-- packages/server/src/sdk/plugins/plugins.ts | 41 ------------------ .../server/src/utilities/fileSystem/plugin.ts | 35 ++++++++-------- packages/server/src/watch.ts | 5 ++- packages/types/src/documents/global/plugin.ts | 26 ++++++++++-- 8 files changed, 92 insertions(+), 75 deletions(-) delete mode 100644 packages/server/src/sdk/plugins/plugins.ts diff --git a/packages/pro b/packages/pro index 45f5673d5e..219be3ef38 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 45f5673d5e5ab3c22deb6663cea2e31a628aa133 +Subproject commit 219be3ef380ac8cb16b34892f62117d31b50eec7 diff --git a/packages/server/src/api/controllers/plugin/index.ts b/packages/server/src/api/controllers/plugin/index.ts index 2a4d69b671..6f7cceca10 100644 --- a/packages/server/src/api/controllers/plugin/index.ts +++ b/packages/server/src/api/controllers/plugin/index.ts @@ -11,6 +11,7 @@ import { UploadPluginResponse, FetchPluginResponse, DeletePluginResponse, + PluginMetadata, } from "@budibase/types" import env from "../../../environment" import { clientAppSocket } from "../../../websockets" @@ -53,10 +54,11 @@ export async function create( const { source, url, headers, githubToken } = ctx.request.body try { - let metadata - let directory + let metadata: PluginMetadata + let directory: string + // Generating random name as a backup and needed for url - let name = "PLUGIN_" + Math.floor(100000 + Math.random() * 900000) + const name = "PLUGIN_" + Math.floor(100000 + Math.random() * 900000) switch (source) { case PluginSource.NPM: { @@ -81,12 +83,14 @@ export async function create( directory = directoryUrl break } + default: + ctx.throw(400, "Invalid source") } - pluginCore.validate(metadata?.schema) + pluginCore.validate(metadata.schema) // Only allow components in cloud - if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) { + if (!env.SELF_HOSTED && metadata.schema?.type !== PluginType.COMPONENT) { throw new Error( "Only component plugins are supported outside of self-host" ) diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts index e3e88c25c4..6d6bf00ee0 100644 --- a/packages/server/src/sdk/index.ts +++ b/packages/server/src/sdk/index.ts @@ -7,7 +7,7 @@ import { default as queries } from "./app/queries" import { default as rows } from "./app/rows" import { default as links } from "./app/links" import { default as users } from "./users" -import { default as plugins } from "./plugins" +import * as plugins from "./plugins" import * as views from "./app/views" import * as permissions from "./app/permissions" import * as rowActions from "./app/rowActions" diff --git a/packages/server/src/sdk/plugins/index.ts b/packages/server/src/sdk/plugins/index.ts index 863cfa0db6..03dd2be256 100644 --- a/packages/server/src/sdk/plugins/index.ts +++ b/packages/server/src/sdk/plugins/index.ts @@ -1,5 +1,41 @@ -import * as plugins from "./plugins" +import { KoaFile, Plugin, PluginSource, PluginType } from "@budibase/types" +import { + db as dbCore, + objectStore, + plugins as pluginCore, + tenancy, +} from "@budibase/backend-core" +import { fileUpload } from "../../api/controllers/plugin/file" +import env from "../../environment" +import { clientAppSocket } from "../../websockets" +import { sdk as pro } from "@budibase/pro" -export default { - ...plugins, +export async function fetch(type?: PluginType): Promise { + const db = tenancy.getGlobalDB() + const response = await db.allDocs( + dbCore.getPluginParams(null, { + include_docs: true, + }) + ) + let plugins = response.rows.map((row: any) => row.doc) as Plugin[] + plugins = await objectStore.enrichPluginURLs(plugins) + if (type) { + return plugins.filter((plugin: Plugin) => plugin.schema?.type === type) + } else { + return plugins + } +} + +export async function processUploaded(plugin: KoaFile, source: PluginSource) { + const { metadata, directory } = await fileUpload(plugin) + pluginCore.validate(metadata.schema) + + // Only allow components in cloud + if (!env.SELF_HOSTED && metadata.schema?.type !== PluginType.COMPONENT) { + throw new Error("Only component plugins are supported outside of self-host") + } + + const doc = await pro.plugins.storePlugin(metadata, directory, source) + clientAppSocket?.emit("plugin-update", { name: doc.name, hash: doc.hash }) + return doc } diff --git a/packages/server/src/sdk/plugins/plugins.ts b/packages/server/src/sdk/plugins/plugins.ts deleted file mode 100644 index bff24dcef7..0000000000 --- a/packages/server/src/sdk/plugins/plugins.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { KoaFile, Plugin, PluginSource, PluginType } from "@budibase/types" -import { - db as dbCore, - objectStore, - plugins as pluginCore, - tenancy, -} from "@budibase/backend-core" -import { fileUpload } from "../../api/controllers/plugin/file" -import env from "../../environment" -import { clientAppSocket } from "../../websockets" -import { sdk as pro } from "@budibase/pro" - -export async function fetch(type?: PluginType): Promise { - const db = tenancy.getGlobalDB() - const response = await db.allDocs( - dbCore.getPluginParams(null, { - include_docs: true, - }) - ) - let plugins = response.rows.map((row: any) => row.doc) as Plugin[] - plugins = await objectStore.enrichPluginURLs(plugins) - if (type) { - return plugins.filter((plugin: Plugin) => plugin.schema?.type === type) - } else { - return plugins - } -} - -export async function processUploaded(plugin: KoaFile, source?: PluginSource) { - const { metadata, directory } = await fileUpload(plugin) - pluginCore.validate(metadata?.schema) - - // Only allow components in cloud - if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) { - throw new Error("Only component plugins are supported outside of self-host") - } - - const doc = await pro.plugins.storePlugin(metadata, directory, source) - clientAppSocket?.emit("plugin-update", { name: doc.name, hash: doc.hash }) - return doc -} diff --git a/packages/server/src/utilities/fileSystem/plugin.ts b/packages/server/src/utilities/fileSystem/plugin.ts index 2949daef61..1c62e9c1e9 100644 --- a/packages/server/src/utilities/fileSystem/plugin.ts +++ b/packages/server/src/utilities/fileSystem/plugin.ts @@ -1,4 +1,4 @@ -import { Plugin } from "@budibase/types" +import { Plugin, PluginUpload } from "@budibase/types" import { budibaseTempDir } from "../budibaseDir" import fs from "fs" import { join } from "path" @@ -8,23 +8,22 @@ import stream from "stream" const DATASOURCE_PATH = join(budibaseTempDir(), "datasource") const AUTOMATION_PATH = join(budibaseTempDir(), "automation") -export const getPluginMetadata = async (path: string) => { - let metadata: any = {} +export const getPluginMetadata = async ( + path: string +): Promise => { + let pkg: any + let schema: any try { - const pkg = fs.readFileSync(join(path, "package.json"), "utf8") - const schema = fs.readFileSync(join(path, "schema.json"), "utf8") - - metadata.schema = JSON.parse(schema) - metadata.package = JSON.parse(pkg) - - if ( - !metadata.package.name || - !metadata.package.version || - !metadata.package.description - ) { - throw new Error( - "package.json is missing one of 'name', 'version' or 'description'." - ) + pkg = JSON.parse(fs.readFileSync(join(path, "pkg.json"), "utf8")) + schema = JSON.parse(fs.readFileSync(join(path, "schema.json"), "utf8")) + if (!pkg.name) { + throw new Error("package.json is missing 'name'.") + } + if (!pkg.version) { + throw new Error("package.json is missing 'version'.") + } + if (!pkg.description) { + throw new Error("package.json is missing 'description'.") } } catch (err: any) { throw new Error( @@ -32,7 +31,7 @@ export const getPluginMetadata = async (path: string) => { ) } - return { metadata, directory: path } + return { metadata: { package: pkg, schema }, directory: path } } async function getPluginImpl(path: string, plugin: Plugin) { diff --git a/packages/server/src/watch.ts b/packages/server/src/watch.ts index 8f3fe22023..16b8800887 100644 --- a/packages/server/src/watch.ts +++ b/packages/server/src/watch.ts @@ -3,7 +3,8 @@ import env from "./environment" import chokidar from "chokidar" import fs from "fs" import { constants, tenancy } from "@budibase/backend-core" -import pluginsSdk from "./sdk/plugins" +import { processUploaded } from "./sdk/plugins" +import { PluginSource } from "@budibase/types" export function watch() { const watchPath = path.join(env.PLUGINS_DIR, "./**/*.tar.gz") @@ -27,7 +28,7 @@ export function watch() { const split = path.split("/") const name = split[split.length - 1] console.log("Importing plugin:", path) - await pluginsSdk.processUploaded({ name, path }) + await processUploaded({ name, path }, PluginSource.FILE) } catch (err: any) { const message = err?.message ? err?.message : err console.error("Failed to import plugin:", message) diff --git a/packages/types/src/documents/global/plugin.ts b/packages/types/src/documents/global/plugin.ts index eeddc04e58..56611d2d24 100644 --- a/packages/types/src/documents/global/plugin.ts +++ b/packages/types/src/documents/global/plugin.ts @@ -24,10 +24,7 @@ export interface Plugin extends Document { source: PluginSource package: { [key: string]: any } hash: string - schema: { - type: PluginType - [key: string]: any - } + schema: PluginSchema iconFileName?: string // Populated on read jsUrl?: string @@ -36,3 +33,24 @@ export interface Plugin extends Document { } export const PLUGIN_TYPE_ARR = Object.values(PluginType) + +export interface PluginSchema { + type: PluginType + [key: string]: any +} + +interface Package { + name: string + version: string + description: string +} + +export interface PluginMetadata { + schema: PluginSchema + package: Package +} + +export interface PluginUpload { + metadata: PluginMetadata + directory: string +} From da8c5be6857c0190f3ec3c3cd99c51a994425ef0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Feb 2025 11:08:18 +0000 Subject: [PATCH 28/51] Fix queryUtils tests. --- packages/server/src/sdk/app/rows/queryUtils.ts | 2 +- packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index 629fb50545..8ab50ea1b7 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -126,5 +126,5 @@ export const getQueryableFields = async ( } result.push(...(await extractTableFields(table, fields, [table._id!]))) - return result + return Array.from(new Set(result)) } diff --git a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts index 100581cb54..26a8431446 100644 --- a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts @@ -425,9 +425,7 @@ describe("query utils", () => { // aux2_2 primitive props "aux2_2._id", - "aux2Table._id", "aux2_2.title", - "aux2Table.title", // table primitive props "table._id", @@ -453,9 +451,7 @@ describe("query utils", () => { // aux1_2 primitive props "aux1_2._id", - "aux1Table._id", "aux1_2.name", - "aux1Table.name", // table primitive props "table._id", From f4846736675ac9fe7cb549c75366931d57e62fa1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Feb 2025 12:43:05 +0000 Subject: [PATCH 29/51] Remove elasticsearch global mock, fix tests. --- .../__mocks__/@elastic/elasticsearch.ts | 24 ---- packages/server/datasource-sha.env | 1 + .../server/src/integrations/elasticsearch.ts | 42 ++++--- .../integrations/tests/elasticsearch.spec.ts | 119 +++++++++--------- .../integrations/tests/utils/elasticsearch.ts | 43 +++++++ .../src/integrations/tests/utils/images.ts | 1 + 6 files changed, 126 insertions(+), 104 deletions(-) delete mode 100644 packages/server/__mocks__/@elastic/elasticsearch.ts create mode 100644 packages/server/src/integrations/tests/utils/elasticsearch.ts diff --git a/packages/server/__mocks__/@elastic/elasticsearch.ts b/packages/server/__mocks__/@elastic/elasticsearch.ts deleted file mode 100644 index 5e13437f29..0000000000 --- a/packages/server/__mocks__/@elastic/elasticsearch.ts +++ /dev/null @@ -1,24 +0,0 @@ -const elastic: any = {} - -elastic.Client = function () { - this.index = jest.fn().mockResolvedValue({ body: [] }) - this.search = jest.fn().mockResolvedValue({ - body: { - hits: { - hits: [ - { - _source: { - name: "test", - }, - }, - ], - }, - }, - }) - this.update = jest.fn().mockResolvedValue({ body: [] }) - this.delete = jest.fn().mockResolvedValue({ body: [] }) - - this.close = jest.fn() -} - -module.exports = elastic diff --git a/packages/server/datasource-sha.env b/packages/server/datasource-sha.env index 61249d530c..383fb191f4 100644 --- a/packages/server/datasource-sha.env +++ b/packages/server/datasource-sha.env @@ -3,3 +3,4 @@ MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588eb POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d MARIADB_SHA=sha256:e59ba8783bf7bc02a4779f103bb0d8751ac0e10f9471089709608377eded7aa8 +ELASTICSEARCH_SHA=sha256:9a6443f55243f6acbfeb4a112d15eb3b9aac74bf25e0e39fa19b3ddd3a6879d0 \ No newline at end of file diff --git a/packages/server/src/integrations/elasticsearch.ts b/packages/server/src/integrations/elasticsearch.ts index af03baaef1..10f9d1e697 100644 --- a/packages/server/src/integrations/elasticsearch.ts +++ b/packages/server/src/integrations/elasticsearch.ts @@ -10,7 +10,7 @@ import { import { Client, ClientOptions } from "@elastic/elasticsearch" import { HOST_ADDRESS } from "./utils" -interface ElasticsearchConfig { +export interface ElasticsearchConfig { url: string ssl?: boolean ca?: string @@ -99,9 +99,9 @@ const SCHEMA: Integration = { }, } -class ElasticSearchIntegration implements IntegrationBase { +export class ElasticSearchIntegration implements IntegrationBase { private config: ElasticsearchConfig - private client + private client: Client constructor(config: ElasticsearchConfig) { this.config = config @@ -132,20 +132,23 @@ class ElasticSearchIntegration implements IntegrationBase { } } - async create(query: { index: string; json: object }) { - const { index, json } = query + async create(query: { + index: string + json: object + extra?: Record + }) { + const { index, json, extra } = query try { const result = await this.client.index({ index, body: json, + ...extra, }) return result.body } catch (err) { console.error("Error writing to elasticsearch", err) throw err - } finally { - await this.client.close() } } @@ -160,41 +163,46 @@ class ElasticSearchIntegration implements IntegrationBase { } catch (err) { console.error("Error querying elasticsearch", err) throw err - } finally { - await this.client.close() } } - async update(query: { id: string; index: string; json: object }) { - const { id, index, json } = query + async update(query: { + id: string + index: string + json: object + extra?: Record + }) { + const { id, index, json, extra } = query try { const result = await this.client.update({ id, index, body: json, + ...extra, }) return result.body } catch (err) { console.error("Error querying elasticsearch", err) throw err - } finally { - await this.client.close() } } - async delete(query: { id: string; index: string }) { - const { id, index } = query + async delete(query: { + id: string + index: string + extra?: Record + }) { + const { id, index, extra } = query try { const result = await this.client.delete({ id, index, + ...extra, }) return result.body } catch (err) { console.error("Error deleting from elasticsearch", err) throw err - } finally { - await this.client.close() } } } diff --git a/packages/server/src/integrations/tests/elasticsearch.spec.ts b/packages/server/src/integrations/tests/elasticsearch.spec.ts index f8a1dd8013..da9c5600ad 100644 --- a/packages/server/src/integrations/tests/elasticsearch.spec.ts +++ b/packages/server/src/integrations/tests/elasticsearch.spec.ts @@ -1,83 +1,76 @@ -import { default as ElasticSearchIntegration } from "../elasticsearch" - -jest.mock("@elastic/elasticsearch") - -class TestConfiguration { - integration: any - - constructor(config: any = {}) { - this.integration = new ElasticSearchIntegration.integration(config) - } -} +import { Datasource } from "@budibase/types" +import { ElasticsearchConfig, ElasticSearchIntegration } from "../elasticsearch" +import * as elasticsearch from "../tests/utils/elasticsearch" +import { generator } from "@budibase/backend-core/tests" describe("Elasticsearch Integration", () => { - let config: any - let indexName = "Users" + let datasource: Datasource + let integration: ElasticSearchIntegration + + let index: string + + beforeAll(async () => { + datasource = await elasticsearch.getDatasource() + }) beforeEach(() => { - config = new TestConfiguration() + index = generator.guid() + integration = new ElasticSearchIntegration( + datasource.config! as ElasticsearchConfig + ) }) - it("calls the create method with the correct params", async () => { - const body = { - name: "Hello", - } - await config.integration.create({ - index: indexName, - json: body, + it("can create a record", async () => { + await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, }) - expect(config.integration.client.index).toHaveBeenCalledWith({ - index: indexName, - body, + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, }) + expect(records).toEqual([{ name: "Hello" }]) }) - it("calls the read method with the correct params", async () => { - const body = { - query: { - term: { - name: "kimchy", - }, - }, - } - const response = await config.integration.read({ - index: indexName, - json: body, + it("can update a record", async () => { + const create = await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, }) - expect(config.integration.client.search).toHaveBeenCalledWith({ - index: indexName, - body, + + await integration.update({ + id: create._id, + index, + json: { doc: { name: "World" } }, + extra: { refresh: "true" }, }) - expect(response).toEqual(expect.any(Array)) + + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([{ name: "World" }]) }) - it("calls the update method with the correct params", async () => { - const body = { - name: "updated", - } - - const response = await config.integration.update({ - id: "1234", - index: indexName, - json: body, + it("can delete a record", async () => { + const create = await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, }) - expect(config.integration.client.update).toHaveBeenCalledWith({ - id: "1234", - index: indexName, - body, + await integration.delete({ + id: create._id, + index, + extra: { refresh: "true" }, }) - expect(response).toEqual(expect.any(Array)) - }) - it("calls the delete method with the correct params", async () => { - const body = { - id: "1234", - } - - const response = await config.integration.delete(body) - - expect(config.integration.client.delete).toHaveBeenCalledWith(body) - expect(response).toEqual(expect.any(Array)) + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([]) }) }) diff --git a/packages/server/src/integrations/tests/utils/elasticsearch.ts b/packages/server/src/integrations/tests/utils/elasticsearch.ts new file mode 100644 index 0000000000..a7229c3f60 --- /dev/null +++ b/packages/server/src/integrations/tests/utils/elasticsearch.ts @@ -0,0 +1,43 @@ +import { Datasource, SourceName } from "@budibase/types" +import { GenericContainer, Wait } from "testcontainers" +import { testContainerUtils } from "@budibase/backend-core/tests" +import { startContainer } from "." +import { ELASTICSEARCH_IMAGE } from "./images" +import { ElasticsearchConfig } from "../../elasticsearch" + +let ports: Promise + +export async function getDatasource(): Promise { + if (!ports) { + ports = startContainer( + new GenericContainer(ELASTICSEARCH_IMAGE) + .withExposedPorts(9200) + .withEnvironment({ + "discovery.type": "single-node", + "xpack.security.enabled": "false", + }) + .withWaitStrategy( + Wait.forHttp( + "/_cluster/health?wait_for_status=yellow&timeout=10s", + 9200 + ).withStartupTimeout(60000) + ) + .withTmpFs({ "/usr/share/elasticsearch/data": "rw" }) + ) + } + + const port = (await ports).find(x => x.container === 9200)?.host + if (!port) { + throw new Error("Elasticsearch port not found") + } + + const config: ElasticsearchConfig = { + url: `http://127.0.0.1:${port}`, + } + + return { + type: "datasource", + source: SourceName.ELASTICSEARCH, + config, + } +} diff --git a/packages/server/src/integrations/tests/utils/images.ts b/packages/server/src/integrations/tests/utils/images.ts index 00686412c6..c09b130ea5 100644 --- a/packages/server/src/integrations/tests/utils/images.ts +++ b/packages/server/src/integrations/tests/utils/images.ts @@ -12,3 +12,4 @@ export const POSTGRES_IMAGE = `postgres@${process.env.POSTGRES_SHA}` export const POSTGRES_LEGACY_IMAGE = `postgres:9.5.25` export const MONGODB_IMAGE = `mongo@${process.env.MONGODB_SHA}` export const MARIADB_IMAGE = `mariadb@${process.env.MARIADB_SHA}` +export const ELASTICSEARCH_IMAGE = `elasticsearch@${process.env.ELASTICSEARCH_SHA}` From 1c23509839517910e1b44a66bdb184ffd8f02c27 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Feb 2025 14:58:52 +0000 Subject: [PATCH 30/51] Integrate Elasticsearch as a test datasource. --- .github/workflows/budibase_ci.yml | 15 +- .../src/api/routes/tests/datasource.spec.ts | 6 +- .../routes/tests/queries/generic-sql.spec.ts | 3 +- .../server/src/api/routes/tests/row.spec.ts | 7 +- .../src/api/routes/tests/search.spec.ts | 7 +- .../server/src/api/routes/tests/table.spec.ts | 7 +- .../src/api/routes/tests/viewV2.spec.ts | 7 +- .../tests/steps/executeQuery.spec.ts | 3 +- .../integrations/tests/elasticsearch.spec.ts | 141 +++++++++--------- .../src/integrations/tests/utils/index.ts | 56 +++++-- .../sdk/app/rows/search/tests/search.spec.ts | 7 +- 11 files changed, 145 insertions(+), 114 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index e294e50e8d..6a9aa3e8d0 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -155,7 +155,18 @@ jobs: strategy: matrix: datasource: - [mssql, mysql, postgres, postgres_legacy, mongodb, mariadb, oracle, sqs, none] + [ + mssql, + mysql, + postgres, + postgres_legacy, + mongodb, + mariadb, + oracle, + sqs, + elasticsearch, + none, + ] steps: - name: Checkout repo uses: actions/checkout@v4 @@ -192,6 +203,8 @@ jobs: docker pull budibase/oracle-database:23.2-slim-faststart elif [ "${{ matrix.datasource }}" == "postgres_legacy" ]; then docker pull postgres:9.5.25 + elif [ "${{ matrix.datasource }}" == "elasticsearch" ]; then + docker pull elasticsearch@${{ steps.dotenv.outputs.ELASTICSEARCH_SHA }} fi docker pull minio/minio & docker pull redis & diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts index 21e9effa77..12029c39c4 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.ts +++ b/packages/server/src/api/routes/tests/datasource.spec.ts @@ -165,7 +165,8 @@ describe("/datasources", () => { }) const descriptions = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS], + plus: true, + exclude: [DatabaseName.SQS], }) if (descriptions.length) { @@ -590,7 +591,8 @@ if (descriptions.length) { } const datasources = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS, DatabaseName.ORACLE], + plus: true, + exclude: [DatabaseName.SQS, DatabaseName.ORACLE], }) if (datasources.length) { diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 863f5b65e0..4a545b253e 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -9,7 +9,8 @@ import { Knex } from "knex" import { generator } from "@budibase/backend-core/tests" const descriptions = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS], + plus: true, + exclude: [DatabaseName.SQS], }) if (descriptions.length) { diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 87002670b7..b349a1df8a 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1,9 +1,6 @@ import * as setup from "./utilities" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import tk from "timekeeper" import emitter from "../../../../src/events" @@ -80,7 +77,7 @@ function encodeJS(binding: string) { return `{{ js "${Buffer.from(binding).toString("base64")}"}}` } -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index ee372914d7..a904c0c469 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1,8 +1,5 @@ import { tableForDatasource } from "../../../tests/utilities/structures" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import { context, db as dbCore, @@ -60,7 +57,7 @@ jest.mock("@budibase/pro", () => ({ }, })) -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 2a7f039ff5..29b576d16a 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -28,17 +28,14 @@ import * as setup from "./utilities" import * as uuid from "uuid" import { generator } from "@budibase/backend-core/tests" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import { tableForDatasource } from "../../../tests/utilities/structures" import timekeeper from "timekeeper" const { basicTable } = setup.structures const ISO_REGEX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/ -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 9531737d30..7eed1811d9 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -37,17 +37,14 @@ import { ViewV2Type, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" -import { - DatabaseName, - datasourceDescribe, -} from "../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../integrations/tests/utils" import merge from "lodash/merge" import { quotas } from "@budibase/pro" import { context, db, events, roles, setEnv } from "@budibase/backend-core" import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai" import nock from "nock" -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( diff --git a/packages/server/src/automations/tests/steps/executeQuery.spec.ts b/packages/server/src/automations/tests/steps/executeQuery.spec.ts index dff3580b7e..a51d335902 100644 --- a/packages/server/src/automations/tests/steps/executeQuery.spec.ts +++ b/packages/server/src/automations/tests/steps/executeQuery.spec.ts @@ -9,7 +9,8 @@ import { generator } from "@budibase/backend-core/tests" import { createAutomationBuilder } from "../utilities/AutomationTestBuilder" const descriptions = datasourceDescribe({ - exclude: [DatabaseName.MONGODB, DatabaseName.SQS], + plus: true, + exclude: [DatabaseName.SQS], }) if (descriptions.length) { diff --git a/packages/server/src/integrations/tests/elasticsearch.spec.ts b/packages/server/src/integrations/tests/elasticsearch.spec.ts index da9c5600ad..14a272d610 100644 --- a/packages/server/src/integrations/tests/elasticsearch.spec.ts +++ b/packages/server/src/integrations/tests/elasticsearch.spec.ts @@ -2,75 +2,80 @@ import { Datasource } from "@budibase/types" import { ElasticsearchConfig, ElasticSearchIntegration } from "../elasticsearch" import * as elasticsearch from "../tests/utils/elasticsearch" import { generator } from "@budibase/backend-core/tests" +import { DatabaseName, datasourceDescribe } from "./utils" -describe("Elasticsearch Integration", () => { - let datasource: Datasource - let integration: ElasticSearchIntegration +const describes = datasourceDescribe({ only: [DatabaseName.ELASTICSEARCH] }) - let index: string +if (describes.length) { + describe.each(describes)("Elasticsearch Integration", () => { + let datasource: Datasource + let integration: ElasticSearchIntegration - beforeAll(async () => { - datasource = await elasticsearch.getDatasource() + let index: string + + beforeAll(async () => { + datasource = await elasticsearch.getDatasource() + }) + + beforeEach(() => { + index = generator.guid() + integration = new ElasticSearchIntegration( + datasource.config! as ElasticsearchConfig + ) + }) + + it("can create a record", async () => { + await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, + }) + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([{ name: "Hello" }]) + }) + + it("can update a record", async () => { + const create = await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, + }) + + await integration.update({ + id: create._id, + index, + json: { doc: { name: "World" } }, + extra: { refresh: "true" }, + }) + + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([{ name: "World" }]) + }) + + it("can delete a record", async () => { + const create = await integration.create({ + index, + json: { name: "Hello" }, + extra: { refresh: "true" }, + }) + + await integration.delete({ + id: create._id, + index, + extra: { refresh: "true" }, + }) + + const records = await integration.read({ + index, + json: { query: { match_all: {} } }, + }) + expect(records).toEqual([]) + }) }) - - beforeEach(() => { - index = generator.guid() - integration = new ElasticSearchIntegration( - datasource.config! as ElasticsearchConfig - ) - }) - - it("can create a record", async () => { - await integration.create({ - index, - json: { name: "Hello" }, - extra: { refresh: "true" }, - }) - const records = await integration.read({ - index, - json: { query: { match_all: {} } }, - }) - expect(records).toEqual([{ name: "Hello" }]) - }) - - it("can update a record", async () => { - const create = await integration.create({ - index, - json: { name: "Hello" }, - extra: { refresh: "true" }, - }) - - await integration.update({ - id: create._id, - index, - json: { doc: { name: "World" } }, - extra: { refresh: "true" }, - }) - - const records = await integration.read({ - index, - json: { query: { match_all: {} } }, - }) - expect(records).toEqual([{ name: "World" }]) - }) - - it("can delete a record", async () => { - const create = await integration.create({ - index, - json: { name: "Hello" }, - extra: { refresh: "true" }, - }) - - await integration.delete({ - id: create._id, - index, - extra: { refresh: "true" }, - }) - - const records = await integration.read({ - index, - json: { query: { match_all: {} } }, - }) - expect(records).toEqual([]) - }) -}) +} diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index 9e2c4f7e70..08777cab89 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -6,6 +6,7 @@ import * as mysql from "./mysql" import * as mssql from "./mssql" import * as mariadb from "./mariadb" import * as oracle from "./oracle" +import * as elasticsearch from "./elasticsearch" import { testContainerUtils } from "@budibase/backend-core/tests" import { Knex } from "knex" import TestConfiguration from "../../../tests/utilities/TestConfiguration" @@ -23,22 +24,32 @@ export enum DatabaseName { MARIADB = "mariadb", ORACLE = "oracle", SQS = "sqs", + ELASTICSEARCH = "elasticsearch", } +const DATASOURCE_PLUS = [ + DatabaseName.POSTGRES, + DatabaseName.POSTGRES_LEGACY, + DatabaseName.MYSQL, + DatabaseName.SQL_SERVER, + DatabaseName.MARIADB, + DatabaseName.ORACLE, + DatabaseName.SQS, +] + const providers: Record = { + // datasource_plus entries [DatabaseName.POSTGRES]: postgres.getDatasource, [DatabaseName.POSTGRES_LEGACY]: postgres.getLegacyDatasource, - [DatabaseName.MONGODB]: mongodb.getDatasource, [DatabaseName.MYSQL]: mysql.getDatasource, [DatabaseName.SQL_SERVER]: mssql.getDatasource, [DatabaseName.MARIADB]: mariadb.getDatasource, [DatabaseName.ORACLE]: oracle.getDatasource, [DatabaseName.SQS]: async () => undefined, -} -export interface DatasourceDescribeOpts { - only?: DatabaseName[] - exclude?: DatabaseName[] + // rest + [DatabaseName.ELASTICSEARCH]: elasticsearch.getDatasource, + [DatabaseName.MONGODB]: mongodb.getDatasource, } export interface DatasourceDescribeReturnPromise { @@ -103,6 +114,20 @@ function createDummyTest() { }) } +interface OnlyOpts { + only: DatabaseName[] +} + +interface PlusOpts { + plus: true + exclude?: DatabaseName[] +} + +export type DatasourceDescribeOpts = OnlyOpts | PlusOpts + +// If you ever want to rename this function, be mindful that you will also need +// to modify src/tests/filters/index.js to make sure that we're correctly +// filtering datasource/non-datasource tests in CI. export function datasourceDescribe(opts: DatasourceDescribeOpts) { // tests that call this need a lot longer timeouts jest.setTimeout(120000) @@ -111,17 +136,15 @@ export function datasourceDescribe(opts: DatasourceDescribeOpts) { createDummyTest() } - const { only, exclude } = opts - - if (only && exclude) { - throw new Error("you can only supply one of 'only' or 'exclude'") - } - - let databases = Object.values(DatabaseName) - if (only) { - databases = only - } else if (exclude) { - databases = databases.filter(db => !exclude.includes(db)) + let databases: DatabaseName[] = [] + if ("only" in opts) { + databases = opts.only + } else if ("plus" in opts) { + databases = Object.values(DatabaseName) + .filter(db => DATASOURCE_PLUS.includes(db)) + .filter(db => !opts.exclude?.includes(db)) + } else { + throw new Error("invalid options") } if (process.env.DATASOURCE) { @@ -156,6 +179,7 @@ export function datasourceDescribe(opts: DatasourceDescribeOpts) { isMSSQL: dbName === DatabaseName.SQL_SERVER, isOracle: dbName === DatabaseName.ORACLE, isMariaDB: dbName === DatabaseName.MARIADB, + isElasticsearch: dbName === DatabaseName.ELASTICSEARCH, })) } diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index b424c3707d..eaf495e25f 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -10,16 +10,13 @@ import { import { search } from "../../../../../sdk/app/rows/search" import { generator } from "@budibase/backend-core/tests" -import { - DatabaseName, - datasourceDescribe, -} from "../../../../../integrations/tests/utils" +import { datasourceDescribe } from "../../../../../integrations/tests/utils" import { tableForDatasource } from "../../../../../tests/utilities/structures" // These test cases are only for things that cannot be tested through the API // (e.g. limiting searches to returning specific fields). If it's possible to // test through the API, it should be done there instead. -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ plus: true }) if (descriptions.length) { describe.each(descriptions)( From 48674833fcca04598f3549b580e1c511a3670541 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Feb 2025 15:09:39 +0000 Subject: [PATCH 31/51] Correct use of dsProvider in elasticsearch.spec.ts. --- .../server/src/integrations/tests/elasticsearch.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/integrations/tests/elasticsearch.spec.ts b/packages/server/src/integrations/tests/elasticsearch.spec.ts index 14a272d610..bcf8def1e9 100644 --- a/packages/server/src/integrations/tests/elasticsearch.spec.ts +++ b/packages/server/src/integrations/tests/elasticsearch.spec.ts @@ -1,20 +1,20 @@ import { Datasource } from "@budibase/types" import { ElasticsearchConfig, ElasticSearchIntegration } from "../elasticsearch" -import * as elasticsearch from "../tests/utils/elasticsearch" import { generator } from "@budibase/backend-core/tests" import { DatabaseName, datasourceDescribe } from "./utils" const describes = datasourceDescribe({ only: [DatabaseName.ELASTICSEARCH] }) if (describes.length) { - describe.each(describes)("Elasticsearch Integration", () => { + describe.each(describes)("Elasticsearch Integration", ({ dsProvider }) => { let datasource: Datasource let integration: ElasticSearchIntegration let index: string beforeAll(async () => { - datasource = await elasticsearch.getDatasource() + const ds = await dsProvider() + datasource = ds.datasource! }) beforeEach(() => { From 9f0da2a65a1252a9754afa2806d07fac30cd5e60 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Feb 2025 15:12:35 +0000 Subject: [PATCH 32/51] Document elasticsearch container params. --- .../src/integrations/tests/utils/elasticsearch.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/server/src/integrations/tests/utils/elasticsearch.ts b/packages/server/src/integrations/tests/utils/elasticsearch.ts index a7229c3f60..a2ea22f73b 100644 --- a/packages/server/src/integrations/tests/utils/elasticsearch.ts +++ b/packages/server/src/integrations/tests/utils/elasticsearch.ts @@ -13,15 +13,26 @@ export async function getDatasource(): Promise { new GenericContainer(ELASTICSEARCH_IMAGE) .withExposedPorts(9200) .withEnvironment({ + // We need to set the discovery type to single-node to avoid the + // cluster waiting for other nodes to join before starting up. "discovery.type": "single-node", + // We disable security to avoid having to do any auth against the + // container, and to disable SSL. With SSL enabled it uses a self + // signed certificate that we'd have to ignore anyway. "xpack.security.enabled": "false", }) .withWaitStrategy( Wait.forHttp( + // Single node clusters never reach status green, so we wait for + // yellow instead. "/_cluster/health?wait_for_status=yellow&timeout=10s", 9200 ).withStartupTimeout(60000) ) + // We gave the container a tmpfs data directory. Without this, I found + // that the default data directory was very small and the container + // easily filled it up. This caused the cluster to go into a red status + // and stop responding to requests. .withTmpFs({ "/usr/share/elasticsearch/data": "rw" }) ) } From b23aee478da9ecdfbbd8d6d4fa0d359f242d651c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Feb 2025 15:18:28 +0000 Subject: [PATCH 33/51] Fix plugin tests. --- packages/server/src/utilities/fileSystem/plugin.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/server/src/utilities/fileSystem/plugin.ts b/packages/server/src/utilities/fileSystem/plugin.ts index 1c62e9c1e9..15238637c1 100644 --- a/packages/server/src/utilities/fileSystem/plugin.ts +++ b/packages/server/src/utilities/fileSystem/plugin.ts @@ -14,7 +14,7 @@ export const getPluginMetadata = async ( let pkg: any let schema: any try { - pkg = JSON.parse(fs.readFileSync(join(path, "pkg.json"), "utf8")) + pkg = JSON.parse(fs.readFileSync(join(path, "package.json"), "utf8")) schema = JSON.parse(fs.readFileSync(join(path, "schema.json"), "utf8")) if (!pkg.name) { throw new Error("package.json is missing 'name'.") @@ -27,7 +27,8 @@ export const getPluginMetadata = async ( } } catch (err: any) { throw new Error( - `Unable to process schema.json/package.json in plugin. ${err.message}` + `Unable to process schema.json/package.json in plugin. ${err.message}`, + { cause: err } ) } From 3be31652524b9849e62b650cf1bd26ccf6769fa1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Feb 2025 15:22:32 +0000 Subject: [PATCH 34/51] Remove mocks from static.spec.ts --- .../src/api/routes/tests/static.spec.ts | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/routes/tests/static.spec.ts b/packages/server/src/api/routes/tests/static.spec.ts index 872085c382..6ec7b54a7f 100644 --- a/packages/server/src/api/routes/tests/static.spec.ts +++ b/packages/server/src/api/routes/tests/static.spec.ts @@ -1,11 +1,3 @@ -// Directly mock the AWS SDK -jest.mock("@aws-sdk/s3-request-presigner", () => ({ - getSignedUrl: jest.fn(() => { - return `http://example.com` - }), -})) -jest.mock("@aws-sdk/client-s3") - import { Datasource, SourceName } from "@budibase/types" import { setEnv } from "../../../environment" import { getRequest, getConfig, afterAll as _afterAll } from "./utilities" @@ -92,7 +84,17 @@ describe("/static", () => { .set(config.defaultHeaders()) .expect("Content-Type", /json/) .expect(200) - expect(res.body.signedUrl).toEqual("http://example.com") + + expect(res.body.signedUrl).toStartWith( + "https://foo.s3.eu-west-1.amazonaws.com/bar?" + ) + expect(res.body.signedUrl).toContain("X-Amz-Algorithm=AWS4-HMAC-SHA256") + expect(res.body.signedUrl).toContain("X-Amz-Credential=bb") + expect(res.body.signedUrl).toContain("X-Amz-Date=") + expect(res.body.signedUrl).toContain("X-Amz-Signature=") + expect(res.body.signedUrl).toContain("X-Amz-Expires=900") + expect(res.body.signedUrl).toContain("X-Amz-SignedHeaders=host") + expect(res.body.publicUrl).toEqual( `https://${bucket}.s3.eu-west-1.amazonaws.com/${key}` ) From 7a63890c1f94a13839a76370066c765b651afadd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 25 Feb 2025 15:32:29 +0000 Subject: [PATCH 35/51] Adding zoom capabilities to QR-code reader to help with smaller qr codes. --- packages/client/manifest.json | 6 ++++++ packages/client/package.json | 2 +- .../client/src/components/app/forms/CodeScanner.svelte | 8 ++++++++ .../src/components/app/forms/CodeScannerField.svelte | 2 ++ yarn.lock | 8 ++++---- 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/client/manifest.json b/packages/client/manifest.json index a2d29d3020..886ad49650 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -4492,6 +4492,12 @@ } ] }, + { + "type": "text", + "label": "Zoom level", + "key": "defaultZoom", + "defaultValue": "1" + }, { "type": "event", "label": "On change", diff --git a/packages/client/package.json b/packages/client/package.json index 72be403698..a7ea04fd0b 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -28,7 +28,7 @@ "apexcharts": "^3.48.0", "dayjs": "^1.10.8", "downloadjs": "1.4.7", - "html5-qrcode": "^2.2.1", + "html5-qrcode": "^2.3.8", "leaflet": "^1.7.1", "sanitize-html": "^2.13.0", "screenfull": "^6.0.1", diff --git a/packages/client/src/components/app/forms/CodeScanner.svelte b/packages/client/src/components/app/forms/CodeScanner.svelte index 008c0f5727..5c5d6ff73b 100644 --- a/packages/client/src/components/app/forms/CodeScanner.svelte +++ b/packages/client/src/components/app/forms/CodeScanner.svelte @@ -20,6 +20,7 @@ export let beepFrequency = 2637 export let customFrequency = 1046 export let preferredCamera = "environment" + export let defaultZoom = 1 export let validator const dispatch = createEventDispatcher() @@ -58,6 +59,13 @@ html5QrCode .start(cameraSetting, cameraConfig, onScanSuccess) .then(() => { + if (defaultZoom > 1) { + const cameraOptions = html5QrCode.getRunningTrackCameraCapabilities() + const zoom = cameraOptions.zoomFeature() + if (zoom.isSupported()) { + zoom.apply(defaultZoom) + } + } resolve({ initialised: true }) }) .catch(err => { diff --git a/packages/client/src/components/app/forms/CodeScannerField.svelte b/packages/client/src/components/app/forms/CodeScannerField.svelte index 7c9948554a..dd9e986804 100644 --- a/packages/client/src/components/app/forms/CodeScannerField.svelte +++ b/packages/client/src/components/app/forms/CodeScannerField.svelte @@ -17,6 +17,7 @@ export let beepFrequency export let customFrequency export let preferredCamera + export let defaultZoom export let helpText = null let fieldState @@ -56,6 +57,7 @@ {beepFrequency} {customFrequency} {preferredCamera} + {defaultZoom} validator={fieldState.validator} /> {/if} diff --git a/yarn.lock b/yarn.lock index ceae41458c..64810291c4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12768,10 +12768,10 @@ html-tag@^2.0.0: is-self-closing "^1.0.1" kind-of "^6.0.0" -html5-qrcode@^2.2.1: - version "2.3.7" - resolved "https://registry.yarnpkg.com/html5-qrcode/-/html5-qrcode-2.3.7.tgz#09ed2ca7473a47bd551088c15fcfcb7cb409a5be" - integrity sha512-Jmlok9Ynm49hgVXkdupWryf8o430proIFoQsRl1LmTg4Rq461W72omylR9yw9tsEMtswMEw3wacUM5y0agOBQA== +html5-qrcode@^2.3.8: + version "2.3.8" + resolved "https://registry.yarnpkg.com/html5-qrcode/-/html5-qrcode-2.3.8.tgz#0b0cdf7a9926cfd4be530e13a51db47592adfa0d" + integrity sha512-jsr4vafJhwoLVEDW3n1KvPnCCXWaQfRng0/EEYk1vNcQGcG/htAdhJX0be8YyqMoSz7+hZvOZSTAepsabiuhiQ== htmlparser2@^8.0.0: version "8.0.1" From a1670c309f53a95f3e890cdfd6298c3db08ab778 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 25 Feb 2025 15:42:50 +0000 Subject: [PATCH 36/51] Linting. --- packages/client/src/components/app/forms/CodeScanner.svelte | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/client/src/components/app/forms/CodeScanner.svelte b/packages/client/src/components/app/forms/CodeScanner.svelte index 5c5d6ff73b..1a046d7168 100644 --- a/packages/client/src/components/app/forms/CodeScanner.svelte +++ b/packages/client/src/components/app/forms/CodeScanner.svelte @@ -60,7 +60,8 @@ .start(cameraSetting, cameraConfig, onScanSuccess) .then(() => { if (defaultZoom > 1) { - const cameraOptions = html5QrCode.getRunningTrackCameraCapabilities() + const cameraOptions = + html5QrCode.getRunningTrackCameraCapabilities() const zoom = cameraOptions.zoomFeature() if (zoom.isSupported()) { zoom.apply(defaultZoom) From eeb6f42681622cde92f294e81a21f9c69d3753b5 Mon Sep 17 00:00:00 2001 From: Dean Date: Tue, 25 Feb 2025 15:53:55 +0000 Subject: [PATCH 37/51] Fix for autocompletion evaluation in the builder --- .../builder/src/components/common/bindings/BindingPanel.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/common/bindings/BindingPanel.svelte b/packages/builder/src/components/common/bindings/BindingPanel.svelte index 9ebbc91309..3715719565 100644 --- a/packages/builder/src/components/common/bindings/BindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/BindingPanel.svelte @@ -90,7 +90,7 @@ $: requestEval(runtimeExpression, context, snippets) $: bindingHelpers = new BindingHelpers(getCaretPosition, insertAtPos) - $: bindingOptions = bindingsToCompletions(bindings, editorMode) + $: bindingOptions = bindingsToCompletions(enrichedBindings, editorMode) $: helperOptions = allowHelpers ? getHelperCompletions(editorMode) : [] $: snippetsOptions = usingJS && useSnippets && snippets?.length ? snippets : [] From daf317b0a72c96eb815e473ea8d283c63586def9 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 26 Feb 2025 13:19:59 +0000 Subject: [PATCH 38/51] Bump version to 3.4.18 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 8ea860e3c4..055b8b5ca8 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.4.17", + "version": "3.4.18", "npmClient": "yarn", "concurrency": 20, "command": { From eefcfb3c0f22e7d20d58a02ee782262472b13e96 Mon Sep 17 00:00:00 2001 From: Christos Alexiou Date: Wed, 26 Feb 2025 15:43:26 +0200 Subject: [PATCH 39/51] Update before installing --- hosting/single/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 1f449e7376..75b1a12d12 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -92,7 +92,7 @@ COPY hosting/single/ssh/sshd_config /etc/ COPY hosting/single/ssh/ssh_setup.sh /tmp # setup letsencrypt certificate -RUN apt-get install -y certbot python3-certbot-nginx +RUN apt-get update && apt-get install -y certbot python3-certbot-nginx COPY hosting/letsencrypt /app/letsencrypt RUN chmod +x /app/letsencrypt/certificate-request.sh /app/letsencrypt/certificate-renew.sh From 7b9cc6eea4263a2f4c716121230ae59864461d95 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 26 Feb 2025 14:11:40 +0000 Subject: [PATCH 40/51] Bump version to 3.4.19 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 055b8b5ca8..03be6e46bc 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.4.18", + "version": "3.4.19", "npmClient": "yarn", "concurrency": 20, "command": { From 606c4475ad8f6aed56b64b14c7fa017cc0e7db39 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 26 Feb 2025 14:29:39 +0000 Subject: [PATCH 41/51] Revert "Improve types around `nodemailer`" --- .../tests/steps/sendSmtpEmail.spec.ts | 20 +++++----- .../server/src/utilities/workerRequests.ts | 40 ++++++++----------- packages/types/package.json | 1 - packages/types/src/api/web/global/email.ts | 9 ++--- .../documents/app/automation/automation.ts | 8 ++-- packages/worker/package.json | 1 - .../src/api/controllers/global/email.ts | 7 +--- packages/worker/src/utilities/email.ts | 7 ++-- yarn.lock | 13 ++---- 9 files changed, 42 insertions(+), 64 deletions(-) diff --git a/packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts b/packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts index 7aff612a97..7452239dfa 100644 --- a/packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts +++ b/packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts @@ -1,4 +1,3 @@ -import { SendEmailResponse } from "@budibase/types" import TestConfiguration from "../../../tests/utilities/TestConfiguration" import * as workerRequests from "../../../utilities/workerRequests" @@ -6,18 +5,17 @@ jest.mock("../../../utilities/workerRequests", () => ({ sendSmtpEmail: jest.fn(), })) -function generateResponse(to: string, from: string): SendEmailResponse { +function generateResponse(to: string, from: string) { return { - message: `Email sent to ${to}.`, - accepted: [to], - envelope: { - from: from, - to: [to], + success: true, + response: { + accepted: [to], + envelope: { + from: from, + to: [to], + }, + message: `Email sent to ${to}.`, }, - messageId: "messageId", - pending: [], - rejected: [], - response: "response", } } diff --git a/packages/server/src/utilities/workerRequests.ts b/packages/server/src/utilities/workerRequests.ts index dd1493b82f..0f487d9f31 100644 --- a/packages/server/src/utilities/workerRequests.ts +++ b/packages/server/src/utilities/workerRequests.ts @@ -8,15 +8,7 @@ import { logging, env as coreEnv, } from "@budibase/backend-core" -import { - Ctx, - User, - EmailInvite, - EmailAttachment, - SendEmailResponse, - SendEmailRequest, - EmailTemplatePurpose, -} from "@budibase/types" +import { Ctx, User, EmailInvite, EmailAttachment } from "@budibase/types" interface Request { ctx?: Ctx @@ -118,23 +110,25 @@ export async function sendSmtpEmail({ invite?: EmailInvite }) { // tenant ID will be set in header - const request: SendEmailRequest = { - email: to, - from, - contents, - subject, - cc, - bcc, - purpose: EmailTemplatePurpose.CUSTOM, - automation, - invite, - attachments, - } const response = await fetch( checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`), - createRequest({ method: "POST", body: request }) + createRequest({ + method: "POST", + body: { + email: to, + from, + contents, + subject, + cc, + bcc, + purpose: "custom", + automation, + invite, + attachments, + }, + }) ) - return (await checkResponse(response, "send email")) as SendEmailResponse + return checkResponse(response, "send email") } export async function removeAppFromUserRoles(ctx: Ctx, appId: string) { diff --git a/packages/types/package.json b/packages/types/package.json index a6e08ab84c..ee3c059bc9 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -17,7 +17,6 @@ "@budibase/nano": "10.1.5", "@types/json-schema": "^7.0.15", "@types/koa": "2.13.4", - "@types/nodemailer": "^6.4.17", "@types/redlock": "4.0.7", "koa-useragent": "^4.1.0", "rimraf": "3.0.2", diff --git a/packages/types/src/api/web/global/email.ts b/packages/types/src/api/web/global/email.ts index 3d2e007231..a0ca0e8485 100644 --- a/packages/types/src/api/web/global/email.ts +++ b/packages/types/src/api/web/global/email.ts @@ -1,5 +1,4 @@ import { EmailAttachment, EmailInvite } from "../../../documents" -import SMTPTransport from "nodemailer/lib/smtp-transport" export enum EmailTemplatePurpose { CORE = "core", @@ -13,17 +12,17 @@ export enum EmailTemplatePurpose { export interface SendEmailRequest { workspaceId?: string email: string - userId?: string + userId: string purpose: EmailTemplatePurpose contents?: string from?: string subject: string - cc?: string - bcc?: string + cc?: boolean + bcc?: boolean automation?: boolean invite?: EmailInvite attachments?: EmailAttachment[] } -export interface SendEmailResponse extends SMTPTransport.SentMessageInfo { +export interface SendEmailResponse extends Record { message: string } diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts index cfe2ba5147..d5ef35d059 100644 --- a/packages/types/src/documents/app/automation/automation.ts +++ b/packages/types/src/documents/app/automation/automation.ts @@ -1,10 +1,10 @@ import { Document } from "../../document" import { User } from "../../global" +import { ReadStream } from "fs" import { Row } from "../row" import { Table } from "../table" import { AutomationStep, AutomationTrigger } from "./schema" import { ContextEmitter } from "../../../sdk" -import { Readable } from "stream" export enum AutomationIOType { OBJECT = "object", @@ -108,8 +108,8 @@ export interface SendEmailOpts { subject: string // info Pass in a structure of information to be stored alongside the invitation. info?: any - cc?: string - bcc?: string + cc?: boolean + bcc?: boolean automation?: boolean invite?: EmailInvite attachments?: EmailAttachment[] @@ -269,7 +269,7 @@ export type AutomationAttachment = { export type AutomationAttachmentContent = { filename: string - content: Readable + content: ReadStream | NodeJS.ReadableStream } export type BucketedContent = AutomationAttachmentContent & { diff --git a/packages/worker/package.json b/packages/worker/package.json index edaab50d78..28728272ca 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -86,7 +86,6 @@ "@types/koa__router": "12.0.4", "@types/lodash": "4.14.200", "@types/node-fetch": "2.6.4", - "@types/nodemailer": "^6.4.17", "@types/server-destroy": "1.0.1", "@types/supertest": "2.0.14", "@types/uuid": "8.3.4", diff --git a/packages/worker/src/api/controllers/global/email.ts b/packages/worker/src/api/controllers/global/email.ts index ed2d9b5125..ad0fc3fa32 100644 --- a/packages/worker/src/api/controllers/global/email.ts +++ b/packages/worker/src/api/controllers/global/email.ts @@ -24,13 +24,10 @@ export async function sendEmail( invite, attachments, } = ctx.request.body - let user: User | undefined = undefined + let user: any if (userId) { const db = tenancy.getGlobalDB() - user = await db.tryGet(userId) - } - if (!user) { - ctx.throw(404, "User not found.") + user = await db.get(userId) } const response = await sendEmailFn(email, purpose, { workspaceId, diff --git a/packages/worker/src/utilities/email.ts b/packages/worker/src/utilities/email.ts index a2b9c3bfc2..fa9dd7a6fa 100644 --- a/packages/worker/src/utilities/email.ts +++ b/packages/worker/src/utilities/email.ts @@ -13,8 +13,7 @@ import { configs, cache, objectStore } from "@budibase/backend-core" import ical from "ical-generator" import _ from "lodash" -import nodemailer from "nodemailer" -import SMTPTransport from "nodemailer/lib/smtp-transport" +const nodemailer = require("nodemailer") const TEST_MODE = env.ENABLE_EMAIL_TEST_MODE && env.isDev() const TYPE = TemplateType.EMAIL @@ -27,7 +26,7 @@ const FULL_EMAIL_PURPOSES = [ ] function createSMTPTransport(config?: SMTPInnerConfig) { - let options: SMTPTransport.Options + let options: any let secure = config?.secure // default it if not specified if (secure == null) { @@ -162,7 +161,7 @@ export async function sendEmail( const code = await getLinkCode(purpose, email, opts.user, opts?.info) let context = await getSettingsTemplateContext(purpose, code) - let message: Parameters[0] = { + let message: any = { from: opts?.from || config?.from, html: await buildEmail(purpose, email, context, { user: opts?.user, diff --git a/yarn.lock b/yarn.lock index 445e0abc18..48eaaf4ecd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2785,9 +2785,9 @@ through2 "^2.0.0" "@budibase/pro@npm:@budibase/pro@latest": - version "3.4.16" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.16.tgz#c482a400e27b7e89ca73092c4c81bdeac1d24581" - integrity sha512-8ECnqOh9jQ10KlQEwmKPFcoVGE+2gGgSybj+vbshwDp1zAW76doyMR2DMNjEatNpWVnpoMnTkDWtE9aqQ5v0vQ== + version "3.4.12" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.12.tgz#60e630944de4e2de970a04179d8f0f57d48ce75e" + integrity sha512-msUBmcWxRDg+ugjZvd27XudERQqtQRdiARsO8MaDVTcp5ejIXgshEIVVshHOCj3hcbRblw9pXvBIMI53iTMUsA== dependencies: "@anthropic-ai/sdk" "^0.27.3" "@budibase/backend-core" "*" @@ -6775,13 +6775,6 @@ dependencies: undici-types "~6.19.2" -"@types/nodemailer@^6.4.17": - version "6.4.17" - resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.17.tgz#5c82a42aee16a3dd6ea31446a1bd6a447f1ac1a4" - integrity sha512-I9CCaIp6DTldEg7vyUTZi8+9Vo0hi1/T8gv3C89yk1rSAAzoKQ8H8ki/jBYJSFoH/BisgLP8tkZMlQ91CIquww== - dependencies: - "@types/node" "*" - "@types/normalize-package-data@^2.4.0": version "2.4.1" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" From d6d32ceaadde4597554f786d6e98317df2674ddf Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 26 Feb 2025 14:49:09 +0000 Subject: [PATCH 42/51] Bump version to 3.4.20 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 03be6e46bc..b391c8e05c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.4.19", + "version": "3.4.20", "npmClient": "yarn", "concurrency": 20, "command": { From 06d00d69b2660ed98648161785453ef93cbea83c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 26 Feb 2025 15:23:49 +0000 Subject: [PATCH 43/51] Debug testcontainers. --- .github/workflows/budibase_ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 6a9aa3e8d0..97cfe88804 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -235,6 +235,7 @@ jobs: - name: Test server env: DATASOURCE: ${{ matrix.datasource }} + DEBUG: testcontainers* run: | if ${{ env.ONLY_AFFECTED_TASKS }}; then AFFECTED=$(yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/server) From f604aba7ba278915e65329858ad39607510c5ebe Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 26 Feb 2025 15:41:09 +0000 Subject: [PATCH 44/51] Try ubuntu-latest --- .github/workflows/budibase_ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 97cfe88804..af3950a439 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -30,7 +30,7 @@ env: jobs: lint: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout repo uses: actions/checkout@v4 @@ -47,7 +47,7 @@ jobs: - run: yarn lint build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout repo uses: actions/checkout@v4 @@ -76,7 +76,7 @@ jobs: fi helm-lint: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout repo uses: actions/checkout@v4 @@ -88,7 +88,7 @@ jobs: - run: cd charts/budibase && helm lint . test-libraries: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout repo uses: actions/checkout@v4 @@ -122,7 +122,7 @@ jobs: fi test-worker: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout repo uses: actions/checkout@v4 @@ -151,7 +151,7 @@ jobs: yarn test --verbose --reporters=default --reporters=github-actions test-server: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest strategy: matrix: datasource: @@ -254,7 +254,7 @@ jobs: yarn test --filter $FILTER --verbose --reporters=default --reporters=github-actions check-pro-submodule: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') steps: - name: Checkout repo and submodules @@ -313,7 +313,7 @@ jobs: fi check-lockfile: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') steps: - name: Checkout repo From 9076d847c19cb9f855101e38213e213c7e402738 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 26 Feb 2025 15:48:26 +0000 Subject: [PATCH 45/51] Update to mssql 2022 to see if that helps. --- .github/workflows/budibase_ci.yml | 16 ++++++++-------- packages/server/datasource-sha.env | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index af3950a439..91c32359d2 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -30,7 +30,7 @@ env: jobs: lint: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -47,7 +47,7 @@ jobs: - run: yarn lint build: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -76,7 +76,7 @@ jobs: fi helm-lint: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -88,7 +88,7 @@ jobs: - run: cd charts/budibase && helm lint . test-libraries: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -122,7 +122,7 @@ jobs: fi test-worker: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -151,7 +151,7 @@ jobs: yarn test --verbose --reporters=default --reporters=github-actions test-server: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: matrix: datasource: @@ -254,7 +254,7 @@ jobs: yarn test --filter $FILTER --verbose --reporters=default --reporters=github-actions check-pro-submodule: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') steps: - name: Checkout repo and submodules @@ -313,7 +313,7 @@ jobs: fi check-lockfile: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') steps: - name: Checkout repo diff --git a/packages/server/datasource-sha.env b/packages/server/datasource-sha.env index 383fb191f4..69750793ce 100644 --- a/packages/server/datasource-sha.env +++ b/packages/server/datasource-sha.env @@ -1,4 +1,4 @@ -MSSQL_SHA=sha256:3b913841850a4d57fcfcb798be06acc88ea0f2acc5418bc0c140a43e91c4a545 +MSSQL_SHA=sha256:d252932ef839c24c61c1139cc98f69c85ca774fa7c6bfaaa0015b7eb02b9dc87 MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d From 82e32b3db84fa2562c332d4190476c8f13bea8b6 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 26 Feb 2025 15:56:32 +0000 Subject: [PATCH 46/51] Fix healthcheck. --- packages/server/src/integrations/tests/utils/mssql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/tests/utils/mssql.ts b/packages/server/src/integrations/tests/utils/mssql.ts index 548631a987..d84fb5d12d 100644 --- a/packages/server/src/integrations/tests/utils/mssql.ts +++ b/packages/server/src/integrations/tests/utils/mssql.ts @@ -23,7 +23,7 @@ export async function getDatasource(): Promise { }) .withWaitStrategy( Wait.forSuccessfulCommand( - "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Password_123 -q 'SELECT 1'" + "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P Password_123 -q 'SELECT 1'" ).withStartupTimeout(20000) ) ) From a684c9f0490d20f5685201e10fcd903c64116170 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 26 Feb 2025 16:09:14 +0000 Subject: [PATCH 47/51] Fix self-signed cert error in tests. --- .github/workflows/budibase_ci.yml | 1 - packages/server/src/integrations/tests/utils/mssql.ts | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 91c32359d2..2e7851b338 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -235,7 +235,6 @@ jobs: - name: Test server env: DATASOURCE: ${{ matrix.datasource }} - DEBUG: testcontainers* run: | if ${{ env.ONLY_AFFECTED_TASKS }}; then AFFECTED=$(yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/server) diff --git a/packages/server/src/integrations/tests/utils/mssql.ts b/packages/server/src/integrations/tests/utils/mssql.ts index d84fb5d12d..453c2b8bc8 100644 --- a/packages/server/src/integrations/tests/utils/mssql.ts +++ b/packages/server/src/integrations/tests/utils/mssql.ts @@ -23,7 +23,7 @@ export async function getDatasource(): Promise { }) .withWaitStrategy( Wait.forSuccessfulCommand( - "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P Password_123 -q 'SELECT 1'" + "/opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P Password_123 -q 'SELECT 1'" ).withStartupTimeout(20000) ) ) @@ -44,7 +44,8 @@ export async function getDatasource(): Promise { user: "sa", password: "Password_123", options: { - encrypt: false, + encrypt: true, + trustServerCertificate: true, }, }, } From 3e07627b7134a41c31f8706ef444d048458d77c0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 27 Feb 2025 09:40:25 +0000 Subject: [PATCH 48/51] Update pro reference. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 219be3ef38..e3843dd4ea 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 219be3ef380ac8cb16b34892f62117d31b50eec7 +Subproject commit e3843dd4eaced68ae063355b77df200dbc789c98 From 300734328f5483a1c7ff9dac39807ffa086687a5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 27 Feb 2025 11:34:19 +0100 Subject: [PATCH 49/51] Remove unused index.js --- packages/client/src/index.js | 142 ----------------------------------- 1 file changed, 142 deletions(-) delete mode 100644 packages/client/src/index.js diff --git a/packages/client/src/index.js b/packages/client/src/index.js deleted file mode 100644 index f19c46a452..0000000000 --- a/packages/client/src/index.js +++ /dev/null @@ -1,142 +0,0 @@ -import ClientApp from "./components/ClientApp.svelte" -import UpdatingApp from "./components/UpdatingApp.svelte" -import { - builderStore, - appStore, - blockStore, - componentStore, - environmentStore, - dndStore, - eventStore, - hoverStore, - stateStore, - routeStore, -} from "./stores" -import loadSpectrumIcons from "@budibase/bbui/spectrum-icons-vite.js" -import { get } from "svelte/store" -import { initWebsocket } from "./websocket.js" - -// Provide svelte and svelte/internal as globals for custom components -import * as svelte from "svelte" -import * as internal from "svelte/internal" - -window.svelte_internal = internal -window.svelte = svelte - -// Initialise spectrum icons -loadSpectrumIcons() - -let app - -const loadBudibase = async () => { - // Update builder store with any builder flags - builderStore.set({ - ...get(builderStore), - inBuilder: !!window["##BUDIBASE_IN_BUILDER##"], - layout: window["##BUDIBASE_PREVIEW_LAYOUT##"], - screen: window["##BUDIBASE_PREVIEW_SCREEN##"], - selectedComponentId: window["##BUDIBASE_SELECTED_COMPONENT_ID##"], - previewId: window["##BUDIBASE_PREVIEW_ID##"], - theme: window["##BUDIBASE_PREVIEW_THEME##"], - customTheme: window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"], - previewDevice: window["##BUDIBASE_PREVIEW_DEVICE##"], - navigation: window["##BUDIBASE_PREVIEW_NAVIGATION##"], - hiddenComponentIds: window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"], - usedPlugins: window["##BUDIBASE_USED_PLUGINS##"], - location: window["##BUDIBASE_LOCATION##"], - snippets: window["##BUDIBASE_SNIPPETS##"], - componentErrors: window["##BUDIBASE_COMPONENT_ERRORS##"], - }) - - // Set app ID - this window flag is set by both the preview and the real - // server rendered app HTML - appStore.actions.setAppId(window["##BUDIBASE_APP_ID##"]) - - // Set the flag used to determine if the app is being loaded via an iframe - appStore.actions.setAppEmbedded( - window["##BUDIBASE_APP_EMBEDDED##"] === "true" - ) - - if (window.MIGRATING_APP) { - new UpdatingApp({ - target: window.document.body, - }) - return - } - - // Fetch environment info - if (!get(environmentStore)?.loaded) { - await environmentStore.actions.fetchEnvironment() - } - - // Register handler for runtime events from the builder - window.handleBuilderRuntimeEvent = (type, data) => { - if (!window["##BUDIBASE_IN_BUILDER##"]) { - return - } - if (type === "event-completed") { - eventStore.actions.resolveEvent(data) - } else if (type === "eject-block") { - const block = blockStore.actions.getBlock(data) - block?.eject() - } else if (type === "dragging-new-component") { - const { dragging, component } = data - if (dragging) { - const definition = - componentStore.actions.getComponentDefinition(component) - dndStore.actions.startDraggingNewComponent({ component, definition }) - } else { - dndStore.actions.reset() - } - } else if (type === "request-context") { - const { selectedComponentInstance, screenslotInstance } = - get(componentStore) - const instance = selectedComponentInstance || screenslotInstance - const context = instance?.getDataContext() - let stringifiedContext = null - try { - stringifiedContext = JSON.stringify(context) - } catch (error) { - // Ignore - invalid context - } - eventStore.actions.dispatchEvent("provide-context", { - context: stringifiedContext, - }) - } else if (type === "hover-component") { - hoverStore.actions.hoverComponent(data, false) - } else if (type === "builder-meta") { - builderStore.actions.setMetadata(data) - } else if (type === "builder-state") { - const [[key, value]] = Object.entries(data) - stateStore.actions.setValue(key, value) - } else if (type === "builder-url-test-data") { - const { route, testValue } = data - routeStore.actions.setTestUrlParams(route, testValue) - } - } - - // Register any custom components - if (window["##BUDIBASE_CUSTOM_COMPONENTS##"]) { - window["##BUDIBASE_CUSTOM_COMPONENTS##"].forEach(component => { - componentStore.actions.registerCustomComponent(component) - }) - } - - // Make a callback available for custom component bundles to register - // themselves at runtime - window.registerCustomComponent = - componentStore.actions.registerCustomComponent - - // Initialise websocket - initWebsocket() - - // Create app if one hasn't been created yet - if (!app) { - app = new ClientApp({ - target: window.document.body, - }) - } -} - -// Attach to window so the HTML template can call this when it loads -window.loadBudibase = loadBudibase From 2865af355681c2aa31440a6493e190ff1a8fef56 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 27 Feb 2025 11:43:15 +0100 Subject: [PATCH 50/51] Don't validate components within plugins --- packages/builder/src/stores/builder/screenComponent.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 310bf2172c..003f8a47ff 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -239,7 +239,10 @@ function getMissingAncestors( ancestors: string[] ): UIComponentError[] { const definition = definitions[component._component] - + if (ancestors.some(a => !a.startsWith(BudibasePrefix))) { + // We don't have a way to know what components are used within a plugin component + return [] + } if (!definition?.requiredAncestors?.length) { return [] } From 6fc325a6fa24f2c80496368a27a91a88cad296e6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 27 Feb 2025 11:49:04 +0100 Subject: [PATCH 51/51] Validate first missing ancestors --- packages/builder/src/stores/builder/screenComponent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 003f8a47ff..2b3dde91f8 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -81,11 +81,11 @@ export const screenComponentErrorList = derived( const errors: UIComponentError[] = [] function checkComponentErrors(component: Component, ancestors: string[]) { + errors.push(...getMissingAncestors(component, definitions, ancestors)) errors.push( ...getInvalidDatasources(screen, component, datasources, definitions) ) errors.push(...getMissingRequiredSettings(component, definitions)) - errors.push(...getMissingAncestors(component, definitions, ancestors)) for (const child of component._children || []) { checkComponentErrors(child, [...ancestors, component._component])