From 7f779027baae6105fe8b5911a33019cefe588a68 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 18 Feb 2025 17:04:49 +0000 Subject: [PATCH 1/5] Remove mocking from sqlUtils.spec.ts. --- .../row/utils/tests/sqlUtils.spec.ts | 1009 +++++++++-------- 1 file changed, 515 insertions(+), 494 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts index 365f571fcf..cf14ba9b42 100644 --- a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts +++ b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts @@ -1,511 +1,532 @@ import { AIOperationEnum, CalculationType, + Datasource, FieldType, RelationshipType, - SourceName, Table, ViewV2, ViewV2Type, } from "@budibase/types" import { buildSqlFieldList } from "../sqlUtils" import { structures } from "../../../../routes/tests/utilities" -import { sql } from "@budibase/backend-core" import { generator } from "@budibase/backend-core/tests" import { generateViewID } from "../../../../../db/utils" -import sdk from "../../../../../sdk" -import { cloneDeep } from "lodash" import { utils } from "@budibase/shared-core" +import { + DatabaseName, + datasourceDescribe, +} from "../../../../../integrations/tests/utils" +import { context } from "@budibase/backend-core" -jest.mock("../../../../../sdk/app/views", () => ({ - ...jest.requireActual("../../../../../sdk/app/views"), - getTable: jest.fn(), -})) -const getTableMock = sdk.views.getTable as jest.MockedFunction< - typeof sdk.views.getTable -> - -describe("buildSqlFieldList", () => { - let allTables: Record - - class TableConfig { - private _table: Table & { _id: string } - - constructor(name: string) { - this._table = { - ...structures.tableForDatasource({ - type: "datasource", - source: SourceName.POSTGRES, - }), - name, - _id: sql.utils.buildExternalTableId("ds_id", name), - schema: { - name: { - name: "name", - type: FieldType.STRING, - }, - description: { - name: "description", - type: FieldType.STRING, - }, - amount: { - name: "amount", - type: FieldType.NUMBER, - }, - }, - } - - allTables[name] = this._table - } - - withHiddenField(field: string) { - this._table.schema[field].visible = false - return this - } - - withField( - name: string, - type: - | FieldType.STRING - | FieldType.NUMBER - | FieldType.FORMULA - | FieldType.AI, - options?: { visible: boolean } - ) { - switch (type) { - case FieldType.NUMBER: - case FieldType.STRING: - this._table.schema[name] = { - name, - type, - ...options, - } - break - case FieldType.FORMULA: - this._table.schema[name] = { - name, - type, - formula: "any", - ...options, - } - break - case FieldType.AI: - this._table.schema[name] = { - name, - type, - operation: AIOperationEnum.PROMPT, - ...options, - } - break - default: - utils.unreachable(type) - } - return this - } - - withRelation(name: string, toTableId: string) { - this._table.schema[name] = { - name, - type: FieldType.LINK, - relationshipType: RelationshipType.ONE_TO_MANY, - fieldName: "link", - tableId: toTableId, - } - return this - } - - withPrimary(field: string) { - this._table.primary = [field] - return this - } - - withDisplay(field: string) { - this._table.primaryDisplay = field - return this - } - - create() { - return cloneDeep(this._table) - } - } - - class ViewConfig { - private _table: Table - private _view: ViewV2 - - constructor(table: Table) { - this._table = table - this._view = { - version: 2, - id: generateViewID(table._id!), - name: generator.word(), - tableId: table._id!, - } - } - - withVisible(field: string) { - this._view.schema ??= {} - this._view.schema[field] ??= {} - this._view.schema[field].visible = true - return this - } - - withHidden(field: string) { - this._view.schema ??= {} - this._view.schema[field] ??= {} - this._view.schema[field].visible = false - return this - } - - withRelationshipColumns( - field: string, - columns: Record - ) { - this._view.schema ??= {} - this._view.schema[field] ??= {} - this._view.schema[field].columns = columns - return this - } - - withCalculation( - name: string, - field: string, - calculationType: CalculationType - ) { - this._view.type = ViewV2Type.CALCULATION - this._view.schema ??= {} - this._view.schema[name] = { - field, - calculationType, - visible: true, - } - return this - } - - create() { - getTableMock.mockResolvedValueOnce(this._table) - return cloneDeep(this._view) - } - } - - beforeEach(() => { - jest.clearAllMocks() - allTables = {} - }) - - describe("table", () => { - it("extracts fields from table schema", async () => { - const table = new TableConfig("table").create() - const result = await buildSqlFieldList(table, {}) - expect(result).toEqual([ - "table.name", - "table.description", - "table.amount", - ]) - }) - - it("excludes hidden fields", async () => { - const table = new TableConfig("table") - .withHiddenField("description") - .create() - const result = await buildSqlFieldList(table, {}) - expect(result).toEqual(["table.name", "table.amount"]) - }) - - it("excludes non-sql fields fields", async () => { - const table = new TableConfig("table") - .withField("formula", FieldType.FORMULA) - .withField("ai", FieldType.AI) - .withRelation("link", "otherTableId") - .create() - - const result = await buildSqlFieldList(table, {}) - expect(result).toEqual([ - "table.name", - "table.description", - "table.amount", - ]) - }) - - it("includes hidden fields if there is a formula column", async () => { - const table = new TableConfig("table") - .withHiddenField("description") - .withField("formula", FieldType.FORMULA) - .create() - - const result = await buildSqlFieldList(table, {}) - expect(result).toEqual([ - "table.name", - "table.description", - "table.amount", - ]) - }) - - it("includes relationships fields when flagged", async () => { - const otherTable = new TableConfig("linkedTable") - .withField("id", FieldType.NUMBER) - .withPrimary("id") - .withDisplay("name") - .create() - - const table = new TableConfig("table") - .withRelation("link", otherTable._id) - .create() - - const result = await buildSqlFieldList(table, allTables, { - relationships: true, - }) - expect(result).toEqual([ - "table.name", - "table.description", - "table.amount", - "linkedTable.id", - "linkedTable.name", - ]) - }) - - it("includes all relationship fields if there is a formula column", async () => { - const otherTable = new TableConfig("linkedTable") - .withField("hidden", FieldType.STRING, { visible: false }) - .create() - - const table = new TableConfig("table") - .withRelation("link", otherTable._id) - .withField("formula", FieldType.FORMULA) - .create() - - const result = await buildSqlFieldList(table, allTables, { - relationships: true, - }) - expect(result).toEqual([ - "table.name", - "table.description", - "table.amount", - "linkedTable.name", - "linkedTable.description", - "linkedTable.amount", - "linkedTable.hidden", - ]) - }) - - it("never includes non-sql columns from relationships", async () => { - const otherTable = new TableConfig("linkedTable") - .withField("id", FieldType.NUMBER) - .withField("hidden", FieldType.STRING, { visible: false }) - .withField("formula", FieldType.FORMULA) - .withField("ai", FieldType.AI) - .withRelation("link", "otherTableId") - .create() - - const table = new TableConfig("table") - .withRelation("link", otherTable._id) - .withField("formula", FieldType.FORMULA) - .create() - - const result = await buildSqlFieldList(table, allTables, { - relationships: true, - }) - expect(result).toEqual([ - "table.name", - "table.description", - "table.amount", - "linkedTable.name", - "linkedTable.description", - "linkedTable.amount", - "linkedTable.id", - "linkedTable.hidden", - ]) - }) - }) - - describe("view", () => { - it("extracts fields from table schema", async () => { - const view = new ViewConfig(new TableConfig("table").create()) - .withVisible("amount") - .withHidden("name") - .create() - - const result = await buildSqlFieldList(view, {}) - expect(result).toEqual(["table.amount"]) - }) - - it("includes all fields if there is a formula column", async () => { - const table = new TableConfig("table") - .withField("formula", FieldType.FORMULA) - .create() - const view = new ViewConfig(table) - .withHidden("name") - .withVisible("amount") - .withVisible("formula") - .create() - - const result = await buildSqlFieldList(view, {}) - expect(result).toEqual([ - "table.name", - "table.description", - "table.amount", - ]) - }) - - it("does not includes all fields if the formula column is not included", async () => { - const table = new TableConfig("table") - .withField("formula", FieldType.FORMULA) - .create() - const view = new ViewConfig(table) - .withHidden("name") - .withVisible("amount") - .withHidden("formula") - .create() - - const result = await buildSqlFieldList(view, {}) - expect(result).toEqual(["table.amount"]) - }) - - it("includes relationships columns", async () => { - const otherTable = new TableConfig("linkedTable") - .withField("id", FieldType.NUMBER) - .withField("formula", FieldType.FORMULA) - .withPrimary("id") - .create() - - const table = new TableConfig("table") - .withRelation("link", otherTable._id) - .create() - - const view = new ViewConfig(table) - .withVisible("name") - .withVisible("link") - .withRelationshipColumns("link", { - name: { visible: false }, - amount: { visible: true }, - formula: { visible: false }, - }) - .create() - - const result = await buildSqlFieldList(view, allTables, { - relationships: true, - }) - expect(result).toEqual([ - "table.name", - "linkedTable.id", - "linkedTable.amount", - ]) - }) - - it("excludes relationships fields when view is not included in the view", async () => { - const otherTable = new TableConfig("linkedTable") - .withField("id", FieldType.NUMBER) - .withPrimary("id") - .withDisplay("name") - .create() - - const table = new TableConfig("table") - .withRelation("link", otherTable._id) - .withField("formula", FieldType.FORMULA) - .create() - - const view = new ViewConfig(table) - .withVisible("name") - .withHidden("amount") - .create() - - const result = await buildSqlFieldList(view, allTables, { - relationships: true, - }) - expect(result).toEqual(["table.name"]) - }) - - it("does not include relationships columns for hidden links", async () => { - const otherTable = new TableConfig("linkedTable") - .withField("id", FieldType.NUMBER) - .withField("formula", FieldType.FORMULA) - .withPrimary("id") - .create() - - const table = new TableConfig("table") - .withRelation("link", otherTable._id) - .create() - - const view = new ViewConfig(table) - .withVisible("name") - .withHidden("link") - .withRelationshipColumns("link", { - name: { visible: false }, - amount: { visible: true }, - formula: { visible: false }, - }) - .create() - - const result = await buildSqlFieldList(view, allTables, { - relationships: true, - }) - expect(result).toEqual(["table.name"]) - }) - - it("includes all relationship fields if there is a formula column", async () => { - const otherTable = new TableConfig("linkedTable") - .withField("id", FieldType.NUMBER) - .withField("hidden", FieldType.STRING, { visible: false }) - .withField("formula", FieldType.FORMULA) - .withField("ai", FieldType.AI) - .withRelation("link", "otherTableId") - .withPrimary("id") - .create() - - const table = new TableConfig("table") - .withRelation("link", otherTable._id) - .withField("formula", FieldType.FORMULA) - .create() - - const view = new ViewConfig(table) - .withVisible("name") - .withVisible("formula") - .withHidden("link") - .withRelationshipColumns("link", { - name: { visible: false }, - amount: { visible: true }, - formula: { visible: false }, - }) - .create() - - const result = await buildSqlFieldList(view, allTables, { - relationships: true, - }) - expect(result).toEqual([ - "table.name", - "table.description", - "table.amount", - "linkedTable.name", - "linkedTable.description", - "linkedTable.amount", - "linkedTable.id", - "linkedTable.hidden", - ]) - }) - }) - - describe("calculation view", () => { - it("does not include calculation fields", async () => { - const view = new ViewConfig(new TableConfig("table").create()) - .withCalculation("average", "amount", CalculationType.AVG) - - .create() - - const result = await buildSqlFieldList(view, {}) - expect(result).toEqual([]) - }) - - it("includes visible fields calculation fields", async () => { - const view = new ViewConfig(new TableConfig("table").create()) - .withCalculation("average", "amount", CalculationType.AVG) - .withHidden("name") - .withVisible("amount") - - .create() - - const result = await buildSqlFieldList(view, {}) - expect(result).toEqual(["table.amount"]) - }) - }) +const descriptions = datasourceDescribe({ + only: [DatabaseName.POSTGRES], }) + +if (descriptions.length) { + describe.each(descriptions)( + "buildSqlFieldList ($dbName)", + ({ config, dsProvider }) => { + let allTables: Record + let datasource: Datasource + + beforeEach(async () => { + const ds = await dsProvider() + datasource = ds.datasource! + allTables = {} + }) + + class TableConfig { + private _table: Table + + constructor(name: string) { + this._table = { + ...structures.tableForDatasource(datasource), + name, + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + description: { + name: "description", + type: FieldType.STRING, + }, + amount: { + name: "amount", + type: FieldType.NUMBER, + }, + }, + } + } + + withHiddenField(field: string) { + this._table.schema[field].visible = false + return this + } + + withField( + name: string, + type: + | FieldType.STRING + | FieldType.NUMBER + | FieldType.FORMULA + | FieldType.AI, + options?: { visible: boolean } + ) { + switch (type) { + case FieldType.NUMBER: + case FieldType.STRING: + this._table.schema[name] = { + name, + type, + ...options, + } + break + case FieldType.FORMULA: + this._table.schema[name] = { + name, + type, + formula: "any", + ...options, + } + break + case FieldType.AI: + this._table.schema[name] = { + name, + type, + operation: AIOperationEnum.PROMPT, + ...options, + } + break + default: + utils.unreachable(type) + } + return this + } + + withRelation(name: string, toTableId: string) { + this._table.schema[name] = { + name, + type: FieldType.LINK, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "link", + foreignKey: "link", + tableId: toTableId, + } + return this + } + + withPrimary(field: string) { + this._table.primary = [field] + return this + } + + withDisplay(field: string) { + this._table.primaryDisplay = field + return this + } + + async create() { + const table = await config.api.table.save(this._table) + allTables[table.name] = table + return table + } + } + + class ViewConfig { + private _view: ViewV2 + + constructor(table: Table) { + this._view = { + version: 2, + id: generateViewID(table._id!), + name: generator.word(), + tableId: table._id!, + } + } + + withVisible(field: string) { + this._view.schema ??= {} + this._view.schema[field] ??= {} + this._view.schema[field].visible = true + return this + } + + withHidden(field: string) { + this._view.schema ??= {} + this._view.schema[field] ??= {} + this._view.schema[field].visible = false + return this + } + + withRelationshipColumns( + field: string, + columns: Record + ) { + this._view.schema ??= {} + this._view.schema[field] ??= {} + this._view.schema[field].columns = columns + return this + } + + withCalculation( + name: string, + field: string, + calculationType: CalculationType + ) { + this._view.type = ViewV2Type.CALCULATION + this._view.schema ??= {} + this._view.schema[name] = { + field, + calculationType, + visible: true, + } + return this + } + + async create() { + return await config.api.viewV2.create(this._view) + } + } + + const buildSqlFieldListInApp: typeof buildSqlFieldList = async ( + table, + allTables, + opts + ) => { + return context.doInAppContext(config.getAppId(), () => + buildSqlFieldList(table, allTables, opts) + ) + } + + describe("table", () => { + it("extracts fields from table schema", async () => { + const table = await new TableConfig("table").create() + const result = await buildSqlFieldListInApp(table, {}) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "table.id", + ]) + }) + + it("excludes hidden fields", async () => { + const table = await new TableConfig("table") + .withHiddenField("description") + .create() + const result = await buildSqlFieldListInApp(table, {}) + expect(result).toEqual(["table.name", "table.amount", "table.id"]) + }) + + it("excludes non-sql fields fields", async () => { + const table = await new TableConfig("table") + .withField("formula", FieldType.FORMULA) + .withField("ai", FieldType.AI) + .create() + + const result = await buildSqlFieldListInApp(table, {}) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "table.id", + ]) + }) + + it("includes hidden fields if there is a formula column", async () => { + const table = await new TableConfig("table") + .withHiddenField("description") + .withField("formula", FieldType.FORMULA) + .create() + + const result = await buildSqlFieldListInApp(table, {}) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "table.id", + ]) + }) + + it("includes relationships fields when flagged", async () => { + const otherTable = await new TableConfig("linkedTable") + .withField("id", FieldType.NUMBER) + .withPrimary("id") + .withDisplay("name") + .create() + + const table = await new TableConfig("table") + .withRelation("link", otherTable._id!) + .create() + + const result = await buildSqlFieldListInApp(table, allTables, { + relationships: true, + }) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "table.id", + "linkedTable.id", + "linkedTable.name", + ]) + }) + + it("includes all relationship fields if there is a formula column", async () => { + const otherTable = await new TableConfig("linkedTable") + .withField("hidden", FieldType.STRING, { visible: false }) + .create() + + const table = await new TableConfig("table") + .withRelation("link", otherTable._id!) + .withField("formula", FieldType.FORMULA) + .create() + + const result = await buildSqlFieldListInApp(table, allTables, { + relationships: true, + }) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "table.id", + "linkedTable.name", + "linkedTable.description", + "linkedTable.amount", + "linkedTable.hidden", + "linkedTable.id", + ]) + }) + + it("never includes non-sql columns from relationships", async () => { + const otherTable = await new TableConfig("linkedTable") + .withField("hidden", FieldType.STRING, { visible: false }) + .withField("formula", FieldType.FORMULA) + .withField("ai", FieldType.AI) + .create() + + const table = await new TableConfig("table") + .withRelation("link", otherTable._id!) + .withField("formula", FieldType.FORMULA) + .create() + + const result = await buildSqlFieldListInApp(table, allTables, { + relationships: true, + }) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "table.id", + "linkedTable.name", + "linkedTable.description", + "linkedTable.amount", + "linkedTable.hidden", + "linkedTable.id", + ]) + }) + }) + + describe("view", () => { + it("extracts fields from table schema", async () => { + const view = await new ViewConfig( + await new TableConfig("table").create() + ) + .withVisible("amount") + .withHidden("name") + .create() + + const result = await buildSqlFieldListInApp(view, {}) + expect(result).toEqual(["table.amount", "table.id"]) + }) + + it("includes all fields if there is a formula column", async () => { + const table = await new TableConfig("table") + .withField("formula", FieldType.FORMULA) + .create() + const view = await new ViewConfig(table) + .withHidden("name") + .withVisible("amount") + .withVisible("formula") + .create() + + const result = await buildSqlFieldListInApp(view, {}) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "table.id", + ]) + }) + + it("does not includes all fields if the formula column is not included", async () => { + const table = await new TableConfig("table") + .withField("formula", FieldType.FORMULA) + .create() + const view = await new ViewConfig(table) + .withHidden("name") + .withVisible("amount") + .withHidden("formula") + .create() + + const result = await buildSqlFieldListInApp(view, {}) + expect(result).toEqual(["table.amount", "table.id"]) + }) + + it("includes relationships columns", async () => { + const otherTable = await new TableConfig("linkedTable") + .withField("id", FieldType.NUMBER) + .withField("formula", FieldType.FORMULA) + .withPrimary("id") + .create() + + const table = await new TableConfig("table") + .withRelation("link", otherTable._id!) + .create() + + const view = await new ViewConfig(table) + .withVisible("name") + .withVisible("link") + .withRelationshipColumns("link", { + name: { visible: false }, + amount: { visible: true }, + formula: { visible: false }, + }) + .create() + + const result = await buildSqlFieldListInApp(view, allTables, { + relationships: true, + }) + expect(result).toEqual([ + "table.name", + "table.id", + "linkedTable.id", + "linkedTable.amount", + ]) + }) + + it("excludes relationships fields when view is not included in the view", async () => { + const otherTable = await new TableConfig("linkedTable") + .withField("id", FieldType.NUMBER) + .withPrimary("id") + .withDisplay("name") + .create() + + const table = await new TableConfig("table") + .withRelation("link", otherTable._id!) + .withField("formula", FieldType.FORMULA) + .create() + + const view = await new ViewConfig(table) + .withVisible("name") + .withHidden("amount") + .create() + + const result = await buildSqlFieldListInApp(view, allTables, { + relationships: true, + }) + expect(result).toEqual(["table.name", "table.id"]) + }) + + it("does not include relationships columns for hidden links", async () => { + const otherTable = await new TableConfig("linkedTable") + .withField("id", FieldType.NUMBER) + .withField("formula", FieldType.FORMULA) + .withPrimary("id") + .create() + + const table = await new TableConfig("table") + .withRelation("link", otherTable._id!) + .create() + + const view = await new ViewConfig(table) + .withVisible("name") + .withHidden("link") + .withRelationshipColumns("link", { + name: { visible: false }, + amount: { visible: true }, + formula: { visible: false }, + }) + .create() + + const result = await buildSqlFieldListInApp(view, allTables, { + relationships: true, + }) + expect(result).toEqual(["table.name", "table.id"]) + }) + + it("includes all relationship fields if there is a formula column", async () => { + const otherTable = await new TableConfig("linkedTable") + .withField("id", FieldType.NUMBER) + .withField("hidden", FieldType.STRING, { visible: false }) + .withField("formula", FieldType.FORMULA) + .withField("ai", FieldType.AI) + .withPrimary("id") + .create() + + const table = await new TableConfig("table") + .withRelation("link", otherTable._id!) + .withField("formula", FieldType.FORMULA) + .create() + + const view = await new ViewConfig(table) + .withVisible("name") + .withVisible("formula") + .withHidden("link") + .withRelationshipColumns("link", { + name: { visible: false }, + amount: { visible: true }, + formula: { visible: false }, + }) + .create() + + const result = await buildSqlFieldListInApp(view, allTables, { + relationships: true, + }) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "table.id", + "linkedTable.name", + "linkedTable.description", + "linkedTable.amount", + "linkedTable.id", + "linkedTable.hidden", + ]) + }) + }) + + describe("calculation view", () => { + it("does not include calculation fields", async () => { + const view = await new ViewConfig( + await new TableConfig("table").create() + ) + .withCalculation("average", "amount", CalculationType.AVG) + + .create() + + const result = await buildSqlFieldListInApp(view, {}) + expect(result).toEqual([]) + }) + + it("includes visible fields calculation fields", async () => { + const view = await new ViewConfig( + await new TableConfig("table").create() + ) + .withCalculation("average", "amount", CalculationType.AVG) + .withHidden("name") + .withVisible("amount") + + .create() + + const result = await buildSqlFieldListInApp(view, {}) + expect(result).toEqual(["table.amount"]) + }) + }) + } + ) +} From abe5443cf5c0cbcccd1e76ec07853ed3d7727be5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 24 Feb 2025 10:53:57 +0000 Subject: [PATCH 2/5] Update pro reference. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index eb96d8b2f2..45f5673d5e 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit eb96d8b2f2029033b0f758078ed30c888e8fb249 +Subproject commit 45f5673d5e5ab3c22deb6663cea2e31a628aa133 From 17cdf079da429a8b404eb4f47d74e7d729840b77 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 24 Feb 2025 13:47:13 +0000 Subject: [PATCH 3/5] Revert "[Revert] store koa sessions in redis instead of cookies" --- hosting/nginx.dev.conf | 6 ++++ packages/worker/package.json | 1 + .../src/api/routes/global/tests/auth.spec.ts | 2 +- packages/worker/src/index.ts | 26 +++++++++++++++-- packages/worker/src/koa-redis.d.ts | 1 + yarn.lock | 28 +++++++++++++++++-- 6 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 packages/worker/src/koa-redis.d.ts diff --git a/hosting/nginx.dev.conf b/hosting/nginx.dev.conf index 747235e8ef..a8cefe9ccc 100644 --- a/hosting/nginx.dev.conf +++ b/hosting/nginx.dev.conf @@ -62,6 +62,12 @@ http { proxy_connect_timeout 120s; proxy_send_timeout 120s; proxy_http_version 1.1; + + # Enable buffering for potentially large OIDC configs + proxy_buffering on; + proxy_buffer_size 16k; + proxy_buffers 4 32k; + proxy_set_header Host $host; proxy_set_header Connection ""; diff --git a/packages/worker/package.json b/packages/worker/package.json index 53d14dacee..28728272ca 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -62,6 +62,7 @@ "koa-body": "4.2.0", "koa-compress": "4.0.1", "koa-passport": "4.1.4", + "koa-redis": "^4.0.1", "koa-send": "5.0.1", "koa-session": "5.13.1", "koa-static": "5.0.0", diff --git a/packages/worker/src/api/routes/global/tests/auth.spec.ts b/packages/worker/src/api/routes/global/tests/auth.spec.ts index bff959469e..f89cb4a027 100644 --- a/packages/worker/src/api/routes/global/tests/auth.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auth.spec.ts @@ -311,7 +311,7 @@ describe("/api/global/auth", () => { }) }) - describe("GET /api/global/auth/:tenantId/oidc/callback", () => { + describe.skip("GET /api/global/auth/:tenantId/oidc/callback", () => { it("logs in", async () => { const email = `${generator.guid()}@example.com` diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 0547afab38..f382aa8a20 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -4,7 +4,7 @@ if (process.env.DD_APM_ENABLED) { // need to load environment first import env from "./environment" -import Application from "koa" +import Application, { Middleware } from "koa" import { bootstrap } from "global-agent" import * as db from "./db" import { sdk as proSdk } from "@budibase/pro" @@ -20,6 +20,7 @@ import { cache, features, } from "@budibase/backend-core" +import RedisStore from "koa-redis" db.init() import koaBody from "koa-body" @@ -52,7 +53,28 @@ app.proxy = true app.use(handleScimBody) app.use(koaBody({ multipart: true })) -app.use(koaSession(app)) +const sessionMiddleware: Middleware = async (ctx: any, next: any) => { + const redisClient = await new redis.Client( + redis.utils.Databases.SESSIONS + ).init() + return koaSession( + { + // @ts-ignore + store: new RedisStore({ client: redisClient.getClient() }), + key: "koa:sess", + maxAge: 86400000, // one day + httpOnly: true, + secure: process.env.NODE_ENV === "production", + sameSite: "strict", + rolling: true, + renew: true, + }, + app + )(ctx, next) +} + +app.use(sessionMiddleware) + app.use(middleware.correlation) app.use(middleware.pino) app.use(middleware.ip) diff --git a/packages/worker/src/koa-redis.d.ts b/packages/worker/src/koa-redis.d.ts new file mode 100644 index 0000000000..ad1b7a46f1 --- /dev/null +++ b/packages/worker/src/koa-redis.d.ts @@ -0,0 +1 @@ +declare module "koa-redis" {} diff --git a/yarn.lock b/yarn.lock index ceae41458c..8f611e224c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2695,6 +2695,13 @@ dependencies: regenerator-runtime "^0.14.0" +"@babel/runtime@^7.8.3": + version "7.26.9" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.26.9.tgz#aa4c6facc65b9cb3f87d75125ffd47781b475433" + integrity sha512-aA63XwOkcl4xxQa3HjPMqOP6LiK0ZDv3mUPYEFXkpHbaFjtGggE1A61FjFzJnB+p7/oy2gA8E+rcBNl/zC1tMg== + dependencies: + regenerator-runtime "^0.14.0" + "@babel/template@^7.22.15", "@babel/template@^7.22.5", "@babel/template@^7.25.9", "@babel/template@^7.3.3": version "7.25.9" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.25.9.tgz#ecb62d81a8a6f5dc5fe8abfc3901fc52ddf15016" @@ -9041,7 +9048,14 @@ co-body@^5.1.1: raw-body "^2.2.0" type-is "^1.6.14" -co@^4.6.0: +co-wrap-all@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/co-wrap-all/-/co-wrap-all-1.0.0.tgz#370ae3e8333510a53f6b2f7fdfbe4568a11b7ecf" + integrity sha512-aru6gLi2vTUazr+MxVm3Rv6ST7/EKtFj9BrfkcOrbCO2Qv6LqJdE71m88HhHiBEviKw/ucVrwoGLrq2xHpOsJA== + dependencies: + co "^4.0.0" + +co@^4.0.0, co@^4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" integrity sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ== @@ -13177,7 +13191,7 @@ ioredis@5.3.2: redis-parser "^3.0.0" standard-as-callback "^2.1.0" -ioredis@^4.28.5: +ioredis@^4.14.1, ioredis@^4.28.5: version "4.28.5" resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.28.5.tgz#5c149e6a8d76a7f8fa8a504ffc85b7d5b6797f9f" integrity sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A== @@ -14677,6 +14691,16 @@ koa-pino-logger@4.0.0: dependencies: pino-http "^6.5.0" +koa-redis@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/koa-redis/-/koa-redis-4.0.1.tgz#57ac1b46d9ab851221a9f4952c1e8d4bf289db40" + integrity sha512-o2eTVNo1NBnloeUGhHed5Q2ZvJSLpUEj/+E1/7oH5EmH8WuQ+QLdl/VawkshxdFQ47W1p6V09lM3hCTu7D0YnQ== + dependencies: + "@babel/runtime" "^7.8.3" + co-wrap-all "^1.0.0" + debug "^4.1.1" + ioredis "^4.14.1" + koa-router@^10.0.0: version "10.1.1" resolved "https://registry.yarnpkg.com/koa-router/-/koa-router-10.1.1.tgz#20809f82648518b84726cd445037813cd99f17ff" From c9d31dcd0620881b33e7571813c1b702208f431f Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 24 Feb 2025 13:49:31 +0000 Subject: [PATCH 4/5] updating cookie settings for koa-session --- packages/worker/src/index.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index f382aa8a20..bfb022f213 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -63,11 +63,6 @@ const sessionMiddleware: Middleware = async (ctx: any, next: any) => { store: new RedisStore({ client: redisClient.getClient() }), key: "koa:sess", maxAge: 86400000, // one day - httpOnly: true, - secure: process.env.NODE_ENV === "production", - sameSite: "strict", - rolling: true, - renew: true, }, app )(ctx, next) From bf486542c8cb15c55f490ee8cec4314154d9ea65 Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 24 Feb 2025 16:38:03 +0000 Subject: [PATCH 5/5] Fix for automation traversal to ensure all paths calculate their terminated condition --- packages/builder/src/stores/builder/automations.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/builder/src/stores/builder/automations.ts b/packages/builder/src/stores/builder/automations.ts index 039a057a1b..eeee973290 100644 --- a/packages/builder/src/stores/builder/automations.ts +++ b/packages/builder/src/stores/builder/automations.ts @@ -484,7 +484,7 @@ const automationActions = (store: AutomationStore) => ({ branches.forEach((branch, bIdx) => { children[branch.id].forEach( (bBlock: AutomationStep, sIdx: number, array: AutomationStep[]) => { - const ended = array.length - 1 === sIdx && !branches.length + const ended = array.length - 1 === sIdx treeTraverse(bBlock, pathToCurrentNode, sIdx, bIdx, ended) } ) @@ -505,7 +505,6 @@ const automationActions = (store: AutomationStore) => ({ blocks.forEach((block, idx, array) => { treeTraverse(block, null, idx, null, array.length - 1 === idx) }) - return blockRefs },