From ea14af8a64eeba84b3a964b13f63a49bfd67906f Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 28 May 2024 11:00:47 +0100 Subject: [PATCH 01/48] Offset button column to account for vertical scrollbar if required --- .../src/components/grid/layout/ButtonColumn.svelte | 7 ++++++- .../src/components/grid/layout/Grid.svelte | 3 ++- .../components/grid/overlays/ScrollOverlay.svelte | 12 ++++++------ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte index 3448042894..2e9bcf43d5 100644 --- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte +++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte @@ -3,6 +3,7 @@ import { Button } from "@budibase/bbui" import GridCell from "../cells/GridCell.svelte" import GridScrollWrapper from "./GridScrollWrapper.svelte" + import { ScrollBarSize } from "../lib/constants" const { renderedRows, @@ -16,6 +17,7 @@ scroll, isDragging, buttonColumnWidth, + showVScrollbar, } = getContext("grid") let measureContainer @@ -67,7 +69,7 @@ selected={rowSelected} highlighted={rowHovered || rowFocused} > -
+
{#each buttons as button}
diff --git a/packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte b/packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte deleted file mode 100644 index 647b1af526..0000000000 --- a/packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte +++ /dev/null @@ -1,25 +0,0 @@ - - -
- {#each options as option} - onClick(option.value)} - size="S" - icon={option.icon} - quiet - selected={option.value === value} - /> - {/each} -
From 151ff27351e9ac0422d47933286f82a4fa1b4533 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 29 May 2024 15:38:30 +0200 Subject: [PATCH 17/48] Styling --- .../grid/controls/ColumnsSettingButton.svelte | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte index 8fe3854d93..2608eea75f 100644 --- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte +++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte @@ -78,7 +78,7 @@ {column.label}
-
+
{#each options as option} toggleColumn(column, option.value)} @@ -86,6 +86,7 @@ icon={option.icon} quiet selected={option.value === columnToPermissionOptions(column)} + noPadding /> {/each}
@@ -101,11 +102,6 @@ flex-direction: column; gap: 12px; } - .buttons { - display: flex; - flex-direction: row; - gap: 8px; - } .columns { display: grid; align-items: center; @@ -118,4 +114,16 @@ display: flex; gap: 8px; } + .permissionPicker { + display: flex; + gap: var(--spacing-xs); + padding-left: calc(var(--spacing-xl) * 2); + } + + .permissionPicker :global(.spectrum-Icon) { + width: 14px; + } + .permissionPicker :global(.spectrum-ActionButton) { + width: 22px; + } From 06549149b6867acac854751a85aa42c77f64f30f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 29 May 2024 16:01:11 +0200 Subject: [PATCH 18/48] Remove readonly --- .../components/grid/controls/ColumnsSettingButton.svelte | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte index 2608eea75f..c3ce6d2743 100644 --- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte +++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte @@ -13,9 +13,8 @@ const toggleColumn = async (column, permission) => { const visible = permission !== PERMISSION_OPTIONS.HIDDEN - const readonly = permission === PERMISSION_OPTIONS.READONLY - datasource.actions.addSchemaMutation(column.name, { visible, readonly }) + datasource.actions.addSchemaMutation(column.name, { visible }) await datasource.actions.saveSchemaMutations() dispatch(visible ? "show-column" : "hide-column") } @@ -27,13 +26,11 @@ const PERMISSION_OPTIONS = { WRITABLE: "writable", - READONLY: "readonly", HIDDEN: "hidden", } const options = [ { icon: "Edit", value: PERMISSION_OPTIONS.WRITABLE }, - { icon: "Visibility", value: PERMISSION_OPTIONS.READONLY }, { icon: "VisibilityOff", value: PERMISSION_OPTIONS.HIDDEN }, ] @@ -42,10 +39,6 @@ return PERMISSION_OPTIONS.HIDDEN } - if (column.readonly) { - return PERMISSION_OPTIONS.READONLY - } - return PERMISSION_OPTIONS.WRITABLE } From 5b42aff1f750f2ae3c175d147e164b183fd46c8b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 29 May 2024 16:21:19 +0200 Subject: [PATCH 19/48] Add picker to sticky columns --- .../bbui/src/ActionButton/ActionButton.svelte | 4 ++++ .../grid/controls/ColumnsSettingButton.svelte | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/bbui/src/ActionButton/ActionButton.svelte b/packages/bbui/src/ActionButton/ActionButton.svelte index 96969d1435..d3cec0f307 100644 --- a/packages/bbui/src/ActionButton/ActionButton.svelte +++ b/packages/bbui/src/ActionButton/ActionButton.svelte @@ -57,6 +57,7 @@ class:fullWidth class="spectrum-ActionButton spectrum-ActionButton--size{size}" class:active + class:disabled {disabled} on:longPress on:click|preventDefault @@ -122,6 +123,9 @@ .is-selected:not(.emphasized) .spectrum-Icon { color: var(--spectrum-global-color-gray-900); } + .is-selected.disabled .spectrum-Icon { + color: var(--spectrum-global-color-gray-500); + } .tooltip { position: absolute; pointer-events: none; diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte index c3ce6d2743..2f8e141a61 100644 --- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte +++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte @@ -64,7 +64,19 @@ {$stickyColumn.label}
- + +
+ {#each options as option} + + {/each} +
{/if} {#each $columns as column}
@@ -99,6 +111,7 @@ display: grid; align-items: center; grid-template-columns: 1fr auto; + gap: 8px; } .columns :global(.spectrum-Switch) { margin-right: 0; From 53968f0725d4b1d9019ebe64e45061b0dd81a79e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 29 May 2024 16:22:32 +0200 Subject: [PATCH 20/48] Copy change --- .../src/components/grid/controls/ColumnsSettingButton.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte index 2f8e141a61..5d398574c3 100644 --- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte +++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte @@ -21,7 +21,7 @@ const getText = columns => { const hidden = columns.filter(col => !col.visible).length - return hidden ? `Columns (${hidden})` : "Columns" + return hidden ? `Columns (${hidden} restricted)` : "Columns" } const PERMISSION_OPTIONS = { From 3d15f71344273796bc13fd68fda52020ba2198d7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 29 May 2024 16:50:07 +0200 Subject: [PATCH 21/48] Break down component --- .../grid/controls/ColumnsSettingButton.svelte | 57 ++++++------------- .../controls/ToggleActionButtonGroup.svelte | 41 +++++++++++++ 2 files changed, 59 insertions(+), 39 deletions(-) create mode 100644 packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte index 5d398574c3..0a85e41966 100644 --- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte +++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte @@ -1,7 +1,8 @@ + +
+ {#each options as option} + dispatch("click", option.value)} + {disabled} + size="S" + icon={option.icon} + quiet + selected={option.value === value} + noPadding + tooltip={option.tooltip} + /> + {/each} +
+ + From 21e5c8ecda3f243718ad8fc9890bdb392f3f2c47 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 29 May 2024 17:19:07 +0200 Subject: [PATCH 22/48] Overflow tooltip --- packages/bbui/src/Popover/Popover.svelte | 5 +++++ .../src/components/grid/controls/ColumnsSettingButton.svelte | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/bbui/src/Popover/Popover.svelte b/packages/bbui/src/Popover/Popover.svelte index aa811afe1e..05d8af1693 100644 --- a/packages/bbui/src/Popover/Popover.svelte +++ b/packages/bbui/src/Popover/Popover.svelte @@ -27,6 +27,7 @@ export let clickOutsideOverride = false export let resizable = true export let wrap = false + export let overflow = false $: target = portalTarget || getContext(Context.PopoverRoot) || ".spectrum" @@ -104,6 +105,7 @@ class="spectrum-Popover is-open" class:customZindex class:hidden={!showPopover} + class:overflow role="presentation" style="height: {customHeight}; --customZindex: {customZindex};" transition:fly|local={{ y: -20, duration: animate ? 260 : 0 }} @@ -122,6 +124,9 @@ overflow: auto; transition: opacity 260ms ease-out; } + .spectrum-Popover.overflow { + overflow: visible; + } .hidden { opacity: 0; pointer-events: none; diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte index 0a85e41966..1c8a8ec6a0 100644 --- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte +++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte @@ -31,7 +31,7 @@ } const options = [ - { icon: "Edit", value: PERMISSION_OPTIONS.WRITABLE, tooltip: "Writable" }, + { icon: "Edit", value: PERMISSION_OPTIONS.WRITABLE, tooltip: "Writeable" }, { icon: "VisibilityOff", value: PERMISSION_OPTIONS.HIDDEN, @@ -61,7 +61,7 @@
- +
{#if $stickyColumn} From fb4cecc93fa9087e60d345deb7f5fa3daa6ad0fc Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 29 May 2024 17:07:29 +0100 Subject: [PATCH 23/48] Update template tests to make sure importing templates works for SQS. --- .../src/api/routes/tests/templates.spec.ts | 55 +++++++++++++++++-- .../src/tests/utilities/TestConfiguration.ts | 10 ++++ 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/routes/tests/templates.spec.ts b/packages/server/src/api/routes/tests/templates.spec.ts index 8a5cfb3537..680ddb39d7 100644 --- a/packages/server/src/api/routes/tests/templates.spec.ts +++ b/packages/server/src/api/routes/tests/templates.spec.ts @@ -1,5 +1,7 @@ import * as setup from "./utilities" +import path from "path" import nock from "nock" +import { generator } from "@budibase/backend-core/tests" interface App { background: string @@ -27,7 +29,7 @@ function setManifest(manifest: Manifest) { function mockApp(key: string, tarPath: string) { nock("https://prod-budi-templates.s3-eu-west-1.amazonaws.com") - .get(`/app/${key}.tar.gz`) + .get(`/templates/app/${key}.tar.gz`) .replyWithFile(200, tarPath) } @@ -54,7 +56,7 @@ function mockAgencyClientPortal() { mockApp( "agency-client-portal", - "packages/server/src/api/routes/tests/data/agency-client-portal.tar.gz" + path.resolve(__dirname, "data", "agency-client-portal.tar.gz") ) } @@ -72,9 +74,52 @@ describe("/templates", () => { describe("fetch", () => { it("should be able to fetch templates", async () => { - const res = await config.api.templates.fetch() - expect(res).toHaveLength(1) - expect(res[0].name).toBe("Agency Client Portal") + const templates = await config.api.templates.fetch() + expect(templates).toHaveLength(1) + expect(templates[0].name).toBe("Agency Client Portal") }) }) + + describe("create app from template", () => { + it.each(["sqs", "lucene"])( + `should be able to create an app from a template (%s)`, + async source => { + const env = { + SQS_SEARCH_ENABLE: source === "sqs" ? "true" : "false", + } + + await config.withEnv(env, async () => { + const name = generator.guid().replaceAll("-", "") + const url = `/${name}` + + const app = await config.api.application.create({ + name, + url, + useTemplate: "true", + templateName: "Agency Client Portal", + templateKey: "app/agency-client-portal", + }) + expect(app.name).toBe(name) + expect(app.url).toBe(url) + + await config.withApp(app, async () => { + const tables = await config.api.table.fetch() + expect(tables).toHaveLength(2) + + tables.sort((a, b) => a.name.localeCompare(b.name)) + const [agencyProjects, users] = tables + expect(agencyProjects.name).toBe("Agency Projects") + expect(users.name).toBe("Users") + + const { rows } = await config.api.row.search(agencyProjects._id!, { + tableId: agencyProjects._id!, + query: {}, + }) + + expect(rows).toHaveLength(3) + }) + }) + } + ) + }) }) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 58dbecd197..325d911f07 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -314,6 +314,16 @@ export default class TestConfiguration { } } + async withApp(app: App | string, f: () => Promise) { + const oldAppId = this.appId + this.appId = typeof app === "string" ? app : app.appId + try { + return await f() + } finally { + this.appId = oldAppId + } + } + // UTILS _req | void, Res>( From 8b22beb0a204baa91fd298cb3d23748d61c9759c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 30 May 2024 10:07:17 +0200 Subject: [PATCH 24/48] Use AbsTooltip --- .../controls/ToggleActionButtonGroup.svelte | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte b/packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte index 4c82da9e21..4f83f069d3 100644 --- a/packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte +++ b/packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte @@ -3,7 +3,7 @@ let dispatch = createEventDispatcher() - import { ActionButton } from "@budibase/bbui" + import { ActionButton, AbsTooltip, TooltipType } from "@budibase/bbui" export let value export let options @@ -12,16 +12,17 @@
{#each options as option} - dispatch("click", option.value)} - {disabled} - size="S" - icon={option.icon} - quiet - selected={option.value === value} - noPadding - tooltip={option.tooltip} - /> + + dispatch("click", option.value)} + {disabled} + size="S" + icon={option.icon} + quiet + selected={option.value === value} + noPadding + /> + {/each}
From a672c0089931e14eda859c00661d338d9791fbeb Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 30 May 2024 10:07:47 +0200 Subject: [PATCH 25/48] Copy changes --- .../src/components/grid/controls/ColumnsSettingButton.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte index 1c8a8ec6a0..7ba746da33 100644 --- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte +++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte @@ -31,7 +31,7 @@ } const options = [ - { icon: "Edit", value: PERMISSION_OPTIONS.WRITABLE, tooltip: "Writeable" }, + { icon: "Edit", value: PERMISSION_OPTIONS.WRITABLE, tooltip: "Writable" }, { icon: "VisibilityOff", value: PERMISSION_OPTIONS.HIDDEN, From b5a036509784576e00555f4a6e25479fc3e60863 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 30 May 2024 10:11:04 +0200 Subject: [PATCH 26/48] Remove new overflow setting --- packages/bbui/src/Popover/Popover.svelte | 5 ----- .../src/components/grid/controls/ColumnsSettingButton.svelte | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/bbui/src/Popover/Popover.svelte b/packages/bbui/src/Popover/Popover.svelte index 05d8af1693..aa811afe1e 100644 --- a/packages/bbui/src/Popover/Popover.svelte +++ b/packages/bbui/src/Popover/Popover.svelte @@ -27,7 +27,6 @@ export let clickOutsideOverride = false export let resizable = true export let wrap = false - export let overflow = false $: target = portalTarget || getContext(Context.PopoverRoot) || ".spectrum" @@ -105,7 +104,6 @@ class="spectrum-Popover is-open" class:customZindex class:hidden={!showPopover} - class:overflow role="presentation" style="height: {customHeight}; --customZindex: {customZindex};" transition:fly|local={{ y: -20, duration: animate ? 260 : 0 }} @@ -124,9 +122,6 @@ overflow: auto; transition: opacity 260ms ease-out; } - .spectrum-Popover.overflow { - overflow: visible; - } .hidden { opacity: 0; pointer-events: none; diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte index 7ba746da33..0a85e41966 100644 --- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte +++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte @@ -61,7 +61,7 @@
- +
{#if $stickyColumn} From adcda035ed9dc736a68b61f6a0809ff44022e344 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 30 May 2024 10:11:11 +0200 Subject: [PATCH 27/48] Change size --- .../components/grid/controls/ToggleActionButtonGroup.svelte | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte b/packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte index 4f83f069d3..e705b5016d 100644 --- a/packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte +++ b/packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte @@ -37,6 +37,7 @@ width: 14px; } .permissionPicker :global(.spectrum-ActionButton) { - width: 22px; + width: 28px; + height: 28px; } From 7b4304d4a2ebace99dbaa01d3c46caedfb7cfb1c Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 30 May 2024 09:11:42 +0100 Subject: [PATCH 28/48] Force wrapping long words in field labels --- packages/client/src/components/app/forms/Field.svelte | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/client/src/components/app/forms/Field.svelte b/packages/client/src/components/app/forms/Field.svelte index 690e328cf4..74ff5442a9 100644 --- a/packages/client/src/components/app/forms/Field.svelte +++ b/packages/client/src/components/app/forms/Field.svelte @@ -126,6 +126,9 @@ display: flex; flex-direction: column; } + label { + word-wrap: break-word; + } label.hidden { padding: 0; } From 19a422ca1ccf7accf621728aeb1995a3dd8b28fc Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 30 May 2024 09:14:56 +0100 Subject: [PATCH 29/48] Typo --- packages/frontend-core/src/components/grid/stores/rows.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js index 5140a7da8b..d354dfd9b3 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.js +++ b/packages/frontend-core/src/components/grid/stores/rows.js @@ -404,7 +404,7 @@ export const createActions = context => { // Save change try { - // Incremenet change count for this row + // Increment change count for this row inProgressChanges.update(state => ({ ...state, [rowId]: (state[rowId] || 0) + 1, From 466a365ac1b0b1bf734ab9097c19e5c8e11efe1a Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 30 May 2024 09:09:35 +0000 Subject: [PATCH 30/48] Bump version to 2.27.5 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 36d217c1bb..335df975af 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.27.4", + "version": "2.27.5", "npmClient": "yarn", "packages": [ "packages/*", From dee797656a10f6be8678c196d793b417b841ed4c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 30 May 2024 14:31:45 +0100 Subject: [PATCH 31/48] Update account-portal submodule to latest master. --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index c167c331ff..a03225549e 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit c167c331ff9b8161fc18e2ecbaaf1ea5815ba964 +Subproject commit a03225549e3ce61f43d0da878da162e08941b939 From 6f02185abe5947dd3e62e2e17b9489c77db31242 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 30 May 2024 14:33:10 +0100 Subject: [PATCH 32/48] Put pro back in line with master. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 1879d8686b..5189b83bea 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 1879d8686b1d9392707595a02cdd4981923e7f99 +Subproject commit 5189b83bea1868574ff7f4c51fe5db38a11badb8 From 25a4e1d99948f19ca1e7f9d274492dca78c25a09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 May 2024 19:53:00 +0000 Subject: [PATCH 33/48] Bump mysql2 from 3.9.7 to 3.9.8 in /packages/server Bumps [mysql2](https://github.com/sidorares/node-mysql2) from 3.9.7 to 3.9.8. - [Release notes](https://github.com/sidorares/node-mysql2/releases) - [Changelog](https://github.com/sidorares/node-mysql2/blob/master/Changelog.md) - [Commits](https://github.com/sidorares/node-mysql2/compare/v3.9.7...v3.9.8) --- updated-dependencies: - dependency-name: mysql2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- packages/server/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/package.json b/packages/server/package.json index e816ad3f18..bd5a82cb29 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -97,7 +97,7 @@ "memorystream": "0.3.1", "mongodb": "^6.3.0", "mssql": "10.0.1", - "mysql2": "3.9.7", + "mysql2": "3.9.8", "node-fetch": "2.6.7", "object-sizeof": "2.6.1", "openai": "^3.2.1", From 1777ac4b046903f2905b93a724eb39ead0e179aa Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 31 May 2024 14:59:15 +0100 Subject: [PATCH 34/48] Fix mariadb healthcheck. --- packages/server/src/integrations/tests/utils/mariadb.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/tests/utils/mariadb.ts b/packages/server/src/integrations/tests/utils/mariadb.ts index fcd79b8e56..c4dd4cf43b 100644 --- a/packages/server/src/integrations/tests/utils/mariadb.ts +++ b/packages/server/src/integrations/tests/utils/mariadb.ts @@ -18,7 +18,7 @@ class MariaDBWaitStrategy extends AbstractWaitStrategy { await logs.waitUntilReady(container, boundPorts, startTime) const command = Wait.forSuccessfulCommand( - `mysqladmin ping -h localhost -P 3306 -u root -ppassword` + `/usr/local/bin/healthcheck.sh --innodb_initialized` ) await command.waitUntilReady(container) } From cbb3c9aa934fb6adee76154606540c7cdd9b7448 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Fri, 31 May 2024 15:34:08 +0100 Subject: [PATCH 35/48] Allow a user invite to be revoked (#13805) * Add free_trial to deploy camunda script * Allow user invites to be deleted * Refactor to pass invite codes * lint * update account-portal * yarn lock * users terminology instead of rows and invites --- .../builder/portal/users/users/index.svelte | 33 ++++++++++--- packages/builder/src/stores/portal/users.js | 5 ++ packages/frontend-core/src/api/user.js | 10 ++++ packages/types/src/api/web/user.ts | 5 ++ .../src/api/controllers/global/users.ts | 16 +++++++ .../worker/src/api/routes/global/users.ts | 5 ++ yarn.lock | 47 +++++++++++++++++++ 7 files changed, 115 insertions(+), 6 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/users/users/index.svelte b/packages/builder/src/pages/builder/portal/users/users/index.svelte index c2fbce5747..58da310104 100644 --- a/packages/builder/src/pages/builder/portal/users/users/index.svelte +++ b/packages/builder/src/pages/builder/portal/users/users/index.svelte @@ -60,6 +60,7 @@ userLimitReachedModal let searchEmail = undefined let selectedRows = [] + let selectedInvites = [] let bulkSaveResponse let customRenderers = [ { column: "email", component: EmailTableRenderer }, @@ -123,7 +124,7 @@ return {} } let pendingSchema = JSON.parse(JSON.stringify(tblSchema)) - pendingSchema.email.displayName = "Pending Invites" + pendingSchema.email.displayName = "Pending Users" return pendingSchema } @@ -132,6 +133,7 @@ const { admin, builder, userGroups, apps } = invite.info return { + _id: invite.code, email: invite.email, builder, admin, @@ -260,9 +262,26 @@ return } - await users.bulkDelete(ids) - notifications.success(`Successfully deleted ${selectedRows.length} rows`) + if (ids.length > 0) { + await users.bulkDelete(ids) + } + + if (selectedInvites.length > 0) { + await users.removeInvites( + selectedInvites.map(invite => ({ + code: invite._id, + })) + ) + pendingInvites = await users.getInvites() + } + + notifications.success( + `Successfully deleted ${ + selectedRows.length + selectedInvites.length + } users` + ) selectedRows = [] + selectedInvites = [] await fetch.refresh() } catch (error) { notifications.error("Error deleting users") @@ -328,15 +347,15 @@
{/if}
- - {#if selectedRows.length > 0} + {#if selectedRows.length > 0 || selectedInvites.length > 0} {/if} +
({ }) }, + /** + * Removes multiple user invites from Redis cache + */ + removeUserInvites: async inviteCodes => { + return await API.post({ + url: "/api/global/users/multi/invite/delete", + body: inviteCodes, + }) + }, + /** * Accepts an invite to join the platform and creates a user. * @param inviteCode the invite code sent in the email diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts index f59bda133b..471ca86616 100644 --- a/packages/types/src/api/web/user.ts +++ b/packages/types/src/api/web/user.ts @@ -45,7 +45,12 @@ export interface InviteUserRequest { userInfo: any } +export interface DeleteInviteUserRequest { + code: string +} + export type InviteUsersRequest = InviteUserRequest[] +export type DeleteInviteUsersRequest = DeleteInviteUserRequest[] export interface InviteUsersResponse { successful: { email: string }[] diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 46bf13284e..cd69281f56 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -10,6 +10,8 @@ import { CreateAdminUserRequest, CreateAdminUserResponse, Ctx, + DeleteInviteUserRequest, + DeleteInviteUsersRequest, InviteUserRequest, InviteUsersRequest, InviteUsersResponse, @@ -335,6 +337,20 @@ export const inviteMultiple = async (ctx: Ctx) => { ctx.body = await userSdk.invite(ctx.request.body) } +export const removeMultipleInvites = async ( + ctx: Ctx +) => { + const inviteCodesToRemove = ctx.request.body.map( + (invite: DeleteInviteUserRequest) => invite.code + ) + for (const code of inviteCodesToRemove) { + await cache.invite.deleteCode(code) + } + ctx.body = { + message: "User invites successfully removed.", + } +} + export const checkInvite = async (ctx: any) => { const { code } = ctx.params let invite diff --git a/packages/worker/src/api/routes/global/users.ts b/packages/worker/src/api/routes/global/users.ts index b40c491830..0372c187f8 100644 --- a/packages/worker/src/api/routes/global/users.ts +++ b/packages/worker/src/api/routes/global/users.ts @@ -108,6 +108,11 @@ router buildInviteMultipleValidation(), controller.inviteMultiple ) + .post( + "/api/global/users/multi/invite/delete", + auth.builderOrAdmin, + controller.removeMultipleInvites + ) // non-global endpoints .get("/api/global/users/invite/:code", controller.checkInvite) diff --git a/yarn.lock b/yarn.lock index 677b7cb441..9daf499918 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11904,6 +11904,17 @@ glob@^10.0.0, glob@^10.2.2: minipass "^7.0.4" path-scurry "^1.10.2" +glob@^10.3.7: + version "10.4.1" + resolved "https://registry.yarnpkg.com/glob/-/glob-10.4.1.tgz#0cfb01ab6a6b438177bfe6a58e2576f6efe909c2" + integrity sha512-2jelhlq3E4ho74ZyVLN03oKdAZVUa6UDZzFLVH1H7dnoax+y9qyaq8zBkfDIggjniU19z0wU18y16jMB2eyVIw== + dependencies: + foreground-child "^3.1.0" + jackspeak "^3.1.2" + minimatch "^9.0.4" + minipass "^7.1.2" + path-scurry "^1.11.1" + glob@^5.0.15: version "5.0.15" resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1" @@ -13472,6 +13483,15 @@ jackspeak@^2.3.6: optionalDependencies: "@pkgjs/parseargs" "^0.11.0" +jackspeak@^3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-3.1.2.tgz#eada67ea949c6b71de50f1b09c92a961897b90ab" + integrity sha512-kWmLKn2tRtfYMF/BakihVVRzBKOxz4gJMiL2Rj91WnAB5TPZumSH99R/Yf1qE1u4uRimvCSJfm6hnxohXeEXjQ== + dependencies: + "@isaacs/cliui" "^8.0.2" + optionalDependencies: + "@pkgjs/parseargs" "^0.11.0" + jake@^10.8.5: version "10.8.5" resolved "https://registry.yarnpkg.com/jake/-/jake-10.8.5.tgz#f2183d2c59382cb274226034543b9c03b8164c46" @@ -15751,6 +15771,13 @@ minimatch@^8.0.2: dependencies: brace-expansion "^2.0.1" +minimatch@^9.0.4: + version "9.0.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.4.tgz#8e49c731d1749cbec05050ee5145147b32496a51" + integrity sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw== + dependencies: + brace-expansion "^2.0.1" + minimist-options@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/minimist-options/-/minimist-options-4.1.0.tgz#c0655713c53a8a2ebd77ffa247d342c40f010619" @@ -15845,6 +15872,11 @@ minipass@^5.0.0: resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.0.4.tgz#dbce03740f50a4786ba994c1fb908844d27b038c" integrity sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ== +minipass@^7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707" + integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw== + minizlib@^2.1.1, minizlib@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931" @@ -17378,6 +17410,14 @@ path-scurry@^1.10.2, path-scurry@^1.6.1: lru-cache "^10.2.0" minipass "^5.0.0 || ^6.0.2 || ^7.0.0" +path-scurry@^1.11.1: + version "1.11.1" + resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.11.1.tgz#7960a668888594a0720b12a911d1a742ab9f11d2" + integrity sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA== + dependencies: + lru-cache "^10.2.0" + minipass "^5.0.0 || ^6.0.2 || ^7.0.0" + path-to-regexp@1.x: version "1.8.0" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.8.0.tgz#887b3ba9d84393e87a0a0b9f4cb756198b53548a" @@ -19318,6 +19358,13 @@ rimraf@^4.4.1: dependencies: glob "^9.2.0" +rimraf@^5.0.7: + version "5.0.7" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-5.0.7.tgz#27bddf202e7d89cb2e0381656380d1734a854a74" + integrity sha512-nV6YcJo5wbLW77m+8KjH8aB/7/rxQy9SZ0HY5shnwULfS+9nmTtVXAJET5NdZmCzA4fPI/Hm1wo/Po/4mopOdg== + dependencies: + glob "^10.3.7" + ripemd160@^2.0.0, ripemd160@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/ripemd160/-/ripemd160-2.0.2.tgz#a1c1a6f624751577ba5d07914cbc92850585890c" From 9912904bd1faf766c4cb7c58877c95634090ad91 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 31 May 2024 16:07:46 +0100 Subject: [PATCH 36/48] Fixing an issue with error cases that have a content-disposition being downloaded as a file. --- packages/server/src/integrations/rest.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index 451c319aa9..86c059bc82 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -149,13 +149,12 @@ class RestIntegration implements IntegrationBase { { downloadImages: this.config.downloadImages } ) let contentLength = response.headers.get("content-length") - if (!contentLength && raw) { - contentLength = Buffer.byteLength(raw, "utf8").toString() - } + let isSuccess = response.status >= 200 && response.status < 300 if ( - contentDisposition.includes("filename") || - contentDisposition.includes("attachment") || - contentDisposition.includes("form-data") + (contentDisposition.includes("filename") || + contentDisposition.includes("attachment") || + contentDisposition.includes("form-data")) && + isSuccess ) { filename = path.basename(parse(contentDisposition).parameters?.filename) || "" @@ -168,6 +167,9 @@ class RestIntegration implements IntegrationBase { return handleFileResponse(response, filename, this.startTimeMs) } else { responseTxt = response.text ? await response.text() : "" + if (!contentLength && responseTxt) { + contentLength = Buffer.byteLength(responseTxt, "utf8").toString() + } const hasContent = (contentLength && parseInt(contentLength) > 0) || responseTxt.length > 0 From 4a1f24d0d8925979a4d6d6322ba34909f7431ffe Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 31 May 2024 16:08:10 +0100 Subject: [PATCH 37/48] Fixing an issue with default parameters not being passed into dynamic parameters. --- packages/server/src/threads/query.ts | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index 54322b1156..1210461bba 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -14,7 +14,13 @@ import { context, cache, auth } from "@budibase/backend-core" import { getGlobalIDFromUserMetadataID } from "../db/utils" import sdk from "../sdk" import { cloneDeep } from "lodash/fp" -import { Datasource, Query, SourceName, Row } from "@budibase/types" +import { + Datasource, + Query, + SourceName, + Row, + QueryParameter, +} from "@budibase/types" import { isSQL } from "../integrations/utils" import { interpolateSQL } from "../integrations/queries/sql" @@ -196,12 +202,22 @@ class QueryRunner { return { rows, keys, info, extra, pagination } } - async runAnotherQuery(queryId: string, parameters: any) { + async runAnotherQuery( + queryId: string, + currentParameters: Record + ) { const db = context.getAppDB() const query = await db.get(queryId) const datasource = await sdk.datasources.get(query.datasourceId, { enriched: true, }) + // enrich parameters with dynamic queries defaults + const defaultParams = query.parameters || [] + for (let param of defaultParams) { + if (!currentParameters[param.name]) { + currentParameters[param.name] = param.default + } + } return new QueryRunner( { schema: query.schema, @@ -210,7 +226,7 @@ class QueryRunner { transformer: query.transformer, nullDefaultSupport: query.nullDefaultSupport, ctx: this.ctx, - parameters, + parameters: currentParameters, datasource, queryId, }, From 6df5315b32b660414dbd0150fe728714a83d1342 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 31 May 2024 16:08:33 +0100 Subject: [PATCH 38/48] Update yarn lock --- yarn.lock | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/yarn.lock b/yarn.lock index 677b7cb441..d85a50e938 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11904,6 +11904,17 @@ glob@^10.0.0, glob@^10.2.2: minipass "^7.0.4" path-scurry "^1.10.2" +glob@^10.3.7: + version "10.4.1" + resolved "https://registry.yarnpkg.com/glob/-/glob-10.4.1.tgz#0cfb01ab6a6b438177bfe6a58e2576f6efe909c2" + integrity sha512-2jelhlq3E4ho74ZyVLN03oKdAZVUa6UDZzFLVH1H7dnoax+y9qyaq8zBkfDIggjniU19z0wU18y16jMB2eyVIw== + dependencies: + foreground-child "^3.1.0" + jackspeak "^3.1.2" + minimatch "^9.0.4" + minipass "^7.1.2" + path-scurry "^1.11.1" + glob@^5.0.15: version "5.0.15" resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1" @@ -13472,6 +13483,15 @@ jackspeak@^2.3.6: optionalDependencies: "@pkgjs/parseargs" "^0.11.0" +jackspeak@^3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-3.1.2.tgz#eada67ea949c6b71de50f1b09c92a961897b90ab" + integrity sha512-kWmLKn2tRtfYMF/BakihVVRzBKOxz4gJMiL2Rj91WnAB5TPZumSH99R/Yf1qE1u4uRimvCSJfm6hnxohXeEXjQ== + dependencies: + "@isaacs/cliui" "^8.0.2" + optionalDependencies: + "@pkgjs/parseargs" "^0.11.0" + jake@^10.8.5: version "10.8.5" resolved "https://registry.yarnpkg.com/jake/-/jake-10.8.5.tgz#f2183d2c59382cb274226034543b9c03b8164c46" @@ -15751,6 +15771,13 @@ minimatch@^8.0.2: dependencies: brace-expansion "^2.0.1" +minimatch@^9.0.4: + version "9.0.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.4.tgz#8e49c731d1749cbec05050ee5145147b32496a51" + integrity sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw== + dependencies: + brace-expansion "^2.0.1" + minimist-options@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/minimist-options/-/minimist-options-4.1.0.tgz#c0655713c53a8a2ebd77ffa247d342c40f010619" @@ -15845,6 +15872,11 @@ minipass@^5.0.0: resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.0.4.tgz#dbce03740f50a4786ba994c1fb908844d27b038c" integrity sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ== +minipass@^7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707" + integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw== + minizlib@^2.1.1, minizlib@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931" @@ -16033,10 +16065,10 @@ mute-stream@~1.0.0: resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-1.0.0.tgz#e31bd9fe62f0aed23520aa4324ea6671531e013e" integrity sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA== -mysql2@3.9.7: - version "3.9.7" - resolved "https://registry.yarnpkg.com/mysql2/-/mysql2-3.9.7.tgz#843755daf65b5ef08afe545fe14b8fb62824741a" - integrity sha512-KnJT8vYRcNAZv73uf9zpXqNbvBG7DJrs+1nACsjZP1HMJ1TgXEy8wnNilXAn/5i57JizXKtrUtwDB7HxT9DDpw== +mysql2@3.9.8: + version "3.9.8" + resolved "https://registry.yarnpkg.com/mysql2/-/mysql2-3.9.8.tgz#fe8a0f975f2c495ed76ca988ddc5505801dc49ce" + integrity sha512-+5JKNjPuks1FNMoy9TYpl77f+5frbTklz7eb3XDwbpsERRLEeXiW2PDEkakYF50UuKU2qwfGnyXpKYvukv8mGA== dependencies: denque "^2.1.0" generate-function "^2.3.1" @@ -17378,6 +17410,14 @@ path-scurry@^1.10.2, path-scurry@^1.6.1: lru-cache "^10.2.0" minipass "^5.0.0 || ^6.0.2 || ^7.0.0" +path-scurry@^1.11.1: + version "1.11.1" + resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.11.1.tgz#7960a668888594a0720b12a911d1a742ab9f11d2" + integrity sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA== + dependencies: + lru-cache "^10.2.0" + minipass "^5.0.0 || ^6.0.2 || ^7.0.0" + path-to-regexp@1.x: version "1.8.0" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.8.0.tgz#887b3ba9d84393e87a0a0b9f4cb756198b53548a" @@ -19318,6 +19358,13 @@ rimraf@^4.4.1: dependencies: glob "^9.2.0" +rimraf@^5.0.7: + version "5.0.7" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-5.0.7.tgz#27bddf202e7d89cb2e0381656380d1734a854a74" + integrity sha512-nV6YcJo5wbLW77m+8KjH8aB/7/rxQy9SZ0HY5shnwULfS+9nmTtVXAJET5NdZmCzA4fPI/Hm1wo/Po/4mopOdg== + dependencies: + glob "^10.3.7" + ripemd160@^2.0.0, ripemd160@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/ripemd160/-/ripemd160-2.0.2.tgz#a1c1a6f624751577ba5d07914cbc92850585890c" From 75501c225198a43ffceae4f8eb8df44faf9c2532 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 31 May 2024 17:57:31 +0100 Subject: [PATCH 39/48] Updating object store stream upload to make sure the stream has finished being processed before trying to upload to AWS (and only uploading a partial stream). --- .../src/objectStore/objectStore.ts | 35 +++++++++++++++---- .../types/src/documents/app/automation.ts | 2 +- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 0ac2c35179..35748d8f76 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -41,10 +41,7 @@ type UploadParams = BaseUploadParams & { path?: string | PathLike } -export type StreamTypes = - | ReadStream - | NodeJS.ReadableStream - | ReadableStream +export type StreamTypes = ReadStream | NodeJS.ReadableStream export type StreamUploadParams = BaseUploadParams & { stream?: StreamTypes @@ -222,6 +219,9 @@ export async function streamUpload({ extra, ttl, }: StreamUploadParams) { + if (!stream) { + throw new Error("Stream to upload is invalid/undefined") + } const extension = filename.split(".").pop() const objectStore = ObjectStore(bucketName) const bucketCreated = await createBucketIfNotExists(objectStore, bucketName) @@ -251,14 +251,35 @@ export async function streamUpload({ : CONTENT_TYPE_MAP.txt } + const bucket = sanitizeBucket(bucketName), + objKey = sanitizeKey(filename) const params = { - Bucket: sanitizeBucket(bucketName), - Key: sanitizeKey(filename), + Bucket: bucket, + Key: objKey, Body: stream, ContentType: contentType, ...extra, } - return objectStore.upload(params).promise() + + // make sure we have the stream before we try to push it to object store + if (stream.on) { + await new Promise((resolve, reject) => { + stream.on("finish", resolve) + stream.on("error", reject) + }) + } + + const details = await objectStore.upload(params).promise() + const headDetails = await objectStore + .headObject({ + Bucket: bucket, + Key: objKey, + }) + .promise() + return { + ...details, + ContentLength: headDetails.ContentLength, + } } /** diff --git a/packages/types/src/documents/app/automation.ts b/packages/types/src/documents/app/automation.ts index 6d1753dc28..63291fa3bb 100644 --- a/packages/types/src/documents/app/automation.ts +++ b/packages/types/src/documents/app/automation.ts @@ -245,7 +245,7 @@ export type AutomationAttachment = { export type AutomationAttachmentContent = { filename: string - content: ReadStream | NodeJS.ReadableStream | ReadableStream + content: ReadStream | NodeJS.ReadableStream } export type BucketedContent = AutomationAttachmentContent & { From d90763dd3c88008454a0cbe0a51175cfa454af0d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 31 May 2024 17:59:16 +0100 Subject: [PATCH 40/48] Getting size parameter right for streams. --- packages/server/src/integrations/utils/utils.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts index 9f04457d7a..be689fbfd3 100644 --- a/packages/server/src/integrations/utils/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -368,13 +368,16 @@ export async function handleFileResponse( size = parseInt(contentLength, 10) } - await objectStore.streamUpload({ + const details = await objectStore.streamUpload({ bucket, filename: key, stream, ttl: 1, type: response.headers["content-type"], }) + if (!size && details.ContentLength) { + size = details.ContentLength + } } presignedUrl = objectStore.getPresignedUrl(bucket, key) return { From 26a0801b755ff704c62d6a7ac1755e8bd8b8ad5a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 3 Jun 2024 10:15:16 +0100 Subject: [PATCH 41/48] Linting. --- packages/server/src/threads/query.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index 1210461bba..d76714661d 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -19,7 +19,6 @@ import { Query, SourceName, Row, - QueryParameter, } from "@budibase/types" import { isSQL } from "../integrations/utils" From 155de99b68c8fc481cc4d0ac37eba751595cb22f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 3 Jun 2024 11:46:20 +0100 Subject: [PATCH 42/48] Streaming to disk before passing onto S3. --- .../src/objectStore/objectStore.ts | 9 +-- packages/server/package.json | 5 +- .../src/integrations/tests/rest.spec.ts | 2 + .../server/src/integrations/utils/utils.ts | 70 +++++++++++-------- yarn.lock | 12 +++- 5 files changed, 56 insertions(+), 42 deletions(-) diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 35748d8f76..de94e3968b 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -14,6 +14,7 @@ import { v4 } from "uuid" import { APP_PREFIX, APP_DEV_PREFIX } from "../db" import fsp from "fs/promises" import { HeadObjectOutput } from "aws-sdk/clients/s3" +import { ReadableStream } from "stream/web" const streamPipeline = promisify(stream.pipeline) // use this as a temporary store of buckets that are being created @@ -261,14 +262,6 @@ export async function streamUpload({ ...extra, } - // make sure we have the stream before we try to push it to object store - if (stream.on) { - await new Promise((resolve, reject) => { - stream.on("finish", resolve) - stream.on("error", reject) - }) - } - const details = await objectStore.upload(params).promise() const headDetails = await objectStore .headObject({ diff --git a/packages/server/package.json b/packages/server/package.json index bd5a82cb29..b3beac7ffb 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -68,7 +68,6 @@ "aws-sdk": "2.1030.0", "bcrypt": "5.1.0", "bcryptjs": "2.4.3", - "bl": "^6.0.12", "bull": "4.10.1", "chokidar": "3.5.3", "content-disposition": "^0.5.4", @@ -116,7 +115,8 @@ "uuid": "^8.3.2", "validate.js": "0.13.1", "worker-farm": "1.7.0", - "xml2js": "0.5.0" + "xml2js": "0.5.0", + "tmp": "0.2.3" }, "devDependencies": { "@babel/preset-env": "7.16.11", @@ -137,6 +137,7 @@ "@types/supertest": "2.0.14", "@types/tar": "6.1.5", "@types/uuid": "8.3.4", + "@types/tmp": "0.2.6", "copyfiles": "2.4.1", "docker-compose": "0.23.17", "jest": "29.7.0", diff --git a/packages/server/src/integrations/tests/rest.spec.ts b/packages/server/src/integrations/tests/rest.spec.ts index f20f369c25..dee17a5497 100644 --- a/packages/server/src/integrations/tests/rest.spec.ts +++ b/packages/server/src/integrations/tests/rest.spec.ts @@ -657,6 +657,7 @@ describe("REST Integration", () => { mockReadable.push(null) ;(fetch as unknown as jest.Mock).mockImplementationOnce(() => Promise.resolve({ + status: 200, headers: { raw: () => ({ "content-type": [contentType], @@ -700,6 +701,7 @@ describe("REST Integration", () => { mockReadable.push(null) ;(fetch as unknown as jest.Mock).mockImplementationOnce(() => Promise.resolve({ + status: 200, headers: { raw: () => ({ "content-type": [contentType], diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts index be689fbfd3..157bdba3bd 100644 --- a/packages/server/src/integrations/utils/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -9,10 +9,12 @@ import { context, objectStore, sql } from "@budibase/backend-core" import { v4 } from "uuid" import { parseStringPromise as xmlParser } from "xml2js" import { formatBytes } from "../../utilities" -import bl from "bl" import env from "../../environment" import { InvalidColumns } from "../../constants" import { helpers, utils } from "@budibase/shared-core" +import { pipeline } from "stream/promises" +import tmp from "tmp" +import fs from "fs" type PrimitiveTypes = | FieldType.STRING @@ -360,38 +362,44 @@ export async function handleFileResponse( const key = `${context.getProdAppId()}/${processedFileName}` const bucket = objectStore.ObjectStoreBuckets.TEMP - const stream = response.body.pipe(bl((error, data) => data)) + // put the response stream to disk temporarily as a buffer + const tmpObj = tmp.fileSync() + try { + await pipeline(response.body, fs.createWriteStream(tmpObj.name)) + if (response.body) { + const contentLength = response.headers.get("content-length") + if (contentLength) { + size = parseInt(contentLength, 10) + } - if (response.body) { - const contentLength = response.headers.get("content-length") - if (contentLength) { - size = parseInt(contentLength, 10) + const details = await objectStore.streamUpload({ + bucket, + filename: key, + stream: fs.createReadStream(tmpObj.name), + ttl: 1, + type: response.headers["content-type"], + }) + if (!size && details.ContentLength) { + size = details.ContentLength + } } - - const details = await objectStore.streamUpload({ - bucket, - filename: key, - stream, - ttl: 1, - type: response.headers["content-type"], - }) - if (!size && details.ContentLength) { - size = details.ContentLength + presignedUrl = objectStore.getPresignedUrl(bucket, key) + return { + data: { + size, + name: processedFileName, + url: presignedUrl, + extension: fileExtension, + key: key, + }, + info: { + code: response.status, + size: formatBytes(size.toString()), + time: `${Math.round(performance.now() - startTime)}ms`, + }, } - } - presignedUrl = objectStore.getPresignedUrl(bucket, key) - return { - data: { - size, - name: processedFileName, - url: presignedUrl, - extension: fileExtension, - key: key, - }, - info: { - code: response.status, - size: formatBytes(size.toString()), - time: `${Math.round(performance.now() - startTime)}ms`, - }, + } finally { + // cleanup tmp + tmpObj.removeCallback() } } diff --git a/yarn.lock b/yarn.lock index d85a50e938..5297fe0cad 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6348,6 +6348,11 @@ dependencies: "@types/estree" "*" +"@types/tmp@0.2.6": + version "0.2.6" + resolved "https://registry.yarnpkg.com/@types/tmp/-/tmp-0.2.6.tgz#d785ee90c52d7cc020e249c948c36f7b32d1e217" + integrity sha512-chhaNf2oKHlRkDGt+tiKE2Z5aJ6qalm7Z9rlLdBwmOiAAf09YQvvoLXjWK4HWPF1xU/fqvMgfNfpVoBscA/tKA== + "@types/tough-cookie@*", "@types/tough-cookie@^4.0.2": version "4.0.2" resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz#6286b4c7228d58ab7866d19716f3696e03a09397" @@ -7700,7 +7705,7 @@ bl@^4.0.3, bl@^4.1.0: inherits "^2.0.4" readable-stream "^3.4.0" -bl@^6.0.12, bl@^6.0.3: +bl@^6.0.3: version "6.0.12" resolved "https://registry.yarnpkg.com/bl/-/bl-6.0.12.tgz#77c35b96e13aeff028496c798b75389ddee9c7f8" integrity sha512-EnEYHilP93oaOa2MnmNEjAcovPS3JlQZOyzGXi3EyEpPhm9qWvdDp7BmAVEVusGzp8LlwQK56Av+OkDoRjzE0w== @@ -21283,6 +21288,11 @@ tlhunter-sorted-set@^0.1.0: resolved "https://registry.yarnpkg.com/tlhunter-sorted-set/-/tlhunter-sorted-set-0.1.0.tgz#1c3eae28c0fa4dff97e9501d2e3c204b86406f4b" integrity sha512-eGYW4bjf1DtrHzUYxYfAcSytpOkA44zsr7G2n3PV7yOUR23vmkGe3LL4R+1jL9OsXtbsFOwe8XtbCrabeaEFnw== +tmp@0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.3.tgz#eb783cc22bc1e8bebd0671476d46ea4eb32a79ae" + integrity sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w== + tmp@^0.0.33: version "0.0.33" resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9" From 38ff7debb46dc32f8d7f1b8d3ce073d5ad893594 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 3 Jun 2024 12:08:54 +0100 Subject: [PATCH 43/48] Linting. --- packages/server/src/threads/query.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index d76714661d..ba451a3325 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -14,12 +14,7 @@ import { context, cache, auth } from "@budibase/backend-core" import { getGlobalIDFromUserMetadataID } from "../db/utils" import sdk from "../sdk" import { cloneDeep } from "lodash/fp" -import { - Datasource, - Query, - SourceName, - Row, -} from "@budibase/types" +import { Datasource, Query, SourceName, Row } from "@budibase/types" import { isSQL } from "../integrations/utils" import { interpolateSQL } from "../integrations/queries/sql" From 64a5accffd13661464197e07bb234eb1631326f7 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 3 Jun 2024 13:02:24 +0000 Subject: [PATCH 44/48] Bump version to 2.27.6 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 335df975af..d90f7732a2 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.27.5", + "version": "2.27.6", "npmClient": "yarn", "packages": [ "packages/*", From 3909bbcfc00cda2173ef10eff463f7e78a67ad83 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 3 Jun 2024 15:05:18 +0100 Subject: [PATCH 45/48] NGINX headers for security audit --- hosting/proxy/nginx.prod.conf | 1 + packages/account-portal | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf index 79007da311..217106b1bf 100644 --- a/hosting/proxy/nginx.prod.conf +++ b/hosting/proxy/nginx.prod.conf @@ -74,6 +74,7 @@ http { add_header X-Content-Type-Options nosniff always; add_header X-XSS-Protection "1; mode=block" always; add_header Content-Security-Policy "${csp_default}; ${csp_script}; ${csp_style}; ${csp_object}; ${csp_base_uri}; ${csp_connect}; ${csp_font}; ${csp_frame}; ${csp_img}; ${csp_manifest}; ${csp_media}; ${csp_worker};" always; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always; # upstreams set $apps ${APPS_UPSTREAM_URL}; diff --git a/packages/account-portal b/packages/account-portal index c167c331ff..2a5022fb94 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit c167c331ff9b8161fc18e2ecbaaf1ea5815ba964 +Subproject commit 2a5022fb946481c9f7a9c38d1413922729972be0 From 020477f1f69abf9e0c265e4f452cfbf382b35e5f Mon Sep 17 00:00:00 2001 From: Christos Alexiou Date: Mon, 3 Jun 2024 17:15:51 +0300 Subject: [PATCH 46/48] qa-arc-runner-set -> ubuntu-latest --- .github/workflows/force-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/force-release.yml b/.github/workflows/force-release.yml index 8a9d444f51..3d96d51484 100644 --- a/.github/workflows/force-release.yml +++ b/.github/workflows/force-release.yml @@ -9,7 +9,7 @@ on: jobs: ensure-is-master-tag: name: Ensure is a master tag - runs-on: qa-arc-runner-set + runs-on: ubuntu-latest steps: - name: Checkout monorepo uses: actions/checkout@v4 From 20c18259a95cf9303ece127c76456aef7011c10f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 3 Jun 2024 17:38:22 +0100 Subject: [PATCH 47/48] Update CouchDB chart from 4.3.0 to 4.5.3. --- charts/budibase/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/budibase/Chart.yaml b/charts/budibase/Chart.yaml index e2c9378f2c..83a72d203f 100644 --- a/charts/budibase/Chart.yaml +++ b/charts/budibase/Chart.yaml @@ -17,6 +17,6 @@ version: 0.0.0 appVersion: 0.0.0 dependencies: - name: couchdb - version: 4.3.0 + version: 4.5.3 repository: https://apache.github.io/couchdb-helm condition: services.couchdb.enabled From 63e7421dd56cbfe18ba304628af4394e899328e8 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 4 Jun 2024 12:41:07 +0100 Subject: [PATCH 48/48] acct portal --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 2a5022fb94..e8136bd1ea 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 2a5022fb946481c9f7a9c38d1413922729972be0 +Subproject commit e8136bd1ea9fa4c61a4bcbeda482abea0b6c3d9f