From ff5e9a468d88b60aab47b05376c9d64e8f0124b1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 14 Sep 2023 12:44:14 +0100 Subject: [PATCH 01/66] Add an error if trying to add new users to the user metadata table within an app - this is invalid and should throw an error. --- packages/server/src/api/controllers/row/index.ts | 5 +++++ packages/server/src/api/controllers/row/utils.ts | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index f0f2462019..6e0a6d979e 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -72,6 +72,11 @@ export const save = async (ctx: UserCtx) => { const tableId = utils.getTableId(ctx) const body = ctx.request.body + // user metadata doesn't exist yet - don't allow creation + if (utils.isUserMetadataTable(tableId) && !body._rev) { + ctx.throw(400, "Cannot create new user entry.") + } + // if it has an ID already then its a patch if (body && body._id) { return patch(ctx as UserCtx) diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index 192ba2109c..5f10fd9ad4 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -175,3 +175,7 @@ export function removeEmptyFilters(filters: SearchFilters) { } return filters } + +export function isUserMetadataTable(tableId: string) { + return tableId === InternalTables.USER_METADATA +} From 217d10f5fb81f7e247ed13ba381bef559eebcfd6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 14 Sep 2023 14:00:49 +0100 Subject: [PATCH 02/66] Fix for more than/less than ranges, zeros were ignored when building up ranges, so that it simply acted like an upper limit, rather than a range. --- packages/server/src/api/controllers/row/utils.ts | 4 ++++ packages/server/src/integrations/base/sql.ts | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index cc27f4c2a3..92db29d303 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -147,6 +147,10 @@ export async function validate({ return { valid: Object.keys(errors).length === 0, errors } } +export function isValidFilter(value: any) { + return value != null && value !== "" +} + // don't do a pure falsy check, as 0 is included // https://github.com/Budibase/budibase/issues/10118 export function removeEmptyFilters(filters: SearchFilters) { diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index bf19ec9afe..3cdded69b4 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -11,6 +11,7 @@ import { QueryOptions } from "../../definitions/datasource" import { isIsoDateString, SqlClient } from "../utils" import SqlTableQueryBuilder from "./sqlTable" import environment from "../../environment" +import { isValidFilter } from "../../api/controllers/row/utils" const envLimit = environment.SQL_MAX_ROWS ? parseInt(environment.SQL_MAX_ROWS) @@ -261,15 +262,15 @@ class InternalBuilder { if (isEmptyObject(value.high)) { value.high = "" } - if (value.low && value.high) { + if (isValidFilter(value.low) && isValidFilter(value.high)) { // Use a between operator if we have 2 valid range values const fnc = allOr ? "orWhereBetween" : "whereBetween" query = query[fnc](key, [value.low, value.high]) - } else if (value.low) { + } else if (isValidFilter(value.low)) { // Use just a single greater than operator if we only have a low const fnc = allOr ? "orWhere" : "where" query = query[fnc](key, ">", value.low) - } else if (value.high) { + } else if (isValidFilter(value.high)) { // Use just a single less than operator if we only have a high const fnc = allOr ? "orWhere" : "where" query = query[fnc](key, "<", value.high) From c466f35a98a3e92ca35afd2d9037147d63dac58c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 13 Sep 2023 15:23:30 +0100 Subject: [PATCH 03/66] Found some discussion of testcontainers being problematic when nearly out of disk space, we have seen issues with the default Github runners as they have extremely limited disk space, this should help a bit removing android and dotnet, two pieces of functionality we will never need. --- .github/workflows/budibase_ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index d670e222d3..fc35575ec6 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -25,6 +25,13 @@ jobs: lint: runs-on: ubuntu-latest steps: + - name: Maximize build space + uses: easimon/maximize-build-space@master + with: + root-reserve-mb: 35000 + swap-size-mb: 1024 + remove-android: 'true' + remove-dotnet: 'true' - name: Checkout repo and submodules uses: actions/checkout@v3 if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' From 8ca3f13a1cea7171d4e6a51b547bebdf28b65d19 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 14 Sep 2023 16:53:36 +0100 Subject: [PATCH 04/66] Quick re-jig based on test failure, seems the base sql.ts is depended on fairly heavily, importing the SDK can create a lot of cycles. --- .../src/api/controllers/row/external.ts | 3 +- .../server/src/api/controllers/row/utils.ts | 33 --------------- packages/server/src/integrations/base/sql.ts | 18 ++++---- packages/server/src/integrations/utils.ts | 42 ++++++++++++++++++- packages/server/src/sdk/app/rows/search.ts | 1 + 5 files changed, 52 insertions(+), 45 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index a04584e6bd..6cc6337e0d 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -18,7 +18,6 @@ import { import sdk from "../../../sdk" import * as utils from "./utils" import { dataFilters } from "@budibase/shared-core" -import { removeEmptyFilters } from "./utils" export async function handleRequest( operation: Operation, @@ -27,7 +26,7 @@ export async function handleRequest( ) { // make sure the filters are cleaned up, no empty strings for equals, fuzzy or string if (opts && opts.filters) { - opts.filters = utils.removeEmptyFilters(opts.filters) + opts.filters = sdk.rows.removeEmptyFilters(opts.filters) } if ( !dataFilters.hasFilters(opts?.filters) && diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index 92db29d303..82e7c7b0d8 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -146,36 +146,3 @@ export async function validate({ } return { valid: Object.keys(errors).length === 0, errors } } - -export function isValidFilter(value: any) { - return value != null && value !== "" -} - -// don't do a pure falsy check, as 0 is included -// https://github.com/Budibase/budibase/issues/10118 -export function removeEmptyFilters(filters: SearchFilters) { - for (let filterField of NoEmptyFilterStrings) { - if (!filters[filterField]) { - continue - } - - for (let filterType of Object.keys(filters)) { - if (filterType !== filterField) { - continue - } - // don't know which one we're checking, type could be anything - const value = filters[filterType] as unknown - if (typeof value === "object") { - for (let [key, value] of Object.entries( - filters[filterType] as object - )) { - if (value == null || value === "") { - // @ts-ignore - delete filters[filterField][key] - } - } - } - } - } - return filters -} diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 3cdded69b4..add7596165 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -1,4 +1,8 @@ import { Knex, knex } from "knex" +import { db as dbCore } from "@budibase/backend-core" +import { QueryOptions } from "../../definitions/datasource" +import { isIsoDateString, SqlClient } from "../utils" +import SqlTableQueryBuilder from "./sqlTable" import { Operation, QueryJson, @@ -6,12 +10,8 @@ import { SearchFilters, SortDirection, } from "@budibase/types" -import { db as dbCore } from "@budibase/backend-core" -import { QueryOptions } from "../../definitions/datasource" -import { isIsoDateString, SqlClient } from "../utils" -import SqlTableQueryBuilder from "./sqlTable" import environment from "../../environment" -import { isValidFilter } from "../../api/controllers/row/utils" +import { isValidFilter } from "../utils" const envLimit = environment.SQL_MAX_ROWS ? parseInt(environment.SQL_MAX_ROWS) @@ -262,15 +262,17 @@ class InternalBuilder { if (isEmptyObject(value.high)) { value.high = "" } - if (isValidFilter(value.low) && isValidFilter(value.high)) { + const lowValid = isValidFilter(value.low), + highValid = isValidFilter(value.high) + if (lowValid && highValid) { // Use a between operator if we have 2 valid range values const fnc = allOr ? "orWhereBetween" : "whereBetween" query = query[fnc](key, [value.low, value.high]) - } else if (isValidFilter(value.low)) { + } else if (lowValid) { // Use just a single greater than operator if we only have a low const fnc = allOr ? "orWhere" : "where" query = query[fnc](key, ">", value.low) - } else if (isValidFilter(value.high)) { + } else if (highValid) { // Use just a single less than operator if we only have a high const fnc = allOr ? "orWhere" : "where" query = query[fnc](key, "<", value.high) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 2883e4471c..75f4bcbfa1 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -1,6 +1,11 @@ -import { SourceName, SqlQuery, Datasource, Table } from "@budibase/types" +import { SqlQuery, Table, SearchFilters } from "@budibase/types" import { DocumentType, SEPARATOR } from "../db/utils" -import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants" +import { + FieldTypes, + BuildSchemaErrors, + InvalidColumns, + NoEmptyFilterStrings, +} from "../constants" import { helpers } from "@budibase/shared-core" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` @@ -343,3 +348,36 @@ export function getPrimaryDisplay(testValue: unknown): string | undefined { } return testValue as string } + +export function isValidFilter(value: any) { + return value != null && value !== "" +} + +// don't do a pure falsy check, as 0 is included +// https://github.com/Budibase/budibase/issues/10118 +export function removeEmptyFilters(filters: SearchFilters) { + for (let filterField of NoEmptyFilterStrings) { + if (!filters[filterField]) { + continue + } + + for (let filterType of Object.keys(filters)) { + if (filterType !== filterField) { + continue + } + // don't know which one we're checking, type could be anything + const value = filters[filterType] as unknown + if (typeof value === "object") { + for (let [key, value] of Object.entries( + filters[filterType] as object + )) { + if (value == null || value === "") { + // @ts-ignore + delete filters[filterField][key] + } + } + } + } + } + return filters +} diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 4861f473ea..1d06108b67 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -3,6 +3,7 @@ import { isExternalTable } from "../../../integrations/utils" import * as internal from "./search/internal" import * as external from "./search/external" import { Format } from "../../../api/controllers/view/exporters" +export { isValidFilter, removeEmptyFilters } from "../../../integrations/utils" export interface ViewParams { calculation: string From 68f31975223a4c1a187f0281161baf4b49491980 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 14 Sep 2023 17:12:09 +0100 Subject: [PATCH 05/66] Moving test to where the functions are now. --- .../tests/utils.spec.ts => sdk/tests/rows/search.spec.ts} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename packages/server/src/{api/controllers/row/tests/utils.spec.ts => sdk/tests/rows/search.spec.ts} (71%) diff --git a/packages/server/src/api/controllers/row/tests/utils.spec.ts b/packages/server/src/sdk/tests/rows/search.spec.ts similarity index 71% rename from packages/server/src/api/controllers/row/tests/utils.spec.ts rename to packages/server/src/sdk/tests/rows/search.spec.ts index e0ad637e9d..feae5e7ee8 100644 --- a/packages/server/src/api/controllers/row/tests/utils.spec.ts +++ b/packages/server/src/sdk/tests/rows/search.spec.ts @@ -1,8 +1,8 @@ -import * as utils from "../utils" +import * as search from "../../app/rows/search" describe("removeEmptyFilters", () => { it("0 should not be removed", () => { - const filters = utils.removeEmptyFilters({ + const filters = search.removeEmptyFilters({ equal: { column: 0, }, @@ -11,7 +11,7 @@ describe("removeEmptyFilters", () => { }) it("empty string should be removed", () => { - const filters = utils.removeEmptyFilters({ + const filters = search.removeEmptyFilters({ equal: { column: "", }, From fb30be6ce4ef3652e762deda1e0631a8031c2ccd Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 14 Sep 2023 17:02:50 +0000 Subject: [PATCH 06/66] Bump version to 2.10.9 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 30aa2afa3c..77df84de36 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.10.8", + "version": "2.10.9", "npmClient": "yarn", "packages": [ "packages/*" From 7f43db09a638c47cbfbdc93b9363c72595866146 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Fri, 15 Sep 2023 10:45:15 +0100 Subject: [PATCH 07/66] Budi 7481 initial form step binding drawer can crash (#11751) * Type safety * Clear localFiles is fieldState is cleared --- .../src/components/common/CodeEditor/CodeEditor.svelte | 2 +- packages/client/src/components/app/forms/S3Upload.svelte | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/common/CodeEditor/CodeEditor.svelte b/packages/builder/src/components/common/CodeEditor/CodeEditor.svelte index 7ab7c5dddf..76d7a58ef1 100644 --- a/packages/builder/src/components/common/CodeEditor/CodeEditor.svelte +++ b/packages/builder/src/components/common/CodeEditor/CodeEditor.svelte @@ -235,7 +235,7 @@ const baseExtensions = buildBaseExtensions() editor = new EditorView({ - doc: value, + doc: value?.toString(), extensions: buildExtensions(baseExtensions), parent: textarea, }) diff --git a/packages/client/src/components/app/forms/S3Upload.svelte b/packages/client/src/components/app/forms/S3Upload.svelte index dfc5032de9..9985c83bb8 100644 --- a/packages/client/src/components/app/forms/S3Upload.svelte +++ b/packages/client/src/components/app/forms/S3Upload.svelte @@ -17,6 +17,13 @@ let fieldApi let localFiles = [] + $: { + // If the field state is reset, clear the local files + if (!fieldState?.value?.length) { + localFiles = [] + } + } + const { API, notificationStore, uploadStore } = getContext("sdk") const component = getContext("component") From 539fdd1d6d4bd7d3ceb44a9115971aba409036bf Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 15 Sep 2023 09:45:38 +0000 Subject: [PATCH 08/66] Bump version to 2.10.10 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 77df84de36..021c74d07c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.10.9", + "version": "2.10.10", "npmClient": "yarn", "packages": [ "packages/*" From 36acd69fddaa2b4dcc8ed9db697c6e0477033135 Mon Sep 17 00:00:00 2001 From: Gerard Burns Date: Mon, 18 Sep 2023 12:14:12 +0100 Subject: [PATCH 09/66] Note about MySQL SSL (#11797) --- packages/server/src/integrations/mysql.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index b88162c59a..8a688c5f3b 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -158,6 +158,12 @@ class MySQLIntegration extends Sql implements DatasourcePlus { ) { config.ssl.rejectUnauthorized = config.rejectUnauthorized } + // The MySQL library we use doesn't directly document the parameters that can be passed in the ssl + // object, it instead points to an older library that it says it is mostly API compatible with, that + // older library actually documents what parameters can be passed in the ssl object. + // https://github.com/sidorares/node-mysql2#api-and-configuration + // https://github.com/mysqljs/mysql#ssl-options + // @ts-ignore delete config.rejectUnauthorized this.config = { From fd518548fddb17c1e28c3f5eddf07b3f5f2a3fea Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 18 Sep 2023 11:14:32 +0000 Subject: [PATCH 10/66] Bump version to 2.10.9-alpha.3 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 1673559ee9..a57875986a 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.10.9-alpha.2", + "version": "2.10.9-alpha.3", "npmClient": "yarn", "packages": [ "packages/*" From 3f7b615d10b439b1a269f0949c0658b081f21992 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 18 Sep 2023 17:58:07 +0200 Subject: [PATCH 11/66] Populate apps from groups while fetching apps --- .../src/sdk/app/applications/applications.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/server/src/sdk/app/applications/applications.ts b/packages/server/src/sdk/app/applications/applications.ts index 865b277504..2604b59fa7 100644 --- a/packages/server/src/sdk/app/applications/applications.ts +++ b/packages/server/src/sdk/app/applications/applications.ts @@ -1,13 +1,14 @@ import { AppStatus } from "../../../db/utils" -import { App, ContextUser } from "@budibase/types" +import { App, ContextUser, User } from "@budibase/types" import { getLocksById } from "../../../utilities/redis" import { enrichApps } from "../../users/sessions" import { checkAppMetadata } from "../../../automations/logging" import { db as dbCore, users } from "@budibase/backend-core" +import { groups } from "@budibase/pro" -export function filterAppList(user: ContextUser, apps: App[]) { +export function filterAppList(user: User, apps: App[]) { let appList: string[] = [] - const roleApps = Object.keys(user.roles || {}) + const roleApps = Object.keys(user.roles) if (users.hasAppBuilderPermissions(user)) { appList = user.builder?.apps || [] appList = appList.concat(roleApps) @@ -23,7 +24,12 @@ export async function fetch(status: AppStatus, user: ContextUser) { const dev = status === AppStatus.DEV const all = status === AppStatus.ALL let apps = (await dbCore.getAllApps({ dev, all })) as App[] - apps = filterAppList(user, apps) + + const enrichedUser = await groups.enrichUserRolesFromGroups({ + ...user, + roles: user.roles || {}, + }) + apps = filterAppList(enrichedUser, apps) const appIds = apps .filter((app: any) => app.status === "development") From bd197bee9ebb36b34f51b7df7a7a6ddbacd4078b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 18 Sep 2023 18:06:03 +0100 Subject: [PATCH 12/66] Frontend work and basic API work for app update support. --- .../components/start/ImportAppModal.svelte | 40 +++++++++++++++++++ .../app/[application]/settings/_layout.svelte | 6 +-- .../{export.svelte => exportImport.svelte} | 26 ++++++++++-- .../scripts/integrations/postgres/init.sql | 8 ++-- .../server/src/api/controllers/application.ts | 2 + packages/server/src/api/routes/application.ts | 5 +++ 6 files changed, 77 insertions(+), 10 deletions(-) create mode 100644 packages/builder/src/components/start/ImportAppModal.svelte rename packages/builder/src/pages/builder/app/[application]/settings/{export.svelte => exportImport.svelte} (71%) diff --git a/packages/builder/src/components/start/ImportAppModal.svelte b/packages/builder/src/components/start/ImportAppModal.svelte new file mode 100644 index 0000000000..329c8286ca --- /dev/null +++ b/packages/builder/src/components/start/ImportAppModal.svelte @@ -0,0 +1,40 @@ + + + + + { + file = e.detail?.[0] + }} + /> + + {#if encrypted} + + {/if} + + diff --git a/packages/builder/src/pages/builder/app/[application]/settings/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/settings/_layout.svelte index 519a9c0f45..801ddd1130 100644 --- a/packages/builder/src/pages/builder/app/[application]/settings/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/settings/_layout.svelte @@ -32,9 +32,9 @@ active={$isActive("./embed")} /> app.devId == $store.appId) $: app = filteredApps.length ? filteredApps[0] : {} $: appDeployed = app?.status === AppStatus.DEPLOYED - let exportModal + let exportModal, importModal let exportPublishedVersion = false const exportApp = opts => { exportPublishedVersion = !!opts?.published exportModal.show() } + + const importApp = () => { + importModal.show() + } + + + + Export your app Export your latest edited or published app - -
+
exportApp({ published: false })}> Export latest edited app @@ -47,10 +55,20 @@ Export latest published app
+ + + Import your app + Import an export to update this app + +
+ importApp()}> + Import app + +
From d0097273b9d87744a9c31bcce9ccd6bdbccadd47 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 19 Sep 2023 10:18:04 +0000 Subject: [PATCH 14/66] Bump version to 2.10.9-alpha.4 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index a57875986a..219091b329 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.10.9-alpha.3", + "version": "2.10.9-alpha.4", "npmClient": "yarn", "packages": [ "packages/*" From 225457d123d92cd09e9d59a2722c014fa99bc688 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 19 Sep 2023 10:28:33 +0000 Subject: [PATCH 15/66] Bump version to 2.10.11 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 021c74d07c..fbecff9293 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.10.10", + "version": "2.10.11", "npmClient": "yarn", "packages": [ "packages/*" From 6d6b3dd9711df5bb15e769c116f665397e88e7f3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 18 Sep 2023 13:10:27 +0200 Subject: [PATCH 16/66] Refactor --- .../tests/core/utilities/structures/shared.ts | 19 -------------- .../tests/core/utilities/structures/sso.ts | 7 +++-- .../tests/core/utilities/structures/users.ts | 26 +++++++++++++++++-- 3 files changed, 27 insertions(+), 25 deletions(-) delete mode 100644 packages/backend-core/tests/core/utilities/structures/shared.ts diff --git a/packages/backend-core/tests/core/utilities/structures/shared.ts b/packages/backend-core/tests/core/utilities/structures/shared.ts deleted file mode 100644 index de0e19486c..0000000000 --- a/packages/backend-core/tests/core/utilities/structures/shared.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { User } from "@budibase/types" -import { generator } from "./generator" -import { uuid } from "./common" - -export const newEmail = () => { - return `${uuid()}@test.com` -} - -export const user = (userProps?: any): User => { - return { - email: newEmail(), - password: "test", - roles: { app_test: "admin" }, - firstName: generator.first(), - lastName: generator.last(), - pictureUrl: "http://test.com", - ...userProps, - } -} diff --git a/packages/backend-core/tests/core/utilities/structures/sso.ts b/packages/backend-core/tests/core/utilities/structures/sso.ts index 4d13635f09..2e3af712a9 100644 --- a/packages/backend-core/tests/core/utilities/structures/sso.ts +++ b/packages/backend-core/tests/core/utilities/structures/sso.ts @@ -13,8 +13,7 @@ import { } from "@budibase/types" import { generator } from "./generator" import { email, uuid } from "./common" -import * as shared from "./shared" -import { user } from "./shared" +import * as users from "./users" import sample from "lodash/sample" export function OAuth(): OAuth2 { @@ -26,7 +25,7 @@ export function OAuth(): OAuth2 { export function authDetails(userDoc?: User): SSOAuthDetails { if (!userDoc) { - userDoc = user() + userDoc = users.user() } const userId = userDoc._id || uuid() @@ -52,7 +51,7 @@ export function providerType(): SSOProviderType { export function ssoProfile(user?: User): SSOProfile { if (!user) { - user = shared.user() + user = users.user() } return { id: user._id!, diff --git a/packages/backend-core/tests/core/utilities/structures/users.ts b/packages/backend-core/tests/core/utilities/structures/users.ts index 0a4f2e8b54..420a9fde0e 100644 --- a/packages/backend-core/tests/core/utilities/structures/users.ts +++ b/packages/backend-core/tests/core/utilities/structures/users.ts @@ -4,11 +4,33 @@ import { BuilderUser, SSOAuthDetails, SSOUser, + User, } from "@budibase/types" -import { user } from "./shared" import { authDetails } from "./sso" +import { uuid } from "./common" +import { generator } from "./generator" +import { tenant } from "." +import { generateGlobalUserID } from "../../../../src/docIds" -export { user, newEmail } from "./shared" +export const newEmail = () => { + return `${uuid()}@test.com` +} + +export const user = (userProps?: Partial>): User => { + const userId = userProps?._id || generateGlobalUserID() + return { + _id: userId, + userId, + email: newEmail(), + password: "test", + roles: { app_test: "admin" }, + firstName: generator.first(), + lastName: generator.last(), + pictureUrl: "http://test.com", + tenantId: tenant.id(), + ...userProps, + } +} export const adminUser = (userProps?: any): AdminUser => { return { From 3336433de82b93e36064376815acb0fc97561291 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 18 Sep 2023 13:39:05 +0200 Subject: [PATCH 17/66] User cache, get in bulk --- packages/backend-core/src/cache/user.ts | 46 +++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts index e2af78adfd..f05cdd42ac 100644 --- a/packages/backend-core/src/cache/user.ts +++ b/packages/backend-core/src/cache/user.ts @@ -27,6 +27,31 @@ async function populateFromDB(userId: string, tenantId: string) { return user } +async function populateUsersFromDB(userIds: string[], tenantId: string) { + const db = tenancy.getTenantDB(tenantId) + const allDocsResponse = await db.allDocs({ + keys: userIds, + include_docs: true, + limit: userIds.length, + }) + + const users = allDocsResponse.rows.map(r => r.doc) + await Promise.all( + users.map(async user => { + user.budibaseAccess = true + if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { + const account = await accounts.getAccount(user.email) + if (account) { + user.account = account + user.accountPortalAccess = true + } + } + }) + ) + + return users +} + /** * Get the requested user by id. * Use redis cache to first read the user. @@ -77,6 +102,27 @@ export async function getUser( return user } +/** + * Get the requested users by id. + * Use redis cache to first read the users. + * If not present fallback to loading the users directly and re-caching. + * @param {*} userIds the ids of the user to get + * @param {*} tenantId the tenant of the users to get + * @returns + */ +export async function getUsers(userIds: string[], tenantId: string) { + const client = await redis.getUserClient() + // try cache + let usersFromCache = await client.bulkGet(userIds) + const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid]) + const usersFromDb = await populateUsersFromDB(missingUsersFromCache, tenantId) + for (const userToCache of usersFromDb) { + await client.store(userToCache._id, userToCache, EXPIRY_SECONDS) + } + const users = [...Object.values(usersFromCache), ...usersFromDb] + return users +} + export async function invalidateUser(userId: string) { const client = await redis.getUserClient() await client.delete(userId) From d3b04ef4de108da40981fc53f8c5395d1579a86f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 18 Sep 2023 13:46:59 +0200 Subject: [PATCH 18/66] Add tests --- .../backend-core/src/cache/tests/user.spec.ts | 95 +++++++++++++++++++ packages/backend-core/src/cache/user.ts | 15 ++- 2 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 packages/backend-core/src/cache/tests/user.spec.ts diff --git a/packages/backend-core/src/cache/tests/user.spec.ts b/packages/backend-core/src/cache/tests/user.spec.ts new file mode 100644 index 0000000000..3d746fc506 --- /dev/null +++ b/packages/backend-core/src/cache/tests/user.spec.ts @@ -0,0 +1,95 @@ +import { User } from "@budibase/types" +import { tenancy } from "../.." +import { generator, structures } from "../../../tests" +import { DBTestConfiguration } from "../../../tests/extra" +import { getUsers } from "../user" +import { getGlobalDB, getGlobalDBName } from "../../context" +import _ from "lodash" +import { getDB } from "../../db" +import type * as TenancyType from "../../tenancy" + +const config = new DBTestConfiguration() + +// This mock is required to ensure that getTenantDB returns always as a singleton. +// This will allow us to spy on the db +const staticDb = getDB(getGlobalDBName(config.tenantId)) +jest.mock("../../tenancy", (): typeof TenancyType => ({ + ...jest.requireActual("../../tenancy"), + getTenantDB: jest.fn().mockImplementation(() => staticDb), +})) + +describe("user cache", () => { + describe("getUsers", () => { + const users: User[] = [] + beforeAll(async () => { + const userCount = 10 + const userIds = generator.arrayOf(() => generator.guid(), { + min: userCount, + max: userCount, + }) + + await config.doInTenant(async () => { + const db = getGlobalDB() + for (const userId of userIds) { + const user = structures.users.user({ _id: userId }) + await db.put(user) + users.push(user) + } + }) + }) + + beforeEach(() => { + jest.clearAllMocks() + }) + + it("when no user is in cache, all of them are retrieved from db", async () => { + const usersToRequest = _.sampleSize(users, 5) + + const userIdsToRequest = usersToRequest.map(x => x._id!) + + jest.spyOn(staticDb, "allDocs") + + const results = await getUsers(userIdsToRequest, config.tenantId) + + expect(results).toHaveLength(5) + expect(results).toEqual( + usersToRequest.map(u => ({ + ...u, + budibaseAccess: true, + _rev: expect.any(String), + })) + ) + + expect(tenancy.getTenantDB).toBeCalledTimes(1) + expect(tenancy.getTenantDB).toBeCalledWith(config.tenantId) + expect(staticDb.allDocs).toBeCalledTimes(1) + expect(staticDb.allDocs).toBeCalledWith({ + keys: userIdsToRequest, + include_docs: true, + limit: 5, + }) + }) + + it("on a second all, all of them are retrieved from cache", async () => { + const usersToRequest = _.sampleSize(users, 5) + + const userIdsToRequest = usersToRequest.map(x => x._id!) + + jest.spyOn(staticDb, "allDocs") + + await getUsers(userIdsToRequest, config.tenantId) + const resultsFromCache = await getUsers(userIdsToRequest, config.tenantId) + + expect(resultsFromCache).toHaveLength(5) + expect(resultsFromCache).toEqual( + usersToRequest.map(u => ({ + ...u, + budibaseAccess: true, + _rev: expect.any(String), + })) + ) + + expect(staticDb.allDocs).toBeCalledTimes(1) + }) + }) +}) diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts index f05cdd42ac..9742b41b65 100644 --- a/packages/backend-core/src/cache/user.ts +++ b/packages/backend-core/src/cache/user.ts @@ -115,11 +115,18 @@ export async function getUsers(userIds: string[], tenantId: string) { // try cache let usersFromCache = await client.bulkGet(userIds) const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid]) - const usersFromDb = await populateUsersFromDB(missingUsersFromCache, tenantId) - for (const userToCache of usersFromDb) { - await client.store(userToCache._id, userToCache, EXPIRY_SECONDS) + const users = Object.values(usersFromCache) + + if (missingUsersFromCache.length) { + const usersFromDb = await populateUsersFromDB( + missingUsersFromCache, + tenantId + ) + for (const userToCache of usersFromDb) { + await client.store(userToCache._id, userToCache, EXPIRY_SECONDS) + } + users.push(...usersFromDb) } - const users = [...Object.values(usersFromCache), ...usersFromDb] return users } From 4311d563d2298bbd7fc23b063aa51c52faf14f70 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 18 Sep 2023 13:57:59 +0200 Subject: [PATCH 19/66] Add tests --- .../backend-core/src/cache/tests/user.spec.ts | 54 ++++++++++++++++--- 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/packages/backend-core/src/cache/tests/user.spec.ts b/packages/backend-core/src/cache/tests/user.spec.ts index 3d746fc506..45537694c7 100644 --- a/packages/backend-core/src/cache/tests/user.spec.ts +++ b/packages/backend-core/src/cache/tests/user.spec.ts @@ -1,5 +1,5 @@ import { User } from "@budibase/types" -import { tenancy } from "../.." +import { cache, tenancy } from "../.." import { generator, structures } from "../../../tests" import { DBTestConfiguration } from "../../../tests/extra" import { getUsers } from "../user" @@ -7,6 +7,7 @@ import { getGlobalDB, getGlobalDBName } from "../../context" import _ from "lodash" import { getDB } from "../../db" import type * as TenancyType from "../../tenancy" +import * as redis from "../../redis/init" const config = new DBTestConfiguration() @@ -38,8 +39,11 @@ describe("user cache", () => { }) }) - beforeEach(() => { + beforeEach(async () => { jest.clearAllMocks() + + const redisClient = await redis.getUserClient() + await redisClient.clear() }) it("when no user is in cache, all of them are retrieved from db", async () => { @@ -82,14 +86,50 @@ describe("user cache", () => { expect(resultsFromCache).toHaveLength(5) expect(resultsFromCache).toEqual( - usersToRequest.map(u => ({ - ...u, - budibaseAccess: true, - _rev: expect.any(String), - })) + expect.arrayContaining( + usersToRequest.map(u => ({ + ...u, + budibaseAccess: true, + _rev: expect.any(String), + })) + ) ) expect(staticDb.allDocs).toBeCalledTimes(1) }) + + it("when some users are cached, only the missing ones are retrieved from db", async () => { + const usersToRequest = _.sampleSize(users, 5) + + const userIdsToRequest = usersToRequest.map(x => x._id!) + + jest.spyOn(staticDb, "allDocs") + + await getUsers( + [userIdsToRequest[0], userIdsToRequest[3]], + config.tenantId + ) + ;(staticDb.allDocs as jest.Mock).mockClear() + + const results = await getUsers(userIdsToRequest, config.tenantId) + + expect(results).toHaveLength(5) + expect(results).toEqual( + expect.arrayContaining( + usersToRequest.map(u => ({ + ...u, + budibaseAccess: true, + _rev: expect.any(String), + })) + ) + ) + + expect(staticDb.allDocs).toBeCalledTimes(1) + expect(staticDb.allDocs).toBeCalledWith({ + keys: [userIdsToRequest[1], userIdsToRequest[2], userIdsToRequest[4]], + include_docs: true, + limit: 3, + }) + }) }) }) From 6f1d0271265627b29d28da9a0dfaae63385eae03 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 18 Sep 2023 14:38:20 +0200 Subject: [PATCH 20/66] Fix test --- .../backend-core/src/middleware/passport/sso/tests/sso.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts b/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts index 484a118cbd..c3ddf220e6 100644 --- a/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts +++ b/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts @@ -102,6 +102,7 @@ describe("sso", () => { // modified external id to match user format ssoUser._id = "us_" + details.userId + delete ssoUser.userId // new sso user won't have a password delete ssoUser.password From 7b4585ce688a4a2cca2d7a147cd39bb535fb043a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 19 Sep 2023 12:08:59 +0200 Subject: [PATCH 21/66] Tenantid optional --- packages/backend-core/src/cache/user.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts index 9742b41b65..e21c8a1e43 100644 --- a/packages/backend-core/src/cache/user.ts +++ b/packages/backend-core/src/cache/user.ts @@ -110,7 +110,7 @@ export async function getUser( * @param {*} tenantId the tenant of the users to get * @returns */ -export async function getUsers(userIds: string[], tenantId: string) { +export async function getUsers(userIds: string[], tenantId?: string) { const client = await redis.getUserClient() // try cache let usersFromCache = await client.bulkGet(userIds) @@ -118,6 +118,7 @@ export async function getUsers(userIds: string[], tenantId: string) { const users = Object.values(usersFromCache) if (missingUsersFromCache.length) { + tenantId ??= context.getTenantId() const usersFromDb = await populateUsersFromDB( missingUsersFromCache, tenantId From 9e1ccc35ee92521d8639fa7885b9553663e82eed Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 19 Sep 2023 12:22:25 +0200 Subject: [PATCH 22/66] Handle missing users --- .../backend-core/src/cache/tests/user.spec.ts | 54 +++++++++++++------ packages/backend-core/src/cache/user.ts | 43 +++++++++++---- packages/backend-core/src/redis/redis.ts | 2 +- 3 files changed, 73 insertions(+), 26 deletions(-) diff --git a/packages/backend-core/src/cache/tests/user.spec.ts b/packages/backend-core/src/cache/tests/user.spec.ts index 45537694c7..490d769b92 100644 --- a/packages/backend-core/src/cache/tests/user.spec.ts +++ b/packages/backend-core/src/cache/tests/user.spec.ts @@ -55,14 +55,14 @@ describe("user cache", () => { const results = await getUsers(userIdsToRequest, config.tenantId) - expect(results).toHaveLength(5) - expect(results).toEqual( - usersToRequest.map(u => ({ + expect(results.users).toHaveLength(5) + expect(results).toEqual({ + users: usersToRequest.map(u => ({ ...u, budibaseAccess: true, _rev: expect.any(String), - })) - ) + })), + }) expect(tenancy.getTenantDB).toBeCalledTimes(1) expect(tenancy.getTenantDB).toBeCalledWith(config.tenantId) @@ -84,16 +84,16 @@ describe("user cache", () => { await getUsers(userIdsToRequest, config.tenantId) const resultsFromCache = await getUsers(userIdsToRequest, config.tenantId) - expect(resultsFromCache).toHaveLength(5) - expect(resultsFromCache).toEqual( - expect.arrayContaining( + expect(resultsFromCache.users).toHaveLength(5) + expect(resultsFromCache).toEqual({ + users: expect.arrayContaining( usersToRequest.map(u => ({ ...u, budibaseAccess: true, _rev: expect.any(String), })) - ) - ) + ), + }) expect(staticDb.allDocs).toBeCalledTimes(1) }) @@ -113,16 +113,16 @@ describe("user cache", () => { const results = await getUsers(userIdsToRequest, config.tenantId) - expect(results).toHaveLength(5) - expect(results).toEqual( - expect.arrayContaining( + expect(results.users).toHaveLength(5) + expect(results).toEqual({ + users: expect.arrayContaining( usersToRequest.map(u => ({ ...u, budibaseAccess: true, _rev: expect.any(String), })) - ) - ) + ), + }) expect(staticDb.allDocs).toBeCalledTimes(1) expect(staticDb.allDocs).toBeCalledWith({ @@ -131,5 +131,29 @@ describe("user cache", () => { limit: 3, }) }) + + it("requesting existing and unexisting ids will return found ones", async () => { + const usersToRequest = _.sampleSize(users, 3) + const missingIds = [generator.guid(), generator.guid()] + + const userIdsToRequest = _.shuffle([ + ...missingIds, + ...usersToRequest.map(x => x._id!), + ]) + + const results = await getUsers(userIdsToRequest, config.tenantId) + + expect(results.users).toHaveLength(3) + expect(results).toEqual({ + users: expect.arrayContaining( + usersToRequest.map(u => ({ + ...u, + budibaseAccess: true, + _rev: expect.any(String), + })) + ), + notFoundIds: expect.arrayContaining(missingIds), + }) + }) }) }) diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts index e21c8a1e43..ccd9946504 100644 --- a/packages/backend-core/src/cache/user.ts +++ b/packages/backend-core/src/cache/user.ts @@ -6,6 +6,7 @@ import env from "../environment" import * as accounts from "../accounts" import { UserDB } from "../users" import { sdk } from "@budibase/shared-core" +import { User } from "@budibase/types" const EXPIRY_SECONDS = 3600 @@ -27,7 +28,10 @@ async function populateFromDB(userId: string, tenantId: string) { return user } -async function populateUsersFromDB(userIds: string[], tenantId: string) { +async function populateUsersFromDB( + userIds: string[], + tenantId: string +): Promise<{ users: User[]; notFoundIds?: string[] }> { const db = tenancy.getTenantDB(tenantId) const allDocsResponse = await db.allDocs({ keys: userIds, @@ -35,9 +39,22 @@ async function populateUsersFromDB(userIds: string[], tenantId: string) { limit: userIds.length, }) - const users = allDocsResponse.rows.map(r => r.doc) + const { users, notFoundIds } = allDocsResponse.rows.reduce( + (p, c) => { + if (c.doc) { + p.users.push(c.doc) + } else { + p.notFoundIds ??= [] + p.notFoundIds.push(c.key) + } + return p + }, + { + users: [], + } as { users: User[]; notFoundIds?: string[] } + ) await Promise.all( - users.map(async user => { + users.map(async (user: any) => { user.budibaseAccess = true if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { const account = await accounts.getAccount(user.email) @@ -49,7 +66,7 @@ async function populateUsersFromDB(userIds: string[], tenantId: string) { }) ) - return users + return { users, notFoundIds: notFoundIds } } /** @@ -110,12 +127,16 @@ export async function getUser( * @param {*} tenantId the tenant of the users to get * @returns */ -export async function getUsers(userIds: string[], tenantId?: string) { +export async function getUsers( + userIds: string[], + tenantId?: string +): Promise<{ users: User[]; notFoundIds?: string[] }> { const client = await redis.getUserClient() // try cache - let usersFromCache = await client.bulkGet(userIds) + let usersFromCache = await client.bulkGet(userIds) const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid]) const users = Object.values(usersFromCache) + let notFoundIds if (missingUsersFromCache.length) { tenantId ??= context.getTenantId() @@ -123,12 +144,14 @@ export async function getUsers(userIds: string[], tenantId?: string) { missingUsersFromCache, tenantId ) - for (const userToCache of usersFromDb) { - await client.store(userToCache._id, userToCache, EXPIRY_SECONDS) + + notFoundIds = usersFromDb.notFoundIds + for (const userToCache of usersFromDb.users) { + await client.store(userToCache._id!, userToCache, EXPIRY_SECONDS) } - users.push(...usersFromDb) + users.push(...usersFromDb.users) } - return users + return { users, notFoundIds: notFoundIds } } export async function invalidateUser(userId: string) { diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 5056a5d549..5eaa2b4b61 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -242,7 +242,7 @@ class RedisWrapper { } } - async bulkGet(keys: string[]) { + async bulkGet(keys: string[]) { const db = this._db if (keys.length === 0) { return {} From 1d63b219b814d518db995a05710f461659d4349d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 19 Sep 2023 13:01:16 +0200 Subject: [PATCH 23/66] Make use of UserDB --- .../backend-core/src/cache/tests/user.spec.ts | 64 ++++++++----------- packages/backend-core/src/cache/user.ts | 43 ++++--------- 2 files changed, 38 insertions(+), 69 deletions(-) diff --git a/packages/backend-core/src/cache/tests/user.spec.ts b/packages/backend-core/src/cache/tests/user.spec.ts index 490d769b92..80e5bc3063 100644 --- a/packages/backend-core/src/cache/tests/user.spec.ts +++ b/packages/backend-core/src/cache/tests/user.spec.ts @@ -1,24 +1,15 @@ import { User } from "@budibase/types" -import { cache, tenancy } from "../.." import { generator, structures } from "../../../tests" import { DBTestConfiguration } from "../../../tests/extra" import { getUsers } from "../user" -import { getGlobalDB, getGlobalDBName } from "../../context" +import { getGlobalDB } from "../../context" import _ from "lodash" -import { getDB } from "../../db" -import type * as TenancyType from "../../tenancy" + import * as redis from "../../redis/init" +import { UserDB } from "../../users" const config = new DBTestConfiguration() -// This mock is required to ensure that getTenantDB returns always as a singleton. -// This will allow us to spy on the db -const staticDb = getDB(getGlobalDBName(config.tenantId)) -jest.mock("../../tenancy", (): typeof TenancyType => ({ - ...jest.requireActual("../../tenancy"), - getTenantDB: jest.fn().mockImplementation(() => staticDb), -})) - describe("user cache", () => { describe("getUsers", () => { const users: User[] = [] @@ -51,9 +42,9 @@ describe("user cache", () => { const userIdsToRequest = usersToRequest.map(x => x._id!) - jest.spyOn(staticDb, "allDocs") + jest.spyOn(UserDB, "bulkGet") - const results = await getUsers(userIdsToRequest, config.tenantId) + const results = await config.doInTenant(() => getUsers(userIdsToRequest)) expect(results.users).toHaveLength(5) expect(results).toEqual({ @@ -64,14 +55,8 @@ describe("user cache", () => { })), }) - expect(tenancy.getTenantDB).toBeCalledTimes(1) - expect(tenancy.getTenantDB).toBeCalledWith(config.tenantId) - expect(staticDb.allDocs).toBeCalledTimes(1) - expect(staticDb.allDocs).toBeCalledWith({ - keys: userIdsToRequest, - include_docs: true, - limit: 5, - }) + expect(UserDB.bulkGet).toBeCalledTimes(1) + expect(UserDB.bulkGet).toBeCalledWith(userIdsToRequest) }) it("on a second all, all of them are retrieved from cache", async () => { @@ -79,10 +64,12 @@ describe("user cache", () => { const userIdsToRequest = usersToRequest.map(x => x._id!) - jest.spyOn(staticDb, "allDocs") + jest.spyOn(UserDB, "bulkGet") - await getUsers(userIdsToRequest, config.tenantId) - const resultsFromCache = await getUsers(userIdsToRequest, config.tenantId) + await config.doInTenant(() => getUsers(userIdsToRequest)) + const resultsFromCache = await config.doInTenant(() => + getUsers(userIdsToRequest) + ) expect(resultsFromCache.users).toHaveLength(5) expect(resultsFromCache).toEqual({ @@ -95,7 +82,7 @@ describe("user cache", () => { ), }) - expect(staticDb.allDocs).toBeCalledTimes(1) + expect(UserDB.bulkGet).toBeCalledTimes(1) }) it("when some users are cached, only the missing ones are retrieved from db", async () => { @@ -103,15 +90,14 @@ describe("user cache", () => { const userIdsToRequest = usersToRequest.map(x => x._id!) - jest.spyOn(staticDb, "allDocs") + jest.spyOn(UserDB, "bulkGet") - await getUsers( - [userIdsToRequest[0], userIdsToRequest[3]], - config.tenantId + await config.doInTenant(() => + getUsers([userIdsToRequest[0], userIdsToRequest[3]]) ) - ;(staticDb.allDocs as jest.Mock).mockClear() + ;(UserDB.bulkGet as jest.Mock).mockClear() - const results = await getUsers(userIdsToRequest, config.tenantId) + const results = await config.doInTenant(() => getUsers(userIdsToRequest)) expect(results.users).toHaveLength(5) expect(results).toEqual({ @@ -124,12 +110,12 @@ describe("user cache", () => { ), }) - expect(staticDb.allDocs).toBeCalledTimes(1) - expect(staticDb.allDocs).toBeCalledWith({ - keys: [userIdsToRequest[1], userIdsToRequest[2], userIdsToRequest[4]], - include_docs: true, - limit: 3, - }) + expect(UserDB.bulkGet).toBeCalledTimes(1) + expect(UserDB.bulkGet).toBeCalledWith([ + userIdsToRequest[1], + userIdsToRequest[2], + userIdsToRequest[4], + ]) }) it("requesting existing and unexisting ids will return found ones", async () => { @@ -141,7 +127,7 @@ describe("user cache", () => { ...usersToRequest.map(x => x._id!), ]) - const results = await getUsers(userIdsToRequest, config.tenantId) + const results = await config.doInTenant(() => getUsers(userIdsToRequest)) expect(results.users).toHaveLength(3) expect(results).toEqual({ diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts index ccd9946504..481d3691e4 100644 --- a/packages/backend-core/src/cache/user.ts +++ b/packages/backend-core/src/cache/user.ts @@ -29,30 +29,15 @@ async function populateFromDB(userId: string, tenantId: string) { } async function populateUsersFromDB( - userIds: string[], - tenantId: string + userIds: string[] ): Promise<{ users: User[]; notFoundIds?: string[] }> { - const db = tenancy.getTenantDB(tenantId) - const allDocsResponse = await db.allDocs({ - keys: userIds, - include_docs: true, - limit: userIds.length, - }) + const getUsersResponse = await UserDB.bulkGet(userIds) + + // Handle missed user ids + const notFoundIds = userIds.filter((uid, i) => !getUsersResponse[i]) + + const users = getUsersResponse.filter(x => x) - const { users, notFoundIds } = allDocsResponse.rows.reduce( - (p, c) => { - if (c.doc) { - p.users.push(c.doc) - } else { - p.notFoundIds ??= [] - p.notFoundIds.push(c.key) - } - return p - }, - { - users: [], - } as { users: User[]; notFoundIds?: string[] } - ) await Promise.all( users.map(async (user: any) => { user.budibaseAccess = true @@ -66,7 +51,10 @@ async function populateUsersFromDB( }) ) - return { users, notFoundIds: notFoundIds } + if (notFoundIds.length) { + return { users, notFoundIds } + } + return { users } } /** @@ -128,8 +116,7 @@ export async function getUser( * @returns */ export async function getUsers( - userIds: string[], - tenantId?: string + userIds: string[] ): Promise<{ users: User[]; notFoundIds?: string[] }> { const client = await redis.getUserClient() // try cache @@ -139,11 +126,7 @@ export async function getUsers( let notFoundIds if (missingUsersFromCache.length) { - tenantId ??= context.getTenantId() - const usersFromDb = await populateUsersFromDB( - missingUsersFromCache, - tenantId - ) + const usersFromDb = await populateUsersFromDB(missingUsersFromCache) notFoundIds = usersFromDb.notFoundIds for (const userToCache of usersFromDb.users) { From e128f1c9216e92bc3fbe2d28202913826a2ca36a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 19 Sep 2023 13:27:42 +0200 Subject: [PATCH 24/66] Fix types --- packages/backend-core/src/cache/user.ts | 2 +- packages/backend-core/src/redis/redis.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts index 481d3691e4..b3fd7c08cd 100644 --- a/packages/backend-core/src/cache/user.ts +++ b/packages/backend-core/src/cache/user.ts @@ -120,7 +120,7 @@ export async function getUsers( ): Promise<{ users: User[]; notFoundIds?: string[] }> { const client = await redis.getUserClient() // try cache - let usersFromCache = await client.bulkGet(userIds) + let usersFromCache = await client.bulkGet(userIds) const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid]) const users = Object.values(usersFromCache) let notFoundIds diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 5eaa2b4b61..78817d0aa0 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -242,7 +242,7 @@ class RedisWrapper { } } - async bulkGet(keys: string[]) { + async bulkGet(keys: string[]) { const db = this._db if (keys.length === 0) { return {} @@ -250,7 +250,7 @@ class RedisWrapper { const prefixedKeys = keys.map(key => addDbPrefix(db, key)) let response = await this.getClient().mget(prefixedKeys) if (Array.isArray(response)) { - let final: any = {} + let final: Record = {} let count = 0 for (let result of response) { if (result) { From dade2314849dc6d7d6df8b6a6357f522a9c7840d Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 19 Sep 2023 12:15:45 +0000 Subject: [PATCH 25/66] Bump version to 2.10.9-alpha.5 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 219091b329..cf21e415f2 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.10.9-alpha.4", + "version": "2.10.9-alpha.5", "npmClient": "yarn", "packages": [ "packages/*" From e8e4dd5c907bd720b1945f581b7da499e39a42fb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 19 Sep 2023 13:45:24 +0100 Subject: [PATCH 26/66] Finishing UI - getting API ready. --- .../components/start/ImportAppModal.svelte | 22 ++++++++++++++++--- packages/frontend-core/src/api/app.js | 18 +++++++++++++++ .../server/src/api/controllers/application.ts | 11 +++++++++- packages/server/src/api/routes/application.ts | 5 +++-- 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/packages/builder/src/components/start/ImportAppModal.svelte b/packages/builder/src/components/start/ImportAppModal.svelte index 329c8286ca..7496b5824d 100644 --- a/packages/builder/src/components/start/ImportAppModal.svelte +++ b/packages/builder/src/components/start/ImportAppModal.svelte @@ -1,5 +1,7 @@ diff --git a/packages/frontend-core/src/api/app.js b/packages/frontend-core/src/api/app.js index 982066f05a..49137cbecd 100644 --- a/packages/frontend-core/src/api/app.js +++ b/packages/frontend-core/src/api/app.js @@ -1,3 +1,5 @@ +import { sdk } from "@budibase/shared-core" + export const buildAppEndpoints = API => ({ /** * Fetches screen definition for an app. @@ -81,6 +83,22 @@ export const buildAppEndpoints = API => ({ }) }, + /** + * Update an application using an export - the body + * should be of type FormData, with a "file" and a "password" if encrypted. + * @param appId The ID of the app to update - this will always be + * converted to development ID. + * @param body a FormData body with a file and password. + */ + updateAppFromExport: async (appId, body) => { + const devId = sdk.applications.getDevAppID(appId) + return await API.post({ + url: `/api/applications/${devId}/import`, + body, + json: false, + }) + }, + /** * Imports an export of all apps. * @param apps the FormData containing the apps to import diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index ad6780e61b..a7d3634a89 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -575,7 +575,16 @@ export async function sync(ctx: UserCtx) { } } -export async function updateWithExport(ctx: UserCtx) {} +export async function importToApp(ctx: UserCtx) { + const { appId } = ctx.params + const appExport = ctx.request.files?.appExport + const password = ctx.request.body.encryptionPassword + if (!appExport) { + ctx.throw(400, "Must supply app export to import") + } + console.log(appExport) + ctx.body = { message: "app updated" } +} export async function updateAppPackage(appPackage: any, appId: any) { return context.doInAppContext(appId, async () => { diff --git a/packages/server/src/api/routes/application.ts b/packages/server/src/api/routes/application.ts index aef32ee559..a21d6a2153 100644 --- a/packages/server/src/api/routes/application.ts +++ b/packages/server/src/api/routes/application.ts @@ -4,6 +4,7 @@ import * as deploymentController from "../controllers/deploy" import authorized from "../../middleware/authorized" import { permissions } from "@budibase/backend-core" import { applicationValidator } from "./utils/validators" +import { importToApp } from "../controllers/application" const router: Router = new Router() @@ -59,9 +60,9 @@ router controller.destroy ) .post( - "/api/applications/:appId/update", + "/api/applications/:appId/import", authorized(permissions.BUILDER), - controller.updateWithExport + controller.importToApp ) export default router From e69ebfb2ab1317a08659fe99c2c95e9d4c3f588c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 19 Sep 2023 14:01:03 +0100 Subject: [PATCH 27/66] Adding test case for row creation, don't allow user table. --- .../server/src/api/routes/tests/row.spec.ts | 19 ++++++++++++++++++- yarn.lock | 13 ------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index a74a9f7960..6a021460ac 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3,7 +3,7 @@ import { databaseTestProviders } from "../../../integrations/tests/utils" import tk from "timekeeper" import { outputProcessing } from "../../../utilities/rowProcessor" import * as setup from "./utilities" -import { context, roles, tenancy } from "@budibase/backend-core" +import { context, InternalTable, roles, tenancy } from "@budibase/backend-core" import { quotas } from "@budibase/pro" import { FieldType, @@ -1415,6 +1415,23 @@ describe.each([ }) }) + isInternal && + it("doesn't allow creating in user table", async () => { + const userTableId = InternalTable.USER_METADATA + const response = await config.api.row.save( + userTableId, + { + tableId: userTableId, + firstName: "Joe", + lastName: "Joe", + email: "joe@joe.com", + roles: {}, + }, + { expectStatus: 400 } + ) + expect(response.message).toBe("Cannot create new user entry.") + }) + describe("permissions", () => { let viewId: string let tableId: string diff --git a/yarn.lock b/yarn.lock index 8c93661665..ab86a87560 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6269,14 +6269,6 @@ "@types/tedious" "*" tarn "^3.0.1" -"@types/node-fetch@2.6.1": - version "2.6.1" - resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975" - integrity sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA== - dependencies: - "@types/node" "*" - form-data "^3.0.0" - "@types/node-fetch@2.6.4": version "2.6.4" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.4.tgz#1bc3a26de814f6bf466b25aeb1473fa1afe6a660" @@ -6298,11 +6290,6 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f" integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA== -"@types/node@14.18.20": - version "14.18.20" - resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.20.tgz#268f028b36eaf51181c3300252f605488c4f0650" - integrity sha512-Q8KKwm9YqEmUBRsqJ2GWJDtXltBDxTdC4m5vTdXBolu2PeQh8LX+f6BTwU+OuXPu37fLxoN6gidqBmnky36FXA== - "@types/node@16.9.1": version "16.9.1" resolved "https://registry.yarnpkg.com/@types/node/-/node-16.9.1.tgz#0611b37db4246c937feef529ddcc018cf8e35708" From 51a43540dda49be660126f4f1432c1163b549232 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 19 Sep 2023 13:34:23 +0000 Subject: [PATCH 28/66] Bump version to 2.10.12-alpha.0 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index fbecff9293..0d35a3532c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.10.11", + "version": "2.10.12-alpha.0", "npmClient": "yarn", "packages": [ "packages/*" From c26e1d11274a97c51cf0ae8244cbeed3bc0e7a15 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 19 Sep 2023 15:02:57 +0000 Subject: [PATCH 29/66] Bump version to 2.10.12-alpha.1 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 0d35a3532c..5d16aa7bf1 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.10.12-alpha.0", + "version": "2.10.12-alpha.1", "npmClient": "yarn", "packages": [ "packages/*" From 8e0cf1d087c473a60e162f08edcb3b6a8bc8796c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 19 Sep 2023 18:38:44 +0100 Subject: [PATCH 30/66] Moving things around ready for implementation of temp db handle. --- .../server/src/api/controllers/application.ts | 12 ++++---- .../server/src/sdk/app/applications/import.ts | 29 +++++++++++++++++++ .../server/src/sdk/app/applications/index.ts | 2 ++ 3 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 packages/server/src/sdk/app/applications/import.ts diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index a7d3634a89..8d4d1c673a 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -39,9 +39,8 @@ import { } from "../../db/defaultData/datasource_bb_default" import { removeAppFromUserRoles } from "../../utilities/workerRequests" import { stringToReadStream } from "../../utilities" -import { doesUserHaveLock, getLocksById } from "../../utilities/redis" +import { doesUserHaveLock } from "../../utilities/redis" import { cleanupAutomations } from "../../automations/utils" -import { checkAppMetadata } from "../../automations/logging" import { getUniqueRows } from "../../utilities/usageQuota/rows" import { groups, licensing, quotas } from "@budibase/pro" import { @@ -51,7 +50,6 @@ import { PlanType, Screen, UserCtx, - ContextUser, } from "@budibase/types" import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts" import sdk from "../../sdk" @@ -578,11 +576,15 @@ export async function sync(ctx: UserCtx) { export async function importToApp(ctx: UserCtx) { const { appId } = ctx.params const appExport = ctx.request.files?.appExport - const password = ctx.request.body.encryptionPassword + const password = ctx.request.body.encryptionPassword as string if (!appExport) { ctx.throw(400, "Must supply app export to import") } - console.log(appExport) + if (Array.isArray(appExport)) { + ctx.throw(400, "Must only supply one app export") + } + const fileAttributes = { type: appExport.type!, path: appExport.path! } + await sdk.applications.updateWithExport(appId, fileAttributes, password) ctx.body = { message: "app updated" } } diff --git a/packages/server/src/sdk/app/applications/import.ts b/packages/server/src/sdk/app/applications/import.ts new file mode 100644 index 0000000000..e72300fa4e --- /dev/null +++ b/packages/server/src/sdk/app/applications/import.ts @@ -0,0 +1,29 @@ +import { db as dbCore } from "@budibase/backend-core" +import backups from "../backups" + +export type FileAttributes = { + type: string + path: string +} + +export async function updateWithExport( + appId: string, + file: FileAttributes, + password?: string +) { + const devId = dbCore.getDevAppID(appId) + // TEMPORARY BEGIN + const appDb = dbCore.getDB(devId) + await appDb.destroy() + // TEMPORARY END + // const tempAppName = `temp_${devId}` + // const tempDb = dbCore.getDB(tempAppName) + const template = { + file: { + type: file.type!, + path: file.path!, + password, + }, + } + await backups.importApp(devId, appDb, template) +} diff --git a/packages/server/src/sdk/app/applications/index.ts b/packages/server/src/sdk/app/applications/index.ts index 963d065ce2..04ed3b2919 100644 --- a/packages/server/src/sdk/app/applications/index.ts +++ b/packages/server/src/sdk/app/applications/index.ts @@ -1,9 +1,11 @@ import * as sync from "./sync" import * as utils from "./utils" import * as applications from "./applications" +import * as imports from "./import" export default { ...sync, ...utils, ...applications, + ...imports, } From ac50ce58328b57108290b2367a9eaa6b00f1dc9f Mon Sep 17 00:00:00 2001 From: Gerard Burns Date: Wed, 20 Sep 2023 08:51:13 +0100 Subject: [PATCH 31/66] Change verification banner implementation to respect portal and design app section layouts (#11813) --- .../common/VerificationPromptBanner.svelte | 102 ++++++++++++++++++ .../builder/src/pages/builder/_layout.svelte | 27 ----- .../builder/app/[application]/_layout.svelte | 2 + .../src/pages/builder/portal/_layout.svelte | 2 + 4 files changed, 106 insertions(+), 27 deletions(-) create mode 100644 packages/builder/src/components/common/VerificationPromptBanner.svelte diff --git a/packages/builder/src/components/common/VerificationPromptBanner.svelte b/packages/builder/src/components/common/VerificationPromptBanner.svelte new file mode 100644 index 0000000000..e9109ae0b1 --- /dev/null +++ b/packages/builder/src/components/common/VerificationPromptBanner.svelte @@ -0,0 +1,102 @@ + + +{#if user?.account?.verified === false} + +{/if} + + diff --git a/packages/builder/src/pages/builder/_layout.svelte b/packages/builder/src/pages/builder/_layout.svelte index 960822a39f..b216958045 100644 --- a/packages/builder/src/pages/builder/_layout.svelte +++ b/packages/builder/src/pages/builder/_layout.svelte @@ -3,7 +3,6 @@ import { admin, auth, licensing } from "stores/portal" import { onMount } from "svelte" import { CookieUtils, Constants } from "@budibase/frontend-core" - import { banner, BANNER_TYPES } from "@budibase/bbui" import { API } from "api" import Branding from "./Branding.svelte" @@ -17,32 +16,6 @@ $: user = $auth.user $: useAccountPortal = cloud && !$admin.disableAccountPortal - let showVerificationPrompt = false - - const checkVerification = user => { - if (!showVerificationPrompt && user?.account?.verified === false) { - showVerificationPrompt = true - banner.queue([ - { - message: `Please verify your account. We've sent the verification link to ${user.email}`, - type: BANNER_TYPES.NEUTRAL, - showCloseButton: false, - extraButtonAction: () => { - fetch(`${$admin.accountPortalUrl}/api/auth/reset`, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ email: user.email }), - }) - }, - extraButtonText: "Resend email", - }, - ]) - } - } - - $: checkVerification(user) const validateTenantId = async () => { const host = window.location.host diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index 872151b4a3..1df2a90250 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -22,6 +22,7 @@ import { isActive, goto, layout, redirect } from "@roxi/routify" import { capitalise } from "helpers" import { onMount, onDestroy } from "svelte" + import VerificationPromptBanner from "components/common/VerificationPromptBanner.svelte" import CommandPalette from "components/commandPalette/CommandPalette.svelte" import TourWrap from "components/portal/onboarding/TourWrap.svelte" import TourPopover from "components/portal/onboarding/TourPopover.svelte" @@ -136,6 +137,7 @@ {/if}
+
{#if $store.initialised}
diff --git a/packages/builder/src/pages/builder/portal/_layout.svelte b/packages/builder/src/pages/builder/portal/_layout.svelte index 006e69daca..9459eefff1 100644 --- a/packages/builder/src/pages/builder/portal/_layout.svelte +++ b/packages/builder/src/pages/builder/portal/_layout.svelte @@ -8,6 +8,7 @@ import Logo from "./_components/Logo.svelte" import UserDropdown from "./_components/UserDropdown.svelte" import HelpMenu from "components/common/HelpMenu.svelte" + import VerificationPromptBanner from "components/common/VerificationPromptBanner.svelte" import { sdk } from "@budibase/shared-core" let loaded = false @@ -55,6 +56,7 @@ {:else}
+
From a1913dbd6f667f283895384f08179757eb43f710 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 20 Sep 2023 07:51:31 +0000 Subject: [PATCH 32/66] Bump version to 2.10.12-alpha.2 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 5d16aa7bf1..67a120b31c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.10.12-alpha.1", + "version": "2.10.12-alpha.2", "npmClient": "yarn", "packages": [ "packages/*" From 36f351e96d51df3a0cd622f4c416b6f5ea7a93bf Mon Sep 17 00:00:00 2001 From: Gerard Burns Date: Wed, 20 Sep 2023 12:08:46 +0100 Subject: [PATCH 33/66] Nginx cors changes to allow communicating with account portal from bb front end (#11831) --- hosting/proxy/nginx.prod.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf index 9ce6b54053..634bce18ac 100644 --- a/hosting/proxy/nginx.prod.conf +++ b/hosting/proxy/nginx.prod.conf @@ -55,7 +55,7 @@ http { set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com"; set $csp_object "object-src 'none'"; set $csp_base_uri "base-uri 'self'"; - set $csp_connect "connect-src 'self' https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com"; + set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibase.qa https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com"; set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com https://js.intercomcdn.com https://fonts.intercomcdn.com"; set $csp_frame "frame-src 'self' https:"; set $csp_img "img-src http: https: data: blob:"; From acb53c075ac3ffb096326a60dc762769736a9811 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Wed, 20 Sep 2023 14:30:35 +0100 Subject: [PATCH 34/66] Scroll to first invalid field (#11824) * Final actions * Add vertical options to ScrollTo * Remove nearest option * Scroll to form field when invalid * Backwards compatibility for ScrollTo action * Revert "Final actions" This reverts commit 910e9e6d396aa8531c9662181ebccc77b90ed0ae. * Scroll to form field when invalid * Remove createAutomations function * Refactor * Refactor * lint --------- Co-authored-by: Michael Drury --- .../ButtonActionEditor.svelte | 37 ------------------- .../actions/ScrollTo.svelte | 7 ---- .../src/components/app/forms/InnerForm.svelte | 17 +++++++-- 3 files changed, 14 insertions(+), 47 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/ButtonActionEditor.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/ButtonActionEditor.svelte index 792777349c..6d9e96a564 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/ButtonActionEditor.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/ButtonActionEditor.svelte @@ -3,7 +3,6 @@ import { createEventDispatcher } from "svelte" import { notifications } from "@budibase/bbui" import ButtonActionDrawer from "./ButtonActionDrawer.svelte" - import { automationStore } from "builderStore" import { cloneDeep } from "lodash/fp" const dispatch = createEventDispatcher() @@ -24,47 +23,11 @@ } const saveEventData = async () => { - // any automations that need created from event triggers - const automationsToCreate = tmpValue.filter( - action => action["##eventHandlerType"] === "Trigger Automation" - ) - for (let action of automationsToCreate) { - await createAutomation(action.parameters) - } - dispatch("change", tmpValue) notifications.success("Component actions saved.") drawer.hide() } - // called by the parent modal when actions are saved - const createAutomation = async parameters => { - if (parameters.automationId || !parameters.newAutomationName) { - return - } - try { - let trigger = automationStore.actions.constructBlock( - "TRIGGER", - "APP", - $automationStore.blockDefinitions.TRIGGER.APP - ) - trigger.inputs = { - fields: Object.keys(parameters.fields ?? {}).reduce((fields, key) => { - fields[key] = "string" - return fields - }, {}), - } - const automation = await automationStore.actions.create( - parameters.newAutomationName, - trigger - ) - parameters.automationId = automation._id - delete parameters.newAutomationName - } catch (error) { - notifications.error("Error creating automation") - } - } - $: actionCount = value?.length $: actionText = `${actionCount || "No"} action${ actionCount !== 1 ? "s" : "" diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ScrollTo.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ScrollTo.svelte index 2198a87ecb..49a93d71dd 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ScrollTo.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ScrollTo.svelte @@ -1,6 +1,5 @@ -
{conditionText}
-
- Configure conditions -
+ {conditionText} @@ -48,10 +45,3 @@ - - diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 75fe287b2a..6788925c31 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -2539,7 +2539,8 @@ { "type": "text", "label": "Default value", - "key": "defaultValue" + "key": "defaultValue", + "supportsConditions": false }, { "type": "event", @@ -2629,7 +2630,8 @@ { "type": "text", "label": "Default value", - "key": "defaultValue" + "key": "defaultValue", + "supportsConditions": false }, { "type": "event", @@ -2685,7 +2687,8 @@ { "type": "text", "label": "Default value", - "key": "defaultValue" + "key": "defaultValue", + "supportsConditions": false }, { "type": "event", @@ -2736,7 +2739,8 @@ { "type": "text", "label": "Default value", - "key": "defaultValue" + "key": "defaultValue", + "supportsConditions": false }, { "type": "event", @@ -2841,7 +2845,8 @@ { "type": "text", "label": "Default value", - "key": "defaultValue" + "key": "defaultValue", + "supportsConditions": false }, { "type": "boolean", @@ -2960,7 +2965,8 @@ { "type": "text", "label": "Default value", - "key": "defaultValue" + "key": "defaultValue", + "supportsConditions": false }, { "type": "event", @@ -3143,7 +3149,8 @@ { "type": "text", "label": "Default value", - "key": "defaultValue" + "key": "defaultValue", + "supportsConditions": false }, { "type": "event", @@ -3200,7 +3207,8 @@ { "type": "text", "label": "Default value", - "key": "defaultValue" + "key": "defaultValue", + "supportsConditions": false }, { "type": "event", @@ -3301,7 +3309,8 @@ { "type": "text", "label": "Default value", - "key": "defaultValue" + "key": "defaultValue", + "supportsConditions": false }, { "type": "event", @@ -3355,7 +3364,8 @@ { "type": "text", "label": "Default value", - "key": "defaultValue" + "key": "defaultValue", + "supportsConditions": false }, { "type": "boolean", @@ -3622,7 +3632,8 @@ { "type": "text", "label": "Default value", - "key": "defaultValue" + "key": "defaultValue", + "supportsConditions": false }, { "type": "event", @@ -3689,7 +3700,8 @@ { "type": "text", "label": "Default value", - "key": "defaultValue" + "key": "defaultValue", + "supportsConditions": false }, { "type": "event", From c5accec85cfe0118d3ff477b448e5f9713488243 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Fri, 22 Sep 2023 13:24:19 +0100 Subject: [PATCH 41/66] Hide popover when viewing published app (#11858) * Hide popover when viewing published app * Refactor --- packages/builder/src/components/deploy/AppActions.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/deploy/AppActions.svelte b/packages/builder/src/components/deploy/AppActions.svelte index 4b366f2412..7259e7e402 100644 --- a/packages/builder/src/components/deploy/AppActions.svelte +++ b/packages/builder/src/components/deploy/AppActions.svelte @@ -224,10 +224,10 @@ { + appActionPopover.hide() if (isPublished) { viewApp() } else { - appActionPopover.hide() updateAppModal.show() } }} From 043bbf08208d2264137c771c47409e33f3af5a5c Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 22 Sep 2023 12:24:41 +0000 Subject: [PATCH 42/66] Bump version to 2.10.12-alpha.4 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 860d724276..2db56a535e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.10.12-alpha.3", + "version": "2.10.12-alpha.4", "npmClient": "yarn", "packages": [ "packages/*" From 07c719215469be236103187bc7d9c05aca409d0e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 22 Sep 2023 13:29:16 +0100 Subject: [PATCH 43/66] Adding documentation for export endpoint. --- packages/server/specs/openapi.json | 58 ++++++++++++++++++- packages/server/specs/openapi.yaml | 40 ++++++++++++- .../server/specs/resources/application.ts | 11 ++++ .../api/controllers/public/applications.ts | 19 +++++- .../src/api/routes/public/applications.ts | 33 ++++++++++- packages/server/src/definitions/openapi.ts | 34 +++++++++++ 6 files changed, 191 insertions(+), 4 deletions(-) diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json index 3652182f9d..18f9dd4245 100644 --- a/packages/server/specs/openapi.json +++ b/packages/server/specs/openapi.json @@ -613,6 +613,23 @@ "data" ] }, + "appExport": { + "type": "object", + "properties": { + "encryptPassword": { + "description": "An optional password used to encrypt the export.", + "type": "string" + }, + "excludeRows": { + "description": "Set whether the internal table rows should be excluded from the export.", + "type": "boolean" + } + }, + "required": [ + "encryptPassword", + "excludeRows" + ] + }, "row": { "description": "The row to be created/updated, based on the table schema.", "type": "object", @@ -2166,7 +2183,8 @@ "/applications/{appId}/import": { "post": { "operationId": "appImport", - "summary": "Import an app to an existing app", + "summary": "Import an app to an existing app 🔒", + "description": "This endpoint is only available on a business or enterprise license.", "tags": [ "applications" ], @@ -2205,6 +2223,44 @@ } } }, + "/applications/{appId}/export": { + "post": { + "operationId": "appExport", + "summary": "Export an app 🔒", + "description": "This endpoint is only available on a business or enterprise license.", + "tags": [ + "applications" + ], + "parameters": [ + { + "$ref": "#/components/parameters/appIdUrl" + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/appExport" + } + } + } + }, + "responses": { + "200": { + "description": "A gzip tarball containing the app export, encrypted if password provided.", + "content": { + "application/gzip": { + "schema": { + "type": "string", + "format": "binary", + "example": "Tarball containing database and object store contents..." + } + } + } + } + } + } + }, "/applications/search": { "post": { "operationId": "appSearch", diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml index c533ee5a9a..4916141569 100644 --- a/packages/server/specs/openapi.yaml +++ b/packages/server/specs/openapi.yaml @@ -587,6 +587,19 @@ components: - appUrl required: - data + appExport: + type: object + properties: + encryptPassword: + description: An optional password used to encrypt the export. + type: string + excludeRows: + description: Set whether the internal table rows should be excluded from the + export. + type: boolean + required: + - encryptPassword + - excludeRows row: description: The row to be created/updated, based on the table schema. type: object @@ -1766,7 +1779,8 @@ paths: "/applications/{appId}/import": post: operationId: appImport - summary: Import an app to an existing app + summary: Import an app to an existing app 🔒 + description: This endpoint is only available on a business or enterprise license. tags: - applications parameters: @@ -1789,6 +1803,30 @@ paths: responses: "204": description: Application has been updated. + "/applications/{appId}/export": + post: + operationId: appExport + summary: Export an app 🔒 + description: This endpoint is only available on a business or enterprise license. + tags: + - applications + parameters: + - $ref: "#/components/parameters/appIdUrl" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/appExport" + responses: + "200": + description: A gzip tarball containing the app export, encrypted if password + provided. + content: + application/gzip: + schema: + type: string + format: binary + example: Tarball containing database and object store contents... /applications/search: post: operationId: appSearch diff --git a/packages/server/specs/resources/application.ts b/packages/server/specs/resources/application.ts index cd7a68c049..081dd9e72a 100644 --- a/packages/server/specs/resources/application.ts +++ b/packages/server/specs/resources/application.ts @@ -134,4 +134,15 @@ export default new Resource() deploymentOutput: object({ data: deploymentOutputSchema, }), + appExport: object({ + encryptPassword: { + description: "An optional password used to encrypt the export.", + type: "string", + }, + excludeRows: { + description: + "Set whether the internal table rows should be excluded from the export.", + type: "boolean", + }, + }), }) diff --git a/packages/server/src/api/controllers/public/applications.ts b/packages/server/src/api/controllers/public/applications.ts index d9a94ed073..f2168ee98a 100644 --- a/packages/server/src/api/controllers/public/applications.ts +++ b/packages/server/src/api/controllers/public/applications.ts @@ -2,6 +2,7 @@ import { db as dbCore, context } from "@budibase/backend-core" import { search as stringSearch, addRev } from "./utils" import * as controller from "../application" import * as deployController from "../deploy" +import * as backupController from "../backup" import { Application } from "../../../definitions/common" import { UserCtx } from "@budibase/types" import { Next } from "koa" @@ -94,8 +95,10 @@ export async function publish(ctx: UserCtx, next: Next) { } export async function importToApp(ctx: UserCtx, next: Next) { + if (!ctx.request.files?.appExport) { + ctx.throw(400, "Must provide app export file for import.") + } await context.doInAppContext(ctx.params.appId, async () => { - // TODO: paid control await controller.importToApp(ctx) ctx.body = undefined ctx.status = 204 @@ -103,6 +106,19 @@ export async function importToApp(ctx: UserCtx, next: Next) { }) } +export async function exportApp(ctx: UserCtx, next: Next) { + const { encryptPassword, excludeRows } = ctx.request.body + await context.doInAppContext(ctx.params.appId, async () => { + // make sure no other inputs + ctx.request.body = { + encryptPassword, + excludeRows, + } + await backupController.exportAppDump(ctx) + await next() + }) +} + export default { create, update, @@ -112,4 +128,5 @@ export default { unpublish, publish, importToApp, + exportApp, } diff --git a/packages/server/src/api/routes/public/applications.ts b/packages/server/src/api/routes/public/applications.ts index 6fff83bd07..5410eb7dcf 100644 --- a/packages/server/src/api/routes/public/applications.ts +++ b/packages/server/src/api/routes/public/applications.ts @@ -142,7 +142,8 @@ write.push( * /applications/{appId}/import: * post: * operationId: appImport - * summary: Import an app to an existing app + * summary: Import an app to an existing app 🔒 + * description: This endpoint is only available on a business or enterprise license. * tags: * - applications * parameters: @@ -170,6 +171,36 @@ write.push( new Endpoint("post", "/applications/:appId/import", controller.importToApp) ) +/** + * @openapi + * /applications/{appId}/export: + * post: + * operationId: appExport + * summary: Export an app 🔒 + * description: This endpoint is only available on a business or enterprise license. + * tags: + * - applications + * parameters: + * - $ref: '#/components/parameters/appIdUrl' + * requestBody: + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/appExport' + * responses: + * 200: + * description: A gzip tarball containing the app export, encrypted if password provided. + * content: + * application/gzip: + * schema: + * type: string + * format: binary + * example: Tarball containing database and object store contents... + */ +read.push( + new Endpoint("post", "/applications/:appId/export", controller.exportApp) +) + /** * @openapi * /applications/{appId}: diff --git a/packages/server/src/definitions/openapi.ts b/packages/server/src/definitions/openapi.ts index 9057e39005..52434494e5 100644 --- a/packages/server/src/definitions/openapi.ts +++ b/packages/server/src/definitions/openapi.ts @@ -19,8 +19,13 @@ export interface paths { post: operations["appPublish"]; }; "/applications/{appId}/import": { + /** This endpoint is only available on a business or enterprise license. */ post: operations["appImport"]; }; + "/applications/{appId}/export": { + /** This endpoint is only available on a business or enterprise license. */ + post: operations["appExport"]; + }; "/applications/search": { /** Based on application properties (currently only name) search for applications. */ post: operations["appSearch"]; @@ -161,6 +166,12 @@ export interface components { appUrl: string; }; }; + appExport: { + /** @description An optional password used to encrypt the export. */ + encryptPassword: string; + /** @description Set whether the internal table rows should be excluded from the export. */ + excludeRows: boolean; + }; /** @description The row to be created/updated, based on the table schema. */ row: { [key: string]: unknown }; searchOutput: { @@ -892,6 +903,7 @@ export interface operations { }; }; }; + /** This endpoint is only available on a business or enterprise license. */ appImport: { parameters: { path: { @@ -917,6 +929,28 @@ export interface operations { }; }; }; + /** This endpoint is only available on a business or enterprise license. */ + appExport: { + parameters: { + path: { + /** The ID of the app which this request is targeting. */ + appId: components["parameters"]["appIdUrl"]; + }; + }; + responses: { + /** A gzip tarball containing the app export, encrypted if password provided. */ + 200: { + content: { + "application/gzip": string; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["appExport"]; + }; + }; + }; /** Based on application properties (currently only name) search for applications. */ appSearch: { responses: { From 799f62d218cc00684ad2fdf2d11e0f76090560d0 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 22 Sep 2023 15:10:23 +0000 Subject: [PATCH 44/66] Bump version to 2.10.12-alpha.5 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 2db56a535e..124446321f 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.10.12-alpha.4", + "version": "2.10.12-alpha.5", "npmClient": "yarn", "packages": [ "packages/*" From 02ab0a134e3aaad0a9350a8b28233dff8a616efa Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 22 Sep 2023 18:17:19 +0200 Subject: [PATCH 45/66] Update ts to 5.x --- package.json | 2 +- packages/backend-core/package.json | 2 +- packages/builder/package.json | 2 +- packages/cli/package.json | 2 +- packages/pro | 2 +- packages/server/package.json | 2 +- packages/shared-core/package.json | 3 +-- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 2 +- qa-core/package.json | 2 +- yarn.lock | 8 ++++---- 12 files changed, 15 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 6df4105e25..68dd6360e0 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "rimraf": "^3.0.2", "rollup-plugin-replace": "^2.2.0", "svelte": "^3.38.2", - "typescript": "4.7.3", + "typescript": "5.2.2", "@babel/core": "^7.22.5", "@babel/eslint-parser": "^7.22.5", "@babel/preset-env": "^7.22.5", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 739469b49a..1c94163d93 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -87,7 +87,7 @@ "timekeeper": "2.2.0", "ts-node": "10.8.1", "tsconfig-paths": "4.0.0", - "typescript": "4.7.3" + "typescript": "5.2.2" }, "nx": { "targets": { diff --git a/packages/builder/package.json b/packages/builder/package.json index 43f1ae3bff..a53f3509cf 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -108,7 +108,7 @@ "svelte-jester": "^1.3.2", "ts-node": "10.8.1", "tsconfig-paths": "4.0.0", - "typescript": "4.7.3", + "typescript": "5.2.2", "vite": "^3.0.8", "vite-plugin-static-copy": "^0.16.0", "vitest": "^0.29.2" diff --git a/packages/cli/package.json b/packages/cli/package.json index 6d4d78b7d7..04eeb523cd 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -62,6 +62,6 @@ "eslint": "^7.20.0", "renamer": "^4.0.0", "ts-node": "^10.9.1", - "typescript": "4.7.3" + "typescript": "5.2.2" } } diff --git a/packages/pro b/packages/pro index 4638ae916e..c1bf60cb8f 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 4638ae916e55ce89166095578cbd01745d0ee9ee +Subproject commit c1bf60cb8f00b488082db4a2fc50528e8c2d3469 diff --git a/packages/server/package.json b/packages/server/package.json index 8851237d8e..4d60d61331 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -174,7 +174,7 @@ "timekeeper": "2.2.0", "ts-node": "10.8.1", "tsconfig-paths": "4.0.0", - "typescript": "4.7.3", + "typescript": "5.2.2", "update-dotenv": "1.1.1" }, "optionalDependencies": { diff --git a/packages/shared-core/package.json b/packages/shared-core/package.json index 160a42f086..9d03df21e0 100644 --- a/packages/shared-core/package.json +++ b/packages/shared-core/package.json @@ -19,7 +19,7 @@ "devDependencies": { "concurrently": "^7.6.0", "rimraf": "3.0.2", - "typescript": "4.7.3" + "typescript": "5.2.2" }, "nx": { "targets": { @@ -43,7 +43,6 @@ } ] } - } } } diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index aee60f4732..317b0fc3af 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -45,6 +45,6 @@ "rollup-plugin-node-globals": "^1.4.0", "rollup-plugin-node-resolve": "^5.2.0", "rollup-plugin-terser": "^7.0.2", - "typescript": "4.7.3" + "typescript": "5.2.2" } } diff --git a/packages/types/package.json b/packages/types/package.json index df49ec3cea..ad190bb1a6 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -24,7 +24,7 @@ "concurrently": "^7.6.0", "koa-body": "4.2.0", "rimraf": "3.0.2", - "typescript": "4.7.3" + "typescript": "5.2.2" }, "dependencies": { "scim-patch": "^0.7.0" diff --git a/packages/worker/package.json b/packages/worker/package.json index a1344d89e3..efb22d97f8 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -100,7 +100,7 @@ "timekeeper": "2.2.0", "ts-node": "10.8.1", "tsconfig-paths": "4.0.0", - "typescript": "4.7.3", + "typescript": "5.2.2", "update-dotenv": "1.1.1" }, "nx": { diff --git a/qa-core/package.json b/qa-core/package.json index 4954991f67..3c789d89e6 100644 --- a/qa-core/package.json +++ b/qa-core/package.json @@ -38,7 +38,7 @@ "ts-jest": "29.1.1", "ts-node": "10.8.1", "tsconfig-paths": "4.0.0", - "typescript": "4.7.3" + "typescript": "5.2.2" }, "dependencies": { "@budibase/backend-core": "^2.3.17", diff --git a/yarn.lock b/yarn.lock index d777dca0bd..bdb71ef06b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -24120,10 +24120,10 @@ typeof@^1.0.0: resolved "https://registry.yarnpkg.com/typeof/-/typeof-1.0.0.tgz#9c84403f2323ae5399167275497638ea1d2f2440" integrity sha512-Pze0mIxYXhaJdpw1ayMzOA7rtGr1OmsTY/Z+FWtRKIqXFz6aoDLjqdbWE/tcIBSC8nhnVXiRrEXujodR/xiFAA== -typescript@4.7.3: - version "4.7.3" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.3.tgz#8364b502d5257b540f9de4c40be84c98e23a129d" - integrity sha512-WOkT3XYvrpXx4vMMqlD+8R8R37fZkjyLGlxavMc4iB8lrl8L0DeTcHbYgw/v0N/z9wAFsgBhcsF0ruoySS22mA== +typescript@5.2.2: + version "5.2.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78" + integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w== "typescript@>=3 < 6": version "5.0.4" From 124ff401595cdfcbc75e9532d99b52f06874f5b1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 22 Sep 2023 18:24:00 +0200 Subject: [PATCH 46/66] Fix types --- packages/cli/src/plugins/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/plugins/index.ts b/packages/cli/src/plugins/index.ts index 45e024be7f..c29de2a297 100644 --- a/packages/cli/src/plugins/index.ts +++ b/packages/cli/src/plugins/index.ts @@ -49,8 +49,8 @@ async function askAboutTopLevel(name: string) { } } -async function init(opts: PluginOpts) { - const type = opts["init"] || (opts as PluginType) +async function init(opts: PluginOpts | PluginType) { + const type = (opts as PluginOpts).init || (opts as PluginType) if (!type || !PLUGIN_TYPE_ARR.includes(type)) { console.log( error( From fe5cc6878bcd68cf6822046923828ba899a3af9e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 22 Sep 2023 17:38:34 +0100 Subject: [PATCH 47/66] Adding licensing checks to import/export API. --- .../tests/core/utilities/mocks/licenses.ts | 4 +-- packages/pro | 2 +- .../server/src/api/controllers/application.ts | 9 +++++- .../api/controllers/public/applications.ts | 32 +++++-------------- .../src/api/routes/public/tests/users.spec.ts | 6 ++-- packages/types/src/sdk/licensing/feature.ts | 2 +- 6 files changed, 23 insertions(+), 32 deletions(-) diff --git a/packages/backend-core/tests/core/utilities/mocks/licenses.ts b/packages/backend-core/tests/core/utilities/mocks/licenses.ts index 309f0fd159..758fd6bf9a 100644 --- a/packages/backend-core/tests/core/utilities/mocks/licenses.ts +++ b/packages/backend-core/tests/core/utilities/mocks/licenses.ts @@ -86,8 +86,8 @@ export const useAuditLogs = () => { return useFeature(Feature.AUDIT_LOGS) } -export const usePublicApiUserRoles = () => { - return useFeature(Feature.USER_ROLE_PUBLIC_API) +export const useExpandedPublicApi = () => { + return useFeature(Feature.EXPANDED_PUBLIC_API) } export const useScimIntegration = () => { diff --git a/packages/pro b/packages/pro index 4638ae916e..6c193aa50b 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 4638ae916e55ce89166095578cbd01745d0ee9ee +Subproject commit 6c193aa50bc5eca7757db886c8457f516dafc1b8 diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 8d4d1c673a..99241a4831 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -584,7 +584,14 @@ export async function importToApp(ctx: UserCtx) { ctx.throw(400, "Must only supply one app export") } const fileAttributes = { type: appExport.type!, path: appExport.path! } - await sdk.applications.updateWithExport(appId, fileAttributes, password) + try { + await sdk.applications.updateWithExport(appId, fileAttributes, password) + } catch (err: any) { + ctx.throw( + 500, + `Unable to perform update, please retry - ${err?.message || err}` + ) + } ctx.body = { message: "app updated" } } diff --git a/packages/server/src/api/controllers/public/applications.ts b/packages/server/src/api/controllers/public/applications.ts index f2168ee98a..316da72377 100644 --- a/packages/server/src/api/controllers/public/applications.ts +++ b/packages/server/src/api/controllers/public/applications.ts @@ -6,6 +6,7 @@ import * as backupController from "../backup" import { Application } from "../../../definitions/common" import { UserCtx } from "@budibase/types" import { Next } from "koa" +import { sdk as proSdk } from "@budibase/pro" function fixAppID(app: Application, params: any) { if (!params) { @@ -94,30 +95,13 @@ export async function publish(ctx: UserCtx, next: Next) { }) } -export async function importToApp(ctx: UserCtx, next: Next) { - if (!ctx.request.files?.appExport) { - ctx.throw(400, "Must provide app export file for import.") - } - await context.doInAppContext(ctx.params.appId, async () => { - await controller.importToApp(ctx) - ctx.body = undefined - ctx.status = 204 - await next() - }) -} - -export async function exportApp(ctx: UserCtx, next: Next) { - const { encryptPassword, excludeRows } = ctx.request.body - await context.doInAppContext(ctx.params.appId, async () => { - // make sure no other inputs - ctx.request.body = { - encryptPassword, - excludeRows, - } - await backupController.exportAppDump(ctx) - await next() - }) -} +// get licensed endpoints from pro +export const importToApp = proSdk.publicApi.applications.buildImportFn( + controller.importToApp +) +export const exportApp = proSdk.publicApi.applications.buildExportFn( + backupController.exportAppDump +) export default { create, diff --git a/packages/server/src/api/routes/public/tests/users.spec.ts b/packages/server/src/api/routes/public/tests/users.spec.ts index c81acca1df..9d38dc4791 100644 --- a/packages/server/src/api/routes/public/tests/users.spec.ts +++ b/packages/server/src/api/routes/public/tests/users.spec.ts @@ -92,7 +92,7 @@ describe("no user role update in free", () => { describe("no user role update in business", () => { beforeAll(() => { updateMock() - mocks.licenses.usePublicApiUserRoles() + mocks.licenses.useExpandedPublicApi() }) it("should allow 'roles' to be updated", async () => { @@ -105,7 +105,7 @@ describe("no user role update in business", () => { }) it("should allow 'admin' to be updated", async () => { - mocks.licenses.usePublicApiUserRoles() + mocks.licenses.useExpandedPublicApi() const res = await makeRequest("post", "/users", { ...base(), admin: { global: true }, @@ -115,7 +115,7 @@ describe("no user role update in business", () => { }) it("should allow 'builder' to be updated", async () => { - mocks.licenses.usePublicApiUserRoles() + mocks.licenses.useExpandedPublicApi() const res = await makeRequest("post", "/users", { ...base(), builder: { global: true }, diff --git a/packages/types/src/sdk/licensing/feature.ts b/packages/types/src/sdk/licensing/feature.ts index bd3a6583bf..732a4a6c77 100644 --- a/packages/types/src/sdk/licensing/feature.ts +++ b/packages/types/src/sdk/licensing/feature.ts @@ -11,7 +11,7 @@ export enum Feature { SYNC_AUTOMATIONS = "syncAutomations", APP_BUILDERS = "appBuilders", OFFLINE = "offline", - USER_ROLE_PUBLIC_API = "userRolePublicApi", + EXPANDED_PUBLIC_API = "expandedPublicApi", VIEW_PERMISSIONS = "viewPermissions", } From c0a2940d8b84f9244448a60289031bc14b0d2a5e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 22 Sep 2023 18:43:18 +0100 Subject: [PATCH 48/66] Adding basic tests for public API app export/import. --- packages/pro | 2 +- .../routes/public/tests/applications.spec.ts | 81 +++++++++++++++++++ .../src/api/routes/public/tests/utils.ts | 67 ++++++++++++--- 3 files changed, 137 insertions(+), 13 deletions(-) create mode 100644 packages/server/src/api/routes/public/tests/applications.spec.ts diff --git a/packages/pro b/packages/pro index 6c193aa50b..874321ba70 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 6c193aa50bc5eca7757db886c8457f516dafc1b8 +Subproject commit 874321ba70cfe6ae3d7515b3b7d66185c4fe03cc diff --git a/packages/server/src/api/routes/public/tests/applications.spec.ts b/packages/server/src/api/routes/public/tests/applications.spec.ts new file mode 100644 index 0000000000..fae049da75 --- /dev/null +++ b/packages/server/src/api/routes/public/tests/applications.spec.ts @@ -0,0 +1,81 @@ +import * as setup from "../../tests/utilities" +import { + generateMakeRequest, + generateMakeRequestWithFormData, + MakeRequestResponse, + MakeRequestWithFormDataResponse, +} from "./utils" +import { User } from "@budibase/types" +import { join } from "path" +import { mocks } from "@budibase/backend-core/tests" + +const PASSWORD = "testtest" +const NO_LICENSE_MSG = "Endpoint unavailable, license required." + +let config = setup.getConfig() +let apiKey: string, + globalUser: User, + makeRequest: MakeRequestResponse, + makeRequestFormData: MakeRequestWithFormDataResponse + +beforeAll(async () => { + await config.init() + globalUser = await config.globalUser() + apiKey = await config.generateApiKey(globalUser._id) + makeRequest = generateMakeRequest(apiKey) + makeRequestFormData = generateMakeRequestWithFormData(apiKey) +}) + +afterAll(setup.afterAll) + +describe("check export/import", () => { + async function runExport() { + return await makeRequest("post", `/applications/${config.appId}/export`, { + encryptionPassword: PASSWORD, + excludeRows: true, + }) + } + + async function runImport() { + const pathToExport = join( + __dirname, + "..", + "..", + "tests", + "assets", + "export.tar.gz" + ) + return await makeRequestFormData( + "post", + `/applications/${config.appId}/import`, + { + encryptionPassword: PASSWORD, + appExport: { path: pathToExport }, + } + ) + } + + it("check licensing for export", async () => { + const res = await runExport() + expect(res.status).toBe(403) + expect(res.body.message).toBe(NO_LICENSE_MSG) + }) + + it("check licensing for import", async () => { + const res = await runImport() + expect(res.status).toBe(403) + expect(res.body.message).toBe(NO_LICENSE_MSG) + }) + + it("should be able to export app", async () => { + mocks.licenses.useExpandedPublicApi() + const res = await runExport() + expect(res.status).toBe(200) + }) + + it("should be able to import app", async () => { + mocks.licenses.useExpandedPublicApi() + const res = await runImport() + expect(res.status).toBe(204) + }) +}) diff --git a/packages/server/src/api/routes/public/tests/utils.ts b/packages/server/src/api/routes/public/tests/utils.ts index 755e2d659f..1b57682af9 100644 --- a/packages/server/src/api/routes/public/tests/utils.ts +++ b/packages/server/src/api/routes/public/tests/utils.ts @@ -11,6 +11,32 @@ export type MakeRequestResponse = ( intAppId?: string ) => Promise +export type MakeRequestWithFormDataResponse = ( + method: HttpMethod, + endpoint: string, + fields: Record, + intAppId?: string +) => Promise + +function base( + apiKey: string, + endpoint: string, + intAppId: string | null, + isInternal: boolean +) { + const extraHeaders: any = { + "x-budibase-api-key": apiKey, + } + if (intAppId) { + extraHeaders["x-budibase-app-id"] = intAppId + } + + const url = isInternal + ? endpoint + : checkSlashesInUrl(`/api/public/v1/${endpoint}`) + return { headers: extraHeaders, url } +} + export function generateMakeRequest( apiKey: string, isInternal = false @@ -23,18 +49,8 @@ export function generateMakeRequest( body?: any, intAppId: string | null = config.getAppId() ) => { - const extraHeaders: any = { - "x-budibase-api-key": apiKey, - } - if (intAppId) { - extraHeaders["x-budibase-app-id"] = intAppId - } - - const url = isInternal - ? endpoint - : checkSlashesInUrl(`/api/public/v1/${endpoint}`) - - const req = request[method](url).set(config.defaultHeaders(extraHeaders)) + const { headers, url } = base(apiKey, endpoint, intAppId, isInternal) + const req = request[method](url).set(config.defaultHeaders(headers)) if (body) { req.send(body) } @@ -43,3 +59,30 @@ export function generateMakeRequest( return res } } + +export function generateMakeRequestWithFormData( + apiKey: string, + isInternal = false +): MakeRequestWithFormDataResponse { + const request = setup.getRequest()! + const config = setup.getConfig()! + return async ( + method: HttpMethod, + endpoint: string, + fields: Record, + intAppId: string | null = config.getAppId() + ) => { + const { headers, url } = base(apiKey, endpoint, intAppId, isInternal) + const req = request[method](url).set(config.defaultHeaders(headers)) + for (let [field, value] of Object.entries(fields)) { + if (typeof value === "string") { + req.field(field, value) + } else { + req.attach(field, value.path) + } + } + const res = await req + expect(res.body).toBeDefined() + return res + } +} From aee590bbba0f6b4c376f532c2e18f6bbd9a4d487 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 25 Sep 2023 09:42:48 +0200 Subject: [PATCH 49/66] Update lint --- package.json | 2 +- packages/server/package.json | 1 - packages/worker/package.json | 1 - yarn.lock | 81 +++++++++++++++++++++--------------- 4 files changed, 48 insertions(+), 37 deletions(-) diff --git a/package.json b/package.json index 68dd6360e0..d758888eab 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "@esbuild-plugins/tsconfig-paths": "^0.1.2", "@nx/js": "16.4.3", "@rollup/plugin-json": "^4.0.2", - "@typescript-eslint/parser": "5.45.0", + "@typescript-eslint/parser": "6.7.2", "esbuild": "^0.18.17", "esbuild-node-externals": "^1.8.0", "eslint": "^8.44.0", diff --git a/packages/server/package.json b/packages/server/package.json index 4d60d61331..c4d4f883b7 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -153,7 +153,6 @@ "@types/server-destroy": "1.0.1", "@types/supertest": "2.0.12", "@types/tar": "6.1.5", - "@typescript-eslint/parser": "5.45.0", "apidoc": "0.50.4", "babel-jest": "29.6.2", "copyfiles": "2.4.1", diff --git a/packages/worker/package.json b/packages/worker/package.json index efb22d97f8..4d1f1581b8 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -88,7 +88,6 @@ "@types/server-destroy": "1.0.1", "@types/supertest": "2.0.12", "@types/uuid": "8.3.4", - "@typescript-eslint/parser": "5.45.0", "copyfiles": "2.4.1", "eslint": "6.8.0", "jest": "29.6.2", diff --git a/yarn.lock b/yarn.lock index bdb71ef06b..34441ee45f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6696,51 +6696,52 @@ dependencies: "@types/yargs-parser" "*" -"@typescript-eslint/parser@5.45.0": - version "5.45.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-5.45.0.tgz#b18a5f6b3cf1c2b3e399e9d2df4be40d6b0ddd0e" - integrity sha512-brvs/WSM4fKUmF5Ot/gEve6qYiCMjm6w4HkHPfS6ZNmxTS0m0iNN4yOChImaCkqc1hRwFGqUyanMXuGal6oyyQ== +"@typescript-eslint/parser@6.7.2": + version "6.7.2" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-6.7.2.tgz#e0ae93771441b9518e67d0660c79e3a105497af4" + integrity sha512-KA3E4ox0ws+SPyxQf9iSI25R6b4Ne78ORhNHeVKrPQnoYsb9UhieoiRoJgrzgEeKGOXhcY1i8YtOeCHHTDa6Fw== dependencies: - "@typescript-eslint/scope-manager" "5.45.0" - "@typescript-eslint/types" "5.45.0" - "@typescript-eslint/typescript-estree" "5.45.0" + "@typescript-eslint/scope-manager" "6.7.2" + "@typescript-eslint/types" "6.7.2" + "@typescript-eslint/typescript-estree" "6.7.2" + "@typescript-eslint/visitor-keys" "6.7.2" debug "^4.3.4" -"@typescript-eslint/scope-manager@5.45.0": - version "5.45.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-5.45.0.tgz#7a4ac1bfa9544bff3f620ab85947945938319a96" - integrity sha512-noDMjr87Arp/PuVrtvN3dXiJstQR1+XlQ4R1EvzG+NMgXi8CuMCXpb8JqNtFHKceVSQ985BZhfRdowJzbv4yKw== +"@typescript-eslint/scope-manager@6.7.2": + version "6.7.2" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-6.7.2.tgz#cf59a2095d2f894770c94be489648ad1c78dc689" + integrity sha512-bgi6plgyZjEqapr7u2mhxGR6E8WCzKNUFWNh6fkpVe9+yzRZeYtDTbsIBzKbcxI+r1qVWt6VIoMSNZ4r2A+6Yw== dependencies: - "@typescript-eslint/types" "5.45.0" - "@typescript-eslint/visitor-keys" "5.45.0" + "@typescript-eslint/types" "6.7.2" + "@typescript-eslint/visitor-keys" "6.7.2" "@typescript-eslint/types@4.33.0": version "4.33.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-4.33.0.tgz#a1e59036a3b53ae8430ceebf2a919dc7f9af6d72" integrity sha512-zKp7CjQzLQImXEpLt2BUw1tvOMPfNoTAfb8l51evhYbOEEzdWyQNmHWWGPR6hwKJDAi+1VXSBmnhL9kyVTTOuQ== -"@typescript-eslint/types@5.45.0": - version "5.45.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.45.0.tgz#794760b9037ee4154c09549ef5a96599621109c5" - integrity sha512-QQij+u/vgskA66azc9dCmx+rev79PzX8uDHpsqSjEFtfF2gBUTRCpvYMh2gw2ghkJabNkPlSUCimsyBEQZd1DA== - "@typescript-eslint/types@5.53.0": version "5.53.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.53.0.tgz#f79eca62b97e518ee124086a21a24f3be267026f" integrity sha512-5kcDL9ZUIP756K6+QOAfPkigJmCPHcLN7Zjdz76lQWWDdzfOhZDTj1irs6gPBKiXx5/6O3L0+AvupAut3z7D2A== -"@typescript-eslint/typescript-estree@5.45.0": - version "5.45.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-5.45.0.tgz#f70a0d646d7f38c0dfd6936a5e171a77f1e5291d" - integrity sha512-maRhLGSzqUpFcZgXxg1qc/+H0bT36lHK4APhp0AEUVrpSwXiRAomm/JGjSG+kNUio5kAa3uekCYu/47cnGn5EQ== +"@typescript-eslint/types@6.7.2": + version "6.7.2" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-6.7.2.tgz#75a615a6dbeca09cafd102fe7f465da1d8a3c066" + integrity sha512-flJYwMYgnUNDAN9/GAI3l8+wTmvTYdv64fcH8aoJK76Y+1FCZ08RtI5zDerM/FYT5DMkAc+19E4aLmd5KqdFyg== + +"@typescript-eslint/typescript-estree@6.7.2": + version "6.7.2" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-6.7.2.tgz#ce5883c23b581a5caf878af641e49dd0349238c7" + integrity sha512-kiJKVMLkoSciGyFU0TOY0fRxnp9qq1AzVOHNeN1+B9erKFCJ4Z8WdjAkKQPP+b1pWStGFqezMLltxO+308dJTQ== dependencies: - "@typescript-eslint/types" "5.45.0" - "@typescript-eslint/visitor-keys" "5.45.0" + "@typescript-eslint/types" "6.7.2" + "@typescript-eslint/visitor-keys" "6.7.2" debug "^4.3.4" globby "^11.1.0" is-glob "^4.0.3" - semver "^7.3.7" - tsutils "^3.21.0" + semver "^7.5.4" + ts-api-utils "^1.0.1" "@typescript-eslint/typescript-estree@^4.33.0": version "4.33.0" @@ -6776,14 +6777,6 @@ "@typescript-eslint/types" "4.33.0" eslint-visitor-keys "^2.0.0" -"@typescript-eslint/visitor-keys@5.45.0": - version "5.45.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-5.45.0.tgz#e0d160e9e7fdb7f8da697a5b78e7a14a22a70528" - integrity sha512-jc6Eccbn2RtQPr1s7th6jJWQHBHI6GBVQkCHoJFQ5UreaKm59Vxw+ynQUPPY2u2Amquc+7tmEoC2G52ApsGNNg== - dependencies: - "@typescript-eslint/types" "5.45.0" - eslint-visitor-keys "^3.3.0" - "@typescript-eslint/visitor-keys@5.53.0": version "5.53.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-5.53.0.tgz#8a5126623937cdd909c30d8fa72f79fa56cc1a9f" @@ -6792,6 +6785,14 @@ "@typescript-eslint/types" "5.53.0" eslint-visitor-keys "^3.3.0" +"@typescript-eslint/visitor-keys@6.7.2": + version "6.7.2" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-6.7.2.tgz#4cb2bd786f1f459731b0ad1584c9f73e1c7a4d5c" + integrity sha512-uVw9VIMFBUTz8rIeaUT3fFe8xIUx8r4ywAdlQv1ifH+6acn/XF8Y6rwJ7XNmkNMDrTW+7+vxFFPIF40nJCVsMQ== + dependencies: + "@typescript-eslint/types" "6.7.2" + eslint-visitor-keys "^3.4.1" + "@vitest/expect@0.29.8": version "0.29.8" resolved "https://registry.yarnpkg.com/@vitest/expect/-/expect-0.29.8.tgz#6ecdd031b4ea8414717d10b65ccd800908384612" @@ -21927,6 +21928,13 @@ semver@^7.3.2, semver@^7.3.7, semver@^7.3.8: dependencies: lru-cache "^6.0.0" +semver@^7.5.4: + version "7.5.4" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e" + integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== + dependencies: + lru-cache "^6.0.0" + semver@~2.3.1: version "2.3.2" resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52" @@ -23882,6 +23890,11 @@ triple-beam@^1.3.0: resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9" integrity sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw== +ts-api-utils@^1.0.1: + version "1.0.3" + resolved "https://registry.yarnpkg.com/ts-api-utils/-/ts-api-utils-1.0.3.tgz#f12c1c781d04427313dbac808f453f050e54a331" + integrity sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg== + ts-graphviz@^1.5.0: version "1.5.4" resolved "https://registry.yarnpkg.com/ts-graphviz/-/ts-graphviz-1.5.4.tgz#61a3059afeac4f6d4be3c6729a4d88546ca9e095" From 2530ebda2bfd5e75bdff87ed5eb083c646826010 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 25 Sep 2023 09:48:27 +0200 Subject: [PATCH 50/66] Fix tests --- .../backend-core/tests/core/utilities/structures/users.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/tests/core/utilities/structures/users.ts b/packages/backend-core/tests/core/utilities/structures/users.ts index 420a9fde0e..66d23696e0 100644 --- a/packages/backend-core/tests/core/utilities/structures/users.ts +++ b/packages/backend-core/tests/core/utilities/structures/users.ts @@ -10,14 +10,13 @@ import { authDetails } from "./sso" import { uuid } from "./common" import { generator } from "./generator" import { tenant } from "." -import { generateGlobalUserID } from "../../../../src/docIds" export const newEmail = () => { return `${uuid()}@test.com` } export const user = (userProps?: Partial>): User => { - const userId = userProps?._id || generateGlobalUserID() + const userId = userProps?._id return { _id: userId, userId, @@ -53,7 +52,7 @@ export const adminOnlyUser = (userProps?: any): AdminOnlyUser => { } } -export const builderUser = (userProps?: any): BuilderUser => { +export const builderUser = (userProps?: Partial): BuilderUser => { return { ...user(userProps), builder: { From f3309a742cd35cc71befa13c0ef2c622275999f0 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Mon, 25 Sep 2023 11:50:01 +0100 Subject: [PATCH 51/66] Add onChange to linked rows, longform text area, and multi select arrays in automations (#11874) * Add onChange to LinkedRowSelector * Add onChange to Multiselect array * Add onChange to TextArea * lint --- .../automation/SetupPanel/RowSelector.svelte | 5 ++--- .../automation/SetupPanel/RowSelectorTypes.svelte | 13 +++++++++++-- .../src/components/common/LinkedRowSelector.svelte | 9 ++++++++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte index 1080fc7305..5039c37ece 100644 --- a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte +++ b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte @@ -83,8 +83,7 @@ if (Array.isArray(value)) { return value } - - return [value] + return value.split(",").map(x => x.trim()) } if (type === "json") { @@ -146,7 +145,7 @@ placeholder={placeholders[schema.type]} panel={AutomationBindingPanel} value={Array.isArray(value[field]) - ? value[field].join(" ") + ? value[field].join(",") : value[field]} on:change={e => onChange(e, field, schema.type)} label={field} diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte index 0e5e13ce1e..8ac9226fa0 100644 --- a/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte +++ b/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte @@ -55,9 +55,14 @@ bind:value={value[field]} label={field} options={schema.constraints.inclusion} + on:change={e => onChange(e, field)} /> {:else if schema.type === "longform"} -