From 4acb8fae99b39ef03a1817af83ecfcda1926c858 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 13 Jan 2025 11:09:22 +0000 Subject: [PATCH 001/112] Convert portal templates store --- packages/builder/src/stores/portal/templates.js | 16 ---------------- packages/builder/src/stores/portal/templates.ts | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 16 deletions(-) delete mode 100644 packages/builder/src/stores/portal/templates.js create mode 100644 packages/builder/src/stores/portal/templates.ts diff --git a/packages/builder/src/stores/portal/templates.js b/packages/builder/src/stores/portal/templates.js deleted file mode 100644 index 2ff23866ab..0000000000 --- a/packages/builder/src/stores/portal/templates.js +++ /dev/null @@ -1,16 +0,0 @@ -import { writable } from "svelte/store" -import { API } from "@/api" - -export function templatesStore() { - const { subscribe, set } = writable([]) - - return { - subscribe, - load: async () => { - const templates = await API.getAppTemplates() - set(templates) - }, - } -} - -export const templates = templatesStore() diff --git a/packages/builder/src/stores/portal/templates.ts b/packages/builder/src/stores/portal/templates.ts new file mode 100644 index 0000000000..caf1a54ced --- /dev/null +++ b/packages/builder/src/stores/portal/templates.ts @@ -0,0 +1,16 @@ +import { API } from "@/api" +import { BudiStore } from "../BudiStore" +import { TemplateMetadata } from "@budibase/types" + +class TemplateStore extends BudiStore { + constructor() { + super([]) + } + + async load() { + const templates = await API.getAppTemplates() + this.set(templates) + } +} + +export const templates = new TemplateStore() From 085617a5220be1dcbc2d31cde6cb645709b6cf67 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 13 Jan 2025 11:17:11 +0000 Subject: [PATCH 002/112] Convert portal temporal store to TS --- .../portal/licensing/LicensingOverlays.svelte | 4 +- .../portal/licensing/licensingBanners.js | 4 +- .../builder/src/stores/portal/temporal.js | 45 ---------------- .../builder/src/stores/portal/temporal.ts | 53 +++++++++++++++++++ 4 files changed, 57 insertions(+), 49 deletions(-) delete mode 100644 packages/builder/src/stores/portal/temporal.js create mode 100644 packages/builder/src/stores/portal/temporal.ts diff --git a/packages/builder/src/components/portal/licensing/LicensingOverlays.svelte b/packages/builder/src/components/portal/licensing/LicensingOverlays.svelte index 3de2b464e0..8280251839 100644 --- a/packages/builder/src/components/portal/licensing/LicensingOverlays.svelte +++ b/packages/builder/src/components/portal/licensing/LicensingOverlays.svelte @@ -20,7 +20,7 @@ const processModals = () => { const defaultCacheFn = key => { - temporalStore.actions.setExpiring(key, {}, oneDayInSeconds) + temporalStore.setExpiring(key, {}, oneDayInSeconds) } const dismissableModals = [ @@ -50,7 +50,7 @@ }, ] return dismissableModals.filter(modal => { - return !temporalStore.actions.getExpiring(modal.key) && modal.criteria() + return !temporalStore.getExpiring(modal.key) && modal.criteria() }) } diff --git a/packages/builder/src/components/portal/licensing/licensingBanners.js b/packages/builder/src/components/portal/licensing/licensingBanners.js index 230b9fe6f6..62ca6caa1b 100644 --- a/packages/builder/src/components/portal/licensing/licensingBanners.js +++ b/packages/builder/src/components/portal/licensing/licensingBanners.js @@ -6,7 +6,7 @@ import { BANNER_TYPES } from "@budibase/bbui" const oneDayInSeconds = 86400 const defaultCacheFn = key => { - temporalStore.actions.setExpiring(key, {}, oneDayInSeconds) + temporalStore.setExpiring(key, {}, oneDayInSeconds) } const upgradeAction = key => { @@ -148,7 +148,7 @@ export const getBanners = () => { buildUsersAboveLimitBanner(ExpiringKeys.LICENSING_USERS_ABOVE_LIMIT_BANNER), ].filter(licensingBanner => { return ( - !temporalStore.actions.getExpiring(licensingBanner.key) && + !temporalStore.getExpiring(licensingBanner.key) && licensingBanner.criteria() ) }) diff --git a/packages/builder/src/stores/portal/temporal.js b/packages/builder/src/stores/portal/temporal.js deleted file mode 100644 index 96b47d1c7f..0000000000 --- a/packages/builder/src/stores/portal/temporal.js +++ /dev/null @@ -1,45 +0,0 @@ -import { createLocalStorageStore } from "@budibase/frontend-core" -import { get } from "svelte/store" - -export const createTemporalStore = () => { - const initialValue = {} - - const localStorageKey = `bb-temporal` - const store = createLocalStorageStore(localStorageKey, initialValue) - - const setExpiring = (key, data, duration) => { - const updated = { - ...data, - expiry: Date.now() + duration * 1000, - } - - store.update(state => ({ - ...state, - [key]: updated, - })) - } - - const getExpiring = key => { - const entry = get(store)[key] - if (!entry) { - return - } - const currentExpiry = entry.expiry - if (currentExpiry < Date.now()) { - store.update(state => { - delete state[key] - return state - }) - return null - } else { - return entry - } - } - - return { - subscribe: store.subscribe, - actions: { setExpiring, getExpiring }, - } -} - -export const temporalStore = createTemporalStore() diff --git a/packages/builder/src/stores/portal/temporal.ts b/packages/builder/src/stores/portal/temporal.ts new file mode 100644 index 0000000000..acbe6feff2 --- /dev/null +++ b/packages/builder/src/stores/portal/temporal.ts @@ -0,0 +1,53 @@ +import { get } from "svelte/store" +import { BudiStore, PersistenceType } from "../BudiStore" + +type TemporalItem = Record & { expiry: number } +type TemporalState = Record + +class TemporalStore extends BudiStore { + constructor() { + super( + {}, + { + persistence: { + key: "bb-temporal", + type: PersistenceType.LOCAL, + }, + } + ) + } + + setExpiring = ( + key: string, + data: Record, + durationSeconds: number + ) => { + const updated: TemporalItem = { + ...data, + expiry: Date.now() + durationSeconds * 1000, + } + this.update(state => ({ + ...state, + [key]: updated, + })) + } + + getExpiring(key: string) { + const entry = get(this.store)[key] + if (!entry) { + return null + } + const currentExpiry = entry.expiry + if (currentExpiry < Date.now()) { + this.update(state => { + delete state[key] + return state + }) + return null + } else { + return entry + } + } +} + +export const temporalStore = new TemporalStore() From 5f3825118f175b66c03c2ac2147c0364ca8f838f Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 13 Jan 2025 11:23:09 +0000 Subject: [PATCH 003/112] Convert portal theme store to TS --- packages/builder/src/stores/portal/theme.js | 37 ----------------- packages/builder/src/stores/portal/theme.ts | 45 +++++++++++++++++++++ 2 files changed, 45 insertions(+), 37 deletions(-) delete mode 100644 packages/builder/src/stores/portal/theme.js create mode 100644 packages/builder/src/stores/portal/theme.ts diff --git a/packages/builder/src/stores/portal/theme.js b/packages/builder/src/stores/portal/theme.js deleted file mode 100644 index 11a87845e1..0000000000 --- a/packages/builder/src/stores/portal/theme.js +++ /dev/null @@ -1,37 +0,0 @@ -import { createLocalStorageStore } from "@budibase/frontend-core" -import { derived } from "svelte/store" -import { - DefaultBuilderTheme, - ensureValidTheme, - getThemeClassNames, - ThemeOptions, - ThemeClassPrefix, -} from "@budibase/shared-core" - -export const getThemeStore = () => { - const themeElement = document.documentElement - const initialValue = { - theme: DefaultBuilderTheme, - } - const store = createLocalStorageStore("bb-theme", initialValue) - const derivedStore = derived(store, $store => ({ - ...$store, - theme: ensureValidTheme($store.theme, DefaultBuilderTheme), - })) - - // Update theme class when store changes - derivedStore.subscribe(({ theme }) => { - const classNames = getThemeClassNames(theme).split(" ") - ThemeOptions.forEach(option => { - const className = `${ThemeClassPrefix}${option.id}` - themeElement.classList.toggle(className, classNames.includes(className)) - }) - }) - - return { - ...store, - subscribe: derivedStore.subscribe, - } -} - -export const themeStore = getThemeStore() diff --git a/packages/builder/src/stores/portal/theme.ts b/packages/builder/src/stores/portal/theme.ts new file mode 100644 index 0000000000..5198555024 --- /dev/null +++ b/packages/builder/src/stores/portal/theme.ts @@ -0,0 +1,45 @@ +import { derived, Writable } from "svelte/store" +import { + DefaultBuilderTheme, + ensureValidTheme, + getThemeClassNames, + ThemeOptions, + ThemeClassPrefix, +} from "@budibase/shared-core" +import { Theme } from "@budibase/types" +import { DerivedBudiStore, PersistenceType } from "../BudiStore" + +interface ThemeState { + theme: Theme +} + +class ThemeStore extends DerivedBudiStore { + constructor() { + const makeDerivedStore = (store: Writable) => { + return derived(store, $store => ({ + ...$store, + theme: ensureValidTheme($store.theme, DefaultBuilderTheme), + })) + } + super({ theme: DefaultBuilderTheme }, makeDerivedStore, { + persistence: { + key: "bb-theme", + type: PersistenceType.LOCAL, + }, + }) + + // Update theme class when store changes + this.subscribe(({ theme }) => { + const classNames = getThemeClassNames(theme).split(" ") + ThemeOptions.forEach(option => { + const className = `${ThemeClassPrefix}${option.id}` + document.documentElement.classList.toggle( + className, + classNames.includes(className) + ) + }) + }) + } +} + +export const themeStore = new ThemeStore() From 65bd89250de002307ce62c1c2845ba1efa3e48ad Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 13 Jan 2025 11:24:04 +0000 Subject: [PATCH 004/112] Convert portal barrel file to TS --- packages/builder/src/stores/portal/{index.js => index.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/builder/src/stores/portal/{index.js => index.ts} (100%) diff --git a/packages/builder/src/stores/portal/index.js b/packages/builder/src/stores/portal/index.ts similarity index 100% rename from packages/builder/src/stores/portal/index.js rename to packages/builder/src/stores/portal/index.ts From 12b283d41ed8b7be2e41b099321adcf47efe1ad0 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 13 Jan 2025 11:28:04 +0000 Subject: [PATCH 005/112] Convert portal navigation store to BudiStore --- .../builder/src/pages/builder/_layout.svelte | 2 +- .../builder/src/stores/portal/navigation.ts | 31 +++++++------------ 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/packages/builder/src/pages/builder/_layout.svelte b/packages/builder/src/pages/builder/_layout.svelte index 209cf2746d..7e765d7366 100644 --- a/packages/builder/src/pages/builder/_layout.svelte +++ b/packages/builder/src/pages/builder/_layout.svelte @@ -18,7 +18,7 @@ $: useAccountPortal = cloud && !$admin.disableAccountPortal - navigation.actions.init($redirect) + navigation.init($redirect) const validateTenantId = async () => { const host = window.location.host diff --git a/packages/builder/src/stores/portal/navigation.ts b/packages/builder/src/stores/portal/navigation.ts index 4eb50bc84f..f289c86bf9 100644 --- a/packages/builder/src/stores/portal/navigation.ts +++ b/packages/builder/src/stores/portal/navigation.ts @@ -1,38 +1,31 @@ -import { writable } from "svelte/store" +import { BudiStore } from "../BudiStore" type GotoFuncType = (path: string) => void -interface PortalNavigationStore { +interface NavigationState { initialisated: boolean goto: GotoFuncType } -export function createNavigationStore() { - const store = writable({ - initialisated: false, - goto: undefined as any, - }) - const { set, subscribe } = store +class NavigationStore extends BudiStore { + constructor() { + super({ + initialisated: false, + goto: undefined as any, + }) + } - const init = (gotoFunc: GotoFuncType) => { + init(gotoFunc: GotoFuncType) { if (typeof gotoFunc !== "function") { throw new Error( `gotoFunc must be a function, found a "${typeof gotoFunc}" instead` ) } - - set({ + this.set({ initialisated: true, goto: gotoFunc, }) } - - return { - subscribe, - actions: { - init, - }, - } } -export const navigation = createNavigationStore() +export const navigation = new NavigationStore() From 502c1605307c3efdcb4c6ea27e5a5a2e08a9ccff Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 13 Jan 2025 11:31:44 +0000 Subject: [PATCH 006/112] Convert admin store to BudiStore --- packages/builder/src/stores/portal/admin.ts | 71 ++++++++++----------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/packages/builder/src/stores/portal/admin.ts b/packages/builder/src/stores/portal/admin.ts index f262d54bff..1cb0be1313 100644 --- a/packages/builder/src/stores/portal/admin.ts +++ b/packages/builder/src/stores/portal/admin.ts @@ -1,4 +1,4 @@ -import { writable, get } from "svelte/store" +import { get } from "svelte/store" import { API } from "@/api" import { auth } from "@/stores/portal" import { banner } from "@budibase/bbui" @@ -7,42 +7,44 @@ import { GetEnvironmentResponse, SystemStatusResponse, } from "@budibase/types" +import { BudiStore } from "../BudiStore" -interface PortalAdminStore extends GetEnvironmentResponse { +interface AdminState extends GetEnvironmentResponse { loaded: boolean checklist?: ConfigChecklistResponse status?: SystemStatusResponse } -export function createAdminStore() { - const admin = writable({ - loaded: false, - multiTenancy: false, - cloud: false, - isDev: false, - disableAccountPortal: false, - offlineMode: false, - maintenance: [], - }) +class AdminStore extends BudiStore { + constructor() { + super({ + loaded: false, + multiTenancy: false, + cloud: false, + isDev: false, + disableAccountPortal: false, + offlineMode: false, + maintenance: [], + }) + } - async function init() { - await getChecklist() - await getEnvironment() + async init() { + await this.getChecklist() + await this.getEnvironment() // enable system status checks in the cloud - if (get(admin).cloud) { - await getSystemStatus() - checkStatus() + if (get(this.store).cloud) { + await this.getSystemStatus() + this.checkStatus() } - - admin.update(store => { + this.update(store => { store.loaded = true return store }) } - async function getEnvironment() { + async getEnvironment() { const environment = await API.getEnvironment() - admin.update(store => { + this.update(store => { store.multiTenancy = environment.multiTenancy store.cloud = environment.cloud store.disableAccountPortal = environment.disableAccountPortal @@ -56,43 +58,36 @@ export function createAdminStore() { }) } - const checkStatus = async () => { - const health = get(admin)?.status?.health + async checkStatus() { + const health = get(this.store).status?.health if (!health?.passing) { await banner.showStatus() } } - async function getSystemStatus() { + async getSystemStatus() { const status = await API.getSystemStatus() - admin.update(store => { + this.update(store => { store.status = status return store }) } - async function getChecklist() { + async getChecklist() { const tenantId = get(auth).tenantId const checklist = await API.getChecklist(tenantId) - admin.update(store => { + this.update(store => { store.checklist = checklist return store }) } - function unload() { - admin.update(store => { + unload() { + this.update(store => { store.loaded = false return store }) } - - return { - subscribe: admin.subscribe, - init, - unload, - getChecklist, - } } -export const admin = createAdminStore() +export const admin = new AdminStore() From 153d905921c48be839c8aa1a74d76743017fe8b3 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 13 Jan 2025 11:33:39 +0000 Subject: [PATCH 007/112] Update exports --- packages/builder/src/stores/portal/admin.test.js | 4 ++-- packages/builder/src/stores/portal/admin.ts | 2 +- packages/builder/src/stores/portal/auditLogs.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/stores/portal/admin.test.js b/packages/builder/src/stores/portal/admin.test.js index 1528042630..261e94eb6f 100644 --- a/packages/builder/src/stores/portal/admin.test.js +++ b/packages/builder/src/stores/portal/admin.test.js @@ -1,5 +1,5 @@ import { it, expect, describe, beforeEach, vi } from "vitest" -import { createAdminStore } from "./admin" +import { AdminStore } from "./admin" import { writable, get } from "svelte/store" import { API } from "@/api" import { auth } from "@/stores/portal" @@ -46,7 +46,7 @@ describe("admin store", () => { ctx.writableReturn = { update: vi.fn(), subscribe: vi.fn() } writable.mockReturnValue(ctx.writableReturn) - ctx.returnedStore = createAdminStore() + ctx.returnedStore = new AdminStore() }) it("returns the created store", ctx => { diff --git a/packages/builder/src/stores/portal/admin.ts b/packages/builder/src/stores/portal/admin.ts index 1cb0be1313..90e3a5cdc9 100644 --- a/packages/builder/src/stores/portal/admin.ts +++ b/packages/builder/src/stores/portal/admin.ts @@ -15,7 +15,7 @@ interface AdminState extends GetEnvironmentResponse { status?: SystemStatusResponse } -class AdminStore extends BudiStore { +export class AdminStore extends BudiStore { constructor() { super({ loaded: false, diff --git a/packages/builder/src/stores/portal/auditLogs.ts b/packages/builder/src/stores/portal/auditLogs.ts index ff29f0cd1b..6f11f228d6 100644 --- a/packages/builder/src/stores/portal/auditLogs.ts +++ b/packages/builder/src/stores/portal/auditLogs.ts @@ -13,7 +13,7 @@ interface PortalAuditLogsStore { logs?: SearchAuditLogsResponse } -export class AuditLogsStore extends BudiStore { +class AuditLogsStore extends BudiStore { constructor() { super({}) } From 9e10921e2981b73b8ca497354d0b55c5089e7fff Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 13 Jan 2025 12:01:29 +0000 Subject: [PATCH 008/112] Update tests --- packages/builder/src/stores/portal/admin.test.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/packages/builder/src/stores/portal/admin.test.js b/packages/builder/src/stores/portal/admin.test.js index 261e94eb6f..8924a5e6fb 100644 --- a/packages/builder/src/stores/portal/admin.test.js +++ b/packages/builder/src/stores/portal/admin.test.js @@ -49,15 +49,6 @@ describe("admin store", () => { ctx.returnedStore = new AdminStore() }) - it("returns the created store", ctx => { - expect(ctx.returnedStore).toEqual({ - subscribe: expect.toBe(ctx.writableReturn.subscribe), - init: expect.toBeFunc(), - unload: expect.toBeFunc(), - getChecklist: expect.toBeFunc(), - }) - }) - describe("init method", () => { beforeEach(async ctx => { let getMockIndex = 0 From 80fbaa1599a277d1d7c8749cb9934d056e14248a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 15 Jan 2025 17:37:37 +0000 Subject: [PATCH 009/112] Remove old migration system. --- packages/backend-core/src/index.ts | 1 - .../src/migrations/definitions.ts | 40 ---- packages/backend-core/src/migrations/index.ts | 2 - .../backend-core/src/migrations/migrations.ts | 186 --------------- .../__snapshots__/migrations.spec.ts.snap | 11 - .../src/migrations/tests/migrations.spec.ts | 64 ------ .../users/{users.spec.js => users.spec.ts} | 21 +- packages/frontend-core/src/api/migrations.ts | 4 +- packages/server/scripts/dev/manage.js | 1 - .../server/src/api/controllers/application.ts | 9 - .../server/src/api/controllers/migrations.ts | 28 +-- packages/server/src/api/routes/migrations.ts | 10 +- packages/server/src/environment.ts | 1 - .../src/migrations/functions/appUrls.ts | 27 --- .../src/migrations/functions/backfill/app.ts | 149 ------------ .../functions/backfill/app/automations.ts | 26 --- .../functions/backfill/app/datasources.ts | 22 -- .../functions/backfill/app/layouts.ts | 29 --- .../functions/backfill/app/queries.ts | 47 ---- .../functions/backfill/app/roles.ts | 22 -- .../functions/backfill/app/screens.ts | 22 -- .../functions/backfill/app/tables.ts | 13 -- .../migrations/functions/backfill/global.ts | 214 ------------------ .../functions/backfill/global/configs.ts | 74 ------ .../functions/backfill/global/quotas.ts | 60 ----- .../functions/backfill/global/users.ts | 53 ----- .../migrations/functions/backfill/index.ts | 7 - .../functions/backfill/installation.ts | 50 ---- .../src/migrations/functions/syncQuotas.ts | 19 -- .../src/migrations/functions/tableSettings.ts | 145 ------------ .../functions/tests/appUrls.spec.js | 26 --- .../functions/tests/tableSettings.spec.ts | 144 ------------ .../tests/userEmailViewCasing.spec.js | 34 --- .../migrations/functions/usageQuotas/index.ts | 3 - .../functions/usageQuotas/syncApps.ts | 13 -- .../functions/usageQuotas/syncCreators.ts | 13 -- .../functions/usageQuotas/syncPlugins.ts | 10 - .../functions/usageQuotas/syncRows.ts | 27 --- .../functions/usageQuotas/syncUsers.ts | 9 - .../usageQuotas/tests/syncApps.spec.ts | 35 --- .../usageQuotas/tests/syncCreators.spec.ts | 26 --- .../usageQuotas/tests/syncRows.spec.ts | 53 ----- .../usageQuotas/tests/syncUsers.spec.ts | 26 --- .../functions/userEmailViewCasing.ts | 13 -- packages/server/src/migrations/index.ts | 115 ---------- .../server/src/migrations/tests/helpers.ts | 40 ---- .../server/src/migrations/tests/index.spec.ts | 137 ----------- .../server/src/migrations/tests/structures.ts | 67 ------ packages/server/src/startup/index.ts | 13 -- .../types/src/api/web/global/oldMigration.ts | 11 +- packages/types/src/api/web/system/index.ts | 1 - .../types/src/api/web/system/migration.ts | 8 - packages/types/src/sdk/index.ts | 1 - packages/types/src/sdk/migrations.ts | 57 ----- .../src/api/controllers/global/users.ts | 6 - .../src/api/controllers/system/migrations.ts | 23 -- packages/worker/src/api/routes/index.ts | 2 - .../src/api/routes/system/migrations.ts | 19 -- packages/worker/src/migrations/index.ts | 62 ----- 59 files changed, 16 insertions(+), 2335 deletions(-) delete mode 100644 packages/backend-core/src/migrations/definitions.ts delete mode 100644 packages/backend-core/src/migrations/index.ts delete mode 100644 packages/backend-core/src/migrations/migrations.ts delete mode 100644 packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap delete mode 100644 packages/backend-core/src/migrations/tests/migrations.spec.ts rename packages/backend-core/tests/core/users/{users.spec.js => users.spec.ts} (67%) delete mode 100644 packages/server/src/migrations/functions/appUrls.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/automations.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/datasources.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/layouts.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/queries.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/roles.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/screens.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/tables.ts delete mode 100644 packages/server/src/migrations/functions/backfill/global.ts delete mode 100644 packages/server/src/migrations/functions/backfill/global/configs.ts delete mode 100644 packages/server/src/migrations/functions/backfill/global/quotas.ts delete mode 100644 packages/server/src/migrations/functions/backfill/global/users.ts delete mode 100644 packages/server/src/migrations/functions/backfill/index.ts delete mode 100644 packages/server/src/migrations/functions/backfill/installation.ts delete mode 100644 packages/server/src/migrations/functions/syncQuotas.ts delete mode 100644 packages/server/src/migrations/functions/tableSettings.ts delete mode 100644 packages/server/src/migrations/functions/tests/appUrls.spec.js delete mode 100644 packages/server/src/migrations/functions/tests/tableSettings.spec.ts delete mode 100644 packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js delete mode 100644 packages/server/src/migrations/functions/usageQuotas/index.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/syncApps.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/syncCreators.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/syncRows.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/syncUsers.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts delete mode 100644 packages/server/src/migrations/functions/userEmailViewCasing.ts delete mode 100644 packages/server/src/migrations/index.ts delete mode 100644 packages/server/src/migrations/tests/helpers.ts delete mode 100644 packages/server/src/migrations/tests/index.spec.ts delete mode 100644 packages/server/src/migrations/tests/structures.ts delete mode 100644 packages/types/src/api/web/system/migration.ts delete mode 100644 packages/types/src/sdk/migrations.ts delete mode 100644 packages/worker/src/api/controllers/system/migrations.ts delete mode 100644 packages/worker/src/api/routes/system/migrations.ts delete mode 100644 packages/worker/src/migrations/index.ts diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index dbdce51c50..d4e6e9a1ec 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -1,6 +1,5 @@ export * as configs from "./configs" export * as events from "./events" -export * as migrations from "./migrations" export * as users from "./users" export * as userUtils from "./users/utils" export * as roles from "./security/roles" diff --git a/packages/backend-core/src/migrations/definitions.ts b/packages/backend-core/src/migrations/definitions.ts deleted file mode 100644 index 0dd57fe639..0000000000 --- a/packages/backend-core/src/migrations/definitions.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { - MigrationType, - MigrationName, - MigrationDefinition, -} from "@budibase/types" - -export const DEFINITIONS: MigrationDefinition[] = [ - { - type: MigrationType.GLOBAL, - name: MigrationName.USER_EMAIL_VIEW_CASING, - }, - { - type: MigrationType.GLOBAL, - name: MigrationName.SYNC_QUOTAS, - }, - { - type: MigrationType.APP, - name: MigrationName.APP_URLS, - }, - { - type: MigrationType.APP, - name: MigrationName.EVENT_APP_BACKFILL, - }, - { - type: MigrationType.APP, - name: MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS, - }, - { - type: MigrationType.GLOBAL, - name: MigrationName.EVENT_GLOBAL_BACKFILL, - }, - { - type: MigrationType.INSTALLATION, - name: MigrationName.EVENT_INSTALLATION_BACKFILL, - }, - { - type: MigrationType.GLOBAL, - name: MigrationName.GLOBAL_INFO_SYNC_USERS, - }, -] diff --git a/packages/backend-core/src/migrations/index.ts b/packages/backend-core/src/migrations/index.ts deleted file mode 100644 index bce0cfc75c..0000000000 --- a/packages/backend-core/src/migrations/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./migrations" -export * from "./definitions" diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts deleted file mode 100644 index c8320b5724..0000000000 --- a/packages/backend-core/src/migrations/migrations.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { DEFAULT_TENANT_ID } from "../constants" -import { - DocumentType, - StaticDatabases, - getAllApps, - getGlobalDBName, - getDB, -} from "../db" -import environment from "../environment" -import * as platform from "../platform" -import * as context from "../context" -import { DEFINITIONS } from "." -import { - Migration, - MigrationOptions, - MigrationType, - MigrationNoOpOptions, - App, -} from "@budibase/types" - -export const getMigrationsDoc = async (db: any) => { - // get the migrations doc - try { - return await db.get(DocumentType.MIGRATIONS) - } catch (err: any) { - if (err.status && err.status === 404) { - return { _id: DocumentType.MIGRATIONS } - } else { - throw err - } - } -} - -export const backPopulateMigrations = async (opts: MigrationNoOpOptions) => { - // filter migrations to the type and populate a no-op migration - const migrations: Migration[] = DEFINITIONS.filter( - def => def.type === opts.type - ).map(d => ({ ...d, fn: async () => {} })) - await runMigrations(migrations, { noOp: opts }) -} - -export const runMigration = async ( - migration: Migration, - options: MigrationOptions = {} -) => { - const migrationType = migration.type - const migrationName = migration.name - const silent = migration.silent - - const log = (message: string) => { - if (!silent) { - console.log(message) - } - } - - // get the db to store the migration in - let dbNames: string[] - if (migrationType === MigrationType.GLOBAL) { - dbNames = [getGlobalDBName()] - } else if (migrationType === MigrationType.APP) { - if (options.noOp) { - if (!options.noOp.appId) { - throw new Error("appId is required for noOp app migration") - } - dbNames = [options.noOp.appId] - } else { - const apps = (await getAllApps(migration.appOpts)) as App[] - dbNames = apps.map(app => app.appId) - } - } else if (migrationType === MigrationType.INSTALLATION) { - dbNames = [StaticDatabases.PLATFORM_INFO.name] - } else { - throw new Error(`Unrecognised migration type [${migrationType}]`) - } - - const length = dbNames.length - let count = 0 - - // run the migration against each db - for (const dbName of dbNames) { - count++ - const lengthStatement = length > 1 ? `[${count}/${length}]` : "" - - const db = getDB(dbName) - - try { - const doc = await getMigrationsDoc(db) - - // the migration has already been run - if (doc[migrationName]) { - // check for force - if ( - options.force && - options.force[migrationType] && - options.force[migrationType].includes(migrationName) - ) { - log(`[Migration: ${migrationName}] [DB: ${dbName}] Forcing`) - } else { - // no force, exit - return - } - } - - // check if the migration is not a no-op - if (!options.noOp) { - log( - `[Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}` - ) - - if (migration.preventRetry) { - // eagerly set the completion date - // so that we never run this migration twice even upon failure - doc[migrationName] = Date.now() - const response = await db.put(doc) - doc._rev = response.rev - } - - // run the migration - if (migrationType === MigrationType.APP) { - await context.doInAppContext(db.name, async () => { - await migration.fn(db) - }) - } else { - await migration.fn(db) - } - - log(`[Migration: ${migrationName}] [DB: ${dbName}] Complete`) - } - - // mark as complete - doc[migrationName] = Date.now() - await db.put(doc) - } catch (err) { - console.error( - `[Migration: ${migrationName}] [DB: ${dbName}] Error: `, - err - ) - throw err - } - } -} - -export const runMigrations = async ( - migrations: Migration[], - options: MigrationOptions = {} -) => { - let tenantIds - - if (environment.MULTI_TENANCY) { - if (options.noOp) { - tenantIds = [options.noOp.tenantId] - } else if (!options.tenantIds || !options.tenantIds.length) { - // run for all tenants - tenantIds = await platform.tenants.getTenantIds() - } else { - tenantIds = options.tenantIds - } - } else { - // single tenancy - tenantIds = [DEFAULT_TENANT_ID] - } - - if (tenantIds.length > 1) { - console.log(`Checking migrations for ${tenantIds.length} tenants`) - } else { - console.log("Checking migrations") - } - - let count = 0 - // for all tenants - for (const tenantId of tenantIds) { - count++ - if (tenantIds.length > 1) { - console.log(`Progress [${count}/${tenantIds.length}]`) - } - // for all migrations - for (const migration of migrations) { - // run the migration - await context.doInTenant( - tenantId, - async () => await runMigration(migration, options) - ) - } - } - console.log("Migrations complete") -} diff --git a/packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap b/packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap deleted file mode 100644 index 377900b5d5..0000000000 --- a/packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap +++ /dev/null @@ -1,11 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`migrations should match snapshot 1`] = ` -{ - "_id": "migrations", - "_rev": "1-2f64479842a0513aa8b97f356b0b9127", - "createdAt": "2020-01-01T00:00:00.000Z", - "test": 1577836800000, - "updatedAt": "2020-01-01T00:00:00.000Z", -} -`; diff --git a/packages/backend-core/src/migrations/tests/migrations.spec.ts b/packages/backend-core/src/migrations/tests/migrations.spec.ts deleted file mode 100644 index af2eb33cf5..0000000000 --- a/packages/backend-core/src/migrations/tests/migrations.spec.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { testEnv, DBTestConfiguration } from "../../../tests/extra" -import * as migrations from "../index" -import * as context from "../../context" -import { MigrationType } from "@budibase/types" - -testEnv.multiTenant() - -describe("migrations", () => { - const config = new DBTestConfiguration() - - const migrationFunction = jest.fn() - - const MIGRATIONS = [ - { - type: MigrationType.GLOBAL, - name: "test" as any, - fn: migrationFunction, - }, - ] - - beforeEach(() => { - config.newTenant() - }) - - afterEach(async () => { - jest.clearAllMocks() - }) - - const migrate = () => { - return migrations.runMigrations(MIGRATIONS, { - tenantIds: [config.tenantId], - }) - } - - it("should run a new migration", async () => { - await config.doInTenant(async () => { - await migrate() - expect(migrationFunction).toHaveBeenCalled() - const db = context.getGlobalDB() - const doc = await migrations.getMigrationsDoc(db) - expect(doc.test).toBeDefined() - }) - }) - - it("should match snapshot", async () => { - await config.doInTenant(async () => { - await migrate() - const doc = await migrations.getMigrationsDoc(context.getGlobalDB()) - expect(doc).toMatchSnapshot() - }) - }) - - it("should skip a previously run migration", async () => { - await config.doInTenant(async () => { - const db = context.getGlobalDB() - await migrate() - const previousDoc = await migrations.getMigrationsDoc(db) - await migrate() - const currentDoc = await migrations.getMigrationsDoc(db) - expect(migrationFunction).toHaveBeenCalledTimes(1) - expect(currentDoc.test).toBe(previousDoc.test) - }) - }) -}) diff --git a/packages/backend-core/tests/core/users/users.spec.js b/packages/backend-core/tests/core/users/users.spec.ts similarity index 67% rename from packages/backend-core/tests/core/users/users.spec.js rename to packages/backend-core/tests/core/users/users.spec.ts index dde0d87fb7..b14f553266 100644 --- a/packages/backend-core/tests/core/users/users.spec.js +++ b/packages/backend-core/tests/core/users/users.spec.ts @@ -1,17 +1,17 @@ -const _ = require("lodash/fp") -const { structures } = require("../../../tests") +import { range } from "lodash/fp" +import { structures } from "../.." jest.mock("../../../src/context") jest.mock("../../../src/db") -const context = require("../../../src/context") -const db = require("../../../src/db") +import * as context from "../../../src/context" +import * as db from "../../../src/db" -const { getCreatorCount } = require("../../../src/users/users") +import { getCreatorCount } from "../../../src/users/users" describe("Users", () => { - let getGlobalDBMock - let paginationMock + let getGlobalDBMock: jest.SpyInstance + let paginationMock: jest.SpyInstance beforeEach(() => { jest.resetAllMocks() @@ -22,11 +22,10 @@ describe("Users", () => { jest.spyOn(db, "getGlobalUserParams") }) - it("Retrieves the number of creators", async () => { - const getUsers = (offset, limit, creators = false) => { - const range = _.range(offset, limit) + it("retrieves the number of creators", async () => { + const getUsers = (offset: number, limit: number, creators = false) => { const opts = creators ? { builder: { global: true } } : undefined - return range.map(() => structures.users.user(opts)) + return range(offset, limit).map(() => structures.users.user(opts)) } const page1Data = getUsers(0, 8) const page2Data = getUsers(8, 12, true) diff --git a/packages/frontend-core/src/api/migrations.ts b/packages/frontend-core/src/api/migrations.ts index 8213691205..35d2f95dbc 100644 --- a/packages/frontend-core/src/api/migrations.ts +++ b/packages/frontend-core/src/api/migrations.ts @@ -1,8 +1,8 @@ -import { GetOldMigrationStatus } from "@budibase/types" +import { GetMigrationStatus } from "@budibase/types" import { BaseAPIClient } from "./types" export interface MigrationEndpoints { - getMigrationStatus: () => Promise + getMigrationStatus: () => Promise } export const buildMigrationEndpoints = ( diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js index 0794c53506..84e1abea69 100644 --- a/packages/server/scripts/dev/manage.js +++ b/packages/server/scripts/dev/manage.js @@ -43,7 +43,6 @@ async function init() { BB_ADMIN_USER_EMAIL: "", BB_ADMIN_USER_PASSWORD: "", PLUGINS_DIR: "", - HTTP_MIGRATIONS: "0", HTTP_LOGGING: "0", VERSION: "0.0.0+local", PASSWORD_MIN_LENGTH: "1", diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 4169087a63..b62d022cbd 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -27,7 +27,6 @@ import { env as envCore, ErrorCode, events, - migrations, objectStore, roles, tenancy, @@ -43,7 +42,6 @@ import { groups, licensing, quotas } from "@budibase/pro" import { App, Layout, - MigrationType, PlanType, Screen, UserCtx, @@ -488,13 +486,6 @@ async function creationEvents(request: BBRequest, app: App) { } async function appPostCreate(ctx: UserCtx, app: App) { - const tenantId = tenancy.getTenantId() - await migrations.backPopulateMigrations({ - type: MigrationType.APP, - tenantId, - appId: app.appId, - }) - await creationEvents(ctx.request, app) // app import, template creation and duplication diff --git a/packages/server/src/api/controllers/migrations.ts b/packages/server/src/api/controllers/migrations.ts index edf4ec6f51..fc3a1a1548 100644 --- a/packages/server/src/api/controllers/migrations.ts +++ b/packages/server/src/api/controllers/migrations.ts @@ -1,35 +1,11 @@ import { context } from "@budibase/backend-core" -import { migrate as migrationImpl, MIGRATIONS } from "../../migrations" -import { - Ctx, - FetchOldMigrationResponse, - GetOldMigrationStatus, - RuneOldMigrationResponse, - RunOldMigrationRequest, -} from "@budibase/types" +import { Ctx, GetMigrationStatus } from "@budibase/types" import { getAppMigrationVersion, getLatestEnabledMigrationId, } from "../../appMigrations" -export async function migrate( - ctx: Ctx -) { - const options = ctx.request.body - // don't await as can take a while, just return - migrationImpl(options) - ctx.body = { message: "Migration started." } -} - -export async function fetchDefinitions( - ctx: Ctx -) { - ctx.body = MIGRATIONS -} - -export async function getMigrationStatus( - ctx: Ctx -) { +export async function getMigrationStatus(ctx: Ctx) { const appId = context.getAppId() if (!appId) { diff --git a/packages/server/src/api/routes/migrations.ts b/packages/server/src/api/routes/migrations.ts index 918b197de2..0ffc334551 100644 --- a/packages/server/src/api/routes/migrations.ts +++ b/packages/server/src/api/routes/migrations.ts @@ -1,16 +1,8 @@ import Router from "@koa/router" import * as migrationsController from "../controllers/migrations" -import { auth } from "@budibase/backend-core" const router: Router = new Router() -router - .post("/api/migrations/run", auth.internalApi, migrationsController.migrate) - .get( - "/api/migrations/definitions", - auth.internalApi, - migrationsController.fetchDefinitions - ) - .get("/api/migrations/status", migrationsController.getMigrationStatus) +router.get("/api/migrations/status", migrationsController.getMigrationStatus) export default router diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 45d675ec3f..a9867b1231 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -54,7 +54,6 @@ const environment = { REDIS_URL: process.env.REDIS_URL, REDIS_PASSWORD: process.env.REDIS_PASSWORD, REDIS_CLUSTERED: process.env.REDIS_CLUSTERED, - HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS, CLUSTER_MODE: process.env.CLUSTER_MODE, API_REQ_LIMIT_PER_SEC: process.env.API_REQ_LIMIT_PER_SEC, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, diff --git a/packages/server/src/migrations/functions/appUrls.ts b/packages/server/src/migrations/functions/appUrls.ts deleted file mode 100644 index be03d3c81e..0000000000 --- a/packages/server/src/migrations/functions/appUrls.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { db as dbCore } from "@budibase/backend-core" -import sdk from "../../sdk" - -/** - * Date: - * January 2022 - * - * Description: - * Add the url to the app metadata if it doesn't exist - */ -export const run = async (appDb: any) => { - let metadata - try { - metadata = await appDb.get(dbCore.DocumentType.APP_METADATA) - } catch (e) { - // sometimes the metadata document doesn't exist - // exit early instead of failing the migration - console.error("Error retrieving app metadata. Skipping", e) - return - } - - if (!metadata.url) { - metadata.url = sdk.applications.getAppUrl({ name: metadata.name }) - console.log(`Adding url to app: ${metadata.url}`) - await appDb.put(metadata) - } -} diff --git a/packages/server/src/migrations/functions/backfill/app.ts b/packages/server/src/migrations/functions/backfill/app.ts deleted file mode 100644 index 51a37108b0..0000000000 --- a/packages/server/src/migrations/functions/backfill/app.ts +++ /dev/null @@ -1,149 +0,0 @@ -import * as automations from "./app/automations" -import * as datasources from "./app/datasources" -import * as layouts from "./app/layouts" -import * as queries from "./app/queries" -import * as roles from "./app/roles" -import * as tables from "./app/tables" -import * as screens from "./app/screens" -import * as global from "./global" -import { App, AppBackfillSucceededEvent, Event } from "@budibase/types" -import { db as dbUtils, events } from "@budibase/backend-core" -import env from "../../../environment" -import { DEFAULT_TIMESTAMP } from "." - -const failGraceful = env.SELF_HOSTED && !env.isDev() - -const handleError = (e: any, errors?: any) => { - if (failGraceful) { - if (errors) { - errors.push(e) - } - return - } - console.trace(e) - throw e -} - -const EVENTS = [ - Event.AUTOMATION_CREATED, - Event.AUTOMATION_STEP_CREATED, - Event.DATASOURCE_CREATED, - Event.LAYOUT_CREATED, - Event.QUERY_CREATED, - Event.ROLE_CREATED, - Event.SCREEN_CREATED, - Event.TABLE_CREATED, - Event.VIEW_CREATED, - Event.VIEW_CALCULATION_CREATED, - Event.VIEW_FILTER_CREATED, - Event.APP_PUBLISHED, - Event.APP_CREATED, -] - -/** - * Date: - * May 2022 - * - * Description: - * Backfill app events. - */ - -export const run = async (appDb: any) => { - try { - if (await global.isComplete()) { - // make sure new apps aren't backfilled - // return if the global migration for this tenant is complete - // which runs after the app migrations - return - } - - // tell the event pipeline to start caching - // events for this tenant - await events.backfillCache.start(EVENTS) - - let timestamp: string | number = DEFAULT_TIMESTAMP - const app: App = await appDb.get(dbUtils.DocumentType.APP_METADATA) - if (app.createdAt) { - timestamp = app.createdAt as string - } - - if (dbUtils.isProdAppID(app.appId)) { - await events.app.published(app, timestamp) - } - - const totals: any = {} - const errors: any = [] - - if (dbUtils.isDevAppID(app.appId)) { - await events.app.created(app, timestamp) - try { - totals.automations = await automations.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.datasources = await datasources.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.layouts = await layouts.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.queries = await queries.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.roles = await roles.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.screens = await screens.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.tables = await tables.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - } - - const properties: AppBackfillSucceededEvent = { - appId: app.appId, - automations: totals.automations, - datasources: totals.datasources, - layouts: totals.layouts, - queries: totals.queries, - roles: totals.roles, - tables: totals.tables, - screens: totals.screens, - } - - if (errors.length) { - properties.errors = errors.map((e: any) => - JSON.stringify(e, Object.getOwnPropertyNames(e)) - ) - properties.errorCount = errors.length - } else { - properties.errorCount = 0 - } - - await events.backfill.appSucceeded(properties) - // tell the event pipeline to stop caching events for this tenant - await events.backfillCache.end() - } catch (e) { - handleError(e) - await events.backfill.appFailed(e) - } -} diff --git a/packages/server/src/migrations/functions/backfill/app/automations.ts b/packages/server/src/migrations/functions/backfill/app/automations.ts deleted file mode 100644 index 20da8fd3c0..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/automations.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getAutomationParams } from "../../../../db/utils" -import { Automation } from "@budibase/types" - -const getAutomations = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getAutomationParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const automations = await getAutomations(appDb) - - for (const automation of automations) { - await events.automation.created(automation, timestamp) - - for (const step of automation.definition.steps) { - await events.automation.stepCreated(automation, step, timestamp) - } - } - - return automations.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/datasources.ts b/packages/server/src/migrations/functions/backfill/app/datasources.ts deleted file mode 100644 index 5d7e1ad866..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/datasources.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getDatasourceParams } from "../../../../db/utils" -import { Datasource } from "@budibase/types" - -const getDatasources = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getDatasourceParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const datasources: Datasource[] = await getDatasources(appDb) - - for (const datasource of datasources) { - await events.datasource.created(datasource, timestamp) - } - - return datasources.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/layouts.ts b/packages/server/src/migrations/functions/backfill/app/layouts.ts deleted file mode 100644 index ee5806459b..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/layouts.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getLayoutParams } from "../../../../db/utils" -import { Layout } from "@budibase/types" - -const getLayouts = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getLayoutParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const layouts: Layout[] = await getLayouts(appDb) - - for (const layout of layouts) { - // exclude default layouts - if ( - layout._id === "layout_private_master" || - layout._id === "layout_public_master" - ) { - continue - } - await events.layout.created(layout, timestamp) - } - - return layouts.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/queries.ts b/packages/server/src/migrations/functions/backfill/app/queries.ts deleted file mode 100644 index e028721bce..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/queries.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getQueryParams } from "../../../../db/utils" -import { Query, Datasource, SourceName } from "@budibase/types" - -const getQueries = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getQueryParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -const getDatasource = async ( - appDb: any, - datasourceId: string -): Promise => { - return appDb.get(datasourceId) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const queries: Query[] = await getQueries(appDb) - - for (const query of queries) { - let datasource: Datasource - - try { - datasource = await getDatasource(appDb, query.datasourceId) - } catch (e: any) { - // handle known bug where a datasource has been deleted - // and the query has not - if (e.status === 404) { - datasource = { - type: "unknown", - _id: query.datasourceId, - source: "unknown" as SourceName, - } - } else { - throw e - } - } - - await events.query.created(datasource, query, timestamp) - } - - return queries.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/roles.ts b/packages/server/src/migrations/functions/backfill/app/roles.ts deleted file mode 100644 index 494b6f6923..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/roles.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getRoleParams } from "../../../../db/utils" -import { Role } from "@budibase/types" - -const getRoles = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getRoleParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const roles = await getRoles(appDb) - - for (const role of roles) { - await events.role.created(role, timestamp) - } - - return roles.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/screens.ts b/packages/server/src/migrations/functions/backfill/app/screens.ts deleted file mode 100644 index ab3b4b9d3c..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/screens.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getScreenParams } from "../../../../db/utils" -import { Screen } from "@budibase/types" - -const getScreens = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getScreenParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const screens = await getScreens(appDb) - - for (const screen of screens) { - await events.screen.created(screen, timestamp) - } - - return screens.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts deleted file mode 100644 index e8437bd529..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/tables.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { events } from "@budibase/backend-core" -import { Database } from "@budibase/types" -import sdk from "../../../../sdk" - -export const backfill = async (appDb: Database, timestamp: string | number) => { - const tables = await sdk.tables.getAllInternalTables(appDb) - - for (const table of tables) { - await events.table.created(table, timestamp) - } - - return tables.length -} diff --git a/packages/server/src/migrations/functions/backfill/global.ts b/packages/server/src/migrations/functions/backfill/global.ts deleted file mode 100644 index 7f718cee2f..0000000000 --- a/packages/server/src/migrations/functions/backfill/global.ts +++ /dev/null @@ -1,214 +0,0 @@ -import * as users from "./global/users" -import * as configs from "./global/configs" -import * as quotas from "./global/quotas" -import { - tenancy, - events, - migrations, - accounts, - db as dbUtils, -} from "@budibase/backend-core" -import { - App, - CloudAccount, - Event, - Hosting, - QuotaUsage, - TenantBackfillSucceededEvent, - User, -} from "@budibase/types" -import env from "../../../environment" -import { DEFAULT_TIMESTAMP } from "." - -const failGraceful = env.SELF_HOSTED && !env.isDev() - -const handleError = (e: any, errors?: any) => { - if (failGraceful) { - if (errors) { - errors.push(e) - } - return - } - throw e -} - -const formatUsage = (usage: QuotaUsage) => { - let maxAutomations = 0 - let maxQueries = 0 - let rows = 0 - - if (usage) { - if (usage.usageQuota) { - rows = usage.usageQuota.rows - } - - if (usage.monthly) { - for (const value of Object.values(usage.monthly)) { - if (value.automations > maxAutomations) { - maxAutomations = value.automations - } - if (value.queries > maxQueries) { - maxQueries = value.queries - } - } - } - } - - return { - maxAutomations, - maxQueries, - rows, - } -} - -const EVENTS = [ - Event.EMAIL_SMTP_CREATED, - Event.AUTH_SSO_CREATED, - Event.AUTH_SSO_ACTIVATED, - Event.ORG_NAME_UPDATED, - Event.ORG_LOGO_UPDATED, - Event.ORG_PLATFORM_URL_UPDATED, - Event.USER_CREATED, - Event.USER_PERMISSION_ADMIN_ASSIGNED, - Event.USER_PERMISSION_BUILDER_ASSIGNED, - Event.ROLE_ASSIGNED, - Event.ROWS_CREATED, - Event.QUERIES_RUN, - Event.AUTOMATIONS_RUN, -] - -/** - * Date: - * May 2022 - * - * Description: - * Backfill global events. - */ - -export const run = async (db: any) => { - try { - const tenantId = tenancy.getTenantId() - let timestamp: string | number = DEFAULT_TIMESTAMP - - const totals: any = {} - const errors: any = [] - - let allUsers: User[] = [] - try { - allUsers = await users.getUsers(db) - } catch (e: any) { - handleError(e, errors) - } - - if (!allUsers || allUsers.length === 0) { - // first time startup - we don't need to backfill anything - // tenant will be identified when admin user is created - if (env.SELF_HOSTED) { - await events.installation.firstStartup() - } - return - } - - try { - const installTimestamp = await getInstallTimestamp(db, allUsers) - if (installTimestamp) { - timestamp = installTimestamp - } - } catch (e) { - handleError(e, errors) - } - - let account: CloudAccount | undefined - if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { - account = await accounts.getAccountByTenantId(tenantId) - } - - try { - await events.identification.identifyTenantGroup( - tenantId, - env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD, - timestamp - ) - } catch (e) { - handleError(e, errors) - } - - // tell the event pipeline to start caching - // events for this tenant - await events.backfillCache.start(EVENTS) - - try { - await configs.backfill(db, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.users = await users.backfill(db, account) - } catch (e) { - handleError(e, errors) - } - - try { - const allApps = (await dbUtils.getAllApps({ dev: true })) as App[] - totals.apps = allApps.length - - totals.usage = await quotas.backfill(allApps) - } catch (e) { - handleError(e, errors) - } - - const properties: TenantBackfillSucceededEvent = { - apps: totals.apps, - users: totals.users, - ...formatUsage(totals.usage), - usage: totals.usage, - } - - if (errors.length) { - properties.errors = errors.map((e: any) => - JSON.stringify(e, Object.getOwnPropertyNames(e)) - ) - properties.errorCount = errors.length - } else { - properties.errorCount = 0 - } - - await events.backfill.tenantSucceeded(properties) - // tell the event pipeline to stop caching events for this tenant - await events.backfillCache.end() - } catch (e) { - handleError(e) - await events.backfill.tenantFailed(e) - } -} - -export const isComplete = async (): Promise => { - const globalDb = tenancy.getGlobalDB() - const migrationsDoc = await migrations.getMigrationsDoc(globalDb) - return !!migrationsDoc.event_global_backfill -} - -export const getInstallTimestamp = async ( - globalDb: any, - allUsers?: User[] -): Promise => { - if (!allUsers) { - allUsers = await users.getUsers(globalDb) - } - - // get the oldest user timestamp - if (allUsers) { - const timestamps = allUsers - .map(user => user.createdAt) - .filter(timestamp => !!timestamp) - .sort( - (a, b) => - new Date(a as number).getTime() - new Date(b as number).getTime() - ) - - if (timestamps.length) { - return timestamps[0] - } - } -} diff --git a/packages/server/src/migrations/functions/backfill/global/configs.ts b/packages/server/src/migrations/functions/backfill/global/configs.ts deleted file mode 100644 index 04eb9caff2..0000000000 --- a/packages/server/src/migrations/functions/backfill/global/configs.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { - events, - DocumentType, - SEPARATOR, - UNICODE_MAX, -} from "@budibase/backend-core" -import { - Config, - isSMTPConfig, - isGoogleConfig, - isOIDCConfig, - isSettingsConfig, - ConfigType, - DatabaseQueryOpts, -} from "@budibase/types" -import env from "./../../../../environment" - -export function getConfigParams(): DatabaseQueryOpts { - return { - include_docs: true, - startkey: `${DocumentType.CONFIG}${SEPARATOR}`, - endkey: `${DocumentType.CONFIG}${SEPARATOR}${UNICODE_MAX}`, - } -} - -const getConfigs = async (globalDb: any): Promise => { - const response = await globalDb.allDocs(getConfigParams()) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async ( - globalDb: any, - timestamp: string | number | undefined -) => { - const configs = await getConfigs(globalDb) - - for (const config of configs) { - if (isSMTPConfig(config)) { - await events.email.SMTPCreated(timestamp) - } - if (isGoogleConfig(config)) { - await events.auth.SSOCreated(ConfigType.GOOGLE, timestamp) - if (config.config.activated) { - await events.auth.SSOActivated(ConfigType.GOOGLE, timestamp) - } - } - if (isOIDCConfig(config)) { - await events.auth.SSOCreated(ConfigType.OIDC, timestamp) - if (config.config.configs[0].activated) { - await events.auth.SSOActivated(ConfigType.OIDC, timestamp) - } - } - if (isSettingsConfig(config)) { - const company = config.config.company - if (company && company !== "Budibase") { - await events.org.nameUpdated(timestamp) - } - - const logoUrl = config.config.logoUrl - if (logoUrl) { - await events.org.logoUpdated(timestamp) - } - - const platformUrl = config.config.platformUrl - if ( - platformUrl && - platformUrl !== "http://localhost:10000" && - env.SELF_HOSTED - ) { - await events.org.platformURLUpdated(timestamp) - } - } - } -} diff --git a/packages/server/src/migrations/functions/backfill/global/quotas.ts b/packages/server/src/migrations/functions/backfill/global/quotas.ts deleted file mode 100644 index 505274a318..0000000000 --- a/packages/server/src/migrations/functions/backfill/global/quotas.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { DEFAULT_TIMESTAMP } from "./../index" -import { events } from "@budibase/backend-core" -import { quotas } from "@budibase/pro" -import { App } from "@budibase/types" - -const getOldestCreatedAt = (allApps: App[]): string | undefined => { - const timestamps = allApps - .filter(app => !!app.createdAt) - .map(app => app.createdAt as string) - .sort((a, b) => new Date(a).getTime() - new Date(b).getTime()) - - if (timestamps.length) { - return timestamps[0] - } -} - -const getMonthTimestamp = (monthString: string): number => { - const parts = monthString.split("-") - const month = parseInt(parts[0]) - 1 // we already do +1 in month string calculation - const year = parseInt(parts[1]) - - // using 0 as the day in next month gives us last day in previous month - const date = new Date(year, month + 1, 0).getTime() - const now = new Date().getTime() - - if (date > now) { - return now - } else { - return date - } -} - -export const backfill = async (allApps: App[]) => { - const usage = await quotas.getQuotaUsage() - - const rows = usage.usageQuota.rows - let timestamp: string | number = DEFAULT_TIMESTAMP - - const oldestAppTimestamp = getOldestCreatedAt(allApps) - if (oldestAppTimestamp) { - timestamp = oldestAppTimestamp - } - - await events.rows.created(rows, timestamp) - - for (const [monthString, quotas] of Object.entries(usage.monthly)) { - if (monthString === "current") { - continue - } - const monthTimestamp = getMonthTimestamp(monthString) - - const queries = quotas.queries - await events.query.run(queries, monthTimestamp) - - const automations = quotas.automations - await events.automation.run(automations, monthTimestamp) - } - - return usage -} diff --git a/packages/server/src/migrations/functions/backfill/global/users.ts b/packages/server/src/migrations/functions/backfill/global/users.ts deleted file mode 100644 index b3dae822d7..0000000000 --- a/packages/server/src/migrations/functions/backfill/global/users.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { - events, - db as dbUtils, - users as usersCore, -} from "@budibase/backend-core" -import { User, CloudAccount } from "@budibase/types" -import { DEFAULT_TIMESTAMP } from ".." - -// manually define user doc params - normally server doesn't read users from the db -const getUserParams = (props: any) => { - return dbUtils.getDocParams(dbUtils.DocumentType.USER, null, props) -} - -export const getUsers = async (globalDb: any): Promise => { - const response = await globalDb.allDocs( - getUserParams({ - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async ( - globalDb: any, - account: CloudAccount | undefined -) => { - const users = await getUsers(globalDb) - - for (const user of users) { - let timestamp: string | number = DEFAULT_TIMESTAMP - if (user.createdAt) { - timestamp = user.createdAt - } - await events.identification.identifyUser(user, account, timestamp) - await events.user.created(user, timestamp) - - if (usersCore.hasAdminPermissions(user)) { - await events.user.permissionAdminAssigned(user, timestamp) - } - - if (usersCore.hasBuilderPermissions(user)) { - await events.user.permissionBuilderAssigned(user, timestamp) - } - - if (user.roles) { - for (const [, role] of Object.entries(user.roles)) { - await events.role.assigned(user, role, timestamp) - } - } - } - - return users.length -} diff --git a/packages/server/src/migrations/functions/backfill/index.ts b/packages/server/src/migrations/functions/backfill/index.ts deleted file mode 100644 index 00c04722b4..0000000000 --- a/packages/server/src/migrations/functions/backfill/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * as app from "./app" -export * as global from "./global" -export * as installation from "./installation" - -// historical events are free in posthog - make sure we default to a -// historical time if no other can be found -export const DEFAULT_TIMESTAMP = new Date(2022, 0, 1).getTime() diff --git a/packages/server/src/migrations/functions/backfill/installation.ts b/packages/server/src/migrations/functions/backfill/installation.ts deleted file mode 100644 index 3c2b8bd3fc..0000000000 --- a/packages/server/src/migrations/functions/backfill/installation.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { DEFAULT_TIMESTAMP } from "./index" -import { events, tenancy, installation } from "@budibase/backend-core" -import { Installation } from "@budibase/types" -import * as global from "./global" -import env from "../../../environment" - -const failGraceful = env.SELF_HOSTED - -const handleError = (e: any, errors?: any) => { - if (failGraceful) { - if (errors) { - errors.push(e) - } - return - } - throw e -} - -/** - * Date: - * May 2022 - * - * Description: - * Backfill installation events. - */ - -export const run = async () => { - try { - // need to use the default tenant to try to get the installation time - await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => { - const db = tenancy.getGlobalDB() - let timestamp: string | number = DEFAULT_TIMESTAMP - - const installTimestamp = await global.getInstallTimestamp(db) - if (installTimestamp) { - timestamp = installTimestamp - } - - const install: Installation = await installation.getInstall() - await events.identification.identifyInstallationGroup( - install.installId, - timestamp - ) - }) - await events.backfill.installationSucceeded() - } catch (e) { - handleError(e) - await events.backfill.installationFailed(e) - } -} diff --git a/packages/server/src/migrations/functions/syncQuotas.ts b/packages/server/src/migrations/functions/syncQuotas.ts deleted file mode 100644 index 83a7670e78..0000000000 --- a/packages/server/src/migrations/functions/syncQuotas.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { runQuotaMigration } from "./usageQuotas" -import * as syncApps from "./usageQuotas/syncApps" -import * as syncRows from "./usageQuotas/syncRows" -import * as syncPlugins from "./usageQuotas/syncPlugins" -import * as syncUsers from "./usageQuotas/syncUsers" -import * as syncCreators from "./usageQuotas/syncCreators" - -/** - * Synchronise quotas to the state of the db. - */ -export const run = async () => { - await runQuotaMigration(async () => { - await syncApps.run() - await syncRows.run() - await syncPlugins.run() - await syncUsers.run() - await syncCreators.run() - }) -} diff --git a/packages/server/src/migrations/functions/tableSettings.ts b/packages/server/src/migrations/functions/tableSettings.ts deleted file mode 100644 index 2db3df0d0f..0000000000 --- a/packages/server/src/migrations/functions/tableSettings.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { getScreenParams } from "../../db/utils" -import { Screen } from "@budibase/types" -import { makePropSafe as safe } from "@budibase/string-templates" -/** - * Date: - * November 2022 - * - * Description: - * Update table settings to use actions instead of links. We do not remove the - * legacy values here as we cannot guarantee that their apps are up-t-date. - * It is safe to simply save both the new and old structure in the definition. - * - * Migration 1: - * Legacy "linkRows", "linkURL", "linkPeek" and "linkColumn" settings on tables - * and table blocks are migrated into a "Navigate To" action under the new - * "onClick" setting. - * - * Migration 2: - * Legacy "titleButtonURL" and "titleButtonPeek" settings on table blocks are - * migrated into a "Navigate To" action under the new "onClickTitleButton" - * setting. - */ -export const run = async (appDb: any) => { - // Get all app screens - let screens: Screen[] - try { - screens = ( - await appDb.allDocs( - getScreenParams(null, { - include_docs: true, - }) - ) - ).rows.map((row: any) => row.doc) - } catch (e) { - // sometimes the metadata document doesn't exist - // exit early instead of failing the migration - console.error("Error retrieving app metadata. Skipping", e) - return - } - - // Recursively update any relevant components and mutate the screen docs - for (let screen of screens) { - const changed = migrateTableSettings(screen.props) - - // Save screen if we updated it - if (changed) { - await appDb.put(screen) - console.log( - `Screen ${screen.routing?.route} contained table settings which were migrated` - ) - } - } -} - -// Recursively searches and mutates a screen doc to migrate table component -// and table block settings -const migrateTableSettings = (component: any) => { - let changed = false - if (!component) { - return changed - } - - // Migration 1: migrate table row click settings - if ( - component._component.endsWith("/table") || - component._component.endsWith("/tableblock") - ) { - const { linkRows, linkURL, linkPeek, linkColumn, onClick } = component - if (linkRows && !onClick) { - const column = linkColumn || "_id" - const action = convertLinkSettingToAction(linkURL, !!linkPeek, column) - if (action) { - changed = true - component.onClick = action - if (component._component.endsWith("/tableblock")) { - component.clickBehaviour = "actions" - } - } - } - } - - // Migration 2: migrate table block title button settings - if (component._component.endsWith("/tableblock")) { - const { - showTitleButton, - titleButtonURL, - titleButtonPeek, - onClickTitleButton, - } = component - if (showTitleButton && !onClickTitleButton) { - const action = convertLinkSettingToAction( - titleButtonURL, - !!titleButtonPeek - ) - if (action) { - changed = true - component.onClickTitleButton = action - component.titleButtonClickBehaviour = "actions" - } - } - } - - // Recurse down the tree as needed - component._children?.forEach((child: any) => { - const childChanged = migrateTableSettings(child) - changed = changed || childChanged - }) - return changed -} - -// Util ti convert the legacy settings into a navigation action structure -const convertLinkSettingToAction = ( - linkURL: string, - linkPeek: boolean, - linkColumn?: string -) => { - // Sanity check we have a URL - if (!linkURL) { - return null - } - - // Default URL to the old URL setting - let url = linkURL - - // If we enriched the old URL with a column, update the url - if (linkColumn && linkURL.includes("/:")) { - // Convert old link URL setting, which is a screen URL, into a valid - // binding using the new clicked row binding - const split = linkURL.split("/:") - const col = linkColumn || "_id" - const binding = `{{ ${safe("eventContext")}.${safe("row")}.${safe(col)} }}` - url = `${split[0]}/${binding}` - } - - // Create action structure - return [ - { - "##eventHandlerType": "Navigate To", - parameters: { - url, - peek: linkPeek, - }, - }, - ] -} diff --git a/packages/server/src/migrations/functions/tests/appUrls.spec.js b/packages/server/src/migrations/functions/tests/appUrls.spec.js deleted file mode 100644 index 7a2f324552..0000000000 --- a/packages/server/src/migrations/functions/tests/appUrls.spec.js +++ /dev/null @@ -1,26 +0,0 @@ -const { db: dbCore } = require("@budibase/backend-core") -const TestConfig = require("../../../tests/utilities/TestConfiguration") - -const migration = require("../appUrls") - -describe("run", () => { - let config = new TestConfig(false) - - beforeAll(async () => { - await config.init() - }) - - afterAll(config.end) - - it("runs successfully", async () => { - const app = await config.createApp("testApp") - const metadata = await dbCore.doWithDB(app.appId, async db => { - const metadataDoc = await db.get(dbCore.DocumentType.APP_METADATA) - delete metadataDoc.url - await db.put(metadataDoc) - await migration.run(db) - return await db.get(dbCore.DocumentType.APP_METADATA) - }) - expect(metadata.url).toEqual("/testapp") - }) -}) diff --git a/packages/server/src/migrations/functions/tests/tableSettings.spec.ts b/packages/server/src/migrations/functions/tests/tableSettings.spec.ts deleted file mode 100644 index 8d28a43322..0000000000 --- a/packages/server/src/migrations/functions/tests/tableSettings.spec.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { App, Screen } from "@budibase/types" - -import { db as dbCore } from "@budibase/backend-core" -import TestConfig from "../../../tests/utilities/TestConfiguration" -import { run as runMigration } from "../tableSettings" - -describe("run", () => { - const config = new TestConfig(false) - let app: App - let screen: Screen - - beforeAll(async () => { - await config.init() - app = await config.createApp("testApp") - screen = await config.createScreen() - }) - - afterAll(config.end) - - it("migrates table block row on click settings", async () => { - // Add legacy table block as first child - screen.props._children = [ - { - _instanceName: "Table Block", - _styles: {}, - _component: "@budibase/standard-components/tableblock", - _id: "foo", - linkRows: true, - linkURL: "/rows/:id", - linkPeek: true, - linkColumn: "name", - }, - ] - await config.createScreen(screen) - - // Run migration - screen = await dbCore.doWithDB(app.appId, async (db: any) => { - await runMigration(db) - return await db.get(screen._id) - }) - - // Verify new "onClick" setting - const onClick = screen.props._children?.[0].onClick - expect(onClick).toBeDefined() - expect(onClick.length).toBe(1) - expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To") - expect(onClick[0].parameters.url).toBe( - `/rows/{{ [eventContext].[row].[name] }}` - ) - expect(onClick[0].parameters.peek).toBeTruthy() - }) - - it("migrates table row on click settings", async () => { - // Add legacy table block as first child - screen.props._children = [ - { - _instanceName: "Table", - _styles: {}, - _component: "@budibase/standard-components/table", - _id: "foo", - linkRows: true, - linkURL: "/rows/:id", - linkPeek: true, - linkColumn: "name", - }, - ] - await config.createScreen(screen) - - // Run migration - screen = await dbCore.doWithDB(app.appId, async (db: any) => { - await runMigration(db) - return await db.get(screen._id) - }) - - // Verify new "onClick" setting - const onClick = screen.props._children?.[0].onClick - expect(onClick).toBeDefined() - expect(onClick.length).toBe(1) - expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To") - expect(onClick[0].parameters.url).toBe( - `/rows/{{ [eventContext].[row].[name] }}` - ) - expect(onClick[0].parameters.peek).toBeTruthy() - }) - - it("migrates table block title button settings", async () => { - // Add legacy table block as first child - screen.props._children = [ - { - _instanceName: "Table Block", - _styles: {}, - _component: "@budibase/standard-components/tableblock", - _id: "foo", - showTitleButton: true, - titleButtonURL: "/url", - titleButtonPeek: true, - }, - ] - await config.createScreen(screen) - - // Run migration - screen = await dbCore.doWithDB(app.appId, async (db: any) => { - await runMigration(db) - return await db.get(screen._id) - }) - - // Verify new "onClickTitleButton" setting - const onClick = screen.props._children?.[0].onClickTitleButton - expect(onClick).toBeDefined() - expect(onClick.length).toBe(1) - expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To") - expect(onClick[0].parameters.url).toBe("/url") - expect(onClick[0].parameters.peek).toBeTruthy() - }) - - it("ignores components that have already been migrated", async () => { - // Add legacy table block as first child - screen.props._children = [ - { - _instanceName: "Table Block", - _styles: {}, - _component: "@budibase/standard-components/tableblock", - _id: "foo", - linkRows: true, - linkURL: "/rows/:id", - linkPeek: true, - linkColumn: "name", - onClick: "foo", - }, - ] - const initialDefinition = JSON.stringify(screen.props._children?.[0]) - await config.createScreen(screen) - - // Run migration - screen = await dbCore.doWithDB(app.appId, async (db: any) => { - await runMigration(db) - return await db.get(screen._id) - }) - - // Verify new "onClick" setting - const newDefinition = JSON.stringify(screen.props._children?.[0]) - expect(initialDefinition).toEqual(newDefinition) - }) -}) diff --git a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js b/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js deleted file mode 100644 index f0da25893c..0000000000 --- a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js +++ /dev/null @@ -1,34 +0,0 @@ -jest.mock("@budibase/backend-core", () => { - const core = jest.requireActual("@budibase/backend-core") - return { - ...core, - db: { - ...core.db, - createNewUserEmailView: jest.fn(), - }, - } -}) -const { context, db: dbCore } = require("@budibase/backend-core") -const TestConfig = require("../../../tests/utilities/TestConfiguration") - -// mock email view creation - -const migration = require("../userEmailViewCasing") - -describe("run", () => { - let config = new TestConfig(false) - - beforeAll(async () => { - await config.init() - }) - - afterAll(config.end) - - it("runs successfully", async () => { - await config.doInTenant(async () => { - const globalDb = context.getGlobalDB() - await migration.run(globalDb) - expect(dbCore.createNewUserEmailView).toHaveBeenCalledTimes(1) - }) - }) -}) diff --git a/packages/server/src/migrations/functions/usageQuotas/index.ts b/packages/server/src/migrations/functions/usageQuotas/index.ts deleted file mode 100644 index e94e993b21..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export const runQuotaMigration = async (migration: () => Promise) => { - await migration() -} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncApps.ts b/packages/server/src/migrations/functions/usageQuotas/syncApps.ts deleted file mode 100644 index 80ed1953d3..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/syncApps.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { db as dbCore } from "@budibase/backend-core" -import { quotas } from "@budibase/pro" -import { QuotaUsageType, StaticQuotaName } from "@budibase/types" - -export const run = async () => { - // get app count - const devApps = await dbCore.getAllApps({ dev: true }) - const appCount = devApps ? devApps.length : 0 - - // sync app count - console.log(`Syncing app count: ${appCount}`) - await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC) -} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts b/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts deleted file mode 100644 index ce53be925a..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { users } from "@budibase/backend-core" -import { quotas } from "@budibase/pro" -import { QuotaUsageType, StaticQuotaName } from "@budibase/types" - -export const run = async () => { - const creatorCount = await users.getCreatorCount() - console.log(`Syncing creator count: ${creatorCount}`) - await quotas.setUsage( - creatorCount, - StaticQuotaName.CREATORS, - QuotaUsageType.STATIC - ) -} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts b/packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts deleted file mode 100644 index b00970aea2..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { logging } from "@budibase/backend-core" -import { plugins } from "@budibase/pro" - -export const run = async () => { - try { - await plugins.checkPluginQuotas() - } catch (err) { - logging.logAlert("Failed to update plugin quotas", err) - } -} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncRows.ts b/packages/server/src/migrations/functions/usageQuotas/syncRows.ts deleted file mode 100644 index 506218a41f..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/syncRows.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { db as dbCore } from "@budibase/backend-core" -import { getUniqueRows } from "../../../utilities/usageQuota/rows" -import { quotas } from "@budibase/pro" -import { StaticQuotaName, QuotaUsageType, App } from "@budibase/types" - -export const run = async () => { - // get all rows in all apps - const allApps = (await dbCore.getAllApps({ all: true })) as App[] - const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : [] - const { appRows } = await getUniqueRows(appIds) - - // get the counts per app - const counts: { [key: string]: number } = {} - let rowCount = 0 - Object.entries(appRows).forEach(([appId, rows]) => { - counts[appId] = rows.length - rowCount += rows.length - }) - - // sync row count - console.log(`Syncing row count: ${rowCount}`) - await quotas.setUsagePerApp( - counts, - StaticQuotaName.ROWS, - QuotaUsageType.STATIC - ) -} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncUsers.ts b/packages/server/src/migrations/functions/usageQuotas/syncUsers.ts deleted file mode 100644 index c9913dced8..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/syncUsers.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { users } from "@budibase/backend-core" -import { quotas } from "@budibase/pro" -import { QuotaUsageType, StaticQuotaName } from "@budibase/types" - -export const run = async () => { - const userCount = await users.getUserCount() - console.log(`Syncing user count: ${userCount}`) - await quotas.setUsage(userCount, StaticQuotaName.USERS, QuotaUsageType.STATIC) -} diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts deleted file mode 100644 index 1d4d4d0f71..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts +++ /dev/null @@ -1,35 +0,0 @@ -import TestConfig from "../../../../tests/utilities/TestConfiguration" -import * as syncApps from "../syncApps" -import { quotas } from "@budibase/pro" -import { QuotaUsageType, StaticQuotaName } from "@budibase/types" - -describe("syncApps", () => { - let config = new TestConfig(false) - - beforeEach(async () => { - await config.init() - }) - - afterAll(config.end) - - it("runs successfully", async () => { - return config.doInContext(undefined, async () => { - // create the usage quota doc and mock usages - await quotas.getQuotaUsage() - await quotas.setUsage(3, StaticQuotaName.APPS, QuotaUsageType.STATIC) - - let usageDoc = await quotas.getQuotaUsage() - expect(usageDoc.usageQuota.apps).toEqual(3) - - // create an extra app to test the migration - await config.createApp("quota-test") - - // migrate - await syncApps.run() - - // assert the migration worked - usageDoc = await quotas.getQuotaUsage() - expect(usageDoc.usageQuota.apps).toEqual(2) - }) - }) -}) diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts deleted file mode 100644 index 93b7d4949b..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts +++ /dev/null @@ -1,26 +0,0 @@ -import TestConfig from "../../../../tests/utilities/TestConfiguration" -import * as syncCreators from "../syncCreators" -import { quotas } from "@budibase/pro" - -describe("syncCreators", () => { - let config = new TestConfig(false) - - beforeEach(async () => { - await config.init() - }) - - afterAll(config.end) - - it("syncs creators", async () => { - return config.doInContext(undefined, async () => { - await config.createUser({ admin: { global: true } }) - - await syncCreators.run() - - const usageDoc = await quotas.getQuotaUsage() - // default + additional creator - const creatorsCount = 2 - expect(usageDoc.usageQuota.creators).toBe(creatorsCount) - }) - }) -}) diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts deleted file mode 100644 index 730278683c..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts +++ /dev/null @@ -1,53 +0,0 @@ -import TestConfig from "../../../../tests/utilities/TestConfiguration" -import * as syncRows from "../syncRows" -import { quotas } from "@budibase/pro" -import { QuotaUsageType, StaticQuotaName } from "@budibase/types" -import { db as dbCore, context } from "@budibase/backend-core" - -describe("syncRows", () => { - const config = new TestConfig() - - beforeEach(async () => { - await config.init() - }) - - afterAll(config.end) - - it("runs successfully", async () => { - return config.doInContext(undefined, async () => { - // create the usage quota doc and mock usages - await quotas.getQuotaUsage() - await quotas.setUsage(300, StaticQuotaName.ROWS, QuotaUsageType.STATIC) - - let usageDoc = await quotas.getQuotaUsage() - expect(usageDoc.usageQuota.rows).toEqual(300) - - // app 1 - const app1 = config.app - await context.doInAppContext(app1!.appId, async () => { - await config.createTable() - await config.createRow() - }) - // app 2 - const app2 = await config.createApp("second-app") - await context.doInAppContext(app2.appId, async () => { - await config.createTable() - await config.createRow() - await config.createRow() - }) - - // migrate - await syncRows.run() - - // assert the migration worked - usageDoc = await quotas.getQuotaUsage() - expect(usageDoc.usageQuota.rows).toEqual(3) - expect( - usageDoc.apps?.[dbCore.getProdAppID(app1!.appId)].usageQuota.rows - ).toEqual(1) - expect( - usageDoc.apps?.[dbCore.getProdAppID(app2.appId)].usageQuota.rows - ).toEqual(2) - }) - }) -}) diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts deleted file mode 100644 index 2731cc041d..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts +++ /dev/null @@ -1,26 +0,0 @@ -import TestConfig from "../../../../tests/utilities/TestConfiguration" -import * as syncUsers from "../syncUsers" -import { quotas } from "@budibase/pro" - -describe("syncUsers", () => { - let config = new TestConfig(false) - - beforeEach(async () => { - await config.init() - }) - - afterAll(config.end) - - it("syncs users", async () => { - return config.doInContext(undefined, async () => { - await config.createUser() - - await syncUsers.run() - - const usageDoc = await quotas.getQuotaUsage() - // default + additional user - const userCount = 2 - expect(usageDoc.usageQuota.users).toBe(userCount) - }) - }) -}) diff --git a/packages/server/src/migrations/functions/userEmailViewCasing.ts b/packages/server/src/migrations/functions/userEmailViewCasing.ts deleted file mode 100644 index 078289cddf..0000000000 --- a/packages/server/src/migrations/functions/userEmailViewCasing.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { db as dbCore } from "@budibase/backend-core" - -/** - * Date: - * October 2021 - * - * Description: - * Recreate the user email view to include latest changes i.e. lower casing the email address - */ - -export const run = async () => { - await dbCore.createNewUserEmailView() -} diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts deleted file mode 100644 index a66d793142..0000000000 --- a/packages/server/src/migrations/index.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { locks, migrations } from "@budibase/backend-core" -import { - Migration, - MigrationOptions, - MigrationName, - LockType, - LockName, -} from "@budibase/types" -import env from "../environment" - -// migration functions -import * as userEmailViewCasing from "./functions/userEmailViewCasing" -import * as syncQuotas from "./functions/syncQuotas" -import * as appUrls from "./functions/appUrls" -import * as tableSettings from "./functions/tableSettings" -import * as backfill from "./functions/backfill" -/** - * Populate the migration function and additional configuration from - * the static migration definitions. - */ -export const buildMigrations = () => { - const definitions = migrations.DEFINITIONS - const serverMigrations: Migration[] = [] - - for (const definition of definitions) { - switch (definition.name) { - case MigrationName.USER_EMAIL_VIEW_CASING: { - serverMigrations.push({ - ...definition, - fn: userEmailViewCasing.run, - }) - break - } - case MigrationName.SYNC_QUOTAS: { - serverMigrations.push({ - ...definition, - fn: syncQuotas.run, - }) - break - } - case MigrationName.APP_URLS: { - serverMigrations.push({ - ...definition, - appOpts: { all: true }, - fn: appUrls.run, - }) - break - } - case MigrationName.EVENT_APP_BACKFILL: { - serverMigrations.push({ - ...definition, - appOpts: { all: true }, - fn: backfill.app.run, - silent: !!env.SELF_HOSTED, // reduce noisy logging - preventRetry: !!env.SELF_HOSTED, // only ever run once - }) - break - } - case MigrationName.EVENT_GLOBAL_BACKFILL: { - serverMigrations.push({ - ...definition, - fn: backfill.global.run, - silent: !!env.SELF_HOSTED, // reduce noisy logging - preventRetry: !!env.SELF_HOSTED, // only ever run once - }) - break - } - case MigrationName.EVENT_INSTALLATION_BACKFILL: { - serverMigrations.push({ - ...definition, - fn: backfill.installation.run, - silent: !!env.SELF_HOSTED, // reduce noisy logging - preventRetry: !!env.SELF_HOSTED, // only ever run once - }) - break - } - case MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS: { - serverMigrations.push({ - ...definition, - appOpts: { dev: true }, - fn: tableSettings.run, - }) - break - } - } - } - - return serverMigrations -} - -export const MIGRATIONS = buildMigrations() - -export const migrate = async (options?: MigrationOptions) => { - if (env.SELF_HOSTED) { - // self host runs migrations on startup - // make sure only a single instance runs them - await migrateWithLock(options) - } else { - await migrations.runMigrations(MIGRATIONS, options) - } -} - -const migrateWithLock = async (options?: MigrationOptions) => { - await locks.doWithLock( - { - type: LockType.TRY_ONCE, - name: LockName.MIGRATIONS, - ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes - systemLock: true, - }, - async () => { - await migrations.runMigrations(MIGRATIONS, options) - } - ) -} diff --git a/packages/server/src/migrations/tests/helpers.ts b/packages/server/src/migrations/tests/helpers.ts deleted file mode 100644 index 35831a2fd0..0000000000 --- a/packages/server/src/migrations/tests/helpers.ts +++ /dev/null @@ -1,40 +0,0 @@ -// Mimic configs test configuration from worker, creation configs directly in database - -import * as structures from "./structures" -import { configs } from "@budibase/backend-core" -import { Config } from "@budibase/types" - -export const saveSettingsConfig = async (globalDb: any) => { - const config = structures.settings() - await saveConfig(config, globalDb) -} - -export const saveGoogleConfig = async (globalDb: any) => { - const config = structures.google() - await saveConfig(config, globalDb) -} - -export const saveOIDCConfig = async (globalDb: any) => { - const config = structures.oidc() - await saveConfig(config, globalDb) -} - -export const saveSmtpConfig = async (globalDb: any) => { - const config = structures.smtp() - await saveConfig(config, globalDb) -} - -const saveConfig = async (config: Config, globalDb: any) => { - config._id = configs.generateConfigID(config.type) - - let response - try { - response = await globalDb.get(config._id) - config._rev = response._rev - await globalDb.put(config) - } catch (e: any) { - if (e.status === 404) { - await globalDb.put(config) - } - } -} diff --git a/packages/server/src/migrations/tests/index.spec.ts b/packages/server/src/migrations/tests/index.spec.ts deleted file mode 100644 index 3a23d8f011..0000000000 --- a/packages/server/src/migrations/tests/index.spec.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { - events, - migrations, - tenancy, - DocumentType, - context, -} from "@budibase/backend-core" -import TestConfig from "../../tests/utilities/TestConfiguration" -import * as structures from "../../tests/utilities/structures" -import { MIGRATIONS } from "../" -import * as helpers from "./helpers" - -import tk from "timekeeper" -import { View } from "@budibase/types" - -const timestamp = new Date().toISOString() -tk.freeze(timestamp) - -const clearMigrations = async () => { - const dbs = [context.getDevAppDB(), context.getProdAppDB()] - for (const db of dbs) { - const doc = await db.get(DocumentType.MIGRATIONS) - const newDoc = { _id: doc._id, _rev: doc._rev } - await db.put(newDoc) - } -} - -describe("migrations", () => { - const config = new TestConfig() - - beforeAll(async () => { - await config.init() - }) - - afterAll(() => { - config.end() - }) - - describe("backfill", () => { - it("runs app db migration", async () => { - await config.doInContext(undefined, async () => { - await clearMigrations() - await config.createAutomation() - await config.createAutomation(structures.newAutomation()) - await config.createDatasource() - await config.createDatasource() - await config.createLayout() - await config.createQuery() - await config.createQuery() - await config.createRole() - await config.createRole() - await config.createTable() - await config.createLegacyView() - await config.createTable() - await config.createLegacyView( - structures.view(config.table!._id!) as View - ) - await config.createScreen() - await config.createScreen() - - jest.clearAllMocks() - const migration = MIGRATIONS.filter( - m => m.name === "event_app_backfill" - )[0] - await migrations.runMigration(migration) - - expect(events.app.created).toHaveBeenCalledTimes(1) - expect(events.app.published).toHaveBeenCalledTimes(1) - expect(events.automation.created).toHaveBeenCalledTimes(2) - expect(events.automation.stepCreated).toHaveBeenCalledTimes(1) - expect(events.datasource.created).toHaveBeenCalledTimes(2) - expect(events.layout.created).toHaveBeenCalledTimes(1) - expect(events.query.created).toHaveBeenCalledTimes(2) - expect(events.role.created).toHaveBeenCalledTimes(3) // created roles + admin (created on table creation) - expect(events.table.created).toHaveBeenCalledTimes(3) - expect(events.backfill.appSucceeded).toHaveBeenCalledTimes(2) - - // to make sure caching is working as expected - expect( - events.processors.analyticsProcessor.processEvent - ).toHaveBeenCalledTimes(20) // Addition of of the events above - }) - }) - }) - - it("runs global db migration", async () => { - await config.doInContext(undefined, async () => { - await clearMigrations() - const appId = config.getProdAppId() - const roles = { [appId]: "role_12345" } - await config.createUser({ - builder: { global: false }, - admin: { global: true }, - roles, - }) // admin only - await config.createUser({ - builder: { global: false }, - admin: { global: false }, - roles, - }) // non admin non builder - await config.createTable() - await config.createRow() - await config.createRow() - - const db = tenancy.getGlobalDB() - await helpers.saveGoogleConfig(db) - await helpers.saveOIDCConfig(db) - await helpers.saveSettingsConfig(db) - await helpers.saveSmtpConfig(db) - - jest.clearAllMocks() - const migration = MIGRATIONS.filter( - m => m.name === "event_global_backfill" - )[0] - await migrations.runMigration(migration) - - expect(events.user.created).toHaveBeenCalledTimes(3) - expect(events.role.assigned).toHaveBeenCalledTimes(2) - expect(events.user.permissionBuilderAssigned).toHaveBeenCalledTimes(1) // default test user - expect(events.user.permissionAdminAssigned).toHaveBeenCalledTimes(1) // admin from above - expect(events.rows.created).toHaveBeenCalledTimes(1) - expect(events.rows.created).toHaveBeenCalledWith(2, timestamp) - expect(events.email.SMTPCreated).toHaveBeenCalledTimes(1) - expect(events.auth.SSOCreated).toHaveBeenCalledTimes(2) - expect(events.auth.SSOActivated).toHaveBeenCalledTimes(2) - expect(events.org.logoUpdated).toHaveBeenCalledTimes(1) - expect(events.org.nameUpdated).toHaveBeenCalledTimes(1) - expect(events.org.platformURLUpdated).toHaveBeenCalledTimes(1) - expect(events.backfill.tenantSucceeded).toHaveBeenCalledTimes(1) - - // to make sure caching is working as expected - expect( - events.processors.analyticsProcessor.processEvent - ).toHaveBeenCalledTimes(19) - }) - }) -}) diff --git a/packages/server/src/migrations/tests/structures.ts b/packages/server/src/migrations/tests/structures.ts deleted file mode 100644 index e2113e6a7c..0000000000 --- a/packages/server/src/migrations/tests/structures.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { utils } from "@budibase/backend-core" -import { - SMTPConfig, - OIDCConfig, - GoogleConfig, - SettingsConfig, - ConfigType, -} from "@budibase/types" - -export const oidc = (conf?: OIDCConfig): OIDCConfig => { - return { - type: ConfigType.OIDC, - config: { - configs: [ - { - configUrl: "http://someconfigurl", - clientID: "clientId", - clientSecret: "clientSecret", - logo: "Microsoft", - name: "Active Directory", - uuid: utils.newid(), - activated: true, - scopes: [], - ...conf, - }, - ], - }, - } -} - -export const google = (conf?: GoogleConfig): GoogleConfig => { - return { - type: ConfigType.GOOGLE, - config: { - clientID: "clientId", - clientSecret: "clientSecret", - activated: true, - ...conf, - }, - } -} - -export const smtp = (conf?: SMTPConfig): SMTPConfig => { - return { - type: ConfigType.SMTP, - config: { - port: 12345, - host: "smtptesthost.com", - from: "testfrom@example.com", - subject: "Hello!", - secure: false, - ...conf, - }, - } -} - -export const settings = (conf?: SettingsConfig): SettingsConfig => { - return { - type: ConfigType.SETTINGS, - config: { - platformUrl: "http://mycustomdomain.com", - logoUrl: "http://mylogourl,com", - company: "mycompany", - ...conf, - }, - } -} diff --git a/packages/server/src/startup/index.ts b/packages/server/src/startup/index.ts index edca64db7d..54e982ae56 100644 --- a/packages/server/src/startup/index.ts +++ b/packages/server/src/startup/index.ts @@ -15,7 +15,6 @@ import { watch } from "../watch" import * as automations from "../automations" import * as fileSystem from "../utilities/fileSystem" import { default as eventEmitter, init as eventInit } from "../events" -import * as migrations from "../migrations" import * as bullboard from "../automations/bullboard" import * as appMigrations from "../appMigrations/queue" import * as pro from "@budibase/pro" @@ -106,18 +105,6 @@ export async function startup( initialiseWebsockets(app, server) } - // run migrations on startup if not done via http - // not recommended in a clustered environment - if (!env.HTTP_MIGRATIONS && !env.isTest()) { - console.log("Running migrations") - try { - await migrations.migrate() - } catch (e) { - logging.logAlert("Error performing migrations. Exiting.", e) - shutdown(server) - } - } - // monitor plugin directory if required if ( env.SELF_HOSTED && diff --git a/packages/types/src/api/web/global/oldMigration.ts b/packages/types/src/api/web/global/oldMigration.ts index 812ee1e593..f0da6c52f3 100644 --- a/packages/types/src/api/web/global/oldMigration.ts +++ b/packages/types/src/api/web/global/oldMigration.ts @@ -1,12 +1,3 @@ -import { Migration, MigrationOptions } from "../../../sdk" - -export interface RunOldMigrationRequest extends MigrationOptions {} -export interface RuneOldMigrationResponse { - message: string -} - -export type FetchOldMigrationResponse = Migration[] - -export interface GetOldMigrationStatus { +export interface GetMigrationStatus { migrated: boolean } diff --git a/packages/types/src/api/web/system/index.ts b/packages/types/src/api/web/system/index.ts index 9b03ddd438..18ed533e9d 100644 --- a/packages/types/src/api/web/system/index.ts +++ b/packages/types/src/api/web/system/index.ts @@ -3,6 +3,5 @@ export * from "./status" export * from "./ops" export * from "./account" export * from "./log" -export * from "./migration" export * from "./restore" export * from "./tenant" diff --git a/packages/types/src/api/web/system/migration.ts b/packages/types/src/api/web/system/migration.ts deleted file mode 100644 index a18112744c..0000000000 --- a/packages/types/src/api/web/system/migration.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { MigrationDefinition, MigrationOptions } from "../../../sdk" - -export interface RunGlobalMigrationRequest extends MigrationOptions {} -export interface RunGlobalMigrationResponse { - message: string -} - -export type FetchMigrationDefinitionsResponse = MigrationDefinition[] diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index 86eb5b1a24..eb9e23b3d1 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -4,7 +4,6 @@ export * from "./hosting" export * from "./context" export * from "./events" export * from "./licensing" -export * from "./migrations" export * from "./datasources" export * from "./search" export * from "./koa" diff --git a/packages/types/src/sdk/migrations.ts b/packages/types/src/sdk/migrations.ts deleted file mode 100644 index 6db0c85879..0000000000 --- a/packages/types/src/sdk/migrations.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { Database } from "./db" - -export interface Migration extends MigrationDefinition { - appOpts?: object - fn: (db: Database) => Promise - silent?: boolean - preventRetry?: boolean -} - -export enum MigrationType { - // run once per tenant, recorded in global db, global db is provided as an argument - GLOBAL = "global", - // run per app, recorded in each app db, app db is provided as an argument - APP = "app", - // run once, recorded in global info db, global info db is provided as an argument - INSTALLATION = "installation", -} - -export interface MigrationNoOpOptions { - type: MigrationType - tenantId: string - appId?: string -} - -/** - * e.g. - * { - * tenantIds: ['bb'], - * force: { - * global: ['quota_1'] - * } - * } - */ -export interface MigrationOptions { - tenantIds?: string[] - force?: { - [type: string]: string[] - } - noOp?: MigrationNoOpOptions -} - -export enum MigrationName { - USER_EMAIL_VIEW_CASING = "user_email_view_casing", - APP_URLS = "app_urls", - EVENT_APP_BACKFILL = "event_app_backfill", - EVENT_GLOBAL_BACKFILL = "event_global_backfill", - EVENT_INSTALLATION_BACKFILL = "event_installation_backfill", - GLOBAL_INFO_SYNC_USERS = "global_info_sync_users", - TABLE_SETTINGS_LINKS_TO_ACTIONS = "table_settings_links_to_actions", - // increment this number to re-activate this migration - SYNC_QUOTAS = "sync_quotas_2", -} - -export interface MigrationDefinition { - type: MigrationType - name: MigrationName -} diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 0bcdadfefc..e36c45a3ba 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -28,7 +28,6 @@ import { LockType, LookupAccountHolderResponse, LookupTenantUserResponse, - MigrationType, PlatformUserByEmail, SaveUserResponse, SearchUsersRequest, @@ -45,7 +44,6 @@ import { cache, ErrorCode, events, - migrations, platform, tenancy, db, @@ -187,10 +185,6 @@ export const adminUser = async ( if (env.MULTI_TENANCY) { // store the new tenant record in the platform db await platform.tenants.addTenant(tenantId) - await migrations.backPopulateMigrations({ - type: MigrationType.GLOBAL, - tenantId, - }) } await tenancy.doInTenant(tenantId, async () => { diff --git a/packages/worker/src/api/controllers/system/migrations.ts b/packages/worker/src/api/controllers/system/migrations.ts deleted file mode 100644 index fc253d839d..0000000000 --- a/packages/worker/src/api/controllers/system/migrations.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { - FetchMigrationDefinitionsResponse, - RunGlobalMigrationRequest, - RunGlobalMigrationResponse, - UserCtx, -} from "@budibase/types" - -const { migrate, MIGRATIONS } = require("../../../migrations") - -export const runMigrations = async ( - ctx: UserCtx -) => { - const options = ctx.request.body - // don't await as can take a while, just return - migrate(options) - ctx.body = { message: "Migration started." } -} - -export const fetchDefinitions = async ( - ctx: UserCtx -) => { - ctx.body = MIGRATIONS -} diff --git a/packages/worker/src/api/routes/index.ts b/packages/worker/src/api/routes/index.ts index 741026543c..d4ddb41522 100644 --- a/packages/worker/src/api/routes/index.ts +++ b/packages/worker/src/api/routes/index.ts @@ -12,7 +12,6 @@ import tenantsRoutes from "./system/tenants" import statusRoutes from "./system/status" import selfRoutes from "./global/self" import licenseRoutes from "./global/license" -import migrationRoutes from "./system/migrations" import accountRoutes from "./system/accounts" import restoreRoutes from "./system/restore" import systemLogRoutes from "./system/logs" @@ -34,7 +33,6 @@ export const routes: Router[] = [ licenseRoutes, pro.groups, pro.auditLogs, - migrationRoutes, accountRoutes, restoreRoutes, eventRoutes, diff --git a/packages/worker/src/api/routes/system/migrations.ts b/packages/worker/src/api/routes/system/migrations.ts deleted file mode 100644 index a8189b5a91..0000000000 --- a/packages/worker/src/api/routes/system/migrations.ts +++ /dev/null @@ -1,19 +0,0 @@ -import Router from "@koa/router" -import * as migrationsController from "../../controllers/system/migrations" -import { auth } from "@budibase/backend-core" - -const router: Router = new Router() - -router - .post( - "/api/system/migrations/run", - auth.internalApi, - migrationsController.runMigrations - ) - .get( - "/api/system/migrations/definitions", - auth.internalApi, - migrationsController.fetchDefinitions - ) - -export default router diff --git a/packages/worker/src/migrations/index.ts b/packages/worker/src/migrations/index.ts deleted file mode 100644 index 642fbeb54e..0000000000 --- a/packages/worker/src/migrations/index.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { migrations, locks } from "@budibase/backend-core" -import { - Migration, - MigrationOptions, - MigrationName, - LockType, - LockName, -} from "@budibase/types" -import env from "../environment" - -// migration functions -import * as syncUserInfo from "./functions/globalInfoSyncUsers" - -/** - * Populate the migration function and additional configuration from - * the static migration definitions. - */ -export const buildMigrations = () => { - const definitions = migrations.DEFINITIONS - const workerMigrations: Migration[] = [] - - for (const definition of definitions) { - switch (definition.name) { - case MigrationName.GLOBAL_INFO_SYNC_USERS: { - // only needed in cloud - if (!env.SELF_HOSTED) { - workerMigrations.push({ - ...definition, - fn: syncUserInfo.run, - }) - } - break - } - } - } - - return workerMigrations -} - -export const MIGRATIONS = buildMigrations() - -export const migrate = async (options?: MigrationOptions) => { - if (env.SELF_HOSTED) { - await migrateWithLock(options) - } else { - await migrations.runMigrations(MIGRATIONS, options) - } -} - -const migrateWithLock = async (options?: MigrationOptions) => { - await locks.doWithLock( - { - type: LockType.TRY_ONCE, - name: LockName.MIGRATIONS, - ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes - systemLock: true, - }, - async () => { - await migrations.runMigrations(MIGRATIONS, options) - } - ) -} From ff4f8f1d666f7a68c2df54987c5f2454192098c0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 15 Jan 2025 17:45:28 +0000 Subject: [PATCH 010/112] Remove unused test. --- .../routes/system/tests/migrations.spec.ts | 63 ------------------- packages/worker/src/tests/api/index.ts | 3 - packages/worker/src/tests/api/migrations.ts | 17 ----- 3 files changed, 83 deletions(-) delete mode 100644 packages/worker/src/api/routes/system/tests/migrations.spec.ts delete mode 100644 packages/worker/src/tests/api/migrations.ts diff --git a/packages/worker/src/api/routes/system/tests/migrations.spec.ts b/packages/worker/src/api/routes/system/tests/migrations.spec.ts deleted file mode 100644 index fe91a1070c..0000000000 --- a/packages/worker/src/api/routes/system/tests/migrations.spec.ts +++ /dev/null @@ -1,63 +0,0 @@ -const migrateFn = jest.fn() - -import { TestConfiguration } from "../../../../tests" - -jest.mock("../../../../migrations", () => { - return { - ...jest.requireActual("../../../../migrations"), - migrate: migrateFn, - } -}) - -describe("/api/system/migrations", () => { - const config = new TestConfiguration() - - beforeAll(async () => { - await config.beforeAll() - }) - - afterAll(async () => { - await config.afterAll() - }) - - afterEach(() => { - jest.clearAllMocks() - }) - - describe("POST /api/system/migrations/run", () => { - it("fails with no internal api key", async () => { - const res = await config.api.migrations.runMigrations({ - headers: {}, - status: 403, - }) - expect(res.body).toEqual({ message: "Unauthorized", status: 403 }) - expect(migrateFn).toHaveBeenCalledTimes(0) - }) - - it("runs migrations", async () => { - const res = await config.api.migrations.runMigrations() - expect(res.body.message).toBeDefined() - expect(migrateFn).toHaveBeenCalledTimes(1) - }) - }) - - describe("DELETE /api/system/migrations/definitions", () => { - it("fails with no internal api key", async () => { - const res = await config.api.migrations.getMigrationDefinitions({ - headers: {}, - status: 403, - }) - expect(res.body).toEqual({ message: "Unauthorized", status: 403 }) - }) - - it("returns definitions", async () => { - const res = await config.api.migrations.getMigrationDefinitions() - expect(res.body).toEqual([ - { - name: "global_info_sync_users", - type: "global", - }, - ]) - }) - }) -}) diff --git a/packages/worker/src/tests/api/index.ts b/packages/worker/src/tests/api/index.ts index fa4e54184c..0f0334c8da 100644 --- a/packages/worker/src/tests/api/index.ts +++ b/packages/worker/src/tests/api/index.ts @@ -6,7 +6,6 @@ import { EmailAPI } from "./email" import { SelfAPI } from "./self" import { UserAPI } from "./users" import { EnvironmentAPI } from "./environment" -import { MigrationAPI } from "./migrations" import { StatusAPI } from "./status" import { RestoreAPI } from "./restore" import { TenantAPI } from "./tenants" @@ -26,7 +25,6 @@ export default class API { self: SelfAPI users: UserAPI environment: EnvironmentAPI - migrations: MigrationAPI status: StatusAPI restore: RestoreAPI tenants: TenantAPI @@ -46,7 +44,6 @@ export default class API { this.self = new SelfAPI(config) this.users = new UserAPI(config) this.environment = new EnvironmentAPI(config) - this.migrations = new MigrationAPI(config) this.status = new StatusAPI(config) this.restore = new RestoreAPI(config) this.tenants = new TenantAPI(config) diff --git a/packages/worker/src/tests/api/migrations.ts b/packages/worker/src/tests/api/migrations.ts deleted file mode 100644 index 6d62fe994a..0000000000 --- a/packages/worker/src/tests/api/migrations.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { TestAPI, TestAPIOpts } from "./base" - -export class MigrationAPI extends TestAPI { - runMigrations = (opts?: TestAPIOpts) => { - return this.request - .post(`/api/system/migrations/run`) - .set(opts?.headers ? opts.headers : this.config.internalAPIHeaders()) - .expect(opts?.status ? opts.status : 200) - } - - getMigrationDefinitions = (opts?: TestAPIOpts) => { - return this.request - .get(`/api/system/migrations/definitions`) - .set(opts?.headers ? opts.headers : this.config.internalAPIHeaders()) - .expect(opts?.status ? opts.status : 200) - } -} From ec1e145eb54a919ea8c9052a0a2778d711840e6f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 15 Jan 2025 18:06:49 +0000 Subject: [PATCH 011/112] Formatting for log lines. --- .../bindings/EvaluationSidePanel.svelte | 41 ++++++++++++++++--- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte index c8bf5529ad..54f7dc7d01 100644 --- a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte +++ b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte @@ -11,11 +11,16 @@ export let expressionError: string | undefined = undefined export let evaluating = false export let expression: string | null = null + export let logging: { log: string; line?: number }[] = [] $: error = expressionError != null $: empty = expression == null || expression?.trim() === "" $: success = !error && !empty $: highlightedResult = highlight(expressionResult) + $: highlightedLogs = logging.map(l => ({ + log: highlight(l.log), + line: l.line, + })) const formatError = (err: any) => { if (err.code === UserScriptError.code) { @@ -25,14 +30,14 @@ } // json can be any primitive type - const highlight = (json?: any | null) => { + const highlight = (json?: JSONValue | null) => { if (json == null) { return "" } // Attempt to parse and then stringify, in case this is valid result try { - json = JSON.stringify(JSON.parse(json), null, 2) + json = JSON.stringify(JSON.parse(json as any), null, 2) } catch (err) { // couldn't parse/stringify, just treat it as the raw input } @@ -90,8 +95,21 @@ {:else if error} {formatError(expressionError)} {:else} - - {@html highlightedResult} +
+ {#each highlightedLogs as logLine} +
+ + {@html logLine.log} + {#if logLine.line} + line {logLine.line} + {/if} +
+ {/each} +
+ + {@html highlightedResult} +
+
{/if} @@ -142,8 +160,21 @@ font-size: 12px; overflow-y: scroll; overflow-x: hidden; - white-space: pre-wrap; + white-space: pre-line; word-wrap: break-word; height: 0; } + .output-lines { + display: flex; + gap: var(--spacing-s); + flex-direction: column; + } + .line { + border-bottom: var(--border-light); + padding-bottom: var(--spacing-s); + display: flex; + flex-direction: row; + justify-content: space-between; + align-items: end; + } From 6f9f36f9eb8d1c13a0c060762698c2a9f167c028 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 15 Jan 2025 18:38:23 +0000 Subject: [PATCH 012/112] Getting string templates ready. --- .../src/helpers/javascript.ts | 22 +++++++++++++-- .../string-templates/src/processors/index.ts | 28 +++++++++++++++---- .../src/processors/postprocessor.ts | 28 +++++++++++-------- .../src/processors/preprocessor.ts | 7 +++-- packages/string-templates/src/types.ts | 5 ++++ 5 files changed, 69 insertions(+), 21 deletions(-) diff --git a/packages/string-templates/src/helpers/javascript.ts b/packages/string-templates/src/helpers/javascript.ts index 4fb1329196..a0fdd3cbe5 100644 --- a/packages/string-templates/src/helpers/javascript.ts +++ b/packages/string-templates/src/helpers/javascript.ts @@ -4,6 +4,7 @@ import { getJsHelperList } from "./list" import { iifeWrapper } from "../iife" import { JsTimeoutError, UserScriptError } from "../errors" import { cloneDeep } from "lodash/fp" +import { Log } from "../types" // The method of executing JS scripts depends on the bundle being built. // This setter is used in the entrypoint (either index.js or index.mjs). @@ -96,10 +97,9 @@ export function processJS(handlebars: string, context: any) { clonedContext = cloneDeep(context) } - const sandboxContext = { + const sandboxContext: Record = { $: (path: string) => getContextValue(path, clonedContext), helpers: getJsHelperList(), - // Proxy to evaluate snippets when running in the browser snippets: new Proxy( {}, @@ -114,8 +114,24 @@ export function processJS(handlebars: string, context: any) { ), } + const logs: Log[] = [] + // logging only supported on frontend + if (!isBackendService()) { + const log = (log: string) => logs.push({ log }) + sandboxContext.console = { + log: log, + info: log, + debug: log, + warn: log, + error: log, + // two below may need special cases + trace: log, + table: log, + } + } + // Create a sandbox with our context and run the JS - const res = { data: runJS(js, sandboxContext) } + const res = { data: runJS(js, sandboxContext), logs } return `{{${LITERAL_MARKER} js_result-${JSON.stringify(res)}}}` } catch (error: any) { onErrorLog && onErrorLog(error) diff --git a/packages/string-templates/src/processors/index.ts b/packages/string-templates/src/processors/index.ts index 79085b0dfe..4454d02738 100644 --- a/packages/string-templates/src/processors/index.ts +++ b/packages/string-templates/src/processors/index.ts @@ -1,9 +1,16 @@ import { FIND_HBS_REGEX } from "../utilities" import * as preprocessor from "./preprocessor" +import type { Preprocessor } from "./preprocessor" import * as postprocessor from "./postprocessor" -import { ProcessOptions } from "../types" +import type { Postprocessor } from "./postprocessor" +import { Log, ProcessOptions } from "../types" -function process(output: string, processors: any[], opts?: ProcessOptions) { +function process( + output: string, + processors: (Preprocessor | Postprocessor)[], + opts?: ProcessOptions +) { + let logs: Log[] = [] for (let processor of processors) { // if a literal statement has occurred stop if (typeof output !== "string") { @@ -16,10 +23,16 @@ function process(output: string, processors: any[], opts?: ProcessOptions) { continue } for (let match of matches) { - output = processor.process(output, match, opts) + const res = processor.process(output, match, opts || {}) + if (typeof res === "object" && "logs" in res && res.logs) { + logs = logs.concat(res.logs) + output = res.result + } else { + output = res as string + } } } - return output + return { result: output, logs } } export function preprocess(string: string, opts: ProcessOptions) { @@ -30,8 +43,13 @@ export function preprocess(string: string, opts: ProcessOptions) { ) } - return process(string, processors, opts) + return process(string, processors, opts).result } + export function postprocess(string: string) { + return process(string, postprocessor.processors).result +} + +export function postprocessWithLogs(string: string) { return process(string, postprocessor.processors) } diff --git a/packages/string-templates/src/processors/postprocessor.ts b/packages/string-templates/src/processors/postprocessor.ts index 6ddc0e67cd..49d5f7d1cf 100644 --- a/packages/string-templates/src/processors/postprocessor.ts +++ b/packages/string-templates/src/processors/postprocessor.ts @@ -1,12 +1,16 @@ import { LITERAL_MARKER } from "../helpers/constants" +import { Log } from "../types" export enum PostProcessorNames { CONVERT_LITERALS = "convert-literals", } -type PostprocessorFn = (statement: string) => string +export type PostprocessorFn = (statement: string) => { + result: any + logs?: Log[] +} -class Postprocessor { +export class Postprocessor { name: PostProcessorNames private readonly fn: PostprocessorFn @@ -23,12 +27,12 @@ class Postprocessor { export const processors = [ new Postprocessor( PostProcessorNames.CONVERT_LITERALS, - (statement: string) => { + (statement: string): { result: any; logs?: Log[] } => { if ( typeof statement !== "string" || !statement.includes(LITERAL_MARKER) ) { - return statement + return { result: statement } } const splitMarkerIndex = statement.indexOf("-") const type = statement.substring(12, splitMarkerIndex) @@ -38,20 +42,22 @@ export const processors = [ ) switch (type) { case "string": - return value + return { result: value } case "number": - return parseFloat(value) + return { result: parseFloat(value) } case "boolean": - return value === "true" + return { result: value === "true" } case "object": - return JSON.parse(value) - case "js_result": + return { result: JSON.parse(value) } + case "js_result": { // We use the literal helper to process the result of JS expressions // as we want to be able to return any types. // We wrap the value in an abject to be able to use undefined properly. - return JSON.parse(value).data + const parsed = JSON.parse(value) + return { result: parsed.data, logs: parsed.logs } + } } - return value + return { result: value } } ), ] diff --git a/packages/string-templates/src/processors/preprocessor.ts b/packages/string-templates/src/processors/preprocessor.ts index 97e5c56fcc..37981f31a8 100644 --- a/packages/string-templates/src/processors/preprocessor.ts +++ b/packages/string-templates/src/processors/preprocessor.ts @@ -11,9 +11,12 @@ export enum PreprocessorNames { NORMALIZE_SPACES = "normalize-spaces", } -type PreprocessorFn = (statement: string, opts?: ProcessOptions) => string +export type PreprocessorFn = ( + statement: string, + opts?: ProcessOptions +) => string -class Preprocessor { +export class Preprocessor { name: string private readonly fn: PreprocessorFn diff --git a/packages/string-templates/src/types.ts b/packages/string-templates/src/types.ts index 2a7a430bee..c973142c93 100644 --- a/packages/string-templates/src/types.ts +++ b/packages/string-templates/src/types.ts @@ -8,3 +8,8 @@ export interface ProcessOptions { onlyFound?: boolean disabledHelpers?: string[] } + +export interface Log { + log: string + line?: number +} From 28958f5a1ce32b46bd6415de0c6d10d55f27a3eb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 16 Jan 2025 15:29:50 +0000 Subject: [PATCH 013/112] Expose the ability to get logs. --- packages/string-templates/src/index.ts | 82 ++++++++++++++++++++------ 1 file changed, 65 insertions(+), 17 deletions(-) diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index 7246bc4942..fafff5d9fb 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -1,7 +1,7 @@ import { createContext, runInNewContext } from "vm" import { create, TemplateDelegate } from "handlebars" import { registerAll, registerMinimum } from "./helpers/index" -import { postprocess, preprocess } from "./processors" +import { postprocess, postprocessWithLogs, preprocess } from "./processors" import { atob, btoa, @@ -15,7 +15,7 @@ import { convertHBSBlock } from "./conversion" import { removeJSRunner, setJSRunner } from "./helpers/javascript" import manifest from "./manifest.json" -import { ProcessOptions } from "./types" +import { Log, ProcessOptions } from "./types" import { UserScriptError } from "./errors" export { helpersToRemoveForJs, getJsHelperList } from "./helpers/list" @@ -187,23 +187,27 @@ export function processObjectSync( return object } -/** - * This will process a single handlebars containing string. If the string passed in has no valid handlebars statements - * then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call. - * @param {string} string The template string which is the filled from the context object. - * @param {object} context An object of information which will be used to enrich the string. - * @param {object|undefined} [opts] optional - specify some options for processing. - * @returns {string} The enriched string, all templates should have been replaced if they can be. - */ -export function processStringSync( +// keep the logging function internal, don't want to add this to the process options directly +// as it can't be used for object processing etc. +function processStringSyncInternal( + str: string, + context?: object, + opts?: ProcessOptions & { logging: false } +): string +function processStringSyncInternal( + str: string, + context?: object, + opts?: ProcessOptions & { logging: true } +): { result: string; logs: Log[] } +function processStringSyncInternal( string: string, context?: object, - opts?: ProcessOptions -): string { + opts?: ProcessOptions & { logging: boolean } +): string | { result: string; logs: Log[] } { // Take a copy of input in case of error const input = string if (typeof string !== "string") { - throw "Cannot process non-string types." + throw new Error("Cannot process non-string types.") } function process(stringPart: string) { // context is needed to check for overlap between helpers and context @@ -217,16 +221,24 @@ export function processStringSync( }, ...context, }) - return postprocess(processedString) + return opts?.logging + ? postprocessWithLogs(processedString) + : postprocess(processedString) } try { if (opts && opts.onlyFound) { + let logs: Log[] = [] const blocks = findHBSBlocks(string) for (let block of blocks) { const outcome = process(block) - string = string.replace(block, outcome) + if (typeof outcome === "object" && "result" in outcome) { + logs = logs.concat(outcome.logs || []) + string = string.replace(block, outcome.result) + } else { + string = string.replace(block, outcome) + } } - return string + return opts?.logging ? string : { result: string, logs } } else { return process(string) } @@ -239,6 +251,42 @@ export function processStringSync( } } +/** + * This will process a single handlebars containing string. If the string passed in has no valid handlebars statements + * then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call. + * @param {string} string The template string which is the filled from the context object. + * @param {object} context An object of information which will be used to enrich the string. + * @param {object|undefined} [opts] optional - specify some options for processing. + * @returns {string} The enriched string, all templates should have been replaced if they can be. + */ +export function processStringSync( + string: string, + context?: object, + opts?: ProcessOptions +): string { + return processStringSyncInternal(string, context, { + ...opts, + logging: false, + }) +} + +/** + * Same as function above, but allows logging to be returned - this is only for JS bindings. + */ +export function processStringWithLogsSync( + string: string, + context?: object, + opts?: ProcessOptions +): { result: string; logs: Log[] } { + if (isBackendService()) { + throw new Error("Logging disabled for backend bindings") + } + return processStringSyncInternal(string, context, { + ...opts, + logging: true, + }) +} + /** * By default with expressions like {{ name }} handlebars will escape various * characters, which can be problematic. To fix this we use the syntax {{{ name }}}, From f74e2c6693a757c6eab28b2323a9d50eaa5018e6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 16 Jan 2025 16:40:51 +0100 Subject: [PATCH 014/112] Convert form block --- .../app/blocks/form/FormBlock.svelte | 31 ++++++++++--------- packages/client/src/index.ts | 4 +++ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte index 656aa5933b..69f51b84e8 100644 --- a/packages/client/src/components/app/blocks/form/FormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte @@ -1,12 +1,14 @@ - + +export type Context = Readable From a466a2d6705ef934f0ce03b839a87fe07e06557b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 16 Jan 2025 16:44:41 +0100 Subject: [PATCH 015/112] Type params --- .../app/blocks/form/FormBlock.svelte | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte index 69f51b84e8..2a0c622750 100644 --- a/packages/client/src/components/app/blocks/form/FormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte @@ -7,21 +7,24 @@ import { Component, Context, SDK } from "../../../../index" import { TableSchema, UIDatasource } from "@budibase/types" - export let actionType + export let actionType: string export let dataSource: UIDatasource - export let size - export let disabled + export let size: string + export let disabled: boolean export let fields - export let buttons - export let buttonPosition - export let title - export let description - export let rowId - export let actionUrl - export let noRowsMessage - export let notificationOverride - export let buttonsCollapsed - export let buttonsCollapsedText + export let buttons: { + "##eventHandlerType": string + parameters: Record + }[] + export let buttonPosition: "top" | "bottom" + export let title: string + export let description: string + export let rowId: string + export let actionUrl: string + export let noRowsMessage: string + export let notificationOverride: boolean + export let buttonsCollapsed: boolean + export let buttonsCollapsedText: string // Legacy export let showDeleteButton From 029b7754ca374719f248da4a77a8855a1fdeec5d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 16 Jan 2025 16:44:55 +0100 Subject: [PATCH 016/112] Type other params --- .../src/components/app/blocks/form/FormBlock.svelte | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte index 2a0c622750..22f21b5790 100644 --- a/packages/client/src/components/app/blocks/form/FormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte @@ -27,10 +27,10 @@ export let buttonsCollapsedText: string // Legacy - export let showDeleteButton - export let showSaveButton - export let saveButtonLabel - export let deleteButtonLabel + export let showDeleteButton: boolean + export let showSaveButton: boolean + export let saveButtonLabel: boolean + export let deleteButtonLabel: boolean const { fetchDatasourceSchema, generateGoldenSample } = getContext("sdk") const component = getContext("component") From d3a2306787d15b9051571572c3b12fcdb6295c76 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 16 Jan 2025 15:48:49 +0000 Subject: [PATCH 017/112] Finishing link up of logs. --- .../src/components/common/bindings/BindingPanel.svelte | 9 +++++++-- .../common/bindings/EvaluationSidePanel.svelte | 5 +++-- packages/string-templates/src/index.ts | 1 + 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/components/common/bindings/BindingPanel.svelte b/packages/builder/src/components/common/bindings/BindingPanel.svelte index 98df69bc06..ffb477012c 100644 --- a/packages/builder/src/components/common/bindings/BindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/BindingPanel.svelte @@ -12,7 +12,7 @@ decodeJSBinding, encodeJSBinding, processObjectSync, - processStringSync, + processStringWithLogsSync, } from "@budibase/string-templates" import { readableToRuntimeBinding } from "@/dataBinding" import CodeEditor from "../CodeEditor/CodeEditor.svelte" @@ -41,6 +41,7 @@ InsertAtPositionFn, JSONValue, } from "@budibase/types" + import type { Log } from "@budibase/string-templates" import type { CompletionContext } from "@codemirror/autocomplete" const dispatch = createEventDispatcher() @@ -66,6 +67,7 @@ let insertAtPos: InsertAtPositionFn | undefined let targetMode: BindingMode | null = null let expressionResult: string | undefined + let expressionLogs: Log[] | undefined let expressionError: string | undefined let evaluating = false @@ -157,7 +159,7 @@ (expression: string | null, context: any, snippets: Snippet[]) => { try { expressionError = undefined - expressionResult = processStringSync( + const output = processStringWithLogsSync( expression || "", { ...context, @@ -167,6 +169,8 @@ noThrow: false, } ) + expressionResult = output.result + expressionLogs = output.logs } catch (err: any) { expressionResult = undefined expressionError = err @@ -421,6 +425,7 @@ diff --git a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte index 54f7dc7d01..41245af4f9 100644 --- a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte +++ b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte @@ -4,20 +4,21 @@ import { Helpers } from "@budibase/bbui" import { fade } from "svelte/transition" import { UserScriptError } from "@budibase/string-templates" + import type { Log } from "@budibase/string-templates" import type { JSONValue } from "@budibase/types" // this can be essentially any primitive response from the JS function export let expressionResult: JSONValue | undefined = undefined export let expressionError: string | undefined = undefined + export let expressionLogs: Log[] = [] export let evaluating = false export let expression: string | null = null - export let logging: { log: string; line?: number }[] = [] $: error = expressionError != null $: empty = expression == null || expression?.trim() === "" $: success = !error && !empty $: highlightedResult = highlight(expressionResult) - $: highlightedLogs = logging.map(l => ({ + $: highlightedLogs = expressionLogs.map(l => ({ log: highlight(l.log), line: l.line, })) diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index fafff5d9fb..553c0e8861 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -18,6 +18,7 @@ import manifest from "./manifest.json" import { Log, ProcessOptions } from "./types" import { UserScriptError } from "./errors" +export type { Log } from "./types" export { helpersToRemoveForJs, getJsHelperList } from "./helpers/list" export { FIND_ANY_HBS_REGEX } from "./utilities" export { setJSRunner, setOnErrorLog } from "./helpers/javascript" From 325fea0c10644732b70a39c9885955aac4187838 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 16 Jan 2025 16:55:24 +0100 Subject: [PATCH 018/112] ViewV2, return 404 if not found --- packages/server/src/api/controllers/view/viewsV2.ts | 6 ++++-- packages/server/src/sdk/app/views/index.ts | 4 +++- packages/server/src/sdk/app/views/internal.ts | 6 ++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 5376ce49c3..ff8daf3bc6 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -123,9 +123,11 @@ async function parseSchema(view: CreateViewRequest) { } export async function get(ctx: Ctx) { - ctx.body = { - data: await sdk.views.getEnriched(ctx.params.viewId), + const view = await sdk.views.getEnriched(ctx.params.viewId) + if (!view) { + ctx.throw(404) } + ctx.body = { data: view } } export async function fetch(ctx: Ctx) { diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index 4f978253d6..157e741acd 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -40,7 +40,9 @@ export async function get(viewId: string): Promise { return pickApi(tableId).get(viewId) } -export async function getEnriched(viewId: string): Promise { +export async function getEnriched( + viewId: string +): Promise { const { tableId } = utils.extractViewInfoFromID(viewId) return pickApi(tableId).getEnriched(viewId) } diff --git a/packages/server/src/sdk/app/views/internal.ts b/packages/server/src/sdk/app/views/internal.ts index 4f7abad357..9eeebc4cc4 100644 --- a/packages/server/src/sdk/app/views/internal.ts +++ b/packages/server/src/sdk/app/views/internal.ts @@ -17,13 +17,15 @@ export async function get(viewId: string): Promise { return ensureQueryUISet(found) } -export async function getEnriched(viewId: string): Promise { +export async function getEnriched( + viewId: string +): Promise { const { tableId } = utils.extractViewInfoFromID(viewId) const table = await sdk.tables.getTable(tableId) const views = Object.values(table.views!).filter(isV2) const found = views.find(v => v.id === viewId) if (!found) { - throw new Error("No view found") + return } return await enrichSchema(ensureQueryUISet(found), table.schema) } From 566da4b2a3ac279407a43c9018448bd44555ef5a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 16 Jan 2025 16:58:57 +0100 Subject: [PATCH 019/112] ViewV2, return 404 if not found --- packages/server/src/sdk/app/views/external.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/views/external.ts b/packages/server/src/sdk/app/views/external.ts index 65e0ff410d..fabee903fe 100644 --- a/packages/server/src/sdk/app/views/external.ts +++ b/packages/server/src/sdk/app/views/external.ts @@ -22,7 +22,9 @@ export async function get(viewId: string): Promise { return ensureQueryUISet(found) } -export async function getEnriched(viewId: string): Promise { +export async function getEnriched( + viewId: string +): Promise { const { tableId } = utils.extractViewInfoFromID(viewId) const { datasourceId, tableName } = breakExternalTableId(tableId) @@ -32,7 +34,7 @@ export async function getEnriched(viewId: string): Promise { const views = Object.values(table.views!).filter(isV2) const found = views.find(v => v.id === viewId) if (!found) { - throw new Error("No view found") + return } return await enrichSchema(ensureQueryUISet(found), table.schema) } From da1f1177db3771ee410b3b0cb142cd033b7287c1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 16 Jan 2025 17:08:44 +0100 Subject: [PATCH 020/112] Type ComponentErrorState --- .../error-states/ComponentErrorState.svelte | 13 ++++++++----- packages/client/src/index.ts | 8 ++++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/packages/client/src/components/error-states/ComponentErrorState.svelte b/packages/client/src/components/error-states/ComponentErrorState.svelte index ec7c5169b0..b846eaa230 100644 --- a/packages/client/src/components/error-states/ComponentErrorState.svelte +++ b/packages/client/src/components/error-states/ComponentErrorState.svelte @@ -1,14 +1,17 @@ - From cd6c3e39459c6b1261c65ac5456a3a440db079e6 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 16 Jan 2025 17:25:13 +0000 Subject: [PATCH 022/112] Extract automation step definitions out into shared-core. --- .../src/api/routes/tests/automation.spec.ts | 4 +- packages/server/src/automations/actions.ts | 47 +++++---- packages/server/src/automations/steps/bash.ts | 50 +--------- .../server/src/automations/steps/collect.ts | 46 +-------- .../server/src/automations/steps/createRow.ts | 70 +------------ .../server/src/automations/steps/delay.ts | 41 +------- .../server/src/automations/steps/deleteRow.ts | 59 +---------- .../server/src/automations/steps/discord.ts | 63 +----------- .../src/automations/steps/executeQuery.ts | 57 ----------- .../src/automations/steps/executeScript.ts | 45 --------- .../server/src/automations/steps/filter.ts | 73 +------------- packages/server/src/automations/steps/make.ts | 58 +---------- packages/server/src/automations/steps/n8n.ts | 62 ------------ .../server/src/automations/steps/openai.ts | 59 +---------- .../src/automations/steps/outgoingWebhook.ts | 69 ------------- .../server/src/automations/steps/queryRows.ts | 72 -------------- .../src/automations/steps/sendSmtpEmail.ts | 98 +------------------ .../server/src/automations/steps/serverLog.ts | 56 +---------- .../server/src/automations/steps/slack.ts | 55 +---------- .../automations/steps/triggerAutomationRun.ts | 53 ---------- .../server/src/automations/steps/updateRow.ts | 70 +------------ .../server/src/automations/steps/zapier.ts | 51 +--------- .../src/automations/tests/filter.spec.ts | 4 +- .../tests/scenarios/scenarios.spec.ts | 4 +- .../tests/utilities/AutomationTestBuilder.ts | 4 +- packages/shared-core/src/automations/index.ts | 1 + .../shared-core/src/automations/steps/bash.ts | 47 +++++++++ .../src/automations/steps/branch.ts | 0 .../src/automations/steps/collect.ts | 43 ++++++++ .../src/automations/steps/createRow.ts | 67 +++++++++++++ .../src/automations/steps/delay.ts | 38 +++++++ .../src/automations/steps/deleteRow.ts | 56 +++++++++++ .../src/automations/steps/discord.ts | 60 ++++++++++++ .../src/automations/steps/executeQuery.ts | 59 +++++++++++ .../src/automations/steps/executeScript.ts | 47 +++++++++ .../src/automations/steps/filter.ts | 69 +++++++++++++ .../src/automations/steps/index.ts | 22 +++++ .../src/automations/steps/loop.ts | 0 .../shared-core/src/automations/steps/make.ts | 55 +++++++++++ .../shared-core/src/automations/steps/n8n.ts | 65 ++++++++++++ .../src/automations/steps/openai.ts | 56 +++++++++++ .../src/automations/steps/outgoingWebhook.ts | 79 +++++++++++++++ .../src/automations/steps/queryRows.ts | 75 ++++++++++++++ .../src/automations/steps/sendSmtpEmail.ts | 95 ++++++++++++++++++ .../src/automations/steps/serverLog.ts | 47 +++++++++ .../src/automations/steps/slack.ts | 52 ++++++++++ .../automations/steps/triggerAutomationRun.ts | 55 +++++++++++ .../src/automations/steps/updateRow.ts | 68 +++++++++++++ .../src/automations/steps/zapier.ts | 48 +++++++++ packages/shared-core/src/index.ts | 1 + 50 files changed, 1255 insertions(+), 1220 deletions(-) create mode 100644 packages/shared-core/src/automations/index.ts create mode 100644 packages/shared-core/src/automations/steps/bash.ts rename packages/{server => shared-core}/src/automations/steps/branch.ts (100%) create mode 100644 packages/shared-core/src/automations/steps/collect.ts create mode 100644 packages/shared-core/src/automations/steps/createRow.ts create mode 100644 packages/shared-core/src/automations/steps/delay.ts create mode 100644 packages/shared-core/src/automations/steps/deleteRow.ts create mode 100644 packages/shared-core/src/automations/steps/discord.ts create mode 100644 packages/shared-core/src/automations/steps/executeQuery.ts create mode 100644 packages/shared-core/src/automations/steps/executeScript.ts create mode 100644 packages/shared-core/src/automations/steps/filter.ts create mode 100644 packages/shared-core/src/automations/steps/index.ts rename packages/{server => shared-core}/src/automations/steps/loop.ts (100%) create mode 100644 packages/shared-core/src/automations/steps/make.ts create mode 100644 packages/shared-core/src/automations/steps/n8n.ts create mode 100644 packages/shared-core/src/automations/steps/openai.ts create mode 100644 packages/shared-core/src/automations/steps/outgoingWebhook.ts create mode 100644 packages/shared-core/src/automations/steps/queryRows.ts create mode 100644 packages/shared-core/src/automations/steps/sendSmtpEmail.ts create mode 100644 packages/shared-core/src/automations/steps/serverLog.ts create mode 100644 packages/shared-core/src/automations/steps/slack.ts create mode 100644 packages/shared-core/src/automations/steps/triggerAutomationRun.ts create mode 100644 packages/shared-core/src/automations/steps/updateRow.ts create mode 100644 packages/shared-core/src/automations/steps/zapier.ts diff --git a/packages/server/src/api/routes/tests/automation.spec.ts b/packages/server/src/api/routes/tests/automation.spec.ts index a517346c82..5c0b86d9a0 100644 --- a/packages/server/src/api/routes/tests/automation.spec.ts +++ b/packages/server/src/api/routes/tests/automation.spec.ts @@ -19,9 +19,11 @@ import { Table, } from "@budibase/types" import { mocks } from "@budibase/backend-core/tests" -import { FilterConditions } from "../../../automations/steps/filter" import { removeDeprecated } from "../../../automations/utils" import { createAutomationBuilder } from "../../../automations/tests/utilities/AutomationTestBuilder" +import { automations } from "@budibase/shared-core" + +const FilterConditions = automations.steps.filter.FilterConditions const MAX_RETRIES = 4 let { diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index 537b6befc3..f35f96babe 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -1,3 +1,4 @@ +import { automations } from "@budibase/shared-core" import * as sendSmtpEmail from "./steps/sendSmtpEmail" import * as createRow from "./steps/createRow" import * as updateRow from "./steps/updateRow" @@ -14,9 +15,7 @@ import * as make from "./steps/make" import * as filter from "./steps/filter" import * as delay from "./steps/delay" import * as queryRow from "./steps/queryRows" -import * as loop from "./steps/loop" import * as collect from "./steps/collect" -import * as branch from "./steps/branch" import * as triggerAutomationRun from "./steps/triggerAutomationRun" import * as openai from "./steps/openai" import env from "../environment" @@ -62,27 +61,27 @@ export const BUILTIN_ACTION_DEFINITIONS: Record< string, AutomationStepDefinition > = { - SEND_EMAIL_SMTP: sendSmtpEmail.definition, - CREATE_ROW: createRow.definition, - UPDATE_ROW: updateRow.definition, - DELETE_ROW: deleteRow.definition, - OUTGOING_WEBHOOK: outgoingWebhook.definition, - EXECUTE_SCRIPT: executeScript.definition, - EXECUTE_QUERY: executeQuery.definition, - SERVER_LOG: serverLog.definition, - DELAY: delay.definition, - FILTER: filter.definition, - QUERY_ROWS: queryRow.definition, - LOOP: loop.definition, - COLLECT: collect.definition, - TRIGGER_AUTOMATION_RUN: triggerAutomationRun.definition, - BRANCH: branch.definition, + SEND_EMAIL_SMTP: automations.steps.sendSmtpEmail.definition, + CREATE_ROW: automations.steps.createRow.definition, + UPDATE_ROW: automations.steps.updateRow.definition, + DELETE_ROW: automations.steps.deleteRow.definition, + OUTGOING_WEBHOOK: automations.steps.outgoingWebhook.definition, + EXECUTE_SCRIPT: automations.steps.executeScript.definition, + EXECUTE_QUERY: automations.steps.executeQuery.definition, + SERVER_LOG: automations.steps.serverLog.definition, + DELAY: automations.steps.delay.definition, + FILTER: automations.steps.filter.definition, + QUERY_ROWS: automations.steps.queryRows.definition, + LOOP: automations.steps.loop.definition, + COLLECT: automations.steps.collect.definition, + TRIGGER_AUTOMATION_RUN: automations.steps.triggerAutomationRun.definition, + BRANCH: automations.steps.branch.definition, // these used to be lowercase step IDs, maintain for backwards compat - discord: discord.definition, - slack: slack.definition, - zapier: zapier.definition, - integromat: make.definition, - n8n: n8n.definition, + discord: automations.steps.discord.definition, + slack: automations.steps.slack.definition, + zapier: automations.steps.zapier.definition, + integromat: automations.steps.make.definition, + n8n: automations.steps.n8n.definition, } // don't add the bash script/definitions unless in self host @@ -97,7 +96,7 @@ if (env.SELF_HOSTED) { BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition if (env.isTest()) { - BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition + BUILTIN_ACTION_DEFINITIONS["OPENAI"] = automations.steps.openai.definition } } @@ -105,7 +104,7 @@ export async function getActionDefinitions(): Promise< Record > { if (env.SELF_HOSTED) { - BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition + BUILTIN_ACTION_DEFINITIONS["OPENAI"] = automations.steps.openai.definition } const actionDefinitions = BUILTIN_ACTION_DEFINITIONS diff --git a/packages/server/src/automations/steps/bash.ts b/packages/server/src/automations/steps/bash.ts index 636e659ffe..f6d2feecf6 100644 --- a/packages/server/src/automations/steps/bash.ts +++ b/packages/server/src/automations/steps/bash.ts @@ -2,55 +2,7 @@ import { execSync } from "child_process" import { processStringSync } from "@budibase/string-templates" import * as automationUtils from "../automationUtils" import environment from "../../environment" -import { - AutomationActionStepId, - AutomationCustomIOType, - AutomationFeature, - AutomationIOType, - AutomationStepDefinition, - AutomationStepType, - BashStepInputs, - BashStepOutputs, -} from "@budibase/types" - -export const definition: AutomationStepDefinition = { - name: "Bash Scripting", - tagline: "Execute a bash command", - icon: "JourneyEvent", - description: "Run a bash script", - type: AutomationStepType.ACTION, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - stepId: AutomationActionStepId.EXECUTE_BASH, - inputs: {}, - schema: { - inputs: { - properties: { - code: { - type: AutomationIOType.STRING, - customType: AutomationCustomIOType.CODE, - title: "Code", - }, - }, - required: ["code"], - }, - outputs: { - properties: { - stdout: { - type: AutomationIOType.STRING, - description: "Standard output of your bash command or script", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the command was successful", - }, - }, - required: ["stdout"], - }, - }, -} +import { BashStepInputs, BashStepOutputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/steps/collect.ts b/packages/server/src/automations/steps/collect.ts index 3b2ca7d21f..689c3d347f 100644 --- a/packages/server/src/automations/steps/collect.ts +++ b/packages/server/src/automations/steps/collect.ts @@ -1,48 +1,4 @@ -import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - CollectStepInputs, - CollectStepOutputs, -} from "@budibase/types" - -export const definition: AutomationStepDefinition = { - name: "Collect Data", - tagline: "Collect data to be sent to design", - icon: "Collection", - description: - "Collects specified data so it can be provided to the design section", - type: AutomationStepType.ACTION, - internal: true, - features: {}, - stepId: AutomationActionStepId.COLLECT, - inputs: {}, - schema: { - inputs: { - properties: { - collection: { - type: AutomationIOType.STRING, - title: "What to Collect", - }, - }, - required: ["collection"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the action was successful", - }, - value: { - type: AutomationIOType.STRING, - description: "Collected data", - }, - }, - required: ["success", "value"], - }, - }, -} +import { CollectStepInputs, CollectStepOutputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/steps/createRow.ts b/packages/server/src/automations/steps/createRow.ts index cd16be2cf3..24dada422d 100644 --- a/packages/server/src/automations/steps/createRow.ts +++ b/packages/server/src/automations/steps/createRow.ts @@ -5,77 +5,9 @@ import { sendAutomationAttachmentsToStorage, } from "../automationUtils" import { buildCtx } from "./utils" -import { - AutomationActionStepId, - AutomationCustomIOType, - AutomationFeature, - AutomationIOType, - AutomationStepDefinition, - AutomationStepType, - CreateRowStepInputs, - CreateRowStepOutputs, -} from "@budibase/types" +import { CreateRowStepInputs, CreateRowStepOutputs } from "@budibase/types" import { EventEmitter } from "events" -export const definition: AutomationStepDefinition = { - name: "Create Row", - tagline: "Create a {{inputs.enriched.table.name}} row", - icon: "TableRowAddBottom", - description: "Add a row to your database", - type: AutomationStepType.ACTION, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - stepId: AutomationActionStepId.CREATE_ROW, - inputs: {}, - schema: { - inputs: { - properties: { - row: { - type: AutomationIOType.OBJECT, - properties: { - tableId: { - type: AutomationIOType.STRING, - customType: AutomationCustomIOType.TABLE, - }, - }, - customType: AutomationCustomIOType.ROW, - title: "Table", - required: ["tableId"], - }, - }, - required: ["row"], - }, - outputs: { - properties: { - row: { - type: AutomationIOType.OBJECT, - customType: AutomationCustomIOType.ROW, - description: "The new row", - }, - response: { - type: AutomationIOType.OBJECT, - description: "The response from the table", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the row creation was successful", - }, - id: { - type: AutomationIOType.STRING, - description: "The identifier of the new row", - }, - revision: { - type: AutomationIOType.STRING, - description: "The revision of the new row", - }, - }, - required: ["success", "id", "revision"], - }, - }, -} - export async function run({ inputs, appId, diff --git a/packages/server/src/automations/steps/delay.ts b/packages/server/src/automations/steps/delay.ts index c623693488..4ca5a16942 100644 --- a/packages/server/src/automations/steps/delay.ts +++ b/packages/server/src/automations/steps/delay.ts @@ -1,44 +1,5 @@ import { wait } from "../../utilities" -import { - AutomationActionStepId, - AutomationIOType, - AutomationStepDefinition, - AutomationStepType, - DelayStepInputs, - DelayStepOutputs, -} from "@budibase/types" - -export const definition: AutomationStepDefinition = { - name: "Delay", - icon: "Clock", - tagline: "Delay for {{inputs.time}} milliseconds", - description: "Delay the automation until an amount of time has passed", - stepId: AutomationActionStepId.DELAY, - internal: true, - features: {}, - inputs: {}, - schema: { - inputs: { - properties: { - time: { - type: AutomationIOType.NUMBER, - title: "Delay in milliseconds", - }, - }, - required: ["time"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the delay was successful", - }, - }, - required: ["success"], - }, - }, - type: AutomationStepType.LOGIC, -} +import { DelayStepInputs, DelayStepOutputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/steps/deleteRow.ts b/packages/server/src/automations/steps/deleteRow.ts index 6b8a68eeb8..7c50fe4dcb 100644 --- a/packages/server/src/automations/steps/deleteRow.ts +++ b/packages/server/src/automations/steps/deleteRow.ts @@ -2,64 +2,7 @@ import { EventEmitter } from "events" import { destroy } from "../../api/controllers/row" import { buildCtx } from "./utils" import { getError } from "../automationUtils" -import { - AutomationActionStepId, - AutomationStepType, - AutomationIOType, - AutomationCustomIOType, - AutomationFeature, - DeleteRowStepInputs, - DeleteRowStepOutputs, - AutomationStepDefinition, -} from "@budibase/types" - -export const definition: AutomationStepDefinition = { - description: "Delete a row from your database", - icon: "TableRowRemoveCenter", - name: "Delete Row", - tagline: "Delete a {{inputs.enriched.table.name}} row", - type: AutomationStepType.ACTION, - stepId: AutomationActionStepId.DELETE_ROW, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - inputs: {}, - schema: { - inputs: { - properties: { - tableId: { - type: AutomationIOType.STRING, - customType: AutomationCustomIOType.TABLE, - title: "Table", - }, - id: { - type: AutomationIOType.STRING, - title: "Row ID", - }, - }, - required: ["tableId", "id"], - }, - outputs: { - properties: { - row: { - type: AutomationIOType.OBJECT, - customType: AutomationCustomIOType.ROW, - description: "The deleted row", - }, - response: { - type: AutomationIOType.OBJECT, - description: "The response from the table", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the deletion was successful", - }, - }, - required: ["row", "success"], - }, - }, -} +import { DeleteRowStepInputs, DeleteRowStepOutputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/steps/discord.ts b/packages/server/src/automations/steps/discord.ts index 1ce21fc89e..30d3e8137b 100644 --- a/packages/server/src/automations/steps/discord.ts +++ b/packages/server/src/automations/steps/discord.ts @@ -1,71 +1,10 @@ import fetch from "node-fetch" import { getFetchResponse } from "./utils" -import { - AutomationActionStepId, - AutomationStepType, - AutomationIOType, - AutomationFeature, - ExternalAppStepOutputs, - DiscordStepInputs, - AutomationStepDefinition, -} from "@budibase/types" +import { ExternalAppStepOutputs, DiscordStepInputs } from "@budibase/types" const DEFAULT_USERNAME = "Budibase Automate" const DEFAULT_AVATAR_URL = "https://i.imgur.com/a1cmTKM.png" -export const definition: AutomationStepDefinition = { - name: "Discord Message", - tagline: "Send a message to a Discord server", - description: "Send a message to a Discord server", - icon: "ri-discord-line", - stepId: AutomationActionStepId.discord, - type: AutomationStepType.ACTION, - internal: false, - features: { - [AutomationFeature.LOOPING]: true, - }, - inputs: {}, - schema: { - inputs: { - properties: { - url: { - type: AutomationIOType.STRING, - title: "Discord Webhook URL", - }, - username: { - type: AutomationIOType.STRING, - title: "Bot Name", - }, - avatar_url: { - type: AutomationIOType.STRING, - title: "Bot Avatar URL", - }, - content: { - type: AutomationIOType.STRING, - title: "Message", - }, - }, - required: ["url", "content"], - }, - outputs: { - properties: { - httpStatus: { - type: AutomationIOType.NUMBER, - description: "The HTTP status code of the request", - }, - response: { - type: AutomationIOType.STRING, - description: "The response from the Discord Webhook", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the message sent successfully", - }, - }, - }, - }, -} - export async function run({ inputs, }: { diff --git a/packages/server/src/automations/steps/executeQuery.ts b/packages/server/src/automations/steps/executeQuery.ts index 7af540093f..9816e31b1e 100644 --- a/packages/server/src/automations/steps/executeQuery.ts +++ b/packages/server/src/automations/steps/executeQuery.ts @@ -3,67 +3,10 @@ import * as queryController from "../../api/controllers/query" import { buildCtx } from "./utils" import * as automationUtils from "../automationUtils" import { - AutomationActionStepId, - AutomationCustomIOType, - AutomationFeature, - AutomationIOType, - AutomationStepDefinition, - AutomationStepType, ExecuteQueryStepInputs, ExecuteQueryStepOutputs, } from "@budibase/types" -export const definition: AutomationStepDefinition = { - name: "External Data Connector", - tagline: "Execute Data Connector", - icon: "Data", - description: "Execute a query in an external data connector", - type: AutomationStepType.ACTION, - stepId: AutomationActionStepId.EXECUTE_QUERY, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - inputs: {}, - schema: { - inputs: { - properties: { - query: { - type: AutomationIOType.OBJECT, - properties: { - queryId: { - type: AutomationIOType.STRING, - customType: AutomationCustomIOType.QUERY, - }, - }, - customType: AutomationCustomIOType.QUERY_PARAMS, - title: "Parameters", - required: ["queryId"], - }, - }, - required: ["query"], - }, - outputs: { - properties: { - response: { - type: AutomationIOType.OBJECT, - description: "The response from the datasource execution", - }, - info: { - type: AutomationIOType.OBJECT, - description: - "Some query types may return extra data, like headers from a REST query", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the action was successful", - }, - }, - required: ["response", "success"], - }, - }, -} - export async function run({ inputs, appId, diff --git a/packages/server/src/automations/steps/executeScript.ts b/packages/server/src/automations/steps/executeScript.ts index 97145f3e14..105543d34c 100644 --- a/packages/server/src/automations/steps/executeScript.ts +++ b/packages/server/src/automations/steps/executeScript.ts @@ -2,56 +2,11 @@ import * as scriptController from "../../api/controllers/script" import { buildCtx } from "./utils" import * as automationUtils from "../automationUtils" import { - AutomationActionStepId, - AutomationCustomIOType, - AutomationFeature, - AutomationIOType, - AutomationStepDefinition, - AutomationStepType, ExecuteScriptStepInputs, ExecuteScriptStepOutputs, } from "@budibase/types" import { EventEmitter } from "events" -export const definition: AutomationStepDefinition = { - name: "JS Scripting", - tagline: "Execute JavaScript Code", - icon: "Code", - description: "Run a piece of JavaScript code in your automation", - type: AutomationStepType.ACTION, - internal: true, - stepId: AutomationActionStepId.EXECUTE_SCRIPT, - inputs: {}, - features: { - [AutomationFeature.LOOPING]: true, - }, - schema: { - inputs: { - properties: { - code: { - type: AutomationIOType.STRING, - customType: AutomationCustomIOType.CODE, - title: "Code", - }, - }, - required: ["code"], - }, - outputs: { - properties: { - value: { - type: AutomationIOType.STRING, - description: "The result of the return statement", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the action was successful", - }, - }, - required: ["success"], - }, - }, -} - export async function run({ inputs, appId, diff --git a/packages/server/src/automations/steps/filter.ts b/packages/server/src/automations/steps/filter.ts index 84cea00b93..9b7e347034 100644 --- a/packages/server/src/automations/steps/filter.ts +++ b/packages/server/src/automations/steps/filter.ts @@ -1,74 +1,7 @@ -import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - FilterStepInputs, - FilterStepOutputs, -} from "@budibase/types" +import { FilterStepInputs, FilterStepOutputs } from "@budibase/types" +import { automations } from "@budibase/shared-core" -export const FilterConditions = { - EQUAL: "EQUAL", - NOT_EQUAL: "NOT_EQUAL", - GREATER_THAN: "GREATER_THAN", - LESS_THAN: "LESS_THAN", -} - -export const PrettyFilterConditions = { - [FilterConditions.EQUAL]: "Equals", - [FilterConditions.NOT_EQUAL]: "Not equals", - [FilterConditions.GREATER_THAN]: "Greater than", - [FilterConditions.LESS_THAN]: "Less than", -} - -export const definition: AutomationStepDefinition = { - name: "Condition", - tagline: "{{inputs.field}} {{inputs.condition}} {{inputs.value}}", - icon: "Branch2", - description: - "Conditionally halt automations which do not meet certain conditions", - type: AutomationStepType.LOGIC, - internal: true, - features: {}, - stepId: AutomationActionStepId.FILTER, - inputs: { - condition: FilterConditions.EQUAL, - }, - schema: { - inputs: { - properties: { - field: { - type: AutomationIOType.STRING, - title: "Reference Value", - }, - condition: { - type: AutomationIOType.STRING, - title: "Condition", - enum: Object.values(FilterConditions), - pretty: Object.values(PrettyFilterConditions), - }, - value: { - type: AutomationIOType.STRING, - title: "Comparison Value", - }, - }, - required: ["field", "condition", "value"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the action was successful", - }, - result: { - type: AutomationIOType.BOOLEAN, - description: "Whether the logic block passed", - }, - }, - required: ["success", "result"], - }, - }, -} +const FilterConditions = automations.steps.filter.FilterConditions export async function run({ inputs, diff --git a/packages/server/src/automations/steps/make.ts b/packages/server/src/automations/steps/make.ts index 62cfd66302..241cccce9e 100644 --- a/packages/server/src/automations/steps/make.ts +++ b/packages/server/src/automations/steps/make.ts @@ -1,62 +1,6 @@ import fetch from "node-fetch" import { getFetchResponse } from "./utils" -import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - AutomationFeature, - ExternalAppStepOutputs, - MakeIntegrationInputs, -} from "@budibase/types" - -export const definition: AutomationStepDefinition = { - name: "Make Integration", - stepTitle: "Make", - tagline: "Trigger a Make scenario", - description: - "Performs a webhook call to Make and gets the response (if configured)", - icon: "ri-shut-down-line", - stepId: AutomationActionStepId.integromat, - type: AutomationStepType.ACTION, - internal: false, - features: { - [AutomationFeature.LOOPING]: true, - }, - inputs: {}, - schema: { - inputs: { - properties: { - url: { - type: AutomationIOType.STRING, - title: "Webhook URL", - }, - body: { - type: AutomationIOType.JSON, - title: "Payload", - }, - }, - required: ["url", "body"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether call was successful", - }, - httpStatus: { - type: AutomationIOType.NUMBER, - description: "The HTTP status code returned", - }, - response: { - type: AutomationIOType.OBJECT, - description: "The webhook response - this can have properties", - }, - }, - required: ["success", "response"], - }, - }, -} +import { ExternalAppStepOutputs, MakeIntegrationInputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/steps/n8n.ts b/packages/server/src/automations/steps/n8n.ts index 8e55d2535b..29883dd327 100644 --- a/packages/server/src/automations/steps/n8n.ts +++ b/packages/server/src/automations/steps/n8n.ts @@ -1,73 +1,11 @@ import fetch, { HeadersInit } from "node-fetch" import { getFetchResponse } from "./utils" import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - AutomationFeature, HttpMethod, ExternalAppStepOutputs, n8nStepInputs, } from "@budibase/types" -export const definition: AutomationStepDefinition = { - name: "n8n Integration", - stepTitle: "n8n", - tagline: "Trigger an n8n workflow", - description: - "Performs a webhook call to n8n and gets the response (if configured)", - icon: "ri-shut-down-line", - stepId: AutomationActionStepId.n8n, - type: AutomationStepType.ACTION, - internal: false, - features: { - [AutomationFeature.LOOPING]: true, - }, - inputs: {}, - schema: { - inputs: { - properties: { - url: { - type: AutomationIOType.STRING, - title: "Webhook URL", - }, - method: { - type: AutomationIOType.STRING, - title: "Method", - enum: Object.values(HttpMethod), - }, - authorization: { - type: AutomationIOType.STRING, - title: "Authorization", - }, - body: { - type: AutomationIOType.JSON, - title: "Payload", - }, - }, - required: ["url", "method"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether call was successful", - }, - httpStatus: { - type: AutomationIOType.NUMBER, - description: "The HTTP status code returned", - }, - response: { - type: AutomationIOType.OBJECT, - description: "The webhook response - this can have properties", - }, - }, - required: ["success", "response"], - }, - }, -} - export async function run({ inputs, }: { diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 53e41ceb09..55f57aff7c 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -1,67 +1,10 @@ import { OpenAI } from "openai" -import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - OpenAIStepInputs, - OpenAIStepOutputs, -} from "@budibase/types" +import { OpenAIStepInputs, OpenAIStepOutputs } from "@budibase/types" import { env } from "@budibase/backend-core" import * as automationUtils from "../automationUtils" import * as pro from "@budibase/pro" -enum Model { - GPT_4O_MINI = "gpt-4o-mini", - GPT_4O = "gpt-4o", - GPT_4 = "gpt-4", - GPT_35_TURBO = "gpt-3.5-turbo", -} - -export const definition: AutomationStepDefinition = { - name: "OpenAI", - tagline: "Send prompts to ChatGPT", - icon: "Algorithm", - description: "Interact with the OpenAI ChatGPT API.", - type: AutomationStepType.ACTION, - internal: true, - features: {}, - stepId: AutomationActionStepId.OPENAI, - inputs: { - prompt: "", - }, - schema: { - inputs: { - properties: { - prompt: { - type: AutomationIOType.STRING, - title: "Prompt", - }, - model: { - type: AutomationIOType.STRING, - title: "Model", - enum: Object.values(Model), - }, - }, - required: ["prompt", "model"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the action was successful", - }, - response: { - type: AutomationIOType.STRING, - description: "What was output", - }, - }, - required: ["success", "response"], - }, - }, -} - /** * Maintains backward compatibility with automation steps created before the introduction * of custom configurations and Budibase AI diff --git a/packages/server/src/automations/steps/outgoingWebhook.ts b/packages/server/src/automations/steps/outgoingWebhook.ts index 70b8b1f476..a1211ea281 100644 --- a/packages/server/src/automations/steps/outgoingWebhook.ts +++ b/packages/server/src/automations/steps/outgoingWebhook.ts @@ -2,12 +2,6 @@ import fetch from "node-fetch" import { getFetchResponse } from "./utils" import * as automationUtils from "../automationUtils" import { - AutomationActionStepId, - AutomationCustomIOType, - AutomationFeature, - AutomationIOType, - AutomationStepDefinition, - AutomationStepType, ExternalAppStepOutputs, OutgoingWebhookStepInputs, } from "@budibase/types" @@ -26,69 +20,6 @@ const BODY_REQUESTS = [RequestType.POST, RequestType.PUT, RequestType.PATCH] * NOTE: this functionality is deprecated - it no longer should be used. */ -export const definition: AutomationStepDefinition = { - deprecated: true, - name: "Outgoing webhook", - tagline: "Send a {{inputs.requestMethod}} request", - icon: "Send", - description: "Send a request of specified method to a URL", - type: AutomationStepType.ACTION, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - stepId: AutomationActionStepId.OUTGOING_WEBHOOK, - inputs: { - requestMethod: "POST", - url: "http://", - requestBody: "{}", - headers: "{}", - }, - schema: { - inputs: { - properties: { - requestMethod: { - type: AutomationIOType.STRING, - enum: Object.values(RequestType), - title: "Request method", - }, - url: { - type: AutomationIOType.STRING, - title: "URL", - }, - requestBody: { - type: AutomationIOType.STRING, - title: "JSON Body", - customType: AutomationCustomIOType.WIDE, - }, - headers: { - type: AutomationIOType.STRING, - title: "Headers", - customType: AutomationCustomIOType.WIDE, - }, - }, - required: ["requestMethod", "url"], - }, - outputs: { - properties: { - response: { - type: AutomationIOType.OBJECT, - description: "The response from the webhook", - }, - httpStatus: { - type: AutomationIOType.NUMBER, - description: "The HTTP status code returned", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the action was successful", - }, - }, - required: ["response", "success"], - }, - }, -} - export async function run({ inputs, }: { diff --git a/packages/server/src/automations/steps/queryRows.ts b/packages/server/src/automations/steps/queryRows.ts index 4a02e80686..9c60a63dac 100644 --- a/packages/server/src/automations/steps/queryRows.ts +++ b/packages/server/src/automations/steps/queryRows.ts @@ -4,84 +4,12 @@ import { buildCtx } from "./utils" import * as automationUtils from "../automationUtils" import { FieldType, - AutomationActionStepId, - AutomationCustomIOType, - AutomationFeature, - AutomationIOType, - AutomationStepDefinition, - AutomationStepType, EmptyFilterOption, SortOrder, QueryRowsStepInputs, QueryRowsStepOutputs, } from "@budibase/types" -const SortOrderPretty = { - [SortOrder.ASCENDING]: "Ascending", - [SortOrder.DESCENDING]: "Descending", -} - -export const definition: AutomationStepDefinition = { - description: "Query rows from the database", - icon: "Search", - name: "Query rows", - tagline: "Query rows from {{inputs.enriched.table.name}} table", - type: AutomationStepType.ACTION, - stepId: AutomationActionStepId.QUERY_ROWS, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - inputs: {}, - schema: { - inputs: { - properties: { - tableId: { - type: AutomationIOType.STRING, - customType: AutomationCustomIOType.TABLE, - title: "Table", - }, - filters: { - type: AutomationIOType.OBJECT, - customType: AutomationCustomIOType.FILTERS, - title: "Filtering", - }, - sortColumn: { - type: AutomationIOType.STRING, - title: "Sort Column", - customType: AutomationCustomIOType.COLUMN, - }, - sortOrder: { - type: AutomationIOType.STRING, - title: "Sort Order", - enum: Object.values(SortOrder), - pretty: Object.values(SortOrderPretty), - }, - limit: { - type: AutomationIOType.NUMBER, - title: "Limit", - customType: AutomationCustomIOType.QUERY_LIMIT, - }, - }, - required: ["tableId"], - }, - outputs: { - properties: { - rows: { - type: AutomationIOType.ARRAY, - customType: AutomationCustomIOType.ROWS, - description: "The rows that were found", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the query was successful", - }, - }, - required: ["rows", "success"], - }, - }, -} - async function getTable(appId: string, tableId: string) { const ctx: any = buildCtx(appId, null, { params: { diff --git a/packages/server/src/automations/steps/sendSmtpEmail.ts b/packages/server/src/automations/steps/sendSmtpEmail.ts index 72ca410528..bfa2fa31ad 100644 --- a/packages/server/src/automations/steps/sendSmtpEmail.ts +++ b/packages/server/src/automations/steps/sendSmtpEmail.ts @@ -1,102 +1,6 @@ import { sendSmtpEmail } from "../../utilities/workerRequests" import * as automationUtils from "../automationUtils" -import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - AutomationFeature, - AutomationCustomIOType, - SmtpEmailStepInputs, - BaseAutomationOutputs, -} from "@budibase/types" - -export const definition: AutomationStepDefinition = { - description: "Send an email using SMTP", - tagline: "Send SMTP email to {{inputs.to}}", - icon: "Email", - name: "Send Email (SMTP)", - type: AutomationStepType.ACTION, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - stepId: AutomationActionStepId.SEND_EMAIL_SMTP, - inputs: {}, - schema: { - inputs: { - properties: { - to: { - type: AutomationIOType.STRING, - title: "Send To", - }, - from: { - type: AutomationIOType.STRING, - title: "Send From", - }, - cc: { - type: AutomationIOType.STRING, - title: "CC", - }, - bcc: { - type: AutomationIOType.STRING, - title: "BCC", - }, - subject: { - type: AutomationIOType.STRING, - title: "Email Subject", - }, - contents: { - type: AutomationIOType.STRING, - title: "HTML Contents", - }, - addInvite: { - type: AutomationIOType.BOOLEAN, - title: "Add calendar invite", - }, - startTime: { - type: AutomationIOType.DATE, - title: "Start Time", - dependsOn: "addInvite", - }, - endTime: { - type: AutomationIOType.DATE, - title: "End Time", - dependsOn: "addInvite", - }, - summary: { - type: AutomationIOType.STRING, - title: "Meeting Summary", - dependsOn: "addInvite", - }, - location: { - type: AutomationIOType.STRING, - title: "Location", - dependsOn: "addInvite", - }, - attachments: { - type: AutomationIOType.ATTACHMENT, - customType: AutomationCustomIOType.MULTI_ATTACHMENTS, - title: "Attachments", - }, - }, - required: ["to", "from", "subject", "contents"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the email was sent", - }, - response: { - type: AutomationIOType.OBJECT, - description: "A response from the email client, this may be an error", - }, - }, - required: ["success"], - }, - }, -} +import { SmtpEmailStepInputs, BaseAutomationOutputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/steps/serverLog.ts b/packages/server/src/automations/steps/serverLog.ts index d0702ab3eb..b3588b7378 100644 --- a/packages/server/src/automations/steps/serverLog.ts +++ b/packages/server/src/automations/steps/serverLog.ts @@ -1,58 +1,4 @@ -import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - AutomationFeature, - ServerLogStepInputs, - ServerLogStepOutputs, -} from "@budibase/types" - -/** - * Note, there is some functionality in this that is not currently exposed as it - * is complex and maybe better to be opinionated here. - * GET/DELETE requests cannot handle body elements so they will not be sent if configured. - */ - -export const definition: AutomationStepDefinition = { - name: "Backend log", - tagline: "Console log a value in the backend", - icon: "Monitoring", - description: "Logs the given text to the server (using console.log)", - type: AutomationStepType.ACTION, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - stepId: AutomationActionStepId.SERVER_LOG, - inputs: { - text: "", - }, - schema: { - inputs: { - properties: { - text: { - type: AutomationIOType.STRING, - title: "Log", - }, - }, - required: ["text"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the action was successful", - }, - message: { - type: AutomationIOType.STRING, - description: "What was output", - }, - }, - required: ["success", "message"], - }, - }, -} +import { ServerLogStepInputs, ServerLogStepOutputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/steps/slack.ts b/packages/server/src/automations/steps/slack.ts index 353611260c..12dcea74c7 100644 --- a/packages/server/src/automations/steps/slack.ts +++ b/packages/server/src/automations/steps/slack.ts @@ -1,59 +1,6 @@ import fetch from "node-fetch" import { getFetchResponse } from "./utils" -import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - AutomationFeature, - ExternalAppStepOutputs, - SlackStepInputs, -} from "@budibase/types" - -export const definition: AutomationStepDefinition = { - name: "Slack Message", - tagline: "Send a message to Slack", - description: "Send a message to Slack", - icon: "ri-slack-line", - stepId: AutomationActionStepId.slack, - type: AutomationStepType.ACTION, - internal: false, - features: { - [AutomationFeature.LOOPING]: true, - }, - inputs: {}, - schema: { - inputs: { - properties: { - url: { - type: AutomationIOType.STRING, - title: "Incoming Webhook URL", - }, - text: { - type: AutomationIOType.STRING, - title: "Message", - }, - }, - required: ["url", "text"], - }, - outputs: { - properties: { - httpStatus: { - type: AutomationIOType.NUMBER, - description: "The HTTP status code of the request", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the message sent successfully", - }, - response: { - type: AutomationIOType.STRING, - description: "The response from the Slack Webhook", - }, - }, - }, - }, -} +import { ExternalAppStepOutputs, SlackStepInputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/steps/triggerAutomationRun.ts b/packages/server/src/automations/steps/triggerAutomationRun.ts index f1cf65b182..5c7959648d 100644 --- a/packages/server/src/automations/steps/triggerAutomationRun.ts +++ b/packages/server/src/automations/steps/triggerAutomationRun.ts @@ -1,10 +1,5 @@ import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, Automation, - AutomationCustomIOType, TriggerAutomationStepInputs, TriggerAutomationStepOutputs, } from "@budibase/types" @@ -13,54 +8,6 @@ import { context } from "@budibase/backend-core" import { features } from "@budibase/pro" import env from "../../environment" -export const definition: AutomationStepDefinition = { - name: "Trigger an automation", - tagline: "Triggers an automation synchronously", - icon: "Sync", - description: "Triggers an automation synchronously", - type: AutomationStepType.ACTION, - internal: true, - features: {}, - stepId: AutomationActionStepId.TRIGGER_AUTOMATION_RUN, - inputs: {}, - schema: { - inputs: { - properties: { - automation: { - type: AutomationIOType.OBJECT, - properties: { - automationId: { - type: AutomationIOType.STRING, - customType: AutomationCustomIOType.AUTOMATION, - }, - }, - customType: AutomationCustomIOType.AUTOMATION_FIELDS, - title: "automatioFields", - required: ["automationId"], - }, - timeout: { - type: AutomationIOType.NUMBER, - title: "Timeout (ms)", - }, - }, - required: ["automationId"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the automation was successful", - }, - value: { - type: AutomationIOType.OBJECT, - description: "Automation Result", - }, - }, - required: ["success", "value"], - }, - }, -} - export async function run({ inputs, }: { diff --git a/packages/server/src/automations/steps/updateRow.ts b/packages/server/src/automations/steps/updateRow.ts index 552d846f18..46ae2a5c74 100644 --- a/packages/server/src/automations/steps/updateRow.ts +++ b/packages/server/src/automations/steps/updateRow.ts @@ -2,76 +2,8 @@ import { EventEmitter } from "events" import * as rowController from "../../api/controllers/row" import * as automationUtils from "../automationUtils" import { buildCtx } from "./utils" -import { - AutomationActionStepId, - AutomationCustomIOType, - AutomationFeature, - AutomationIOType, - AutomationStepDefinition, - AutomationStepType, - UpdateRowStepInputs, - UpdateRowStepOutputs, -} from "@budibase/types" +import { UpdateRowStepInputs, UpdateRowStepOutputs } from "@budibase/types" -export const definition: AutomationStepDefinition = { - name: "Update Row", - tagline: "Update a {{inputs.enriched.table.name}} row", - icon: "Refresh", - description: "Update a row in your database", - type: AutomationStepType.ACTION, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - stepId: AutomationActionStepId.UPDATE_ROW, - inputs: {}, - schema: { - inputs: { - properties: { - meta: { - type: AutomationIOType.OBJECT, - title: "Field settings", - }, - row: { - type: AutomationIOType.OBJECT, - customType: AutomationCustomIOType.ROW, - title: "Table", - }, - rowId: { - type: AutomationIOType.STRING, - title: "Row ID", - }, - }, - required: ["row", "rowId"], - }, - outputs: { - properties: { - row: { - type: AutomationIOType.OBJECT, - customType: AutomationCustomIOType.ROW, - description: "The updated row", - }, - response: { - type: AutomationIOType.OBJECT, - description: "The response from the table", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the action was successful", - }, - id: { - type: AutomationIOType.STRING, - description: "The identifier of the updated row", - }, - revision: { - type: AutomationIOType.STRING, - description: "The revision of the updated row", - }, - }, - required: ["success", "id", "revision"], - }, - }, -} export async function run({ inputs, appId, diff --git a/packages/server/src/automations/steps/zapier.ts b/packages/server/src/automations/steps/zapier.ts index 888dae1e68..d17323cf0d 100644 --- a/packages/server/src/automations/steps/zapier.ts +++ b/packages/server/src/automations/steps/zapier.ts @@ -1,55 +1,6 @@ import fetch from "node-fetch" import { getFetchResponse } from "./utils" -import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - AutomationFeature, - ZapierStepInputs, - ZapierStepOutputs, -} from "@budibase/types" - -export const definition: AutomationStepDefinition = { - name: "Zapier Webhook", - stepId: AutomationActionStepId.zapier, - type: AutomationStepType.ACTION, - internal: false, - features: { - [AutomationFeature.LOOPING]: true, - }, - description: "Trigger a Zapier Zap via webhooks", - tagline: "Trigger a Zapier Zap", - icon: "ri-flashlight-line", - inputs: {}, - schema: { - inputs: { - properties: { - url: { - type: AutomationIOType.STRING, - title: "Webhook URL", - }, - body: { - type: AutomationIOType.JSON, - title: "Payload", - }, - }, - required: ["url"], - }, - outputs: { - properties: { - httpStatus: { - type: AutomationIOType.NUMBER, - description: "The HTTP status code of the request", - }, - response: { - type: AutomationIOType.STRING, - description: "The response from Zapier", - }, - }, - }, - }, -} +import { ZapierStepInputs, ZapierStepOutputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/tests/filter.spec.ts b/packages/server/src/automations/tests/filter.spec.ts index a1ed72dd74..674516517a 100644 --- a/packages/server/src/automations/tests/filter.spec.ts +++ b/packages/server/src/automations/tests/filter.spec.ts @@ -1,5 +1,7 @@ import * as setup from "./utilities" -import { FilterConditions } from "../steps/filter" +import { automations } from "@budibase/shared-core" + +const FilterConditions = automations.steps.filter.FilterConditions describe("test the filter logic", () => { const config = setup.getConfig() diff --git a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts index 45b251f4c1..3e203b7959 100644 --- a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts +++ b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts @@ -6,9 +6,11 @@ import { DatabaseName, datasourceDescribe, } from "../../../integrations/tests/utils" -import { FilterConditions } from "../../../automations/steps/filter" import { Knex } from "knex" import { generator } from "@budibase/backend-core/tests" +import { automations } from "@budibase/shared-core" + +const FilterConditions = automations.steps.filter.FilterConditions describe("Automation Scenarios", () => { let config = setup.getConfig() diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index acc35a0eeb..830d2ee5ca 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -42,7 +42,7 @@ import { } from "@budibase/types" import TestConfiguration from "../../../tests/utilities/TestConfiguration" import * as setup from "../utilities" -import { definition } from "../../../automations/steps/branch" +import { automations } from "@budibase/shared-core" type TriggerOutputs = | RowCreatedTriggerOutputs @@ -103,7 +103,7 @@ class BaseStepBuilder { branchStepInputs.children![branchId] = stepBuilder.build() }) const branchStep: AutomationStep = { - ...definition, + ...automations.steps.branch.definition, id: uuidv4(), stepId: AutomationActionStepId.BRANCH, inputs: branchStepInputs, diff --git a/packages/shared-core/src/automations/index.ts b/packages/shared-core/src/automations/index.ts new file mode 100644 index 0000000000..064796845e --- /dev/null +++ b/packages/shared-core/src/automations/index.ts @@ -0,0 +1 @@ +export * as steps from "./steps/index" diff --git a/packages/shared-core/src/automations/steps/bash.ts b/packages/shared-core/src/automations/steps/bash.ts new file mode 100644 index 0000000000..2980e18f0e --- /dev/null +++ b/packages/shared-core/src/automations/steps/bash.ts @@ -0,0 +1,47 @@ +import { + AutomationActionStepId, + AutomationCustomIOType, + AutomationFeature, + AutomationIOType, + AutomationStepDefinition, + AutomationStepType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Bash Scripting", + tagline: "Execute a bash command", + icon: "JourneyEvent", + description: "Run a bash script", + type: AutomationStepType.ACTION, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + stepId: AutomationActionStepId.EXECUTE_BASH, + inputs: {}, + schema: { + inputs: { + properties: { + code: { + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.CODE, + title: "Code", + }, + }, + required: ["code"], + }, + outputs: { + properties: { + stdout: { + type: AutomationIOType.STRING, + description: "Standard output of your bash command or script", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the command was successful", + }, + }, + required: ["stdout"], + }, + }, +} diff --git a/packages/server/src/automations/steps/branch.ts b/packages/shared-core/src/automations/steps/branch.ts similarity index 100% rename from packages/server/src/automations/steps/branch.ts rename to packages/shared-core/src/automations/steps/branch.ts diff --git a/packages/shared-core/src/automations/steps/collect.ts b/packages/shared-core/src/automations/steps/collect.ts new file mode 100644 index 0000000000..d4a6f6ec98 --- /dev/null +++ b/packages/shared-core/src/automations/steps/collect.ts @@ -0,0 +1,43 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Collect Data", + tagline: "Collect data to be sent to design", + icon: "Collection", + description: + "Collects specified data so it can be provided to the design section", + type: AutomationStepType.ACTION, + internal: true, + features: {}, + stepId: AutomationActionStepId.COLLECT, + inputs: {}, + schema: { + inputs: { + properties: { + collection: { + type: AutomationIOType.STRING, + title: "What to Collect", + }, + }, + required: ["collection"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + value: { + type: AutomationIOType.STRING, + description: "Collected data", + }, + }, + required: ["success", "value"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/createRow.ts b/packages/shared-core/src/automations/steps/createRow.ts new file mode 100644 index 0000000000..f51fdabb67 --- /dev/null +++ b/packages/shared-core/src/automations/steps/createRow.ts @@ -0,0 +1,67 @@ +import { + AutomationActionStepId, + AutomationCustomIOType, + AutomationFeature, + AutomationIOType, + AutomationStepDefinition, + AutomationStepType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Create Row", + tagline: "Create a {{inputs.enriched.table.name}} row", + icon: "TableRowAddBottom", + description: "Add a row to your database", + type: AutomationStepType.ACTION, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + stepId: AutomationActionStepId.CREATE_ROW, + inputs: {}, + schema: { + inputs: { + properties: { + row: { + type: AutomationIOType.OBJECT, + properties: { + tableId: { + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.TABLE, + }, + }, + customType: AutomationCustomIOType.ROW, + title: "Table", + required: ["tableId"], + }, + }, + required: ["row"], + }, + outputs: { + properties: { + row: { + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.ROW, + description: "The new row", + }, + response: { + type: AutomationIOType.OBJECT, + description: "The response from the table", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the row creation was successful", + }, + id: { + type: AutomationIOType.STRING, + description: "The identifier of the new row", + }, + revision: { + type: AutomationIOType.STRING, + description: "The revision of the new row", + }, + }, + required: ["success", "id", "revision"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/delay.ts b/packages/shared-core/src/automations/steps/delay.ts new file mode 100644 index 0000000000..627bc9ed2f --- /dev/null +++ b/packages/shared-core/src/automations/steps/delay.ts @@ -0,0 +1,38 @@ +import { + AutomationActionStepId, + AutomationIOType, + AutomationStepDefinition, + AutomationStepType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Delay", + icon: "Clock", + tagline: "Delay for {{inputs.time}} milliseconds", + description: "Delay the automation until an amount of time has passed", + stepId: AutomationActionStepId.DELAY, + internal: true, + features: {}, + inputs: {}, + schema: { + inputs: { + properties: { + time: { + type: AutomationIOType.NUMBER, + title: "Delay in milliseconds", + }, + }, + required: ["time"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the delay was successful", + }, + }, + required: ["success"], + }, + }, + type: AutomationStepType.LOGIC, +} diff --git a/packages/shared-core/src/automations/steps/deleteRow.ts b/packages/shared-core/src/automations/steps/deleteRow.ts new file mode 100644 index 0000000000..5a9de74392 --- /dev/null +++ b/packages/shared-core/src/automations/steps/deleteRow.ts @@ -0,0 +1,56 @@ +import { + AutomationActionStepId, + AutomationStepType, + AutomationIOType, + AutomationCustomIOType, + AutomationFeature, + AutomationStepDefinition, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + description: "Delete a row from your database", + icon: "TableRowRemoveCenter", + name: "Delete Row", + tagline: "Delete a {{inputs.enriched.table.name}} row", + type: AutomationStepType.ACTION, + stepId: AutomationActionStepId.DELETE_ROW, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + inputs: {}, + schema: { + inputs: { + properties: { + tableId: { + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.TABLE, + title: "Table", + }, + id: { + type: AutomationIOType.STRING, + title: "Row ID", + }, + }, + required: ["tableId", "id"], + }, + outputs: { + properties: { + row: { + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.ROW, + description: "The deleted row", + }, + response: { + type: AutomationIOType.OBJECT, + description: "The response from the table", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the deletion was successful", + }, + }, + required: ["row", "success"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/discord.ts b/packages/shared-core/src/automations/steps/discord.ts new file mode 100644 index 0000000000..906b970e1d --- /dev/null +++ b/packages/shared-core/src/automations/steps/discord.ts @@ -0,0 +1,60 @@ +import { + AutomationActionStepId, + AutomationStepType, + AutomationIOType, + AutomationFeature, + AutomationStepDefinition, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Discord Message", + tagline: "Send a message to a Discord server", + description: "Send a message to a Discord server", + icon: "ri-discord-line", + stepId: AutomationActionStepId.discord, + type: AutomationStepType.ACTION, + internal: false, + features: { + [AutomationFeature.LOOPING]: true, + }, + inputs: {}, + schema: { + inputs: { + properties: { + url: { + type: AutomationIOType.STRING, + title: "Discord Webhook URL", + }, + username: { + type: AutomationIOType.STRING, + title: "Bot Name", + }, + avatar_url: { + type: AutomationIOType.STRING, + title: "Bot Avatar URL", + }, + content: { + type: AutomationIOType.STRING, + title: "Message", + }, + }, + required: ["url", "content"], + }, + outputs: { + properties: { + httpStatus: { + type: AutomationIOType.NUMBER, + description: "The HTTP status code of the request", + }, + response: { + type: AutomationIOType.STRING, + description: "The response from the Discord Webhook", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the message sent successfully", + }, + }, + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/executeQuery.ts b/packages/shared-core/src/automations/steps/executeQuery.ts new file mode 100644 index 0000000000..bbb8b93d6b --- /dev/null +++ b/packages/shared-core/src/automations/steps/executeQuery.ts @@ -0,0 +1,59 @@ +import { + AutomationActionStepId, + AutomationCustomIOType, + AutomationFeature, + AutomationIOType, + AutomationStepDefinition, + AutomationStepType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "External Data Connector", + tagline: "Execute Data Connector", + icon: "Data", + description: "Execute a query in an external data connector", + type: AutomationStepType.ACTION, + stepId: AutomationActionStepId.EXECUTE_QUERY, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + inputs: {}, + schema: { + inputs: { + properties: { + query: { + type: AutomationIOType.OBJECT, + properties: { + queryId: { + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.QUERY, + }, + }, + customType: AutomationCustomIOType.QUERY_PARAMS, + title: "Parameters", + required: ["queryId"], + }, + }, + required: ["query"], + }, + outputs: { + properties: { + response: { + type: AutomationIOType.OBJECT, + description: "The response from the datasource execution", + }, + info: { + type: AutomationIOType.OBJECT, + description: + "Some query types may return extra data, like headers from a REST query", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + }, + required: ["response", "success"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/executeScript.ts b/packages/shared-core/src/automations/steps/executeScript.ts new file mode 100644 index 0000000000..dcc52f2905 --- /dev/null +++ b/packages/shared-core/src/automations/steps/executeScript.ts @@ -0,0 +1,47 @@ +import { + AutomationActionStepId, + AutomationCustomIOType, + AutomationFeature, + AutomationIOType, + AutomationStepDefinition, + AutomationStepType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "JS Scripting", + tagline: "Execute JavaScript Code", + icon: "Code", + description: "Run a piece of JavaScript code in your automation", + type: AutomationStepType.ACTION, + internal: true, + stepId: AutomationActionStepId.EXECUTE_SCRIPT, + inputs: {}, + features: { + [AutomationFeature.LOOPING]: true, + }, + schema: { + inputs: { + properties: { + code: { + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.CODE, + title: "Code", + }, + }, + required: ["code"], + }, + outputs: { + properties: { + value: { + type: AutomationIOType.STRING, + description: "The result of the return statement", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + }, + required: ["success"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/filter.ts b/packages/shared-core/src/automations/steps/filter.ts new file mode 100644 index 0000000000..70dcb6f66e --- /dev/null +++ b/packages/shared-core/src/automations/steps/filter.ts @@ -0,0 +1,69 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, +} from "@budibase/types" + +export const FilterConditions = { + EQUAL: "EQUAL", + NOT_EQUAL: "NOT_EQUAL", + GREATER_THAN: "GREATER_THAN", + LESS_THAN: "LESS_THAN", +} + +export const PrettyFilterConditions = { + [FilterConditions.EQUAL]: "Equals", + [FilterConditions.NOT_EQUAL]: "Not equals", + [FilterConditions.GREATER_THAN]: "Greater than", + [FilterConditions.LESS_THAN]: "Less than", +} + +export const definition: AutomationStepDefinition = { + name: "Condition", + tagline: "{{inputs.field}} {{inputs.condition}} {{inputs.value}}", + icon: "Branch2", + description: + "Conditionally halt automations which do not meet certain conditions", + type: AutomationStepType.LOGIC, + internal: true, + features: {}, + stepId: AutomationActionStepId.FILTER, + inputs: { + condition: FilterConditions.EQUAL, + }, + schema: { + inputs: { + properties: { + field: { + type: AutomationIOType.STRING, + title: "Reference Value", + }, + condition: { + type: AutomationIOType.STRING, + title: "Condition", + enum: Object.values(FilterConditions), + pretty: Object.values(PrettyFilterConditions), + }, + value: { + type: AutomationIOType.STRING, + title: "Comparison Value", + }, + }, + required: ["field", "condition", "value"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + result: { + type: AutomationIOType.BOOLEAN, + description: "Whether the logic block passed", + }, + }, + required: ["success", "result"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/index.ts b/packages/shared-core/src/automations/steps/index.ts new file mode 100644 index 0000000000..d23c5622b8 --- /dev/null +++ b/packages/shared-core/src/automations/steps/index.ts @@ -0,0 +1,22 @@ +export * as bash from "./bash" +export * as branch from "./branch" +export * as collect from "./collect" +export * as createRow from "./createRow" +export * as delay from "./delay" +export * as deleteRow from "./deleteRow" +export * as discord from "./discord" +export * as executeQuery from "./executeQuery" +export * as executeScript from "./executeScript" +export * as filter from "./filter" +export * as loop from "./loop" +export * as make from "./make" +export * as n8n from "./n8n" +export * as openai from "./openai" +export * as outgoingWebhook from "./outgoingWebhook" +export * as queryRows from "./queryRows" +export * as sendSmtpEmail from "./sendSmtpEmail" +export * as serverLog from "./serverLog" +export * as slack from "./slack" +export * as triggerAutomationRun from "./triggerAutomationRun" +export * as updateRow from "./updateRow" +export * as zapier from "./zapier" diff --git a/packages/server/src/automations/steps/loop.ts b/packages/shared-core/src/automations/steps/loop.ts similarity index 100% rename from packages/server/src/automations/steps/loop.ts rename to packages/shared-core/src/automations/steps/loop.ts diff --git a/packages/shared-core/src/automations/steps/make.ts b/packages/shared-core/src/automations/steps/make.ts new file mode 100644 index 0000000000..4f9e6c40ad --- /dev/null +++ b/packages/shared-core/src/automations/steps/make.ts @@ -0,0 +1,55 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, + AutomationFeature, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Make Integration", + stepTitle: "Make", + tagline: "Trigger a Make scenario", + description: + "Performs a webhook call to Make and gets the response (if configured)", + icon: "ri-shut-down-line", + stepId: AutomationActionStepId.integromat, + type: AutomationStepType.ACTION, + internal: false, + features: { + [AutomationFeature.LOOPING]: true, + }, + inputs: {}, + schema: { + inputs: { + properties: { + url: { + type: AutomationIOType.STRING, + title: "Webhook URL", + }, + body: { + type: AutomationIOType.JSON, + title: "Payload", + }, + }, + required: ["url", "body"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether call was successful", + }, + httpStatus: { + type: AutomationIOType.NUMBER, + description: "The HTTP status code returned", + }, + response: { + type: AutomationIOType.OBJECT, + description: "The webhook response - this can have properties", + }, + }, + required: ["success", "response"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/n8n.ts b/packages/shared-core/src/automations/steps/n8n.ts new file mode 100644 index 0000000000..fcb407bcbf --- /dev/null +++ b/packages/shared-core/src/automations/steps/n8n.ts @@ -0,0 +1,65 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, + AutomationFeature, + HttpMethod, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "n8n Integration", + stepTitle: "n8n", + tagline: "Trigger an n8n workflow", + description: + "Performs a webhook call to n8n and gets the response (if configured)", + icon: "ri-shut-down-line", + stepId: AutomationActionStepId.n8n, + type: AutomationStepType.ACTION, + internal: false, + features: { + [AutomationFeature.LOOPING]: true, + }, + inputs: {}, + schema: { + inputs: { + properties: { + url: { + type: AutomationIOType.STRING, + title: "Webhook URL", + }, + method: { + type: AutomationIOType.STRING, + title: "Method", + enum: Object.values(HttpMethod), + }, + authorization: { + type: AutomationIOType.STRING, + title: "Authorization", + }, + body: { + type: AutomationIOType.JSON, + title: "Payload", + }, + }, + required: ["url", "method"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether call was successful", + }, + httpStatus: { + type: AutomationIOType.NUMBER, + description: "The HTTP status code returned", + }, + response: { + type: AutomationIOType.OBJECT, + description: "The webhook response - this can have properties", + }, + }, + required: ["success", "response"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/openai.ts b/packages/shared-core/src/automations/steps/openai.ts new file mode 100644 index 0000000000..dc25e62f96 --- /dev/null +++ b/packages/shared-core/src/automations/steps/openai.ts @@ -0,0 +1,56 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, +} from "@budibase/types" + +enum Model { + GPT_4O_MINI = "gpt-4o-mini", + GPT_4O = "gpt-4o", + GPT_4 = "gpt-4", + GPT_35_TURBO = "gpt-3.5-turbo", +} + +export const definition: AutomationStepDefinition = { + name: "OpenAI", + tagline: "Send prompts to ChatGPT", + icon: "Algorithm", + description: "Interact with the OpenAI ChatGPT API.", + type: AutomationStepType.ACTION, + internal: true, + features: {}, + stepId: AutomationActionStepId.OPENAI, + inputs: { + prompt: "", + }, + schema: { + inputs: { + properties: { + prompt: { + type: AutomationIOType.STRING, + title: "Prompt", + }, + model: { + type: AutomationIOType.STRING, + title: "Model", + enum: Object.values(Model), + }, + }, + required: ["prompt", "model"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + response: { + type: AutomationIOType.STRING, + description: "What was output", + }, + }, + required: ["success", "response"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/outgoingWebhook.ts b/packages/shared-core/src/automations/steps/outgoingWebhook.ts new file mode 100644 index 0000000000..9e0b1b375b --- /dev/null +++ b/packages/shared-core/src/automations/steps/outgoingWebhook.ts @@ -0,0 +1,79 @@ +import { + AutomationActionStepId, + AutomationCustomIOType, + AutomationFeature, + AutomationIOType, + AutomationStepDefinition, + AutomationStepType, +} from "@budibase/types" + +enum RequestType { + POST = "POST", + GET = "GET", + PUT = "PUT", + DELETE = "DELETE", + PATCH = "PATCH", +} + +export const definition: AutomationStepDefinition = { + deprecated: true, + name: "Outgoing webhook", + tagline: "Send a {{inputs.requestMethod}} request", + icon: "Send", + description: "Send a request of specified method to a URL", + type: AutomationStepType.ACTION, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + stepId: AutomationActionStepId.OUTGOING_WEBHOOK, + inputs: { + requestMethod: "POST", + url: "http://", + requestBody: "{}", + headers: "{}", + }, + schema: { + inputs: { + properties: { + requestMethod: { + type: AutomationIOType.STRING, + enum: Object.values(RequestType), + title: "Request method", + }, + url: { + type: AutomationIOType.STRING, + title: "URL", + }, + requestBody: { + type: AutomationIOType.STRING, + title: "JSON Body", + customType: AutomationCustomIOType.WIDE, + }, + headers: { + type: AutomationIOType.STRING, + title: "Headers", + customType: AutomationCustomIOType.WIDE, + }, + }, + required: ["requestMethod", "url"], + }, + outputs: { + properties: { + response: { + type: AutomationIOType.OBJECT, + description: "The response from the webhook", + }, + httpStatus: { + type: AutomationIOType.NUMBER, + description: "The HTTP status code returned", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + }, + required: ["response", "success"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/queryRows.ts b/packages/shared-core/src/automations/steps/queryRows.ts new file mode 100644 index 0000000000..31fec1e112 --- /dev/null +++ b/packages/shared-core/src/automations/steps/queryRows.ts @@ -0,0 +1,75 @@ +import { + AutomationActionStepId, + AutomationCustomIOType, + AutomationFeature, + AutomationIOType, + AutomationStepDefinition, + AutomationStepType, + SortOrder, +} from "@budibase/types" + +const SortOrderPretty = { + [SortOrder.ASCENDING]: "Ascending", + [SortOrder.DESCENDING]: "Descending", +} + +export const definition: AutomationStepDefinition = { + description: "Query rows from the database", + icon: "Search", + name: "Query rows", + tagline: "Query rows from {{inputs.enriched.table.name}} table", + type: AutomationStepType.ACTION, + stepId: AutomationActionStepId.QUERY_ROWS, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + inputs: {}, + schema: { + inputs: { + properties: { + tableId: { + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.TABLE, + title: "Table", + }, + filters: { + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.FILTERS, + title: "Filtering", + }, + sortColumn: { + type: AutomationIOType.STRING, + title: "Sort Column", + customType: AutomationCustomIOType.COLUMN, + }, + sortOrder: { + type: AutomationIOType.STRING, + title: "Sort Order", + enum: Object.values(SortOrder), + pretty: Object.values(SortOrderPretty), + }, + limit: { + type: AutomationIOType.NUMBER, + title: "Limit", + customType: AutomationCustomIOType.QUERY_LIMIT, + }, + }, + required: ["tableId"], + }, + outputs: { + properties: { + rows: { + type: AutomationIOType.ARRAY, + customType: AutomationCustomIOType.ROWS, + description: "The rows that were found", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the query was successful", + }, + }, + required: ["rows", "success"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/sendSmtpEmail.ts b/packages/shared-core/src/automations/steps/sendSmtpEmail.ts new file mode 100644 index 0000000000..99ded7fdb3 --- /dev/null +++ b/packages/shared-core/src/automations/steps/sendSmtpEmail.ts @@ -0,0 +1,95 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, + AutomationFeature, + AutomationCustomIOType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + description: "Send an email using SMTP", + tagline: "Send SMTP email to {{inputs.to}}", + icon: "Email", + name: "Send Email (SMTP)", + type: AutomationStepType.ACTION, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + stepId: AutomationActionStepId.SEND_EMAIL_SMTP, + inputs: {}, + schema: { + inputs: { + properties: { + to: { + type: AutomationIOType.STRING, + title: "Send To", + }, + from: { + type: AutomationIOType.STRING, + title: "Send From", + }, + cc: { + type: AutomationIOType.STRING, + title: "CC", + }, + bcc: { + type: AutomationIOType.STRING, + title: "BCC", + }, + subject: { + type: AutomationIOType.STRING, + title: "Email Subject", + }, + contents: { + type: AutomationIOType.STRING, + title: "HTML Contents", + }, + addInvite: { + type: AutomationIOType.BOOLEAN, + title: "Add calendar invite", + }, + startTime: { + type: AutomationIOType.DATE, + title: "Start Time", + dependsOn: "addInvite", + }, + endTime: { + type: AutomationIOType.DATE, + title: "End Time", + dependsOn: "addInvite", + }, + summary: { + type: AutomationIOType.STRING, + title: "Meeting Summary", + dependsOn: "addInvite", + }, + location: { + type: AutomationIOType.STRING, + title: "Location", + dependsOn: "addInvite", + }, + attachments: { + type: AutomationIOType.ATTACHMENT, + customType: AutomationCustomIOType.MULTI_ATTACHMENTS, + title: "Attachments", + }, + }, + required: ["to", "from", "subject", "contents"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the email was sent", + }, + response: { + type: AutomationIOType.OBJECT, + description: "A response from the email client, this may be an error", + }, + }, + required: ["success"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/serverLog.ts b/packages/shared-core/src/automations/steps/serverLog.ts new file mode 100644 index 0000000000..44ffb84049 --- /dev/null +++ b/packages/shared-core/src/automations/steps/serverLog.ts @@ -0,0 +1,47 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, + AutomationFeature, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Backend log", + tagline: "Console log a value in the backend", + icon: "Monitoring", + description: "Logs the given text to the server (using console.log)", + type: AutomationStepType.ACTION, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + stepId: AutomationActionStepId.SERVER_LOG, + inputs: { + text: "", + }, + schema: { + inputs: { + properties: { + text: { + type: AutomationIOType.STRING, + title: "Log", + }, + }, + required: ["text"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + message: { + type: AutomationIOType.STRING, + description: "What was output", + }, + }, + required: ["success", "message"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/slack.ts b/packages/shared-core/src/automations/steps/slack.ts new file mode 100644 index 0000000000..185e00a8ed --- /dev/null +++ b/packages/shared-core/src/automations/steps/slack.ts @@ -0,0 +1,52 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, + AutomationFeature, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Slack Message", + tagline: "Send a message to Slack", + description: "Send a message to Slack", + icon: "ri-slack-line", + stepId: AutomationActionStepId.slack, + type: AutomationStepType.ACTION, + internal: false, + features: { + [AutomationFeature.LOOPING]: true, + }, + inputs: {}, + schema: { + inputs: { + properties: { + url: { + type: AutomationIOType.STRING, + title: "Incoming Webhook URL", + }, + text: { + type: AutomationIOType.STRING, + title: "Message", + }, + }, + required: ["url", "text"], + }, + outputs: { + properties: { + httpStatus: { + type: AutomationIOType.NUMBER, + description: "The HTTP status code of the request", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the message sent successfully", + }, + response: { + type: AutomationIOType.STRING, + description: "The response from the Slack Webhook", + }, + }, + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/triggerAutomationRun.ts b/packages/shared-core/src/automations/steps/triggerAutomationRun.ts new file mode 100644 index 0000000000..3aa78b6015 --- /dev/null +++ b/packages/shared-core/src/automations/steps/triggerAutomationRun.ts @@ -0,0 +1,55 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, + AutomationCustomIOType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Trigger an automation", + tagline: "Triggers an automation synchronously", + icon: "Sync", + description: "Triggers an automation synchronously", + type: AutomationStepType.ACTION, + internal: true, + features: {}, + stepId: AutomationActionStepId.TRIGGER_AUTOMATION_RUN, + inputs: {}, + schema: { + inputs: { + properties: { + automation: { + type: AutomationIOType.OBJECT, + properties: { + automationId: { + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.AUTOMATION, + }, + }, + customType: AutomationCustomIOType.AUTOMATION_FIELDS, + title: "automatioFields", + required: ["automationId"], + }, + timeout: { + type: AutomationIOType.NUMBER, + title: "Timeout (ms)", + }, + }, + required: ["automationId"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the automation was successful", + }, + value: { + type: AutomationIOType.OBJECT, + description: "Automation Result", + }, + }, + required: ["success", "value"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/updateRow.ts b/packages/shared-core/src/automations/steps/updateRow.ts new file mode 100644 index 0000000000..30f86214bb --- /dev/null +++ b/packages/shared-core/src/automations/steps/updateRow.ts @@ -0,0 +1,68 @@ +import { + AutomationActionStepId, + AutomationCustomIOType, + AutomationFeature, + AutomationIOType, + AutomationStepDefinition, + AutomationStepType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Update Row", + tagline: "Update a {{inputs.enriched.table.name}} row", + icon: "Refresh", + description: "Update a row in your database", + type: AutomationStepType.ACTION, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + stepId: AutomationActionStepId.UPDATE_ROW, + inputs: {}, + schema: { + inputs: { + properties: { + meta: { + type: AutomationIOType.OBJECT, + title: "Field settings", + }, + row: { + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.ROW, + title: "Table", + }, + rowId: { + type: AutomationIOType.STRING, + title: "Row ID", + }, + }, + required: ["row", "rowId"], + }, + outputs: { + properties: { + row: { + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.ROW, + description: "The updated row", + }, + response: { + type: AutomationIOType.OBJECT, + description: "The response from the table", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + id: { + type: AutomationIOType.STRING, + description: "The identifier of the updated row", + }, + revision: { + type: AutomationIOType.STRING, + description: "The revision of the updated row", + }, + }, + required: ["success", "id", "revision"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/zapier.ts b/packages/shared-core/src/automations/steps/zapier.ts new file mode 100644 index 0000000000..b1afd31def --- /dev/null +++ b/packages/shared-core/src/automations/steps/zapier.ts @@ -0,0 +1,48 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, + AutomationFeature, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Zapier Webhook", + stepId: AutomationActionStepId.zapier, + type: AutomationStepType.ACTION, + internal: false, + features: { + [AutomationFeature.LOOPING]: true, + }, + description: "Trigger a Zapier Zap via webhooks", + tagline: "Trigger a Zapier Zap", + icon: "ri-flashlight-line", + inputs: {}, + schema: { + inputs: { + properties: { + url: { + type: AutomationIOType.STRING, + title: "Webhook URL", + }, + body: { + type: AutomationIOType.JSON, + title: "Payload", + }, + }, + required: ["url"], + }, + outputs: { + properties: { + httpStatus: { + type: AutomationIOType.NUMBER, + description: "The HTTP status code of the request", + }, + response: { + type: AutomationIOType.STRING, + description: "The response from Zapier", + }, + }, + }, + }, +} diff --git a/packages/shared-core/src/index.ts b/packages/shared-core/src/index.ts index 0833f999ee..a4f4208f4a 100644 --- a/packages/shared-core/src/index.ts +++ b/packages/shared-core/src/index.ts @@ -5,3 +5,4 @@ export * as utils from "./utils" export * as sdk from "./sdk" export * from "./table" export * from "./themes" +export * as automations from "./automations" From e6d536bcc83e172926fb753ea30ecf95ba2860a1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 16 Jan 2025 18:15:54 +0000 Subject: [PATCH 023/112] Getting the line number calculated correctly, as well as adding some basic test cases. --- .../bindings/EvaluationSidePanel.svelte | 2 +- .../src/helpers/javascript.ts | 23 +++++++++++-- packages/string-templates/src/index.ts | 16 ++------- packages/string-templates/src/utilities.ts | 17 ++++++++++ .../string-templates/test/jsLogging.spec.ts | 33 +++++++++++++++++++ 5 files changed, 74 insertions(+), 17 deletions(-) create mode 100644 packages/string-templates/test/jsLogging.spec.ts diff --git a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte index 41245af4f9..984fba9b7a 100644 --- a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte +++ b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte @@ -102,7 +102,7 @@ {@html logLine.log} {#if logLine.line} - line {logLine.line} + :{logLine.line} {/if} {/each} diff --git a/packages/string-templates/src/helpers/javascript.ts b/packages/string-templates/src/helpers/javascript.ts index a0fdd3cbe5..a1bfb7a824 100644 --- a/packages/string-templates/src/helpers/javascript.ts +++ b/packages/string-templates/src/helpers/javascript.ts @@ -1,4 +1,9 @@ -import { atob, isBackendService, isJSAllowed } from "../utilities" +import { + atob, + frontendWrapJS, + isBackendService, + isJSAllowed, +} from "../utilities" import { LITERAL_MARKER } from "../helpers/constants" import { getJsHelperList } from "./list" import { iifeWrapper } from "../iife" @@ -117,7 +122,21 @@ export function processJS(handlebars: string, context: any) { const logs: Log[] = [] // logging only supported on frontend if (!isBackendService()) { - const log = (log: string) => logs.push({ log }) + // this counts the lines in the wrapped JS *before* the user's code, so that we can minus it + const jsLineCount = frontendWrapJS(js).split(js)[0].split("\n").length + const log = (log: string) => { + // quick way to find out what line this is being called from + // its an anonymous function and we look for the overall length to find the + // line number we care about (from the users function) + // JS stack traces are in the format function:line:column + const lineNumber = new Error().stack?.match( + /:(\d+):\d+/ + )?.[1] + logs.push({ + log, + line: lineNumber ? parseInt(lineNumber) - jsLineCount : undefined, + }) + } sandboxContext.console = { log: log, info: log, diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index 553c0e8861..a21bfdb755 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -8,6 +8,7 @@ import { FIND_ANY_HBS_REGEX, FIND_HBS_REGEX, findDoubleHbsInstances, + frontendWrapJS, isBackendService, prefixStrings, } from "./utilities" @@ -511,20 +512,7 @@ export function browserJSSetup() { setJSRunner((js: string, context: Record) => { createContext(context) - const wrappedJs = ` - result = { - result: null, - error: null, - }; - - try { - result.result = ${js}; - } catch (e) { - result.error = e; - } - - result; - ` + const wrappedJs = frontendWrapJS(js) const result = runInNewContext(wrappedJs, context, { timeout: 1000 }) if (result.error) { diff --git a/packages/string-templates/src/utilities.ts b/packages/string-templates/src/utilities.ts index 779bef3735..dba1faab17 100644 --- a/packages/string-templates/src/utilities.ts +++ b/packages/string-templates/src/utilities.ts @@ -86,3 +86,20 @@ export const prefixStrings = ( const regexPattern = new RegExp(`\\b(${escapedStrings.join("|")})\\b`, "g") return baseString.replace(regexPattern, `${prefix}$1`) } + +export function frontendWrapJS(js: string) { + return ` + result = { + result: null, + error: null, + }; + + try { + result.result = ${js}; + } catch (e) { + result.error = e; + } + + result; + ` +} diff --git a/packages/string-templates/test/jsLogging.spec.ts b/packages/string-templates/test/jsLogging.spec.ts new file mode 100644 index 0000000000..9b2bb945d2 --- /dev/null +++ b/packages/string-templates/test/jsLogging.spec.ts @@ -0,0 +1,33 @@ +import { + processStringWithLogsSync, + encodeJSBinding, + defaultJSSetup, +} from "../src/index" + +const processJS = (js: string, context?: object) => { + return processStringWithLogsSync(encodeJSBinding(js), context) +} + +describe("Javascript", () => { + beforeAll(() => { + defaultJSSetup() + }) + + describe("Test logging in JS bindings", () => { + it("should execute a simple expression", () => { + const output = processJS( + `console.log("hello"); + console.log("world"); + console.log("foo"); + return "hello"` + ) + expect(output.result).toBe("hello") + expect(output.logs[0].log).toBe("hello") + expect(output.logs[0].line).toBe(1) + expect(output.logs[1].log).toBe("world") + expect(output.logs[1].line).toBe(2) + expect(output.logs[2].log).toBe("foo") + expect(output.logs[2].line).toBe(3) + }) + }) +}) From 8cf7375202a3bebb97baee2ac33a75950cdc13e3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 17 Jan 2025 10:33:01 +0100 Subject: [PATCH 024/112] Type schema object --- packages/client/src/utils/schema.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/client/src/utils/schema.ts b/packages/client/src/utils/schema.ts index 5400d62087..9ea868751d 100644 --- a/packages/client/src/utils/schema.ts +++ b/packages/client/src/utils/schema.ts @@ -1,5 +1,6 @@ import { API } from "api" import { DataFetchMap, DataFetchType } from "@budibase/frontend-core" +import { FieldType, TableSchema } from "@budibase/types" /** * Constructs a fetch instance for a given datasource. @@ -42,14 +43,14 @@ export const fetchDatasourceSchema = async < } // Get the normal schema as long as we aren't wanting a form schema - let schema: any + let schema: TableSchema | undefined if (datasource?.type !== "query" || !options?.formSchema) { - schema = instance.getSchema(definition as any) + schema = instance.getSchema(definition as any) as TableSchema } else if ("parameters" in definition && definition.parameters?.length) { schema = {} - definition.parameters.forEach(param => { - schema[param.name] = { ...param, type: "string" } - }) + for (const param of definition.parameters) { + schema[param.name] = { ...param, type: FieldType.STRING } + } } if (!schema) { return null @@ -57,11 +58,11 @@ export const fetchDatasourceSchema = async < // Strip hidden fields from views if (datasource.type === "viewV2") { - Object.keys(schema).forEach(field => { + for (const field of Object.keys(schema)) { if (!schema[field].visible) { delete schema[field] } - }) + } } // Enrich schema with relationships if required From 84dbcb0f6914979285356c7576cf1162974ef410 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 17 Jan 2025 11:51:48 +0100 Subject: [PATCH 025/112] Change errorState to boolean --- packages/client/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index 97c0525f55..80ac725df3 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -17,7 +17,7 @@ export interface SDK { export type Component = Writable<{ id: string styles: any - errorState: string + errorState: boolean }> export type Context = Readable From ed35acc01affb459b38a0ea3a0f7a454ecacf5c8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 17 Jan 2025 11:57:38 +0100 Subject: [PATCH 026/112] Revert Component store to readable --- packages/client/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index 80ac725df3..ada30d53ec 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -1,6 +1,6 @@ import { APIClient } from "@budibase/frontend-core" import type { ActionTypes } from "./constants" -import { Readable, Writable } from "svelte/store" +import { Readable } from "svelte/store" export interface SDK { API: APIClient @@ -14,7 +14,7 @@ export interface SDK { }> } -export type Component = Writable<{ +export type Component = Readable<{ id: string styles: any errorState: boolean From e146d995ebcd89c661e769b21743875ceddd81c0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 17 Jan 2025 11:06:55 +0000 Subject: [PATCH 027/112] Adding in support for multi-parameter logs and actual logging to console. --- .../bindings/EvaluationSidePanel.svelte | 2 +- .../src/helpers/javascript.ts | 48 +++++++++++-------- packages/string-templates/src/types.ts | 2 +- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte index 984fba9b7a..dc3f585033 100644 --- a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte +++ b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte @@ -19,7 +19,7 @@ $: success = !error && !empty $: highlightedResult = highlight(expressionResult) $: highlightedLogs = expressionLogs.map(l => ({ - log: highlight(l.log), + log: highlight(l.log.join(", ")), line: l.line, })) diff --git a/packages/string-templates/src/helpers/javascript.ts b/packages/string-templates/src/helpers/javascript.ts index a1bfb7a824..5a4f69de5b 100644 --- a/packages/string-templates/src/helpers/javascript.ts +++ b/packages/string-templates/src/helpers/javascript.ts @@ -124,28 +124,38 @@ export function processJS(handlebars: string, context: any) { if (!isBackendService()) { // this counts the lines in the wrapped JS *before* the user's code, so that we can minus it const jsLineCount = frontendWrapJS(js).split(js)[0].split("\n").length - const log = (log: string) => { - // quick way to find out what line this is being called from - // its an anonymous function and we look for the overall length to find the - // line number we care about (from the users function) - // JS stack traces are in the format function:line:column - const lineNumber = new Error().stack?.match( - /:(\d+):\d+/ - )?.[1] - logs.push({ - log, - line: lineNumber ? parseInt(lineNumber) - jsLineCount : undefined, - }) + const buildLogResponse = ( + type: "log" | "info" | "debug" | "warn" | "error" | "trace" | "table" + ) => { + return (...props: any[]) => { + console[type](...props) + props.forEach((prop, index) => { + if (typeof prop === "object") { + props[index] = JSON.stringify(prop) + } + }) + // quick way to find out what line this is being called from + // its an anonymous function and we look for the overall length to find the + // line number we care about (from the users function) + // JS stack traces are in the format function:line:column + const lineNumber = new Error().stack?.match( + /:(\d+):\d+/ + )?.[1] + logs.push({ + log: props, + line: lineNumber ? parseInt(lineNumber) - jsLineCount : undefined, + }) + } } sandboxContext.console = { - log: log, - info: log, - debug: log, - warn: log, - error: log, + log: buildLogResponse("log"), + info: buildLogResponse("info"), + debug: buildLogResponse("debug"), + warn: buildLogResponse("warn"), + error: buildLogResponse("error"), // two below may need special cases - trace: log, - table: log, + trace: buildLogResponse("trace"), + table: buildLogResponse("table"), } } diff --git a/packages/string-templates/src/types.ts b/packages/string-templates/src/types.ts index c973142c93..a32149c8bb 100644 --- a/packages/string-templates/src/types.ts +++ b/packages/string-templates/src/types.ts @@ -10,6 +10,6 @@ export interface ProcessOptions { } export interface Log { - log: string + log: any[] line?: number } From 14bc491d2c83583c00b62e83cccc62c42112c3be Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 17 Jan 2025 12:44:39 +0100 Subject: [PATCH 028/112] More typings --- packages/bbui/src/Icon/Icon.svelte | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/bbui/src/Icon/Icon.svelte b/packages/bbui/src/Icon/Icon.svelte index 1a851248b9..35f176035e 100644 --- a/packages/bbui/src/Icon/Icon.svelte +++ b/packages/bbui/src/Icon/Icon.svelte @@ -5,19 +5,19 @@ TooltipType, } from "../Tooltip/AbsTooltip.svelte" - export let name = "Add" - export let hidden = false + export let name: string = "Add" + export let hidden: boolean = false export let size = "M" - export let hoverable = false - export let disabled = false + export let hoverable: boolean = false + export let disabled: boolean = false export let color: string = undefined - export let hoverColor = undefined - export let tooltip = undefined + export let hoverColor: string = undefined + export let tooltip: string = undefined export let tooltipPosition = TooltipPosition.Bottom export let tooltipType = TooltipType.Default - export let tooltipColor = undefined - export let tooltipWrap = true - export let newStyles = false + export let tooltipColor: string = undefined + export let tooltipWrap: boolean = true + export let newStyles: boolean = false Date: Fri, 17 Jan 2025 17:18:42 +0000 Subject: [PATCH 029/112] Logging with types - allows for coloured outputs. --- packages/bbui/src/bbui.css | 5 +++ .../bindings/EvaluationSidePanel.svelte | 36 ++++++++++++++----- .../src/helpers/javascript.ts | 11 +++--- packages/string-templates/src/index.ts | 2 +- packages/string-templates/src/types.ts | 3 ++ 5 files changed, 42 insertions(+), 15 deletions(-) diff --git a/packages/bbui/src/bbui.css b/packages/bbui/src/bbui.css index dd0588818e..810c5ff2c0 100644 --- a/packages/bbui/src/bbui.css +++ b/packages/bbui/src/bbui.css @@ -45,6 +45,11 @@ --purple: #806fde; --purple-dark: #130080; + --error-bg: rgba(226, 109, 105, 0.3); + --warning-bg: rgba(255, 210, 106, 0.3); + --error-content: rgba(226, 109, 105, 0.6); + --warning-content: rgba(255, 210, 106, 0.6); + --rounded-small: 4px; --rounded-medium: 8px; --rounded-large: 16px; diff --git a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte index dc3f585033..fcd23bb816 100644 --- a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte +++ b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte @@ -21,6 +21,7 @@ $: highlightedLogs = expressionLogs.map(l => ({ log: highlight(l.log.join(", ")), line: l.line, + type: l.type, })) const formatError = (err: any) => { @@ -67,7 +68,7 @@
{#if error} - +
Error
{#if evaluating}
@@ -98,9 +99,24 @@ {:else}
{#each highlightedLogs as logLine} -
- - {@html logLine.log} +
+
+ {#if logLine.type === "error"} + + {:else if logLine.type === "warn"} + + {/if} + + {@html logLine.log} +
{#if logLine.line} :{logLine.line} {/if} @@ -149,10 +165,9 @@ height: 100%; z-index: 1; position: absolute; - opacity: 10%; } .header.error::before { - background: var(--spectrum-global-color-red-400); + background: var(--error-bg); } .body { flex: 1 1 auto; @@ -167,15 +182,20 @@ } .output-lines { display: flex; - gap: var(--spacing-s); flex-direction: column; + gap: var(--spacing-xs); } .line { border-bottom: var(--border-light); - padding-bottom: var(--spacing-s); display: flex; flex-direction: row; justify-content: space-between; align-items: end; + padding: var(--spacing-s); + } + .icon-log { + display: flex; + gap: var(--spacing-s); + align-items: start; } diff --git a/packages/string-templates/src/helpers/javascript.ts b/packages/string-templates/src/helpers/javascript.ts index 5a4f69de5b..997ca7b6ec 100644 --- a/packages/string-templates/src/helpers/javascript.ts +++ b/packages/string-templates/src/helpers/javascript.ts @@ -9,7 +9,7 @@ import { getJsHelperList } from "./list" import { iifeWrapper } from "../iife" import { JsTimeoutError, UserScriptError } from "../errors" import { cloneDeep } from "lodash/fp" -import { Log } from "../types" +import { Log, LogType } from "../types" // The method of executing JS scripts depends on the bundle being built. // This setter is used in the entrypoint (either index.js or index.mjs). @@ -124,9 +124,7 @@ export function processJS(handlebars: string, context: any) { if (!isBackendService()) { // this counts the lines in the wrapped JS *before* the user's code, so that we can minus it const jsLineCount = frontendWrapJS(js).split(js)[0].split("\n").length - const buildLogResponse = ( - type: "log" | "info" | "debug" | "warn" | "error" | "trace" | "table" - ) => { + const buildLogResponse = (type: LogType) => { return (...props: any[]) => { console[type](...props) props.forEach((prop, index) => { @@ -144,6 +142,7 @@ export function processJS(handlebars: string, context: any) { logs.push({ log: props, line: lineNumber ? parseInt(lineNumber) - jsLineCount : undefined, + type, }) } } @@ -153,8 +152,8 @@ export function processJS(handlebars: string, context: any) { debug: buildLogResponse("debug"), warn: buildLogResponse("warn"), error: buildLogResponse("error"), - // two below may need special cases - trace: buildLogResponse("trace"), + // table should be treated differently, but works the same + // as the rest of the logs for now table: buildLogResponse("table"), } } diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index a21bfdb755..67ccde727e 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -19,7 +19,7 @@ import manifest from "./manifest.json" import { Log, ProcessOptions } from "./types" import { UserScriptError } from "./errors" -export type { Log } from "./types" +export type { Log, LogType } from "./types" export { helpersToRemoveForJs, getJsHelperList } from "./helpers/list" export { FIND_ANY_HBS_REGEX } from "./utilities" export { setJSRunner, setOnErrorLog } from "./helpers/javascript" diff --git a/packages/string-templates/src/types.ts b/packages/string-templates/src/types.ts index a32149c8bb..f6ec7098f9 100644 --- a/packages/string-templates/src/types.ts +++ b/packages/string-templates/src/types.ts @@ -9,7 +9,10 @@ export interface ProcessOptions { disabledHelpers?: string[] } +export type LogType = "log" | "info" | "debug" | "warn" | "error" | "table" + export interface Log { log: any[] line?: number + type?: LogType } From f84ddd2b57e6625fa2533fa6637b75e6336202f3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 17 Jan 2025 21:17:39 +0100 Subject: [PATCH 030/112] Fix types after merge --- packages/bbui/src/Icon/Icon.svelte | 8 ++++---- packages/bbui/src/Tooltip/AbsTooltip.svelte | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/bbui/src/Icon/Icon.svelte b/packages/bbui/src/Icon/Icon.svelte index 35f176035e..7438fab5fd 100644 --- a/packages/bbui/src/Icon/Icon.svelte +++ b/packages/bbui/src/Icon/Icon.svelte @@ -10,12 +10,12 @@ export let size = "M" export let hoverable: boolean = false export let disabled: boolean = false - export let color: string = undefined - export let hoverColor: string = undefined - export let tooltip: string = undefined + export let color: string | undefined = undefined + export let hoverColor: string | undefined = undefined + export let tooltip: string | undefined = undefined export let tooltipPosition = TooltipPosition.Bottom export let tooltipType = TooltipType.Default - export let tooltipColor: string = undefined + export let tooltipColor: string | undefined = undefined export let tooltipWrap: boolean = true export let newStyles: boolean = false diff --git a/packages/bbui/src/Tooltip/AbsTooltip.svelte b/packages/bbui/src/Tooltip/AbsTooltip.svelte index b85f4e1c03..a887db4102 100644 --- a/packages/bbui/src/Tooltip/AbsTooltip.svelte +++ b/packages/bbui/src/Tooltip/AbsTooltip.svelte @@ -23,7 +23,7 @@ export let type = TooltipType.Default export let text = "" export let fixed = false - export let color = null + export let color = "" export let noWrap = false let wrapper From fa930de15e8653a8adb5337f27b2297c711531ea Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 09:45:22 +0100 Subject: [PATCH 031/112] Type fields --- .../components/app/blocks/form/FormBlock.svelte | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte index 22f21b5790..ab3c498ca3 100644 --- a/packages/client/src/components/app/blocks/form/FormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte @@ -7,11 +7,13 @@ import { Component, Context, SDK } from "../../../../index" import { TableSchema, UIDatasource } from "@budibase/types" + type Field = { name: string; active: boolean } + export let actionType: string export let dataSource: UIDatasource export let size: string export let disabled: boolean - export let fields + export let fields: (Field | string)[] export let buttons: { "##eventHandlerType": string parameters: Record @@ -66,11 +68,11 @@ } } - const convertOldFieldFormat = (fields: any[]) => { + const convertOldFieldFormat = (fields: (Field | string)[]): Field[] => { if (!fields) { return [] } - return fields.map((field: any) => { + return fields.map(field => { if (typeof field === "string") { // existed but was a string return { @@ -87,14 +89,11 @@ }) } - const getDefaultFields = ( - fields: { name: string; active: boolean }[], - schema: TableSchema - ) => { + const getDefaultFields = (fields: Field[], schema: TableSchema) => { if (!schema) { return [] } - let defaultFields: { name: string; active: boolean }[] = [] + let defaultFields: Field[] = [] if (!fields || fields.length === 0) { Object.values(schema) From 0d1f5c698e529664e1a9ed0e30dce9eb52d0e778 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 09:54:12 +0100 Subject: [PATCH 032/112] Add basic datasource validation error --- packages/client/manifest.json | 3 ++- .../client/src/components/Component.svelte | 24 ++++++++++++++++++- .../error-states/ComponentErrorState.svelte | 4 ++++ .../client/src/utils/componentsValidator.ts | 13 ++++++++++ 4 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 packages/client/src/utils/componentsValidator.ts diff --git a/packages/client/manifest.json b/packages/client/manifest.json index c236dd1ad9..7da7c60abe 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -7206,7 +7206,8 @@ { "type": "table", "label": "Data", - "key": "dataSource" + "key": "dataSource", + "validator": "checkValidDatasource" }, { "type": "radio", diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index 79b4ca6f68..4926965e45 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -40,6 +40,7 @@ getActionDependentContextKeys, } from "../utils/buttonActions.js" import { gridLayout } from "utils/grid" + import { validateComponentSetting } from "utils/componentsValidator" export let instance = {} export let parent = null @@ -103,6 +104,7 @@ let settingsDefinition let settingsDefinitionMap let missingRequiredSettings = false + let invalidSettings = false // Temporary styles which can be added in the app preview for things like // DND. We clear these whenever a new instance is received. @@ -141,12 +143,16 @@ $: showEmptyState = definition?.showEmptyState !== false $: hasMissingRequiredSettings = missingRequiredSettings?.length > 0 $: editable = !!definition?.editable && !hasMissingRequiredSettings + $: hasInvalidSettings = invalidSettings?.length > 0 $: requiredAncestors = definition?.requiredAncestors || [] $: missingRequiredAncestors = requiredAncestors.filter( ancestor => !$component.ancestors.includes(`${BudibasePrefix}${ancestor}`) ) $: hasMissingRequiredAncestors = missingRequiredAncestors?.length > 0 - $: errorState = hasMissingRequiredSettings || hasMissingRequiredAncestors + $: errorState = + hasMissingRequiredSettings || + hasMissingRequiredAncestors || + hasInvalidSettings // Interactive components can be selected, dragged and highlighted inside // the builder preview @@ -338,6 +344,21 @@ return missing }) + // Check for invalid settings + invalidSettings = settingsDefinition.reduce((invalidSettings, setting) => { + if (setting.validator) { + const error = validateComponentSetting( + setting.validator, + instance[setting.key] + ) + if (error) { + invalidSettings.push(error) + } + } + + return invalidSettings + }, []) + // When considering bindings we can ignore children, so we remove that // before storing the reference stringified version const noChildren = JSON.stringify({ ...instance, _children: null }) @@ -692,6 +713,7 @@ {:else} diff --git a/packages/client/src/components/error-states/ComponentErrorState.svelte b/packages/client/src/components/error-states/ComponentErrorState.svelte index b846eaa230..ba55e3cdc1 100644 --- a/packages/client/src/components/error-states/ComponentErrorState.svelte +++ b/packages/client/src/components/error-states/ComponentErrorState.svelte @@ -9,6 +9,7 @@ | { key: string; label: string }[] | undefined export let missingRequiredAncestors: string[] | undefined + export let invalidSettings: string[] | undefined const component = getContext("component") const { styleable, builderStore } = getContext("sdk") @@ -16,6 +17,7 @@ $: styles = { ...$component.styles, normal: {}, custom: null, empty: true } $: requiredSetting = missingRequiredSettings?.[0] $: requiredAncestor = missingRequiredAncestors?.[0] + $: invalidSetting = invalidSettings?.[0] {#if $builderStore.inBuilder} @@ -24,6 +26,8 @@ {#if requiredAncestor} + {:else if invalidSetting} + {invalidSetting} {:else if requiredSetting} {/if} diff --git a/packages/client/src/utils/componentsValidator.ts b/packages/client/src/utils/componentsValidator.ts new file mode 100644 index 0000000000..6609f92ed6 --- /dev/null +++ b/packages/client/src/utils/componentsValidator.ts @@ -0,0 +1,13 @@ +const validators = { + checkValidDatasource: (a: any) => { + return `Ups... "${a.label}" not found` + }, +} + +export function validateComponentSetting( + key: keyof typeof validators, + value: any +) { + const validator = validators[key] + return validator(value) +} From bd5e55480e045886597a0c5d24938756954ec4be Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 11:16:19 +0000 Subject: [PATCH 033/112] Adding more test cases. --- .../string-templates/test/jsLogging.spec.ts | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/packages/string-templates/test/jsLogging.spec.ts b/packages/string-templates/test/jsLogging.spec.ts index 9b2bb945d2..44b3b392ba 100644 --- a/packages/string-templates/test/jsLogging.spec.ts +++ b/packages/string-templates/test/jsLogging.spec.ts @@ -21,13 +21,33 @@ describe("Javascript", () => { console.log("foo"); return "hello"` ) - expect(output.result).toBe("hello") - expect(output.logs[0].log).toBe("hello") - expect(output.logs[0].line).toBe(1) - expect(output.logs[1].log).toBe("world") - expect(output.logs[1].line).toBe(2) - expect(output.logs[2].log).toBe("foo") - expect(output.logs[2].line).toBe(3) + expect(output.result).toEqual("hello") + expect(output.logs[0].log).toEqual(["hello"]) + expect(output.logs[0].line).toEqual(1) + expect(output.logs[1].log).toEqual(["world"]) + expect(output.logs[1].line).toEqual(2) + expect(output.logs[2].log).toEqual(["foo"]) + expect(output.logs[2].line).toEqual(3) }) }) + + it("should log comma separated values", () => { + const output = processJS(`console.log(1, { a: 1 }); return 1`) + expect(output.logs[0].log).toEqual([1, JSON.stringify({ a: 1 })]) + expect(output.logs[0].line).toEqual(1) + }) + + it("should return the type working with warn", () => { + const output = processJS(`console.warn("warning"); return 1`) + expect(output.logs[0].log).toEqual(["warning"]) + expect(output.logs[0].line).toEqual(1) + expect(output.logs[0].type).toEqual("warn") + }) + + it("should return the type working with error", () => { + const output = processJS(`console.error("error"); return 1`) + expect(output.logs[0].log).toEqual(["error"]) + expect(output.logs[0].line).toEqual(1) + expect(output.logs[0].type).toEqual("error") + }) }) From 0b73f53974c301667d4f2972d4b3392629e2d64b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 14:51:37 +0100 Subject: [PATCH 034/112] Automatically type getContext response --- packages/client/src/components/app/DataProvider.svelte | 5 ++--- .../client/src/components/app/blocks/form/FormBlock.svelte | 7 +++---- .../src/components/error-states/ComponentErrorState.svelte | 5 ++--- packages/client/src/context.d.ts | 7 +++++++ 4 files changed, 14 insertions(+), 10 deletions(-) create mode 100644 packages/client/src/context.d.ts diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte index 154f69d4de..a80b9a5f74 100644 --- a/packages/client/src/components/app/DataProvider.svelte +++ b/packages/client/src/components/app/DataProvider.svelte @@ -14,7 +14,6 @@ GroupUserDatasource, DataFetchOptions, } from "@budibase/types" - import { SDK, Component } from "../../index" type ProviderDatasource = Exclude< DataFetchDatasource, @@ -29,8 +28,8 @@ export let paginate: boolean export let autoRefresh: number - const { styleable, Provider, ActionTypes, API } = getContext("sdk") - const component = getContext("component") + const { styleable, Provider, ActionTypes, API } = getContext("sdk") + const component = getContext("component") let interval: ReturnType let queryExtensions: Record = {} diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte index ab3c498ca3..3f44aee1d7 100644 --- a/packages/client/src/components/app/blocks/form/FormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte @@ -4,7 +4,6 @@ import { Utils } from "@budibase/frontend-core" import FormBlockWrapper from "./FormBlockWrapper.svelte" import { get } from "svelte/store" - import { Component, Context, SDK } from "../../../../index" import { TableSchema, UIDatasource } from "@budibase/types" type Field = { name: string; active: boolean } @@ -34,9 +33,9 @@ export let saveButtonLabel: boolean export let deleteButtonLabel: boolean - const { fetchDatasourceSchema, generateGoldenSample } = getContext("sdk") - const component = getContext("component") - const context = getContext("context") + const { fetchDatasourceSchema, generateGoldenSample } = getContext("sdk") + const component = getContext("component") + const context = getContext("context") let schema: TableSchema diff --git a/packages/client/src/components/error-states/ComponentErrorState.svelte b/packages/client/src/components/error-states/ComponentErrorState.svelte index b846eaa230..d30f4916da 100644 --- a/packages/client/src/components/error-states/ComponentErrorState.svelte +++ b/packages/client/src/components/error-states/ComponentErrorState.svelte @@ -3,15 +3,14 @@ import { Icon } from "@budibase/bbui" import MissingRequiredSetting from "./MissingRequiredSetting.svelte" import MissingRequiredAncestor from "./MissingRequiredAncestor.svelte" - import { Component, SDK } from "../../index" export let missingRequiredSettings: | { key: string; label: string }[] | undefined export let missingRequiredAncestors: string[] | undefined - const component = getContext("component") - const { styleable, builderStore } = getContext("sdk") + const component = getContext("component") + const { styleable, builderStore } = getContext("sdk") $: styles = { ...$component.styles, normal: {}, custom: null, empty: true } $: requiredSetting = missingRequiredSettings?.[0] diff --git a/packages/client/src/context.d.ts b/packages/client/src/context.d.ts new file mode 100644 index 0000000000..1bc0e3f7ad --- /dev/null +++ b/packages/client/src/context.d.ts @@ -0,0 +1,7 @@ +import { Component, Context, SDK } from "." + +declare module "svelte" { + function getContext(name: "sdk"): SDK + function getContext(name: "component"): Component + function getContext(name: "context"): Context +} From cbabaaf8f7c4b52d7fc5d8e21fa07ce5867b1134 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 15:09:02 +0100 Subject: [PATCH 035/112] Move types --- packages/client/src/context.d.ts | 7 ++++--- packages/client/src/index.ts | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/client/src/context.d.ts b/packages/client/src/context.d.ts index 1bc0e3f7ad..6ac29df547 100644 --- a/packages/client/src/context.d.ts +++ b/packages/client/src/context.d.ts @@ -1,7 +1,8 @@ +import { Readable } from "svelte/store" import { Component, Context, SDK } from "." declare module "svelte" { - function getContext(name: "sdk"): SDK - function getContext(name: "component"): Component - function getContext(name: "context"): Context + export function getContext(key: "sdk"): SDK + export function getContext(key: "component"): Readable + export function getContext(key: "context"): Readable } diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index ada30d53ec..dabe66dab2 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -14,10 +14,10 @@ export interface SDK { }> } -export type Component = Readable<{ +export type Component = { id: string styles: any errorState: boolean -}> +} -export type Context = Readable +export type Context = {} From 199aa80e25433fe5ec103ef449ce29346138289e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 15:10:03 +0100 Subject: [PATCH 036/112] Move types --- packages/client/src/context.d.ts | 5 ++--- packages/client/src/index.ts | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/client/src/context.d.ts b/packages/client/src/context.d.ts index 6ac29df547..994f9ef172 100644 --- a/packages/client/src/context.d.ts +++ b/packages/client/src/context.d.ts @@ -1,8 +1,7 @@ -import { Readable } from "svelte/store" import { Component, Context, SDK } from "." declare module "svelte" { export function getContext(key: "sdk"): SDK - export function getContext(key: "component"): Readable - export function getContext(key: "context"): Readable + export function getContext(key: "component"): Component + export function getContext(key: "context"): Context } diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index dabe66dab2..2a435c2f8c 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -14,10 +14,10 @@ export interface SDK { }> } -export type Component = { +export type Component = Readable<{ id: string styles: any errorState: boolean -} +}> -export type Context = {} +export type Context = Readable<{}> From 0b1ff21da5f52b5e0a38836ec83df8b58dba814e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 Jan 2025 14:44:06 +0000 Subject: [PATCH 037/112] Fix tests. --- packages/server/src/automations/actions.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index f35f96babe..bdbd9d509d 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -88,12 +88,10 @@ export const BUILTIN_ACTION_DEFINITIONS: Record< // the fact this isn't included in any definitions means it cannot be // ran at all if (env.SELF_HOSTED) { - const bash = require("./steps/bash") - // @ts-ignore - ACTION_IMPLS["EXECUTE_BASH"] = bash.run + ACTION_IMPLS["EXECUTE_BASH"] = automations.steps.bash.run // @ts-ignore - BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition + BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = automations.steps.bash.definition if (env.isTest()) { BUILTIN_ACTION_DEFINITIONS["OPENAI"] = automations.steps.openai.definition From ba2a61841f292276169bc3c4decc659a58c98f61 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 15:55:00 +0100 Subject: [PATCH 038/112] Allow importing @budibase/client/manifest.json --- eslint-local-rules/index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eslint-local-rules/index.js b/eslint-local-rules/index.js index d9d894c33e..9348706399 100644 --- a/eslint-local-rules/index.js +++ b/eslint-local-rules/index.js @@ -41,11 +41,12 @@ module.exports = { if ( /^@budibase\/[^/]+\/.*$/.test(importPath) && importPath !== "@budibase/backend-core/tests" && - importPath !== "@budibase/string-templates/test/utils" + importPath !== "@budibase/string-templates/test/utils" && + importPath !== "@budibase/client/manifest.json" ) { context.report({ node, - message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests and @budibase/string-templates/test/utils.`, + message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests, @budibase/string-templates/test/utils and @budibase/client/manifest.json.`, }) } }, From 7e029386190964097cdf1582f91568b7a694e7ef Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 15:55:17 +0100 Subject: [PATCH 039/112] Screen helpers, findComponentsBySettingsType --- packages/builder/src/helpers/screen.ts | 30 ++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 packages/builder/src/helpers/screen.ts diff --git a/packages/builder/src/helpers/screen.ts b/packages/builder/src/helpers/screen.ts new file mode 100644 index 0000000000..e83805a511 --- /dev/null +++ b/packages/builder/src/helpers/screen.ts @@ -0,0 +1,30 @@ +import { Component, Screen, ScreenProps } from "@budibase/types" +import clientManifest from "@budibase/client/manifest.json" + +export function findComponentsBySettingsType(screen: Screen, type: string) { + const result: Component[] = [] + function recurseFieldComponentsInChildren( + component: ScreenProps, + type: string + ) { + if (!component) { + return + } + + const componentType = component._component.split("/").slice(-1)[0] + const definition = + clientManifest[componentType as keyof typeof clientManifest] + if ( + "settings" in definition && + definition.settings.some((s: any) => s.type === type) + ) { + result.push(component) + } + component._children?.forEach(child => { + recurseFieldComponentsInChildren(child, type) + }) + } + + recurseFieldComponentsInChildren(screen?.props, type) + return result +} From 3c4ac9ad5a7610645b83c8b8f3cdeb3b8c52c70a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 16:04:39 +0100 Subject: [PATCH 040/112] Errors from builder store --- packages/client/manifest.json | 3 +-- .../client/src/components/Component.svelte | 20 ++++--------------- packages/client/src/stores/builder.js | 3 +++ .../src/stores/derived/componentErrors.ts | 6 ++++++ packages/client/src/stores/derived/index.js | 1 + .../client/src/utils/componentsValidator.ts | 13 ------------ 6 files changed, 15 insertions(+), 31 deletions(-) create mode 100644 packages/client/src/stores/derived/componentErrors.ts delete mode 100644 packages/client/src/utils/componentsValidator.ts diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 7da7c60abe..c236dd1ad9 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -7206,8 +7206,7 @@ { "type": "table", "label": "Data", - "key": "dataSource", - "validator": "checkValidDatasource" + "key": "dataSource" }, { "type": "radio", diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index 4926965e45..7f825c0601 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -23,6 +23,7 @@ appStore, dndComponentPath, dndIsDragging, + componentErrors, } from "stores" import { Helpers } from "@budibase/bbui" import { getActiveConditions, reduceConditionActions } from "utils/conditions" @@ -40,7 +41,6 @@ getActionDependentContextKeys, } from "../utils/buttonActions.js" import { gridLayout } from "utils/grid" - import { validateComponentSetting } from "utils/componentsValidator" export let instance = {} export let parent = null @@ -344,21 +344,6 @@ return missing }) - // Check for invalid settings - invalidSettings = settingsDefinition.reduce((invalidSettings, setting) => { - if (setting.validator) { - const error = validateComponentSetting( - setting.validator, - instance[setting.key] - ) - if (error) { - invalidSettings.push(error) - } - } - - return invalidSettings - }, []) - // When considering bindings we can ignore children, so we remove that // before storing the reference stringified version const noChildren = JSON.stringify({ ...instance, _children: null }) @@ -389,6 +374,9 @@ } } + // Check for invalid settings + $: invalidSettings = $componentErrors[id] + // Extracts a map of all context keys which are required by action settings // to provide the functions to evaluate at runtime. This needs done manually // as the action definitions themselves do not specify bindings for action diff --git a/packages/client/src/stores/builder.js b/packages/client/src/stores/builder.js index faa37eddca..62123c07e5 100644 --- a/packages/client/src/stores/builder.js +++ b/packages/client/src/stores/builder.js @@ -19,6 +19,9 @@ const createBuilderStore = () => { eventResolvers: {}, metadata: null, snippets: null, + componentErrors: { + c5ea93132725c48b2a365fcc1facaee86: ["Ups...!"], + }, // TODO // Legacy - allow the builder to specify a layout layout: null, diff --git a/packages/client/src/stores/derived/componentErrors.ts b/packages/client/src/stores/derived/componentErrors.ts new file mode 100644 index 0000000000..48185de9c3 --- /dev/null +++ b/packages/client/src/stores/derived/componentErrors.ts @@ -0,0 +1,6 @@ +import { derived } from "svelte/store" +import { builderStore } from "../builder.js" + +export const componentErrors = derived([builderStore], ([$builderStore]) => { + return $builderStore.componentErrors as Record +}) diff --git a/packages/client/src/stores/derived/index.js b/packages/client/src/stores/derived/index.js index 337c73831f..e7e70d8952 100644 --- a/packages/client/src/stores/derived/index.js +++ b/packages/client/src/stores/derived/index.js @@ -5,3 +5,4 @@ export { currentRole } from "./currentRole.js" export { dndComponentPath } from "./dndComponentPath.js" export { devToolsEnabled } from "./devToolsEnabled.js" export { snippets } from "./snippets.js" +export { componentErrors } from "./componentErrors" diff --git a/packages/client/src/utils/componentsValidator.ts b/packages/client/src/utils/componentsValidator.ts deleted file mode 100644 index 6609f92ed6..0000000000 --- a/packages/client/src/utils/componentsValidator.ts +++ /dev/null @@ -1,13 +0,0 @@ -const validators = { - checkValidDatasource: (a: any) => { - return `Ups... "${a.label}" not found` - }, -} - -export function validateComponentSetting( - key: keyof typeof validators, - value: any -) { - const validator = validators[key] - return validator(value) -} From 75dac8f3343540eb8d0e6e843e8fed909be931a1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 Jan 2025 15:08:50 +0000 Subject: [PATCH 041/112] Fix tests (again). --- packages/server/src/automations/actions.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index bdbd9d509d..65a57c2586 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -18,6 +18,7 @@ import * as queryRow from "./steps/queryRows" import * as collect from "./steps/collect" import * as triggerAutomationRun from "./steps/triggerAutomationRun" import * as openai from "./steps/openai" +import * as bash from "./steps/bash" import env from "../environment" import { PluginType, @@ -88,9 +89,8 @@ export const BUILTIN_ACTION_DEFINITIONS: Record< // the fact this isn't included in any definitions means it cannot be // ran at all if (env.SELF_HOSTED) { - // @ts-ignore - ACTION_IMPLS["EXECUTE_BASH"] = automations.steps.bash.run - // @ts-ignore + // @ts-expect-error + ACTION_IMPLS["EXECUTE_BASH"] = bash.run BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = automations.steps.bash.definition if (env.isTest()) { From 3b03515253bd6f5203b65b67be2d9eb6ebd91111 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 15:23:00 +0000 Subject: [PATCH 042/112] Fixing test failure. --- packages/string-templates/src/processors/index.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/string-templates/src/processors/index.ts b/packages/string-templates/src/processors/index.ts index 4454d02738..0916c791cd 100644 --- a/packages/string-templates/src/processors/index.ts +++ b/packages/string-templates/src/processors/index.ts @@ -24,8 +24,10 @@ function process( } for (let match of matches) { const res = processor.process(output, match, opts || {}) - if (typeof res === "object" && "logs" in res && res.logs) { - logs = logs.concat(res.logs) + if (typeof res === "object") { + if ("logs" in res && res.logs) { + logs = logs.concat(res.logs) + } output = res.result } else { output = res as string From a920be3207f850c71a52758b7d675e86a2dc7643 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 15:33:06 +0000 Subject: [PATCH 043/112] Remove error. --- packages/string-templates/test/jsLogging.spec.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/string-templates/test/jsLogging.spec.ts b/packages/string-templates/test/jsLogging.spec.ts index 44b3b392ba..f328b76c7c 100644 --- a/packages/string-templates/test/jsLogging.spec.ts +++ b/packages/string-templates/test/jsLogging.spec.ts @@ -43,11 +43,4 @@ describe("Javascript", () => { expect(output.logs[0].line).toEqual(1) expect(output.logs[0].type).toEqual("warn") }) - - it("should return the type working with error", () => { - const output = processJS(`console.error("error"); return 1`) - expect(output.logs[0].log).toEqual(["error"]) - expect(output.logs[0].line).toEqual(1) - expect(output.logs[0].type).toEqual("error") - }) }) From ae73c0147ffc481206a4e7bc29e0d942cbba511e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 15:41:08 +0000 Subject: [PATCH 044/112] Adding test checks. --- packages/string-templates/src/environment.ts | 11 +++++++++++ packages/string-templates/src/helpers/javascript.ts | 5 ++++- packages/string-templates/test/jsLogging.spec.ts | 7 +++++++ 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 packages/string-templates/src/environment.ts diff --git a/packages/string-templates/src/environment.ts b/packages/string-templates/src/environment.ts new file mode 100644 index 0000000000..ede52591b1 --- /dev/null +++ b/packages/string-templates/src/environment.ts @@ -0,0 +1,11 @@ +function isJest() { + return ( + process.env.NODE_ENV === "jest" || + (process.env.JEST_WORKER_ID != null && + process.env.JEST_WORKER_ID !== "null") + ) +} + +export function isTest() { + return isJest() +} diff --git a/packages/string-templates/src/helpers/javascript.ts b/packages/string-templates/src/helpers/javascript.ts index 997ca7b6ec..91f2f9a0ce 100644 --- a/packages/string-templates/src/helpers/javascript.ts +++ b/packages/string-templates/src/helpers/javascript.ts @@ -10,6 +10,7 @@ import { iifeWrapper } from "../iife" import { JsTimeoutError, UserScriptError } from "../errors" import { cloneDeep } from "lodash/fp" import { Log, LogType } from "../types" +import { isTest } from "../environment" // The method of executing JS scripts depends on the bundle being built. // This setter is used in the entrypoint (either index.js or index.mjs). @@ -126,7 +127,9 @@ export function processJS(handlebars: string, context: any) { const jsLineCount = frontendWrapJS(js).split(js)[0].split("\n").length const buildLogResponse = (type: LogType) => { return (...props: any[]) => { - console[type](...props) + if (!isTest()) { + console[type](...props) + } props.forEach((prop, index) => { if (typeof prop === "object") { props[index] = JSON.stringify(prop) diff --git a/packages/string-templates/test/jsLogging.spec.ts b/packages/string-templates/test/jsLogging.spec.ts index f328b76c7c..44b3b392ba 100644 --- a/packages/string-templates/test/jsLogging.spec.ts +++ b/packages/string-templates/test/jsLogging.spec.ts @@ -43,4 +43,11 @@ describe("Javascript", () => { expect(output.logs[0].line).toEqual(1) expect(output.logs[0].type).toEqual("warn") }) + + it("should return the type working with error", () => { + const output = processJS(`console.error("error"); return 1`) + expect(output.logs[0].log).toEqual(["error"]) + expect(output.logs[0].line).toEqual(1) + expect(output.logs[0].type).toEqual("error") + }) }) From 9c65f1ab41cfdb2e98282ed52d1af7e224152865 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 15:58:11 +0000 Subject: [PATCH 045/112] Another quick fix. --- packages/string-templates/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index 67ccde727e..b2097b0d4c 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -240,7 +240,7 @@ function processStringSyncInternal( string = string.replace(block, outcome) } } - return opts?.logging ? string : { result: string, logs } + return !opts?.logging ? string : { result: string, logs } } else { return process(string) } From 98bd824d7ae01e59df1141ed2f68c8b7b3e233b0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 16:33:21 +0000 Subject: [PATCH 046/112] Adding the ability to configure whether or not string templates is testing backend JS or frontend. --- packages/server/src/api/routes/tests/row.spec.ts | 2 ++ packages/string-templates/src/environment.ts | 12 ++++++++++++ packages/string-templates/src/index.ts | 1 + packages/string-templates/src/utilities.ts | 11 ++++++++--- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 576f0bb663..2a145e1ed9 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -50,9 +50,11 @@ import { JsTimeoutError } from "@budibase/string-templates" import { isDate } from "../../../utilities" import nock from "nock" import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai" +import { setTestingBackendJS } from "@budibase/string-templates" const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString() tk.freeze(timestamp) +setTestingBackendJS() interface WaitOptions { name: string matchFn?: (event: any) => boolean diff --git a/packages/string-templates/src/environment.ts b/packages/string-templates/src/environment.ts index ede52591b1..6bee6fd3a9 100644 --- a/packages/string-templates/src/environment.ts +++ b/packages/string-templates/src/environment.ts @@ -9,3 +9,15 @@ function isJest() { export function isTest() { return isJest() } + +export const isJSAllowed = () => { + return process && !process.env.NO_JS +} + +export const isTestingBackendJS = () => { + return process && process.env.BACKEND_JS +} + +export const setTestingBackendJS = () => { + process.env.BACKEND_JS = "1" +} diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index b2097b0d4c..8dda8b71ab 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -20,6 +20,7 @@ import { Log, ProcessOptions } from "./types" import { UserScriptError } from "./errors" export type { Log, LogType } from "./types" +export { setTestingBackendJS } from "./environment" export { helpersToRemoveForJs, getJsHelperList } from "./helpers/list" export { FIND_ANY_HBS_REGEX } from "./utilities" export { setJSRunner, setOnErrorLog } from "./helpers/javascript" diff --git a/packages/string-templates/src/utilities.ts b/packages/string-templates/src/utilities.ts index dba1faab17..b05945f075 100644 --- a/packages/string-templates/src/utilities.ts +++ b/packages/string-templates/src/utilities.ts @@ -1,15 +1,20 @@ +import { isTest, isTestingBackendJS } from "./environment" + const ALPHA_NUMERIC_REGEX = /^[A-Za-z0-9]+$/g export const FIND_HBS_REGEX = /{{([^{].*?)}}/g export const FIND_ANY_HBS_REGEX = /{?{{([^{].*?)}}}?/g export const FIND_TRIPLE_HBS_REGEX = /{{{([^{].*?)}}}/g -const isJest = () => typeof jest !== "undefined" - export const isBackendService = () => { + // allow configuring backend JS mode when testing - we default to assuming + // frontend, but need a method to control this + if (isTest() && isTestingBackendJS()) { + return true + } // We consider the tests for string-templates to be frontend, so that they // test the frontend JS functionality. - if (isJest()) { + if (isTest()) { return false } return typeof window === "undefined" From 68374bce29126c0f38da9f030acfdc82c2ed8e60 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 16:40:08 +0000 Subject: [PATCH 047/112] Testing backend JS further. --- packages/server/src/jsRunner/tests/jsRunner.spec.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/server/src/jsRunner/tests/jsRunner.spec.ts b/packages/server/src/jsRunner/tests/jsRunner.spec.ts index 006df19fa6..e10e9c4d43 100644 --- a/packages/server/src/jsRunner/tests/jsRunner.spec.ts +++ b/packages/server/src/jsRunner/tests/jsRunner.spec.ts @@ -1,5 +1,9 @@ import { validate as isValidUUID } from "uuid" -import { processStringSync, encodeJSBinding } from "@budibase/string-templates" +import { + processStringSync, + encodeJSBinding, + setTestingBackendJS, +} from "@budibase/string-templates" import { runJsHelpersTests } from "@budibase/string-templates/test/utils" @@ -7,6 +11,7 @@ import tk from "timekeeper" import { init } from ".." import TestConfiguration from "../../tests/utilities/TestConfiguration" +setTestingBackendJS() const DATE = "2021-01-21T12:00:00" tk.freeze(DATE) From 04a7878ce9ad4d6ea5656ae6409eb6cbbb68bb53 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 17:02:57 +0000 Subject: [PATCH 048/112] Changing how we enforce backend JS. --- packages/server/src/api/routes/tests/row.spec.ts | 2 -- .../automations/tests/utilities/AutomationTestBuilder.ts | 1 + packages/server/src/jsRunner/index.ts | 5 +++++ packages/server/src/jsRunner/tests/jsRunner.spec.ts | 7 +------ packages/string-templates/src/helpers/javascript.ts | 2 +- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 2a145e1ed9..576f0bb663 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -50,11 +50,9 @@ import { JsTimeoutError } from "@budibase/string-templates" import { isDate } from "../../../utilities" import nock from "nock" import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai" -import { setTestingBackendJS } from "@budibase/string-templates" const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString() tk.freeze(timestamp) -setTestingBackendJS() interface WaitOptions { name: string matchFn?: (event: any) => boolean diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index 830d2ee5ca..f89c815752 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -43,6 +43,7 @@ import { import TestConfiguration from "../../../tests/utilities/TestConfiguration" import * as setup from "../utilities" import { automations } from "@budibase/shared-core" +import { setTestingBackendJS } from "@budibase/string-templates" type TriggerOutputs = | RowCreatedTriggerOutputs diff --git a/packages/server/src/jsRunner/index.ts b/packages/server/src/jsRunner/index.ts index e17529a687..a29e952b6d 100644 --- a/packages/server/src/jsRunner/index.ts +++ b/packages/server/src/jsRunner/index.ts @@ -4,12 +4,17 @@ import { JsTimeoutError, setJSRunner, setOnErrorLog, + setTestingBackendJS, } from "@budibase/string-templates" import { context, logging } from "@budibase/backend-core" import tracer from "dd-trace" import { IsolatedVM } from "./vm" export function init() { + // enforce that if we're using isolated-VM runner then we are running backend JS + if (env.isTest()) { + setTestingBackendJS() + } setJSRunner((js: string, ctx: Record) => { return tracer.trace("runJS", {}, () => { try { diff --git a/packages/server/src/jsRunner/tests/jsRunner.spec.ts b/packages/server/src/jsRunner/tests/jsRunner.spec.ts index e10e9c4d43..006df19fa6 100644 --- a/packages/server/src/jsRunner/tests/jsRunner.spec.ts +++ b/packages/server/src/jsRunner/tests/jsRunner.spec.ts @@ -1,9 +1,5 @@ import { validate as isValidUUID } from "uuid" -import { - processStringSync, - encodeJSBinding, - setTestingBackendJS, -} from "@budibase/string-templates" +import { processStringSync, encodeJSBinding } from "@budibase/string-templates" import { runJsHelpersTests } from "@budibase/string-templates/test/utils" @@ -11,7 +7,6 @@ import tk from "timekeeper" import { init } from ".." import TestConfiguration from "../../tests/utilities/TestConfiguration" -setTestingBackendJS() const DATE = "2021-01-21T12:00:00" tk.freeze(DATE) diff --git a/packages/string-templates/src/helpers/javascript.ts b/packages/string-templates/src/helpers/javascript.ts index 91f2f9a0ce..6132adf892 100644 --- a/packages/string-templates/src/helpers/javascript.ts +++ b/packages/string-templates/src/helpers/javascript.ts @@ -88,7 +88,7 @@ export function processJS(handlebars: string, context: any) { let clonedContext: Record if (isBackendService()) { - // On the backned, values are copied across the isolated-vm boundary and + // On the backend, values are copied across the isolated-vm boundary and // so we don't need to do any cloning here. This does create a fundamental // difference in how JS executes on the frontend vs the backend, e.g. // consider this snippet: From d51491a19adc19b70fe80532d5bee7bad07c4f23 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 17:06:31 +0000 Subject: [PATCH 049/112] Linting. --- .../src/automations/tests/utilities/AutomationTestBuilder.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index f89c815752..830d2ee5ca 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -43,7 +43,6 @@ import { import TestConfiguration from "../../../tests/utilities/TestConfiguration" import * as setup from "../utilities" import { automations } from "@budibase/shared-core" -import { setTestingBackendJS } from "@budibase/string-templates" type TriggerOutputs = | RowCreatedTriggerOutputs From 5bc316916ff6c9525f5882fdd11b89eb73d98cd8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 Jan 2025 17:18:29 +0000 Subject: [PATCH 050/112] First iteration of single-step automation test endpoint. --- .../server/src/api/controllers/automation.ts | 91 +++++++++++++++---- packages/server/src/api/routes/automation.ts | 12 ++- packages/server/src/events/NoopEmitter.ts | 39 ++++++++ packages/server/src/events/index.ts | 1 + packages/server/src/threads/automation.ts | 36 +++++--- packages/server/src/utilities/index.ts | 41 ++------- packages/server/src/utilities/redis.ts | 17 ++-- packages/types/src/api/web/app/automation.ts | 7 ++ .../documents/app/automation/automation.ts | 4 +- 9 files changed, 172 insertions(+), 76 deletions(-) create mode 100644 packages/server/src/events/NoopEmitter.ts diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index abc0e492c0..a77014cf31 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -2,7 +2,7 @@ import * as triggers from "../../automations/triggers" import { sdk as coreSdk } from "@budibase/shared-core" import { DocumentType } from "../../db/utils" import { updateTestHistory, removeDeprecated } from "../../automations/utils" -import { setTestFlag, clearTestFlag } from "../../utilities/redis" +import { withTestFlag } from "../../utilities/redis" import { context, cache, events, db as dbCore } from "@budibase/backend-core" import { automations, features } from "@budibase/pro" import { @@ -28,11 +28,18 @@ import { TriggerAutomationResponse, TestAutomationRequest, TestAutomationResponse, + TestAutomationStepRequest, + TestAutomationStepResponse, } from "@budibase/types" -import { getActionDefinitions as actionDefs } from "../../automations/actions" +import { + getActionDefinitions as actionDefs, + getAction, +} from "../../automations/actions" import sdk from "../../sdk" import { builderSocket } from "../../websockets" import env from "../../environment" +import { NoopEmitter } from "../../events" +import { enrichBaseContext } from "../../threads/automation" async function getActionDefinitions() { return removeDeprecated(await actionDefs()) @@ -231,24 +238,68 @@ export async function test( ctx: UserCtx ) { const db = context.getAppDB() - let automation = await db.get(ctx.params.id) - await setTestFlag(automation._id!) - const testInput = prepareTestInput(ctx.request.body) - const response = await triggers.externalTrigger( - automation, - { - ...testInput, - appId: ctx.appId, - user: sdk.users.getUserContextBindings(ctx.user), - }, - { getResponses: true } - ) - // save a test history run - await updateTestHistory(ctx.appId, automation, { - ...ctx.request.body, - occurredAt: new Date().getTime(), + const automation = await db.tryGet(ctx.params.id) + if (!automation) { + ctx.throw(404, `Automation ${ctx.params.id} not found`) + } + + const { request, appId } = ctx + const { body } = request + + ctx.body = await withTestFlag(automation._id!, async () => { + const occurredAt = new Date().getTime() + await updateTestHistory(appId, automation, { ...body, occurredAt }) + + const user = sdk.users.getUserContextBindings(ctx.user) + return await triggers.externalTrigger( + automation, + { ...prepareTestInput(body), appId, user }, + { getResponses: true } + ) }) - await clearTestFlag(automation._id!) - ctx.body = response + await events.automation.tested(automation) } + +export async function testStep( + ctx: UserCtx +) { + const { id, stepId } = ctx.params + const db = context.getAppDB() + const automation = await db.tryGet(id) + if (!automation) { + ctx.throw(404, `Automation ${ctx.params.id} not found`) + } + + const step = automation.definition.steps.find(s => s.stepId === stepId) + if (!step) { + ctx.throw(404, `Step ${stepId} not found on automation ${id}`) + } + + if (step.stepId === AutomationActionStepId.BRANCH) { + ctx.throw(400, "Branch steps cannot be tested directly") + } + if (step.stepId === AutomationActionStepId.LOOP) { + ctx.throw(400, "Loop steps cannot be tested directly") + } + + const { body } = ctx.request + + const fn = await getAction(step.stepId) + if (!fn) { + ctx.throw(400, `Step ${stepId} is not a valid step`) + } + + const output = await withTestFlag( + automation._id!, + async () => + await fn({ + inputs: body.inputs, + context: await enrichBaseContext(body.context), + appId: ctx.appId, + emitter: new NoopEmitter(), + }) + ) + + ctx.body = output +} diff --git a/packages/server/src/api/routes/automation.ts b/packages/server/src/api/routes/automation.ts index 489487271c..ea905be0cd 100644 --- a/packages/server/src/api/routes/automation.ts +++ b/packages/server/src/api/routes/automation.ts @@ -1,6 +1,6 @@ import Router from "@koa/router" import * as controller from "../controllers/automation" -import authorized from "../../middleware/authorized" +import authorized, { authorizedResource } from "../../middleware/authorized" import { permissions } from "@budibase/backend-core" import { bodyResource, paramResource } from "../../middleware/resourceId" import { @@ -82,5 +82,15 @@ router ), controller.test ) + .post( + "/api/automations/:id/step/:stepId/test", + appInfoMiddleware({ appType: AppType.DEV }), + authorizedResource( + permissions.PermissionType.AUTOMATION, + permissions.PermissionLevel.EXECUTE, + "id" + ), + controller.testStep + ) export default router diff --git a/packages/server/src/events/NoopEmitter.ts b/packages/server/src/events/NoopEmitter.ts new file mode 100644 index 0000000000..ed87618ead --- /dev/null +++ b/packages/server/src/events/NoopEmitter.ts @@ -0,0 +1,39 @@ +import { EventEmitter } from "events" +import { + Table, + Row, + ContextEmitter, + EventType, + UserBindings, +} from "@budibase/types" + +export class NoopEmitter extends EventEmitter implements ContextEmitter { + emitRow(values: { + eventName: EventType.ROW_SAVE + appId: string + row: Row + table: Table + user: UserBindings + }): void + emitRow(values: { + eventName: EventType.ROW_UPDATE + appId: string + row: Row + table: Table + oldRow: Row + user: UserBindings + }): void + emitRow(values: { + eventName: EventType.ROW_DELETE + appId: string + row: Row + user: UserBindings + }): void + emitRow(_values: unknown): void { + return + } + + emitTable(_eventName: string, _appId: string, _table?: Table) { + return + } +} diff --git a/packages/server/src/events/index.ts b/packages/server/src/events/index.ts index 23c3f3e512..90bf932bcf 100644 --- a/packages/server/src/events/index.ts +++ b/packages/server/src/events/index.ts @@ -2,5 +2,6 @@ import BudibaseEmitter from "./BudibaseEmitter" const emitter = new BudibaseEmitter() +export { NoopEmitter } from "./NoopEmitter" export { init } from "./docUpdates" export default emitter diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 2d10f5d1fb..2790d8fda6 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -29,6 +29,7 @@ import { LoopStep, UserBindings, isBasicSearchOperator, + ContextEmitter, } from "@budibase/types" import { AutomationContext, @@ -71,6 +72,24 @@ function getLoopIterations(loopStep: LoopStep) { return 0 } +export async function enrichBaseContext(context: Record) { + context.env = await sdkUtils.getEnvironmentVariables() + + try { + const { config } = await configs.getSettingsConfigDoc() + context.settings = { + url: config.platformUrl, + logo: config.logoUrl, + company: config.company, + } + } catch (e) { + // if settings doc doesn't exist, make the settings blank + context.settings = {} + } + + return context +} + /** * The automation orchestrator is a class responsible for executing automations. * It handles the context of the automation and makes sure each step gets the correct @@ -80,7 +99,7 @@ class Orchestrator { private chainCount: number private appId: string private automation: Automation - private emitter: any + private emitter: ContextEmitter private context: AutomationContext private job: Job private loopStepOutputs: LoopStep[] @@ -270,20 +289,9 @@ class Orchestrator { appId: this.appId, automationId: this.automation._id, }) - this.context.env = await sdkUtils.getEnvironmentVariables() - this.context.user = this.currentUser - try { - const { config } = await configs.getSettingsConfigDoc() - this.context.settings = { - url: config.platformUrl, - logo: config.logoUrl, - company: config.company, - } - } catch (e) { - // if settings doc doesn't exist, make the settings blank - this.context.settings = {} - } + await enrichBaseContext(this.context) + this.context.user = this.currentUser let metadata diff --git a/packages/server/src/utilities/index.ts b/packages/server/src/utilities/index.ts index db57b4ec12..f1b32c81f3 100644 --- a/packages/server/src/utilities/index.ts +++ b/packages/server/src/utilities/index.ts @@ -58,30 +58,14 @@ export function checkSlashesInUrl(url: string) { export async function updateEntityMetadata( type: string, entityId: string, - updateFn: any + updateFn: (metadata: Document) => Document ) { const db = context.getAppDB() const id = generateMetadataID(type, entityId) - // read it to see if it exists, we'll overwrite it no matter what - let rev, metadata: Document - try { - const oldMetadata = await db.get(id) - rev = oldMetadata._rev - metadata = updateFn(oldMetadata) - } catch (err) { - rev = null - metadata = updateFn({}) - } + const metadata = updateFn((await db.tryGet(id)) || {}) metadata._id = id - if (rev) { - metadata._rev = rev - } const response = await db.put(metadata) - return { - ...metadata, - _id: id, - _rev: response.rev, - } + return { ...metadata, _id: id, _rev: response.rev } } export async function saveEntityMetadata( @@ -89,26 +73,17 @@ export async function saveEntityMetadata( entityId: string, metadata: Document ): Promise { - return updateEntityMetadata(type, entityId, () => { - return metadata - }) + return updateEntityMetadata(type, entityId, () => metadata) } export async function deleteEntityMetadata(type: string, entityId: string) { const db = context.getAppDB() const id = generateMetadataID(type, entityId) - let rev - try { - const metadata = await db.get(id) - if (metadata) { - rev = metadata._rev - } - } catch (err) { - // don't need to error if it doesn't exist - } - if (id && rev) { - await db.remove(id, rev) + const metadata = await db.tryGet(id) + if (!metadata) { + return } + await db.remove(metadata) } export function escapeDangerousCharacters(string: string) { diff --git a/packages/server/src/utilities/redis.ts b/packages/server/src/utilities/redis.ts index a4154b7b95..a3ce655316 100644 --- a/packages/server/src/utilities/redis.ts +++ b/packages/server/src/utilities/redis.ts @@ -89,17 +89,22 @@ export async function setDebounce(id: string, seconds: number) { await debounceClient.store(id, "debouncing", seconds) } -export async function setTestFlag(id: string) { - await flagClient.store(id, { testing: true }, AUTOMATION_TEST_FLAG_SECONDS) -} - export async function checkTestFlag(id: string) { const flag = await flagClient?.get(id) return !!(flag && flag.testing) } -export async function clearTestFlag(id: string) { - await devAppClient.delete(id) +export async function withTestFlag(id: string, fn: () => Promise) { + // TODO(samwho): this has a bit of a problem where if 2 automations are tested + // at the same time, the second one will overwrite the first one's flag. We + // should instead use an atomic counter and only clear the flag when the + // counter reaches 0. + await flagClient.store(id, { testing: true }, AUTOMATION_TEST_FLAG_SECONDS) + try { + return await fn() + } finally { + await devAppClient.delete(id) + } } export function getSocketPubSubClients() { diff --git a/packages/types/src/api/web/app/automation.ts b/packages/types/src/api/web/app/automation.ts index 40f69fc467..572e6499b6 100644 --- a/packages/types/src/api/web/app/automation.ts +++ b/packages/types/src/api/web/app/automation.ts @@ -75,3 +75,10 @@ export interface TestAutomationRequest { row?: Row } export interface TestAutomationResponse {} + +export interface TestAutomationStepRequest { + inputs: Record + context: Record +} + +export type TestAutomationStepResponse = any diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts index d56f0de879..a7556c2ce3 100644 --- a/packages/types/src/documents/app/automation/automation.ts +++ b/packages/types/src/documents/app/automation/automation.ts @@ -1,10 +1,10 @@ import { Document } from "../../document" -import { EventEmitter } from "events" import { User } from "../../global" import { ReadStream } from "fs" import { Row } from "../row" import { Table } from "../table" import { AutomationStep, AutomationTrigger } from "./schema" +import { ContextEmitter } from "../../../sdk" export enum AutomationIOType { OBJECT = "object", @@ -218,7 +218,7 @@ export interface AutomationLogPage { export interface AutomationStepInputBase { context: Record - emitter: EventEmitter + emitter: ContextEmitter appId: string apiKey?: string } From 99cf4feb07dce88d97155de19c1e922731ba15b7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 Jan 2025 17:30:36 +0000 Subject: [PATCH 051/112] Remove old automation test flag mechanism from Redis. --- .../server/src/api/controllers/automation.ts | 39 +++++++------------ packages/server/src/automations/triggers.ts | 11 +----- 2 files changed, 17 insertions(+), 33 deletions(-) diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index a77014cf31..8c58cd4a19 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -2,7 +2,6 @@ import * as triggers from "../../automations/triggers" import { sdk as coreSdk } from "@budibase/shared-core" import { DocumentType } from "../../db/utils" import { updateTestHistory, removeDeprecated } from "../../automations/utils" -import { withTestFlag } from "../../utilities/redis" import { context, cache, events, db as dbCore } from "@budibase/backend-core" import { automations, features } from "@budibase/pro" import { @@ -246,17 +245,15 @@ export async function test( const { request, appId } = ctx const { body } = request - ctx.body = await withTestFlag(automation._id!, async () => { - const occurredAt = new Date().getTime() - await updateTestHistory(appId, automation, { ...body, occurredAt }) + const occurredAt = new Date().getTime() + await updateTestHistory(appId, automation, { ...body, occurredAt }) - const user = sdk.users.getUserContextBindings(ctx.user) - return await triggers.externalTrigger( - automation, - { ...prepareTestInput(body), appId, user }, - { getResponses: true } - ) - }) + const user = sdk.users.getUserContextBindings(ctx.user) + ctx.body = await triggers.externalTrigger( + automation, + { ...prepareTestInput(body), appId, user }, + { getResponses: true } + ) await events.automation.tested(automation) } @@ -271,7 +268,7 @@ export async function testStep( ctx.throw(404, `Automation ${ctx.params.id} not found`) } - const step = automation.definition.steps.find(s => s.stepId === stepId) + const step = automation.definition.steps.find(s => s.id === stepId) if (!step) { ctx.throw(404, `Step ${stepId} not found on automation ${id}`) } @@ -290,16 +287,10 @@ export async function testStep( ctx.throw(400, `Step ${stepId} is not a valid step`) } - const output = await withTestFlag( - automation._id!, - async () => - await fn({ - inputs: body.inputs, - context: await enrichBaseContext(body.context), - appId: ctx.appId, - emitter: new NoopEmitter(), - }) - ) - - ctx.body = output + ctx.body = await fn({ + inputs: body.inputs, + context: await enrichBaseContext(body.context), + appId: ctx.appId, + emitter: new NoopEmitter(), + }) } diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 67d2dcb911..10830a4046 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -82,11 +82,7 @@ async function queueRelevantRowAutomations( // don't queue events which are for dev apps, only way to test automations is // running tests on them, in production the test flag will never // be checked due to lazy evaluation (first always false) - if ( - !env.ALLOW_DEV_AUTOMATIONS && - isDevAppID(event.appId) && - !(await checkTestFlag(automation._id!)) - ) { + if (!env.ALLOW_DEV_AUTOMATIONS && isDevAppID(event.appId)) { continue } @@ -170,10 +166,7 @@ export async function externalTrigger( throw new Error("Automation is disabled") } - if ( - sdk.automations.isAppAction(automation) && - !(await checkTestFlag(automation._id!)) - ) { + if (sdk.automations.isAppAction(automation) && !isDevAppID(params.appId)) { // values are likely to be submitted as strings, so we shall convert to correct type const coercedFields: any = {} const fields = automation.definition.trigger.inputs.fields From 0670c89e83946323f0ff014a5a5687e6166fac71 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 Jan 2025 17:30:49 +0000 Subject: [PATCH 052/112] Remove unused import. --- packages/server/src/automations/triggers.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 10830a4046..f2082e5c0c 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -4,7 +4,6 @@ import { coerce } from "../utilities/rowProcessor" import { definitions } from "./triggerInfo" // need this to call directly, so we can get a response import { automationQueue } from "./bullboard" -import { checkTestFlag } from "../utilities/redis" import * as utils from "./utils" import env from "../environment" import { context, logging, db as dbCore } from "@budibase/backend-core" From f96c4f352d3dbefb8ad28331b24c2f4d8c42b19e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 Jan 2025 17:38:28 +0000 Subject: [PATCH 053/112] Revert "Remove unused import." This reverts commit 0670c89e83946323f0ff014a5a5687e6166fac71. --- packages/server/src/automations/triggers.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index f2082e5c0c..10830a4046 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -4,6 +4,7 @@ import { coerce } from "../utilities/rowProcessor" import { definitions } from "./triggerInfo" // need this to call directly, so we can get a response import { automationQueue } from "./bullboard" +import { checkTestFlag } from "../utilities/redis" import * as utils from "./utils" import env from "../environment" import { context, logging, db as dbCore } from "@budibase/backend-core" From 5afab49e18d12f0b1c007e80dd0cc158fd312740 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 Jan 2025 17:38:38 +0000 Subject: [PATCH 054/112] Revert "Remove old automation test flag mechanism from Redis." This reverts commit 99cf4feb07dce88d97155de19c1e922731ba15b7. --- .../server/src/api/controllers/automation.ts | 39 ++++++++++++------- packages/server/src/automations/triggers.ts | 11 +++++- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index 8c58cd4a19..a77014cf31 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -2,6 +2,7 @@ import * as triggers from "../../automations/triggers" import { sdk as coreSdk } from "@budibase/shared-core" import { DocumentType } from "../../db/utils" import { updateTestHistory, removeDeprecated } from "../../automations/utils" +import { withTestFlag } from "../../utilities/redis" import { context, cache, events, db as dbCore } from "@budibase/backend-core" import { automations, features } from "@budibase/pro" import { @@ -245,15 +246,17 @@ export async function test( const { request, appId } = ctx const { body } = request - const occurredAt = new Date().getTime() - await updateTestHistory(appId, automation, { ...body, occurredAt }) + ctx.body = await withTestFlag(automation._id!, async () => { + const occurredAt = new Date().getTime() + await updateTestHistory(appId, automation, { ...body, occurredAt }) - const user = sdk.users.getUserContextBindings(ctx.user) - ctx.body = await triggers.externalTrigger( - automation, - { ...prepareTestInput(body), appId, user }, - { getResponses: true } - ) + const user = sdk.users.getUserContextBindings(ctx.user) + return await triggers.externalTrigger( + automation, + { ...prepareTestInput(body), appId, user }, + { getResponses: true } + ) + }) await events.automation.tested(automation) } @@ -268,7 +271,7 @@ export async function testStep( ctx.throw(404, `Automation ${ctx.params.id} not found`) } - const step = automation.definition.steps.find(s => s.id === stepId) + const step = automation.definition.steps.find(s => s.stepId === stepId) if (!step) { ctx.throw(404, `Step ${stepId} not found on automation ${id}`) } @@ -287,10 +290,16 @@ export async function testStep( ctx.throw(400, `Step ${stepId} is not a valid step`) } - ctx.body = await fn({ - inputs: body.inputs, - context: await enrichBaseContext(body.context), - appId: ctx.appId, - emitter: new NoopEmitter(), - }) + const output = await withTestFlag( + automation._id!, + async () => + await fn({ + inputs: body.inputs, + context: await enrichBaseContext(body.context), + appId: ctx.appId, + emitter: new NoopEmitter(), + }) + ) + + ctx.body = output } diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 10830a4046..67d2dcb911 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -82,7 +82,11 @@ async function queueRelevantRowAutomations( // don't queue events which are for dev apps, only way to test automations is // running tests on them, in production the test flag will never // be checked due to lazy evaluation (first always false) - if (!env.ALLOW_DEV_AUTOMATIONS && isDevAppID(event.appId)) { + if ( + !env.ALLOW_DEV_AUTOMATIONS && + isDevAppID(event.appId) && + !(await checkTestFlag(automation._id!)) + ) { continue } @@ -166,7 +170,10 @@ export async function externalTrigger( throw new Error("Automation is disabled") } - if (sdk.automations.isAppAction(automation) && !isDevAppID(params.appId)) { + if ( + sdk.automations.isAppAction(automation) && + !(await checkTestFlag(automation._id!)) + ) { // values are likely to be submitted as strings, so we shall convert to correct type const coercedFields: any = {} const fields = automation.definition.trigger.inputs.fields From 31fc2e45c9252a442f5dc7e77613493523ad629b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 Jan 2025 18:08:14 +0000 Subject: [PATCH 055/112] Improve some typing around automation testing. --- .../tests/utilities/AutomationTestBuilder.ts | 28 +++++++------- packages/server/src/automations/triggers.ts | 33 +++++++++------- .../src/tests/utilities/api/automation.ts | 38 ++++++++++++++++++- packages/types/src/api/web/app/automation.ts | 10 ++++- .../documents/app/automation/automation.ts | 8 ++++ 5 files changed, 86 insertions(+), 31 deletions(-) diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index 830d2ee5ca..50527d97af 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -1,5 +1,4 @@ import { v4 as uuidv4 } from "uuid" -import { testAutomation } from "../../../api/routes/tests/utilities/TestFunctions" import { BUILTIN_ACTION_DEFINITIONS } from "../../actions" import { TRIGGER_DEFINITIONS } from "../../triggers" import { @@ -7,7 +6,6 @@ import { AppActionTriggerOutputs, Automation, AutomationActionStepId, - AutomationResults, AutomationStep, AutomationStepInputs, AutomationTrigger, @@ -24,6 +22,7 @@ import { ExecuteQueryStepInputs, ExecuteScriptStepInputs, FilterStepInputs, + isDidNotTriggerResponse, LoopStepInputs, OpenAIStepInputs, QueryRowsStepInputs, @@ -36,6 +35,7 @@ import { SearchFilters, ServerLogStepInputs, SmtpEmailStepInputs, + TestAutomationRequest, UpdateRowStepInputs, WebhookTriggerInputs, WebhookTriggerOutputs, @@ -279,7 +279,7 @@ class StepBuilder extends BaseStepBuilder { class AutomationBuilder extends BaseStepBuilder { private automationConfig: Automation private config: TestConfiguration - private triggerOutputs: any + private triggerOutputs: TriggerOutputs private triggerSet = false constructor( @@ -398,21 +398,19 @@ class AutomationBuilder extends BaseStepBuilder { async run() { const automation = await this.save() - const results = await testAutomation( - this.config, - automation, - this.triggerOutputs + const response = await this.config.api.automation.test( + automation._id!, + this.triggerOutputs as TestAutomationRequest ) - return this.processResults(results) - } - private processResults(results: { - body: AutomationResults - }): AutomationResults { - results.body.steps.shift() + if (isDidNotTriggerResponse(response)) { + throw new Error(response.message) + } + + response.steps.shift() return { - trigger: results.body.trigger, - steps: results.body.steps, + trigger: response.trigger, + steps: response.steps, } } } diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 67d2dcb911..a9317772d9 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -21,6 +21,7 @@ import { AutomationRowEvent, UserBindings, AutomationResults, + DidNotTriggerResponse, } from "@budibase/types" import { executeInThread } from "../threads/automation" import { dataFilters, sdk } from "@budibase/shared-core" @@ -33,14 +34,6 @@ const JOB_OPTS = { import * as automationUtils from "../automations/automationUtils" import { doesTableExist } from "../sdk/app/tables/getters" -type DidNotTriggerResponse = { - outputs: { - success: false - status: AutomationStatus.STOPPED - } - message: AutomationStoppedReason.TRIGGER_FILTER_NOT_MET -} - async function getAllAutomations() { const db = context.getAppDB() let automations = await db.allDocs( @@ -156,14 +149,26 @@ export function isAutomationResults( ) } +interface AutomationTriggerParams { + fields: Record + timeout?: number + appId?: string + user?: UserBindings +} + export async function externalTrigger( automation: Automation, - params: { - fields: Record - timeout?: number - appId?: string - user?: UserBindings - }, + params: AutomationTriggerParams, + options: { getResponses: true } +): Promise +export async function externalTrigger( + automation: Automation, + params: AutomationTriggerParams, + options?: { getResponses: false } +): Promise +export async function externalTrigger( + automation: Automation, + params: AutomationTriggerParams, { getResponses }: { getResponses?: boolean } = {} ): Promise { if (automation.disabled) { diff --git a/packages/server/src/tests/utilities/api/automation.ts b/packages/server/src/tests/utilities/api/automation.ts index 9d9a27e891..6041664999 100644 --- a/packages/server/src/tests/utilities/api/automation.ts +++ b/packages/server/src/tests/utilities/api/automation.ts @@ -1,4 +1,11 @@ -import { Automation, FetchAutomationResponse } from "@budibase/types" +import { + Automation, + FetchAutomationResponse, + TestAutomationRequest, + TestAutomationResponse, + TestAutomationStepRequest, + TestAutomationStepResponse, +} from "@budibase/types" import { Expectations, TestAPI } from "./base" export class AutomationAPI extends TestAPI { @@ -33,4 +40,33 @@ export class AutomationAPI extends TestAPI { }) return result } + + test = async ( + id: string, + body: TestAutomationRequest, + expectations?: Expectations + ): Promise => { + return await this._post( + `/api/automations/${id}/test`, + { + body, + expectations, + } + ) + } + + testStep = async ( + id: string, + stepId: string, + body: TestAutomationStepRequest, + expectations?: Expectations + ): Promise => { + return await this._post( + `/api/automations/${id}/steps/${stepId}/test`, + { + body, + expectations, + } + ) + } } diff --git a/packages/types/src/api/web/app/automation.ts b/packages/types/src/api/web/app/automation.ts index 572e6499b6..edff4b5eaf 100644 --- a/packages/types/src/api/web/app/automation.ts +++ b/packages/types/src/api/web/app/automation.ts @@ -2,10 +2,12 @@ import { Automation, AutomationActionStepId, AutomationLogPage, + AutomationResults, AutomationStatus, AutomationStepDefinition, AutomationTriggerDefinition, AutomationTriggerStepId, + DidNotTriggerResponse, Row, } from "../../../documents" import { DocumentDestroyResponse } from "@budibase/nano" @@ -74,7 +76,13 @@ export interface TestAutomationRequest { fields: Record row?: Row } -export interface TestAutomationResponse {} +export type TestAutomationResponse = AutomationResults | DidNotTriggerResponse + +export function isDidNotTriggerResponse( + response: TestAutomationResponse +): response is DidNotTriggerResponse { + return !!("message" in response && response.message) +} export interface TestAutomationStepRequest { inputs: Record diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts index a7556c2ce3..0314701d72 100644 --- a/packages/types/src/documents/app/automation/automation.ts +++ b/packages/types/src/documents/app/automation/automation.ts @@ -205,6 +205,14 @@ export interface AutomationResults { }[] } +export interface DidNotTriggerResponse { + outputs: { + success: false + status: AutomationStatus.STOPPED + } + message: AutomationStoppedReason.TRIGGER_FILTER_NOT_MET +} + export interface AutomationLog extends AutomationResults, Document { automationName: string _rev?: string From 75dab572e956e7669ba8282c56c74be4e235f474 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 20:06:02 +0100 Subject: [PATCH 056/112] Move data to builder --- .../design/[screenId]/_components/AppPreview.svelte | 3 +++ packages/client/src/index.js | 1 + packages/client/src/stores/builder.js | 4 +--- .../server/src/api/controllers/static/templates/preview.hbs | 4 +++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index 55a4dc4de4..661e985194 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -68,6 +68,9 @@ port: window.location.port, }, snippets: $snippets, + componentErrors: { + c5ea93132725c48b2a365fcc1facaee86: ["Ups...!"], + }, // TODO } // Refresh the preview when required diff --git a/packages/client/src/index.js b/packages/client/src/index.js index 9cef52bb1e..7cb9ed4430 100644 --- a/packages/client/src/index.js +++ b/packages/client/src/index.js @@ -43,6 +43,7 @@ const loadBudibase = async () => { usedPlugins: window["##BUDIBASE_USED_PLUGINS##"], location: window["##BUDIBASE_LOCATION##"], snippets: window["##BUDIBASE_SNIPPETS##"], + componentErrors: window["##BUDIBASE_COMPONENT_ERRORS##"], }) // Set app ID - this window flag is set by both the preview and the real diff --git a/packages/client/src/stores/builder.js b/packages/client/src/stores/builder.js index 62123c07e5..1ae7d3a670 100644 --- a/packages/client/src/stores/builder.js +++ b/packages/client/src/stores/builder.js @@ -19,9 +19,7 @@ const createBuilderStore = () => { eventResolvers: {}, metadata: null, snippets: null, - componentErrors: { - c5ea93132725c48b2a365fcc1facaee86: ["Ups...!"], - }, // TODO + componentErrors: {}, // Legacy - allow the builder to specify a layout layout: null, diff --git a/packages/server/src/api/controllers/static/templates/preview.hbs b/packages/server/src/api/controllers/static/templates/preview.hbs index 87b9ad6ea3..750a780897 100644 --- a/packages/server/src/api/controllers/static/templates/preview.hbs +++ b/packages/server/src/api/controllers/static/templates/preview.hbs @@ -73,7 +73,8 @@ hiddenComponentIds, usedPlugins, location, - snippets + snippets, + componentErrors } = parsed // Set some flags so the app knows we're in the builder @@ -91,6 +92,7 @@ window["##BUDIBASE_USED_PLUGINS##"] = usedPlugins window["##BUDIBASE_LOCATION##"] = location window["##BUDIBASE_SNIPPETS##"] = snippets + window['##BUDIBASE_COMPONENT_ERRORS##'] = componentErrors // Initialise app try { From 655332715dae0a64866b0a797076425b4299817b Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Tue, 21 Jan 2025 09:23:18 +0000 Subject: [PATCH 057/112] login platform user after forced password reset (#15406) --- packages/builder/src/pages/builder/auth/reset.svelte | 6 ++++++ packages/builder/src/stores/portal/auth.ts | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/pages/builder/auth/reset.svelte b/packages/builder/src/pages/builder/auth/reset.svelte index 1179618e40..64438c22d5 100644 --- a/packages/builder/src/pages/builder/auth/reset.svelte +++ b/packages/builder/src/pages/builder/auth/reset.svelte @@ -30,10 +30,16 @@ try { loading = true if (forceResetPassword) { + const email = $auth.user.email + const tenantId = $auth.user.tenantId await auth.updateSelf({ password, forceResetPassword: false, }) + if (!$auth.user) { + // Update self will clear the platform user, so need to login + await auth.login(email, password, tenantId) + } $goto("../portal/") } else { await auth.resetPassword(password, resetCode) diff --git a/packages/builder/src/stores/portal/auth.ts b/packages/builder/src/stores/portal/auth.ts index 171b2b43ae..c3dcaa3663 100644 --- a/packages/builder/src/stores/portal/auth.ts +++ b/packages/builder/src/stores/portal/auth.ts @@ -121,8 +121,8 @@ class AuthStore extends BudiStore { } } - async login(username: string, password: string) { - const tenantId = get(this.store).tenantId + async login(username: string, password: string, targetTenantId?: string) { + const tenantId = targetTenantId || get(this.store).tenantId await API.logIn(tenantId, username, password) await this.getSelf() } From d1294c8d44e5ef9fac098ed823ae0b17ec37b24f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 12:38:31 +0100 Subject: [PATCH 058/112] Errors from store --- .../[screenId]/_components/AppPreview.svelte | 5 +- packages/builder/src/stores/builder/index.js | 2 + .../src/stores/builder/screenComponent.ts | 61 +++++++++++++++++++ 3 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 packages/builder/src/stores/builder/screenComponent.ts diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index 661e985194..fc0b67f63d 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -11,6 +11,7 @@ selectedScreen, hoverStore, componentTreeNodesStore, + screenComponentStore, snippets, } from "@/stores/builder" import ConfirmDialog from "@/components/common/ConfirmDialog.svelte" @@ -68,9 +69,7 @@ port: window.location.port, }, snippets: $snippets, - componentErrors: { - c5ea93132725c48b2a365fcc1facaee86: ["Ups...!"], - }, // TODO + componentErrors: $screenComponentStore.errors, } // Refresh the preview when required diff --git a/packages/builder/src/stores/builder/index.js b/packages/builder/src/stores/builder/index.js index 08d87bebf5..23491996d1 100644 --- a/packages/builder/src/stores/builder/index.js +++ b/packages/builder/src/stores/builder/index.js @@ -16,6 +16,7 @@ import { userStore, userSelectedResourceMap, isOnlyUser } from "./users.js" import { deploymentStore } from "./deployments.js" import { contextMenuStore } from "./contextMenu.js" import { snippets } from "./snippets" +import { screenComponentStore } from "./screenComponent" // Backend import { tables } from "./tables" @@ -67,6 +68,7 @@ export { snippets, rowActions, appPublished, + screenComponentStore, } export const reset = () => { diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts new file mode 100644 index 0000000000..b97bb7ba98 --- /dev/null +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -0,0 +1,61 @@ +import { derived } from "svelte/store" +import { tables, selectedScreen } from "@/stores/builder" +import { DerivedBudiStore } from "../BudiStore" +import { findComponentsBySettingsType } from "@/helpers/screen" +import { Screen } from "@budibase/types" + +interface BuilderScreenComponentStore {} + +interface DerivedScreenComponentStore extends BuilderScreenComponentStore { + errors: Record +} + +export class ScreenComponentStore extends DerivedBudiStore< + BuilderScreenComponentStore, + DerivedScreenComponentStore +> { + constructor() { + const makeDerivedStore = () => { + return derived( + [selectedScreen, tables], + ([$selectedScreen, $tables]): DerivedScreenComponentStore => { + function getErrors() { + const datasources = $tables.list.reduce( + (list, table) => ({ + ...list, + [table._id!]: table, + }), + {} + ) + return { + ...getInvalidDatasources($selectedScreen, datasources), + } + } + + return { + errors: getErrors(), + } + } + ) + } + + super({}, makeDerivedStore) + } +} + +export const screenComponentStore = new ScreenComponentStore() + +function getInvalidDatasources( + screen: Screen, + datasources: Record +) { + const result: Record = {} + for (const component of findComponentsBySettingsType(screen, "table")) { + const { resourceId, type, label } = component.dataSource + if (!datasources[resourceId]) { + result[component._id!] = [`The ${type} named "${label}" was removed`] + } + } + + return result +} From d5f34970ad9c632c06b8b9266bd247872584e8c3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 12:48:28 +0100 Subject: [PATCH 059/112] Use friendly name --- .../builder/src/stores/builder/screenComponent.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index b97bb7ba98..cb7e26bf93 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -53,9 +53,19 @@ function getInvalidDatasources( for (const component of findComponentsBySettingsType(screen, "table")) { const { resourceId, type, label } = component.dataSource if (!datasources[resourceId]) { - result[component._id!] = [`The ${type} named "${label}" was removed`] + const friendlyTypeName = + friendlyNameByType[type as keyof typeof friendlyNameByType] + result[component._id!] = [ + `The ${friendlyTypeName} named "${label}" was removed`, + ] } } return result } + +const friendlyNameByType = { + table: "table", + view: "view", + viewV2: "view", +} From c5e4edcc9713f6286644fe10f65c19ac02a5c66c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 21 Jan 2025 11:54:27 +0000 Subject: [PATCH 060/112] Setting overflow-y in evaluation panel to auto. --- .../src/components/common/bindings/EvaluationSidePanel.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte index fcd23bb816..c47840ea83 100644 --- a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte +++ b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte @@ -174,7 +174,7 @@ padding: var(--spacing-m) var(--spacing-l); font-family: var(--font-mono); font-size: 12px; - overflow-y: scroll; + overflow-y: auto; overflow-x: hidden; white-space: pre-line; word-wrap: break-word; From 56f666f15a3a9b9c8793b1b69e01ca5cead8abbd Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 12:57:30 +0100 Subject: [PATCH 061/112] Display TableSelect the same way we do for DataSourceSelect --- .../settings/controls/TableSelect.svelte | 50 +++++++++++++++---- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/TableSelect.svelte b/packages/builder/src/components/design/settings/controls/TableSelect.svelte index 706c4ca74e..a91bde74ba 100644 --- a/packages/builder/src/components/design/settings/controls/TableSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/TableSelect.svelte @@ -1,22 +1,30 @@ - +
+ +
+ + {#if views?.length} + + {/if} +
+ From ece99aa751ed9c473d90e1842dc41954b4cfb431 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 13:15:53 +0100 Subject: [PATCH 062/112] DRY --- .../DataSourceSelect/DataSourceSelect.svelte | 16 +------- .../settings/controls/TableSelect.svelte | 8 ++-- .../builder/src/stores/builder/builder.ts | 39 +++++++++++++++++-- 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index b23ef5348d..20ba4c8552 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -18,7 +18,6 @@ } from "@budibase/bbui" import { createEventDispatcher } from "svelte" import { - tables as tablesStore, queries as queriesStore, viewsV2 as viewsV2Store, views as viewsStore, @@ -26,6 +25,7 @@ componentStore, datasources, integrations, + builderStore, } from "@/stores/builder" import BindingBuilder from "@/components/integration/QueryBindingBuilder.svelte" import IntegrationQueryEditor from "@/components/integration/index.svelte" @@ -51,19 +51,7 @@ let modal $: text = value?.label ?? "Choose an option" - $: tables = $tablesStore.list - .map(table => format.table(table, $datasources.list)) - .sort((a, b) => { - // sort tables alphabetically, grouped by datasource - const dsA = a.datasourceName ?? "" - const dsB = b.datasourceName ?? "" - - const dsComparison = dsA.localeCompare(dsB) - if (dsComparison !== 0) { - return dsComparison - } - return a.label.localeCompare(b.label) - }) + $: tables = $builderStore.formatedTableNames $: viewsV1 = $viewsStore.list.map(view => ({ ...view, label: view.name, diff --git a/packages/builder/src/components/design/settings/controls/TableSelect.svelte b/packages/builder/src/components/design/settings/controls/TableSelect.svelte index a91bde74ba..4c7c59037c 100644 --- a/packages/builder/src/components/design/settings/controls/TableSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/TableSelect.svelte @@ -1,8 +1,8 @@ {#if dividerState} @@ -29,7 +32,9 @@ on:click={() => onSelect(data)} > - {data.datasourceName ? `${data.datasourceName} - ` : ""}{data.label} + {data.datasourceName && displayDatasourceName + ? `${data.datasourceName} - ` + : ""}{data.label} Date: Tue, 21 Jan 2025 15:02:37 +0100 Subject: [PATCH 072/112] Fix paddings --- .../design/settings/controls/TableSelect.svelte | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/builder/src/components/design/settings/controls/TableSelect.svelte b/packages/builder/src/components/design/settings/controls/TableSelect.svelte index 146decda91..85209a92a0 100644 --- a/packages/builder/src/components/design/settings/controls/TableSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/TableSelect.svelte @@ -66,3 +66,20 @@ {/if}
+ + From 42e86554c881c01bc3f5fdc49bb83303af7158d7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 15:18:31 +0100 Subject: [PATCH 073/112] Validate views --- .../src/stores/builder/screenComponent.ts | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index cb7e26bf93..dd2f7a8b1c 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -1,5 +1,5 @@ import { derived } from "svelte/store" -import { tables, selectedScreen } from "@/stores/builder" +import { tables, selectedScreen, viewsV2 } from "@/stores/builder" import { DerivedBudiStore } from "../BudiStore" import { findComponentsBySettingsType } from "@/helpers/screen" import { Screen } from "@budibase/types" @@ -17,16 +17,25 @@ export class ScreenComponentStore extends DerivedBudiStore< constructor() { const makeDerivedStore = () => { return derived( - [selectedScreen, tables], - ([$selectedScreen, $tables]): DerivedScreenComponentStore => { + [selectedScreen, tables, viewsV2], + ([$selectedScreen, $tables, $viewsV2]): DerivedScreenComponentStore => { function getErrors() { - const datasources = $tables.list.reduce( - (list, table) => ({ - ...list, - [table._id!]: table, - }), - {} - ) + const datasources = { + ...$tables.list.reduce( + (list, table) => ({ + ...list, + [table._id!]: table, + }), + {} + ), + ...$viewsV2.list.reduce( + (list, view) => ({ + ...list, + [view.id]: view, + }), + {} + ), + } return { ...getInvalidDatasources($selectedScreen, datasources), } From ec930372412b4ffc47878ddbc8de2d16aa2691a5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 15:19:45 +0100 Subject: [PATCH 074/112] Change error message --- packages/builder/src/stores/builder/screenComponent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index dd2f7a8b1c..4b591cf0cf 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -65,7 +65,7 @@ function getInvalidDatasources( const friendlyTypeName = friendlyNameByType[type as keyof typeof friendlyNameByType] result[component._id!] = [ - `The ${friendlyTypeName} named "${label}" was removed`, + `The ${friendlyTypeName} named "${label}" does not exist`, ] } } From f32910b033863216393ef6466e6f73d5bdfa47bb Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 15:30:04 +0100 Subject: [PATCH 075/112] Clean code --- .../src/stores/builder/screenComponent.ts | 53 +++++++++++-------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 4b591cf0cf..baa6520832 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -2,7 +2,7 @@ import { derived } from "svelte/store" import { tables, selectedScreen, viewsV2 } from "@/stores/builder" import { DerivedBudiStore } from "../BudiStore" import { findComponentsBySettingsType } from "@/helpers/screen" -import { Screen } from "@budibase/types" +import { Screen, Table, ViewV2 } from "@budibase/types" interface BuilderScreenComponentStore {} @@ -20,22 +20,10 @@ export class ScreenComponentStore extends DerivedBudiStore< [selectedScreen, tables, viewsV2], ([$selectedScreen, $tables, $viewsV2]): DerivedScreenComponentStore => { function getErrors() { - const datasources = { - ...$tables.list.reduce( - (list, table) => ({ - ...list, - [table._id!]: table, - }), - {} - ), - ...$viewsV2.list.reduce( - (list, view) => ({ - ...list, - [view.id]: view, - }), - {} - ), - } + const datasources = flattenTablesAndViews( + $tables.list, + $viewsV2.list + ) return { ...getInvalidDatasources($selectedScreen, datasources), } @@ -54,10 +42,35 @@ export class ScreenComponentStore extends DerivedBudiStore< export const screenComponentStore = new ScreenComponentStore() +function flattenTablesAndViews(tables: Table[], views: ViewV2[]) { + return { + ...tables.reduce( + (list, table) => ({ + ...list, + [table._id!]: table, + }), + {} + ), + ...views.reduce( + (list, view) => ({ + ...list, + [view.id]: view, + }), + {} + ), + } +} + function getInvalidDatasources( screen: Screen, datasources: Record ) { + const friendlyNameByType = { + table: "table", + view: "view", + viewV2: "view", + } + const result: Record = {} for (const component of findComponentsBySettingsType(screen, "table")) { const { resourceId, type, label } = component.dataSource @@ -72,9 +85,3 @@ function getInvalidDatasources( return result } - -const friendlyNameByType = { - table: "table", - view: "view", - viewV2: "view", -} From e1cc8da9dde735579b7f751b4513cdd33784782f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 15:31:16 +0100 Subject: [PATCH 076/112] Clean types --- packages/client/src/stores/derived/componentErrors.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/stores/derived/componentErrors.ts b/packages/client/src/stores/derived/componentErrors.ts index 48185de9c3..68e87a061d 100644 --- a/packages/client/src/stores/derived/componentErrors.ts +++ b/packages/client/src/stores/derived/componentErrors.ts @@ -2,5 +2,5 @@ import { derived } from "svelte/store" import { builderStore } from "../builder.js" export const componentErrors = derived([builderStore], ([$builderStore]) => { - return $builderStore.componentErrors as Record + return $builderStore.componentErrors }) From fc599767c21c23abe1d3aa14445fe5c614b42bea Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 16:06:21 +0100 Subject: [PATCH 077/112] Fix imports --- .../src/stores/builder/screenComponent.ts | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index baa6520832..b4eb01a3b7 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -1,5 +1,7 @@ import { derived } from "svelte/store" -import { tables, selectedScreen, viewsV2 } from "@/stores/builder" +import { tables } from "./tables" +import { selectedScreen } from "./screens" +import { viewsV2 } from "./viewsV2" import { DerivedBudiStore } from "../BudiStore" import { findComponentsBySettingsType } from "@/helpers/screen" import { Screen, Table, ViewV2 } from "@budibase/types" @@ -19,18 +21,9 @@ export class ScreenComponentStore extends DerivedBudiStore< return derived( [selectedScreen, tables, viewsV2], ([$selectedScreen, $tables, $viewsV2]): DerivedScreenComponentStore => { - function getErrors() { - const datasources = flattenTablesAndViews( - $tables.list, - $viewsV2.list - ) - return { - ...getInvalidDatasources($selectedScreen, datasources), - } - } - + const datasources = flattenTablesAndViews($tables.list, $viewsV2.list) return { - errors: getErrors(), + errors: getInvalidDatasources($selectedScreen, datasources), } } ) From 221021ae9b8051e3ca3a9257ef00c8bf33a0e08e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 21 Jan 2025 17:00:40 +0000 Subject: [PATCH 078/112] Support for our own vm-browserify implementation which re-uses the iframe for running JS in the frontend, to improve performance. --- packages/string-templates/package.json | 1 + packages/string-templates/src/index.ts | 6 +++--- yarn.lock | 18 +++++++++++++++--- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 1d7a4507ab..74d9aaa85a 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -23,6 +23,7 @@ }, "dependencies": { "@budibase/handlebars-helpers": "^0.13.2", + "@budibase/vm-browserify": "^1.1.3", "dayjs": "^1.10.8", "handlebars": "^4.7.8", "lodash.clonedeep": "^4.5.0" diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index 8dda8b71ab..bd008dd4d2 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -1,4 +1,4 @@ -import { createContext, runInNewContext } from "vm" +import vm from "@budibase/vm-browserify" import { create, TemplateDelegate } from "handlebars" import { registerAll, registerMinimum } from "./helpers/index" import { postprocess, postprocessWithLogs, preprocess } from "./processors" @@ -511,11 +511,11 @@ export function browserJSSetup() { * Use polyfilled vm to run JS scripts in a browser Env */ setJSRunner((js: string, context: Record) => { - createContext(context) + vm.createContext(context) const wrappedJs = frontendWrapJS(js) - const result = runInNewContext(wrappedJs, context, { timeout: 1000 }) + const result = vm.runInNewContext(wrappedJs, context) if (result.error) { throw new UserScriptError(result.error) } diff --git a/yarn.lock b/yarn.lock index 453dc45128..e25ff97747 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2131,9 +2131,9 @@ through2 "^2.0.0" "@budibase/pro@npm:@budibase/pro@latest": - version "3.2.44" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.2.44.tgz#90367bb2167aafd8c809e000a57d349e5dc4bb78" - integrity sha512-Zv2PBVUZUS6/psOpIRIDlW3jrOHWWPhpQXzCk00kIQJaqjkdcvuTXSedQ70u537sQmLu8JsSWbui9MdfF8ksVw== + version "3.2.47" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.2.47.tgz#150d7b16b14932d03c84bdb0e6d570d490c28a5c" + integrity sha512-UeTIq7yzMUK6w/akUsRafoD/Kif6PXv4d7K1arn8GTMjwFm9QYu2hg1YkQ+duNdwyZ/GEPlEAV5SYK+NDgtpdA== dependencies: "@anthropic-ai/sdk" "^0.27.3" "@budibase/backend-core" "*" @@ -2152,6 +2152,13 @@ scim-patch "^0.8.1" scim2-parse-filter "^0.2.8" +"@budibase/vm-browserify@^1.1.3": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@budibase/vm-browserify/-/vm-browserify-1.1.3.tgz#63f7917671a0f0cb760e3aa37cfd5dfa32e997ed" + integrity sha512-CuoNb2xwS8TT2ZfG9YqC8QCTcG3ZPLwH4m00sfPDluwmdp3U3HGg/UKWRIqKC6Wv8Mywy1q6bxmSx6Vf40V52w== + dependencies: + indexof "^0.0.1" + "@bull-board/api@5.10.2": version "5.10.2" resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3" @@ -11925,6 +11932,11 @@ indexes-of@^1.0.1: resolved "https://registry.yarnpkg.com/indexes-of/-/indexes-of-1.0.1.tgz#f30f716c8e2bd346c7b67d3df3915566a7c05607" integrity sha512-bup+4tap3Hympa+JBJUG7XuOsdNQ6fxt0MHyXMKuLBKn0OqsTfvUxkUrroEX1+B2VsSHvCjiIcZVxRtYa4nllA== +indexof@^0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/indexof/-/indexof-0.0.1.tgz#82dc336d232b9062179d05ab3293a66059fd435d" + integrity sha512-i0G7hLJ1z0DE8dsqJa2rycj9dBmNKgXBvotXtZYXakU9oivfB9Uj2ZBC27qqef2U58/ZLwalxa1X/RDCdkHtVg== + infer-owner@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/infer-owner/-/infer-owner-1.0.4.tgz#c4cefcaa8e51051c2a40ba2ce8a3d27295af9467" From 0e6ad4db930d7a50cd23264e9a2be3ef6c853585 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 21 Jan 2025 17:06:09 +0000 Subject: [PATCH 079/112] Remove testStep endpoint and fix types. --- .../server/src/api/controllers/automation.ts | 52 +------------------ packages/server/src/api/routes/automation.ts | 12 +---- .../server/src/automations/steps/createRow.ts | 9 ++-- .../server/src/automations/steps/deleteRow.ts | 9 ++-- .../src/automations/steps/executeQuery.ts | 4 +- .../src/automations/steps/executeScript.ts | 4 +- .../server/src/automations/steps/updateRow.ts | 9 ++-- .../server/src/automations/steps/utils.ts | 4 +- packages/server/src/events/NoopEmitter.ts | 39 -------------- packages/server/src/events/index.ts | 1 - .../src/tests/utilities/api/automation.ts | 17 ------ packages/types/src/api/web/app/automation.ts | 7 --- 12 files changed, 26 insertions(+), 141 deletions(-) delete mode 100644 packages/server/src/events/NoopEmitter.ts diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index a77014cf31..13d057ebb7 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -28,18 +28,11 @@ import { TriggerAutomationResponse, TestAutomationRequest, TestAutomationResponse, - TestAutomationStepRequest, - TestAutomationStepResponse, } from "@budibase/types" -import { - getActionDefinitions as actionDefs, - getAction, -} from "../../automations/actions" +import { getActionDefinitions as actionDefs } from "../../automations/actions" import sdk from "../../sdk" import { builderSocket } from "../../websockets" import env from "../../environment" -import { NoopEmitter } from "../../events" -import { enrichBaseContext } from "../../threads/automation" async function getActionDefinitions() { return removeDeprecated(await actionDefs()) @@ -260,46 +253,3 @@ export async function test( await events.automation.tested(automation) } - -export async function testStep( - ctx: UserCtx -) { - const { id, stepId } = ctx.params - const db = context.getAppDB() - const automation = await db.tryGet(id) - if (!automation) { - ctx.throw(404, `Automation ${ctx.params.id} not found`) - } - - const step = automation.definition.steps.find(s => s.stepId === stepId) - if (!step) { - ctx.throw(404, `Step ${stepId} not found on automation ${id}`) - } - - if (step.stepId === AutomationActionStepId.BRANCH) { - ctx.throw(400, "Branch steps cannot be tested directly") - } - if (step.stepId === AutomationActionStepId.LOOP) { - ctx.throw(400, "Loop steps cannot be tested directly") - } - - const { body } = ctx.request - - const fn = await getAction(step.stepId) - if (!fn) { - ctx.throw(400, `Step ${stepId} is not a valid step`) - } - - const output = await withTestFlag( - automation._id!, - async () => - await fn({ - inputs: body.inputs, - context: await enrichBaseContext(body.context), - appId: ctx.appId, - emitter: new NoopEmitter(), - }) - ) - - ctx.body = output -} diff --git a/packages/server/src/api/routes/automation.ts b/packages/server/src/api/routes/automation.ts index ea905be0cd..489487271c 100644 --- a/packages/server/src/api/routes/automation.ts +++ b/packages/server/src/api/routes/automation.ts @@ -1,6 +1,6 @@ import Router from "@koa/router" import * as controller from "../controllers/automation" -import authorized, { authorizedResource } from "../../middleware/authorized" +import authorized from "../../middleware/authorized" import { permissions } from "@budibase/backend-core" import { bodyResource, paramResource } from "../../middleware/resourceId" import { @@ -82,15 +82,5 @@ router ), controller.test ) - .post( - "/api/automations/:id/step/:stepId/test", - appInfoMiddleware({ appType: AppType.DEV }), - authorizedResource( - permissions.PermissionType.AUTOMATION, - permissions.PermissionLevel.EXECUTE, - "id" - ), - controller.testStep - ) export default router diff --git a/packages/server/src/automations/steps/createRow.ts b/packages/server/src/automations/steps/createRow.ts index 24dada422d..cf915dd300 100644 --- a/packages/server/src/automations/steps/createRow.ts +++ b/packages/server/src/automations/steps/createRow.ts @@ -5,8 +5,11 @@ import { sendAutomationAttachmentsToStorage, } from "../automationUtils" import { buildCtx } from "./utils" -import { CreateRowStepInputs, CreateRowStepOutputs } from "@budibase/types" -import { EventEmitter } from "events" +import { + ContextEmitter, + CreateRowStepInputs, + CreateRowStepOutputs, +} from "@budibase/types" export async function run({ inputs, @@ -15,7 +18,7 @@ export async function run({ }: { inputs: CreateRowStepInputs appId: string - emitter: EventEmitter + emitter: ContextEmitter }): Promise { if (inputs.row == null || inputs.row.tableId == null) { return { diff --git a/packages/server/src/automations/steps/deleteRow.ts b/packages/server/src/automations/steps/deleteRow.ts index 7c50fe4dcb..2498a4e4de 100644 --- a/packages/server/src/automations/steps/deleteRow.ts +++ b/packages/server/src/automations/steps/deleteRow.ts @@ -1,8 +1,11 @@ -import { EventEmitter } from "events" import { destroy } from "../../api/controllers/row" import { buildCtx } from "./utils" import { getError } from "../automationUtils" -import { DeleteRowStepInputs, DeleteRowStepOutputs } from "@budibase/types" +import { + ContextEmitter, + DeleteRowStepInputs, + DeleteRowStepOutputs, +} from "@budibase/types" export async function run({ inputs, @@ -11,7 +14,7 @@ export async function run({ }: { inputs: DeleteRowStepInputs appId: string - emitter: EventEmitter + emitter: ContextEmitter }): Promise { if (inputs.id == null) { return { diff --git a/packages/server/src/automations/steps/executeQuery.ts b/packages/server/src/automations/steps/executeQuery.ts index 9816e31b1e..ad99240eb8 100644 --- a/packages/server/src/automations/steps/executeQuery.ts +++ b/packages/server/src/automations/steps/executeQuery.ts @@ -1,8 +1,8 @@ -import { EventEmitter } from "events" import * as queryController from "../../api/controllers/query" import { buildCtx } from "./utils" import * as automationUtils from "../automationUtils" import { + ContextEmitter, ExecuteQueryStepInputs, ExecuteQueryStepOutputs, } from "@budibase/types" @@ -14,7 +14,7 @@ export async function run({ }: { inputs: ExecuteQueryStepInputs appId: string - emitter: EventEmitter + emitter: ContextEmitter }): Promise { if (inputs.query == null) { return { diff --git a/packages/server/src/automations/steps/executeScript.ts b/packages/server/src/automations/steps/executeScript.ts index 105543d34c..db05d0937a 100644 --- a/packages/server/src/automations/steps/executeScript.ts +++ b/packages/server/src/automations/steps/executeScript.ts @@ -2,10 +2,10 @@ import * as scriptController from "../../api/controllers/script" import { buildCtx } from "./utils" import * as automationUtils from "../automationUtils" import { + ContextEmitter, ExecuteScriptStepInputs, ExecuteScriptStepOutputs, } from "@budibase/types" -import { EventEmitter } from "events" export async function run({ inputs, @@ -16,7 +16,7 @@ export async function run({ inputs: ExecuteScriptStepInputs appId: string context: object - emitter: EventEmitter + emitter: ContextEmitter }): Promise { if (inputs.code == null) { return { diff --git a/packages/server/src/automations/steps/updateRow.ts b/packages/server/src/automations/steps/updateRow.ts index 46ae2a5c74..7a62e40706 100644 --- a/packages/server/src/automations/steps/updateRow.ts +++ b/packages/server/src/automations/steps/updateRow.ts @@ -1,8 +1,11 @@ -import { EventEmitter } from "events" import * as rowController from "../../api/controllers/row" import * as automationUtils from "../automationUtils" import { buildCtx } from "./utils" -import { UpdateRowStepInputs, UpdateRowStepOutputs } from "@budibase/types" +import { + ContextEmitter, + UpdateRowStepInputs, + UpdateRowStepOutputs, +} from "@budibase/types" export async function run({ inputs, @@ -11,7 +14,7 @@ export async function run({ }: { inputs: UpdateRowStepInputs appId: string - emitter: EventEmitter + emitter: ContextEmitter }): Promise { if (inputs.rowId == null || inputs.row == null) { return { diff --git a/packages/server/src/automations/steps/utils.ts b/packages/server/src/automations/steps/utils.ts index 8b99044303..20f1e67589 100644 --- a/packages/server/src/automations/steps/utils.ts +++ b/packages/server/src/automations/steps/utils.ts @@ -1,4 +1,4 @@ -import { EventEmitter } from "events" +import { ContextEmitter } from "@budibase/types" export async function getFetchResponse(fetched: any) { let status = fetched.status, @@ -22,7 +22,7 @@ export async function getFetchResponse(fetched: any) { // opts can contain, body, params and version export function buildCtx( appId: string, - emitter?: EventEmitter | null, + emitter?: ContextEmitter | null, opts: any = {} ) { const ctx: any = { diff --git a/packages/server/src/events/NoopEmitter.ts b/packages/server/src/events/NoopEmitter.ts deleted file mode 100644 index ed87618ead..0000000000 --- a/packages/server/src/events/NoopEmitter.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { EventEmitter } from "events" -import { - Table, - Row, - ContextEmitter, - EventType, - UserBindings, -} from "@budibase/types" - -export class NoopEmitter extends EventEmitter implements ContextEmitter { - emitRow(values: { - eventName: EventType.ROW_SAVE - appId: string - row: Row - table: Table - user: UserBindings - }): void - emitRow(values: { - eventName: EventType.ROW_UPDATE - appId: string - row: Row - table: Table - oldRow: Row - user: UserBindings - }): void - emitRow(values: { - eventName: EventType.ROW_DELETE - appId: string - row: Row - user: UserBindings - }): void - emitRow(_values: unknown): void { - return - } - - emitTable(_eventName: string, _appId: string, _table?: Table) { - return - } -} diff --git a/packages/server/src/events/index.ts b/packages/server/src/events/index.ts index 90bf932bcf..23c3f3e512 100644 --- a/packages/server/src/events/index.ts +++ b/packages/server/src/events/index.ts @@ -2,6 +2,5 @@ import BudibaseEmitter from "./BudibaseEmitter" const emitter = new BudibaseEmitter() -export { NoopEmitter } from "./NoopEmitter" export { init } from "./docUpdates" export default emitter diff --git a/packages/server/src/tests/utilities/api/automation.ts b/packages/server/src/tests/utilities/api/automation.ts index 6041664999..3f51385251 100644 --- a/packages/server/src/tests/utilities/api/automation.ts +++ b/packages/server/src/tests/utilities/api/automation.ts @@ -3,8 +3,6 @@ import { FetchAutomationResponse, TestAutomationRequest, TestAutomationResponse, - TestAutomationStepRequest, - TestAutomationStepResponse, } from "@budibase/types" import { Expectations, TestAPI } from "./base" @@ -54,19 +52,4 @@ export class AutomationAPI extends TestAPI { } ) } - - testStep = async ( - id: string, - stepId: string, - body: TestAutomationStepRequest, - expectations?: Expectations - ): Promise => { - return await this._post( - `/api/automations/${id}/steps/${stepId}/test`, - { - body, - expectations, - } - ) - } } diff --git a/packages/types/src/api/web/app/automation.ts b/packages/types/src/api/web/app/automation.ts index edff4b5eaf..b97dee0baf 100644 --- a/packages/types/src/api/web/app/automation.ts +++ b/packages/types/src/api/web/app/automation.ts @@ -83,10 +83,3 @@ export function isDidNotTriggerResponse( ): response is DidNotTriggerResponse { return !!("message" in response && response.message) } - -export interface TestAutomationStepRequest { - inputs: Record - context: Record -} - -export type TestAutomationStepResponse = any From cad6a08bf87570c2485cafc9538460a2cfee8f7e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 11:14:50 +0100 Subject: [PATCH 080/112] Move reactiviness --- packages/client/src/components/Component.svelte | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index 7f825c0601..57daf8cd12 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -139,6 +139,7 @@ // Derive definition properties which can all be optional, so need to be // coerced to booleans + $: invalidSettings = $componentErrors[instance._id] $: hasChildren = !!definition?.hasChildren $: showEmptyState = definition?.showEmptyState !== false $: hasMissingRequiredSettings = missingRequiredSettings?.length > 0 @@ -374,9 +375,6 @@ } } - // Check for invalid settings - $: invalidSettings = $componentErrors[id] - // Extracts a map of all context keys which are required by action settings // to provide the functions to evaluate at runtime. This needs done manually // as the action definitions themselves do not specify bindings for action From 4d85006c35e4d9b2643192902d077147069c6de8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 11:53:50 +0100 Subject: [PATCH 081/112] server nodemon, watch .hbs --- packages/server/nodemon.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/nodemon.json b/packages/server/nodemon.json index ac8c38ccb2..9871b9339e 100644 --- a/packages/server/nodemon.json +++ b/packages/server/nodemon.json @@ -7,7 +7,7 @@ "../shared-core/src", "../string-templates/src" ], - "ext": "js,ts,json,svelte", + "ext": "js,ts,json,svelte,hbs", "ignore": [ "**/*.spec.ts", "**/*.spec.js", From 1294f83ccb9d80f7bcacaf89e9cab7b43f6c4d70 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 22 Jan 2025 11:09:31 +0000 Subject: [PATCH 082/112] Bump version to 3.3.0 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 50582f0a95..a7d534ff01 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.2.47", + "version": "3.3.0", "npmClient": "yarn", "concurrency": 20, "command": { From 2213cd56c46c8f8204c98cd961dbd39482b57271 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 12:18:55 +0100 Subject: [PATCH 083/112] Remove unnecessary derived --- packages/client/src/components/Component.svelte | 3 +-- packages/client/src/stores/derived/componentErrors.ts | 6 ------ packages/client/src/stores/derived/index.js | 1 - 3 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 packages/client/src/stores/derived/componentErrors.ts diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index 57daf8cd12..8d79de2ac4 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -23,7 +23,6 @@ appStore, dndComponentPath, dndIsDragging, - componentErrors, } from "stores" import { Helpers } from "@budibase/bbui" import { getActiveConditions, reduceConditionActions } from "utils/conditions" @@ -139,7 +138,7 @@ // Derive definition properties which can all be optional, so need to be // coerced to booleans - $: invalidSettings = $componentErrors[instance._id] + $: invalidSettings = $builderStore.componentErrors[instance._id] $: hasChildren = !!definition?.hasChildren $: showEmptyState = definition?.showEmptyState !== false $: hasMissingRequiredSettings = missingRequiredSettings?.length > 0 diff --git a/packages/client/src/stores/derived/componentErrors.ts b/packages/client/src/stores/derived/componentErrors.ts deleted file mode 100644 index 68e87a061d..0000000000 --- a/packages/client/src/stores/derived/componentErrors.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { derived } from "svelte/store" -import { builderStore } from "../builder.js" - -export const componentErrors = derived([builderStore], ([$builderStore]) => { - return $builderStore.componentErrors -}) diff --git a/packages/client/src/stores/derived/index.js b/packages/client/src/stores/derived/index.js index e7e70d8952..337c73831f 100644 --- a/packages/client/src/stores/derived/index.js +++ b/packages/client/src/stores/derived/index.js @@ -5,4 +5,3 @@ export { currentRole } from "./currentRole.js" export { dndComponentPath } from "./dndComponentPath.js" export { devToolsEnabled } from "./devToolsEnabled.js" export { snippets } from "./snippets.js" -export { componentErrors } from "./componentErrors" From d3b22e461e3f0f328bab54e4b2b58eb6d833e28e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 16:18:29 +0100 Subject: [PATCH 084/112] Handle errors as a part of the instance to avoid extra refreshes --- packages/client/src/components/Component.svelte | 2 +- packages/client/src/stores/screens.js | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index 8d79de2ac4..a86d43c676 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -138,7 +138,7 @@ // Derive definition properties which can all be optional, so need to be // coerced to booleans - $: invalidSettings = $builderStore.componentErrors[instance._id] + $: invalidSettings = instance?._meta?.errors $: hasChildren = !!definition?.hasChildren $: showEmptyState = definition?.showEmptyState !== false $: hasMissingRequiredSettings = missingRequiredSettings?.length > 0 diff --git a/packages/client/src/stores/screens.js b/packages/client/src/stores/screens.js index bc87216660..491d8d4236 100644 --- a/packages/client/src/stores/screens.js +++ b/packages/client/src/stores/screens.js @@ -42,6 +42,14 @@ const createScreenStore = () => { if ($builderStore.layout) { activeLayout = $builderStore.layout } + + // Attach meta + const errors = $builderStore.componentErrors || {} + const attachComponentMeta = component => { + component._meta = { errors: errors[component._id] || [] } + component._children?.forEach(attachComponentMeta) + } + attachComponentMeta(activeScreen.props) } else { // Find the correct screen by matching the current route screens = $appStore.screens || [] From 84c8507bad7e3ff8df572cf389e004f546be264b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 16:20:33 +0100 Subject: [PATCH 085/112] Renames --- packages/client/src/components/Component.svelte | 10 +++++----- .../components/error-states/ComponentErrorState.svelte | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index a86d43c676..236bcb7c7e 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -103,7 +103,7 @@ let settingsDefinition let settingsDefinitionMap let missingRequiredSettings = false - let invalidSettings = false + let componentErrors = false // Temporary styles which can be added in the app preview for things like // DND. We clear these whenever a new instance is received. @@ -138,12 +138,12 @@ // Derive definition properties which can all be optional, so need to be // coerced to booleans - $: invalidSettings = instance?._meta?.errors + $: componentErrors = instance?._meta?.errors $: hasChildren = !!definition?.hasChildren $: showEmptyState = definition?.showEmptyState !== false $: hasMissingRequiredSettings = missingRequiredSettings?.length > 0 $: editable = !!definition?.editable && !hasMissingRequiredSettings - $: hasInvalidSettings = invalidSettings?.length > 0 + $: hasComponentErrors = componentErrors?.length > 0 $: requiredAncestors = definition?.requiredAncestors || [] $: missingRequiredAncestors = requiredAncestors.filter( ancestor => !$component.ancestors.includes(`${BudibasePrefix}${ancestor}`) @@ -152,7 +152,7 @@ $: errorState = hasMissingRequiredSettings || hasMissingRequiredAncestors || - hasInvalidSettings + hasComponentErrors // Interactive components can be selected, dragged and highlighted inside // the builder preview @@ -698,7 +698,7 @@ {:else} diff --git a/packages/client/src/components/error-states/ComponentErrorState.svelte b/packages/client/src/components/error-states/ComponentErrorState.svelte index 8cf1ad6dfb..9eace07018 100644 --- a/packages/client/src/components/error-states/ComponentErrorState.svelte +++ b/packages/client/src/components/error-states/ComponentErrorState.svelte @@ -8,7 +8,7 @@ | { key: string; label: string }[] | undefined export let missingRequiredAncestors: string[] | undefined - export let invalidSettings: string[] | undefined + export let componentErrors: string[] | undefined const component = getContext("component") const { styleable, builderStore } = getContext("sdk") @@ -16,7 +16,7 @@ $: styles = { ...$component.styles, normal: {}, custom: null, empty: true } $: requiredSetting = missingRequiredSettings?.[0] $: requiredAncestor = missingRequiredAncestors?.[0] - $: invalidSetting = invalidSettings?.[0] + $: errorMessage = componentErrors?.[0] {#if $builderStore.inBuilder} @@ -25,8 +25,8 @@ {#if requiredAncestor} - {:else if invalidSetting} - {invalidSetting} + {:else if errorMessage} + {errorMessage} {:else if requiredSetting} {/if} From c9feae9665a33466e9486c7ca97a2f42b067142c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 16:26:09 +0100 Subject: [PATCH 086/112] Simplify derived screenComponentErrors --- .../[screenId]/_components/AppPreview.svelte | 4 +- packages/builder/src/stores/builder/index.js | 4 +- .../src/stores/builder/screenComponent.ts | 112 +++++++----------- 3 files changed, 49 insertions(+), 71 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index fc0b67f63d..3951c0e902 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -11,7 +11,7 @@ selectedScreen, hoverStore, componentTreeNodesStore, - screenComponentStore, + screenComponentErrors, snippets, } from "@/stores/builder" import ConfirmDialog from "@/components/common/ConfirmDialog.svelte" @@ -69,7 +69,7 @@ port: window.location.port, }, snippets: $snippets, - componentErrors: $screenComponentStore.errors, + componentErrors: $screenComponentErrors, } // Refresh the preview when required diff --git a/packages/builder/src/stores/builder/index.js b/packages/builder/src/stores/builder/index.js index 23491996d1..892b72c7ab 100644 --- a/packages/builder/src/stores/builder/index.js +++ b/packages/builder/src/stores/builder/index.js @@ -16,7 +16,7 @@ import { userStore, userSelectedResourceMap, isOnlyUser } from "./users.js" import { deploymentStore } from "./deployments.js" import { contextMenuStore } from "./contextMenu.js" import { snippets } from "./snippets" -import { screenComponentStore } from "./screenComponent" +import { screenComponentErrors } from "./screenComponent" // Backend import { tables } from "./tables" @@ -68,7 +68,7 @@ export { snippets, rowActions, appPublished, - screenComponentStore, + screenComponentErrors, } export const reset = () => { diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index b4eb01a3b7..19bafeade3 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -2,79 +2,57 @@ import { derived } from "svelte/store" import { tables } from "./tables" import { selectedScreen } from "./screens" import { viewsV2 } from "./viewsV2" -import { DerivedBudiStore } from "../BudiStore" import { findComponentsBySettingsType } from "@/helpers/screen" import { Screen, Table, ViewV2 } from "@budibase/types" -interface BuilderScreenComponentStore {} +export const screenComponentErrors = derived( + [selectedScreen, tables, viewsV2], + ([$selectedScreen, $tables, $viewsV2]): Record => { + function flattenTablesAndViews(tables: Table[], views: ViewV2[]) { + return { + ...tables.reduce( + (list, table) => ({ + ...list, + [table._id!]: table, + }), + {} + ), + ...views.reduce( + (list, view) => ({ + ...list, + [view.id]: view, + }), + {} + ), + } + } -interface DerivedScreenComponentStore extends BuilderScreenComponentStore { - errors: Record -} + function getInvalidDatasources( + screen: Screen, + datasources: Record + ) { + const friendlyNameByType = { + table: "table", + view: "view", + viewV2: "view", + } -export class ScreenComponentStore extends DerivedBudiStore< - BuilderScreenComponentStore, - DerivedScreenComponentStore -> { - constructor() { - const makeDerivedStore = () => { - return derived( - [selectedScreen, tables, viewsV2], - ([$selectedScreen, $tables, $viewsV2]): DerivedScreenComponentStore => { - const datasources = flattenTablesAndViews($tables.list, $viewsV2.list) - return { - errors: getInvalidDatasources($selectedScreen, datasources), - } + const result: Record = {} + for (const component of findComponentsBySettingsType(screen, "table")) { + const { resourceId, type, label } = component.dataSource + if (!datasources[resourceId]) { + const friendlyTypeName = + friendlyNameByType[type as keyof typeof friendlyNameByType] + result[component._id!] = [ + `The ${friendlyTypeName} named "${label}" does not exist`, + ] } - ) + } + + return result } - super({}, makeDerivedStore) + const datasources = flattenTablesAndViews($tables.list, $viewsV2.list) + return getInvalidDatasources($selectedScreen, datasources) } -} - -export const screenComponentStore = new ScreenComponentStore() - -function flattenTablesAndViews(tables: Table[], views: ViewV2[]) { - return { - ...tables.reduce( - (list, table) => ({ - ...list, - [table._id!]: table, - }), - {} - ), - ...views.reduce( - (list, view) => ({ - ...list, - [view.id]: view, - }), - {} - ), - } -} - -function getInvalidDatasources( - screen: Screen, - datasources: Record -) { - const friendlyNameByType = { - table: "table", - view: "view", - viewV2: "view", - } - - const result: Record = {} - for (const component of findComponentsBySettingsType(screen, "table")) { - const { resourceId, type, label } = component.dataSource - if (!datasources[resourceId]) { - const friendlyTypeName = - friendlyNameByType[type as keyof typeof friendlyNameByType] - result[component._id!] = [ - `The ${friendlyTypeName} named "${label}" does not exist`, - ] - } - } - - return result -} +) From 2a5865ecaf6eb9de7a5a6e808bb0580e792ae630 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 16:38:29 +0100 Subject: [PATCH 087/112] Fix creating new table screen modal --- packages/builder/src/helpers/data/format.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/helpers/data/format.js b/packages/builder/src/helpers/data/format.js index 428ce273b2..ba274d5464 100644 --- a/packages/builder/src/helpers/data/format.js +++ b/packages/builder/src/helpers/data/format.js @@ -11,7 +11,7 @@ export const datasourceSelect = { }, viewV2: (view, datasources) => { const datasource = datasources - .filter(f => f.entities) + ?.filter(f => f.entities) .flatMap(d => d.entities) .find(ds => ds._id === view.tableId) return { From abc1ba33356b3a2a8cb67ca586e53c8e712d74dd Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 22 Jan 2025 15:46:49 +0000 Subject: [PATCH 088/112] Bump version to 3.3.1 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index a7d534ff01..13040cb50c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.3.0", + "version": "3.3.1", "npmClient": "yarn", "concurrency": 20, "command": { From fbbe1738db7a4006e7bbe8dfea73de26e7b4c33d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 22 Jan 2025 16:11:54 +0000 Subject: [PATCH 089/112] Using sandboxed iframe to limit to scripting only. --- packages/string-templates/package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 74d9aaa85a..b1b4b9ef55 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -23,7 +23,7 @@ }, "dependencies": { "@budibase/handlebars-helpers": "^0.13.2", - "@budibase/vm-browserify": "^1.1.3", + "@budibase/vm-browserify": "^1.1.4", "dayjs": "^1.10.8", "handlebars": "^4.7.8", "lodash.clonedeep": "^4.5.0" diff --git a/yarn.lock b/yarn.lock index e25ff97747..a375c05ffd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2152,10 +2152,10 @@ scim-patch "^0.8.1" scim2-parse-filter "^0.2.8" -"@budibase/vm-browserify@^1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@budibase/vm-browserify/-/vm-browserify-1.1.3.tgz#63f7917671a0f0cb760e3aa37cfd5dfa32e997ed" - integrity sha512-CuoNb2xwS8TT2ZfG9YqC8QCTcG3ZPLwH4m00sfPDluwmdp3U3HGg/UKWRIqKC6Wv8Mywy1q6bxmSx6Vf40V52w== +"@budibase/vm-browserify@^1.1.4": + version "1.1.4" + resolved "https://registry.yarnpkg.com/@budibase/vm-browserify/-/vm-browserify-1.1.4.tgz#eecb001bd9521cb7647e26fb4d2d29d0a4dce262" + integrity sha512-/dyOLj+jQNKe6sVfLP6NdwA79OZxEWHCa41VGsjKJC9DYo6l2fEcL5BNXq2pATqrbgWmOlEbcRulfZ+7W0QRUg== dependencies: indexof "^0.0.1" From 5c9cc915ff9c7e1da7538e282c02b6503bd006e3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 17:32:34 +0100 Subject: [PATCH 090/112] Remove magic string for settings --- packages/builder/src/helpers/screen.ts | 30 ++++++++++++++----- .../src/stores/builder/screenComponent.ts | 7 +++-- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/packages/builder/src/helpers/screen.ts b/packages/builder/src/helpers/screen.ts index e83805a511..71623844de 100644 --- a/packages/builder/src/helpers/screen.ts +++ b/packages/builder/src/helpers/screen.ts @@ -2,7 +2,13 @@ import { Component, Screen, ScreenProps } from "@budibase/types" import clientManifest from "@budibase/client/manifest.json" export function findComponentsBySettingsType(screen: Screen, type: string) { - const result: Component[] = [] + const result: { + component: Component + setting: { + type: string + key: string + } + }[] = [] function recurseFieldComponentsInChildren( component: ScreenProps, type: string @@ -11,14 +17,15 @@ export function findComponentsBySettingsType(screen: Screen, type: string) { return } - const componentType = component._component.split("/").slice(-1)[0] - const definition = - clientManifest[componentType as keyof typeof clientManifest] - if ( + const definition = getManifestDefinition(component) + const setting = "settings" in definition && - definition.settings.some((s: any) => s.type === type) - ) { - result.push(component) + definition.settings.find((s: any) => s.type === type) + if (setting && "type" in setting) { + result.push({ + component, + setting: { type: setting.type!, key: setting.key! }, + }) } component._children?.forEach(child => { recurseFieldComponentsInChildren(child, type) @@ -28,3 +35,10 @@ export function findComponentsBySettingsType(screen: Screen, type: string) { recurseFieldComponentsInChildren(screen?.props, type) return result } + +function getManifestDefinition(component: Component) { + const componentType = component._component.split("/").slice(-1)[0] + const definition = + clientManifest[componentType as keyof typeof clientManifest] + return definition +} diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 19bafeade3..a061158e6a 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -38,8 +38,11 @@ export const screenComponentErrors = derived( } const result: Record = {} - for (const component of findComponentsBySettingsType(screen, "table")) { - const { resourceId, type, label } = component.dataSource + for (const { component, setting } of findComponentsBySettingsType( + screen, + "table" + )) { + const { resourceId, type, label } = component[setting.key] if (!datasources[resourceId]) { const friendlyTypeName = friendlyNameByType[type as keyof typeof friendlyNameByType] From 5d60da471484ab190129f06490f3a1aec9744639 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 11:40:18 +0100 Subject: [PATCH 091/112] Fix null reference on viewV1 get schema --- packages/frontend-core/src/fetch/ViewFetch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/fetch/ViewFetch.ts b/packages/frontend-core/src/fetch/ViewFetch.ts index 6555896ae8..720f91eaab 100644 --- a/packages/frontend-core/src/fetch/ViewFetch.ts +++ b/packages/frontend-core/src/fetch/ViewFetch.ts @@ -21,7 +21,7 @@ export default class ViewFetch extends BaseDataFetch { getSchema(definition: Table) { const { datasource } = this.options - return definition?.views?.[datasource.name]?.schema + return definition?.views?.[datasource?.name]?.schema } async getData() { From fe2e93ee3e35ac35a64faf6569603eb80e8ac261 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 11:53:09 +0100 Subject: [PATCH 092/112] Remove todos --- .../src/components/grid/stores/datasource.ts | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.ts b/packages/frontend-core/src/components/grid/stores/datasource.ts index 588f373152..ebeade1050 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.ts +++ b/packages/frontend-core/src/components/grid/stores/datasource.ts @@ -1,5 +1,3 @@ -// TODO: datasource and defitions are unions of the different implementations. At this point, the datasource does not know what type is being used, and the assignations will cause TS exceptions. Casting it "as any" for now. This should be fixed improving the type usages. - import { derived, get, Readable, Writable } from "svelte/store" import { DataFetchDefinition, @@ -10,12 +8,10 @@ import { enrichSchemaWithRelColumns, memo } from "../../../utils" import { cloneDeep } from "lodash" import { SaveRowRequest, - SaveTableRequest, UIDatasource, UIFieldMutation, UIFieldSchema, UIRow, - UpdateViewRequest, ViewV2Type, } from "@budibase/types" import { Store as StoreContext, BaseStoreProps } from "." @@ -79,7 +75,7 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => { const schema = derived(definition, $definition => { const schema: Record | undefined = getDatasourceSchema({ API, - datasource: get(datasource) as any, // TODO: see line 1 + datasource: get(datasource), definition: $definition ?? undefined, }) if (!schema) { @@ -137,7 +133,7 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => { let type = $datasource?.type // @ts-expect-error if (type === "provider") { - type = ($datasource as any).value?.datasource?.type // TODO: see line 1 + type = ($datasource as any).value?.datasource?.type } // Handle calculation views if ( @@ -196,15 +192,13 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => { const refreshDefinition = async () => { const def = await getDatasourceDefinition({ API, - datasource: get(datasource) as any, // TODO: see line 1 + datasource: get(datasource), }) - definition.set(def as any) // TODO: see line 1 + definition.set(def ?? null) } // Saves the datasource definition - const saveDefinition = async ( - newDefinition: SaveTableRequest | UpdateViewRequest - ) => { + const saveDefinition = async (newDefinition: DataFetchDefinition) => { // Update local state const originalDefinition = get(definition) definition.set(newDefinition) @@ -212,7 +206,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => { // Update server if (get(config).canSaveSchema) { try { - await getAPI()?.actions.saveDefinition(newDefinition as never) + await getAPI()?.actions.saveDefinition(newDefinition) // Broadcast change so external state can be updated, as this change // will not be received by the builder websocket because we caused it @@ -245,7 +239,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => { delete newDefinition.schema[column].default } } - return await saveDefinition(newDefinition as any) // TODO: see line 1 + return await saveDefinition(newDefinition) } // Adds a schema mutation for a single field @@ -321,7 +315,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => { await saveDefinition({ ...$definition, schema: newSchema, - } as any) // TODO: see line 1 + }) resetSchemaMutations() } From 5f508ad7ca35b45151fdda4f931a46c1c476a714 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 12:07:12 +0100 Subject: [PATCH 093/112] Fix type --- packages/frontend-core/src/components/grid/stores/datasource.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.ts b/packages/frontend-core/src/components/grid/stores/datasource.ts index ebeade1050..5934c7c636 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.ts +++ b/packages/frontend-core/src/components/grid/stores/datasource.ts @@ -206,7 +206,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => { // Update server if (get(config).canSaveSchema) { try { - await getAPI()?.actions.saveDefinition(newDefinition) + await getAPI()?.actions.saveDefinition(newDefinition as never) // Broadcast change so external state can be updated, as this change // will not be received by the builder websocket because we caused it From 19bfd71096788096e584018088ca3514ed266282 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 12:25:10 +0100 Subject: [PATCH 094/112] Update types --- packages/frontend-core/src/fetch/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts index e12e74340c..547043145d 100644 --- a/packages/frontend-core/src/fetch/index.ts +++ b/packages/frontend-core/src/fetch/index.ts @@ -101,12 +101,12 @@ export const fetchData = < // Creates an empty fetch instance with no datasource configured, so no data // will initially be loaded -const createEmptyFetchInstance = ({ +const createEmptyFetchInstance = ({ API, datasource, }: { API: APIClient - datasource: DataFetchDatasource + datasource: T }) => { const handler = DataFetchMap[datasource?.type] if (!handler) { @@ -114,7 +114,7 @@ const createEmptyFetchInstance = ({ } return new handler({ API, - datasource: null as never, + datasource: datasource as any, query: null as any, }) } From 4c48ad6526d46308a22e7b38733447d07fbe30ec Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 15:26:18 +0100 Subject: [PATCH 095/112] Fix selected item for views v1 --- .../DataSourceSelect/DataSourceCategory.svelte | 16 ++++++++++++++-- .../DataSourceSelect/DataSourceSelect.svelte | 1 + 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte index 72e2fbf638..4ea8c63087 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte @@ -7,8 +7,21 @@ export let dataSet export let value export let onSelect + export let identifiers = ["resourceId"] $: displayDatasourceName = $datasources.list.length > 1 + + function isSelected(entry) { + if (!identifiers.length) { + return false + } + for (const identifier of identifiers) { + if (entry[identifier] !== value?.[identifier]) { + return false + } + } + return true + } {#if dividerState} @@ -24,8 +37,7 @@ {#each dataSet as data}
  • {/if} {#if queries?.length} From 8f02dff5cdfcbbf7f93f3d85be95b2cdeff1cef8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 15:29:59 +0100 Subject: [PATCH 096/112] Fix link selector --- .../settings/controls/DataSourceSelect/DataSourceSelect.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index 7c13032e42..1b7acd4a11 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -310,6 +310,7 @@ dataSet={links} {value} onSelect={handleSelected} + identifiers={["tableId", "fieldName"]} /> {/if} {#if fields?.length} From 102fbe9372e05e0a570bdc47068e15cd99309a2e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 15:31:07 +0100 Subject: [PATCH 097/112] Fix provider selector --- .../settings/controls/DataSourceSelect/DataSourceSelect.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index 1b7acd4a11..f8ee0876e7 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -338,6 +338,7 @@ dataSet={dataProviders} {value} onSelect={handleSelected} + identifiers={["providerId"]} /> {/if} Date: Thu, 23 Jan 2025 15:38:09 +0100 Subject: [PATCH 098/112] Fix fields selector --- .../settings/controls/DataSourceSelect/DataSourceSelect.svelte | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index f8ee0876e7..1b9fdcdf10 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -301,6 +301,7 @@ dataSet={queries} {value} onSelect={handleSelected} + identifiers={["_id"]} /> {/if} {#if links?.length} @@ -320,6 +321,7 @@ dataSet={fields} {value} onSelect={handleSelected} + identifiers={["providerId", "tableId", "fieldName"]} /> {/if} {#if jsonArrays?.length} @@ -329,6 +331,7 @@ dataSet={jsonArrays} {value} onSelect={handleSelected} + identifiers={["providerId", "tableId", "fieldName"]} /> {/if} {#if showDataProviders && dataProviders?.length} From 10a669e1d7a68093fa36f603d951ba653246ab73 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 17:16:11 +0100 Subject: [PATCH 099/112] Support searching for multiple types --- packages/builder/src/helpers/screen.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/builder/src/helpers/screen.ts b/packages/builder/src/helpers/screen.ts index 71623844de..296a597adb 100644 --- a/packages/builder/src/helpers/screen.ts +++ b/packages/builder/src/helpers/screen.ts @@ -1,7 +1,12 @@ import { Component, Screen, ScreenProps } from "@budibase/types" import clientManifest from "@budibase/client/manifest.json" -export function findComponentsBySettingsType(screen: Screen, type: string) { +export function findComponentsBySettingsType( + screen: Screen, + type: string | string[] +) { + const typesArray = Array.isArray(type) ? type : [type] + const result: { component: Component setting: { @@ -9,10 +14,7 @@ export function findComponentsBySettingsType(screen: Screen, type: string) { key: string } }[] = [] - function recurseFieldComponentsInChildren( - component: ScreenProps, - type: string - ) { + function recurseFieldComponentsInChildren(component: ScreenProps) { if (!component) { return } @@ -20,7 +22,7 @@ export function findComponentsBySettingsType(screen: Screen, type: string) { const definition = getManifestDefinition(component) const setting = "settings" in definition && - definition.settings.find((s: any) => s.type === type) + definition.settings.find((s: any) => typesArray.includes(s.type)) if (setting && "type" in setting) { result.push({ component, @@ -28,11 +30,11 @@ export function findComponentsBySettingsType(screen: Screen, type: string) { }) } component._children?.forEach(child => { - recurseFieldComponentsInChildren(child, type) + recurseFieldComponentsInChildren(child) }) } - recurseFieldComponentsInChildren(screen?.props, type) + recurseFieldComponentsInChildren(screen?.props) return result } From 1c23763813e49d9b320a12faa821f09d0f815c45 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 17:38:08 +0100 Subject: [PATCH 100/112] Handle dataSources as well --- packages/builder/src/stores/builder/screenComponent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index a061158e6a..426fcbc58a 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -40,7 +40,7 @@ export const screenComponentErrors = derived( const result: Record = {} for (const { component, setting } of findComponentsBySettingsType( screen, - "table" + ["table", "dataSource"] )) { const { resourceId, type, label } = component[setting.key] if (!datasources[resourceId]) { From 1f3c466028ca1166ea94d4714e2e27e80803d4cf Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 18:00:03 +0100 Subject: [PATCH 101/112] DRY --- .../src/stores/builder/screenComponent.ts | 44 +++++++++---------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 426fcbc58a..6db9f43241 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -3,37 +3,29 @@ import { tables } from "./tables" import { selectedScreen } from "./screens" import { viewsV2 } from "./viewsV2" import { findComponentsBySettingsType } from "@/helpers/screen" -import { Screen, Table, ViewV2 } from "@budibase/types" +import { Screen } from "@budibase/types" + +function reduceBy( + key: TKey, + list: TItem[] +) { + return list.reduce( + (result, item) => ({ + ...result, + [item[key] as string]: item, + }), + {} + ) +} export const screenComponentErrors = derived( [selectedScreen, tables, viewsV2], ([$selectedScreen, $tables, $viewsV2]): Record => { - function flattenTablesAndViews(tables: Table[], views: ViewV2[]) { - return { - ...tables.reduce( - (list, table) => ({ - ...list, - [table._id!]: table, - }), - {} - ), - ...views.reduce( - (list, view) => ({ - ...list, - [view.id]: view, - }), - {} - ), - } - } - function getInvalidDatasources( screen: Screen, datasources: Record ) { const friendlyNameByType = { - table: "table", - view: "view", viewV2: "view", } @@ -45,7 +37,7 @@ export const screenComponentErrors = derived( const { resourceId, type, label } = component[setting.key] if (!datasources[resourceId]) { const friendlyTypeName = - friendlyNameByType[type as keyof typeof friendlyNameByType] + friendlyNameByType[type as keyof typeof friendlyNameByType] ?? type result[component._id!] = [ `The ${friendlyTypeName} named "${label}" does not exist`, ] @@ -55,7 +47,11 @@ export const screenComponentErrors = derived( return result } - const datasources = flattenTablesAndViews($tables.list, $viewsV2.list) + const datasources = { + ...reduceBy("_id", $tables.list), + ...reduceBy("id", $viewsV2.list), + } + return getInvalidDatasources($selectedScreen, datasources) } ) From 41b65a6b1df471d37fd04e9146f2ea7e9057f7dc Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 11:03:52 +0100 Subject: [PATCH 102/112] Validate queries --- .../src/stores/builder/screenComponent.ts | 29 +++++++++++++++---- .../types/src/documents/app/datasource.ts | 2 ++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 6db9f43241..a9444aee8c 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -3,7 +3,8 @@ import { tables } from "./tables" import { selectedScreen } from "./screens" import { viewsV2 } from "./viewsV2" import { findComponentsBySettingsType } from "@/helpers/screen" -import { Screen } from "@budibase/types" +import { DatasourceType, Screen } from "@budibase/types" +import { queries } from "./queries" function reduceBy( key: TKey, @@ -19,22 +20,39 @@ function reduceBy( } export const screenComponentErrors = derived( - [selectedScreen, tables, viewsV2], - ([$selectedScreen, $tables, $viewsV2]): Record => { + [selectedScreen, tables, viewsV2, queries], + ([$selectedScreen, $tables, $viewsV2, $queries]): Record< + string, + string[] + > => { function getInvalidDatasources( screen: Screen, datasources: Record ) { - const friendlyNameByType = { + const friendlyNameByType: Partial> = { viewV2: "view", } + const primaryKeyByType: Record = { + table: "resourceId", + view: "TODO", + viewV2: "resourceId", + query: "_id", + custom: "" as never, + } + const result: Record = {} for (const { component, setting } of findComponentsBySettingsType( screen, ["table", "dataSource"] )) { - const { resourceId, type, label } = component[setting.key] + const componentSettings = component[setting.key] + const { type, label } = componentSettings + if (type === "custom") { + continue + } + const resourceId = + componentSettings[primaryKeyByType[type as DatasourceType]] if (!datasources[resourceId]) { const friendlyTypeName = friendlyNameByType[type as keyof typeof friendlyNameByType] ?? type @@ -50,6 +68,7 @@ export const screenComponentErrors = derived( const datasources = { ...reduceBy("_id", $tables.list), ...reduceBy("id", $viewsV2.list), + ...reduceBy("_id", $queries.list), } return getInvalidDatasources($selectedScreen, datasources) diff --git a/packages/types/src/documents/app/datasource.ts b/packages/types/src/documents/app/datasource.ts index a0be7bd80d..27828c5455 100644 --- a/packages/types/src/documents/app/datasource.ts +++ b/packages/types/src/documents/app/datasource.ts @@ -57,3 +57,5 @@ export interface RestConfig { } dynamicVariables?: DynamicVariable[] } + +export type DatasourceType = "table" | "view" | "viewV2" | "query" | "custom" From 6e615a9907b0dc4508ccb921e5113bf3fa93779e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 11:19:49 +0100 Subject: [PATCH 103/112] Validate views v1 --- .../builder/src/stores/builder/screenComponent.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index a9444aee8c..2803acc953 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -5,6 +5,7 @@ import { viewsV2 } from "./viewsV2" import { findComponentsBySettingsType } from "@/helpers/screen" import { DatasourceType, Screen } from "@budibase/types" import { queries } from "./queries" +import { views } from "./views" function reduceBy( key: TKey, @@ -20,8 +21,8 @@ function reduceBy( } export const screenComponentErrors = derived( - [selectedScreen, tables, viewsV2, queries], - ([$selectedScreen, $tables, $viewsV2, $queries]): Record< + [selectedScreen, tables, views, viewsV2, queries], + ([$selectedScreen, $tables, $views, $viewsV2, $queries]): Record< string, string[] > => { @@ -34,9 +35,9 @@ export const screenComponentErrors = derived( } const primaryKeyByType: Record = { - table: "resourceId", - view: "TODO", - viewV2: "resourceId", + table: "tableId", + view: "name", + viewV2: "id", query: "_id", custom: "" as never, } @@ -67,6 +68,7 @@ export const screenComponentErrors = derived( const datasources = { ...reduceBy("_id", $tables.list), + ...reduceBy("name", $views.list), ...reduceBy("id", $viewsV2.list), ...reduceBy("_id", $queries.list), } From 92e2ae46f54f7e2f88baec2a72a08613106a4c76 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 16:17:08 +0100 Subject: [PATCH 104/112] Center error message --- .../src/components/error-states/ComponentErrorState.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/components/error-states/ComponentErrorState.svelte b/packages/client/src/components/error-states/ComponentErrorState.svelte index 9eace07018..7069b7a431 100644 --- a/packages/client/src/components/error-states/ComponentErrorState.svelte +++ b/packages/client/src/components/error-states/ComponentErrorState.svelte @@ -38,7 +38,7 @@ .component-placeholder { display: flex; flex-direction: row; - justify-content: flex-start; + justify-content: center; align-items: center; color: var(--spectrum-global-color-gray-600); font-size: var(--font-size-s); From afb9f86bf7aa2f7b10814e215da1afc5ecf3993b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 23 Jan 2025 17:23:08 +0000 Subject: [PATCH 105/112] Using node vm for string-template test cases. --- packages/string-templates/src/index.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index bd008dd4d2..79b0aa0699 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -1,4 +1,5 @@ -import vm from "@budibase/vm-browserify" +import browserVM from "@budibase/vm-browserify" +import vm from "vm" import { create, TemplateDelegate } from "handlebars" import { registerAll, registerMinimum } from "./helpers/index" import { postprocess, postprocessWithLogs, preprocess } from "./processors" @@ -14,10 +15,10 @@ import { } from "./utilities" import { convertHBSBlock } from "./conversion" import { removeJSRunner, setJSRunner } from "./helpers/javascript" - import manifest from "./manifest.json" import { Log, ProcessOptions } from "./types" import { UserScriptError } from "./errors" +import { isTest } from "./environment" export type { Log, LogType } from "./types" export { setTestingBackendJS } from "./environment" @@ -507,15 +508,15 @@ export function convertToJS(hbs: string) { export { JsTimeoutError, UserScriptError } from "./errors" export function browserJSSetup() { - /** - * Use polyfilled vm to run JS scripts in a browser Env - */ + // tests are in jest - we need to use node VM for these + const jsSandbox = isTest() ? vm : browserVM + // Use polyfilled vm to run JS scripts in a browser Env setJSRunner((js: string, context: Record) => { - vm.createContext(context) + jsSandbox.createContext(context) const wrappedJs = frontendWrapJS(js) - const result = vm.runInNewContext(wrappedJs, context) + const result = jsSandbox.runInNewContext(wrappedJs, context) if (result.error) { throw new UserScriptError(result.error) } From cf9b61c8c80a889c41a3cc017e431239aa70d3fc Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 23 Jan 2025 17:35:50 +0000 Subject: [PATCH 106/112] Fix selecting 'old row' in row updated trigger. --- .../automation/SetupPanel/AutomationBlockSetup.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 16183ea59a..e713d9bc85 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -293,7 +293,7 @@ type: RowSelector, props: { row: inputData["oldRow"] || { - tableId: inputData["row"].tableId, + tableId: inputData["row"]?.tableId, }, meta: { fields: inputData["meta"]?.oldFields || {}, From 388a94aee1f7b13e2baf3527a89103efff649499 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 19:34:03 +0100 Subject: [PATCH 107/112] Add feature flag for datasource setting checks --- packages/builder/src/helpers/index.ts | 1 + packages/builder/src/stores/builder/screenComponent.ts | 6 ++++++ packages/types/src/sdk/featureFlag.ts | 2 ++ 3 files changed, 9 insertions(+) diff --git a/packages/builder/src/helpers/index.ts b/packages/builder/src/helpers/index.ts index 687723e361..81afc696a3 100644 --- a/packages/builder/src/helpers/index.ts +++ b/packages/builder/src/helpers/index.ts @@ -9,3 +9,4 @@ export { lowercase, isBuilderInputFocused, } from "./helpers" +export * as featureFlag from "./featureFlags" diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index a061158e6a..079401891c 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -5,9 +5,15 @@ import { viewsV2 } from "./viewsV2" import { findComponentsBySettingsType } from "@/helpers/screen" import { Screen, Table, ViewV2 } from "@budibase/types" +import { featureFlag } from "@/helpers" + export const screenComponentErrors = derived( [selectedScreen, tables, viewsV2], ([$selectedScreen, $tables, $viewsV2]): Record => { + if (!featureFlag.isEnabled("CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS")) { + return {} + } + function flattenTablesAndViews(tables: Table[], views: ViewV2[]) { return { ...tables.reduce( diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts index 996d3bba8d..d9f092c80a 100644 --- a/packages/types/src/sdk/featureFlag.ts +++ b/packages/types/src/sdk/featureFlag.ts @@ -1,5 +1,6 @@ export enum FeatureFlag { USE_ZOD_VALIDATOR = "USE_ZOD_VALIDATOR", + CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS = "CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS", // Account-portal DIRECT_LOGIN_TO_ACCOUNT_PORTAL = "DIRECT_LOGIN_TO_ACCOUNT_PORTAL", @@ -7,6 +8,7 @@ export enum FeatureFlag { export const FeatureFlagDefaults = { [FeatureFlag.USE_ZOD_VALIDATOR]: false, + [FeatureFlag.CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS]: false, // Account-portal [FeatureFlag.DIRECT_LOGIN_TO_ACCOUNT_PORTAL]: false, From afee1ac993a4547791f8504d975f447ac40fd70b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 24 Jan 2025 09:42:29 +0100 Subject: [PATCH 108/112] Whitelist checks --- .../src/stores/builder/screenComponent.ts | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 2803acc953..6578334310 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -20,6 +20,18 @@ function reduceBy( ) } +const friendlyNameByType: Partial> = { + viewV2: "view", +} + +const validationKeyByType: Record = { + table: "tableId", + view: "name", + viewV2: "id", + query: "_id", + custom: null, +} + export const screenComponentErrors = derived( [selectedScreen, tables, views, viewsV2, queries], ([$selectedScreen, $tables, $views, $viewsV2, $queries]): Record< @@ -30,35 +42,24 @@ export const screenComponentErrors = derived( screen: Screen, datasources: Record ) { - const friendlyNameByType: Partial> = { - viewV2: "view", - } - - const primaryKeyByType: Record = { - table: "tableId", - view: "name", - viewV2: "id", - query: "_id", - custom: "" as never, - } - const result: Record = {} for (const { component, setting } of findComponentsBySettingsType( screen, ["table", "dataSource"] )) { const componentSettings = component[setting.key] - const { type, label } = componentSettings - if (type === "custom") { + const { label } = componentSettings + const type = componentSettings as DatasourceType + + const validationKey = validationKeyByType[type] + if (!validationKey) { continue } - const resourceId = - componentSettings[primaryKeyByType[type as DatasourceType]] + const resourceId = componentSettings[validationKey] if (!datasources[resourceId]) { - const friendlyTypeName = - friendlyNameByType[type as keyof typeof friendlyNameByType] ?? type + const friendlyTypeName = friendlyNameByType[type] ?? type result[component._id!] = [ - `The ${friendlyTypeName} named "${label}" does not exist`, + `The ${friendlyTypeName} named "${label}" could not be found`, ] } } From 8497a060407870973d8fe0494b71056df00d932c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 24 Jan 2025 09:53:37 +0100 Subject: [PATCH 109/112] Fix --- packages/builder/src/stores/builder/screenComponent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 6578334310..6fb1a7d134 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -49,7 +49,7 @@ export const screenComponentErrors = derived( )) { const componentSettings = component[setting.key] const { label } = componentSettings - const type = componentSettings as DatasourceType + const type = componentSettings.type as DatasourceType const validationKey = validationKeyByType[type] if (!validationKey) { From 1f28bf978b79e804d6dbf5bb44db9a4f887574e8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 24 Jan 2025 10:23:02 +0100 Subject: [PATCH 110/112] Move type to ui --- packages/builder/src/stores/builder/screenComponent.ts | 8 ++++---- packages/types/src/documents/app/datasource.ts | 2 -- packages/types/src/ui/datasource.ts | 1 + packages/types/src/ui/index.ts | 1 + 4 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 packages/types/src/ui/datasource.ts diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 547a8170a8..d8169fdedb 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -3,7 +3,7 @@ import { tables } from "./tables" import { selectedScreen } from "./screens" import { viewsV2 } from "./viewsV2" import { findComponentsBySettingsType } from "@/helpers/screen" -import { DatasourceType, Screen } from "@budibase/types" +import { UIDatasourceType, Screen } from "@budibase/types" import { queries } from "./queries" import { views } from "./views" import { featureFlag } from "@/helpers" @@ -21,11 +21,11 @@ function reduceBy( ) } -const friendlyNameByType: Partial> = { +const friendlyNameByType: Partial> = { viewV2: "view", } -const validationKeyByType: Record = { +const validationKeyByType: Record = { table: "tableId", view: "name", viewV2: "id", @@ -53,7 +53,7 @@ export const screenComponentErrors = derived( )) { const componentSettings = component[setting.key] const { label } = componentSettings - const type = componentSettings.type as DatasourceType + const type = componentSettings.type as UIDatasourceType const validationKey = validationKeyByType[type] if (!validationKey) { diff --git a/packages/types/src/documents/app/datasource.ts b/packages/types/src/documents/app/datasource.ts index 27828c5455..a0be7bd80d 100644 --- a/packages/types/src/documents/app/datasource.ts +++ b/packages/types/src/documents/app/datasource.ts @@ -57,5 +57,3 @@ export interface RestConfig { } dynamicVariables?: DynamicVariable[] } - -export type DatasourceType = "table" | "view" | "viewV2" | "query" | "custom" diff --git a/packages/types/src/ui/datasource.ts b/packages/types/src/ui/datasource.ts new file mode 100644 index 0000000000..53740e8c4d --- /dev/null +++ b/packages/types/src/ui/datasource.ts @@ -0,0 +1 @@ +export type UIDatasourceType = "table" | "view" | "viewV2" | "query" | "custom" diff --git a/packages/types/src/ui/index.ts b/packages/types/src/ui/index.ts index 907f4ec0b5..6e5f37608c 100644 --- a/packages/types/src/ui/index.ts +++ b/packages/types/src/ui/index.ts @@ -2,3 +2,4 @@ export * from "./stores" export * from "./bindings" export * from "./components" export * from "./dataFetch" +export * from "./datasource" From bf05fea3edbeef0706dc6d28dbd74165ad87881b Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Fri, 24 Jan 2025 11:41:27 +0000 Subject: [PATCH 111/112] Removing while loop test - frontend JS cannot timeout. --- .../string-templates/test/javascript.spec.ts | 5 --- yarn.lock | 36 +++++-------------- 2 files changed, 8 insertions(+), 33 deletions(-) diff --git a/packages/string-templates/test/javascript.spec.ts b/packages/string-templates/test/javascript.spec.ts index 9134005acb..631fe828ae 100644 --- a/packages/string-templates/test/javascript.spec.ts +++ b/packages/string-templates/test/javascript.spec.ts @@ -125,11 +125,6 @@ describe("Javascript", () => { expect(processJS(`throw "Error"`)).toEqual("Error") }) - it("should timeout after one second", () => { - const output = processJS(`while (true) {}`) - expect(output).toBe("Timed out while executing JS") - }) - it("should prevent access to the process global", async () => { expect(processJS(`return process`)).toEqual( "ReferenceError: process is not defined" diff --git a/yarn.lock b/yarn.lock index a375c05ffd..8647f40b79 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11807,6 +11807,11 @@ husky@^8.0.3: resolved "https://registry.yarnpkg.com/husky/-/husky-8.0.3.tgz#4936d7212e46d1dea28fef29bb3a108872cd9184" integrity sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg== +husky@^9.1.4: + version "9.1.7" + resolved "https://registry.yarnpkg.com/husky/-/husky-9.1.7.tgz#d46a38035d101b46a70456a850ff4201344c0b2d" + integrity sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA== + ical-generator@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/ical-generator/-/ical-generator-4.1.0.tgz#2a336c951864c5583a2aa715d16f2edcdfd2d90b" @@ -18658,16 +18663,7 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -18759,7 +18755,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -18773,13 +18769,6 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -20527,7 +20516,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -20545,15 +20534,6 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From 8dd1366df27d82d15f3a05e71d4f6db57893408b Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Fri, 24 Jan 2025 13:02:00 +0000 Subject: [PATCH 112/112] Fixing yarn.lock (after removing account portal ref). --- yarn.lock | 5 ----- 1 file changed, 5 deletions(-) diff --git a/yarn.lock b/yarn.lock index 8647f40b79..60294cc5ab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11807,11 +11807,6 @@ husky@^8.0.3: resolved "https://registry.yarnpkg.com/husky/-/husky-8.0.3.tgz#4936d7212e46d1dea28fef29bb3a108872cd9184" integrity sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg== -husky@^9.1.4: - version "9.1.7" - resolved "https://registry.yarnpkg.com/husky/-/husky-9.1.7.tgz#d46a38035d101b46a70456a850ff4201344c0b2d" - integrity sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA== - ical-generator@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/ical-generator/-/ical-generator-4.1.0.tgz#2a336c951864c5583a2aa715d16f2edcdfd2d90b"