From 401bfa812cd9d464efe385e81d0bb02ec6bf7e97 Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 6 Jan 2025 11:45:53 +0000 Subject: [PATCH 001/106] Screen store to ts and test updates. Minor changes required for component store --- .../builder/src/stores/builder/components.ts | 39 ++- packages/builder/src/stores/builder/index.js | 2 +- .../stores/builder/{screens.js => screens.ts} | 169 ++++++---- .../src/stores/builder/tests/screens.test.js | 299 +++++++++--------- packages/frontend-core/src/utils/utils.js | 2 +- 5 files changed, 300 insertions(+), 211 deletions(-) rename packages/builder/src/stores/builder/{screens.js => screens.ts} (77%) diff --git a/packages/builder/src/stores/builder/components.ts b/packages/builder/src/stores/builder/components.ts index 9ad9a75f84..bce7fcb71d 100644 --- a/packages/builder/src/stores/builder/components.ts +++ b/packages/builder/src/stores/builder/components.ts @@ -33,7 +33,16 @@ import { Utils } from "@budibase/frontend-core" import { Component, FieldType, Screen, Table } from "@budibase/types" import { utils } from "@budibase/shared-core" -interface ComponentDefinition { +export interface ComponentState { + components: Record + customComponents: string[] + selectedComponentId?: string | null + componentToPaste?: Component | null + settingsCache: Record + selectedScreenId?: string | null +} + +export interface ComponentDefinition { component: string name: string friendlyName?: string @@ -41,9 +50,11 @@ interface ComponentDefinition { settings?: ComponentSetting[] features?: Record typeSupportPresets?: Record + legalDirectChildren: string[] + illegalChildren: string[] } -interface ComponentSetting { +export interface ComponentSetting { key: string type: string section?: string @@ -54,15 +65,6 @@ interface ComponentSetting { settings?: ComponentSetting[] } -interface ComponentState { - components: Record - customComponents: string[] - selectedComponentId: string | null - componentToPaste?: Component | null - settingsCache: Record - selectedScreenId?: string | null -} - export const INITIAL_COMPONENTS_STATE: ComponentState = { components: {}, customComponents: [], @@ -440,6 +442,11 @@ export class ComponentStore extends BudiStore { * @returns */ createInstance(componentName: string, presetProps: any, parent: any) { + const screen = get(selectedScreen) + if (!screen || !selectedScreen) { + throw "A valid screen must be selected" + } + const definition = this.getDefinition(componentName) if (!definition) { return null @@ -461,7 +468,7 @@ export class ComponentStore extends BudiStore { // Standard post processing this.enrichEmptySettings(instance, { parent, - screen: get(selectedScreen), + screen, useDefaultValues: true, }) @@ -481,7 +488,7 @@ export class ComponentStore extends BudiStore { // Add step name to form steps if (componentName.endsWith("/formstep")) { const parentForm = findClosestMatchingComponent( - get(selectedScreen).props, + screen.props, get(selectedComponent)._id, (component: Component) => component._component.endsWith("/form") ) @@ -841,6 +848,9 @@ export class ComponentStore extends BudiStore { const state = get(this.store) const componentId = state.selectedComponentId const screen = get(selectedScreen) + if (!screen) { + throw "A valid screen must be selected" + } const parent = findComponentParent(screen.props, componentId) const index = parent?._children.findIndex( (x: Component) => x._id === componentId @@ -890,6 +900,9 @@ export class ComponentStore extends BudiStore { const component = get(selectedComponent) const componentId = component?._id const screen = get(selectedScreen) + if (!screen) { + throw "A valid screen must be selected" + } const parent = findComponentParent(screen.props, componentId) const index = parent?._children.findIndex( (x: Component) => x._id === componentId diff --git a/packages/builder/src/stores/builder/index.js b/packages/builder/src/stores/builder/index.js index 08d87bebf5..0d4682b551 100644 --- a/packages/builder/src/stores/builder/index.js +++ b/packages/builder/src/stores/builder/index.js @@ -3,7 +3,7 @@ import { appStore } from "./app.js" import { componentStore, selectedComponent } from "./components" import { navigationStore } from "./navigation.js" import { themeStore } from "./theme.js" -import { screenStore, selectedScreen, sortedScreens } from "./screens.js" +import { screenStore, selectedScreen, sortedScreens } from "./screens" import { builderStore } from "./builder.js" import { hoverStore } from "./hover.js" import { previewStore } from "./preview.js" diff --git a/packages/builder/src/stores/builder/screens.js b/packages/builder/src/stores/builder/screens.ts similarity index 77% rename from packages/builder/src/stores/builder/screens.js rename to packages/builder/src/stores/builder/screens.ts index 8298a1469d..fd16cbfae8 100644 --- a/packages/builder/src/stores/builder/screens.js +++ b/packages/builder/src/stores/builder/screens.ts @@ -13,15 +13,32 @@ import { import { createHistoryStore } from "@/stores/builder/history" import { API } from "@/api" import { BudiStore } from "../BudiStore" +import { + FetchAppPackageResponse, + DeleteScreenResponse, + Screen, + Component, +} from "@budibase/types" +import { ComponentDefinition } from "./components" -export const INITIAL_SCREENS_STATE = { - screens: [], - selectedScreenId: null, +interface ScreenState { + screens: Screen[] + selectedScreenId?: string + selected?: Screen } -export class ScreenStore extends BudiStore { +export const initialScreenState: ScreenState = { + screens: [], +} + +// Review the nulls +export class ScreenStore extends BudiStore { + history: any + delete: any + save: any + constructor() { - super(INITIAL_SCREENS_STATE) + super(initialScreenState) // Bind scope this.select = this.select.bind(this) @@ -34,12 +51,15 @@ export class ScreenStore extends BudiStore { this.deleteScreen = this.deleteScreen.bind(this) this.syncScreenData = this.syncScreenData.bind(this) this.updateSetting = this.updateSetting.bind(this) + // TODO review this behaviour this.sequentialScreenPatch = this.sequentialScreenPatch.bind(this) this.removeCustomLayout = this.removeCustomLayout.bind(this) this.history = createHistoryStore({ - getDoc: id => get(this.store).screens?.find(screen => screen._id === id), + getDoc: (id: string) => + get(this.store).screens?.find(screen => screen._id === id), selectDoc: this.select, + beforeAction: () => {}, afterAction: () => { // Ensure a valid component is selected if (!get(selectedComponent)) { @@ -59,14 +79,14 @@ export class ScreenStore extends BudiStore { * Reset entire store back to base config */ reset() { - this.store.set({ ...INITIAL_SCREENS_STATE }) + this.store.set({ ...initialScreenState }) } /** * Replace ALL store screens with application package screens * @param {object} pkg */ - syncAppScreens(pkg) { + syncAppScreens(pkg: FetchAppPackageResponse) { this.update(state => ({ ...state, screens: [...pkg.screens], @@ -79,7 +99,7 @@ export class ScreenStore extends BudiStore { * @param {string} screenId * @returns */ - select(screenId) { + select(screenId: string) { // Check this screen exists const state = get(this.store) const screen = state.screens.find(screen => screen._id === screenId) @@ -107,14 +127,14 @@ export class ScreenStore extends BudiStore { * @throws Will throw an error containing the name of the component causing * the invalid screen state */ - validate(screen) { + validate(screen: Screen) { // Recursive function to find any illegal children in component trees const findIllegalChild = ( - component, - illegalChildren = [], - legalDirectChildren = [] - ) => { - const type = component._component + component: Component, + illegalChildren: string[] = [], + legalDirectChildren: string[] = [] + ): string | undefined => { + const type: string = component._component if (illegalChildren.includes(type)) { return type @@ -137,7 +157,13 @@ export class ScreenStore extends BudiStore { illegalChildren = [] } - const definition = componentStore.getDefinition(component._component) + const definition: ComponentDefinition | null = + componentStore.getDefinition(component._component) + + if (definition == null) { + throw `Invalid defintion ${component._component}` + } + // Reset whitelist for direct children legalDirectChildren = [] if (definition?.legalDirectChildren?.length) { @@ -172,7 +198,7 @@ export class ScreenStore extends BudiStore { const illegalChild = findIllegalChild(screen.props) if (illegalChild) { const def = componentStore.getDefinition(illegalChild) - throw `You can't place a ${def.name} here` + throw `You can't place a ${def?.name} here` } } @@ -183,7 +209,7 @@ export class ScreenStore extends BudiStore { * @param {object} screen * @returns {object} */ - async saveScreen(screen) { + async saveScreen(screen: Screen) { const appState = get(appStore) // Validate screen structure if the app supports it @@ -230,7 +256,7 @@ export class ScreenStore extends BudiStore { * After saving a screen, sync plugins and routes to the appStore * @param {object} savedScreen */ - async syncScreenData(savedScreen) { + async syncScreenData(savedScreen: Screen) { const appState = get(appStore) // If plugins changed we need to fetch the latest app metadata let usedPlugins = appState.usedPlugins @@ -256,28 +282,51 @@ export class ScreenStore extends BudiStore { * This is slightly better than just a traditional "patch" endpoint and this * supports deeply mutating the current doc rather than just appending data. */ - sequentialScreenPatch = Utils.sequential(async (patchFn, screenId) => { - const state = get(this.store) - const screen = state.screens.find(screen => screen._id === screenId) - if (!screen) { - return - } - let clone = cloneDeep(screen) - const result = patchFn(clone) + // sequentialScreenPatch = ( + // patchFn: (screen: Screen) => any, + // screenId: string + // ) => { + // return Utils.sequential(async () => { + // const state = get(this.store) + // const screen = state.screens.find(screen => screen._id === screenId) + // if (!screen) { + // return + // } + // let clone = cloneDeep(screen) + // const result = patchFn(clone) - // An explicit false result means skip this change - if (result === false) { - return + // // An explicit false result means skip this change + // if (result === false) { + // return + // } + // return this.save(clone) + // }) + // } + + sequentialScreenPatch = Utils.sequential( + async (patchFn: (screen: Screen) => any, screenId: string) => { + const state = get(this.store) + const screen = state.screens.find(screen => screen._id === screenId) + if (!screen) { + return + } + let clone = cloneDeep(screen) + const result = patchFn(clone) + + // An explicit false result means skip this change + if (result === false) { + return + } + return this.save(clone) } - return this.save(clone) - }) + ) /** * @param {function} patchFn * @param {string | null} screenId * @returns */ - async patch(patchFn, screenId) { + async patch(patchFn: (screen: Screen) => any, screenId?: string | null) { // Default to the currently selected screen if (!screenId) { const state = get(this.store) @@ -298,7 +347,7 @@ export class ScreenStore extends BudiStore { * @param {object} screen * @returns */ - async replace(screenId, screen) { + async replace(screenId: string, screen: Screen) { if (!screenId) { return } @@ -337,17 +386,25 @@ export class ScreenStore extends BudiStore { * @param {object | array} screens * @returns */ - async deleteScreen(screens) { + async deleteScreen(screens: Screen[]) { const screensToDelete = Array.isArray(screens) ? screens : [screens] // Build array of promises to speed up bulk deletions - let promises = [] - let deleteUrls = [] - screensToDelete.forEach(screen => { - // Delete the screen - promises.push(API.deleteScreen(screen._id, screen._rev)) - // Remove links to this screen - deleteUrls.push(screen.routing.route) - }) + let promises: Promise[] = [] + let deleteUrls: string[] = [] + + // In this instance _id will have been set + // Underline the expectation that _id and _rev will be set after filtering + screensToDelete + .filter( + (screen): screen is Screen & { _id: string; _rev: string } => + !!screen._id || !!screen._rev + ) + .forEach(screen => { + // Delete the screen + promises.push(API.deleteScreen(screen._id, screen._rev)) + // Remove links to this screen + deleteUrls.push(screen.routing.route) + }) await Promise.all(promises) await navigationStore.deleteLink(deleteUrls) const deletedIds = screensToDelete.map(screen => screen._id) @@ -359,8 +416,11 @@ export class ScreenStore extends BudiStore { }) // Deselect the current screen if it was deleted - if (deletedIds.includes(state.selectedScreenId)) { - state.selectedScreenId = null + if ( + state.selectedScreenId && + deletedIds.includes(state.selectedScreenId) + ) { + delete state.selectedScreenId componentStore.update(state => ({ ...state, selectedComponentId: null, @@ -389,13 +449,13 @@ export class ScreenStore extends BudiStore { * @param {any} value * @returns */ - async updateSetting(screen, name, value) { + async updateSetting(screen: Screen, name: string, value: any) { if (!screen || !name) { return } // Apply setting update - const patchFn = screen => { + const patchFn = (screen: Screen) => { if (!screen) { return false } @@ -422,7 +482,7 @@ export class ScreenStore extends BudiStore { ) }) if (otherHomeScreens.length && updatedScreen.routing.homeScreen) { - const patchFn = screen => { + const patchFn = (screen: Screen) => { screen.routing.homeScreen = false } for (let otherHomeScreen of otherHomeScreens) { @@ -432,11 +492,11 @@ export class ScreenStore extends BudiStore { } // Move to layouts store - async removeCustomLayout(screen) { + async removeCustomLayout(screen: Screen) { // Pull relevant settings from old layout, if required const layout = get(layoutStore).layouts.find(x => x._id === screen.layoutId) - const patchFn = screen => { - screen.layoutId = null + const patchFn = (screen: Screen) => { + delete screen.layoutId screen.showNavigation = layout?.props.navigation !== "None" screen.width = layout?.props.width || "Large" } @@ -448,9 +508,12 @@ export class ScreenStore extends BudiStore { * and up-to-date. Ensures stability after a product update. * @param {object} screen */ - async enrichEmptySettings(screen) { + async enrichEmptySettings(screen: Screen) { // Flatten the recursive component tree - const components = findAllMatchingComponents(screen.props, x => x) + const components = findAllMatchingComponents( + screen.props, + (x: Component) => x + ) // Iterate over all components and run checks components.forEach(component => { diff --git a/packages/builder/src/stores/builder/tests/screens.test.js b/packages/builder/src/stores/builder/tests/screens.test.js index 430605b77a..87eb03ea04 100644 --- a/packages/builder/src/stores/builder/tests/screens.test.js +++ b/packages/builder/src/stores/builder/tests/screens.test.js @@ -3,7 +3,7 @@ import { get, writable } from "svelte/store" import { API } from "@/api" import { Constants } from "@budibase/frontend-core" import { componentStore, appStore } from "@/stores/builder" -import { INITIAL_SCREENS_STATE, ScreenStore } from "@/stores/builder/screens" +import { initialScreenState, ScreenStore } from "@/stores/builder/screens" import { getScreenFixture, getComponentFixture, @@ -73,7 +73,7 @@ describe("Screens store", () => { vi.clearAllMocks() const screenStore = new ScreenStore() - ctx.test = { + ctx.bb = { get store() { return get(screenStore) }, @@ -81,74 +81,76 @@ describe("Screens store", () => { } }) - it("Create base screen store with defaults", ctx => { - expect(ctx.test.store).toStrictEqual(INITIAL_SCREENS_STATE) + it("Create base screen store with defaults", ({ bb }) => { + expect(bb.store).toStrictEqual(initialScreenState) }) - it("Syncs all screens from the app package", ctx => { - expect(ctx.test.store.screens.length).toBe(0) + it("Syncs all screens from the app package", ({ bb }) => { + expect(bb.store.screens.length).toBe(0) const screens = Array(2) .fill() .map(() => getScreenFixture().json()) - ctx.test.screenStore.syncAppScreens({ screens }) + bb.screenStore.syncAppScreens({ screens }) - expect(ctx.test.store.screens).toStrictEqual(screens) + expect(bb.store.screens).toStrictEqual(screens) }) - it("Reset the screen store back to the default state", ctx => { - expect(ctx.test.store.screens.length).toBe(0) + it("Reset the screen store back to the default state", ({ bb }) => { + expect(bb.store.screens.length).toBe(0) const screens = Array(2) .fill() .map(() => getScreenFixture().json()) - ctx.test.screenStore.syncAppScreens({ screens }) - expect(ctx.test.store.screens).toStrictEqual(screens) + bb.screenStore.syncAppScreens({ screens }) + expect(bb.store.screens).toStrictEqual(screens) - ctx.test.screenStore.update(state => ({ + bb.screenStore.update(state => ({ ...state, selectedScreenId: screens[0]._id, })) - ctx.test.screenStore.reset() + bb.screenStore.reset() - expect(ctx.test.store).toStrictEqual(INITIAL_SCREENS_STATE) + expect(bb.store).toStrictEqual(initialScreenState) }) - it("Marks a valid screen as selected", ctx => { + it("Marks a valid screen as selected", ({ bb }) => { const screens = Array(2) .fill() .map(() => getScreenFixture().json()) - ctx.test.screenStore.syncAppScreens({ screens }) - expect(ctx.test.store.screens.length).toBe(2) + bb.screenStore.syncAppScreens({ screens }) + expect(bb.store.screens.length).toBe(2) - ctx.test.screenStore.select(screens[0]._id) + bb.screenStore.select(screens[0]._id) - expect(ctx.test.store.selectedScreenId).toEqual(screens[0]._id) + expect(bb.store.selectedScreenId).toEqual(screens[0]._id) }) - it("Skip selecting a screen if it is not present", ctx => { + it("Skip selecting a screen if it is not present", ({ bb }) => { const screens = Array(2) .fill() .map(() => getScreenFixture().json()) - ctx.test.screenStore.syncAppScreens({ screens }) - expect(ctx.test.store.screens.length).toBe(2) + bb.screenStore.syncAppScreens({ screens }) + expect(bb.store.screens.length).toBe(2) - ctx.test.screenStore.select("screen_abc") + bb.screenStore.select("screen_abc") - expect(ctx.test.store.selectedScreenId).toBeNull() + expect(bb.store.selectedScreenId).toBeUndefined() }) - it("Approve a valid empty screen config", ctx => { + it("Approve a valid empty screen config", ({ bb }) => { const coreScreen = getScreenFixture() - ctx.test.screenStore.validate(coreScreen.json()) + bb.screenStore.validate(coreScreen.json()) }) - it("Approve a valid screen config with one component and no illegal children", ctx => { + it("Approve a valid screen config with one component and no illegal children", ({ + bb, + }) => { const coreScreen = getScreenFixture() const formBlock = getComponentFixture(`${COMP_PREFIX}/formblock`) @@ -157,12 +159,12 @@ describe("Screens store", () => { const defSpy = vi.spyOn(componentStore, "getDefinition") defSpy.mockReturnValueOnce(COMPONENT_DEFINITIONS.formblock) - ctx.test.screenStore.validate(coreScreen.json()) + bb.screenStore.validate(coreScreen.json()) expect(defSpy).toHaveBeenCalled() }) - it("Reject an attempt to nest invalid components", ctx => { + it("Reject an attempt to nest invalid components", ({ bb }) => { const coreScreen = getScreenFixture() const formOne = getComponentFixture(`${COMP_PREFIX}/form`) @@ -178,14 +180,14 @@ describe("Screens store", () => { return defMap[comp] }) - expect(() => ctx.test.screenStore.validate(coreScreen.json())).toThrowError( + expect(() => bb.screenStore.validate(coreScreen.json())).toThrowError( `You can't place a ${COMPONENT_DEFINITIONS.form.name} here` ) expect(defSpy).toHaveBeenCalled() }) - it("Reject an attempt to deeply nest invalid components", ctx => { + it("Reject an attempt to deeply nest invalid components", ({ bb }) => { const coreScreen = getScreenFixture() const formOne = getComponentFixture(`${COMP_PREFIX}/form`) @@ -210,14 +212,16 @@ describe("Screens store", () => { return defMap[comp] }) - expect(() => ctx.test.screenStore.validate(coreScreen.json())).toThrowError( + expect(() => bb.screenStore.validate(coreScreen.json())).toThrowError( `You can't place a ${COMPONENT_DEFINITIONS.form.name} here` ) expect(defSpy).toHaveBeenCalled() }) - it("Save a brand new screen and add it to the store. No validation", async ctx => { + it("Save a brand new screen and add it to the store. No validation", async ({ + bb, + }) => { const coreScreen = getScreenFixture() const formOne = getComponentFixture(`${COMP_PREFIX}/form`) @@ -225,7 +229,7 @@ describe("Screens store", () => { appStore.set({ features: { componentValidation: false } }) - expect(ctx.test.store.screens.length).toBe(0) + expect(bb.store.screens.length).toBe(0) const newDocId = getScreenDocId() const newDoc = { ...coreScreen.json(), _id: newDocId } @@ -235,15 +239,15 @@ describe("Screens store", () => { vi.spyOn(API, "fetchAppRoutes").mockResolvedValue({ routes: [], }) - await ctx.test.screenStore.save(coreScreen.json()) + await bb.screenStore.save(coreScreen.json()) expect(saveSpy).toHaveBeenCalled() - expect(ctx.test.store.screens.length).toBe(1) + expect(bb.store.screens.length).toBe(1) - expect(ctx.test.store.screens[0]).toStrictEqual(newDoc) + expect(bb.store.screens[0]).toStrictEqual(newDoc) - expect(ctx.test.store.selectedScreenId).toBe(newDocId) + expect(bb.store.selectedScreenId).toBe(newDocId) // The new screen should be selected expect(get(componentStore).selectedComponentId).toBe( @@ -251,7 +255,7 @@ describe("Screens store", () => { ) }) - it("Sync an updated screen to the screen store on save", async ctx => { + it("Sync an updated screen to the screen store on save", async ({ bb }) => { const existingScreens = Array(4) .fill() .map(() => { @@ -261,7 +265,7 @@ describe("Screens store", () => { return screenDoc }) - ctx.test.screenStore.update(state => ({ + bb.screenStore.update(state => ({ ...state, screens: existingScreens.map(screen => screen.json()), })) @@ -279,16 +283,18 @@ describe("Screens store", () => { }) // Saved the existing screen having modified it. - await ctx.test.screenStore.save(existingScreens[2].json()) + await bb.screenStore.save(existingScreens[2].json()) expect(routeSpy).toHaveBeenCalled() expect(saveSpy).toHaveBeenCalled() // On save, the screen is spliced back into the store with the saved content - expect(ctx.test.store.screens[2]).toStrictEqual(existingScreens[2].json()) + expect(bb.store.screens[2]).toStrictEqual(existingScreens[2].json()) }) - it("Sync API data to relevant stores on save. Updated plugins", async ctx => { + it("Sync API data to relevant stores on save. Updated plugins", async ({ + bb, + }) => { const coreScreen = getScreenFixture() const newDocId = getScreenDocId() @@ -318,7 +324,7 @@ describe("Screens store", () => { routes: [], }) - await ctx.test.screenStore.syncScreenData(newDoc) + await bb.screenStore.syncScreenData(newDoc) expect(routeSpy).toHaveBeenCalled() expect(appPackageSpy).toHaveBeenCalled() @@ -326,7 +332,9 @@ describe("Screens store", () => { expect(get(appStore).usedPlugins).toStrictEqual(plugins) }) - it("Sync API updates to relevant stores on save. Plugins unchanged", async ctx => { + it("Sync API updates to relevant stores on save. Plugins unchanged", async ({ + bb, + }) => { const coreScreen = getScreenFixture() const newDocId = getScreenDocId() @@ -343,7 +351,7 @@ describe("Screens store", () => { routes: [], }) - await ctx.test.screenStore.syncScreenData(newDoc) + await bb.screenStore.syncScreenData(newDoc) expect(routeSpy).toHaveBeenCalled() expect(appPackageSpy).not.toHaveBeenCalled() @@ -352,46 +360,48 @@ describe("Screens store", () => { expect(get(appStore).usedPlugins).toStrictEqual([plugin]) }) - it("Proceed to patch if appropriate config are supplied", async ctx => { - vi.spyOn(ctx.test.screenStore, "sequentialScreenPatch").mockImplementation( - () => { - return false - } - ) + it("Proceed to patch if appropriate config are supplied", async ({ bb }) => { + vi.spyOn(bb.screenStore, "sequentialScreenPatch").mockImplementation(() => { + return false + }) const noop = () => {} - await ctx.test.screenStore.patch(noop, "test") - expect(ctx.test.screenStore.sequentialScreenPatch).toHaveBeenCalledWith( + await bb.screenStore.patch(noop, "test") + expect(bb.screenStore.sequentialScreenPatch).toHaveBeenCalledWith( noop, "test" ) }) - it("Return from the patch if all valid config are not present", async ctx => { - vi.spyOn(ctx.test.screenStore, "sequentialScreenPatch") - await ctx.test.screenStore.patch() - expect(ctx.test.screenStore.sequentialScreenPatch).not.toBeCalled() + it("Return from the patch if all valid config are not present", async ({ + bb, + }) => { + vi.spyOn(bb.screenStore, "sequentialScreenPatch") + await bb.screenStore.patch() + expect(bb.screenStore.sequentialScreenPatch).not.toBeCalled() }) - it("Acquire the currently selected screen on patch, if not specified", async ctx => { - vi.spyOn(ctx.test.screenStore, "sequentialScreenPatch") - await ctx.test.screenStore.patch() + it("Acquire the currently selected screen on patch, if not specified", async ({ + bb, + }) => { + vi.spyOn(bb.screenStore, "sequentialScreenPatch") + await bb.screenStore.patch() const noop = () => {} - ctx.test.screenStore.update(state => ({ + bb.screenStore.update(state => ({ ...state, selectedScreenId: "screen_123", })) - await ctx.test.screenStore.patch(noop) - expect(ctx.test.screenStore.sequentialScreenPatch).toHaveBeenCalledWith( + await bb.screenStore.patch(noop) + expect(bb.screenStore.sequentialScreenPatch).toHaveBeenCalledWith( noop, "screen_123" ) }) // Used by the websocket - it("Ignore a call to replace if no screenId is provided", ctx => { + it("Ignore a call to replace if no screenId is provided", ({ bb }) => { const existingScreens = Array(4) .fill() .map(() => { @@ -400,14 +410,16 @@ describe("Screens store", () => { screenDoc._json._id = existingDocId return screenDoc.json() }) - ctx.test.screenStore.syncAppScreens({ screens: existingScreens }) + bb.screenStore.syncAppScreens({ screens: existingScreens }) - ctx.test.screenStore.replace() + bb.screenStore.replace() - expect(ctx.test.store.screens).toStrictEqual(existingScreens) + expect(bb.store.screens).toStrictEqual(existingScreens) }) - it("Remove a screen from the store if a single screenId is supplied", ctx => { + it("Remove a screen from the store if a single screenId is supplied", ({ + bb, + }) => { const existingScreens = Array(4) .fill() .map(() => { @@ -416,17 +428,17 @@ describe("Screens store", () => { screenDoc._json._id = existingDocId return screenDoc.json() }) - ctx.test.screenStore.syncAppScreens({ screens: existingScreens }) + bb.screenStore.syncAppScreens({ screens: existingScreens }) - ctx.test.screenStore.replace(existingScreens[1]._id) + bb.screenStore.replace(existingScreens[1]._id) const filtered = existingScreens.filter( screen => screen._id != existingScreens[1]._id ) - expect(ctx.test.store.screens).toStrictEqual(filtered) + expect(bb.store.screens).toStrictEqual(filtered) }) - it("Replace an existing screen with a new version of itself", ctx => { + it("Replace an existing screen with a new version of itself", ({ bb }) => { const existingScreens = Array(4) .fill() .map(() => { @@ -436,7 +448,7 @@ describe("Screens store", () => { return screenDoc }) - ctx.test.screenStore.update(state => ({ + bb.screenStore.update(state => ({ ...state, screens: existingScreens.map(screen => screen.json()), })) @@ -444,15 +456,14 @@ describe("Screens store", () => { const formBlock = getComponentFixture(`${COMP_PREFIX}/formblock`) existingScreens[2].addChild(formBlock) - ctx.test.screenStore.replace( - existingScreens[2]._id, - existingScreens[2].json() - ) + bb.screenStore.replace(existingScreens[2]._id, existingScreens[2].json()) - expect(ctx.test.store.screens.length).toBe(4) + expect(bb.store.screens.length).toBe(4) }) - it("Add a screen when attempting to replace one not present in the store", ctx => { + it("Add a screen when attempting to replace one not present in the store", ({ + bb, + }) => { const existingScreens = Array(4) .fill() .map(() => { @@ -462,7 +473,7 @@ describe("Screens store", () => { return screenDoc }) - ctx.test.screenStore.update(state => ({ + bb.screenStore.update(state => ({ ...state, screens: existingScreens.map(screen => screen.json()), })) @@ -470,13 +481,13 @@ describe("Screens store", () => { const newScreenDoc = getScreenFixture() newScreenDoc._json._id = getScreenDocId() - ctx.test.screenStore.replace(newScreenDoc._json._id, newScreenDoc.json()) + bb.screenStore.replace(newScreenDoc._json._id, newScreenDoc.json()) - expect(ctx.test.store.screens.length).toBe(5) - expect(ctx.test.store.screens[4]).toStrictEqual(newScreenDoc.json()) + expect(bb.store.screens.length).toBe(5) + expect(bb.store.screens[4]).toStrictEqual(newScreenDoc.json()) }) - it("Delete a single screen and remove it from the store", async ctx => { + it("Delete a single screen and remove it from the store", async ({ bb }) => { const existingScreens = Array(3) .fill() .map(() => { @@ -486,14 +497,14 @@ describe("Screens store", () => { return screenDoc }) - ctx.test.screenStore.update(state => ({ + bb.screenStore.update(state => ({ ...state, screens: existingScreens.map(screen => screen.json()), })) const deleteSpy = vi.spyOn(API, "deleteScreen") - await ctx.test.screenStore.delete(existingScreens[2].json()) + await bb.screenStore.delete(existingScreens[2].json()) vi.spyOn(API, "fetchAppRoutes").mockResolvedValue({ routes: [], @@ -501,13 +512,15 @@ describe("Screens store", () => { expect(deleteSpy).toBeCalled() - expect(ctx.test.store.screens.length).toBe(2) + expect(bb.store.screens.length).toBe(2) // Just confirm that the routes at are being initialised expect(get(appStore).routes).toEqual([]) }) - it("Upon delete, reset selected screen and component ids if the screen was selected", async ctx => { + it("Upon delete, reset selected screen and component ids if the screen was selected", async ({ + bb, + }) => { const existingScreens = Array(3) .fill() .map(() => { @@ -517,7 +530,7 @@ describe("Screens store", () => { return screenDoc }) - ctx.test.screenStore.update(state => ({ + bb.screenStore.update(state => ({ ...state, screens: existingScreens.map(screen => screen.json()), selectedScreenId: existingScreens[2]._json._id, @@ -528,14 +541,16 @@ describe("Screens store", () => { selectedComponentId: existingScreens[2]._json._id, })) - await ctx.test.screenStore.delete(existingScreens[2].json()) + await bb.screenStore.delete(existingScreens[2].json()) - expect(ctx.test.store.screens.length).toBe(2) + expect(bb.store.screens.length).toBe(2) expect(get(componentStore).selectedComponentId).toBeNull() - expect(ctx.test.store.selectedScreenId).toBeNull() + expect(bb.store.selectedScreenId).toBeUndefined() }) - it("Delete multiple is not supported and should leave the store unchanged", async ctx => { + it("Delete multiple is not supported and should leave the store unchanged", async ({ + bb, + }) => { const existingScreens = Array(3) .fill() .map(() => { @@ -547,7 +562,7 @@ describe("Screens store", () => { const storeScreens = existingScreens.map(screen => screen.json()) - ctx.test.screenStore.update(state => ({ + bb.screenStore.update(state => ({ ...state, screens: existingScreens.map(screen => screen.json()), })) @@ -556,42 +571,40 @@ describe("Screens store", () => { const deleteSpy = vi.spyOn(API, "deleteScreen") - await ctx.test.screenStore.delete(targets) + await bb.screenStore.delete(targets) expect(deleteSpy).not.toHaveBeenCalled() - expect(ctx.test.store.screens.length).toBe(3) - expect(ctx.test.store.screens).toStrictEqual(storeScreens) + expect(bb.store.screens.length).toBe(3) + expect(bb.store.screens).toStrictEqual(storeScreens) }) - it("Update a screen setting", async ctx => { + it("Update a screen setting", async ({ bb }) => { const screenDoc = getScreenFixture() const existingDocId = getScreenDocId() screenDoc._json._id = existingDocId - await ctx.test.screenStore.update(state => ({ + await bb.screenStore.update(state => ({ ...state, screens: [screenDoc.json()], })) const patchedDoc = screenDoc.json() const patchSpy = vi - .spyOn(ctx.test.screenStore, "patch") + .spyOn(bb.screenStore, "patch") .mockImplementation(async patchFn => { patchFn(patchedDoc) return }) - await ctx.test.screenStore.updateSetting( - patchedDoc, - "showNavigation", - false - ) + await bb.screenStore.updateSetting(patchedDoc, "showNavigation", false) expect(patchSpy).toBeCalled() expect(patchedDoc.showNavigation).toBe(false) }) - it("Ensure only one homescreen per role after updating setting. All screens same role", async ctx => { + it("Ensure only one homescreen per role after updating setting. All screens same role", async ({ + bb, + }) => { const existingScreens = Array(3) .fill() .map(() => { @@ -611,23 +624,21 @@ describe("Screens store", () => { // Set the 2nd screen as the home screen storeScreens[1].routing.homeScreen = true - await ctx.test.screenStore.update(state => ({ + await bb.screenStore.update(state => ({ ...state, screens: storeScreens, })) const patchSpy = vi - .spyOn(ctx.test.screenStore, "patch") + .spyOn(bb.screenStore, "patch") .mockImplementation(async (patchFn, screenId) => { - const target = ctx.test.store.screens.find( - screen => screen._id === screenId - ) + const target = bb.store.screens.find(screen => screen._id === screenId) patchFn(target) - await ctx.test.screenStore.replace(screenId, target) + await bb.screenStore.replace(screenId, target) }) - await ctx.test.screenStore.updateSetting( + await bb.screenStore.updateSetting( storeScreens[0], "routing.homeScreen", true @@ -637,13 +648,15 @@ describe("Screens store", () => { expect(patchSpy).toBeCalledTimes(2) // The new homescreen for BASIC - expect(ctx.test.store.screens[0].routing.homeScreen).toBe(true) + expect(bb.store.screens[0].routing.homeScreen).toBe(true) // The previous home screen for the BASIC role is now unset - expect(ctx.test.store.screens[1].routing.homeScreen).toBe(false) + expect(bb.store.screens[1].routing.homeScreen).toBe(false) }) - it("Ensure only one homescreen per role when updating screen setting. Multiple screen roles", async ctx => { + it("Ensure only one homescreen per role when updating screen setting. Multiple screen roles", async ({ + bb, + }) => { const expectedRoles = [ Constants.Roles.BASIC, Constants.Roles.POWER, @@ -675,30 +688,24 @@ describe("Screens store", () => { sorted[9].routing.homeScreen = true // Set screens state - await ctx.test.screenStore.update(state => ({ + await bb.screenStore.update(state => ({ ...state, screens: sorted, })) const patchSpy = vi - .spyOn(ctx.test.screenStore, "patch") + .spyOn(bb.screenStore, "patch") .mockImplementation(async (patchFn, screenId) => { - const target = ctx.test.store.screens.find( - screen => screen._id === screenId - ) + const target = bb.store.screens.find(screen => screen._id === screenId) patchFn(target) - await ctx.test.screenStore.replace(screenId, target) + await bb.screenStore.replace(screenId, target) }) // ADMIN homeScreen updated from 0 to 2 - await ctx.test.screenStore.updateSetting( - sorted[2], - "routing.homeScreen", - true - ) + await bb.screenStore.updateSetting(sorted[2], "routing.homeScreen", true) - const results = ctx.test.store.screens.reduce((acc, screen) => { + const results = bb.store.screens.reduce((acc, screen) => { if (screen.routing.homeScreen) { acc[screen.routing.roleId] = acc[screen.routing.roleId] || [] acc[screen.routing.roleId].push(screen) @@ -706,7 +713,7 @@ describe("Screens store", () => { return acc }, {}) - const screens = ctx.test.store.screens + const screens = bb.store.screens // Should still only be one of each homescreen expect(results[Constants.Roles.ADMIN].length).toBe(1) expect(screens[2].routing.homeScreen).toBe(true) @@ -724,74 +731,80 @@ describe("Screens store", () => { expect(patchSpy).toBeCalledTimes(2) }) - it("Sequential patch check. Exit if the screenId is not valid.", async ctx => { + it("Sequential patch check. Exit if the screenId is not valid.", async ({ + bb, + }) => { const screenDoc = getScreenFixture() const existingDocId = getScreenDocId() screenDoc._json._id = existingDocId const original = screenDoc.json() - await ctx.test.screenStore.update(state => ({ + await bb.screenStore.update(state => ({ ...state, screens: [original], })) const saveSpy = vi - .spyOn(ctx.test.screenStore, "save") + .spyOn(bb.screenStore, "save") .mockImplementation(async () => { return }) // A screen with this Id does not exist - await ctx.test.screenStore.sequentialScreenPatch(() => {}, "123") + await bb.screenStore.sequentialScreenPatch(() => {}, "123") expect(saveSpy).not.toBeCalled() }) - it("Sequential patch check. Exit if the patchFn result is false", async ctx => { + it("Sequential patch check. Exit if the patchFn result is false", async ({ + bb, + }) => { const screenDoc = getScreenFixture() const existingDocId = getScreenDocId() screenDoc._json._id = existingDocId const original = screenDoc.json() // Set screens state - await ctx.test.screenStore.update(state => ({ + await bb.screenStore.update(state => ({ ...state, screens: [original], })) const saveSpy = vi - .spyOn(ctx.test.screenStore, "save") + .spyOn(bb.screenStore, "save") .mockImplementation(async () => { return }) // Returning false from the patch will abort the save - await ctx.test.screenStore.sequentialScreenPatch(() => { + await bb.screenStore.sequentialScreenPatch(() => { return false }, "123") expect(saveSpy).not.toBeCalled() }) - it("Sequential patch check. Patch applied and save requested", async ctx => { + it("Sequential patch check. Patch applied and save requested", async ({ + bb, + }) => { const screenDoc = getScreenFixture() const existingDocId = getScreenDocId() screenDoc._json._id = existingDocId const original = screenDoc.json() - await ctx.test.screenStore.update(state => ({ + await bb.screenStore.update(state => ({ ...state, screens: [original], })) const saveSpy = vi - .spyOn(ctx.test.screenStore, "save") + .spyOn(bb.screenStore, "save") .mockImplementation(async () => { return }) - await ctx.test.screenStore.sequentialScreenPatch(screen => { + await bb.screenStore.sequentialScreenPatch(screen => { screen.name = "updated" }, existingDocId) diff --git a/packages/frontend-core/src/utils/utils.js b/packages/frontend-core/src/utils/utils.js index c424aea5b2..eeff561215 100644 --- a/packages/frontend-core/src/utils/utils.js +++ b/packages/frontend-core/src/utils/utils.js @@ -8,7 +8,7 @@ export const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) * Utility to wrap an async function and ensure all invocations happen * sequentially. * @param fn the async function to run - * @return {Promise} a sequential version of the function + * @return {Function} a sequential version of the function */ export const sequential = fn => { let queue = [] From f1d57906b589642e90c457eaf956b6f48753ff3e Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 6 Jan 2025 14:25:13 +0000 Subject: [PATCH 002/106] Clean up jsdoc comments and remove unnecessary comments --- .../builder/src/stores/builder/screens.ts | 57 ++++++------------- 1 file changed, 17 insertions(+), 40 deletions(-) diff --git a/packages/builder/src/stores/builder/screens.ts b/packages/builder/src/stores/builder/screens.ts index fd16cbfae8..a749ded1f9 100644 --- a/packages/builder/src/stores/builder/screens.ts +++ b/packages/builder/src/stores/builder/screens.ts @@ -18,6 +18,7 @@ import { DeleteScreenResponse, Screen, Component, + SaveScreenResponse, } from "@budibase/types" import { ComponentDefinition } from "./components" @@ -51,7 +52,6 @@ export class ScreenStore extends BudiStore { this.deleteScreen = this.deleteScreen.bind(this) this.syncScreenData = this.syncScreenData.bind(this) this.updateSetting = this.updateSetting.bind(this) - // TODO review this behaviour this.sequentialScreenPatch = this.sequentialScreenPatch.bind(this) this.removeCustomLayout = this.removeCustomLayout.bind(this) @@ -84,7 +84,7 @@ export class ScreenStore extends BudiStore { /** * Replace ALL store screens with application package screens - * @param {object} pkg + * @param {FetchAppPackageResponse} pkg */ syncAppScreens(pkg: FetchAppPackageResponse) { this.update(state => ({ @@ -123,7 +123,7 @@ export class ScreenStore extends BudiStore { * Recursively parses the entire screen doc and checks for components * violating illegal child configurations. * - * @param {object} screen + * @param {Screen} screen * @throws Will throw an error containing the name of the component causing * the invalid screen state */ @@ -206,8 +206,7 @@ export class ScreenStore extends BudiStore { * Core save method. If creating a new screen, the store will sync the target * screen id to ensure that it is selected in the builder * - * @param {object} screen - * @returns {object} + * @param {Screen} screen The screen being modified/created */ async saveScreen(screen: Screen) { const appState = get(appStore) @@ -254,7 +253,7 @@ export class ScreenStore extends BudiStore { /** * After saving a screen, sync plugins and routes to the appStore - * @param {object} savedScreen + * @param {Screen} savedScreen */ async syncScreenData(savedScreen: Screen) { const appState = get(appStore) @@ -282,27 +281,6 @@ export class ScreenStore extends BudiStore { * This is slightly better than just a traditional "patch" endpoint and this * supports deeply mutating the current doc rather than just appending data. */ - // sequentialScreenPatch = ( - // patchFn: (screen: Screen) => any, - // screenId: string - // ) => { - // return Utils.sequential(async () => { - // const state = get(this.store) - // const screen = state.screens.find(screen => screen._id === screenId) - // if (!screen) { - // return - // } - // let clone = cloneDeep(screen) - // const result = patchFn(clone) - - // // An explicit false result means skip this change - // if (result === false) { - // return - // } - // return this.save(clone) - // }) - // } - sequentialScreenPatch = Utils.sequential( async (patchFn: (screen: Screen) => any, screenId: string) => { const state = get(this.store) @@ -322,11 +300,13 @@ export class ScreenStore extends BudiStore { ) /** - * @param {function} patchFn + * @param {Function} patchFn the patch action to be applied * @param {string | null} screenId - * @returns */ - async patch(patchFn: (screen: Screen) => any, screenId?: string | null) { + async patch( + patchFn: (screen: Screen) => any, + screenId?: string | null + ): Promise { // Default to the currently selected screen if (!screenId) { const state = get(this.store) @@ -343,9 +323,9 @@ export class ScreenStore extends BudiStore { * the screen supplied. If no screen is provided, the target has * been removed by another user and will be filtered from the store. * Used to marshal updates for the websocket - * @param {string} screenId - * @param {object} screen - * @returns + * + * @param {string} screenId the target screen id + * @param {Screen} screen the replacement screen */ async replace(screenId: string, screen: Screen) { if (!screenId) { @@ -383,10 +363,9 @@ export class ScreenStore extends BudiStore { * Any deleted screens will then have their routes/links purged * * Wrapped by {@link delete} - * @param {object | array} screens - * @returns + * @param {Screen | Screen[]} screens */ - async deleteScreen(screens: Screen[]) { + async deleteScreen(screens: Screen | Screen[]) { const screensToDelete = Array.isArray(screens) ? screens : [screens] // Build array of promises to speed up bulk deletions let promises: Promise[] = [] @@ -435,7 +414,6 @@ export class ScreenStore extends BudiStore { return state }) - return null } /** @@ -444,10 +422,9 @@ export class ScreenStore extends BudiStore { * After a successful update, this method ensures that there is only * ONE home screen per user Role. * - * @param {object} screen + * @param {Screen} screen * @param {string} name e.g "routing.homeScreen" or "showNavigation" * @param {any} value - * @returns */ async updateSetting(screen: Screen, name: string, value: any) { if (!screen || !name) { @@ -506,7 +483,7 @@ export class ScreenStore extends BudiStore { /** * Parse the entire screen component tree and ensure settings are valid * and up-to-date. Ensures stability after a product update. - * @param {object} screen + * @param {Screen} screen */ async enrichEmptySettings(screen: Screen) { // Flatten the recursive component tree From 744b1d3dbcba03f6fd848c4d38dab4828f5fa5f8 Mon Sep 17 00:00:00 2001 From: Dean Date: Tue, 7 Jan 2025 09:32:02 +0000 Subject: [PATCH 003/106] Screen type fixes --- .../builder/src/stores/builder/componentTreeNodes.ts | 7 ++++++- packages/builder/src/stores/builder/websocket.ts | 9 ++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/stores/builder/componentTreeNodes.ts b/packages/builder/src/stores/builder/componentTreeNodes.ts index 420c540e37..cec4e3a4a0 100644 --- a/packages/builder/src/stores/builder/componentTreeNodes.ts +++ b/packages/builder/src/stores/builder/componentTreeNodes.ts @@ -49,7 +49,12 @@ export class ComponentTreeNodesStore extends BudiStore { // Will ensure all parents of a node are expanded so that it is visible in the tree makeNodeVisible(componentId: string) { - const selectedScreen: Screen = get(selectedScreenStore) + const selectedScreen: Screen | undefined = get(selectedScreenStore) + + if (!selectedScreen) { + console.error("Invalid node " + componentId) + return {} + } const path = findComponentPath(selectedScreen.props, componentId) diff --git a/packages/builder/src/stores/builder/websocket.ts b/packages/builder/src/stores/builder/websocket.ts index bd9e2c8d4d..a56fec2227 100644 --- a/packages/builder/src/stores/builder/websocket.ts +++ b/packages/builder/src/stores/builder/websocket.ts @@ -16,7 +16,14 @@ import { auth, appsStore } from "@/stores/portal" import { screenStore } from "./screens" import { SocketEvent, BuilderSocketEvent, helpers } from "@budibase/shared-core" import { notifications } from "@budibase/bbui" -import { Automation, Datasource, Role, Table, UIUser } from "@budibase/types" +import { + Automation, + Datasource, + Role, + Table, + UIUser, + Screen, +} from "@budibase/types" export const createBuilderWebsocket = (appId: string) => { const socket = createWebsocket("/socket/builder") From 4acb8fae99b39ef03a1817af83ecfcda1926c858 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 13 Jan 2025 11:09:22 +0000 Subject: [PATCH 004/106] Convert portal templates store --- packages/builder/src/stores/portal/templates.js | 16 ---------------- packages/builder/src/stores/portal/templates.ts | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 16 deletions(-) delete mode 100644 packages/builder/src/stores/portal/templates.js create mode 100644 packages/builder/src/stores/portal/templates.ts diff --git a/packages/builder/src/stores/portal/templates.js b/packages/builder/src/stores/portal/templates.js deleted file mode 100644 index 2ff23866ab..0000000000 --- a/packages/builder/src/stores/portal/templates.js +++ /dev/null @@ -1,16 +0,0 @@ -import { writable } from "svelte/store" -import { API } from "@/api" - -export function templatesStore() { - const { subscribe, set } = writable([]) - - return { - subscribe, - load: async () => { - const templates = await API.getAppTemplates() - set(templates) - }, - } -} - -export const templates = templatesStore() diff --git a/packages/builder/src/stores/portal/templates.ts b/packages/builder/src/stores/portal/templates.ts new file mode 100644 index 0000000000..caf1a54ced --- /dev/null +++ b/packages/builder/src/stores/portal/templates.ts @@ -0,0 +1,16 @@ +import { API } from "@/api" +import { BudiStore } from "../BudiStore" +import { TemplateMetadata } from "@budibase/types" + +class TemplateStore extends BudiStore { + constructor() { + super([]) + } + + async load() { + const templates = await API.getAppTemplates() + this.set(templates) + } +} + +export const templates = new TemplateStore() From 085617a5220be1dcbc2d31cde6cb645709b6cf67 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 13 Jan 2025 11:17:11 +0000 Subject: [PATCH 005/106] Convert portal temporal store to TS --- .../portal/licensing/LicensingOverlays.svelte | 4 +- .../portal/licensing/licensingBanners.js | 4 +- .../builder/src/stores/portal/temporal.js | 45 ---------------- .../builder/src/stores/portal/temporal.ts | 53 +++++++++++++++++++ 4 files changed, 57 insertions(+), 49 deletions(-) delete mode 100644 packages/builder/src/stores/portal/temporal.js create mode 100644 packages/builder/src/stores/portal/temporal.ts diff --git a/packages/builder/src/components/portal/licensing/LicensingOverlays.svelte b/packages/builder/src/components/portal/licensing/LicensingOverlays.svelte index 3de2b464e0..8280251839 100644 --- a/packages/builder/src/components/portal/licensing/LicensingOverlays.svelte +++ b/packages/builder/src/components/portal/licensing/LicensingOverlays.svelte @@ -20,7 +20,7 @@ const processModals = () => { const defaultCacheFn = key => { - temporalStore.actions.setExpiring(key, {}, oneDayInSeconds) + temporalStore.setExpiring(key, {}, oneDayInSeconds) } const dismissableModals = [ @@ -50,7 +50,7 @@ }, ] return dismissableModals.filter(modal => { - return !temporalStore.actions.getExpiring(modal.key) && modal.criteria() + return !temporalStore.getExpiring(modal.key) && modal.criteria() }) } diff --git a/packages/builder/src/components/portal/licensing/licensingBanners.js b/packages/builder/src/components/portal/licensing/licensingBanners.js index 230b9fe6f6..62ca6caa1b 100644 --- a/packages/builder/src/components/portal/licensing/licensingBanners.js +++ b/packages/builder/src/components/portal/licensing/licensingBanners.js @@ -6,7 +6,7 @@ import { BANNER_TYPES } from "@budibase/bbui" const oneDayInSeconds = 86400 const defaultCacheFn = key => { - temporalStore.actions.setExpiring(key, {}, oneDayInSeconds) + temporalStore.setExpiring(key, {}, oneDayInSeconds) } const upgradeAction = key => { @@ -148,7 +148,7 @@ export const getBanners = () => { buildUsersAboveLimitBanner(ExpiringKeys.LICENSING_USERS_ABOVE_LIMIT_BANNER), ].filter(licensingBanner => { return ( - !temporalStore.actions.getExpiring(licensingBanner.key) && + !temporalStore.getExpiring(licensingBanner.key) && licensingBanner.criteria() ) }) diff --git a/packages/builder/src/stores/portal/temporal.js b/packages/builder/src/stores/portal/temporal.js deleted file mode 100644 index 96b47d1c7f..0000000000 --- a/packages/builder/src/stores/portal/temporal.js +++ /dev/null @@ -1,45 +0,0 @@ -import { createLocalStorageStore } from "@budibase/frontend-core" -import { get } from "svelte/store" - -export const createTemporalStore = () => { - const initialValue = {} - - const localStorageKey = `bb-temporal` - const store = createLocalStorageStore(localStorageKey, initialValue) - - const setExpiring = (key, data, duration) => { - const updated = { - ...data, - expiry: Date.now() + duration * 1000, - } - - store.update(state => ({ - ...state, - [key]: updated, - })) - } - - const getExpiring = key => { - const entry = get(store)[key] - if (!entry) { - return - } - const currentExpiry = entry.expiry - if (currentExpiry < Date.now()) { - store.update(state => { - delete state[key] - return state - }) - return null - } else { - return entry - } - } - - return { - subscribe: store.subscribe, - actions: { setExpiring, getExpiring }, - } -} - -export const temporalStore = createTemporalStore() diff --git a/packages/builder/src/stores/portal/temporal.ts b/packages/builder/src/stores/portal/temporal.ts new file mode 100644 index 0000000000..acbe6feff2 --- /dev/null +++ b/packages/builder/src/stores/portal/temporal.ts @@ -0,0 +1,53 @@ +import { get } from "svelte/store" +import { BudiStore, PersistenceType } from "../BudiStore" + +type TemporalItem = Record & { expiry: number } +type TemporalState = Record + +class TemporalStore extends BudiStore { + constructor() { + super( + {}, + { + persistence: { + key: "bb-temporal", + type: PersistenceType.LOCAL, + }, + } + ) + } + + setExpiring = ( + key: string, + data: Record, + durationSeconds: number + ) => { + const updated: TemporalItem = { + ...data, + expiry: Date.now() + durationSeconds * 1000, + } + this.update(state => ({ + ...state, + [key]: updated, + })) + } + + getExpiring(key: string) { + const entry = get(this.store)[key] + if (!entry) { + return null + } + const currentExpiry = entry.expiry + if (currentExpiry < Date.now()) { + this.update(state => { + delete state[key] + return state + }) + return null + } else { + return entry + } + } +} + +export const temporalStore = new TemporalStore() From 5f3825118f175b66c03c2ac2147c0364ca8f838f Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 13 Jan 2025 11:23:09 +0000 Subject: [PATCH 006/106] Convert portal theme store to TS --- packages/builder/src/stores/portal/theme.js | 37 ----------------- packages/builder/src/stores/portal/theme.ts | 45 +++++++++++++++++++++ 2 files changed, 45 insertions(+), 37 deletions(-) delete mode 100644 packages/builder/src/stores/portal/theme.js create mode 100644 packages/builder/src/stores/portal/theme.ts diff --git a/packages/builder/src/stores/portal/theme.js b/packages/builder/src/stores/portal/theme.js deleted file mode 100644 index 11a87845e1..0000000000 --- a/packages/builder/src/stores/portal/theme.js +++ /dev/null @@ -1,37 +0,0 @@ -import { createLocalStorageStore } from "@budibase/frontend-core" -import { derived } from "svelte/store" -import { - DefaultBuilderTheme, - ensureValidTheme, - getThemeClassNames, - ThemeOptions, - ThemeClassPrefix, -} from "@budibase/shared-core" - -export const getThemeStore = () => { - const themeElement = document.documentElement - const initialValue = { - theme: DefaultBuilderTheme, - } - const store = createLocalStorageStore("bb-theme", initialValue) - const derivedStore = derived(store, $store => ({ - ...$store, - theme: ensureValidTheme($store.theme, DefaultBuilderTheme), - })) - - // Update theme class when store changes - derivedStore.subscribe(({ theme }) => { - const classNames = getThemeClassNames(theme).split(" ") - ThemeOptions.forEach(option => { - const className = `${ThemeClassPrefix}${option.id}` - themeElement.classList.toggle(className, classNames.includes(className)) - }) - }) - - return { - ...store, - subscribe: derivedStore.subscribe, - } -} - -export const themeStore = getThemeStore() diff --git a/packages/builder/src/stores/portal/theme.ts b/packages/builder/src/stores/portal/theme.ts new file mode 100644 index 0000000000..5198555024 --- /dev/null +++ b/packages/builder/src/stores/portal/theme.ts @@ -0,0 +1,45 @@ +import { derived, Writable } from "svelte/store" +import { + DefaultBuilderTheme, + ensureValidTheme, + getThemeClassNames, + ThemeOptions, + ThemeClassPrefix, +} from "@budibase/shared-core" +import { Theme } from "@budibase/types" +import { DerivedBudiStore, PersistenceType } from "../BudiStore" + +interface ThemeState { + theme: Theme +} + +class ThemeStore extends DerivedBudiStore { + constructor() { + const makeDerivedStore = (store: Writable) => { + return derived(store, $store => ({ + ...$store, + theme: ensureValidTheme($store.theme, DefaultBuilderTheme), + })) + } + super({ theme: DefaultBuilderTheme }, makeDerivedStore, { + persistence: { + key: "bb-theme", + type: PersistenceType.LOCAL, + }, + }) + + // Update theme class when store changes + this.subscribe(({ theme }) => { + const classNames = getThemeClassNames(theme).split(" ") + ThemeOptions.forEach(option => { + const className = `${ThemeClassPrefix}${option.id}` + document.documentElement.classList.toggle( + className, + classNames.includes(className) + ) + }) + }) + } +} + +export const themeStore = new ThemeStore() From 65bd89250de002307ce62c1c2845ba1efa3e48ad Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 13 Jan 2025 11:24:04 +0000 Subject: [PATCH 007/106] Convert portal barrel file to TS --- packages/builder/src/stores/portal/{index.js => index.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/builder/src/stores/portal/{index.js => index.ts} (100%) diff --git a/packages/builder/src/stores/portal/index.js b/packages/builder/src/stores/portal/index.ts similarity index 100% rename from packages/builder/src/stores/portal/index.js rename to packages/builder/src/stores/portal/index.ts From 12b283d41ed8b7be2e41b099321adcf47efe1ad0 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 13 Jan 2025 11:28:04 +0000 Subject: [PATCH 008/106] Convert portal navigation store to BudiStore --- .../builder/src/pages/builder/_layout.svelte | 2 +- .../builder/src/stores/portal/navigation.ts | 31 +++++++------------ 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/packages/builder/src/pages/builder/_layout.svelte b/packages/builder/src/pages/builder/_layout.svelte index 209cf2746d..7e765d7366 100644 --- a/packages/builder/src/pages/builder/_layout.svelte +++ b/packages/builder/src/pages/builder/_layout.svelte @@ -18,7 +18,7 @@ $: useAccountPortal = cloud && !$admin.disableAccountPortal - navigation.actions.init($redirect) + navigation.init($redirect) const validateTenantId = async () => { const host = window.location.host diff --git a/packages/builder/src/stores/portal/navigation.ts b/packages/builder/src/stores/portal/navigation.ts index 4eb50bc84f..f289c86bf9 100644 --- a/packages/builder/src/stores/portal/navigation.ts +++ b/packages/builder/src/stores/portal/navigation.ts @@ -1,38 +1,31 @@ -import { writable } from "svelte/store" +import { BudiStore } from "../BudiStore" type GotoFuncType = (path: string) => void -interface PortalNavigationStore { +interface NavigationState { initialisated: boolean goto: GotoFuncType } -export function createNavigationStore() { - const store = writable({ - initialisated: false, - goto: undefined as any, - }) - const { set, subscribe } = store +class NavigationStore extends BudiStore { + constructor() { + super({ + initialisated: false, + goto: undefined as any, + }) + } - const init = (gotoFunc: GotoFuncType) => { + init(gotoFunc: GotoFuncType) { if (typeof gotoFunc !== "function") { throw new Error( `gotoFunc must be a function, found a "${typeof gotoFunc}" instead` ) } - - set({ + this.set({ initialisated: true, goto: gotoFunc, }) } - - return { - subscribe, - actions: { - init, - }, - } } -export const navigation = createNavigationStore() +export const navigation = new NavigationStore() From 502c1605307c3efdcb4c6ea27e5a5a2e08a9ccff Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 13 Jan 2025 11:31:44 +0000 Subject: [PATCH 009/106] Convert admin store to BudiStore --- packages/builder/src/stores/portal/admin.ts | 71 ++++++++++----------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/packages/builder/src/stores/portal/admin.ts b/packages/builder/src/stores/portal/admin.ts index f262d54bff..1cb0be1313 100644 --- a/packages/builder/src/stores/portal/admin.ts +++ b/packages/builder/src/stores/portal/admin.ts @@ -1,4 +1,4 @@ -import { writable, get } from "svelte/store" +import { get } from "svelte/store" import { API } from "@/api" import { auth } from "@/stores/portal" import { banner } from "@budibase/bbui" @@ -7,42 +7,44 @@ import { GetEnvironmentResponse, SystemStatusResponse, } from "@budibase/types" +import { BudiStore } from "../BudiStore" -interface PortalAdminStore extends GetEnvironmentResponse { +interface AdminState extends GetEnvironmentResponse { loaded: boolean checklist?: ConfigChecklistResponse status?: SystemStatusResponse } -export function createAdminStore() { - const admin = writable({ - loaded: false, - multiTenancy: false, - cloud: false, - isDev: false, - disableAccountPortal: false, - offlineMode: false, - maintenance: [], - }) +class AdminStore extends BudiStore { + constructor() { + super({ + loaded: false, + multiTenancy: false, + cloud: false, + isDev: false, + disableAccountPortal: false, + offlineMode: false, + maintenance: [], + }) + } - async function init() { - await getChecklist() - await getEnvironment() + async init() { + await this.getChecklist() + await this.getEnvironment() // enable system status checks in the cloud - if (get(admin).cloud) { - await getSystemStatus() - checkStatus() + if (get(this.store).cloud) { + await this.getSystemStatus() + this.checkStatus() } - - admin.update(store => { + this.update(store => { store.loaded = true return store }) } - async function getEnvironment() { + async getEnvironment() { const environment = await API.getEnvironment() - admin.update(store => { + this.update(store => { store.multiTenancy = environment.multiTenancy store.cloud = environment.cloud store.disableAccountPortal = environment.disableAccountPortal @@ -56,43 +58,36 @@ export function createAdminStore() { }) } - const checkStatus = async () => { - const health = get(admin)?.status?.health + async checkStatus() { + const health = get(this.store).status?.health if (!health?.passing) { await banner.showStatus() } } - async function getSystemStatus() { + async getSystemStatus() { const status = await API.getSystemStatus() - admin.update(store => { + this.update(store => { store.status = status return store }) } - async function getChecklist() { + async getChecklist() { const tenantId = get(auth).tenantId const checklist = await API.getChecklist(tenantId) - admin.update(store => { + this.update(store => { store.checklist = checklist return store }) } - function unload() { - admin.update(store => { + unload() { + this.update(store => { store.loaded = false return store }) } - - return { - subscribe: admin.subscribe, - init, - unload, - getChecklist, - } } -export const admin = createAdminStore() +export const admin = new AdminStore() From 153d905921c48be839c8aa1a74d76743017fe8b3 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 13 Jan 2025 11:33:39 +0000 Subject: [PATCH 010/106] Update exports --- packages/builder/src/stores/portal/admin.test.js | 4 ++-- packages/builder/src/stores/portal/admin.ts | 2 +- packages/builder/src/stores/portal/auditLogs.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/stores/portal/admin.test.js b/packages/builder/src/stores/portal/admin.test.js index 1528042630..261e94eb6f 100644 --- a/packages/builder/src/stores/portal/admin.test.js +++ b/packages/builder/src/stores/portal/admin.test.js @@ -1,5 +1,5 @@ import { it, expect, describe, beforeEach, vi } from "vitest" -import { createAdminStore } from "./admin" +import { AdminStore } from "./admin" import { writable, get } from "svelte/store" import { API } from "@/api" import { auth } from "@/stores/portal" @@ -46,7 +46,7 @@ describe("admin store", () => { ctx.writableReturn = { update: vi.fn(), subscribe: vi.fn() } writable.mockReturnValue(ctx.writableReturn) - ctx.returnedStore = createAdminStore() + ctx.returnedStore = new AdminStore() }) it("returns the created store", ctx => { diff --git a/packages/builder/src/stores/portal/admin.ts b/packages/builder/src/stores/portal/admin.ts index 1cb0be1313..90e3a5cdc9 100644 --- a/packages/builder/src/stores/portal/admin.ts +++ b/packages/builder/src/stores/portal/admin.ts @@ -15,7 +15,7 @@ interface AdminState extends GetEnvironmentResponse { status?: SystemStatusResponse } -class AdminStore extends BudiStore { +export class AdminStore extends BudiStore { constructor() { super({ loaded: false, diff --git a/packages/builder/src/stores/portal/auditLogs.ts b/packages/builder/src/stores/portal/auditLogs.ts index ff29f0cd1b..6f11f228d6 100644 --- a/packages/builder/src/stores/portal/auditLogs.ts +++ b/packages/builder/src/stores/portal/auditLogs.ts @@ -13,7 +13,7 @@ interface PortalAuditLogsStore { logs?: SearchAuditLogsResponse } -export class AuditLogsStore extends BudiStore { +class AuditLogsStore extends BudiStore { constructor() { super({}) } From 9e10921e2981b73b8ca497354d0b55c5089e7fff Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 13 Jan 2025 12:01:29 +0000 Subject: [PATCH 011/106] Update tests --- packages/builder/src/stores/portal/admin.test.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/packages/builder/src/stores/portal/admin.test.js b/packages/builder/src/stores/portal/admin.test.js index 261e94eb6f..8924a5e6fb 100644 --- a/packages/builder/src/stores/portal/admin.test.js +++ b/packages/builder/src/stores/portal/admin.test.js @@ -49,15 +49,6 @@ describe("admin store", () => { ctx.returnedStore = new AdminStore() }) - it("returns the created store", ctx => { - expect(ctx.returnedStore).toEqual({ - subscribe: expect.toBe(ctx.writableReturn.subscribe), - init: expect.toBeFunc(), - unload: expect.toBeFunc(), - getChecklist: expect.toBeFunc(), - }) - }) - describe("init method", () => { beforeEach(async ctx => { let getMockIndex = 0 From ec1e145eb54a919ea8c9052a0a2778d711840e6f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 15 Jan 2025 18:06:49 +0000 Subject: [PATCH 012/106] Formatting for log lines. --- .../bindings/EvaluationSidePanel.svelte | 41 ++++++++++++++++--- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte index c8bf5529ad..54f7dc7d01 100644 --- a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte +++ b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte @@ -11,11 +11,16 @@ export let expressionError: string | undefined = undefined export let evaluating = false export let expression: string | null = null + export let logging: { log: string; line?: number }[] = [] $: error = expressionError != null $: empty = expression == null || expression?.trim() === "" $: success = !error && !empty $: highlightedResult = highlight(expressionResult) + $: highlightedLogs = logging.map(l => ({ + log: highlight(l.log), + line: l.line, + })) const formatError = (err: any) => { if (err.code === UserScriptError.code) { @@ -25,14 +30,14 @@ } // json can be any primitive type - const highlight = (json?: any | null) => { + const highlight = (json?: JSONValue | null) => { if (json == null) { return "" } // Attempt to parse and then stringify, in case this is valid result try { - json = JSON.stringify(JSON.parse(json), null, 2) + json = JSON.stringify(JSON.parse(json as any), null, 2) } catch (err) { // couldn't parse/stringify, just treat it as the raw input } @@ -90,8 +95,21 @@ {:else if error} {formatError(expressionError)} {:else} - - {@html highlightedResult} +
+ {#each highlightedLogs as logLine} +
+ + {@html logLine.log} + {#if logLine.line} + line {logLine.line} + {/if} +
+ {/each} +
+ + {@html highlightedResult} +
+
{/if} @@ -142,8 +160,21 @@ font-size: 12px; overflow-y: scroll; overflow-x: hidden; - white-space: pre-wrap; + white-space: pre-line; word-wrap: break-word; height: 0; } + .output-lines { + display: flex; + gap: var(--spacing-s); + flex-direction: column; + } + .line { + border-bottom: var(--border-light); + padding-bottom: var(--spacing-s); + display: flex; + flex-direction: row; + justify-content: space-between; + align-items: end; + } From 6f9f36f9eb8d1c13a0c060762698c2a9f167c028 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 15 Jan 2025 18:38:23 +0000 Subject: [PATCH 013/106] Getting string templates ready. --- .../src/helpers/javascript.ts | 22 +++++++++++++-- .../string-templates/src/processors/index.ts | 28 +++++++++++++++---- .../src/processors/postprocessor.ts | 28 +++++++++++-------- .../src/processors/preprocessor.ts | 7 +++-- packages/string-templates/src/types.ts | 5 ++++ 5 files changed, 69 insertions(+), 21 deletions(-) diff --git a/packages/string-templates/src/helpers/javascript.ts b/packages/string-templates/src/helpers/javascript.ts index 4fb1329196..a0fdd3cbe5 100644 --- a/packages/string-templates/src/helpers/javascript.ts +++ b/packages/string-templates/src/helpers/javascript.ts @@ -4,6 +4,7 @@ import { getJsHelperList } from "./list" import { iifeWrapper } from "../iife" import { JsTimeoutError, UserScriptError } from "../errors" import { cloneDeep } from "lodash/fp" +import { Log } from "../types" // The method of executing JS scripts depends on the bundle being built. // This setter is used in the entrypoint (either index.js or index.mjs). @@ -96,10 +97,9 @@ export function processJS(handlebars: string, context: any) { clonedContext = cloneDeep(context) } - const sandboxContext = { + const sandboxContext: Record = { $: (path: string) => getContextValue(path, clonedContext), helpers: getJsHelperList(), - // Proxy to evaluate snippets when running in the browser snippets: new Proxy( {}, @@ -114,8 +114,24 @@ export function processJS(handlebars: string, context: any) { ), } + const logs: Log[] = [] + // logging only supported on frontend + if (!isBackendService()) { + const log = (log: string) => logs.push({ log }) + sandboxContext.console = { + log: log, + info: log, + debug: log, + warn: log, + error: log, + // two below may need special cases + trace: log, + table: log, + } + } + // Create a sandbox with our context and run the JS - const res = { data: runJS(js, sandboxContext) } + const res = { data: runJS(js, sandboxContext), logs } return `{{${LITERAL_MARKER} js_result-${JSON.stringify(res)}}}` } catch (error: any) { onErrorLog && onErrorLog(error) diff --git a/packages/string-templates/src/processors/index.ts b/packages/string-templates/src/processors/index.ts index 79085b0dfe..4454d02738 100644 --- a/packages/string-templates/src/processors/index.ts +++ b/packages/string-templates/src/processors/index.ts @@ -1,9 +1,16 @@ import { FIND_HBS_REGEX } from "../utilities" import * as preprocessor from "./preprocessor" +import type { Preprocessor } from "./preprocessor" import * as postprocessor from "./postprocessor" -import { ProcessOptions } from "../types" +import type { Postprocessor } from "./postprocessor" +import { Log, ProcessOptions } from "../types" -function process(output: string, processors: any[], opts?: ProcessOptions) { +function process( + output: string, + processors: (Preprocessor | Postprocessor)[], + opts?: ProcessOptions +) { + let logs: Log[] = [] for (let processor of processors) { // if a literal statement has occurred stop if (typeof output !== "string") { @@ -16,10 +23,16 @@ function process(output: string, processors: any[], opts?: ProcessOptions) { continue } for (let match of matches) { - output = processor.process(output, match, opts) + const res = processor.process(output, match, opts || {}) + if (typeof res === "object" && "logs" in res && res.logs) { + logs = logs.concat(res.logs) + output = res.result + } else { + output = res as string + } } } - return output + return { result: output, logs } } export function preprocess(string: string, opts: ProcessOptions) { @@ -30,8 +43,13 @@ export function preprocess(string: string, opts: ProcessOptions) { ) } - return process(string, processors, opts) + return process(string, processors, opts).result } + export function postprocess(string: string) { + return process(string, postprocessor.processors).result +} + +export function postprocessWithLogs(string: string) { return process(string, postprocessor.processors) } diff --git a/packages/string-templates/src/processors/postprocessor.ts b/packages/string-templates/src/processors/postprocessor.ts index 6ddc0e67cd..49d5f7d1cf 100644 --- a/packages/string-templates/src/processors/postprocessor.ts +++ b/packages/string-templates/src/processors/postprocessor.ts @@ -1,12 +1,16 @@ import { LITERAL_MARKER } from "../helpers/constants" +import { Log } from "../types" export enum PostProcessorNames { CONVERT_LITERALS = "convert-literals", } -type PostprocessorFn = (statement: string) => string +export type PostprocessorFn = (statement: string) => { + result: any + logs?: Log[] +} -class Postprocessor { +export class Postprocessor { name: PostProcessorNames private readonly fn: PostprocessorFn @@ -23,12 +27,12 @@ class Postprocessor { export const processors = [ new Postprocessor( PostProcessorNames.CONVERT_LITERALS, - (statement: string) => { + (statement: string): { result: any; logs?: Log[] } => { if ( typeof statement !== "string" || !statement.includes(LITERAL_MARKER) ) { - return statement + return { result: statement } } const splitMarkerIndex = statement.indexOf("-") const type = statement.substring(12, splitMarkerIndex) @@ -38,20 +42,22 @@ export const processors = [ ) switch (type) { case "string": - return value + return { result: value } case "number": - return parseFloat(value) + return { result: parseFloat(value) } case "boolean": - return value === "true" + return { result: value === "true" } case "object": - return JSON.parse(value) - case "js_result": + return { result: JSON.parse(value) } + case "js_result": { // We use the literal helper to process the result of JS expressions // as we want to be able to return any types. // We wrap the value in an abject to be able to use undefined properly. - return JSON.parse(value).data + const parsed = JSON.parse(value) + return { result: parsed.data, logs: parsed.logs } + } } - return value + return { result: value } } ), ] diff --git a/packages/string-templates/src/processors/preprocessor.ts b/packages/string-templates/src/processors/preprocessor.ts index 97e5c56fcc..37981f31a8 100644 --- a/packages/string-templates/src/processors/preprocessor.ts +++ b/packages/string-templates/src/processors/preprocessor.ts @@ -11,9 +11,12 @@ export enum PreprocessorNames { NORMALIZE_SPACES = "normalize-spaces", } -type PreprocessorFn = (statement: string, opts?: ProcessOptions) => string +export type PreprocessorFn = ( + statement: string, + opts?: ProcessOptions +) => string -class Preprocessor { +export class Preprocessor { name: string private readonly fn: PreprocessorFn diff --git a/packages/string-templates/src/types.ts b/packages/string-templates/src/types.ts index 2a7a430bee..c973142c93 100644 --- a/packages/string-templates/src/types.ts +++ b/packages/string-templates/src/types.ts @@ -8,3 +8,8 @@ export interface ProcessOptions { onlyFound?: boolean disabledHelpers?: string[] } + +export interface Log { + log: string + line?: number +} From 28958f5a1ce32b46bd6415de0c6d10d55f27a3eb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 16 Jan 2025 15:29:50 +0000 Subject: [PATCH 014/106] Expose the ability to get logs. --- packages/string-templates/src/index.ts | 82 ++++++++++++++++++++------ 1 file changed, 65 insertions(+), 17 deletions(-) diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index 7246bc4942..fafff5d9fb 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -1,7 +1,7 @@ import { createContext, runInNewContext } from "vm" import { create, TemplateDelegate } from "handlebars" import { registerAll, registerMinimum } from "./helpers/index" -import { postprocess, preprocess } from "./processors" +import { postprocess, postprocessWithLogs, preprocess } from "./processors" import { atob, btoa, @@ -15,7 +15,7 @@ import { convertHBSBlock } from "./conversion" import { removeJSRunner, setJSRunner } from "./helpers/javascript" import manifest from "./manifest.json" -import { ProcessOptions } from "./types" +import { Log, ProcessOptions } from "./types" import { UserScriptError } from "./errors" export { helpersToRemoveForJs, getJsHelperList } from "./helpers/list" @@ -187,23 +187,27 @@ export function processObjectSync( return object } -/** - * This will process a single handlebars containing string. If the string passed in has no valid handlebars statements - * then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call. - * @param {string} string The template string which is the filled from the context object. - * @param {object} context An object of information which will be used to enrich the string. - * @param {object|undefined} [opts] optional - specify some options for processing. - * @returns {string} The enriched string, all templates should have been replaced if they can be. - */ -export function processStringSync( +// keep the logging function internal, don't want to add this to the process options directly +// as it can't be used for object processing etc. +function processStringSyncInternal( + str: string, + context?: object, + opts?: ProcessOptions & { logging: false } +): string +function processStringSyncInternal( + str: string, + context?: object, + opts?: ProcessOptions & { logging: true } +): { result: string; logs: Log[] } +function processStringSyncInternal( string: string, context?: object, - opts?: ProcessOptions -): string { + opts?: ProcessOptions & { logging: boolean } +): string | { result: string; logs: Log[] } { // Take a copy of input in case of error const input = string if (typeof string !== "string") { - throw "Cannot process non-string types." + throw new Error("Cannot process non-string types.") } function process(stringPart: string) { // context is needed to check for overlap between helpers and context @@ -217,16 +221,24 @@ export function processStringSync( }, ...context, }) - return postprocess(processedString) + return opts?.logging + ? postprocessWithLogs(processedString) + : postprocess(processedString) } try { if (opts && opts.onlyFound) { + let logs: Log[] = [] const blocks = findHBSBlocks(string) for (let block of blocks) { const outcome = process(block) - string = string.replace(block, outcome) + if (typeof outcome === "object" && "result" in outcome) { + logs = logs.concat(outcome.logs || []) + string = string.replace(block, outcome.result) + } else { + string = string.replace(block, outcome) + } } - return string + return opts?.logging ? string : { result: string, logs } } else { return process(string) } @@ -239,6 +251,42 @@ export function processStringSync( } } +/** + * This will process a single handlebars containing string. If the string passed in has no valid handlebars statements + * then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call. + * @param {string} string The template string which is the filled from the context object. + * @param {object} context An object of information which will be used to enrich the string. + * @param {object|undefined} [opts] optional - specify some options for processing. + * @returns {string} The enriched string, all templates should have been replaced if they can be. + */ +export function processStringSync( + string: string, + context?: object, + opts?: ProcessOptions +): string { + return processStringSyncInternal(string, context, { + ...opts, + logging: false, + }) +} + +/** + * Same as function above, but allows logging to be returned - this is only for JS bindings. + */ +export function processStringWithLogsSync( + string: string, + context?: object, + opts?: ProcessOptions +): { result: string; logs: Log[] } { + if (isBackendService()) { + throw new Error("Logging disabled for backend bindings") + } + return processStringSyncInternal(string, context, { + ...opts, + logging: true, + }) +} + /** * By default with expressions like {{ name }} handlebars will escape various * characters, which can be problematic. To fix this we use the syntax {{{ name }}}, From d3a2306787d15b9051571572c3b12fcdb6295c76 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 16 Jan 2025 15:48:49 +0000 Subject: [PATCH 015/106] Finishing link up of logs. --- .../src/components/common/bindings/BindingPanel.svelte | 9 +++++++-- .../common/bindings/EvaluationSidePanel.svelte | 5 +++-- packages/string-templates/src/index.ts | 1 + 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/components/common/bindings/BindingPanel.svelte b/packages/builder/src/components/common/bindings/BindingPanel.svelte index 98df69bc06..ffb477012c 100644 --- a/packages/builder/src/components/common/bindings/BindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/BindingPanel.svelte @@ -12,7 +12,7 @@ decodeJSBinding, encodeJSBinding, processObjectSync, - processStringSync, + processStringWithLogsSync, } from "@budibase/string-templates" import { readableToRuntimeBinding } from "@/dataBinding" import CodeEditor from "../CodeEditor/CodeEditor.svelte" @@ -41,6 +41,7 @@ InsertAtPositionFn, JSONValue, } from "@budibase/types" + import type { Log } from "@budibase/string-templates" import type { CompletionContext } from "@codemirror/autocomplete" const dispatch = createEventDispatcher() @@ -66,6 +67,7 @@ let insertAtPos: InsertAtPositionFn | undefined let targetMode: BindingMode | null = null let expressionResult: string | undefined + let expressionLogs: Log[] | undefined let expressionError: string | undefined let evaluating = false @@ -157,7 +159,7 @@ (expression: string | null, context: any, snippets: Snippet[]) => { try { expressionError = undefined - expressionResult = processStringSync( + const output = processStringWithLogsSync( expression || "", { ...context, @@ -167,6 +169,8 @@ noThrow: false, } ) + expressionResult = output.result + expressionLogs = output.logs } catch (err: any) { expressionResult = undefined expressionError = err @@ -421,6 +425,7 @@ diff --git a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte index 54f7dc7d01..41245af4f9 100644 --- a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte +++ b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte @@ -4,20 +4,21 @@ import { Helpers } from "@budibase/bbui" import { fade } from "svelte/transition" import { UserScriptError } from "@budibase/string-templates" + import type { Log } from "@budibase/string-templates" import type { JSONValue } from "@budibase/types" // this can be essentially any primitive response from the JS function export let expressionResult: JSONValue | undefined = undefined export let expressionError: string | undefined = undefined + export let expressionLogs: Log[] = [] export let evaluating = false export let expression: string | null = null - export let logging: { log: string; line?: number }[] = [] $: error = expressionError != null $: empty = expression == null || expression?.trim() === "" $: success = !error && !empty $: highlightedResult = highlight(expressionResult) - $: highlightedLogs = logging.map(l => ({ + $: highlightedLogs = expressionLogs.map(l => ({ log: highlight(l.log), line: l.line, })) diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index fafff5d9fb..553c0e8861 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -18,6 +18,7 @@ import manifest from "./manifest.json" import { Log, ProcessOptions } from "./types" import { UserScriptError } from "./errors" +export type { Log } from "./types" export { helpersToRemoveForJs, getJsHelperList } from "./helpers/list" export { FIND_ANY_HBS_REGEX } from "./utilities" export { setJSRunner, setOnErrorLog } from "./helpers/javascript" From e6d536bcc83e172926fb753ea30ecf95ba2860a1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 16 Jan 2025 18:15:54 +0000 Subject: [PATCH 016/106] Getting the line number calculated correctly, as well as adding some basic test cases. --- .../bindings/EvaluationSidePanel.svelte | 2 +- .../src/helpers/javascript.ts | 23 +++++++++++-- packages/string-templates/src/index.ts | 16 ++------- packages/string-templates/src/utilities.ts | 17 ++++++++++ .../string-templates/test/jsLogging.spec.ts | 33 +++++++++++++++++++ 5 files changed, 74 insertions(+), 17 deletions(-) create mode 100644 packages/string-templates/test/jsLogging.spec.ts diff --git a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte index 41245af4f9..984fba9b7a 100644 --- a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte +++ b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte @@ -102,7 +102,7 @@ {@html logLine.log} {#if logLine.line} - line {logLine.line} + :{logLine.line} {/if} {/each} diff --git a/packages/string-templates/src/helpers/javascript.ts b/packages/string-templates/src/helpers/javascript.ts index a0fdd3cbe5..a1bfb7a824 100644 --- a/packages/string-templates/src/helpers/javascript.ts +++ b/packages/string-templates/src/helpers/javascript.ts @@ -1,4 +1,9 @@ -import { atob, isBackendService, isJSAllowed } from "../utilities" +import { + atob, + frontendWrapJS, + isBackendService, + isJSAllowed, +} from "../utilities" import { LITERAL_MARKER } from "../helpers/constants" import { getJsHelperList } from "./list" import { iifeWrapper } from "../iife" @@ -117,7 +122,21 @@ export function processJS(handlebars: string, context: any) { const logs: Log[] = [] // logging only supported on frontend if (!isBackendService()) { - const log = (log: string) => logs.push({ log }) + // this counts the lines in the wrapped JS *before* the user's code, so that we can minus it + const jsLineCount = frontendWrapJS(js).split(js)[0].split("\n").length + const log = (log: string) => { + // quick way to find out what line this is being called from + // its an anonymous function and we look for the overall length to find the + // line number we care about (from the users function) + // JS stack traces are in the format function:line:column + const lineNumber = new Error().stack?.match( + /:(\d+):\d+/ + )?.[1] + logs.push({ + log, + line: lineNumber ? parseInt(lineNumber) - jsLineCount : undefined, + }) + } sandboxContext.console = { log: log, info: log, diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index 553c0e8861..a21bfdb755 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -8,6 +8,7 @@ import { FIND_ANY_HBS_REGEX, FIND_HBS_REGEX, findDoubleHbsInstances, + frontendWrapJS, isBackendService, prefixStrings, } from "./utilities" @@ -511,20 +512,7 @@ export function browserJSSetup() { setJSRunner((js: string, context: Record) => { createContext(context) - const wrappedJs = ` - result = { - result: null, - error: null, - }; - - try { - result.result = ${js}; - } catch (e) { - result.error = e; - } - - result; - ` + const wrappedJs = frontendWrapJS(js) const result = runInNewContext(wrappedJs, context, { timeout: 1000 }) if (result.error) { diff --git a/packages/string-templates/src/utilities.ts b/packages/string-templates/src/utilities.ts index 779bef3735..dba1faab17 100644 --- a/packages/string-templates/src/utilities.ts +++ b/packages/string-templates/src/utilities.ts @@ -86,3 +86,20 @@ export const prefixStrings = ( const regexPattern = new RegExp(`\\b(${escapedStrings.join("|")})\\b`, "g") return baseString.replace(regexPattern, `${prefix}$1`) } + +export function frontendWrapJS(js: string) { + return ` + result = { + result: null, + error: null, + }; + + try { + result.result = ${js}; + } catch (e) { + result.error = e; + } + + result; + ` +} diff --git a/packages/string-templates/test/jsLogging.spec.ts b/packages/string-templates/test/jsLogging.spec.ts new file mode 100644 index 0000000000..9b2bb945d2 --- /dev/null +++ b/packages/string-templates/test/jsLogging.spec.ts @@ -0,0 +1,33 @@ +import { + processStringWithLogsSync, + encodeJSBinding, + defaultJSSetup, +} from "../src/index" + +const processJS = (js: string, context?: object) => { + return processStringWithLogsSync(encodeJSBinding(js), context) +} + +describe("Javascript", () => { + beforeAll(() => { + defaultJSSetup() + }) + + describe("Test logging in JS bindings", () => { + it("should execute a simple expression", () => { + const output = processJS( + `console.log("hello"); + console.log("world"); + console.log("foo"); + return "hello"` + ) + expect(output.result).toBe("hello") + expect(output.logs[0].log).toBe("hello") + expect(output.logs[0].line).toBe(1) + expect(output.logs[1].log).toBe("world") + expect(output.logs[1].line).toBe(2) + expect(output.logs[2].log).toBe("foo") + expect(output.logs[2].line).toBe(3) + }) + }) +}) From e146d995ebcd89c661e769b21743875ceddd81c0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 17 Jan 2025 11:06:55 +0000 Subject: [PATCH 017/106] Adding in support for multi-parameter logs and actual logging to console. --- .../bindings/EvaluationSidePanel.svelte | 2 +- .../src/helpers/javascript.ts | 48 +++++++++++-------- packages/string-templates/src/types.ts | 2 +- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte index 984fba9b7a..dc3f585033 100644 --- a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte +++ b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte @@ -19,7 +19,7 @@ $: success = !error && !empty $: highlightedResult = highlight(expressionResult) $: highlightedLogs = expressionLogs.map(l => ({ - log: highlight(l.log), + log: highlight(l.log.join(", ")), line: l.line, })) diff --git a/packages/string-templates/src/helpers/javascript.ts b/packages/string-templates/src/helpers/javascript.ts index a1bfb7a824..5a4f69de5b 100644 --- a/packages/string-templates/src/helpers/javascript.ts +++ b/packages/string-templates/src/helpers/javascript.ts @@ -124,28 +124,38 @@ export function processJS(handlebars: string, context: any) { if (!isBackendService()) { // this counts the lines in the wrapped JS *before* the user's code, so that we can minus it const jsLineCount = frontendWrapJS(js).split(js)[0].split("\n").length - const log = (log: string) => { - // quick way to find out what line this is being called from - // its an anonymous function and we look for the overall length to find the - // line number we care about (from the users function) - // JS stack traces are in the format function:line:column - const lineNumber = new Error().stack?.match( - /:(\d+):\d+/ - )?.[1] - logs.push({ - log, - line: lineNumber ? parseInt(lineNumber) - jsLineCount : undefined, - }) + const buildLogResponse = ( + type: "log" | "info" | "debug" | "warn" | "error" | "trace" | "table" + ) => { + return (...props: any[]) => { + console[type](...props) + props.forEach((prop, index) => { + if (typeof prop === "object") { + props[index] = JSON.stringify(prop) + } + }) + // quick way to find out what line this is being called from + // its an anonymous function and we look for the overall length to find the + // line number we care about (from the users function) + // JS stack traces are in the format function:line:column + const lineNumber = new Error().stack?.match( + /:(\d+):\d+/ + )?.[1] + logs.push({ + log: props, + line: lineNumber ? parseInt(lineNumber) - jsLineCount : undefined, + }) + } } sandboxContext.console = { - log: log, - info: log, - debug: log, - warn: log, - error: log, + log: buildLogResponse("log"), + info: buildLogResponse("info"), + debug: buildLogResponse("debug"), + warn: buildLogResponse("warn"), + error: buildLogResponse("error"), // two below may need special cases - trace: log, - table: log, + trace: buildLogResponse("trace"), + table: buildLogResponse("table"), } } diff --git a/packages/string-templates/src/types.ts b/packages/string-templates/src/types.ts index c973142c93..a32149c8bb 100644 --- a/packages/string-templates/src/types.ts +++ b/packages/string-templates/src/types.ts @@ -10,6 +10,6 @@ export interface ProcessOptions { } export interface Log { - log: string + log: any[] line?: number } From 272bbf5f8bbf4f59307baf525f02f338633b3391 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 17 Jan 2025 17:18:42 +0000 Subject: [PATCH 018/106] Logging with types - allows for coloured outputs. --- packages/bbui/src/bbui.css | 5 +++ .../bindings/EvaluationSidePanel.svelte | 36 ++++++++++++++----- .../src/helpers/javascript.ts | 11 +++--- packages/string-templates/src/index.ts | 2 +- packages/string-templates/src/types.ts | 3 ++ 5 files changed, 42 insertions(+), 15 deletions(-) diff --git a/packages/bbui/src/bbui.css b/packages/bbui/src/bbui.css index dd0588818e..810c5ff2c0 100644 --- a/packages/bbui/src/bbui.css +++ b/packages/bbui/src/bbui.css @@ -45,6 +45,11 @@ --purple: #806fde; --purple-dark: #130080; + --error-bg: rgba(226, 109, 105, 0.3); + --warning-bg: rgba(255, 210, 106, 0.3); + --error-content: rgba(226, 109, 105, 0.6); + --warning-content: rgba(255, 210, 106, 0.6); + --rounded-small: 4px; --rounded-medium: 8px; --rounded-large: 16px; diff --git a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte index dc3f585033..fcd23bb816 100644 --- a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte +++ b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte @@ -21,6 +21,7 @@ $: highlightedLogs = expressionLogs.map(l => ({ log: highlight(l.log.join(", ")), line: l.line, + type: l.type, })) const formatError = (err: any) => { @@ -67,7 +68,7 @@
{#if error} - +
Error
{#if evaluating}
@@ -98,9 +99,24 @@ {:else}
{#each highlightedLogs as logLine} -
- - {@html logLine.log} +
+
+ {#if logLine.type === "error"} + + {:else if logLine.type === "warn"} + + {/if} + + {@html logLine.log} +
{#if logLine.line} :{logLine.line} {/if} @@ -149,10 +165,9 @@ height: 100%; z-index: 1; position: absolute; - opacity: 10%; } .header.error::before { - background: var(--spectrum-global-color-red-400); + background: var(--error-bg); } .body { flex: 1 1 auto; @@ -167,15 +182,20 @@ } .output-lines { display: flex; - gap: var(--spacing-s); flex-direction: column; + gap: var(--spacing-xs); } .line { border-bottom: var(--border-light); - padding-bottom: var(--spacing-s); display: flex; flex-direction: row; justify-content: space-between; align-items: end; + padding: var(--spacing-s); + } + .icon-log { + display: flex; + gap: var(--spacing-s); + align-items: start; } diff --git a/packages/string-templates/src/helpers/javascript.ts b/packages/string-templates/src/helpers/javascript.ts index 5a4f69de5b..997ca7b6ec 100644 --- a/packages/string-templates/src/helpers/javascript.ts +++ b/packages/string-templates/src/helpers/javascript.ts @@ -9,7 +9,7 @@ import { getJsHelperList } from "./list" import { iifeWrapper } from "../iife" import { JsTimeoutError, UserScriptError } from "../errors" import { cloneDeep } from "lodash/fp" -import { Log } from "../types" +import { Log, LogType } from "../types" // The method of executing JS scripts depends on the bundle being built. // This setter is used in the entrypoint (either index.js or index.mjs). @@ -124,9 +124,7 @@ export function processJS(handlebars: string, context: any) { if (!isBackendService()) { // this counts the lines in the wrapped JS *before* the user's code, so that we can minus it const jsLineCount = frontendWrapJS(js).split(js)[0].split("\n").length - const buildLogResponse = ( - type: "log" | "info" | "debug" | "warn" | "error" | "trace" | "table" - ) => { + const buildLogResponse = (type: LogType) => { return (...props: any[]) => { console[type](...props) props.forEach((prop, index) => { @@ -144,6 +142,7 @@ export function processJS(handlebars: string, context: any) { logs.push({ log: props, line: lineNumber ? parseInt(lineNumber) - jsLineCount : undefined, + type, }) } } @@ -153,8 +152,8 @@ export function processJS(handlebars: string, context: any) { debug: buildLogResponse("debug"), warn: buildLogResponse("warn"), error: buildLogResponse("error"), - // two below may need special cases - trace: buildLogResponse("trace"), + // table should be treated differently, but works the same + // as the rest of the logs for now table: buildLogResponse("table"), } } diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index a21bfdb755..67ccde727e 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -19,7 +19,7 @@ import manifest from "./manifest.json" import { Log, ProcessOptions } from "./types" import { UserScriptError } from "./errors" -export type { Log } from "./types" +export type { Log, LogType } from "./types" export { helpersToRemoveForJs, getJsHelperList } from "./helpers/list" export { FIND_ANY_HBS_REGEX } from "./utilities" export { setJSRunner, setOnErrorLog } from "./helpers/javascript" diff --git a/packages/string-templates/src/types.ts b/packages/string-templates/src/types.ts index a32149c8bb..f6ec7098f9 100644 --- a/packages/string-templates/src/types.ts +++ b/packages/string-templates/src/types.ts @@ -9,7 +9,10 @@ export interface ProcessOptions { disabledHelpers?: string[] } +export type LogType = "log" | "info" | "debug" | "warn" | "error" | "table" + export interface Log { log: any[] line?: number + type?: LogType } From 0d1f5c698e529664e1a9ed0e30dce9eb52d0e778 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 09:54:12 +0100 Subject: [PATCH 019/106] Add basic datasource validation error --- packages/client/manifest.json | 3 ++- .../client/src/components/Component.svelte | 24 ++++++++++++++++++- .../error-states/ComponentErrorState.svelte | 4 ++++ .../client/src/utils/componentsValidator.ts | 13 ++++++++++ 4 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 packages/client/src/utils/componentsValidator.ts diff --git a/packages/client/manifest.json b/packages/client/manifest.json index c236dd1ad9..7da7c60abe 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -7206,7 +7206,8 @@ { "type": "table", "label": "Data", - "key": "dataSource" + "key": "dataSource", + "validator": "checkValidDatasource" }, { "type": "radio", diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index 79b4ca6f68..4926965e45 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -40,6 +40,7 @@ getActionDependentContextKeys, } from "../utils/buttonActions.js" import { gridLayout } from "utils/grid" + import { validateComponentSetting } from "utils/componentsValidator" export let instance = {} export let parent = null @@ -103,6 +104,7 @@ let settingsDefinition let settingsDefinitionMap let missingRequiredSettings = false + let invalidSettings = false // Temporary styles which can be added in the app preview for things like // DND. We clear these whenever a new instance is received. @@ -141,12 +143,16 @@ $: showEmptyState = definition?.showEmptyState !== false $: hasMissingRequiredSettings = missingRequiredSettings?.length > 0 $: editable = !!definition?.editable && !hasMissingRequiredSettings + $: hasInvalidSettings = invalidSettings?.length > 0 $: requiredAncestors = definition?.requiredAncestors || [] $: missingRequiredAncestors = requiredAncestors.filter( ancestor => !$component.ancestors.includes(`${BudibasePrefix}${ancestor}`) ) $: hasMissingRequiredAncestors = missingRequiredAncestors?.length > 0 - $: errorState = hasMissingRequiredSettings || hasMissingRequiredAncestors + $: errorState = + hasMissingRequiredSettings || + hasMissingRequiredAncestors || + hasInvalidSettings // Interactive components can be selected, dragged and highlighted inside // the builder preview @@ -338,6 +344,21 @@ return missing }) + // Check for invalid settings + invalidSettings = settingsDefinition.reduce((invalidSettings, setting) => { + if (setting.validator) { + const error = validateComponentSetting( + setting.validator, + instance[setting.key] + ) + if (error) { + invalidSettings.push(error) + } + } + + return invalidSettings + }, []) + // When considering bindings we can ignore children, so we remove that // before storing the reference stringified version const noChildren = JSON.stringify({ ...instance, _children: null }) @@ -692,6 +713,7 @@ {:else} diff --git a/packages/client/src/components/error-states/ComponentErrorState.svelte b/packages/client/src/components/error-states/ComponentErrorState.svelte index b846eaa230..ba55e3cdc1 100644 --- a/packages/client/src/components/error-states/ComponentErrorState.svelte +++ b/packages/client/src/components/error-states/ComponentErrorState.svelte @@ -9,6 +9,7 @@ | { key: string; label: string }[] | undefined export let missingRequiredAncestors: string[] | undefined + export let invalidSettings: string[] | undefined const component = getContext("component") const { styleable, builderStore } = getContext("sdk") @@ -16,6 +17,7 @@ $: styles = { ...$component.styles, normal: {}, custom: null, empty: true } $: requiredSetting = missingRequiredSettings?.[0] $: requiredAncestor = missingRequiredAncestors?.[0] + $: invalidSetting = invalidSettings?.[0] {#if $builderStore.inBuilder} @@ -24,6 +26,8 @@ {#if requiredAncestor} + {:else if invalidSetting} + {invalidSetting} {:else if requiredSetting} {/if} diff --git a/packages/client/src/utils/componentsValidator.ts b/packages/client/src/utils/componentsValidator.ts new file mode 100644 index 0000000000..6609f92ed6 --- /dev/null +++ b/packages/client/src/utils/componentsValidator.ts @@ -0,0 +1,13 @@ +const validators = { + checkValidDatasource: (a: any) => { + return `Ups... "${a.label}" not found` + }, +} + +export function validateComponentSetting( + key: keyof typeof validators, + value: any +) { + const validator = validators[key] + return validator(value) +} From 1578c1a64b7508bd674d179296b57d7ce801baaf Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 20 Jan 2025 10:48:00 +0000 Subject: [PATCH 020/106] PR Feedback --- .../builder/src/stores/builder/components.ts | 18 ++++++++---------- packages/builder/src/stores/builder/screens.ts | 11 +++++------ .../src/stores/builder/tests/screens.test.js | 2 +- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/packages/builder/src/stores/builder/components.ts b/packages/builder/src/stores/builder/components.ts index bce7fcb71d..7eb351e9da 100644 --- a/packages/builder/src/stores/builder/components.ts +++ b/packages/builder/src/stores/builder/components.ts @@ -36,8 +36,8 @@ import { utils } from "@budibase/shared-core" export interface ComponentState { components: Record customComponents: string[] - selectedComponentId?: string | null - componentToPaste?: Component | null + selectedComponentId?: string + componentToPaste?: Component settingsCache: Record selectedScreenId?: string | null } @@ -68,8 +68,6 @@ export interface ComponentSetting { export const INITIAL_COMPONENTS_STATE: ComponentState = { components: {}, customComponents: [], - selectedComponentId: null, - componentToPaste: null, settingsCache: {}, } @@ -443,7 +441,7 @@ export class ComponentStore extends BudiStore { */ createInstance(componentName: string, presetProps: any, parent: any) { const screen = get(selectedScreen) - if (!screen || !selectedScreen) { + if (!screen) { throw "A valid screen must be selected" } @@ -548,7 +546,7 @@ export class ComponentStore extends BudiStore { // Find the selected component let selectedComponentId = state.selectedComponentId if (selectedComponentId?.startsWith(`${screen._id}-`)) { - selectedComponentId = screen.props._id || null + selectedComponentId = screen.props._id } const currentComponent = findComponent( screen.props, @@ -659,7 +657,7 @@ export class ComponentStore extends BudiStore { // Determine the next component to select, and select it before deletion // to avoid an intermediate state of no component selection const state = get(this.store) - let nextId: string | null = "" + let nextId: string = "" if (state.selectedComponentId === component._id) { nextId = this.getNext() if (!nextId) { @@ -746,7 +744,7 @@ export class ComponentStore extends BudiStore { if (!state.componentToPaste) { return } - let newComponentId: string | null = "" + let newComponentId: string = "" // Remove copied component if cutting, regardless if pasting works let componentToPaste = cloneDeep(state.componentToPaste) @@ -1169,7 +1167,7 @@ export class ComponentStore extends BudiStore { } async handleEjectBlock(componentId: string, ejectedDefinition: Component) { - let nextSelectedComponentId: string | null = null + let nextSelectedComponentId: string | undefined await screenStore.patch((screen: Screen) => { const block = findComponent(screen.props, componentId) @@ -1205,7 +1203,7 @@ export class ComponentStore extends BudiStore { (x: Component) => x._id === componentId ) parent._children[index] = ejectedDefinition - nextSelectedComponentId = ejectedDefinition._id ?? null + nextSelectedComponentId = ejectedDefinition._id }, null) // Select new root component diff --git a/packages/builder/src/stores/builder/screens.ts b/packages/builder/src/stores/builder/screens.ts index a749ded1f9..5163c6a3ea 100644 --- a/packages/builder/src/stores/builder/screens.ts +++ b/packages/builder/src/stores/builder/screens.ts @@ -25,7 +25,6 @@ import { ComponentDefinition } from "./components" interface ScreenState { screens: Screen[] selectedScreenId?: string - selected?: Screen } export const initialScreenState: ScreenState = { @@ -65,7 +64,7 @@ export class ScreenStore extends BudiStore { if (!get(selectedComponent)) { this.update(state => ({ ...state, - selectedComponentId: get(this.store).selected?.props._id, + selectedComponentId: get(selectedScreen)?.props._id, })) } }, @@ -400,10 +399,10 @@ export class ScreenStore extends BudiStore { deletedIds.includes(state.selectedScreenId) ) { delete state.selectedScreenId - componentStore.update(state => ({ - ...state, - selectedComponentId: null, - })) + componentStore.update(state => { + delete state.selectedComponentId + return state + }) } // Update routing diff --git a/packages/builder/src/stores/builder/tests/screens.test.js b/packages/builder/src/stores/builder/tests/screens.test.js index 87eb03ea04..ea916c8d59 100644 --- a/packages/builder/src/stores/builder/tests/screens.test.js +++ b/packages/builder/src/stores/builder/tests/screens.test.js @@ -544,7 +544,7 @@ describe("Screens store", () => { await bb.screenStore.delete(existingScreens[2].json()) expect(bb.store.screens.length).toBe(2) - expect(get(componentStore).selectedComponentId).toBeNull() + expect(get(componentStore).selectedComponentId).toBeUndefined() expect(bb.store.selectedScreenId).toBeUndefined() }) From 51a98229e83fbf0858ad587b09d07c9aca74d544 Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 20 Jan 2025 11:05:17 +0000 Subject: [PATCH 021/106] Lint --- packages/builder/src/stores/builder/components.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/stores/builder/components.ts b/packages/builder/src/stores/builder/components.ts index 7eb351e9da..46d3e07eae 100644 --- a/packages/builder/src/stores/builder/components.ts +++ b/packages/builder/src/stores/builder/components.ts @@ -657,7 +657,7 @@ export class ComponentStore extends BudiStore { // Determine the next component to select, and select it before deletion // to avoid an intermediate state of no component selection const state = get(this.store) - let nextId: string = "" + let nextId = "" if (state.selectedComponentId === component._id) { nextId = this.getNext() if (!nextId) { @@ -744,7 +744,7 @@ export class ComponentStore extends BudiStore { if (!state.componentToPaste) { return } - let newComponentId: string = "" + let newComponentId = "" // Remove copied component if cutting, regardless if pasting works let componentToPaste = cloneDeep(state.componentToPaste) From bd5e55480e045886597a0c5d24938756954ec4be Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 11:16:19 +0000 Subject: [PATCH 022/106] Adding more test cases. --- .../string-templates/test/jsLogging.spec.ts | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/packages/string-templates/test/jsLogging.spec.ts b/packages/string-templates/test/jsLogging.spec.ts index 9b2bb945d2..44b3b392ba 100644 --- a/packages/string-templates/test/jsLogging.spec.ts +++ b/packages/string-templates/test/jsLogging.spec.ts @@ -21,13 +21,33 @@ describe("Javascript", () => { console.log("foo"); return "hello"` ) - expect(output.result).toBe("hello") - expect(output.logs[0].log).toBe("hello") - expect(output.logs[0].line).toBe(1) - expect(output.logs[1].log).toBe("world") - expect(output.logs[1].line).toBe(2) - expect(output.logs[2].log).toBe("foo") - expect(output.logs[2].line).toBe(3) + expect(output.result).toEqual("hello") + expect(output.logs[0].log).toEqual(["hello"]) + expect(output.logs[0].line).toEqual(1) + expect(output.logs[1].log).toEqual(["world"]) + expect(output.logs[1].line).toEqual(2) + expect(output.logs[2].log).toEqual(["foo"]) + expect(output.logs[2].line).toEqual(3) }) }) + + it("should log comma separated values", () => { + const output = processJS(`console.log(1, { a: 1 }); return 1`) + expect(output.logs[0].log).toEqual([1, JSON.stringify({ a: 1 })]) + expect(output.logs[0].line).toEqual(1) + }) + + it("should return the type working with warn", () => { + const output = processJS(`console.warn("warning"); return 1`) + expect(output.logs[0].log).toEqual(["warning"]) + expect(output.logs[0].line).toEqual(1) + expect(output.logs[0].type).toEqual("warn") + }) + + it("should return the type working with error", () => { + const output = processJS(`console.error("error"); return 1`) + expect(output.logs[0].log).toEqual(["error"]) + expect(output.logs[0].line).toEqual(1) + expect(output.logs[0].type).toEqual("error") + }) }) From ba2a61841f292276169bc3c4decc659a58c98f61 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 15:55:00 +0100 Subject: [PATCH 023/106] Allow importing @budibase/client/manifest.json --- eslint-local-rules/index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eslint-local-rules/index.js b/eslint-local-rules/index.js index d9d894c33e..9348706399 100644 --- a/eslint-local-rules/index.js +++ b/eslint-local-rules/index.js @@ -41,11 +41,12 @@ module.exports = { if ( /^@budibase\/[^/]+\/.*$/.test(importPath) && importPath !== "@budibase/backend-core/tests" && - importPath !== "@budibase/string-templates/test/utils" + importPath !== "@budibase/string-templates/test/utils" && + importPath !== "@budibase/client/manifest.json" ) { context.report({ node, - message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests and @budibase/string-templates/test/utils.`, + message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests, @budibase/string-templates/test/utils and @budibase/client/manifest.json.`, }) } }, From 7e029386190964097cdf1582f91568b7a694e7ef Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 15:55:17 +0100 Subject: [PATCH 024/106] Screen helpers, findComponentsBySettingsType --- packages/builder/src/helpers/screen.ts | 30 ++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 packages/builder/src/helpers/screen.ts diff --git a/packages/builder/src/helpers/screen.ts b/packages/builder/src/helpers/screen.ts new file mode 100644 index 0000000000..e83805a511 --- /dev/null +++ b/packages/builder/src/helpers/screen.ts @@ -0,0 +1,30 @@ +import { Component, Screen, ScreenProps } from "@budibase/types" +import clientManifest from "@budibase/client/manifest.json" + +export function findComponentsBySettingsType(screen: Screen, type: string) { + const result: Component[] = [] + function recurseFieldComponentsInChildren( + component: ScreenProps, + type: string + ) { + if (!component) { + return + } + + const componentType = component._component.split("/").slice(-1)[0] + const definition = + clientManifest[componentType as keyof typeof clientManifest] + if ( + "settings" in definition && + definition.settings.some((s: any) => s.type === type) + ) { + result.push(component) + } + component._children?.forEach(child => { + recurseFieldComponentsInChildren(child, type) + }) + } + + recurseFieldComponentsInChildren(screen?.props, type) + return result +} From 3c4ac9ad5a7610645b83c8b8f3cdeb3b8c52c70a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 16:04:39 +0100 Subject: [PATCH 025/106] Errors from builder store --- packages/client/manifest.json | 3 +-- .../client/src/components/Component.svelte | 20 ++++--------------- packages/client/src/stores/builder.js | 3 +++ .../src/stores/derived/componentErrors.ts | 6 ++++++ packages/client/src/stores/derived/index.js | 1 + .../client/src/utils/componentsValidator.ts | 13 ------------ 6 files changed, 15 insertions(+), 31 deletions(-) create mode 100644 packages/client/src/stores/derived/componentErrors.ts delete mode 100644 packages/client/src/utils/componentsValidator.ts diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 7da7c60abe..c236dd1ad9 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -7206,8 +7206,7 @@ { "type": "table", "label": "Data", - "key": "dataSource", - "validator": "checkValidDatasource" + "key": "dataSource" }, { "type": "radio", diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index 4926965e45..7f825c0601 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -23,6 +23,7 @@ appStore, dndComponentPath, dndIsDragging, + componentErrors, } from "stores" import { Helpers } from "@budibase/bbui" import { getActiveConditions, reduceConditionActions } from "utils/conditions" @@ -40,7 +41,6 @@ getActionDependentContextKeys, } from "../utils/buttonActions.js" import { gridLayout } from "utils/grid" - import { validateComponentSetting } from "utils/componentsValidator" export let instance = {} export let parent = null @@ -344,21 +344,6 @@ return missing }) - // Check for invalid settings - invalidSettings = settingsDefinition.reduce((invalidSettings, setting) => { - if (setting.validator) { - const error = validateComponentSetting( - setting.validator, - instance[setting.key] - ) - if (error) { - invalidSettings.push(error) - } - } - - return invalidSettings - }, []) - // When considering bindings we can ignore children, so we remove that // before storing the reference stringified version const noChildren = JSON.stringify({ ...instance, _children: null }) @@ -389,6 +374,9 @@ } } + // Check for invalid settings + $: invalidSettings = $componentErrors[id] + // Extracts a map of all context keys which are required by action settings // to provide the functions to evaluate at runtime. This needs done manually // as the action definitions themselves do not specify bindings for action diff --git a/packages/client/src/stores/builder.js b/packages/client/src/stores/builder.js index faa37eddca..62123c07e5 100644 --- a/packages/client/src/stores/builder.js +++ b/packages/client/src/stores/builder.js @@ -19,6 +19,9 @@ const createBuilderStore = () => { eventResolvers: {}, metadata: null, snippets: null, + componentErrors: { + c5ea93132725c48b2a365fcc1facaee86: ["Ups...!"], + }, // TODO // Legacy - allow the builder to specify a layout layout: null, diff --git a/packages/client/src/stores/derived/componentErrors.ts b/packages/client/src/stores/derived/componentErrors.ts new file mode 100644 index 0000000000..48185de9c3 --- /dev/null +++ b/packages/client/src/stores/derived/componentErrors.ts @@ -0,0 +1,6 @@ +import { derived } from "svelte/store" +import { builderStore } from "../builder.js" + +export const componentErrors = derived([builderStore], ([$builderStore]) => { + return $builderStore.componentErrors as Record +}) diff --git a/packages/client/src/stores/derived/index.js b/packages/client/src/stores/derived/index.js index 337c73831f..e7e70d8952 100644 --- a/packages/client/src/stores/derived/index.js +++ b/packages/client/src/stores/derived/index.js @@ -5,3 +5,4 @@ export { currentRole } from "./currentRole.js" export { dndComponentPath } from "./dndComponentPath.js" export { devToolsEnabled } from "./devToolsEnabled.js" export { snippets } from "./snippets.js" +export { componentErrors } from "./componentErrors" diff --git a/packages/client/src/utils/componentsValidator.ts b/packages/client/src/utils/componentsValidator.ts deleted file mode 100644 index 6609f92ed6..0000000000 --- a/packages/client/src/utils/componentsValidator.ts +++ /dev/null @@ -1,13 +0,0 @@ -const validators = { - checkValidDatasource: (a: any) => { - return `Ups... "${a.label}" not found` - }, -} - -export function validateComponentSetting( - key: keyof typeof validators, - value: any -) { - const validator = validators[key] - return validator(value) -} From 3b03515253bd6f5203b65b67be2d9eb6ebd91111 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 15:23:00 +0000 Subject: [PATCH 026/106] Fixing test failure. --- packages/string-templates/src/processors/index.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/string-templates/src/processors/index.ts b/packages/string-templates/src/processors/index.ts index 4454d02738..0916c791cd 100644 --- a/packages/string-templates/src/processors/index.ts +++ b/packages/string-templates/src/processors/index.ts @@ -24,8 +24,10 @@ function process( } for (let match of matches) { const res = processor.process(output, match, opts || {}) - if (typeof res === "object" && "logs" in res && res.logs) { - logs = logs.concat(res.logs) + if (typeof res === "object") { + if ("logs" in res && res.logs) { + logs = logs.concat(res.logs) + } output = res.result } else { output = res as string From a920be3207f850c71a52758b7d675e86a2dc7643 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 15:33:06 +0000 Subject: [PATCH 027/106] Remove error. --- packages/string-templates/test/jsLogging.spec.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/string-templates/test/jsLogging.spec.ts b/packages/string-templates/test/jsLogging.spec.ts index 44b3b392ba..f328b76c7c 100644 --- a/packages/string-templates/test/jsLogging.spec.ts +++ b/packages/string-templates/test/jsLogging.spec.ts @@ -43,11 +43,4 @@ describe("Javascript", () => { expect(output.logs[0].line).toEqual(1) expect(output.logs[0].type).toEqual("warn") }) - - it("should return the type working with error", () => { - const output = processJS(`console.error("error"); return 1`) - expect(output.logs[0].log).toEqual(["error"]) - expect(output.logs[0].line).toEqual(1) - expect(output.logs[0].type).toEqual("error") - }) }) From ae73c0147ffc481206a4e7bc29e0d942cbba511e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 15:41:08 +0000 Subject: [PATCH 028/106] Adding test checks. --- packages/string-templates/src/environment.ts | 11 +++++++++++ packages/string-templates/src/helpers/javascript.ts | 5 ++++- packages/string-templates/test/jsLogging.spec.ts | 7 +++++++ 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 packages/string-templates/src/environment.ts diff --git a/packages/string-templates/src/environment.ts b/packages/string-templates/src/environment.ts new file mode 100644 index 0000000000..ede52591b1 --- /dev/null +++ b/packages/string-templates/src/environment.ts @@ -0,0 +1,11 @@ +function isJest() { + return ( + process.env.NODE_ENV === "jest" || + (process.env.JEST_WORKER_ID != null && + process.env.JEST_WORKER_ID !== "null") + ) +} + +export function isTest() { + return isJest() +} diff --git a/packages/string-templates/src/helpers/javascript.ts b/packages/string-templates/src/helpers/javascript.ts index 997ca7b6ec..91f2f9a0ce 100644 --- a/packages/string-templates/src/helpers/javascript.ts +++ b/packages/string-templates/src/helpers/javascript.ts @@ -10,6 +10,7 @@ import { iifeWrapper } from "../iife" import { JsTimeoutError, UserScriptError } from "../errors" import { cloneDeep } from "lodash/fp" import { Log, LogType } from "../types" +import { isTest } from "../environment" // The method of executing JS scripts depends on the bundle being built. // This setter is used in the entrypoint (either index.js or index.mjs). @@ -126,7 +127,9 @@ export function processJS(handlebars: string, context: any) { const jsLineCount = frontendWrapJS(js).split(js)[0].split("\n").length const buildLogResponse = (type: LogType) => { return (...props: any[]) => { - console[type](...props) + if (!isTest()) { + console[type](...props) + } props.forEach((prop, index) => { if (typeof prop === "object") { props[index] = JSON.stringify(prop) diff --git a/packages/string-templates/test/jsLogging.spec.ts b/packages/string-templates/test/jsLogging.spec.ts index f328b76c7c..44b3b392ba 100644 --- a/packages/string-templates/test/jsLogging.spec.ts +++ b/packages/string-templates/test/jsLogging.spec.ts @@ -43,4 +43,11 @@ describe("Javascript", () => { expect(output.logs[0].line).toEqual(1) expect(output.logs[0].type).toEqual("warn") }) + + it("should return the type working with error", () => { + const output = processJS(`console.error("error"); return 1`) + expect(output.logs[0].log).toEqual(["error"]) + expect(output.logs[0].line).toEqual(1) + expect(output.logs[0].type).toEqual("error") + }) }) From 9c65f1ab41cfdb2e98282ed52d1af7e224152865 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 15:58:11 +0000 Subject: [PATCH 029/106] Another quick fix. --- packages/string-templates/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index 67ccde727e..b2097b0d4c 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -240,7 +240,7 @@ function processStringSyncInternal( string = string.replace(block, outcome) } } - return opts?.logging ? string : { result: string, logs } + return !opts?.logging ? string : { result: string, logs } } else { return process(string) } From 98bd824d7ae01e59df1141ed2f68c8b7b3e233b0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 16:33:21 +0000 Subject: [PATCH 030/106] Adding the ability to configure whether or not string templates is testing backend JS or frontend. --- packages/server/src/api/routes/tests/row.spec.ts | 2 ++ packages/string-templates/src/environment.ts | 12 ++++++++++++ packages/string-templates/src/index.ts | 1 + packages/string-templates/src/utilities.ts | 11 ++++++++--- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 576f0bb663..2a145e1ed9 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -50,9 +50,11 @@ import { JsTimeoutError } from "@budibase/string-templates" import { isDate } from "../../../utilities" import nock from "nock" import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai" +import { setTestingBackendJS } from "@budibase/string-templates" const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString() tk.freeze(timestamp) +setTestingBackendJS() interface WaitOptions { name: string matchFn?: (event: any) => boolean diff --git a/packages/string-templates/src/environment.ts b/packages/string-templates/src/environment.ts index ede52591b1..6bee6fd3a9 100644 --- a/packages/string-templates/src/environment.ts +++ b/packages/string-templates/src/environment.ts @@ -9,3 +9,15 @@ function isJest() { export function isTest() { return isJest() } + +export const isJSAllowed = () => { + return process && !process.env.NO_JS +} + +export const isTestingBackendJS = () => { + return process && process.env.BACKEND_JS +} + +export const setTestingBackendJS = () => { + process.env.BACKEND_JS = "1" +} diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index b2097b0d4c..8dda8b71ab 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -20,6 +20,7 @@ import { Log, ProcessOptions } from "./types" import { UserScriptError } from "./errors" export type { Log, LogType } from "./types" +export { setTestingBackendJS } from "./environment" export { helpersToRemoveForJs, getJsHelperList } from "./helpers/list" export { FIND_ANY_HBS_REGEX } from "./utilities" export { setJSRunner, setOnErrorLog } from "./helpers/javascript" diff --git a/packages/string-templates/src/utilities.ts b/packages/string-templates/src/utilities.ts index dba1faab17..b05945f075 100644 --- a/packages/string-templates/src/utilities.ts +++ b/packages/string-templates/src/utilities.ts @@ -1,15 +1,20 @@ +import { isTest, isTestingBackendJS } from "./environment" + const ALPHA_NUMERIC_REGEX = /^[A-Za-z0-9]+$/g export const FIND_HBS_REGEX = /{{([^{].*?)}}/g export const FIND_ANY_HBS_REGEX = /{?{{([^{].*?)}}}?/g export const FIND_TRIPLE_HBS_REGEX = /{{{([^{].*?)}}}/g -const isJest = () => typeof jest !== "undefined" - export const isBackendService = () => { + // allow configuring backend JS mode when testing - we default to assuming + // frontend, but need a method to control this + if (isTest() && isTestingBackendJS()) { + return true + } // We consider the tests for string-templates to be frontend, so that they // test the frontend JS functionality. - if (isJest()) { + if (isTest()) { return false } return typeof window === "undefined" From 68374bce29126c0f38da9f030acfdc82c2ed8e60 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 16:40:08 +0000 Subject: [PATCH 031/106] Testing backend JS further. --- packages/server/src/jsRunner/tests/jsRunner.spec.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/server/src/jsRunner/tests/jsRunner.spec.ts b/packages/server/src/jsRunner/tests/jsRunner.spec.ts index 006df19fa6..e10e9c4d43 100644 --- a/packages/server/src/jsRunner/tests/jsRunner.spec.ts +++ b/packages/server/src/jsRunner/tests/jsRunner.spec.ts @@ -1,5 +1,9 @@ import { validate as isValidUUID } from "uuid" -import { processStringSync, encodeJSBinding } from "@budibase/string-templates" +import { + processStringSync, + encodeJSBinding, + setTestingBackendJS, +} from "@budibase/string-templates" import { runJsHelpersTests } from "@budibase/string-templates/test/utils" @@ -7,6 +11,7 @@ import tk from "timekeeper" import { init } from ".." import TestConfiguration from "../../tests/utilities/TestConfiguration" +setTestingBackendJS() const DATE = "2021-01-21T12:00:00" tk.freeze(DATE) From 04a7878ce9ad4d6ea5656ae6409eb6cbbb68bb53 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 17:02:57 +0000 Subject: [PATCH 032/106] Changing how we enforce backend JS. --- packages/server/src/api/routes/tests/row.spec.ts | 2 -- .../automations/tests/utilities/AutomationTestBuilder.ts | 1 + packages/server/src/jsRunner/index.ts | 5 +++++ packages/server/src/jsRunner/tests/jsRunner.spec.ts | 7 +------ packages/string-templates/src/helpers/javascript.ts | 2 +- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 2a145e1ed9..576f0bb663 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -50,11 +50,9 @@ import { JsTimeoutError } from "@budibase/string-templates" import { isDate } from "../../../utilities" import nock from "nock" import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai" -import { setTestingBackendJS } from "@budibase/string-templates" const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString() tk.freeze(timestamp) -setTestingBackendJS() interface WaitOptions { name: string matchFn?: (event: any) => boolean diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index 830d2ee5ca..f89c815752 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -43,6 +43,7 @@ import { import TestConfiguration from "../../../tests/utilities/TestConfiguration" import * as setup from "../utilities" import { automations } from "@budibase/shared-core" +import { setTestingBackendJS } from "@budibase/string-templates" type TriggerOutputs = | RowCreatedTriggerOutputs diff --git a/packages/server/src/jsRunner/index.ts b/packages/server/src/jsRunner/index.ts index e17529a687..a29e952b6d 100644 --- a/packages/server/src/jsRunner/index.ts +++ b/packages/server/src/jsRunner/index.ts @@ -4,12 +4,17 @@ import { JsTimeoutError, setJSRunner, setOnErrorLog, + setTestingBackendJS, } from "@budibase/string-templates" import { context, logging } from "@budibase/backend-core" import tracer from "dd-trace" import { IsolatedVM } from "./vm" export function init() { + // enforce that if we're using isolated-VM runner then we are running backend JS + if (env.isTest()) { + setTestingBackendJS() + } setJSRunner((js: string, ctx: Record) => { return tracer.trace("runJS", {}, () => { try { diff --git a/packages/server/src/jsRunner/tests/jsRunner.spec.ts b/packages/server/src/jsRunner/tests/jsRunner.spec.ts index e10e9c4d43..006df19fa6 100644 --- a/packages/server/src/jsRunner/tests/jsRunner.spec.ts +++ b/packages/server/src/jsRunner/tests/jsRunner.spec.ts @@ -1,9 +1,5 @@ import { validate as isValidUUID } from "uuid" -import { - processStringSync, - encodeJSBinding, - setTestingBackendJS, -} from "@budibase/string-templates" +import { processStringSync, encodeJSBinding } from "@budibase/string-templates" import { runJsHelpersTests } from "@budibase/string-templates/test/utils" @@ -11,7 +7,6 @@ import tk from "timekeeper" import { init } from ".." import TestConfiguration from "../../tests/utilities/TestConfiguration" -setTestingBackendJS() const DATE = "2021-01-21T12:00:00" tk.freeze(DATE) diff --git a/packages/string-templates/src/helpers/javascript.ts b/packages/string-templates/src/helpers/javascript.ts index 91f2f9a0ce..6132adf892 100644 --- a/packages/string-templates/src/helpers/javascript.ts +++ b/packages/string-templates/src/helpers/javascript.ts @@ -88,7 +88,7 @@ export function processJS(handlebars: string, context: any) { let clonedContext: Record if (isBackendService()) { - // On the backned, values are copied across the isolated-vm boundary and + // On the backend, values are copied across the isolated-vm boundary and // so we don't need to do any cloning here. This does create a fundamental // difference in how JS executes on the frontend vs the backend, e.g. // consider this snippet: From d51491a19adc19b70fe80532d5bee7bad07c4f23 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Jan 2025 17:06:31 +0000 Subject: [PATCH 033/106] Linting. --- .../src/automations/tests/utilities/AutomationTestBuilder.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index f89c815752..830d2ee5ca 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -43,7 +43,6 @@ import { import TestConfiguration from "../../../tests/utilities/TestConfiguration" import * as setup from "../utilities" import { automations } from "@budibase/shared-core" -import { setTestingBackendJS } from "@budibase/string-templates" type TriggerOutputs = | RowCreatedTriggerOutputs From 5bc316916ff6c9525f5882fdd11b89eb73d98cd8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 Jan 2025 17:18:29 +0000 Subject: [PATCH 034/106] First iteration of single-step automation test endpoint. --- .../server/src/api/controllers/automation.ts | 91 +++++++++++++++---- packages/server/src/api/routes/automation.ts | 12 ++- packages/server/src/events/NoopEmitter.ts | 39 ++++++++ packages/server/src/events/index.ts | 1 + packages/server/src/threads/automation.ts | 36 +++++--- packages/server/src/utilities/index.ts | 41 ++------- packages/server/src/utilities/redis.ts | 17 ++-- packages/types/src/api/web/app/automation.ts | 7 ++ .../documents/app/automation/automation.ts | 4 +- 9 files changed, 172 insertions(+), 76 deletions(-) create mode 100644 packages/server/src/events/NoopEmitter.ts diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index abc0e492c0..a77014cf31 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -2,7 +2,7 @@ import * as triggers from "../../automations/triggers" import { sdk as coreSdk } from "@budibase/shared-core" import { DocumentType } from "../../db/utils" import { updateTestHistory, removeDeprecated } from "../../automations/utils" -import { setTestFlag, clearTestFlag } from "../../utilities/redis" +import { withTestFlag } from "../../utilities/redis" import { context, cache, events, db as dbCore } from "@budibase/backend-core" import { automations, features } from "@budibase/pro" import { @@ -28,11 +28,18 @@ import { TriggerAutomationResponse, TestAutomationRequest, TestAutomationResponse, + TestAutomationStepRequest, + TestAutomationStepResponse, } from "@budibase/types" -import { getActionDefinitions as actionDefs } from "../../automations/actions" +import { + getActionDefinitions as actionDefs, + getAction, +} from "../../automations/actions" import sdk from "../../sdk" import { builderSocket } from "../../websockets" import env from "../../environment" +import { NoopEmitter } from "../../events" +import { enrichBaseContext } from "../../threads/automation" async function getActionDefinitions() { return removeDeprecated(await actionDefs()) @@ -231,24 +238,68 @@ export async function test( ctx: UserCtx ) { const db = context.getAppDB() - let automation = await db.get(ctx.params.id) - await setTestFlag(automation._id!) - const testInput = prepareTestInput(ctx.request.body) - const response = await triggers.externalTrigger( - automation, - { - ...testInput, - appId: ctx.appId, - user: sdk.users.getUserContextBindings(ctx.user), - }, - { getResponses: true } - ) - // save a test history run - await updateTestHistory(ctx.appId, automation, { - ...ctx.request.body, - occurredAt: new Date().getTime(), + const automation = await db.tryGet(ctx.params.id) + if (!automation) { + ctx.throw(404, `Automation ${ctx.params.id} not found`) + } + + const { request, appId } = ctx + const { body } = request + + ctx.body = await withTestFlag(automation._id!, async () => { + const occurredAt = new Date().getTime() + await updateTestHistory(appId, automation, { ...body, occurredAt }) + + const user = sdk.users.getUserContextBindings(ctx.user) + return await triggers.externalTrigger( + automation, + { ...prepareTestInput(body), appId, user }, + { getResponses: true } + ) }) - await clearTestFlag(automation._id!) - ctx.body = response + await events.automation.tested(automation) } + +export async function testStep( + ctx: UserCtx +) { + const { id, stepId } = ctx.params + const db = context.getAppDB() + const automation = await db.tryGet(id) + if (!automation) { + ctx.throw(404, `Automation ${ctx.params.id} not found`) + } + + const step = automation.definition.steps.find(s => s.stepId === stepId) + if (!step) { + ctx.throw(404, `Step ${stepId} not found on automation ${id}`) + } + + if (step.stepId === AutomationActionStepId.BRANCH) { + ctx.throw(400, "Branch steps cannot be tested directly") + } + if (step.stepId === AutomationActionStepId.LOOP) { + ctx.throw(400, "Loop steps cannot be tested directly") + } + + const { body } = ctx.request + + const fn = await getAction(step.stepId) + if (!fn) { + ctx.throw(400, `Step ${stepId} is not a valid step`) + } + + const output = await withTestFlag( + automation._id!, + async () => + await fn({ + inputs: body.inputs, + context: await enrichBaseContext(body.context), + appId: ctx.appId, + emitter: new NoopEmitter(), + }) + ) + + ctx.body = output +} diff --git a/packages/server/src/api/routes/automation.ts b/packages/server/src/api/routes/automation.ts index 489487271c..ea905be0cd 100644 --- a/packages/server/src/api/routes/automation.ts +++ b/packages/server/src/api/routes/automation.ts @@ -1,6 +1,6 @@ import Router from "@koa/router" import * as controller from "../controllers/automation" -import authorized from "../../middleware/authorized" +import authorized, { authorizedResource } from "../../middleware/authorized" import { permissions } from "@budibase/backend-core" import { bodyResource, paramResource } from "../../middleware/resourceId" import { @@ -82,5 +82,15 @@ router ), controller.test ) + .post( + "/api/automations/:id/step/:stepId/test", + appInfoMiddleware({ appType: AppType.DEV }), + authorizedResource( + permissions.PermissionType.AUTOMATION, + permissions.PermissionLevel.EXECUTE, + "id" + ), + controller.testStep + ) export default router diff --git a/packages/server/src/events/NoopEmitter.ts b/packages/server/src/events/NoopEmitter.ts new file mode 100644 index 0000000000..ed87618ead --- /dev/null +++ b/packages/server/src/events/NoopEmitter.ts @@ -0,0 +1,39 @@ +import { EventEmitter } from "events" +import { + Table, + Row, + ContextEmitter, + EventType, + UserBindings, +} from "@budibase/types" + +export class NoopEmitter extends EventEmitter implements ContextEmitter { + emitRow(values: { + eventName: EventType.ROW_SAVE + appId: string + row: Row + table: Table + user: UserBindings + }): void + emitRow(values: { + eventName: EventType.ROW_UPDATE + appId: string + row: Row + table: Table + oldRow: Row + user: UserBindings + }): void + emitRow(values: { + eventName: EventType.ROW_DELETE + appId: string + row: Row + user: UserBindings + }): void + emitRow(_values: unknown): void { + return + } + + emitTable(_eventName: string, _appId: string, _table?: Table) { + return + } +} diff --git a/packages/server/src/events/index.ts b/packages/server/src/events/index.ts index 23c3f3e512..90bf932bcf 100644 --- a/packages/server/src/events/index.ts +++ b/packages/server/src/events/index.ts @@ -2,5 +2,6 @@ import BudibaseEmitter from "./BudibaseEmitter" const emitter = new BudibaseEmitter() +export { NoopEmitter } from "./NoopEmitter" export { init } from "./docUpdates" export default emitter diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 2d10f5d1fb..2790d8fda6 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -29,6 +29,7 @@ import { LoopStep, UserBindings, isBasicSearchOperator, + ContextEmitter, } from "@budibase/types" import { AutomationContext, @@ -71,6 +72,24 @@ function getLoopIterations(loopStep: LoopStep) { return 0 } +export async function enrichBaseContext(context: Record) { + context.env = await sdkUtils.getEnvironmentVariables() + + try { + const { config } = await configs.getSettingsConfigDoc() + context.settings = { + url: config.platformUrl, + logo: config.logoUrl, + company: config.company, + } + } catch (e) { + // if settings doc doesn't exist, make the settings blank + context.settings = {} + } + + return context +} + /** * The automation orchestrator is a class responsible for executing automations. * It handles the context of the automation and makes sure each step gets the correct @@ -80,7 +99,7 @@ class Orchestrator { private chainCount: number private appId: string private automation: Automation - private emitter: any + private emitter: ContextEmitter private context: AutomationContext private job: Job private loopStepOutputs: LoopStep[] @@ -270,20 +289,9 @@ class Orchestrator { appId: this.appId, automationId: this.automation._id, }) - this.context.env = await sdkUtils.getEnvironmentVariables() - this.context.user = this.currentUser - try { - const { config } = await configs.getSettingsConfigDoc() - this.context.settings = { - url: config.platformUrl, - logo: config.logoUrl, - company: config.company, - } - } catch (e) { - // if settings doc doesn't exist, make the settings blank - this.context.settings = {} - } + await enrichBaseContext(this.context) + this.context.user = this.currentUser let metadata diff --git a/packages/server/src/utilities/index.ts b/packages/server/src/utilities/index.ts index db57b4ec12..f1b32c81f3 100644 --- a/packages/server/src/utilities/index.ts +++ b/packages/server/src/utilities/index.ts @@ -58,30 +58,14 @@ export function checkSlashesInUrl(url: string) { export async function updateEntityMetadata( type: string, entityId: string, - updateFn: any + updateFn: (metadata: Document) => Document ) { const db = context.getAppDB() const id = generateMetadataID(type, entityId) - // read it to see if it exists, we'll overwrite it no matter what - let rev, metadata: Document - try { - const oldMetadata = await db.get(id) - rev = oldMetadata._rev - metadata = updateFn(oldMetadata) - } catch (err) { - rev = null - metadata = updateFn({}) - } + const metadata = updateFn((await db.tryGet(id)) || {}) metadata._id = id - if (rev) { - metadata._rev = rev - } const response = await db.put(metadata) - return { - ...metadata, - _id: id, - _rev: response.rev, - } + return { ...metadata, _id: id, _rev: response.rev } } export async function saveEntityMetadata( @@ -89,26 +73,17 @@ export async function saveEntityMetadata( entityId: string, metadata: Document ): Promise { - return updateEntityMetadata(type, entityId, () => { - return metadata - }) + return updateEntityMetadata(type, entityId, () => metadata) } export async function deleteEntityMetadata(type: string, entityId: string) { const db = context.getAppDB() const id = generateMetadataID(type, entityId) - let rev - try { - const metadata = await db.get(id) - if (metadata) { - rev = metadata._rev - } - } catch (err) { - // don't need to error if it doesn't exist - } - if (id && rev) { - await db.remove(id, rev) + const metadata = await db.tryGet(id) + if (!metadata) { + return } + await db.remove(metadata) } export function escapeDangerousCharacters(string: string) { diff --git a/packages/server/src/utilities/redis.ts b/packages/server/src/utilities/redis.ts index a4154b7b95..a3ce655316 100644 --- a/packages/server/src/utilities/redis.ts +++ b/packages/server/src/utilities/redis.ts @@ -89,17 +89,22 @@ export async function setDebounce(id: string, seconds: number) { await debounceClient.store(id, "debouncing", seconds) } -export async function setTestFlag(id: string) { - await flagClient.store(id, { testing: true }, AUTOMATION_TEST_FLAG_SECONDS) -} - export async function checkTestFlag(id: string) { const flag = await flagClient?.get(id) return !!(flag && flag.testing) } -export async function clearTestFlag(id: string) { - await devAppClient.delete(id) +export async function withTestFlag(id: string, fn: () => Promise) { + // TODO(samwho): this has a bit of a problem where if 2 automations are tested + // at the same time, the second one will overwrite the first one's flag. We + // should instead use an atomic counter and only clear the flag when the + // counter reaches 0. + await flagClient.store(id, { testing: true }, AUTOMATION_TEST_FLAG_SECONDS) + try { + return await fn() + } finally { + await devAppClient.delete(id) + } } export function getSocketPubSubClients() { diff --git a/packages/types/src/api/web/app/automation.ts b/packages/types/src/api/web/app/automation.ts index 40f69fc467..572e6499b6 100644 --- a/packages/types/src/api/web/app/automation.ts +++ b/packages/types/src/api/web/app/automation.ts @@ -75,3 +75,10 @@ export interface TestAutomationRequest { row?: Row } export interface TestAutomationResponse {} + +export interface TestAutomationStepRequest { + inputs: Record + context: Record +} + +export type TestAutomationStepResponse = any diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts index d56f0de879..a7556c2ce3 100644 --- a/packages/types/src/documents/app/automation/automation.ts +++ b/packages/types/src/documents/app/automation/automation.ts @@ -1,10 +1,10 @@ import { Document } from "../../document" -import { EventEmitter } from "events" import { User } from "../../global" import { ReadStream } from "fs" import { Row } from "../row" import { Table } from "../table" import { AutomationStep, AutomationTrigger } from "./schema" +import { ContextEmitter } from "../../../sdk" export enum AutomationIOType { OBJECT = "object", @@ -218,7 +218,7 @@ export interface AutomationLogPage { export interface AutomationStepInputBase { context: Record - emitter: EventEmitter + emitter: ContextEmitter appId: string apiKey?: string } From 99cf4feb07dce88d97155de19c1e922731ba15b7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 Jan 2025 17:30:36 +0000 Subject: [PATCH 035/106] Remove old automation test flag mechanism from Redis. --- .../server/src/api/controllers/automation.ts | 39 +++++++------------ packages/server/src/automations/triggers.ts | 11 +----- 2 files changed, 17 insertions(+), 33 deletions(-) diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index a77014cf31..8c58cd4a19 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -2,7 +2,6 @@ import * as triggers from "../../automations/triggers" import { sdk as coreSdk } from "@budibase/shared-core" import { DocumentType } from "../../db/utils" import { updateTestHistory, removeDeprecated } from "../../automations/utils" -import { withTestFlag } from "../../utilities/redis" import { context, cache, events, db as dbCore } from "@budibase/backend-core" import { automations, features } from "@budibase/pro" import { @@ -246,17 +245,15 @@ export async function test( const { request, appId } = ctx const { body } = request - ctx.body = await withTestFlag(automation._id!, async () => { - const occurredAt = new Date().getTime() - await updateTestHistory(appId, automation, { ...body, occurredAt }) + const occurredAt = new Date().getTime() + await updateTestHistory(appId, automation, { ...body, occurredAt }) - const user = sdk.users.getUserContextBindings(ctx.user) - return await triggers.externalTrigger( - automation, - { ...prepareTestInput(body), appId, user }, - { getResponses: true } - ) - }) + const user = sdk.users.getUserContextBindings(ctx.user) + ctx.body = await triggers.externalTrigger( + automation, + { ...prepareTestInput(body), appId, user }, + { getResponses: true } + ) await events.automation.tested(automation) } @@ -271,7 +268,7 @@ export async function testStep( ctx.throw(404, `Automation ${ctx.params.id} not found`) } - const step = automation.definition.steps.find(s => s.stepId === stepId) + const step = automation.definition.steps.find(s => s.id === stepId) if (!step) { ctx.throw(404, `Step ${stepId} not found on automation ${id}`) } @@ -290,16 +287,10 @@ export async function testStep( ctx.throw(400, `Step ${stepId} is not a valid step`) } - const output = await withTestFlag( - automation._id!, - async () => - await fn({ - inputs: body.inputs, - context: await enrichBaseContext(body.context), - appId: ctx.appId, - emitter: new NoopEmitter(), - }) - ) - - ctx.body = output + ctx.body = await fn({ + inputs: body.inputs, + context: await enrichBaseContext(body.context), + appId: ctx.appId, + emitter: new NoopEmitter(), + }) } diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 67d2dcb911..10830a4046 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -82,11 +82,7 @@ async function queueRelevantRowAutomations( // don't queue events which are for dev apps, only way to test automations is // running tests on them, in production the test flag will never // be checked due to lazy evaluation (first always false) - if ( - !env.ALLOW_DEV_AUTOMATIONS && - isDevAppID(event.appId) && - !(await checkTestFlag(automation._id!)) - ) { + if (!env.ALLOW_DEV_AUTOMATIONS && isDevAppID(event.appId)) { continue } @@ -170,10 +166,7 @@ export async function externalTrigger( throw new Error("Automation is disabled") } - if ( - sdk.automations.isAppAction(automation) && - !(await checkTestFlag(automation._id!)) - ) { + if (sdk.automations.isAppAction(automation) && !isDevAppID(params.appId)) { // values are likely to be submitted as strings, so we shall convert to correct type const coercedFields: any = {} const fields = automation.definition.trigger.inputs.fields From 0670c89e83946323f0ff014a5a5687e6166fac71 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 Jan 2025 17:30:49 +0000 Subject: [PATCH 036/106] Remove unused import. --- packages/server/src/automations/triggers.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 10830a4046..f2082e5c0c 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -4,7 +4,6 @@ import { coerce } from "../utilities/rowProcessor" import { definitions } from "./triggerInfo" // need this to call directly, so we can get a response import { automationQueue } from "./bullboard" -import { checkTestFlag } from "../utilities/redis" import * as utils from "./utils" import env from "../environment" import { context, logging, db as dbCore } from "@budibase/backend-core" From f96c4f352d3dbefb8ad28331b24c2f4d8c42b19e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 Jan 2025 17:38:28 +0000 Subject: [PATCH 037/106] Revert "Remove unused import." This reverts commit 0670c89e83946323f0ff014a5a5687e6166fac71. --- packages/server/src/automations/triggers.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index f2082e5c0c..10830a4046 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -4,6 +4,7 @@ import { coerce } from "../utilities/rowProcessor" import { definitions } from "./triggerInfo" // need this to call directly, so we can get a response import { automationQueue } from "./bullboard" +import { checkTestFlag } from "../utilities/redis" import * as utils from "./utils" import env from "../environment" import { context, logging, db as dbCore } from "@budibase/backend-core" From 5afab49e18d12f0b1c007e80dd0cc158fd312740 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 Jan 2025 17:38:38 +0000 Subject: [PATCH 038/106] Revert "Remove old automation test flag mechanism from Redis." This reverts commit 99cf4feb07dce88d97155de19c1e922731ba15b7. --- .../server/src/api/controllers/automation.ts | 39 ++++++++++++------- packages/server/src/automations/triggers.ts | 11 +++++- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index 8c58cd4a19..a77014cf31 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -2,6 +2,7 @@ import * as triggers from "../../automations/triggers" import { sdk as coreSdk } from "@budibase/shared-core" import { DocumentType } from "../../db/utils" import { updateTestHistory, removeDeprecated } from "../../automations/utils" +import { withTestFlag } from "../../utilities/redis" import { context, cache, events, db as dbCore } from "@budibase/backend-core" import { automations, features } from "@budibase/pro" import { @@ -245,15 +246,17 @@ export async function test( const { request, appId } = ctx const { body } = request - const occurredAt = new Date().getTime() - await updateTestHistory(appId, automation, { ...body, occurredAt }) + ctx.body = await withTestFlag(automation._id!, async () => { + const occurredAt = new Date().getTime() + await updateTestHistory(appId, automation, { ...body, occurredAt }) - const user = sdk.users.getUserContextBindings(ctx.user) - ctx.body = await triggers.externalTrigger( - automation, - { ...prepareTestInput(body), appId, user }, - { getResponses: true } - ) + const user = sdk.users.getUserContextBindings(ctx.user) + return await triggers.externalTrigger( + automation, + { ...prepareTestInput(body), appId, user }, + { getResponses: true } + ) + }) await events.automation.tested(automation) } @@ -268,7 +271,7 @@ export async function testStep( ctx.throw(404, `Automation ${ctx.params.id} not found`) } - const step = automation.definition.steps.find(s => s.id === stepId) + const step = automation.definition.steps.find(s => s.stepId === stepId) if (!step) { ctx.throw(404, `Step ${stepId} not found on automation ${id}`) } @@ -287,10 +290,16 @@ export async function testStep( ctx.throw(400, `Step ${stepId} is not a valid step`) } - ctx.body = await fn({ - inputs: body.inputs, - context: await enrichBaseContext(body.context), - appId: ctx.appId, - emitter: new NoopEmitter(), - }) + const output = await withTestFlag( + automation._id!, + async () => + await fn({ + inputs: body.inputs, + context: await enrichBaseContext(body.context), + appId: ctx.appId, + emitter: new NoopEmitter(), + }) + ) + + ctx.body = output } diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 10830a4046..67d2dcb911 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -82,7 +82,11 @@ async function queueRelevantRowAutomations( // don't queue events which are for dev apps, only way to test automations is // running tests on them, in production the test flag will never // be checked due to lazy evaluation (first always false) - if (!env.ALLOW_DEV_AUTOMATIONS && isDevAppID(event.appId)) { + if ( + !env.ALLOW_DEV_AUTOMATIONS && + isDevAppID(event.appId) && + !(await checkTestFlag(automation._id!)) + ) { continue } @@ -166,7 +170,10 @@ export async function externalTrigger( throw new Error("Automation is disabled") } - if (sdk.automations.isAppAction(automation) && !isDevAppID(params.appId)) { + if ( + sdk.automations.isAppAction(automation) && + !(await checkTestFlag(automation._id!)) + ) { // values are likely to be submitted as strings, so we shall convert to correct type const coercedFields: any = {} const fields = automation.definition.trigger.inputs.fields From 31fc2e45c9252a442f5dc7e77613493523ad629b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 Jan 2025 18:08:14 +0000 Subject: [PATCH 039/106] Improve some typing around automation testing. --- .../tests/utilities/AutomationTestBuilder.ts | 28 +++++++------- packages/server/src/automations/triggers.ts | 33 +++++++++------- .../src/tests/utilities/api/automation.ts | 38 ++++++++++++++++++- packages/types/src/api/web/app/automation.ts | 10 ++++- .../documents/app/automation/automation.ts | 8 ++++ 5 files changed, 86 insertions(+), 31 deletions(-) diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index 830d2ee5ca..50527d97af 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -1,5 +1,4 @@ import { v4 as uuidv4 } from "uuid" -import { testAutomation } from "../../../api/routes/tests/utilities/TestFunctions" import { BUILTIN_ACTION_DEFINITIONS } from "../../actions" import { TRIGGER_DEFINITIONS } from "../../triggers" import { @@ -7,7 +6,6 @@ import { AppActionTriggerOutputs, Automation, AutomationActionStepId, - AutomationResults, AutomationStep, AutomationStepInputs, AutomationTrigger, @@ -24,6 +22,7 @@ import { ExecuteQueryStepInputs, ExecuteScriptStepInputs, FilterStepInputs, + isDidNotTriggerResponse, LoopStepInputs, OpenAIStepInputs, QueryRowsStepInputs, @@ -36,6 +35,7 @@ import { SearchFilters, ServerLogStepInputs, SmtpEmailStepInputs, + TestAutomationRequest, UpdateRowStepInputs, WebhookTriggerInputs, WebhookTriggerOutputs, @@ -279,7 +279,7 @@ class StepBuilder extends BaseStepBuilder { class AutomationBuilder extends BaseStepBuilder { private automationConfig: Automation private config: TestConfiguration - private triggerOutputs: any + private triggerOutputs: TriggerOutputs private triggerSet = false constructor( @@ -398,21 +398,19 @@ class AutomationBuilder extends BaseStepBuilder { async run() { const automation = await this.save() - const results = await testAutomation( - this.config, - automation, - this.triggerOutputs + const response = await this.config.api.automation.test( + automation._id!, + this.triggerOutputs as TestAutomationRequest ) - return this.processResults(results) - } - private processResults(results: { - body: AutomationResults - }): AutomationResults { - results.body.steps.shift() + if (isDidNotTriggerResponse(response)) { + throw new Error(response.message) + } + + response.steps.shift() return { - trigger: results.body.trigger, - steps: results.body.steps, + trigger: response.trigger, + steps: response.steps, } } } diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 67d2dcb911..a9317772d9 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -21,6 +21,7 @@ import { AutomationRowEvent, UserBindings, AutomationResults, + DidNotTriggerResponse, } from "@budibase/types" import { executeInThread } from "../threads/automation" import { dataFilters, sdk } from "@budibase/shared-core" @@ -33,14 +34,6 @@ const JOB_OPTS = { import * as automationUtils from "../automations/automationUtils" import { doesTableExist } from "../sdk/app/tables/getters" -type DidNotTriggerResponse = { - outputs: { - success: false - status: AutomationStatus.STOPPED - } - message: AutomationStoppedReason.TRIGGER_FILTER_NOT_MET -} - async function getAllAutomations() { const db = context.getAppDB() let automations = await db.allDocs( @@ -156,14 +149,26 @@ export function isAutomationResults( ) } +interface AutomationTriggerParams { + fields: Record + timeout?: number + appId?: string + user?: UserBindings +} + export async function externalTrigger( automation: Automation, - params: { - fields: Record - timeout?: number - appId?: string - user?: UserBindings - }, + params: AutomationTriggerParams, + options: { getResponses: true } +): Promise +export async function externalTrigger( + automation: Automation, + params: AutomationTriggerParams, + options?: { getResponses: false } +): Promise +export async function externalTrigger( + automation: Automation, + params: AutomationTriggerParams, { getResponses }: { getResponses?: boolean } = {} ): Promise { if (automation.disabled) { diff --git a/packages/server/src/tests/utilities/api/automation.ts b/packages/server/src/tests/utilities/api/automation.ts index 9d9a27e891..6041664999 100644 --- a/packages/server/src/tests/utilities/api/automation.ts +++ b/packages/server/src/tests/utilities/api/automation.ts @@ -1,4 +1,11 @@ -import { Automation, FetchAutomationResponse } from "@budibase/types" +import { + Automation, + FetchAutomationResponse, + TestAutomationRequest, + TestAutomationResponse, + TestAutomationStepRequest, + TestAutomationStepResponse, +} from "@budibase/types" import { Expectations, TestAPI } from "./base" export class AutomationAPI extends TestAPI { @@ -33,4 +40,33 @@ export class AutomationAPI extends TestAPI { }) return result } + + test = async ( + id: string, + body: TestAutomationRequest, + expectations?: Expectations + ): Promise => { + return await this._post( + `/api/automations/${id}/test`, + { + body, + expectations, + } + ) + } + + testStep = async ( + id: string, + stepId: string, + body: TestAutomationStepRequest, + expectations?: Expectations + ): Promise => { + return await this._post( + `/api/automations/${id}/steps/${stepId}/test`, + { + body, + expectations, + } + ) + } } diff --git a/packages/types/src/api/web/app/automation.ts b/packages/types/src/api/web/app/automation.ts index 572e6499b6..edff4b5eaf 100644 --- a/packages/types/src/api/web/app/automation.ts +++ b/packages/types/src/api/web/app/automation.ts @@ -2,10 +2,12 @@ import { Automation, AutomationActionStepId, AutomationLogPage, + AutomationResults, AutomationStatus, AutomationStepDefinition, AutomationTriggerDefinition, AutomationTriggerStepId, + DidNotTriggerResponse, Row, } from "../../../documents" import { DocumentDestroyResponse } from "@budibase/nano" @@ -74,7 +76,13 @@ export interface TestAutomationRequest { fields: Record row?: Row } -export interface TestAutomationResponse {} +export type TestAutomationResponse = AutomationResults | DidNotTriggerResponse + +export function isDidNotTriggerResponse( + response: TestAutomationResponse +): response is DidNotTriggerResponse { + return !!("message" in response && response.message) +} export interface TestAutomationStepRequest { inputs: Record diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts index a7556c2ce3..0314701d72 100644 --- a/packages/types/src/documents/app/automation/automation.ts +++ b/packages/types/src/documents/app/automation/automation.ts @@ -205,6 +205,14 @@ export interface AutomationResults { }[] } +export interface DidNotTriggerResponse { + outputs: { + success: false + status: AutomationStatus.STOPPED + } + message: AutomationStoppedReason.TRIGGER_FILTER_NOT_MET +} + export interface AutomationLog extends AutomationResults, Document { automationName: string _rev?: string From 75dab572e956e7669ba8282c56c74be4e235f474 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 20:06:02 +0100 Subject: [PATCH 040/106] Move data to builder --- .../design/[screenId]/_components/AppPreview.svelte | 3 +++ packages/client/src/index.js | 1 + packages/client/src/stores/builder.js | 4 +--- .../server/src/api/controllers/static/templates/preview.hbs | 4 +++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index 55a4dc4de4..661e985194 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -68,6 +68,9 @@ port: window.location.port, }, snippets: $snippets, + componentErrors: { + c5ea93132725c48b2a365fcc1facaee86: ["Ups...!"], + }, // TODO } // Refresh the preview when required diff --git a/packages/client/src/index.js b/packages/client/src/index.js index 9cef52bb1e..7cb9ed4430 100644 --- a/packages/client/src/index.js +++ b/packages/client/src/index.js @@ -43,6 +43,7 @@ const loadBudibase = async () => { usedPlugins: window["##BUDIBASE_USED_PLUGINS##"], location: window["##BUDIBASE_LOCATION##"], snippets: window["##BUDIBASE_SNIPPETS##"], + componentErrors: window["##BUDIBASE_COMPONENT_ERRORS##"], }) // Set app ID - this window flag is set by both the preview and the real diff --git a/packages/client/src/stores/builder.js b/packages/client/src/stores/builder.js index 62123c07e5..1ae7d3a670 100644 --- a/packages/client/src/stores/builder.js +++ b/packages/client/src/stores/builder.js @@ -19,9 +19,7 @@ const createBuilderStore = () => { eventResolvers: {}, metadata: null, snippets: null, - componentErrors: { - c5ea93132725c48b2a365fcc1facaee86: ["Ups...!"], - }, // TODO + componentErrors: {}, // Legacy - allow the builder to specify a layout layout: null, diff --git a/packages/server/src/api/controllers/static/templates/preview.hbs b/packages/server/src/api/controllers/static/templates/preview.hbs index 87b9ad6ea3..750a780897 100644 --- a/packages/server/src/api/controllers/static/templates/preview.hbs +++ b/packages/server/src/api/controllers/static/templates/preview.hbs @@ -73,7 +73,8 @@ hiddenComponentIds, usedPlugins, location, - snippets + snippets, + componentErrors } = parsed // Set some flags so the app knows we're in the builder @@ -91,6 +92,7 @@ window["##BUDIBASE_USED_PLUGINS##"] = usedPlugins window["##BUDIBASE_LOCATION##"] = location window["##BUDIBASE_SNIPPETS##"] = snippets + window['##BUDIBASE_COMPONENT_ERRORS##'] = componentErrors // Initialise app try { From 655332715dae0a64866b0a797076425b4299817b Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Tue, 21 Jan 2025 09:23:18 +0000 Subject: [PATCH 041/106] login platform user after forced password reset (#15406) --- packages/builder/src/pages/builder/auth/reset.svelte | 6 ++++++ packages/builder/src/stores/portal/auth.ts | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/pages/builder/auth/reset.svelte b/packages/builder/src/pages/builder/auth/reset.svelte index 1179618e40..64438c22d5 100644 --- a/packages/builder/src/pages/builder/auth/reset.svelte +++ b/packages/builder/src/pages/builder/auth/reset.svelte @@ -30,10 +30,16 @@ try { loading = true if (forceResetPassword) { + const email = $auth.user.email + const tenantId = $auth.user.tenantId await auth.updateSelf({ password, forceResetPassword: false, }) + if (!$auth.user) { + // Update self will clear the platform user, so need to login + await auth.login(email, password, tenantId) + } $goto("../portal/") } else { await auth.resetPassword(password, resetCode) diff --git a/packages/builder/src/stores/portal/auth.ts b/packages/builder/src/stores/portal/auth.ts index 171b2b43ae..c3dcaa3663 100644 --- a/packages/builder/src/stores/portal/auth.ts +++ b/packages/builder/src/stores/portal/auth.ts @@ -121,8 +121,8 @@ class AuthStore extends BudiStore { } } - async login(username: string, password: string) { - const tenantId = get(this.store).tenantId + async login(username: string, password: string, targetTenantId?: string) { + const tenantId = targetTenantId || get(this.store).tenantId await API.logIn(tenantId, username, password) await this.getSelf() } From d1294c8d44e5ef9fac098ed823ae0b17ec37b24f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 12:38:31 +0100 Subject: [PATCH 042/106] Errors from store --- .../[screenId]/_components/AppPreview.svelte | 5 +- packages/builder/src/stores/builder/index.js | 2 + .../src/stores/builder/screenComponent.ts | 61 +++++++++++++++++++ 3 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 packages/builder/src/stores/builder/screenComponent.ts diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index 661e985194..fc0b67f63d 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -11,6 +11,7 @@ selectedScreen, hoverStore, componentTreeNodesStore, + screenComponentStore, snippets, } from "@/stores/builder" import ConfirmDialog from "@/components/common/ConfirmDialog.svelte" @@ -68,9 +69,7 @@ port: window.location.port, }, snippets: $snippets, - componentErrors: { - c5ea93132725c48b2a365fcc1facaee86: ["Ups...!"], - }, // TODO + componentErrors: $screenComponentStore.errors, } // Refresh the preview when required diff --git a/packages/builder/src/stores/builder/index.js b/packages/builder/src/stores/builder/index.js index 08d87bebf5..23491996d1 100644 --- a/packages/builder/src/stores/builder/index.js +++ b/packages/builder/src/stores/builder/index.js @@ -16,6 +16,7 @@ import { userStore, userSelectedResourceMap, isOnlyUser } from "./users.js" import { deploymentStore } from "./deployments.js" import { contextMenuStore } from "./contextMenu.js" import { snippets } from "./snippets" +import { screenComponentStore } from "./screenComponent" // Backend import { tables } from "./tables" @@ -67,6 +68,7 @@ export { snippets, rowActions, appPublished, + screenComponentStore, } export const reset = () => { diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts new file mode 100644 index 0000000000..b97bb7ba98 --- /dev/null +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -0,0 +1,61 @@ +import { derived } from "svelte/store" +import { tables, selectedScreen } from "@/stores/builder" +import { DerivedBudiStore } from "../BudiStore" +import { findComponentsBySettingsType } from "@/helpers/screen" +import { Screen } from "@budibase/types" + +interface BuilderScreenComponentStore {} + +interface DerivedScreenComponentStore extends BuilderScreenComponentStore { + errors: Record +} + +export class ScreenComponentStore extends DerivedBudiStore< + BuilderScreenComponentStore, + DerivedScreenComponentStore +> { + constructor() { + const makeDerivedStore = () => { + return derived( + [selectedScreen, tables], + ([$selectedScreen, $tables]): DerivedScreenComponentStore => { + function getErrors() { + const datasources = $tables.list.reduce( + (list, table) => ({ + ...list, + [table._id!]: table, + }), + {} + ) + return { + ...getInvalidDatasources($selectedScreen, datasources), + } + } + + return { + errors: getErrors(), + } + } + ) + } + + super({}, makeDerivedStore) + } +} + +export const screenComponentStore = new ScreenComponentStore() + +function getInvalidDatasources( + screen: Screen, + datasources: Record +) { + const result: Record = {} + for (const component of findComponentsBySettingsType(screen, "table")) { + const { resourceId, type, label } = component.dataSource + if (!datasources[resourceId]) { + result[component._id!] = [`The ${type} named "${label}" was removed`] + } + } + + return result +} From d5f34970ad9c632c06b8b9266bd247872584e8c3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 12:48:28 +0100 Subject: [PATCH 043/106] Use friendly name --- .../builder/src/stores/builder/screenComponent.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index b97bb7ba98..cb7e26bf93 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -53,9 +53,19 @@ function getInvalidDatasources( for (const component of findComponentsBySettingsType(screen, "table")) { const { resourceId, type, label } = component.dataSource if (!datasources[resourceId]) { - result[component._id!] = [`The ${type} named "${label}" was removed`] + const friendlyTypeName = + friendlyNameByType[type as keyof typeof friendlyNameByType] + result[component._id!] = [ + `The ${friendlyTypeName} named "${label}" was removed`, + ] } } return result } + +const friendlyNameByType = { + table: "table", + view: "view", + viewV2: "view", +} From c5e4edcc9713f6286644fe10f65c19ac02a5c66c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 21 Jan 2025 11:54:27 +0000 Subject: [PATCH 044/106] Setting overflow-y in evaluation panel to auto. --- .../src/components/common/bindings/EvaluationSidePanel.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte index fcd23bb816..c47840ea83 100644 --- a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte +++ b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte @@ -174,7 +174,7 @@ padding: var(--spacing-m) var(--spacing-l); font-family: var(--font-mono); font-size: 12px; - overflow-y: scroll; + overflow-y: auto; overflow-x: hidden; white-space: pre-line; word-wrap: break-word; From 56f666f15a3a9b9c8793b1b69e01ca5cead8abbd Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 12:57:30 +0100 Subject: [PATCH 045/106] Display TableSelect the same way we do for DataSourceSelect --- .../settings/controls/TableSelect.svelte | 50 +++++++++++++++---- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/TableSelect.svelte b/packages/builder/src/components/design/settings/controls/TableSelect.svelte index 706c4ca74e..a91bde74ba 100644 --- a/packages/builder/src/components/design/settings/controls/TableSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/TableSelect.svelte @@ -1,22 +1,30 @@ - +
+ +
+ + {#if views?.length} + + {/if} +
+ From ece99aa751ed9c473d90e1842dc41954b4cfb431 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 13:15:53 +0100 Subject: [PATCH 046/106] DRY --- .../DataSourceSelect/DataSourceSelect.svelte | 16 +------- .../settings/controls/TableSelect.svelte | 8 ++-- .../builder/src/stores/builder/builder.ts | 39 +++++++++++++++++-- 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index b23ef5348d..20ba4c8552 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -18,7 +18,6 @@ } from "@budibase/bbui" import { createEventDispatcher } from "svelte" import { - tables as tablesStore, queries as queriesStore, viewsV2 as viewsV2Store, views as viewsStore, @@ -26,6 +25,7 @@ componentStore, datasources, integrations, + builderStore, } from "@/stores/builder" import BindingBuilder from "@/components/integration/QueryBindingBuilder.svelte" import IntegrationQueryEditor from "@/components/integration/index.svelte" @@ -51,19 +51,7 @@ let modal $: text = value?.label ?? "Choose an option" - $: tables = $tablesStore.list - .map(table => format.table(table, $datasources.list)) - .sort((a, b) => { - // sort tables alphabetically, grouped by datasource - const dsA = a.datasourceName ?? "" - const dsB = b.datasourceName ?? "" - - const dsComparison = dsA.localeCompare(dsB) - if (dsComparison !== 0) { - return dsComparison - } - return a.label.localeCompare(b.label) - }) + $: tables = $builderStore.formatedTableNames $: viewsV1 = $viewsStore.list.map(view => ({ ...view, label: view.name, diff --git a/packages/builder/src/components/design/settings/controls/TableSelect.svelte b/packages/builder/src/components/design/settings/controls/TableSelect.svelte index a91bde74ba..4c7c59037c 100644 --- a/packages/builder/src/components/design/settings/controls/TableSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/TableSelect.svelte @@ -1,8 +1,8 @@ {#if dividerState} @@ -29,7 +32,9 @@ on:click={() => onSelect(data)} > - {data.datasourceName ? `${data.datasourceName} - ` : ""}{data.label} + {data.datasourceName && displayDatasourceName + ? `${data.datasourceName} - ` + : ""}{data.label} Date: Tue, 21 Jan 2025 15:02:37 +0100 Subject: [PATCH 056/106] Fix paddings --- .../design/settings/controls/TableSelect.svelte | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/builder/src/components/design/settings/controls/TableSelect.svelte b/packages/builder/src/components/design/settings/controls/TableSelect.svelte index 146decda91..85209a92a0 100644 --- a/packages/builder/src/components/design/settings/controls/TableSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/TableSelect.svelte @@ -66,3 +66,20 @@ {/if}
+ + From 42e86554c881c01bc3f5fdc49bb83303af7158d7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 15:18:31 +0100 Subject: [PATCH 057/106] Validate views --- .../src/stores/builder/screenComponent.ts | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index cb7e26bf93..dd2f7a8b1c 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -1,5 +1,5 @@ import { derived } from "svelte/store" -import { tables, selectedScreen } from "@/stores/builder" +import { tables, selectedScreen, viewsV2 } from "@/stores/builder" import { DerivedBudiStore } from "../BudiStore" import { findComponentsBySettingsType } from "@/helpers/screen" import { Screen } from "@budibase/types" @@ -17,16 +17,25 @@ export class ScreenComponentStore extends DerivedBudiStore< constructor() { const makeDerivedStore = () => { return derived( - [selectedScreen, tables], - ([$selectedScreen, $tables]): DerivedScreenComponentStore => { + [selectedScreen, tables, viewsV2], + ([$selectedScreen, $tables, $viewsV2]): DerivedScreenComponentStore => { function getErrors() { - const datasources = $tables.list.reduce( - (list, table) => ({ - ...list, - [table._id!]: table, - }), - {} - ) + const datasources = { + ...$tables.list.reduce( + (list, table) => ({ + ...list, + [table._id!]: table, + }), + {} + ), + ...$viewsV2.list.reduce( + (list, view) => ({ + ...list, + [view.id]: view, + }), + {} + ), + } return { ...getInvalidDatasources($selectedScreen, datasources), } From ec930372412b4ffc47878ddbc8de2d16aa2691a5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 15:19:45 +0100 Subject: [PATCH 058/106] Change error message --- packages/builder/src/stores/builder/screenComponent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index dd2f7a8b1c..4b591cf0cf 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -65,7 +65,7 @@ function getInvalidDatasources( const friendlyTypeName = friendlyNameByType[type as keyof typeof friendlyNameByType] result[component._id!] = [ - `The ${friendlyTypeName} named "${label}" was removed`, + `The ${friendlyTypeName} named "${label}" does not exist`, ] } } From f32910b033863216393ef6466e6f73d5bdfa47bb Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 15:30:04 +0100 Subject: [PATCH 059/106] Clean code --- .../src/stores/builder/screenComponent.ts | 53 +++++++++++-------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 4b591cf0cf..baa6520832 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -2,7 +2,7 @@ import { derived } from "svelte/store" import { tables, selectedScreen, viewsV2 } from "@/stores/builder" import { DerivedBudiStore } from "../BudiStore" import { findComponentsBySettingsType } from "@/helpers/screen" -import { Screen } from "@budibase/types" +import { Screen, Table, ViewV2 } from "@budibase/types" interface BuilderScreenComponentStore {} @@ -20,22 +20,10 @@ export class ScreenComponentStore extends DerivedBudiStore< [selectedScreen, tables, viewsV2], ([$selectedScreen, $tables, $viewsV2]): DerivedScreenComponentStore => { function getErrors() { - const datasources = { - ...$tables.list.reduce( - (list, table) => ({ - ...list, - [table._id!]: table, - }), - {} - ), - ...$viewsV2.list.reduce( - (list, view) => ({ - ...list, - [view.id]: view, - }), - {} - ), - } + const datasources = flattenTablesAndViews( + $tables.list, + $viewsV2.list + ) return { ...getInvalidDatasources($selectedScreen, datasources), } @@ -54,10 +42,35 @@ export class ScreenComponentStore extends DerivedBudiStore< export const screenComponentStore = new ScreenComponentStore() +function flattenTablesAndViews(tables: Table[], views: ViewV2[]) { + return { + ...tables.reduce( + (list, table) => ({ + ...list, + [table._id!]: table, + }), + {} + ), + ...views.reduce( + (list, view) => ({ + ...list, + [view.id]: view, + }), + {} + ), + } +} + function getInvalidDatasources( screen: Screen, datasources: Record ) { + const friendlyNameByType = { + table: "table", + view: "view", + viewV2: "view", + } + const result: Record = {} for (const component of findComponentsBySettingsType(screen, "table")) { const { resourceId, type, label } = component.dataSource @@ -72,9 +85,3 @@ function getInvalidDatasources( return result } - -const friendlyNameByType = { - table: "table", - view: "view", - viewV2: "view", -} From e1cc8da9dde735579b7f751b4513cdd33784782f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 15:31:16 +0100 Subject: [PATCH 060/106] Clean types --- packages/client/src/stores/derived/componentErrors.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/stores/derived/componentErrors.ts b/packages/client/src/stores/derived/componentErrors.ts index 48185de9c3..68e87a061d 100644 --- a/packages/client/src/stores/derived/componentErrors.ts +++ b/packages/client/src/stores/derived/componentErrors.ts @@ -2,5 +2,5 @@ import { derived } from "svelte/store" import { builderStore } from "../builder.js" export const componentErrors = derived([builderStore], ([$builderStore]) => { - return $builderStore.componentErrors as Record + return $builderStore.componentErrors }) From fc599767c21c23abe1d3aa14445fe5c614b42bea Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 21 Jan 2025 16:06:21 +0100 Subject: [PATCH 061/106] Fix imports --- .../src/stores/builder/screenComponent.ts | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index baa6520832..b4eb01a3b7 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -1,5 +1,7 @@ import { derived } from "svelte/store" -import { tables, selectedScreen, viewsV2 } from "@/stores/builder" +import { tables } from "./tables" +import { selectedScreen } from "./screens" +import { viewsV2 } from "./viewsV2" import { DerivedBudiStore } from "../BudiStore" import { findComponentsBySettingsType } from "@/helpers/screen" import { Screen, Table, ViewV2 } from "@budibase/types" @@ -19,18 +21,9 @@ export class ScreenComponentStore extends DerivedBudiStore< return derived( [selectedScreen, tables, viewsV2], ([$selectedScreen, $tables, $viewsV2]): DerivedScreenComponentStore => { - function getErrors() { - const datasources = flattenTablesAndViews( - $tables.list, - $viewsV2.list - ) - return { - ...getInvalidDatasources($selectedScreen, datasources), - } - } - + const datasources = flattenTablesAndViews($tables.list, $viewsV2.list) return { - errors: getErrors(), + errors: getInvalidDatasources($selectedScreen, datasources), } } ) From 221021ae9b8051e3ca3a9257ef00c8bf33a0e08e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 21 Jan 2025 17:00:40 +0000 Subject: [PATCH 062/106] Support for our own vm-browserify implementation which re-uses the iframe for running JS in the frontend, to improve performance. --- packages/string-templates/package.json | 1 + packages/string-templates/src/index.ts | 6 +++--- yarn.lock | 18 +++++++++++++++--- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 1d7a4507ab..74d9aaa85a 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -23,6 +23,7 @@ }, "dependencies": { "@budibase/handlebars-helpers": "^0.13.2", + "@budibase/vm-browserify": "^1.1.3", "dayjs": "^1.10.8", "handlebars": "^4.7.8", "lodash.clonedeep": "^4.5.0" diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index 8dda8b71ab..bd008dd4d2 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -1,4 +1,4 @@ -import { createContext, runInNewContext } from "vm" +import vm from "@budibase/vm-browserify" import { create, TemplateDelegate } from "handlebars" import { registerAll, registerMinimum } from "./helpers/index" import { postprocess, postprocessWithLogs, preprocess } from "./processors" @@ -511,11 +511,11 @@ export function browserJSSetup() { * Use polyfilled vm to run JS scripts in a browser Env */ setJSRunner((js: string, context: Record) => { - createContext(context) + vm.createContext(context) const wrappedJs = frontendWrapJS(js) - const result = runInNewContext(wrappedJs, context, { timeout: 1000 }) + const result = vm.runInNewContext(wrappedJs, context) if (result.error) { throw new UserScriptError(result.error) } diff --git a/yarn.lock b/yarn.lock index 453dc45128..e25ff97747 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2131,9 +2131,9 @@ through2 "^2.0.0" "@budibase/pro@npm:@budibase/pro@latest": - version "3.2.44" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.2.44.tgz#90367bb2167aafd8c809e000a57d349e5dc4bb78" - integrity sha512-Zv2PBVUZUS6/psOpIRIDlW3jrOHWWPhpQXzCk00kIQJaqjkdcvuTXSedQ70u537sQmLu8JsSWbui9MdfF8ksVw== + version "3.2.47" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.2.47.tgz#150d7b16b14932d03c84bdb0e6d570d490c28a5c" + integrity sha512-UeTIq7yzMUK6w/akUsRafoD/Kif6PXv4d7K1arn8GTMjwFm9QYu2hg1YkQ+duNdwyZ/GEPlEAV5SYK+NDgtpdA== dependencies: "@anthropic-ai/sdk" "^0.27.3" "@budibase/backend-core" "*" @@ -2152,6 +2152,13 @@ scim-patch "^0.8.1" scim2-parse-filter "^0.2.8" +"@budibase/vm-browserify@^1.1.3": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@budibase/vm-browserify/-/vm-browserify-1.1.3.tgz#63f7917671a0f0cb760e3aa37cfd5dfa32e997ed" + integrity sha512-CuoNb2xwS8TT2ZfG9YqC8QCTcG3ZPLwH4m00sfPDluwmdp3U3HGg/UKWRIqKC6Wv8Mywy1q6bxmSx6Vf40V52w== + dependencies: + indexof "^0.0.1" + "@bull-board/api@5.10.2": version "5.10.2" resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3" @@ -11925,6 +11932,11 @@ indexes-of@^1.0.1: resolved "https://registry.yarnpkg.com/indexes-of/-/indexes-of-1.0.1.tgz#f30f716c8e2bd346c7b67d3df3915566a7c05607" integrity sha512-bup+4tap3Hympa+JBJUG7XuOsdNQ6fxt0MHyXMKuLBKn0OqsTfvUxkUrroEX1+B2VsSHvCjiIcZVxRtYa4nllA== +indexof@^0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/indexof/-/indexof-0.0.1.tgz#82dc336d232b9062179d05ab3293a66059fd435d" + integrity sha512-i0G7hLJ1z0DE8dsqJa2rycj9dBmNKgXBvotXtZYXakU9oivfB9Uj2ZBC27qqef2U58/ZLwalxa1X/RDCdkHtVg== + infer-owner@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/infer-owner/-/infer-owner-1.0.4.tgz#c4cefcaa8e51051c2a40ba2ce8a3d27295af9467" From 0e6ad4db930d7a50cd23264e9a2be3ef6c853585 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 21 Jan 2025 17:06:09 +0000 Subject: [PATCH 063/106] Remove testStep endpoint and fix types. --- .../server/src/api/controllers/automation.ts | 52 +------------------ packages/server/src/api/routes/automation.ts | 12 +---- .../server/src/automations/steps/createRow.ts | 9 ++-- .../server/src/automations/steps/deleteRow.ts | 9 ++-- .../src/automations/steps/executeQuery.ts | 4 +- .../src/automations/steps/executeScript.ts | 4 +- .../server/src/automations/steps/updateRow.ts | 9 ++-- .../server/src/automations/steps/utils.ts | 4 +- packages/server/src/events/NoopEmitter.ts | 39 -------------- packages/server/src/events/index.ts | 1 - .../src/tests/utilities/api/automation.ts | 17 ------ packages/types/src/api/web/app/automation.ts | 7 --- 12 files changed, 26 insertions(+), 141 deletions(-) delete mode 100644 packages/server/src/events/NoopEmitter.ts diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index a77014cf31..13d057ebb7 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -28,18 +28,11 @@ import { TriggerAutomationResponse, TestAutomationRequest, TestAutomationResponse, - TestAutomationStepRequest, - TestAutomationStepResponse, } from "@budibase/types" -import { - getActionDefinitions as actionDefs, - getAction, -} from "../../automations/actions" +import { getActionDefinitions as actionDefs } from "../../automations/actions" import sdk from "../../sdk" import { builderSocket } from "../../websockets" import env from "../../environment" -import { NoopEmitter } from "../../events" -import { enrichBaseContext } from "../../threads/automation" async function getActionDefinitions() { return removeDeprecated(await actionDefs()) @@ -260,46 +253,3 @@ export async function test( await events.automation.tested(automation) } - -export async function testStep( - ctx: UserCtx -) { - const { id, stepId } = ctx.params - const db = context.getAppDB() - const automation = await db.tryGet(id) - if (!automation) { - ctx.throw(404, `Automation ${ctx.params.id} not found`) - } - - const step = automation.definition.steps.find(s => s.stepId === stepId) - if (!step) { - ctx.throw(404, `Step ${stepId} not found on automation ${id}`) - } - - if (step.stepId === AutomationActionStepId.BRANCH) { - ctx.throw(400, "Branch steps cannot be tested directly") - } - if (step.stepId === AutomationActionStepId.LOOP) { - ctx.throw(400, "Loop steps cannot be tested directly") - } - - const { body } = ctx.request - - const fn = await getAction(step.stepId) - if (!fn) { - ctx.throw(400, `Step ${stepId} is not a valid step`) - } - - const output = await withTestFlag( - automation._id!, - async () => - await fn({ - inputs: body.inputs, - context: await enrichBaseContext(body.context), - appId: ctx.appId, - emitter: new NoopEmitter(), - }) - ) - - ctx.body = output -} diff --git a/packages/server/src/api/routes/automation.ts b/packages/server/src/api/routes/automation.ts index ea905be0cd..489487271c 100644 --- a/packages/server/src/api/routes/automation.ts +++ b/packages/server/src/api/routes/automation.ts @@ -1,6 +1,6 @@ import Router from "@koa/router" import * as controller from "../controllers/automation" -import authorized, { authorizedResource } from "../../middleware/authorized" +import authorized from "../../middleware/authorized" import { permissions } from "@budibase/backend-core" import { bodyResource, paramResource } from "../../middleware/resourceId" import { @@ -82,15 +82,5 @@ router ), controller.test ) - .post( - "/api/automations/:id/step/:stepId/test", - appInfoMiddleware({ appType: AppType.DEV }), - authorizedResource( - permissions.PermissionType.AUTOMATION, - permissions.PermissionLevel.EXECUTE, - "id" - ), - controller.testStep - ) export default router diff --git a/packages/server/src/automations/steps/createRow.ts b/packages/server/src/automations/steps/createRow.ts index 24dada422d..cf915dd300 100644 --- a/packages/server/src/automations/steps/createRow.ts +++ b/packages/server/src/automations/steps/createRow.ts @@ -5,8 +5,11 @@ import { sendAutomationAttachmentsToStorage, } from "../automationUtils" import { buildCtx } from "./utils" -import { CreateRowStepInputs, CreateRowStepOutputs } from "@budibase/types" -import { EventEmitter } from "events" +import { + ContextEmitter, + CreateRowStepInputs, + CreateRowStepOutputs, +} from "@budibase/types" export async function run({ inputs, @@ -15,7 +18,7 @@ export async function run({ }: { inputs: CreateRowStepInputs appId: string - emitter: EventEmitter + emitter: ContextEmitter }): Promise { if (inputs.row == null || inputs.row.tableId == null) { return { diff --git a/packages/server/src/automations/steps/deleteRow.ts b/packages/server/src/automations/steps/deleteRow.ts index 7c50fe4dcb..2498a4e4de 100644 --- a/packages/server/src/automations/steps/deleteRow.ts +++ b/packages/server/src/automations/steps/deleteRow.ts @@ -1,8 +1,11 @@ -import { EventEmitter } from "events" import { destroy } from "../../api/controllers/row" import { buildCtx } from "./utils" import { getError } from "../automationUtils" -import { DeleteRowStepInputs, DeleteRowStepOutputs } from "@budibase/types" +import { + ContextEmitter, + DeleteRowStepInputs, + DeleteRowStepOutputs, +} from "@budibase/types" export async function run({ inputs, @@ -11,7 +14,7 @@ export async function run({ }: { inputs: DeleteRowStepInputs appId: string - emitter: EventEmitter + emitter: ContextEmitter }): Promise { if (inputs.id == null) { return { diff --git a/packages/server/src/automations/steps/executeQuery.ts b/packages/server/src/automations/steps/executeQuery.ts index 9816e31b1e..ad99240eb8 100644 --- a/packages/server/src/automations/steps/executeQuery.ts +++ b/packages/server/src/automations/steps/executeQuery.ts @@ -1,8 +1,8 @@ -import { EventEmitter } from "events" import * as queryController from "../../api/controllers/query" import { buildCtx } from "./utils" import * as automationUtils from "../automationUtils" import { + ContextEmitter, ExecuteQueryStepInputs, ExecuteQueryStepOutputs, } from "@budibase/types" @@ -14,7 +14,7 @@ export async function run({ }: { inputs: ExecuteQueryStepInputs appId: string - emitter: EventEmitter + emitter: ContextEmitter }): Promise { if (inputs.query == null) { return { diff --git a/packages/server/src/automations/steps/executeScript.ts b/packages/server/src/automations/steps/executeScript.ts index 105543d34c..db05d0937a 100644 --- a/packages/server/src/automations/steps/executeScript.ts +++ b/packages/server/src/automations/steps/executeScript.ts @@ -2,10 +2,10 @@ import * as scriptController from "../../api/controllers/script" import { buildCtx } from "./utils" import * as automationUtils from "../automationUtils" import { + ContextEmitter, ExecuteScriptStepInputs, ExecuteScriptStepOutputs, } from "@budibase/types" -import { EventEmitter } from "events" export async function run({ inputs, @@ -16,7 +16,7 @@ export async function run({ inputs: ExecuteScriptStepInputs appId: string context: object - emitter: EventEmitter + emitter: ContextEmitter }): Promise { if (inputs.code == null) { return { diff --git a/packages/server/src/automations/steps/updateRow.ts b/packages/server/src/automations/steps/updateRow.ts index 46ae2a5c74..7a62e40706 100644 --- a/packages/server/src/automations/steps/updateRow.ts +++ b/packages/server/src/automations/steps/updateRow.ts @@ -1,8 +1,11 @@ -import { EventEmitter } from "events" import * as rowController from "../../api/controllers/row" import * as automationUtils from "../automationUtils" import { buildCtx } from "./utils" -import { UpdateRowStepInputs, UpdateRowStepOutputs } from "@budibase/types" +import { + ContextEmitter, + UpdateRowStepInputs, + UpdateRowStepOutputs, +} from "@budibase/types" export async function run({ inputs, @@ -11,7 +14,7 @@ export async function run({ }: { inputs: UpdateRowStepInputs appId: string - emitter: EventEmitter + emitter: ContextEmitter }): Promise { if (inputs.rowId == null || inputs.row == null) { return { diff --git a/packages/server/src/automations/steps/utils.ts b/packages/server/src/automations/steps/utils.ts index 8b99044303..20f1e67589 100644 --- a/packages/server/src/automations/steps/utils.ts +++ b/packages/server/src/automations/steps/utils.ts @@ -1,4 +1,4 @@ -import { EventEmitter } from "events" +import { ContextEmitter } from "@budibase/types" export async function getFetchResponse(fetched: any) { let status = fetched.status, @@ -22,7 +22,7 @@ export async function getFetchResponse(fetched: any) { // opts can contain, body, params and version export function buildCtx( appId: string, - emitter?: EventEmitter | null, + emitter?: ContextEmitter | null, opts: any = {} ) { const ctx: any = { diff --git a/packages/server/src/events/NoopEmitter.ts b/packages/server/src/events/NoopEmitter.ts deleted file mode 100644 index ed87618ead..0000000000 --- a/packages/server/src/events/NoopEmitter.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { EventEmitter } from "events" -import { - Table, - Row, - ContextEmitter, - EventType, - UserBindings, -} from "@budibase/types" - -export class NoopEmitter extends EventEmitter implements ContextEmitter { - emitRow(values: { - eventName: EventType.ROW_SAVE - appId: string - row: Row - table: Table - user: UserBindings - }): void - emitRow(values: { - eventName: EventType.ROW_UPDATE - appId: string - row: Row - table: Table - oldRow: Row - user: UserBindings - }): void - emitRow(values: { - eventName: EventType.ROW_DELETE - appId: string - row: Row - user: UserBindings - }): void - emitRow(_values: unknown): void { - return - } - - emitTable(_eventName: string, _appId: string, _table?: Table) { - return - } -} diff --git a/packages/server/src/events/index.ts b/packages/server/src/events/index.ts index 90bf932bcf..23c3f3e512 100644 --- a/packages/server/src/events/index.ts +++ b/packages/server/src/events/index.ts @@ -2,6 +2,5 @@ import BudibaseEmitter from "./BudibaseEmitter" const emitter = new BudibaseEmitter() -export { NoopEmitter } from "./NoopEmitter" export { init } from "./docUpdates" export default emitter diff --git a/packages/server/src/tests/utilities/api/automation.ts b/packages/server/src/tests/utilities/api/automation.ts index 6041664999..3f51385251 100644 --- a/packages/server/src/tests/utilities/api/automation.ts +++ b/packages/server/src/tests/utilities/api/automation.ts @@ -3,8 +3,6 @@ import { FetchAutomationResponse, TestAutomationRequest, TestAutomationResponse, - TestAutomationStepRequest, - TestAutomationStepResponse, } from "@budibase/types" import { Expectations, TestAPI } from "./base" @@ -54,19 +52,4 @@ export class AutomationAPI extends TestAPI { } ) } - - testStep = async ( - id: string, - stepId: string, - body: TestAutomationStepRequest, - expectations?: Expectations - ): Promise => { - return await this._post( - `/api/automations/${id}/steps/${stepId}/test`, - { - body, - expectations, - } - ) - } } diff --git a/packages/types/src/api/web/app/automation.ts b/packages/types/src/api/web/app/automation.ts index edff4b5eaf..b97dee0baf 100644 --- a/packages/types/src/api/web/app/automation.ts +++ b/packages/types/src/api/web/app/automation.ts @@ -83,10 +83,3 @@ export function isDidNotTriggerResponse( ): response is DidNotTriggerResponse { return !!("message" in response && response.message) } - -export interface TestAutomationStepRequest { - inputs: Record - context: Record -} - -export type TestAutomationStepResponse = any From cad6a08bf87570c2485cafc9538460a2cfee8f7e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 11:14:50 +0100 Subject: [PATCH 064/106] Move reactiviness --- packages/client/src/components/Component.svelte | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index 7f825c0601..57daf8cd12 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -139,6 +139,7 @@ // Derive definition properties which can all be optional, so need to be // coerced to booleans + $: invalidSettings = $componentErrors[instance._id] $: hasChildren = !!definition?.hasChildren $: showEmptyState = definition?.showEmptyState !== false $: hasMissingRequiredSettings = missingRequiredSettings?.length > 0 @@ -374,9 +375,6 @@ } } - // Check for invalid settings - $: invalidSettings = $componentErrors[id] - // Extracts a map of all context keys which are required by action settings // to provide the functions to evaluate at runtime. This needs done manually // as the action definitions themselves do not specify bindings for action From 4d85006c35e4d9b2643192902d077147069c6de8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 11:53:50 +0100 Subject: [PATCH 065/106] server nodemon, watch .hbs --- packages/server/nodemon.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/nodemon.json b/packages/server/nodemon.json index ac8c38ccb2..9871b9339e 100644 --- a/packages/server/nodemon.json +++ b/packages/server/nodemon.json @@ -7,7 +7,7 @@ "../shared-core/src", "../string-templates/src" ], - "ext": "js,ts,json,svelte", + "ext": "js,ts,json,svelte,hbs", "ignore": [ "**/*.spec.ts", "**/*.spec.js", From 1294f83ccb9d80f7bcacaf89e9cab7b43f6c4d70 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 22 Jan 2025 11:09:31 +0000 Subject: [PATCH 066/106] Bump version to 3.3.0 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 50582f0a95..a7d534ff01 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.2.47", + "version": "3.3.0", "npmClient": "yarn", "concurrency": 20, "command": { From 2213cd56c46c8f8204c98cd961dbd39482b57271 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 12:18:55 +0100 Subject: [PATCH 067/106] Remove unnecessary derived --- packages/client/src/components/Component.svelte | 3 +-- packages/client/src/stores/derived/componentErrors.ts | 6 ------ packages/client/src/stores/derived/index.js | 1 - 3 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 packages/client/src/stores/derived/componentErrors.ts diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index 57daf8cd12..8d79de2ac4 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -23,7 +23,6 @@ appStore, dndComponentPath, dndIsDragging, - componentErrors, } from "stores" import { Helpers } from "@budibase/bbui" import { getActiveConditions, reduceConditionActions } from "utils/conditions" @@ -139,7 +138,7 @@ // Derive definition properties which can all be optional, so need to be // coerced to booleans - $: invalidSettings = $componentErrors[instance._id] + $: invalidSettings = $builderStore.componentErrors[instance._id] $: hasChildren = !!definition?.hasChildren $: showEmptyState = definition?.showEmptyState !== false $: hasMissingRequiredSettings = missingRequiredSettings?.length > 0 diff --git a/packages/client/src/stores/derived/componentErrors.ts b/packages/client/src/stores/derived/componentErrors.ts deleted file mode 100644 index 68e87a061d..0000000000 --- a/packages/client/src/stores/derived/componentErrors.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { derived } from "svelte/store" -import { builderStore } from "../builder.js" - -export const componentErrors = derived([builderStore], ([$builderStore]) => { - return $builderStore.componentErrors -}) diff --git a/packages/client/src/stores/derived/index.js b/packages/client/src/stores/derived/index.js index e7e70d8952..337c73831f 100644 --- a/packages/client/src/stores/derived/index.js +++ b/packages/client/src/stores/derived/index.js @@ -5,4 +5,3 @@ export { currentRole } from "./currentRole.js" export { dndComponentPath } from "./dndComponentPath.js" export { devToolsEnabled } from "./devToolsEnabled.js" export { snippets } from "./snippets.js" -export { componentErrors } from "./componentErrors" From d3b22e461e3f0f328bab54e4b2b58eb6d833e28e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 16:18:29 +0100 Subject: [PATCH 068/106] Handle errors as a part of the instance to avoid extra refreshes --- packages/client/src/components/Component.svelte | 2 +- packages/client/src/stores/screens.js | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index 8d79de2ac4..a86d43c676 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -138,7 +138,7 @@ // Derive definition properties which can all be optional, so need to be // coerced to booleans - $: invalidSettings = $builderStore.componentErrors[instance._id] + $: invalidSettings = instance?._meta?.errors $: hasChildren = !!definition?.hasChildren $: showEmptyState = definition?.showEmptyState !== false $: hasMissingRequiredSettings = missingRequiredSettings?.length > 0 diff --git a/packages/client/src/stores/screens.js b/packages/client/src/stores/screens.js index bc87216660..491d8d4236 100644 --- a/packages/client/src/stores/screens.js +++ b/packages/client/src/stores/screens.js @@ -42,6 +42,14 @@ const createScreenStore = () => { if ($builderStore.layout) { activeLayout = $builderStore.layout } + + // Attach meta + const errors = $builderStore.componentErrors || {} + const attachComponentMeta = component => { + component._meta = { errors: errors[component._id] || [] } + component._children?.forEach(attachComponentMeta) + } + attachComponentMeta(activeScreen.props) } else { // Find the correct screen by matching the current route screens = $appStore.screens || [] From 84c8507bad7e3ff8df572cf389e004f546be264b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 16:20:33 +0100 Subject: [PATCH 069/106] Renames --- packages/client/src/components/Component.svelte | 10 +++++----- .../components/error-states/ComponentErrorState.svelte | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index a86d43c676..236bcb7c7e 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -103,7 +103,7 @@ let settingsDefinition let settingsDefinitionMap let missingRequiredSettings = false - let invalidSettings = false + let componentErrors = false // Temporary styles which can be added in the app preview for things like // DND. We clear these whenever a new instance is received. @@ -138,12 +138,12 @@ // Derive definition properties which can all be optional, so need to be // coerced to booleans - $: invalidSettings = instance?._meta?.errors + $: componentErrors = instance?._meta?.errors $: hasChildren = !!definition?.hasChildren $: showEmptyState = definition?.showEmptyState !== false $: hasMissingRequiredSettings = missingRequiredSettings?.length > 0 $: editable = !!definition?.editable && !hasMissingRequiredSettings - $: hasInvalidSettings = invalidSettings?.length > 0 + $: hasComponentErrors = componentErrors?.length > 0 $: requiredAncestors = definition?.requiredAncestors || [] $: missingRequiredAncestors = requiredAncestors.filter( ancestor => !$component.ancestors.includes(`${BudibasePrefix}${ancestor}`) @@ -152,7 +152,7 @@ $: errorState = hasMissingRequiredSettings || hasMissingRequiredAncestors || - hasInvalidSettings + hasComponentErrors // Interactive components can be selected, dragged and highlighted inside // the builder preview @@ -698,7 +698,7 @@ {:else} diff --git a/packages/client/src/components/error-states/ComponentErrorState.svelte b/packages/client/src/components/error-states/ComponentErrorState.svelte index 8cf1ad6dfb..9eace07018 100644 --- a/packages/client/src/components/error-states/ComponentErrorState.svelte +++ b/packages/client/src/components/error-states/ComponentErrorState.svelte @@ -8,7 +8,7 @@ | { key: string; label: string }[] | undefined export let missingRequiredAncestors: string[] | undefined - export let invalidSettings: string[] | undefined + export let componentErrors: string[] | undefined const component = getContext("component") const { styleable, builderStore } = getContext("sdk") @@ -16,7 +16,7 @@ $: styles = { ...$component.styles, normal: {}, custom: null, empty: true } $: requiredSetting = missingRequiredSettings?.[0] $: requiredAncestor = missingRequiredAncestors?.[0] - $: invalidSetting = invalidSettings?.[0] + $: errorMessage = componentErrors?.[0] {#if $builderStore.inBuilder} @@ -25,8 +25,8 @@ {#if requiredAncestor} - {:else if invalidSetting} - {invalidSetting} + {:else if errorMessage} + {errorMessage} {:else if requiredSetting} {/if} From c9feae9665a33466e9486c7ca97a2f42b067142c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 16:26:09 +0100 Subject: [PATCH 070/106] Simplify derived screenComponentErrors --- .../[screenId]/_components/AppPreview.svelte | 4 +- packages/builder/src/stores/builder/index.js | 4 +- .../src/stores/builder/screenComponent.ts | 112 +++++++----------- 3 files changed, 49 insertions(+), 71 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index fc0b67f63d..3951c0e902 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -11,7 +11,7 @@ selectedScreen, hoverStore, componentTreeNodesStore, - screenComponentStore, + screenComponentErrors, snippets, } from "@/stores/builder" import ConfirmDialog from "@/components/common/ConfirmDialog.svelte" @@ -69,7 +69,7 @@ port: window.location.port, }, snippets: $snippets, - componentErrors: $screenComponentStore.errors, + componentErrors: $screenComponentErrors, } // Refresh the preview when required diff --git a/packages/builder/src/stores/builder/index.js b/packages/builder/src/stores/builder/index.js index 23491996d1..892b72c7ab 100644 --- a/packages/builder/src/stores/builder/index.js +++ b/packages/builder/src/stores/builder/index.js @@ -16,7 +16,7 @@ import { userStore, userSelectedResourceMap, isOnlyUser } from "./users.js" import { deploymentStore } from "./deployments.js" import { contextMenuStore } from "./contextMenu.js" import { snippets } from "./snippets" -import { screenComponentStore } from "./screenComponent" +import { screenComponentErrors } from "./screenComponent" // Backend import { tables } from "./tables" @@ -68,7 +68,7 @@ export { snippets, rowActions, appPublished, - screenComponentStore, + screenComponentErrors, } export const reset = () => { diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index b4eb01a3b7..19bafeade3 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -2,79 +2,57 @@ import { derived } from "svelte/store" import { tables } from "./tables" import { selectedScreen } from "./screens" import { viewsV2 } from "./viewsV2" -import { DerivedBudiStore } from "../BudiStore" import { findComponentsBySettingsType } from "@/helpers/screen" import { Screen, Table, ViewV2 } from "@budibase/types" -interface BuilderScreenComponentStore {} +export const screenComponentErrors = derived( + [selectedScreen, tables, viewsV2], + ([$selectedScreen, $tables, $viewsV2]): Record => { + function flattenTablesAndViews(tables: Table[], views: ViewV2[]) { + return { + ...tables.reduce( + (list, table) => ({ + ...list, + [table._id!]: table, + }), + {} + ), + ...views.reduce( + (list, view) => ({ + ...list, + [view.id]: view, + }), + {} + ), + } + } -interface DerivedScreenComponentStore extends BuilderScreenComponentStore { - errors: Record -} + function getInvalidDatasources( + screen: Screen, + datasources: Record + ) { + const friendlyNameByType = { + table: "table", + view: "view", + viewV2: "view", + } -export class ScreenComponentStore extends DerivedBudiStore< - BuilderScreenComponentStore, - DerivedScreenComponentStore -> { - constructor() { - const makeDerivedStore = () => { - return derived( - [selectedScreen, tables, viewsV2], - ([$selectedScreen, $tables, $viewsV2]): DerivedScreenComponentStore => { - const datasources = flattenTablesAndViews($tables.list, $viewsV2.list) - return { - errors: getInvalidDatasources($selectedScreen, datasources), - } + const result: Record = {} + for (const component of findComponentsBySettingsType(screen, "table")) { + const { resourceId, type, label } = component.dataSource + if (!datasources[resourceId]) { + const friendlyTypeName = + friendlyNameByType[type as keyof typeof friendlyNameByType] + result[component._id!] = [ + `The ${friendlyTypeName} named "${label}" does not exist`, + ] } - ) + } + + return result } - super({}, makeDerivedStore) + const datasources = flattenTablesAndViews($tables.list, $viewsV2.list) + return getInvalidDatasources($selectedScreen, datasources) } -} - -export const screenComponentStore = new ScreenComponentStore() - -function flattenTablesAndViews(tables: Table[], views: ViewV2[]) { - return { - ...tables.reduce( - (list, table) => ({ - ...list, - [table._id!]: table, - }), - {} - ), - ...views.reduce( - (list, view) => ({ - ...list, - [view.id]: view, - }), - {} - ), - } -} - -function getInvalidDatasources( - screen: Screen, - datasources: Record -) { - const friendlyNameByType = { - table: "table", - view: "view", - viewV2: "view", - } - - const result: Record = {} - for (const component of findComponentsBySettingsType(screen, "table")) { - const { resourceId, type, label } = component.dataSource - if (!datasources[resourceId]) { - const friendlyTypeName = - friendlyNameByType[type as keyof typeof friendlyNameByType] - result[component._id!] = [ - `The ${friendlyTypeName} named "${label}" does not exist`, - ] - } - } - - return result -} +) From 2a5865ecaf6eb9de7a5a6e808bb0580e792ae630 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 16:38:29 +0100 Subject: [PATCH 071/106] Fix creating new table screen modal --- packages/builder/src/helpers/data/format.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/helpers/data/format.js b/packages/builder/src/helpers/data/format.js index 428ce273b2..ba274d5464 100644 --- a/packages/builder/src/helpers/data/format.js +++ b/packages/builder/src/helpers/data/format.js @@ -11,7 +11,7 @@ export const datasourceSelect = { }, viewV2: (view, datasources) => { const datasource = datasources - .filter(f => f.entities) + ?.filter(f => f.entities) .flatMap(d => d.entities) .find(ds => ds._id === view.tableId) return { From abc1ba33356b3a2a8cb67ca586e53c8e712d74dd Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 22 Jan 2025 15:46:49 +0000 Subject: [PATCH 072/106] Bump version to 3.3.1 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index a7d534ff01..13040cb50c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.3.0", + "version": "3.3.1", "npmClient": "yarn", "concurrency": 20, "command": { From fbbe1738db7a4006e7bbe8dfea73de26e7b4c33d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 22 Jan 2025 16:11:54 +0000 Subject: [PATCH 073/106] Using sandboxed iframe to limit to scripting only. --- packages/string-templates/package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 74d9aaa85a..b1b4b9ef55 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -23,7 +23,7 @@ }, "dependencies": { "@budibase/handlebars-helpers": "^0.13.2", - "@budibase/vm-browserify": "^1.1.3", + "@budibase/vm-browserify": "^1.1.4", "dayjs": "^1.10.8", "handlebars": "^4.7.8", "lodash.clonedeep": "^4.5.0" diff --git a/yarn.lock b/yarn.lock index e25ff97747..a375c05ffd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2152,10 +2152,10 @@ scim-patch "^0.8.1" scim2-parse-filter "^0.2.8" -"@budibase/vm-browserify@^1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@budibase/vm-browserify/-/vm-browserify-1.1.3.tgz#63f7917671a0f0cb760e3aa37cfd5dfa32e997ed" - integrity sha512-CuoNb2xwS8TT2ZfG9YqC8QCTcG3ZPLwH4m00sfPDluwmdp3U3HGg/UKWRIqKC6Wv8Mywy1q6bxmSx6Vf40V52w== +"@budibase/vm-browserify@^1.1.4": + version "1.1.4" + resolved "https://registry.yarnpkg.com/@budibase/vm-browserify/-/vm-browserify-1.1.4.tgz#eecb001bd9521cb7647e26fb4d2d29d0a4dce262" + integrity sha512-/dyOLj+jQNKe6sVfLP6NdwA79OZxEWHCa41VGsjKJC9DYo6l2fEcL5BNXq2pATqrbgWmOlEbcRulfZ+7W0QRUg== dependencies: indexof "^0.0.1" From 5c9cc915ff9c7e1da7538e282c02b6503bd006e3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 17:32:34 +0100 Subject: [PATCH 074/106] Remove magic string for settings --- packages/builder/src/helpers/screen.ts | 30 ++++++++++++++----- .../src/stores/builder/screenComponent.ts | 7 +++-- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/packages/builder/src/helpers/screen.ts b/packages/builder/src/helpers/screen.ts index e83805a511..71623844de 100644 --- a/packages/builder/src/helpers/screen.ts +++ b/packages/builder/src/helpers/screen.ts @@ -2,7 +2,13 @@ import { Component, Screen, ScreenProps } from "@budibase/types" import clientManifest from "@budibase/client/manifest.json" export function findComponentsBySettingsType(screen: Screen, type: string) { - const result: Component[] = [] + const result: { + component: Component + setting: { + type: string + key: string + } + }[] = [] function recurseFieldComponentsInChildren( component: ScreenProps, type: string @@ -11,14 +17,15 @@ export function findComponentsBySettingsType(screen: Screen, type: string) { return } - const componentType = component._component.split("/").slice(-1)[0] - const definition = - clientManifest[componentType as keyof typeof clientManifest] - if ( + const definition = getManifestDefinition(component) + const setting = "settings" in definition && - definition.settings.some((s: any) => s.type === type) - ) { - result.push(component) + definition.settings.find((s: any) => s.type === type) + if (setting && "type" in setting) { + result.push({ + component, + setting: { type: setting.type!, key: setting.key! }, + }) } component._children?.forEach(child => { recurseFieldComponentsInChildren(child, type) @@ -28,3 +35,10 @@ export function findComponentsBySettingsType(screen: Screen, type: string) { recurseFieldComponentsInChildren(screen?.props, type) return result } + +function getManifestDefinition(component: Component) { + const componentType = component._component.split("/").slice(-1)[0] + const definition = + clientManifest[componentType as keyof typeof clientManifest] + return definition +} diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 19bafeade3..a061158e6a 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -38,8 +38,11 @@ export const screenComponentErrors = derived( } const result: Record = {} - for (const component of findComponentsBySettingsType(screen, "table")) { - const { resourceId, type, label } = component.dataSource + for (const { component, setting } of findComponentsBySettingsType( + screen, + "table" + )) { + const { resourceId, type, label } = component[setting.key] if (!datasources[resourceId]) { const friendlyTypeName = friendlyNameByType[type as keyof typeof friendlyNameByType] From 5d60da471484ab190129f06490f3a1aec9744639 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 11:40:18 +0100 Subject: [PATCH 075/106] Fix null reference on viewV1 get schema --- packages/frontend-core/src/fetch/ViewFetch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/fetch/ViewFetch.ts b/packages/frontend-core/src/fetch/ViewFetch.ts index 6555896ae8..720f91eaab 100644 --- a/packages/frontend-core/src/fetch/ViewFetch.ts +++ b/packages/frontend-core/src/fetch/ViewFetch.ts @@ -21,7 +21,7 @@ export default class ViewFetch extends BaseDataFetch { getSchema(definition: Table) { const { datasource } = this.options - return definition?.views?.[datasource.name]?.schema + return definition?.views?.[datasource?.name]?.schema } async getData() { From fe2e93ee3e35ac35a64faf6569603eb80e8ac261 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 11:53:09 +0100 Subject: [PATCH 076/106] Remove todos --- .../src/components/grid/stores/datasource.ts | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.ts b/packages/frontend-core/src/components/grid/stores/datasource.ts index 588f373152..ebeade1050 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.ts +++ b/packages/frontend-core/src/components/grid/stores/datasource.ts @@ -1,5 +1,3 @@ -// TODO: datasource and defitions are unions of the different implementations. At this point, the datasource does not know what type is being used, and the assignations will cause TS exceptions. Casting it "as any" for now. This should be fixed improving the type usages. - import { derived, get, Readable, Writable } from "svelte/store" import { DataFetchDefinition, @@ -10,12 +8,10 @@ import { enrichSchemaWithRelColumns, memo } from "../../../utils" import { cloneDeep } from "lodash" import { SaveRowRequest, - SaveTableRequest, UIDatasource, UIFieldMutation, UIFieldSchema, UIRow, - UpdateViewRequest, ViewV2Type, } from "@budibase/types" import { Store as StoreContext, BaseStoreProps } from "." @@ -79,7 +75,7 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => { const schema = derived(definition, $definition => { const schema: Record | undefined = getDatasourceSchema({ API, - datasource: get(datasource) as any, // TODO: see line 1 + datasource: get(datasource), definition: $definition ?? undefined, }) if (!schema) { @@ -137,7 +133,7 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => { let type = $datasource?.type // @ts-expect-error if (type === "provider") { - type = ($datasource as any).value?.datasource?.type // TODO: see line 1 + type = ($datasource as any).value?.datasource?.type } // Handle calculation views if ( @@ -196,15 +192,13 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => { const refreshDefinition = async () => { const def = await getDatasourceDefinition({ API, - datasource: get(datasource) as any, // TODO: see line 1 + datasource: get(datasource), }) - definition.set(def as any) // TODO: see line 1 + definition.set(def ?? null) } // Saves the datasource definition - const saveDefinition = async ( - newDefinition: SaveTableRequest | UpdateViewRequest - ) => { + const saveDefinition = async (newDefinition: DataFetchDefinition) => { // Update local state const originalDefinition = get(definition) definition.set(newDefinition) @@ -212,7 +206,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => { // Update server if (get(config).canSaveSchema) { try { - await getAPI()?.actions.saveDefinition(newDefinition as never) + await getAPI()?.actions.saveDefinition(newDefinition) // Broadcast change so external state can be updated, as this change // will not be received by the builder websocket because we caused it @@ -245,7 +239,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => { delete newDefinition.schema[column].default } } - return await saveDefinition(newDefinition as any) // TODO: see line 1 + return await saveDefinition(newDefinition) } // Adds a schema mutation for a single field @@ -321,7 +315,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => { await saveDefinition({ ...$definition, schema: newSchema, - } as any) // TODO: see line 1 + }) resetSchemaMutations() } From 5f508ad7ca35b45151fdda4f931a46c1c476a714 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 12:07:12 +0100 Subject: [PATCH 077/106] Fix type --- packages/frontend-core/src/components/grid/stores/datasource.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.ts b/packages/frontend-core/src/components/grid/stores/datasource.ts index ebeade1050..5934c7c636 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.ts +++ b/packages/frontend-core/src/components/grid/stores/datasource.ts @@ -206,7 +206,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => { // Update server if (get(config).canSaveSchema) { try { - await getAPI()?.actions.saveDefinition(newDefinition) + await getAPI()?.actions.saveDefinition(newDefinition as never) // Broadcast change so external state can be updated, as this change // will not be received by the builder websocket because we caused it From 19bfd71096788096e584018088ca3514ed266282 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 12:25:10 +0100 Subject: [PATCH 078/106] Update types --- packages/frontend-core/src/fetch/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts index e12e74340c..547043145d 100644 --- a/packages/frontend-core/src/fetch/index.ts +++ b/packages/frontend-core/src/fetch/index.ts @@ -101,12 +101,12 @@ export const fetchData = < // Creates an empty fetch instance with no datasource configured, so no data // will initially be loaded -const createEmptyFetchInstance = ({ +const createEmptyFetchInstance = ({ API, datasource, }: { API: APIClient - datasource: DataFetchDatasource + datasource: T }) => { const handler = DataFetchMap[datasource?.type] if (!handler) { @@ -114,7 +114,7 @@ const createEmptyFetchInstance = ({ } return new handler({ API, - datasource: null as never, + datasource: datasource as any, query: null as any, }) } From 4c48ad6526d46308a22e7b38733447d07fbe30ec Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 15:26:18 +0100 Subject: [PATCH 079/106] Fix selected item for views v1 --- .../DataSourceSelect/DataSourceCategory.svelte | 16 ++++++++++++++-- .../DataSourceSelect/DataSourceSelect.svelte | 1 + 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte index 72e2fbf638..4ea8c63087 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte @@ -7,8 +7,21 @@ export let dataSet export let value export let onSelect + export let identifiers = ["resourceId"] $: displayDatasourceName = $datasources.list.length > 1 + + function isSelected(entry) { + if (!identifiers.length) { + return false + } + for (const identifier of identifiers) { + if (entry[identifier] !== value?.[identifier]) { + return false + } + } + return true + } {#if dividerState} @@ -24,8 +37,7 @@ {#each dataSet as data}
  • {/if} {#if queries?.length} From 8f02dff5cdfcbbf7f93f3d85be95b2cdeff1cef8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 15:29:59 +0100 Subject: [PATCH 080/106] Fix link selector --- .../settings/controls/DataSourceSelect/DataSourceSelect.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index 7c13032e42..1b7acd4a11 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -310,6 +310,7 @@ dataSet={links} {value} onSelect={handleSelected} + identifiers={["tableId", "fieldName"]} /> {/if} {#if fields?.length} From 102fbe9372e05e0a570bdc47068e15cd99309a2e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 15:31:07 +0100 Subject: [PATCH 081/106] Fix provider selector --- .../settings/controls/DataSourceSelect/DataSourceSelect.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index 1b7acd4a11..f8ee0876e7 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -338,6 +338,7 @@ dataSet={dataProviders} {value} onSelect={handleSelected} + identifiers={["providerId"]} /> {/if} Date: Thu, 23 Jan 2025 15:38:09 +0100 Subject: [PATCH 082/106] Fix fields selector --- .../settings/controls/DataSourceSelect/DataSourceSelect.svelte | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index f8ee0876e7..1b9fdcdf10 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -301,6 +301,7 @@ dataSet={queries} {value} onSelect={handleSelected} + identifiers={["_id"]} /> {/if} {#if links?.length} @@ -320,6 +321,7 @@ dataSet={fields} {value} onSelect={handleSelected} + identifiers={["providerId", "tableId", "fieldName"]} /> {/if} {#if jsonArrays?.length} @@ -329,6 +331,7 @@ dataSet={jsonArrays} {value} onSelect={handleSelected} + identifiers={["providerId", "tableId", "fieldName"]} /> {/if} {#if showDataProviders && dataProviders?.length} From 10a669e1d7a68093fa36f603d951ba653246ab73 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 17:16:11 +0100 Subject: [PATCH 083/106] Support searching for multiple types --- packages/builder/src/helpers/screen.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/builder/src/helpers/screen.ts b/packages/builder/src/helpers/screen.ts index 71623844de..296a597adb 100644 --- a/packages/builder/src/helpers/screen.ts +++ b/packages/builder/src/helpers/screen.ts @@ -1,7 +1,12 @@ import { Component, Screen, ScreenProps } from "@budibase/types" import clientManifest from "@budibase/client/manifest.json" -export function findComponentsBySettingsType(screen: Screen, type: string) { +export function findComponentsBySettingsType( + screen: Screen, + type: string | string[] +) { + const typesArray = Array.isArray(type) ? type : [type] + const result: { component: Component setting: { @@ -9,10 +14,7 @@ export function findComponentsBySettingsType(screen: Screen, type: string) { key: string } }[] = [] - function recurseFieldComponentsInChildren( - component: ScreenProps, - type: string - ) { + function recurseFieldComponentsInChildren(component: ScreenProps) { if (!component) { return } @@ -20,7 +22,7 @@ export function findComponentsBySettingsType(screen: Screen, type: string) { const definition = getManifestDefinition(component) const setting = "settings" in definition && - definition.settings.find((s: any) => s.type === type) + definition.settings.find((s: any) => typesArray.includes(s.type)) if (setting && "type" in setting) { result.push({ component, @@ -28,11 +30,11 @@ export function findComponentsBySettingsType(screen: Screen, type: string) { }) } component._children?.forEach(child => { - recurseFieldComponentsInChildren(child, type) + recurseFieldComponentsInChildren(child) }) } - recurseFieldComponentsInChildren(screen?.props, type) + recurseFieldComponentsInChildren(screen?.props) return result } From 1c23763813e49d9b320a12faa821f09d0f815c45 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 17:38:08 +0100 Subject: [PATCH 084/106] Handle dataSources as well --- packages/builder/src/stores/builder/screenComponent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index a061158e6a..426fcbc58a 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -40,7 +40,7 @@ export const screenComponentErrors = derived( const result: Record = {} for (const { component, setting } of findComponentsBySettingsType( screen, - "table" + ["table", "dataSource"] )) { const { resourceId, type, label } = component[setting.key] if (!datasources[resourceId]) { From 1f3c466028ca1166ea94d4714e2e27e80803d4cf Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 Jan 2025 18:00:03 +0100 Subject: [PATCH 085/106] DRY --- .../src/stores/builder/screenComponent.ts | 44 +++++++++---------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 426fcbc58a..6db9f43241 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -3,37 +3,29 @@ import { tables } from "./tables" import { selectedScreen } from "./screens" import { viewsV2 } from "./viewsV2" import { findComponentsBySettingsType } from "@/helpers/screen" -import { Screen, Table, ViewV2 } from "@budibase/types" +import { Screen } from "@budibase/types" + +function reduceBy( + key: TKey, + list: TItem[] +) { + return list.reduce( + (result, item) => ({ + ...result, + [item[key] as string]: item, + }), + {} + ) +} export const screenComponentErrors = derived( [selectedScreen, tables, viewsV2], ([$selectedScreen, $tables, $viewsV2]): Record => { - function flattenTablesAndViews(tables: Table[], views: ViewV2[]) { - return { - ...tables.reduce( - (list, table) => ({ - ...list, - [table._id!]: table, - }), - {} - ), - ...views.reduce( - (list, view) => ({ - ...list, - [view.id]: view, - }), - {} - ), - } - } - function getInvalidDatasources( screen: Screen, datasources: Record ) { const friendlyNameByType = { - table: "table", - view: "view", viewV2: "view", } @@ -45,7 +37,7 @@ export const screenComponentErrors = derived( const { resourceId, type, label } = component[setting.key] if (!datasources[resourceId]) { const friendlyTypeName = - friendlyNameByType[type as keyof typeof friendlyNameByType] + friendlyNameByType[type as keyof typeof friendlyNameByType] ?? type result[component._id!] = [ `The ${friendlyTypeName} named "${label}" does not exist`, ] @@ -55,7 +47,11 @@ export const screenComponentErrors = derived( return result } - const datasources = flattenTablesAndViews($tables.list, $viewsV2.list) + const datasources = { + ...reduceBy("_id", $tables.list), + ...reduceBy("id", $viewsV2.list), + } + return getInvalidDatasources($selectedScreen, datasources) } ) From 41b65a6b1df471d37fd04e9146f2ea7e9057f7dc Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 11:03:52 +0100 Subject: [PATCH 086/106] Validate queries --- .../src/stores/builder/screenComponent.ts | 29 +++++++++++++++---- .../types/src/documents/app/datasource.ts | 2 ++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 6db9f43241..a9444aee8c 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -3,7 +3,8 @@ import { tables } from "./tables" import { selectedScreen } from "./screens" import { viewsV2 } from "./viewsV2" import { findComponentsBySettingsType } from "@/helpers/screen" -import { Screen } from "@budibase/types" +import { DatasourceType, Screen } from "@budibase/types" +import { queries } from "./queries" function reduceBy( key: TKey, @@ -19,22 +20,39 @@ function reduceBy( } export const screenComponentErrors = derived( - [selectedScreen, tables, viewsV2], - ([$selectedScreen, $tables, $viewsV2]): Record => { + [selectedScreen, tables, viewsV2, queries], + ([$selectedScreen, $tables, $viewsV2, $queries]): Record< + string, + string[] + > => { function getInvalidDatasources( screen: Screen, datasources: Record ) { - const friendlyNameByType = { + const friendlyNameByType: Partial> = { viewV2: "view", } + const primaryKeyByType: Record = { + table: "resourceId", + view: "TODO", + viewV2: "resourceId", + query: "_id", + custom: "" as never, + } + const result: Record = {} for (const { component, setting } of findComponentsBySettingsType( screen, ["table", "dataSource"] )) { - const { resourceId, type, label } = component[setting.key] + const componentSettings = component[setting.key] + const { type, label } = componentSettings + if (type === "custom") { + continue + } + const resourceId = + componentSettings[primaryKeyByType[type as DatasourceType]] if (!datasources[resourceId]) { const friendlyTypeName = friendlyNameByType[type as keyof typeof friendlyNameByType] ?? type @@ -50,6 +68,7 @@ export const screenComponentErrors = derived( const datasources = { ...reduceBy("_id", $tables.list), ...reduceBy("id", $viewsV2.list), + ...reduceBy("_id", $queries.list), } return getInvalidDatasources($selectedScreen, datasources) diff --git a/packages/types/src/documents/app/datasource.ts b/packages/types/src/documents/app/datasource.ts index a0be7bd80d..27828c5455 100644 --- a/packages/types/src/documents/app/datasource.ts +++ b/packages/types/src/documents/app/datasource.ts @@ -57,3 +57,5 @@ export interface RestConfig { } dynamicVariables?: DynamicVariable[] } + +export type DatasourceType = "table" | "view" | "viewV2" | "query" | "custom" From 6e615a9907b0dc4508ccb921e5113bf3fa93779e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 11:19:49 +0100 Subject: [PATCH 087/106] Validate views v1 --- .../builder/src/stores/builder/screenComponent.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index a9444aee8c..2803acc953 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -5,6 +5,7 @@ import { viewsV2 } from "./viewsV2" import { findComponentsBySettingsType } from "@/helpers/screen" import { DatasourceType, Screen } from "@budibase/types" import { queries } from "./queries" +import { views } from "./views" function reduceBy( key: TKey, @@ -20,8 +21,8 @@ function reduceBy( } export const screenComponentErrors = derived( - [selectedScreen, tables, viewsV2, queries], - ([$selectedScreen, $tables, $viewsV2, $queries]): Record< + [selectedScreen, tables, views, viewsV2, queries], + ([$selectedScreen, $tables, $views, $viewsV2, $queries]): Record< string, string[] > => { @@ -34,9 +35,9 @@ export const screenComponentErrors = derived( } const primaryKeyByType: Record = { - table: "resourceId", - view: "TODO", - viewV2: "resourceId", + table: "tableId", + view: "name", + viewV2: "id", query: "_id", custom: "" as never, } @@ -67,6 +68,7 @@ export const screenComponentErrors = derived( const datasources = { ...reduceBy("_id", $tables.list), + ...reduceBy("name", $views.list), ...reduceBy("id", $viewsV2.list), ...reduceBy("_id", $queries.list), } From 92e2ae46f54f7e2f88baec2a72a08613106a4c76 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 16:17:08 +0100 Subject: [PATCH 088/106] Center error message --- .../src/components/error-states/ComponentErrorState.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/components/error-states/ComponentErrorState.svelte b/packages/client/src/components/error-states/ComponentErrorState.svelte index 9eace07018..7069b7a431 100644 --- a/packages/client/src/components/error-states/ComponentErrorState.svelte +++ b/packages/client/src/components/error-states/ComponentErrorState.svelte @@ -38,7 +38,7 @@ .component-placeholder { display: flex; flex-direction: row; - justify-content: flex-start; + justify-content: center; align-items: center; color: var(--spectrum-global-color-gray-600); font-size: var(--font-size-s); From afb9f86bf7aa2f7b10814e215da1afc5ecf3993b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 23 Jan 2025 17:23:08 +0000 Subject: [PATCH 089/106] Using node vm for string-template test cases. --- packages/string-templates/src/index.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index bd008dd4d2..79b0aa0699 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -1,4 +1,5 @@ -import vm from "@budibase/vm-browserify" +import browserVM from "@budibase/vm-browserify" +import vm from "vm" import { create, TemplateDelegate } from "handlebars" import { registerAll, registerMinimum } from "./helpers/index" import { postprocess, postprocessWithLogs, preprocess } from "./processors" @@ -14,10 +15,10 @@ import { } from "./utilities" import { convertHBSBlock } from "./conversion" import { removeJSRunner, setJSRunner } from "./helpers/javascript" - import manifest from "./manifest.json" import { Log, ProcessOptions } from "./types" import { UserScriptError } from "./errors" +import { isTest } from "./environment" export type { Log, LogType } from "./types" export { setTestingBackendJS } from "./environment" @@ -507,15 +508,15 @@ export function convertToJS(hbs: string) { export { JsTimeoutError, UserScriptError } from "./errors" export function browserJSSetup() { - /** - * Use polyfilled vm to run JS scripts in a browser Env - */ + // tests are in jest - we need to use node VM for these + const jsSandbox = isTest() ? vm : browserVM + // Use polyfilled vm to run JS scripts in a browser Env setJSRunner((js: string, context: Record) => { - vm.createContext(context) + jsSandbox.createContext(context) const wrappedJs = frontendWrapJS(js) - const result = vm.runInNewContext(wrappedJs, context) + const result = jsSandbox.runInNewContext(wrappedJs, context) if (result.error) { throw new UserScriptError(result.error) } From cf9b61c8c80a889c41a3cc017e431239aa70d3fc Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 23 Jan 2025 17:35:50 +0000 Subject: [PATCH 090/106] Fix selecting 'old row' in row updated trigger. --- .../automation/SetupPanel/AutomationBlockSetup.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 16183ea59a..e713d9bc85 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -293,7 +293,7 @@ type: RowSelector, props: { row: inputData["oldRow"] || { - tableId: inputData["row"].tableId, + tableId: inputData["row"]?.tableId, }, meta: { fields: inputData["meta"]?.oldFields || {}, From 388a94aee1f7b13e2baf3527a89103efff649499 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 Jan 2025 19:34:03 +0100 Subject: [PATCH 091/106] Add feature flag for datasource setting checks --- packages/builder/src/helpers/index.ts | 1 + packages/builder/src/stores/builder/screenComponent.ts | 6 ++++++ packages/types/src/sdk/featureFlag.ts | 2 ++ 3 files changed, 9 insertions(+) diff --git a/packages/builder/src/helpers/index.ts b/packages/builder/src/helpers/index.ts index 687723e361..81afc696a3 100644 --- a/packages/builder/src/helpers/index.ts +++ b/packages/builder/src/helpers/index.ts @@ -9,3 +9,4 @@ export { lowercase, isBuilderInputFocused, } from "./helpers" +export * as featureFlag from "./featureFlags" diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index a061158e6a..079401891c 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -5,9 +5,15 @@ import { viewsV2 } from "./viewsV2" import { findComponentsBySettingsType } from "@/helpers/screen" import { Screen, Table, ViewV2 } from "@budibase/types" +import { featureFlag } from "@/helpers" + export const screenComponentErrors = derived( [selectedScreen, tables, viewsV2], ([$selectedScreen, $tables, $viewsV2]): Record => { + if (!featureFlag.isEnabled("CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS")) { + return {} + } + function flattenTablesAndViews(tables: Table[], views: ViewV2[]) { return { ...tables.reduce( diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts index 996d3bba8d..d9f092c80a 100644 --- a/packages/types/src/sdk/featureFlag.ts +++ b/packages/types/src/sdk/featureFlag.ts @@ -1,5 +1,6 @@ export enum FeatureFlag { USE_ZOD_VALIDATOR = "USE_ZOD_VALIDATOR", + CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS = "CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS", // Account-portal DIRECT_LOGIN_TO_ACCOUNT_PORTAL = "DIRECT_LOGIN_TO_ACCOUNT_PORTAL", @@ -7,6 +8,7 @@ export enum FeatureFlag { export const FeatureFlagDefaults = { [FeatureFlag.USE_ZOD_VALIDATOR]: false, + [FeatureFlag.CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS]: false, // Account-portal [FeatureFlag.DIRECT_LOGIN_TO_ACCOUNT_PORTAL]: false, From afee1ac993a4547791f8504d975f447ac40fd70b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 24 Jan 2025 09:42:29 +0100 Subject: [PATCH 092/106] Whitelist checks --- .../src/stores/builder/screenComponent.ts | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 2803acc953..6578334310 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -20,6 +20,18 @@ function reduceBy( ) } +const friendlyNameByType: Partial> = { + viewV2: "view", +} + +const validationKeyByType: Record = { + table: "tableId", + view: "name", + viewV2: "id", + query: "_id", + custom: null, +} + export const screenComponentErrors = derived( [selectedScreen, tables, views, viewsV2, queries], ([$selectedScreen, $tables, $views, $viewsV2, $queries]): Record< @@ -30,35 +42,24 @@ export const screenComponentErrors = derived( screen: Screen, datasources: Record ) { - const friendlyNameByType: Partial> = { - viewV2: "view", - } - - const primaryKeyByType: Record = { - table: "tableId", - view: "name", - viewV2: "id", - query: "_id", - custom: "" as never, - } - const result: Record = {} for (const { component, setting } of findComponentsBySettingsType( screen, ["table", "dataSource"] )) { const componentSettings = component[setting.key] - const { type, label } = componentSettings - if (type === "custom") { + const { label } = componentSettings + const type = componentSettings as DatasourceType + + const validationKey = validationKeyByType[type] + if (!validationKey) { continue } - const resourceId = - componentSettings[primaryKeyByType[type as DatasourceType]] + const resourceId = componentSettings[validationKey] if (!datasources[resourceId]) { - const friendlyTypeName = - friendlyNameByType[type as keyof typeof friendlyNameByType] ?? type + const friendlyTypeName = friendlyNameByType[type] ?? type result[component._id!] = [ - `The ${friendlyTypeName} named "${label}" does not exist`, + `The ${friendlyTypeName} named "${label}" could not be found`, ] } } From 8497a060407870973d8fe0494b71056df00d932c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 24 Jan 2025 09:53:37 +0100 Subject: [PATCH 093/106] Fix --- packages/builder/src/stores/builder/screenComponent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 6578334310..6fb1a7d134 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -49,7 +49,7 @@ export const screenComponentErrors = derived( )) { const componentSettings = component[setting.key] const { label } = componentSettings - const type = componentSettings as DatasourceType + const type = componentSettings.type as DatasourceType const validationKey = validationKeyByType[type] if (!validationKey) { From 1f28bf978b79e804d6dbf5bb44db9a4f887574e8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 24 Jan 2025 10:23:02 +0100 Subject: [PATCH 094/106] Move type to ui --- packages/builder/src/stores/builder/screenComponent.ts | 8 ++++---- packages/types/src/documents/app/datasource.ts | 2 -- packages/types/src/ui/datasource.ts | 1 + packages/types/src/ui/index.ts | 1 + 4 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 packages/types/src/ui/datasource.ts diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 547a8170a8..d8169fdedb 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -3,7 +3,7 @@ import { tables } from "./tables" import { selectedScreen } from "./screens" import { viewsV2 } from "./viewsV2" import { findComponentsBySettingsType } from "@/helpers/screen" -import { DatasourceType, Screen } from "@budibase/types" +import { UIDatasourceType, Screen } from "@budibase/types" import { queries } from "./queries" import { views } from "./views" import { featureFlag } from "@/helpers" @@ -21,11 +21,11 @@ function reduceBy( ) } -const friendlyNameByType: Partial> = { +const friendlyNameByType: Partial> = { viewV2: "view", } -const validationKeyByType: Record = { +const validationKeyByType: Record = { table: "tableId", view: "name", viewV2: "id", @@ -53,7 +53,7 @@ export const screenComponentErrors = derived( )) { const componentSettings = component[setting.key] const { label } = componentSettings - const type = componentSettings.type as DatasourceType + const type = componentSettings.type as UIDatasourceType const validationKey = validationKeyByType[type] if (!validationKey) { diff --git a/packages/types/src/documents/app/datasource.ts b/packages/types/src/documents/app/datasource.ts index 27828c5455..a0be7bd80d 100644 --- a/packages/types/src/documents/app/datasource.ts +++ b/packages/types/src/documents/app/datasource.ts @@ -57,5 +57,3 @@ export interface RestConfig { } dynamicVariables?: DynamicVariable[] } - -export type DatasourceType = "table" | "view" | "viewV2" | "query" | "custom" diff --git a/packages/types/src/ui/datasource.ts b/packages/types/src/ui/datasource.ts new file mode 100644 index 0000000000..53740e8c4d --- /dev/null +++ b/packages/types/src/ui/datasource.ts @@ -0,0 +1 @@ +export type UIDatasourceType = "table" | "view" | "viewV2" | "query" | "custom" diff --git a/packages/types/src/ui/index.ts b/packages/types/src/ui/index.ts index 907f4ec0b5..6e5f37608c 100644 --- a/packages/types/src/ui/index.ts +++ b/packages/types/src/ui/index.ts @@ -2,3 +2,4 @@ export * from "./stores" export * from "./bindings" export * from "./components" export * from "./dataFetch" +export * from "./datasource" From bf05fea3edbeef0706dc6d28dbd74165ad87881b Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Fri, 24 Jan 2025 11:41:27 +0000 Subject: [PATCH 095/106] Removing while loop test - frontend JS cannot timeout. --- .../string-templates/test/javascript.spec.ts | 5 --- yarn.lock | 36 +++++-------------- 2 files changed, 8 insertions(+), 33 deletions(-) diff --git a/packages/string-templates/test/javascript.spec.ts b/packages/string-templates/test/javascript.spec.ts index 9134005acb..631fe828ae 100644 --- a/packages/string-templates/test/javascript.spec.ts +++ b/packages/string-templates/test/javascript.spec.ts @@ -125,11 +125,6 @@ describe("Javascript", () => { expect(processJS(`throw "Error"`)).toEqual("Error") }) - it("should timeout after one second", () => { - const output = processJS(`while (true) {}`) - expect(output).toBe("Timed out while executing JS") - }) - it("should prevent access to the process global", async () => { expect(processJS(`return process`)).toEqual( "ReferenceError: process is not defined" diff --git a/yarn.lock b/yarn.lock index a375c05ffd..8647f40b79 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11807,6 +11807,11 @@ husky@^8.0.3: resolved "https://registry.yarnpkg.com/husky/-/husky-8.0.3.tgz#4936d7212e46d1dea28fef29bb3a108872cd9184" integrity sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg== +husky@^9.1.4: + version "9.1.7" + resolved "https://registry.yarnpkg.com/husky/-/husky-9.1.7.tgz#d46a38035d101b46a70456a850ff4201344c0b2d" + integrity sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA== + ical-generator@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/ical-generator/-/ical-generator-4.1.0.tgz#2a336c951864c5583a2aa715d16f2edcdfd2d90b" @@ -18658,16 +18663,7 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -18759,7 +18755,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -18773,13 +18769,6 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -20527,7 +20516,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -20545,15 +20534,6 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From 8dd1366df27d82d15f3a05e71d4f6db57893408b Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Fri, 24 Jan 2025 13:02:00 +0000 Subject: [PATCH 096/106] Fixing yarn.lock (after removing account portal ref). --- yarn.lock | 5 ----- 1 file changed, 5 deletions(-) diff --git a/yarn.lock b/yarn.lock index 8647f40b79..60294cc5ab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11807,11 +11807,6 @@ husky@^8.0.3: resolved "https://registry.yarnpkg.com/husky/-/husky-8.0.3.tgz#4936d7212e46d1dea28fef29bb3a108872cd9184" integrity sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg== -husky@^9.1.4: - version "9.1.7" - resolved "https://registry.yarnpkg.com/husky/-/husky-9.1.7.tgz#d46a38035d101b46a70456a850ff4201344c0b2d" - integrity sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA== - ical-generator@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/ical-generator/-/ical-generator-4.1.0.tgz#2a336c951864c5583a2aa715d16f2edcdfd2d90b" From b8d38159d09659b4b7d04077385e1cbb4832b1ce Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 24 Jan 2025 14:48:27 +0100 Subject: [PATCH 097/106] Extract binding utils --- .../DataSourceSelect/DataSourceSelect.svelte | 43 ++-------------- packages/builder/src/helpers/bindings.ts | 50 +++++++++++++++++++ packages/builder/src/helpers/index.ts | 1 + packages/types/src/ui/bindings/binding.ts | 13 +++++ 4 files changed, 67 insertions(+), 40 deletions(-) create mode 100644 packages/builder/src/helpers/bindings.ts diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index 1b9fdcdf10..6ccde7fede 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -31,6 +31,7 @@ import IntegrationQueryEditor from "@/components/integration/index.svelte" import { makePropSafe as safe } from "@budibase/string-templates" import { findAllComponents } from "@/helpers/components" + import { extractFields, extractRelationships } from "@/helpers/bindings" import ClientBindingPanel from "@/components/common/bindings/ClientBindingPanel.svelte" import DataSourceCategory from "@/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte" import { API } from "@/api" @@ -81,46 +82,8 @@ value: `{{ literal ${safe(provider._id)} }}`, type: "provider", })) - $: links = bindings - // Get only link bindings - .filter(x => x.fieldSchema?.type === "link") - // Filter out bindings provided by forms - .filter(x => !x.component?.endsWith("/form")) - .map(binding => { - const { providerId, readableBinding, fieldSchema } = binding || {} - const { name, tableId } = fieldSchema || {} - const safeProviderId = safe(providerId) - return { - providerId, - label: readableBinding, - fieldName: name, - tableId, - type: "link", - // These properties will be enriched by the client library and provide - // details of the parent row of the relationship field, from context - rowId: `{{ ${safeProviderId}.${safe("_id")} }}`, - rowTableId: `{{ ${safeProviderId}.${safe("tableId")} }}`, - } - }) - $: fields = bindings - .filter( - x => - x.fieldSchema?.type === "attachment" || - (x.fieldSchema?.type === "array" && x.tableId) - ) - .map(binding => { - const { providerId, readableBinding, runtimeBinding } = binding - const { name, type, tableId } = binding.fieldSchema - return { - providerId, - label: readableBinding, - fieldName: name, - fieldType: type, - tableId, - type: "field", - value: `{{ literal ${runtimeBinding} }}`, - } - }) + $: links = extractRelationships(bindings) + $: fields = extractFields(bindings) $: jsonArrays = bindings .filter( x => diff --git a/packages/builder/src/helpers/bindings.ts b/packages/builder/src/helpers/bindings.ts new file mode 100644 index 0000000000..2d1410c76a --- /dev/null +++ b/packages/builder/src/helpers/bindings.ts @@ -0,0 +1,50 @@ +import { makePropSafe } from "@budibase/string-templates" +import { UIBinding } from "@budibase/types" + +export function extractRelationships(bindings: UIBinding[]) { + return ( + bindings + // Get only link bindings + .filter(x => x.fieldSchema?.type === "link") + // Filter out bindings provided by forms + .filter(x => !x.component?.endsWith("/form")) + .map(binding => { + const { providerId, readableBinding, fieldSchema } = binding || {} + const { name, tableId } = fieldSchema || {} + const safeProviderId = makePropSafe(providerId) + return { + providerId, + label: readableBinding, + fieldName: name, + tableId, + type: "link", + // These properties will be enriched by the client library and provide + // details of the parent row of the relationship field, from context + rowId: `{{ ${safeProviderId}.${makePropSafe("_id")} }}`, + rowTableId: `{{ ${safeProviderId}.${makePropSafe("tableId")} }}`, + } + }) + ) +} + +export function extractFields(bindings: UIBinding[]) { + return bindings + .filter( + x => + x.fieldSchema?.type === "attachment" || + (x.fieldSchema?.type === "array" && x.tableId) + ) + .map(binding => { + const { providerId, readableBinding, runtimeBinding } = binding + const { name, type, tableId } = binding.fieldSchema! + return { + providerId, + label: readableBinding, + fieldName: name, + fieldType: type, + tableId, + type: "field", + value: `{{ literal ${runtimeBinding} }}`, + } + }) +} diff --git a/packages/builder/src/helpers/index.ts b/packages/builder/src/helpers/index.ts index 81afc696a3..0e61eeb9c6 100644 --- a/packages/builder/src/helpers/index.ts +++ b/packages/builder/src/helpers/index.ts @@ -10,3 +10,4 @@ export { isBuilderInputFocused, } from "./helpers" export * as featureFlag from "./featureFlags" +export * as bindings from "./bindings" diff --git a/packages/types/src/ui/bindings/binding.ts b/packages/types/src/ui/bindings/binding.ts index 2cfb23ed2d..10e6968409 100644 --- a/packages/types/src/ui/bindings/binding.ts +++ b/packages/types/src/ui/bindings/binding.ts @@ -24,3 +24,16 @@ export type InsertAtPositionFn = (_: { value: string cursor?: { anchor: number } }) => void + +export interface UIBinding { + tableId?: string + fieldSchema?: { + name: string + tableId: string + type: string + } + component?: string + providerId: string + readableBinding?: string + runtimeBinding?: string +} From 5f3aaf458b593f0ecbab53553713249b4e60f7fc Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 24 Jan 2025 14:50:06 +0100 Subject: [PATCH 098/106] Extract binding utils --- .../DataSourceSelect/DataSourceSelect.svelte | 28 ++++--------------- packages/builder/src/helpers/bindings.ts | 24 ++++++++++++++++ packages/types/src/ui/bindings/binding.ts | 2 ++ 3 files changed, 32 insertions(+), 22 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index 6ccde7fede..f5028b1eea 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -31,7 +31,11 @@ import IntegrationQueryEditor from "@/components/integration/index.svelte" import { makePropSafe as safe } from "@budibase/string-templates" import { findAllComponents } from "@/helpers/components" - import { extractFields, extractRelationships } from "@/helpers/bindings" + import { + extractFields, + extractJSONArrayFields, + extractRelationships, + } from "@/helpers/bindings" import ClientBindingPanel from "@/components/common/bindings/ClientBindingPanel.svelte" import DataSourceCategory from "@/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte" import { API } from "@/api" @@ -84,27 +88,7 @@ })) $: links = extractRelationships(bindings) $: fields = extractFields(bindings) - $: jsonArrays = bindings - .filter( - x => - x.fieldSchema?.type === "jsonarray" || - (x.fieldSchema?.type === "json" && x.fieldSchema?.subtype === "array") - ) - .map(binding => { - const { providerId, readableBinding, runtimeBinding, tableId } = binding - const { name, type, prefixKeys, subtype } = binding.fieldSchema - return { - providerId, - label: readableBinding, - fieldName: name, - fieldType: type, - tableId, - prefixKeys, - type: type === "jsonarray" ? "jsonarray" : "queryarray", - subtype, - value: `{{ literal ${runtimeBinding} }}`, - } - }) + $: jsonArrays = extractJSONArrayFields(bindings) $: custom = { type: "custom", label: "JSON / CSV", diff --git a/packages/builder/src/helpers/bindings.ts b/packages/builder/src/helpers/bindings.ts index 2d1410c76a..66a13d9ba3 100644 --- a/packages/builder/src/helpers/bindings.ts +++ b/packages/builder/src/helpers/bindings.ts @@ -48,3 +48,27 @@ export function extractFields(bindings: UIBinding[]) { } }) } + +export function extractJSONArrayFields(bindings: UIBinding[]) { + return bindings + .filter( + x => + x.fieldSchema?.type === "jsonarray" || + (x.fieldSchema?.type === "json" && x.fieldSchema?.subtype === "array") + ) + .map(binding => { + const { providerId, readableBinding, runtimeBinding, tableId } = binding + const { name, type, prefixKeys, subtype } = binding.fieldSchema! + return { + providerId, + label: readableBinding, + fieldName: name, + fieldType: type, + tableId, + prefixKeys, + type: type === "jsonarray" ? "jsonarray" : "queryarray", + subtype, + value: `{{ literal ${runtimeBinding} }}`, + } + }) +} diff --git a/packages/types/src/ui/bindings/binding.ts b/packages/types/src/ui/bindings/binding.ts index 10e6968409..a770a25a3e 100644 --- a/packages/types/src/ui/bindings/binding.ts +++ b/packages/types/src/ui/bindings/binding.ts @@ -31,6 +31,8 @@ export interface UIBinding { name: string tableId: string type: string + subtype?: string + prefixKeys?: string } component?: string providerId: string From 08c4cfcec0a4c5ce3691b5dc87b243048d6a65e4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 27 Jan 2025 10:21:31 +0100 Subject: [PATCH 099/106] Validate links --- .../src/stores/builder/screenComponent.ts | 32 +++++++++++++++---- packages/types/src/ui/datasource.ts | 8 ++++- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index d8169fdedb..aa6a854f77 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -6,7 +6,8 @@ import { findComponentsBySettingsType } from "@/helpers/screen" import { UIDatasourceType, Screen } from "@budibase/types" import { queries } from "./queries" import { views } from "./views" -import { featureFlag } from "@/helpers" +import { bindings, featureFlag } from "@/helpers" +import { screenComponentBindableProperties } from "./bindings" function reduceBy( key: TKey, @@ -31,14 +32,26 @@ const validationKeyByType: Record = { viewV2: "id", query: "_id", custom: null, + link: "rowId", } export const screenComponentErrors = derived( - [selectedScreen, tables, views, viewsV2, queries], - ([$selectedScreen, $tables, $views, $viewsV2, $queries]): Record< - string, - string[] - > => { + [ + selectedScreen, + tables, + views, + viewsV2, + queries, + screenComponentBindableProperties, + ], + ([ + $selectedScreen, + $tables, + $views, + $viewsV2, + $queries, + $screenComponentBindableProperties, + ]): Record => { if (!featureFlag.isEnabled("CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS")) { return {} } @@ -56,6 +69,9 @@ export const screenComponentErrors = derived( const type = componentSettings.type as UIDatasourceType const validationKey = validationKeyByType[type] + if (type === "link") { + debugger + } if (!validationKey) { continue } @@ -76,6 +92,10 @@ export const screenComponentErrors = derived( ...reduceBy("name", $views.list), ...reduceBy("id", $viewsV2.list), ...reduceBy("_id", $queries.list), + ...reduceBy( + "rowId", + bindings.extractRelationships($screenComponentBindableProperties) + ), } return getInvalidDatasources($selectedScreen, datasources) diff --git a/packages/types/src/ui/datasource.ts b/packages/types/src/ui/datasource.ts index 53740e8c4d..a121d929c8 100644 --- a/packages/types/src/ui/datasource.ts +++ b/packages/types/src/ui/datasource.ts @@ -1 +1,7 @@ -export type UIDatasourceType = "table" | "view" | "viewV2" | "query" | "custom" +export type UIDatasourceType = + | "table" + | "view" + | "viewV2" + | "query" + | "custom" + | "link" From 52330a30b84e8c231e92d03a020f05f5fc984eb9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 27 Jan 2025 10:23:26 +0100 Subject: [PATCH 100/106] Remove debugger --- packages/builder/src/stores/builder/screenComponent.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index aa6a854f77..1af10e8a80 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -69,9 +69,6 @@ export const screenComponentErrors = derived( const type = componentSettings.type as UIDatasourceType const validationKey = validationKeyByType[type] - if (type === "link") { - debugger - } if (!validationKey) { continue } From 92606c6129458bd499320d6994b7ddae7f72cabc Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 27 Jan 2025 10:28:54 +0100 Subject: [PATCH 101/106] Validate in all screen --- .../src/stores/builder/screenComponent.ts | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 1af10e8a80..3afb96994f 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -7,12 +7,12 @@ import { UIDatasourceType, Screen } from "@budibase/types" import { queries } from "./queries" import { views } from "./views" import { bindings, featureFlag } from "@/helpers" -import { screenComponentBindableProperties } from "./bindings" +import { getBindableProperties } from "@/dataBinding" function reduceBy( key: TKey, list: TItem[] -) { +): Record { return list.reduce( (result, item) => ({ ...result, @@ -36,22 +36,11 @@ const validationKeyByType: Record = { } export const screenComponentErrors = derived( - [ - selectedScreen, - tables, - views, - viewsV2, - queries, - screenComponentBindableProperties, - ], - ([ - $selectedScreen, - $tables, - $views, - $viewsV2, - $queries, - $screenComponentBindableProperties, - ]): Record => { + [selectedScreen, tables, views, viewsV2, queries], + ([$selectedScreen, $tables, $views, $viewsV2, $queries]): Record< + string, + string[] + > => { if (!featureFlag.isEnabled("CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS")) { return {} } @@ -72,8 +61,21 @@ export const screenComponentErrors = derived( if (!validationKey) { continue } + + const componentBindings = getBindableProperties( + $selectedScreen, + component._id + ) + + const componentDatasources = { + ...reduceBy( + "rowId", + bindings.extractRelationships(componentBindings) + ), + } + const resourceId = componentSettings[validationKey] - if (!datasources[resourceId]) { + if (!{ ...datasources, ...componentDatasources }[resourceId]) { const friendlyTypeName = friendlyNameByType[type] ?? type result[component._id!] = [ `The ${friendlyTypeName} named "${label}" could not be found`, @@ -89,10 +91,6 @@ export const screenComponentErrors = derived( ...reduceBy("name", $views.list), ...reduceBy("id", $viewsV2.list), ...reduceBy("_id", $queries.list), - ...reduceBy( - "rowId", - bindings.extractRelationships($screenComponentBindableProperties) - ), } return getInvalidDatasources($selectedScreen, datasources) From f666b35bd181a9aa8f311f58e236344f3f91caef Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 27 Jan 2025 11:48:00 +0100 Subject: [PATCH 102/106] Validate by field --- packages/builder/src/stores/builder/screenComponent.ts | 2 ++ packages/types/src/ui/datasource.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 3afb96994f..aecc27c4e5 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -33,6 +33,7 @@ const validationKeyByType: Record = { query: "_id", custom: null, link: "rowId", + field: "label", } export const screenComponentErrors = derived( @@ -72,6 +73,7 @@ export const screenComponentErrors = derived( "rowId", bindings.extractRelationships(componentBindings) ), + ...reduceBy("label", bindings.extractFields(componentBindings)), } const resourceId = componentSettings[validationKey] diff --git a/packages/types/src/ui/datasource.ts b/packages/types/src/ui/datasource.ts index a121d929c8..c6b1ed01d1 100644 --- a/packages/types/src/ui/datasource.ts +++ b/packages/types/src/ui/datasource.ts @@ -5,3 +5,4 @@ export type UIDatasourceType = | "query" | "custom" | "link" + | "field" From 73d7f985bfaa523715bb2eee0eb6b4120e594079 Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 27 Jan 2025 10:59:42 +0000 Subject: [PATCH 103/106] Do not proceed to validation if a screen hasn't been selected --- packages/builder/src/stores/builder/screenComponent.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index d8169fdedb..434fa27ae5 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -78,6 +78,11 @@ export const screenComponentErrors = derived( ...reduceBy("_id", $queries.list), } + if (!$selectedScreen) { + // Skip validation if a screen is not selected. + return {} + } + return getInvalidDatasources($selectedScreen, datasources) } ) From 1557b026826911ba2e87ec978698bbb04d91f5db Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 27 Jan 2025 12:17:25 +0100 Subject: [PATCH 104/106] Use proper field --- packages/builder/src/stores/builder/screenComponent.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index aecc27c4e5..1c1070ad17 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -33,7 +33,7 @@ const validationKeyByType: Record = { query: "_id", custom: null, link: "rowId", - field: "label", + field: "value", } export const screenComponentErrors = derived( @@ -73,7 +73,7 @@ export const screenComponentErrors = derived( "rowId", bindings.extractRelationships(componentBindings) ), - ...reduceBy("label", bindings.extractFields(componentBindings)), + ...reduceBy("value", bindings.extractFields(componentBindings)), } const resourceId = componentSettings[validationKey] From f9dadf83a2914da94ea793a9fadf09b056d894e0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 27 Jan 2025 12:16:07 +0100 Subject: [PATCH 105/106] Validate json arrays --- packages/builder/src/stores/builder/screenComponent.ts | 5 +++++ packages/types/src/ui/datasource.ts | 1 + 2 files changed, 6 insertions(+) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 1c1070ad17..4100e1e1fc 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -34,6 +34,7 @@ const validationKeyByType: Record = { custom: null, link: "rowId", field: "value", + jsonarray: "value", } export const screenComponentErrors = derived( @@ -74,6 +75,10 @@ export const screenComponentErrors = derived( bindings.extractRelationships(componentBindings) ), ...reduceBy("value", bindings.extractFields(componentBindings)), + ...reduceBy( + "value", + bindings.extractJSONArrayFields(componentBindings) + ), } const resourceId = componentSettings[validationKey] diff --git a/packages/types/src/ui/datasource.ts b/packages/types/src/ui/datasource.ts index c6b1ed01d1..8cc9b4ff94 100644 --- a/packages/types/src/ui/datasource.ts +++ b/packages/types/src/ui/datasource.ts @@ -6,3 +6,4 @@ export type UIDatasourceType = | "custom" | "link" | "field" + | "jsonarray" From 9c6ce76f682012dda63889c7b1a4a10f5278c390 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 27 Jan 2025 13:45:35 +0100 Subject: [PATCH 106/106] Fix undefined reference --- packages/builder/src/stores/builder/screenComponent.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 3fdaf97660..226d233090 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -56,6 +56,9 @@ export const screenComponentErrors = derived( ["table", "dataSource"] )) { const componentSettings = component[setting.key] + if (!componentSettings) { + continue + } const { label } = componentSettings const type = componentSettings.type as UIDatasourceType