diff --git a/.eslintignore b/.eslintignore index 8d4c64d960..f2c53c2fdc 100644 --- a/.eslintignore +++ b/.eslintignore @@ -6,6 +6,7 @@ packages/server/coverage packages/worker/coverage packages/backend-core/coverage packages/server/client +packages/server/coverage packages/builder/.routify packages/sdk/sdk packages/account-portal/packages/server/build diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index ae86695168..6cea7efeba 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -10,7 +10,7 @@ import { StaticDatabases, DEFAULT_TENANT_ID, } from "../constants" -import { Database, IdentityContext } from "@budibase/types" +import { Database, IdentityContext, Snippet, App } from "@budibase/types" import { ContextMap } from "./types" let TEST_APP_ID: string | null = null @@ -122,10 +122,10 @@ export async function doInAutomationContext(params: { automationId: string task: () => T }): Promise { - const tenantId = getTenantIDFromAppID(params.appId) + await ensureSnippetContext() return newContext( { - tenantId, + tenantId: getTenantIDFromAppID(params.appId), appId: params.appId, automationId: params.automationId, }, @@ -281,6 +281,27 @@ export function doInScimContext(task: any) { return newContext(updates, task) } +export async function ensureSnippetContext() { + const ctx = getCurrentContext() + + // If we've already added snippets to context, continue + if (!ctx || ctx.snippets) { + return + } + + // Otherwise get snippets for this app and update context + let snippets: Snippet[] | undefined + const db = getAppDB() + if (db && !env.isTest()) { + const app = await db.get(DocumentType.APP_METADATA) + snippets = app.snippets + } + + // Always set snippets to a non-null value so that we can tell we've attempted + // to load snippets + ctx.snippets = snippets || [] +} + export function getEnvironmentVariables() { const context = Context.get() if (!context.environmentVariables) { diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index 8ea544a53c..f297d3089f 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -1,4 +1,4 @@ -import { IdentityContext, VM } from "@budibase/types" +import { IdentityContext, Snippet, VM } from "@budibase/types" // keep this out of Budibase types, don't want to expose context info export type ContextMap = { @@ -11,4 +11,5 @@ export type ContextMap = { isMigrating?: boolean vm?: VM cleanup?: (() => void | Promise)[] + snippets?: Snippet[] } diff --git a/packages/bbui/src/Actions/click_outside.js b/packages/bbui/src/Actions/click_outside.js index 62416ae88d..12c4c4d002 100644 --- a/packages/bbui/src/Actions/click_outside.js +++ b/packages/bbui/src/Actions/click_outside.js @@ -33,8 +33,8 @@ const handleClick = event => { } // Ignore clicks for drawers, unless the handler is registered from a drawer - const sourceInDrawer = handler.anchor.closest(".drawer-container") != null - const clickInDrawer = event.target.closest(".drawer-container") != null + const sourceInDrawer = handler.anchor.closest(".drawer-wrapper") != null + const clickInDrawer = event.target.closest(".drawer-wrapper") != null if (clickInDrawer && !sourceInDrawer) { return } diff --git a/packages/bbui/src/Drawer/Drawer.svelte b/packages/bbui/src/Drawer/Drawer.svelte index 757f86b471..89ee92726d 100644 --- a/packages/bbui/src/Drawer/Drawer.svelte +++ b/packages/bbui/src/Drawer/Drawer.svelte @@ -57,8 +57,10 @@
-
-
- {#each editorTabs as tab} - changeMode(tab)} - > - {capitalise(tab)} - - {/each} + {#if showTabBar} +
+
+ {#each editorModeOptions as editorMode} + changeMode(editorMode)} + > + {capitalise(editorMode)} + + {/each} +
+
+ {#each sidePanelOptions as panel} + changeSidePanel(panel)} + > + + + {/each} +
-
- {#each sideTabs as tab} - changeSidePanel(tab)} - > - - - {/each} - {#if drawerContext && get(drawerContext.resizable)} - drawerContext.modal.set(!drawerIsModal)} - > - - - {/if} -
-
+ {/if}
{#if mode === Modes.Text} {#key hbsCompletions} @@ -228,7 +295,8 @@ bind:insertAtPos completions={hbsCompletions} autofocus={autofocusEditor} - placeholder="Add bindings by typing {{ or use the menu on the right" + placeholder={placeholder || + "Add bindings by typing {{ or use the menu on the right"} jsBindingWrapping={false} /> {/key} @@ -242,7 +310,8 @@ bind:getCaretPosition bind:insertAtPos autofocus={autofocusEditor} - placeholder="Add bindings by typing $ or use the menu on the right" + placeholder={placeholder || + "Add bindings by typing $ or use the menu on the right"} jsBindingWrapping /> {/key} @@ -289,6 +358,11 @@ {evaluating} expression={editorValue} /> + {:else if sidePanel === SidePanels.Snippets} + bindingHelpers.onSelectSnippet(snippet)} + {snippets} + /> {/if}
diff --git a/packages/builder/src/components/common/bindings/BindingSidePanel.svelte b/packages/builder/src/components/common/bindings/BindingSidePanel.svelte index d990451005..6ef2d35a6c 100644 --- a/packages/builder/src/components/common/bindings/BindingSidePanel.svelte +++ b/packages/builder/src/components/common/bindings/BindingSidePanel.svelte @@ -12,6 +12,7 @@ export let context = null let search = "" + let searching = false let popover let popoverAnchor let hoverTarget @@ -74,6 +75,13 @@ if (!context || !binding.value || binding.value === "") { return } + + // Roles have always been broken for JS. We need to exclude them from + // showing a popover as it will show "Error while executing JS". + if (binding.category === "Role") { + return + } + stopHidingPopover() popoverAnchor = target hoverTarget = { @@ -112,6 +120,17 @@ hideTimeout = null } } + + const startSearching = async () => { + searching = true + search = "" + } + + const stopSearching = e => { + e.stopPropagation() + searching = false + search = "" + } -
{#if selectedCategory} @@ -158,25 +176,34 @@ {#if !selectedCategory}
- - + +
+ - - { - search = null - }} - class:searching={search} - > - - + {:else} +
Bindings
+ + {/if}
{/if} - {#if !selectedCategory && !search}
    {#each categoryNames as categoryName} @@ -281,18 +308,15 @@ background: var(--background); z-index: 1; } - .header :global(input) { border: none; border-radius: 0; background: none; padding: 0; } - .search-input { - flex: 1; - } - .search-input-icon.searching { - cursor: pointer; + .search-input, + .title { + flex: 1 1 auto; } ul.category-list { diff --git a/packages/builder/src/components/common/bindings/ClientBindingPanel.svelte b/packages/builder/src/components/common/bindings/ClientBindingPanel.svelte index 843dec8c89..cb65d2bbe4 100644 --- a/packages/builder/src/components/common/bindings/ClientBindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/ClientBindingPanel.svelte @@ -1,6 +1,6 @@ + + + + {#if snippet} + {snippet.name} + {:else} +
    + Name + + {#if nameError} + + + + {/if} +
    + {/if} +     + + {#if snippet} + + {/if} + + + + {#key key} + (code = e.detail)} + > +
    + +
    +     + {/key} +     +     + + + + diff --git a/packages/builder/src/components/common/bindings/SnippetSidePanel.svelte b/packages/builder/src/components/common/bindings/SnippetSidePanel.svelte new file mode 100644 index 0000000000..c68699fc0f --- /dev/null +++ b/packages/builder/src/components/common/bindings/SnippetSidePanel.svelte @@ -0,0 +1,278 @@ + + + + +
    + +
    + {#if enableSnippets} + {#if searching} +
    + +
    + + {:else} +
    Snippets
    + + + {/if} + {:else} +
    + Snippets + + Premium + +
    + {/if} +
    +
    + {#if enableSnippets && filteredSnippets?.length} + {#each filteredSnippets as snippet} +
    showSnippet(snippet, e.target)} + on:mouseleave={hidePopover} + on:click={() => addSnippet(snippet)} + > + {snippet.name} + editSnippet(e, snippet)} + /> +
    + {/each} + {:else} +
    + + Snippets let you create reusable JS functions and values that can + all be managed in one place + + {#if enableSnippets} + + {:else} + + {/if} +
    + {/if} +
    +     +
    + + +
    + {#key hoveredSnippet} + + {/key} +
    +     + + + + diff --git a/packages/builder/src/components/common/bindings/utils.js b/packages/builder/src/components/common/bindings/utils.js index a086cd0394..c60374f0f7 100644 --- a/packages/builder/src/components/common/bindings/utils.js +++ b/packages/builder/src/components/common/bindings/utils.js @@ -38,4 +38,11 @@ export class BindingHelpers { this.insertAtPos({ start, end, value: insertVal }) } } + + // Adds a snippet to the expression + onSelectSnippet(snippet) { + const pos = this.getCaretPosition() + const { start, end } = pos + this.insertAtPos({ start, end, value: `snippets.${snippet.name}` }) + } } diff --git a/packages/builder/src/helpers/duplicate.js b/packages/builder/src/helpers/duplicate.js index 1547fcd4d1..c2a924f97b 100644 --- a/packages/builder/src/helpers/duplicate.js +++ b/packages/builder/src/helpers/duplicate.js @@ -48,3 +48,53 @@ export const duplicateName = (name, allNames) => { return `${baseName} ${number}` } + +/** + * More flexible alternative to the above function, which handles getting the + * next sequential name from an array of existing items while accounting for + * any type of prefix, and being able to deeply retrieve that name from the + * existing item array. + * + * Examples with a prefix of "foo": + * [] => "foo" + * ["foo"] => "foo2" + * ["foo", "foo6"] => "foo7" + * + * Examples with a prefix of "foo " (space at the end): + * [] => "foo" + * ["foo"] => "foo 2" + * ["foo", "foo 6"] => "foo 7" + * + * @param items the array of existing items + * @param prefix the string prefix of each name, including any spaces desired + * @param getName optional function to extract the name for an item, if not a + * flat array of strings + */ +export const getSequentialName = (items, prefix, getName = x => x) => { + if (!prefix?.length || !getName) { + return null + } + const trimmedPrefix = prefix.trim() + if (!items?.length) { + return trimmedPrefix + } + let max = 0 + items.forEach(item => { + const name = getName(item) + if (typeof name !== "string" || !name.startsWith(trimmedPrefix)) { + return + } + const split = name.split(trimmedPrefix) + if (split.length !== 2) { + return + } + if (split[1].trim() === "") { + split[1] = "1" + } + const num = parseInt(split[1]) + if (num > max) { + max = num + } + }) + return max === 0 ? trimmedPrefix : `${prefix}${max + 1}` +} diff --git a/packages/builder/src/helpers/tests/duplicate.test.js b/packages/builder/src/helpers/tests/duplicate.test.js index 400abed0aa..7e51c5ff2a 100644 --- a/packages/builder/src/helpers/tests/duplicate.test.js +++ b/packages/builder/src/helpers/tests/duplicate.test.js @@ -1,5 +1,5 @@ import { expect, describe, it } from "vitest" -import { duplicateName } from "../duplicate" +import { duplicateName, getSequentialName } from "../duplicate" describe("duplicate", () => { describe("duplicates a name ", () => { @@ -40,3 +40,64 @@ describe("duplicate", () => { }) }) }) + +describe("getSequentialName", () => { + it("handles nullish items", async () => { + const name = getSequentialName(null, "foo", () => {}) + expect(name).toBe("foo") + }) + + it("handles nullish prefix", async () => { + const name = getSequentialName([], null, () => {}) + expect(name).toBe(null) + }) + + it("handles nullish getName function", async () => { + const name = getSequentialName([], "foo", null) + expect(name).toBe(null) + }) + + it("handles just the prefix", async () => { + const name = getSequentialName(["foo"], "foo", x => x) + expect(name).toBe("foo2") + }) + + it("handles continuous ranges", async () => { + const name = getSequentialName(["foo", "foo2", "foo3"], "foo", x => x) + expect(name).toBe("foo4") + }) + + it("handles discontinuous ranges", async () => { + const name = getSequentialName(["foo", "foo3"], "foo", x => x) + expect(name).toBe("foo4") + }) + + it("handles a space inside the prefix", async () => { + const name = getSequentialName(["foo", "foo 2", "foo 3"], "foo ", x => x) + expect(name).toBe("foo 4") + }) + + it("handles a space inside the prefix with just the prefix", async () => { + const name = getSequentialName(["foo"], "foo ", x => x) + expect(name).toBe("foo 2") + }) + + it("handles no matches", async () => { + const name = getSequentialName(["aaa", "bbb"], "foo", x => x) + expect(name).toBe("foo") + }) + + it("handles similar names", async () => { + const name = getSequentialName( + ["fooo1", "2foo", "a3foo4", "5foo5"], + "foo", + x => x + ) + expect(name).toBe("foo") + }) + + it("handles non-string names", async () => { + const name = getSequentialName([null, 4123, [], {}], "foo", x => x) + expect(name).toBe("foo") + }) +}) diff --git a/packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte index c4ee060149..57180625b1 100644 --- a/packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte @@ -40,7 +40,7 @@
    -
    +
    {#if $automationStore.automations?.length} {:else} diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index a910036a4a..95e7a66be9 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -10,6 +10,7 @@ navigationStore, selectedScreen, hoverStore, + snippets, } from "stores/builder" import ConfirmDialog from "components/common/ConfirmDialog.svelte" import { @@ -68,6 +69,7 @@ hostname: window.location.hostname, port: window.location.port, }, + snippets: $snippets, } // Refresh the preview when required diff --git a/packages/builder/src/stores/builder/index.js b/packages/builder/src/stores/builder/index.js index bac9fb8826..4df4bc3c41 100644 --- a/packages/builder/src/stores/builder/index.js +++ b/packages/builder/src/stores/builder/index.js @@ -18,6 +18,7 @@ import { } from "./automations.js" import { userStore, userSelectedResourceMap, isOnlyUser } from "./users.js" import { deploymentStore } from "./deployments.js" +import { snippets } from "./snippets" // Backend import { tables } from "./tables" @@ -62,6 +63,7 @@ export { queries, flags, hoverStore, + snippets, } export const reset = () => { @@ -101,6 +103,7 @@ export const initialise = async pkg => { builderStore.init(application) navigationStore.syncAppNavigation(application?.navigation) themeStore.syncAppTheme(application) + snippets.syncMetadata(application) screenStore.syncAppScreens(pkg) layoutStore.syncAppLayouts(pkg) resetBuilderHistory() diff --git a/packages/builder/src/stores/builder/snippets.js b/packages/builder/src/stores/builder/snippets.js new file mode 100644 index 0000000000..72ab274730 --- /dev/null +++ b/packages/builder/src/stores/builder/snippets.js @@ -0,0 +1,41 @@ +import { writable, get } from "svelte/store" +import { API } from "api" +import { appStore } from "./app" + +const createsnippets = () => { + const store = writable([]) + + const syncMetadata = metadata => { + store.set(metadata?.snippets || []) + } + + const saveSnippet = async updatedSnippet => { + const snippets = [ + ...get(store).filter(snippet => snippet.name !== updatedSnippet.name), + updatedSnippet, + ] + const app = await API.saveAppMetadata({ + appId: get(appStore).appId, + metadata: { snippets }, + }) + syncMetadata(app) + } + + const deleteSnippet = async snippetName => { + const snippets = get(store).filter(snippet => snippet.name !== snippetName) + const app = await API.saveAppMetadata({ + appId: get(appStore).appId, + metadata: { snippets }, + }) + syncMetadata(app) + } + + return { + ...store, + syncMetadata, + saveSnippet, + deleteSnippet, + } +} + +export const snippets = createsnippets() diff --git a/packages/builder/src/stores/builder/websocket.js b/packages/builder/src/stores/builder/websocket.js index bbdf7b6d63..0704f26439 100644 --- a/packages/builder/src/stores/builder/websocket.js +++ b/packages/builder/src/stores/builder/websocket.js @@ -6,6 +6,7 @@ import { themeStore, navigationStore, deploymentStore, + snippets, datasources, tables, } from "stores/builder" @@ -64,6 +65,7 @@ export const createBuilderWebsocket = appId => { appStore.syncMetadata(metadata) themeStore.syncMetadata(metadata) navigationStore.syncMetadata(metadata) + snippets.syncMetadata(metadata) }) socket.onOther( BuilderSocketEvent.AppPublishChange, diff --git a/packages/client/src/components/ClientApp.svelte b/packages/client/src/components/ClientApp.svelte index 7e30e21ae8..e6cda03270 100644 --- a/packages/client/src/components/ClientApp.svelte +++ b/packages/client/src/components/ClientApp.svelte @@ -39,6 +39,7 @@ import FreeFooter from "components/FreeFooter.svelte" import MaintenanceScreen from "components/MaintenanceScreen.svelte" import licensing from "../licensing" + import SnippetsProvider from "./context/SnippetsProvider.svelte" // Provide contexts setContext("sdk", SDK) @@ -121,114 +122,116 @@ - - - {#key $builderStore.selectedComponentId} - {#if $builderStore.inBuilder} - - {/if} - {/key} - - -
    - -
    - {#if showDevTools} - + + + + {#key $builderStore.selectedComponentId} + {#if $builderStore.inBuilder} + {/if} + {/key} -
    - {#if permissionError} -
    - - - {@html ErrorSVG} - - You don't have permission to use this app - - - Ask your administrator to grant you access - - -
    - {:else if !$screenStore.activeLayout} -
    - - - {@html ErrorSVG} - - Something went wrong rendering your app - - - Get in touch with support if this issue persists - - -
    - {:else if embedNoScreens} -
    - - - {@html ErrorSVG} - - This Budibase app is not publicly accessible - - -
    - {:else} - - {#key $screenStore.activeLayout._id} - - {/key} + +
    + +
    + {#if showDevTools} + + {/if} - + {@html ErrorSVG} + + You don't have permission to use this app + + + Ask your administrator to grant you access + + +
    + {:else if !$screenStore.activeLayout} +
    + + + {@html ErrorSVG} + + Something went wrong rendering your app + + + Get in touch with support if this issue persists + + +
    + {:else if embedNoScreens} +
    + + + {@html ErrorSVG} + + This Budibase app is not publicly accessible + + +
    + {:else} + + {#key $screenStore.activeLayout._id} + + {/key} + + -
    +
    - -
    + +
    - - - - - - {/if} + + + + + + {/if} - {#if showDevTools} - + {#if showDevTools} + + {/if} +
    + + {#if !$builderStore.inBuilder && licensing.logoEnabled()} + {/if}
    - {#if !$builderStore.inBuilder && licensing.logoEnabled()} - + + {#if $appStore.isDevApp} + + {/if} + {#if $builderStore.inBuilder || $devToolsStore.allowSelection} + + {/if} + {#if $builderStore.inBuilder} + + {/if}
    - - - {#if $appStore.isDevApp} - - {/if} - {#if $builderStore.inBuilder || $devToolsStore.allowSelection} - - {/if} - {#if $builderStore.inBuilder} - - - {/if} -
    + diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index 3d267ec623..7dbe0c0e44 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -565,7 +565,8 @@ // If we don't know, check and cache if (used == null) { - used = bindingString.indexOf(`[${key}]`) !== -1 + const searchString = key === "snippets" ? key : `[${key}]` + used = bindingString.indexOf(searchString) !== -1 knownContextKeyMap[key] = used } diff --git a/packages/client/src/components/context/SnippetsProvider.svelte b/packages/client/src/components/context/SnippetsProvider.svelte new file mode 100644 index 0000000000..53fa1e8b7f --- /dev/null +++ b/packages/client/src/components/context/SnippetsProvider.svelte @@ -0,0 +1,8 @@ + + + + + diff --git a/packages/client/src/index.js b/packages/client/src/index.js index 1ce7101466..9c249dd5b3 100644 --- a/packages/client/src/index.js +++ b/packages/client/src/index.js @@ -42,6 +42,7 @@ const loadBudibase = async () => { hiddenComponentIds: window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"], usedPlugins: window["##BUDIBASE_USED_PLUGINS##"], location: window["##BUDIBASE_LOCATION##"], + snippets: window["##BUDIBASE_SNIPPETS##"], }) // Set app ID - this window flag is set by both the preview and the real diff --git a/packages/client/src/stores/builder.js b/packages/client/src/stores/builder.js index 2e745885b5..5440fc3a79 100644 --- a/packages/client/src/stores/builder.js +++ b/packages/client/src/stores/builder.js @@ -18,6 +18,7 @@ const createBuilderStore = () => { usedPlugins: null, eventResolvers: {}, metadata: null, + snippets: null, // Legacy - allow the builder to specify a layout layout: null, diff --git a/packages/client/src/stores/derived/index.js b/packages/client/src/stores/derived/index.js index d9a7743be6..337c73831f 100644 --- a/packages/client/src/stores/derived/index.js +++ b/packages/client/src/stores/derived/index.js @@ -4,3 +4,4 @@ export { currentRole } from "./currentRole.js" export { dndComponentPath } from "./dndComponentPath.js" export { devToolsEnabled } from "./devToolsEnabled.js" +export { snippets } from "./snippets.js" diff --git a/packages/client/src/stores/derived/snippets.js b/packages/client/src/stores/derived/snippets.js new file mode 100644 index 0000000000..74b2797643 --- /dev/null +++ b/packages/client/src/stores/derived/snippets.js @@ -0,0 +1,10 @@ +import { appStore } from "../app.js" +import { builderStore } from "../builder.js" +import { derivedMemo } from "@budibase/frontend-core" + +export const snippets = derivedMemo( + [appStore, builderStore], + ([$appStore, $builderStore]) => { + return $builderStore?.snippets || $appStore?.application?.snippets || [] + } +) diff --git a/packages/client/src/utils/enrichDataBinding.js b/packages/client/src/utils/enrichDataBinding.js index 0f142ab297..3756d8789a 100644 --- a/packages/client/src/utils/enrichDataBinding.js +++ b/packages/client/src/utils/enrichDataBinding.js @@ -1,23 +1,5 @@ import { Helpers } from "@budibase/bbui" -import { processString, processObjectSync } from "@budibase/string-templates" - -// Regex to test inputs with to see if they are likely candidates for template strings -const looksLikeTemplate = /{{.*}}/ - -/** - * Enriches a given input with a row from the database. - */ -export const enrichDataBinding = async (input, context) => { - // Only accept string inputs - if (!input || typeof input !== "string") { - return input - } - // Do a fast regex check if this looks like a template string - if (!looksLikeTemplate.test(input)) { - return input - } - return processString(input, context) -} +import { processObjectSync } from "@budibase/string-templates" /** * Recursively enriches all props in a props object and returns the new props. diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte index 097544439c..657f618759 100644 --- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte @@ -317,7 +317,7 @@ align="right" offset={0} popoverTarget={document.getElementById(`grid-${rand}`)} - customZindex={100} + customZindex={50} > {#if editIsOpen}
    { diff --git a/packages/server/package.json b/packages/server/package.json index 97de17eb58..bac65c768e 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -13,9 +13,10 @@ "build": "node ./scripts/build.js", "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/", "check:types": "tsc -p tsconfig.json --noEmit --paths null", + "build:isolated-vm-lib:snippets": "esbuild --minify --bundle src/jsRunner/bundles/snippets.ts --outfile=src/jsRunner/bundles/snippets.ivm.bundle.js --platform=node --format=iife --global-name=snippets", "build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=iife --external:handlebars --global-name=helpers", "build:isolated-vm-lib:bson": "esbuild --minify --bundle src/jsRunner/bundles/bsonPackage.ts --outfile=src/jsRunner/bundles/bson.ivm.bundle.js --platform=node --format=iife --global-name=bson", - "build:isolated-vm-libs": "yarn build:isolated-vm-lib:string-templates && yarn build:isolated-vm-lib:bson", + "build:isolated-vm-libs": "yarn build:isolated-vm-lib:string-templates && yarn build:isolated-vm-lib:bson && yarn build:isolated-vm-lib:snippets", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js", "jest": "NODE_OPTIONS=\"--no-node-snapshot $NODE_OPTIONS\" jest", diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 685af4e98e..814b57567f 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -437,11 +437,11 @@ export class ExternalRequest { return { row: newRow, manyRelationships } } - processRelationshipFields( + async processRelationshipFields( table: Table, row: Row, relationships: RelationshipsJson[] - ): Row { + ): Promise { for (let relationship of relationships) { const linkedTable = this.tables[relationship.tableName] if (!linkedTable || !row[relationship.column]) { @@ -457,7 +457,7 @@ export class ExternalRequest { } // process additional types relatedRow = processDates(table, relatedRow) - relatedRow = processFormulas(linkedTable, relatedRow) + relatedRow = await processFormulas(linkedTable, relatedRow) row[relationship.column][key] = relatedRow } } @@ -521,7 +521,7 @@ export class ExternalRequest { return rows } - outputProcessing( + async outputProcessing( rows: Row[] = [], table: Table, relationships: RelationshipsJson[] @@ -561,9 +561,12 @@ export class ExternalRequest { } // make sure all related rows are correct - let finalRowArray = Object.values(finalRows).map(row => - this.processRelationshipFields(table, row, relationships) - ) + let finalRowArray = [] + for (let row of Object.values(finalRows)) { + finalRowArray.push( + await this.processRelationshipFields(table, row, relationships) + ) + } // process some additional types finalRowArray = processDates(table, finalRowArray) @@ -934,7 +937,11 @@ export class ExternalRequest { processed.manyRelationships ) } - const output = this.outputProcessing(responseRows, table, relationships) + const output = await this.outputProcessing( + responseRows, + table, + relationships + ) // if reading it'll just be an array of rows, return whole thing if (operation === Operation.READ) { return ( diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index 0ea8b3560e..a75a6cd2cc 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -110,7 +110,7 @@ export async function updateAllFormulasInTable(table: Table) { (enriched: Row) => enriched._id === row._id ) if (enrichedRow) { - const processed = processFormulas(table, cloneDeep(row), { + const processed = await processFormulas(table, cloneDeep(row), { dynamic: false, contextRows: [enrichedRow], }) @@ -143,7 +143,7 @@ export async function finaliseRow( squash: false, })) as Row // use enriched row to generate formulas for saving, specifically only use as context - row = processFormulas(table, row, { + row = await processFormulas(table, row, { dynamic: false, contextRows: [enrichedRow], }) @@ -179,7 +179,7 @@ export async function finaliseRow( const response = await db.put(row) // for response, calculate the formulas for the enriched row enrichedRow._rev = response.rev - enrichedRow = processFormulas(table, enrichedRow, { + enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false, }) // this updates the related formulas in other rows based on the relations to this row diff --git a/packages/server/src/api/controllers/script.ts b/packages/server/src/api/controllers/script.ts index b69fc430a6..4317f5fcde 100644 --- a/packages/server/src/api/controllers/script.ts +++ b/packages/server/src/api/controllers/script.ts @@ -1,6 +1,6 @@ import { Ctx } from "@budibase/types" import { IsolatedVM } from "../../jsRunner/vm" -import { iifeWrapper } from "../../jsRunner/utilities" +import { iifeWrapper } from "@budibase/string-templates" export async function execute(ctx: Ctx) { const { script, context } = ctx.request.body diff --git a/packages/server/src/api/controllers/static/templates/preview.hbs b/packages/server/src/api/controllers/static/templates/preview.hbs index 63c61baa9f..54b5b1a4e4 100644 --- a/packages/server/src/api/controllers/static/templates/preview.hbs +++ b/packages/server/src/api/controllers/static/templates/preview.hbs @@ -72,7 +72,8 @@ navigation, hiddenComponentIds, usedPlugins, - location + location, + snippets } = parsed // Set some flags so the app knows we're in the builder @@ -89,6 +90,7 @@ window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"] = hiddenComponentIds window["##BUDIBASE_USED_PLUGINS##"] = usedPlugins window["##BUDIBASE_LOCATION##"] = location + window["##BUDIBASE_SNIPPETS##"] = snippets // Initialise app try { diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts index 55766cd120..424d0d6c79 100644 --- a/packages/server/src/api/routes/utils/validators.ts +++ b/packages/server/src/api/routes/utils/validators.ts @@ -2,6 +2,7 @@ import { auth, permissions } from "@budibase/backend-core" import { DataSourceOperation } from "../../../constants" import { WebhookActionType } from "@budibase/types" import Joi from "joi" +import { ValidSnippetNameRegex } from "@budibase/shared-core" const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("") const OPTIONAL_NUMBER = Joi.number().optional().allow(null) @@ -226,6 +227,21 @@ export function applicationValidator(opts = { isCreate: true }) { base.name = appNameValidator.optional() } + const snippetValidator = Joi.array() + .optional() + .items( + Joi.object({ + name: Joi.string() + .pattern(new RegExp(ValidSnippetNameRegex)) + .error( + new Error( + "Snippet name cannot include spaces or special characters, and cannot start with a number" + ) + ), + code: OPTIONAL_STRING, + }) + ) + return auth.joiValidator.body( Joi.object({ _id: OPTIONAL_STRING, @@ -235,6 +251,7 @@ export function applicationValidator(opts = { isCreate: true }) { template: Joi.object({ templateString: OPTIONAL_STRING, }).unknown(true), + snippets: snippetValidator, }).unknown(true) ) } diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 7af3f9392f..513e0c0df2 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -202,7 +202,8 @@ export async function attachFullLinkedDocs( table => table._id === linkedTableId ) if (linkedTable) { - row[link.fieldName].push(processFormulas(linkedTable, linkedRow)) + const processed = await processFormulas(linkedTable, linkedRow) + row[link.fieldName].push(processed) } } } diff --git a/packages/server/src/jsRunner/bundles/index.ts b/packages/server/src/jsRunner/bundles/index.ts index 9e2960807a..f7685206a6 100644 --- a/packages/server/src/jsRunner/bundles/index.ts +++ b/packages/server/src/jsRunner/bundles/index.ts @@ -5,11 +5,13 @@ import fs from "fs" export const enum BundleType { HELPERS = "helpers", BSON = "bson", + SNIPPETS = "snippets", } const bundleSourceFile: Record = { [BundleType.HELPERS]: "./index-helpers.ivm.bundle.js", [BundleType.BSON]: "./bson.ivm.bundle.js", + [BundleType.SNIPPETS]: "./snippets.ivm.bundle.js", } const bundleSourceCode: Partial> = {} diff --git a/packages/server/src/jsRunner/bundles/snippets.ivm.bundle.js b/packages/server/src/jsRunner/bundles/snippets.ivm.bundle.js new file mode 100644 index 0000000000..bad9049af0 --- /dev/null +++ b/packages/server/src/jsRunner/bundles/snippets.ivm.bundle.js @@ -0,0 +1,3 @@ +"use strict";var snippets=(()=>{var u=Object.create;var n=Object.defineProperty;var a=Object.getOwnPropertyDescriptor;var h=Object.getOwnPropertyNames;var x=Object.getPrototypeOf,C=Object.prototype.hasOwnProperty;var l=(i,e)=>()=>(e||i((e={exports:{}}).exports,e),e.exports),W=(i,e)=>{for(var p in e)n(i,p,{get:e[p],enumerable:!0})},f=(i,e,p,r)=>{if(e&&typeof e=="object"||typeof e=="function")for(let t of h(e))!C.call(i,t)&&t!==p&&n(i,t,{get:()=>e[t],enumerable:!(r=a(e,t))||r.enumerable});return i};var d=(i,e,p)=>(p=i!=null?u(x(i)):{},f(e||!i||!i.__esModule?n(p,"default",{value:i,enumerable:!0}):p,i)),g=i=>f(n({},"__esModule",{value:!0}),i);var s=l((D,o)=>{o.exports.iifeWrapper=i=>`(function(){ +${i} +})();`});var w={};W(w,{default:()=>v});var c=d(s()),v=new Proxy({},{get:function(i,e){return e in snippetCache||(snippetCache[e]=[eval][0]((0,c.iifeWrapper)(snippetDefinitions[e]))),snippetCache[e]}});return g(w);})(); diff --git a/packages/server/src/jsRunner/bundles/snippets.ts b/packages/server/src/jsRunner/bundles/snippets.ts new file mode 100644 index 0000000000..258d501a27 --- /dev/null +++ b/packages/server/src/jsRunner/bundles/snippets.ts @@ -0,0 +1,24 @@ +// @ts-ignore +// eslint-disable-next-line local-rules/no-budibase-imports +import { iifeWrapper } from "@budibase/string-templates/iife" + +export default new Proxy( + {}, + { + get: function (_, name) { + // Both snippetDefinitions and snippetCache are injected to the isolate + // global scope before this bundle is loaded, so we can access it from + // there. + // See https://esbuild.github.io/content-types/#direct-eval for info on + // why eval is being called this way. + // Snippets are cached and reused once they have been evaluated. + // @ts-ignore + if (!(name in snippetCache)) { + // @ts-ignore + snippetCache[name] = [eval][0](iifeWrapper(snippetDefinitions[name])) + } + // @ts-ignore + return snippetCache[name] + }, + } +) diff --git a/packages/server/src/jsRunner/index.ts b/packages/server/src/jsRunner/index.ts index 19bf0fa6b5..b5c8e036f6 100644 --- a/packages/server/src/jsRunner/index.ts +++ b/packages/server/src/jsRunner/index.ts @@ -14,16 +14,19 @@ export function init() { setJSRunner((js: string, ctx: Record) => { return tracer.trace("runJS", {}, span => { try { + // Reuse an existing isolate from context, or make a new one const bbCtx = context.getCurrentContext() - const vm = bbCtx?.vm || new IsolatedVM({ memoryLimit: env.JS_RUNNER_MEMORY_LIMIT, invocationTimeout: env.JS_PER_INVOCATION_TIMEOUT_MS, isolateAccumulatedTimeout: env.JS_PER_REQUEST_TIMEOUT_MS, - }).withHelpers() + }) + .withHelpers() + .withSnippets(bbCtx?.snippets) + // Persist isolate in context so we can reuse it if (bbCtx && !bbCtx.vm) { bbCtx.vm = vm bbCtx.cleanup = bbCtx.cleanup || [] @@ -33,7 +36,7 @@ export function init() { // Because we can't pass functions into an Isolate, we remove them from // the passed context and rely on the withHelpers() method to add them // back in. - const { helpers, ...rest } = ctx + const { helpers, snippets, ...rest } = ctx return vm.withContext(rest, () => vm.execute(js)) } catch (error: any) { if (error.message === "Script execution timed out.") { diff --git a/packages/server/src/jsRunner/tests/isolatedVM.spec.ts b/packages/server/src/jsRunner/tests/isolatedVM.spec.ts index 5296598ef1..5a9bc05d76 100644 --- a/packages/server/src/jsRunner/tests/isolatedVM.spec.ts +++ b/packages/server/src/jsRunner/tests/isolatedVM.spec.ts @@ -1,7 +1,7 @@ import fs from "fs" import path from "path" import { IsolatedVM } from "../vm" -import { iifeWrapper } from "../utilities" +import { iifeWrapper } from "@budibase/string-templates" function runJSWithIsolatedVM(script: string, context: Record) { const runner = new IsolatedVM() diff --git a/packages/server/src/jsRunner/utilities.ts b/packages/server/src/jsRunner/utilities.ts deleted file mode 100644 index fa398ec239..0000000000 --- a/packages/server/src/jsRunner/utilities.ts +++ /dev/null @@ -1,3 +0,0 @@ -export function iifeWrapper(script: string) { - return `(function(){\n${script}\n})();` -} diff --git a/packages/server/src/jsRunner/vm/isolated-vm.ts b/packages/server/src/jsRunner/vm/isolated-vm.ts index 53858bd6ff..e89d420ec5 100644 --- a/packages/server/src/jsRunner/vm/isolated-vm.ts +++ b/packages/server/src/jsRunner/vm/isolated-vm.ts @@ -6,8 +6,8 @@ import crypto from "crypto" import querystring from "querystring" import { BundleType, loadBundle } from "../bundles" -import { VM } from "@budibase/types" -import { iifeWrapper } from "../utilities" +import { Snippet, VM } from "@budibase/types" +import { iifeWrapper } from "@budibase/string-templates" import environment from "../../environment" class ExecutionTimeoutError extends Error { @@ -98,6 +98,26 @@ export class IsolatedVM implements VM { return this } + withSnippets(snippets?: Snippet[]) { + // Transform snippets into a map for faster access + let snippetMap: Record = {} + for (let snippet of snippets || []) { + snippetMap[snippet.name] = snippet.code + } + const snippetsSource = loadBundle(BundleType.SNIPPETS) + const script = this.isolate.compileScriptSync(` + const snippetDefinitions = ${JSON.stringify(snippetMap)}; + const snippetCache = {}; + ${snippetsSource}; + snippets = snippets.default; + `) + script.runSync(this.vm, { timeout: this.invocationTimeout, release: false }) + new Promise(() => { + script.release() + }) + return this + } + withContext(context: Record, executeWithContext: () => T) { this.addToContext(context) diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index a4938bb138..4e33fadce6 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -625,6 +625,7 @@ export async function executeInThread(job: Job) { }) return await context.doInAppContext(appId, async () => { + await context.ensureSnippetContext() const envVars = await sdkUtils.getEnvironmentVariables() // put into automation thread for whole context return await context.doInEnvironmentContext(envVars, async () => { diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index baee2d5c05..9f5e02bf69 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -8,9 +8,8 @@ import { QueryResponse, } from "./definitions" import { IsolatedVM } from "../jsRunner/vm" -import { iifeWrapper } from "../jsRunner/utilities" +import { iifeWrapper, processStringSync } from "@budibase/string-templates" import { getIntegration } from "../integrations" -import { processStringSync } from "@budibase/string-templates" import { context, cache, auth } from "@budibase/backend-core" import { getGlobalIDFromUserMetadataID } from "../db/utils" import sdk from "../sdk" diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index e3ca576fb7..d956a94d0b 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -245,7 +245,7 @@ export async function outputProcessing( } // process formulas after the complex types had been processed - enriched = processFormulas(table, enriched, { dynamic: true }) + enriched = await processFormulas(table, enriched, { dynamic: true }) if (opts.squash) { enriched = (await linkRows.squashLinksToPrimaryDisplay( diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index d0fc82b3ee..8201680f13 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -10,6 +10,7 @@ import { FieldType, } from "@budibase/types" import tracer from "dd-trace" +import { context } from "@budibase/backend-core" interface FormulaOpts { dynamic?: boolean @@ -44,16 +45,19 @@ export function fixAutoColumnSubType( /** * Looks through the rows provided and finds formulas - which it then processes. */ -export function processFormulas( +export async function processFormulas( table: Table, inputRows: T, { dynamic, contextRows }: FormulaOpts = { dynamic: true } -): T { - return tracer.trace("processFormulas", {}, span => { +): Promise { + return tracer.trace("processFormulas", {}, async span => { const numRows = Array.isArray(inputRows) ? inputRows.length : 1 span?.addTags({ table_id: table._id, dynamic, numRows }) const rows = Array.isArray(inputRows) ? inputRows : [inputRows] if (rows) { + // Ensure we have snippet context + await context.ensureSnippetContext() + for (let [column, schema] of Object.entries(table.schema)) { if (schema.type !== FieldType.FORMULA) { continue diff --git a/packages/shared-core/src/constants/index.ts b/packages/shared-core/src/constants/index.ts index 99fb5c2a73..633fd36e45 100644 --- a/packages/shared-core/src/constants/index.ts +++ b/packages/shared-core/src/constants/index.ts @@ -98,6 +98,7 @@ export enum BuilderSocketEvent { export const SocketSessionTTL = 60 export const ValidQueryNameRegex = /^[^()]*$/ export const ValidColumnNameRegex = /^[_a-zA-Z0-9\s]*$/g +export const ValidSnippetNameRegex = /^[a-z_][a-z0-9_]*$/i export const REBOOT_CRON = "@reboot" diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index ceafd5364f..340d74ef8a 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -12,7 +12,8 @@ "import": "./dist/bundle.mjs" }, "./package.json": "./package.json", - "./test/utils": "./test/utils.js" + "./test/utils": "./test/utils.js", + "./iife": "./src/iife.js" }, "files": [ "dist", diff --git a/packages/string-templates/src/helpers/javascript.js b/packages/string-templates/src/helpers/javascript.js index 7827736812..5be2619463 100644 --- a/packages/string-templates/src/helpers/javascript.js +++ b/packages/string-templates/src/helpers/javascript.js @@ -2,6 +2,7 @@ const { atob, isBackendService, isJSAllowed } = require("../utilities") const cloneDeep = require("lodash.clonedeep") const { LITERAL_MARKER } = require("../helpers/constants") const { getJsHelperList } = require("./list") +const { iifeWrapper } = require("../iife") // The method of executing JS scripts depends on the bundle being built. // This setter is used in the entrypoint (either index.js or index.mjs). @@ -48,14 +49,36 @@ module.exports.processJS = (handlebars, context) => { try { // Wrap JS in a function and immediately invoke it. // This is required to allow the final `return` statement to be valid. - const js = `(function(){${atob(handlebars)}})();` + const js = iifeWrapper(atob(handlebars)) + + // Transform snippets into an object for faster access, and cache previously + // evaluated snippets + let snippetMap = {} + let snippetCache = {} + for (let snippet of context.snippets || []) { + snippetMap[snippet.name] = snippet.code + } // Our $ context function gets a value from context. // We clone the context to avoid mutation in the binding affecting real // app context. + const clonedContext = cloneDeep({ ...context, snippets: null }) const sandboxContext = { - $: path => getContextValue(path, cloneDeep(context)), + $: path => getContextValue(path, clonedContext), helpers: getJsHelperList(), + + // Proxy to evaluate snippets when running in the browser + snippets: new Proxy( + {}, + { + get: function (_, name) { + if (!(name in snippetCache)) { + snippetCache[name] = eval(iifeWrapper(snippetMap[name])) + } + return snippetCache[name] + }, + } + ), } // Create a sandbox with our context and run the JS diff --git a/packages/string-templates/src/iife.js b/packages/string-templates/src/iife.js new file mode 100644 index 0000000000..d043c14565 --- /dev/null +++ b/packages/string-templates/src/iife.js @@ -0,0 +1,3 @@ +module.exports.iifeWrapper = script => { + return `(function(){\n${script}\n})();` +} diff --git a/packages/string-templates/src/index.js b/packages/string-templates/src/index.js index 0125b9e0ab..5ae773516f 100644 --- a/packages/string-templates/src/index.js +++ b/packages/string-templates/src/index.js @@ -3,6 +3,7 @@ const handlebars = require("handlebars") const { registerAll, registerMinimum } = require("./helpers/index") const processors = require("./processors") const { atob, btoa, isBackendService } = require("./utilities") +const { iifeWrapper } = require("./iife") const manifest = require("../manifest.json") const { FIND_HBS_REGEX, @@ -426,3 +427,4 @@ function defaultJSSetup() { defaultJSSetup() module.exports.defaultJSSetup = defaultJSSetup +module.exports.iifeWrapper = iifeWrapper diff --git a/packages/types/src/documents/app/app.ts b/packages/types/src/documents/app/app.ts index ae4f3fa6da..3b7f481253 100644 --- a/packages/types/src/documents/app/app.ts +++ b/packages/types/src/documents/app/app.ts @@ -1,4 +1,4 @@ -import { User, Document, Plugin } from "../" +import { User, Document, Plugin, Snippet } from "../" import { SocketSession } from "../../sdk" export type AppMetadataErrors = { [key: string]: string[] } @@ -26,6 +26,7 @@ export interface App extends Document { automations?: AutomationSettings usedPlugins?: Plugin[] upgradableVersion?: string + snippets?: Snippet[] } export interface AppInstance { diff --git a/packages/types/src/documents/app/index.ts b/packages/types/src/documents/app/index.ts index b81c9e36ac..a58b708de3 100644 --- a/packages/types/src/documents/app/index.ts +++ b/packages/types/src/documents/app/index.ts @@ -14,3 +14,4 @@ export * from "./backup" export * from "./webhook" export * from "./links" export * from "./component" +export * from "./snippet" diff --git a/packages/types/src/documents/app/snippet.ts b/packages/types/src/documents/app/snippet.ts new file mode 100644 index 0000000000..1b8433b32e --- /dev/null +++ b/packages/types/src/documents/app/snippet.ts @@ -0,0 +1,4 @@ +export interface Snippet { + name: string + code: string +}