From b186f6a856fcdd9f928fd3e406f08d62a303d84f Mon Sep 17 00:00:00 2001 From: Christos Alexiou Date: Thu, 8 Aug 2024 19:16:48 +0300 Subject: [PATCH 01/59] fix: drop temp variable --- hosting/single/runner.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index b89bf8ffd4..3ec31993dc 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -45,8 +45,7 @@ fi # randomise any unset environment variables for ENV_VAR in "${ENV_VARS[@]}" do - temp=$(eval "echo \$$ENV_VAR") - if [[ -z "${temp}" ]]; then + if [[ -z "${!ENV_VAR}" ]]; then eval "export $ENV_VAR=$(uuidgen | sed -e 's/-//g')" fi done From 165eff2e5afb52708247667d9c4dd6ccec2e550d Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Wed, 14 Aug 2024 17:21:40 +0100 Subject: [PATCH 02/59] First pass - no tests yet, had to make some changes to how pre-processing works, as well as updating the string based on context, if there is any overlap between the helpers and context it will prefix the overlap with ./ - this means to look in context. --- packages/string-templates/src/index.ts | 45 ++++++++++++++++--- .../string-templates/src/processors/index.ts | 4 +- .../src/processors/preprocessor.ts | 10 ++++- packages/string-templates/src/types.ts | 1 + packages/string-templates/src/utilities.ts | 13 ++++++ 5 files changed, 62 insertions(+), 11 deletions(-) diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index 0992813e9d..291807d5f0 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -1,17 +1,18 @@ import { Context, createContext, runInNewContext } from "vm" import { create, TemplateDelegate } from "handlebars" import { registerAll, registerMinimum } from "./helpers/index" -import { preprocess, postprocess } from "./processors" +import { postprocess, preprocess } from "./processors" import { atob, btoa, - isBackendService, - FIND_HBS_REGEX, FIND_ANY_HBS_REGEX, + FIND_HBS_REGEX, findDoubleHbsInstances, + isBackendService, + prefixStrings, } from "./utilities" import { convertHBSBlock } from "./conversion" -import { setJSRunner, removeJSRunner } from "./helpers/javascript" +import { removeJSRunner, setJSRunner } from "./helpers/javascript" import manifest from "./manifest.json" import { ProcessOptions } from "./types" @@ -23,6 +24,7 @@ export { iifeWrapper } from "./iife" const hbsInstance = create() registerAll(hbsInstance) +const helperNames = Object.keys(hbsInstance.helpers) const hbsInstanceNoHelpers = create() registerMinimum(hbsInstanceNoHelpers) const defaultOpts: ProcessOptions = { @@ -45,11 +47,20 @@ function testObject(object: any) { } } +function findOverlappingHelpers(context: object) { + const contextKeys = Object.keys(context) + return contextKeys.filter(key => helperNames.includes(key)) +} + /** * Creates a HBS template function for a given string, and optionally caches it. */ const templateCache: Record> = {} -function createTemplate(string: string, opts?: ProcessOptions) { +function createTemplate( + string: string, + opts?: ProcessOptions, + context?: object +) { opts = { ...defaultOpts, ...opts } // Finalising adds a helper, can't do this with no helpers @@ -60,7 +71,25 @@ function createTemplate(string: string, opts?: ProcessOptions) { return templateCache[key] } - string = preprocess(string, opts) + const overlappingHelpers = !opts?.noHelpers + ? findOverlappingHelpers(context) + : [] + + string = preprocess(string, { + ...opts, + disabledHelpers: overlappingHelpers, + }) + + if (context && !opts?.noHelpers) { + if (overlappingHelpers.length > 0) { + for (let block of findHBSBlocks(string)) { + string = string.replace( + block, + prefixStrings(block, overlappingHelpers, "./") + ) + } + } + } // Optionally disable built in HBS escaping if (opts.noEscaping) { @@ -70,6 +99,7 @@ function createTemplate(string: string, opts?: ProcessOptions) { // This does not throw an error when template can't be fulfilled, // have to try correct beforehand const instance = opts.noHelpers ? hbsInstanceNoHelpers : hbsInstance + const template = instance.compile(string, { strict: false, }) @@ -171,7 +201,8 @@ export function processStringSync( throw "Cannot process non-string types." } function process(stringPart: string) { - const template = createTemplate(stringPart, opts) + // context is needed to check for overlap between helpers and context + const template = createTemplate(stringPart, opts, context) const now = Math.floor(Date.now() / 1000) * 1000 const processedString = template({ now: new Date(now).toISOString(), diff --git a/packages/string-templates/src/processors/index.ts b/packages/string-templates/src/processors/index.ts index 308ac5adf4..79085b0dfe 100644 --- a/packages/string-templates/src/processors/index.ts +++ b/packages/string-templates/src/processors/index.ts @@ -29,9 +29,9 @@ export function preprocess(string: string, opts: ProcessOptions) { processor => processor.name !== preprocessor.PreprocessorNames.FINALISE ) } + return process(string, processors, opts) } export function postprocess(string: string) { - let processors = postprocessor.processors - return process(string, processors) + return process(string, postprocessor.processors) } diff --git a/packages/string-templates/src/processors/preprocessor.ts b/packages/string-templates/src/processors/preprocessor.ts index 5e96336e32..ee76b5dbcf 100644 --- a/packages/string-templates/src/processors/preprocessor.ts +++ b/packages/string-templates/src/processors/preprocessor.ts @@ -13,10 +13,12 @@ export const PreprocessorNames = { class Preprocessor { name: string private fn: any + private helperNames: string[] constructor(name: string, fn: any) { this.name = name this.fn = fn + this.helperNames = HelperNames() } process(fullString: string, statement: string, opts: Object) { @@ -56,7 +58,10 @@ export const processors = [ }), new Preprocessor( PreprocessorNames.FINALISE, - (statement: string, opts: { noHelpers: any }) => { + ( + statement: string, + opts: { noHelpers: any; disabledHelpers?: string[] } + ) => { const noHelpers = opts && opts.noHelpers let insideStatement = statement.slice(2, statement.length - 2) if (insideStatement.charAt(0) === " ") { @@ -75,7 +80,8 @@ export const processors = [ const testHelper = possibleHelper.trim().toLowerCase() if ( !noHelpers && - HelperNames().some(option => testHelper === option.toLowerCase()) + !opts.disabledHelpers?.includes(testHelper) && + this.helperNames.some(option => testHelper === option.toLowerCase()) ) { insideStatement = `(${insideStatement})` } diff --git a/packages/string-templates/src/types.ts b/packages/string-templates/src/types.ts index 1e1ef048a4..19785b9273 100644 --- a/packages/string-templates/src/types.ts +++ b/packages/string-templates/src/types.ts @@ -5,4 +5,5 @@ export interface ProcessOptions { noFinalise?: boolean escapeNewlines?: boolean onlyFound?: boolean + disabledHelpers?: string[] } diff --git a/packages/string-templates/src/utilities.ts b/packages/string-templates/src/utilities.ts index bcb9987c89..18e0926b96 100644 --- a/packages/string-templates/src/utilities.ts +++ b/packages/string-templates/src/utilities.ts @@ -66,3 +66,16 @@ export const btoa = (plainText: string) => { export const atob = (base64: string) => { return Buffer.from(base64, "base64").toString("utf-8") } + +export const prefixStrings = ( + baseString: string, + strings: string[], + prefix: string +) => { + // Escape any special characters in the strings to avoid regex errors + const escapedStrings = strings.map(str => + str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + ) + const regexPattern = new RegExp(`\\b(${escapedStrings.join("|")})\\b`, "g") + return baseString.replace(regexPattern, `${prefix}$1`) +} From 87999db65928ab8cc3586c2b619cea20be87ef7a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 14 Aug 2024 18:12:33 +0100 Subject: [PATCH 03/59] Adding test cases. --- packages/string-templates/src/index.ts | 5 +++- .../src/processors/preprocessor.ts | 4 +--- .../string-templates/test/helpers.spec.ts | 23 +++++++++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index 291807d5f0..4a24328265 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -47,7 +47,10 @@ function testObject(object: any) { } } -function findOverlappingHelpers(context: object) { +function findOverlappingHelpers(context?: object) { + if (!context) { + return [] + } const contextKeys = Object.keys(context) return contextKeys.filter(key => helperNames.includes(key)) } diff --git a/packages/string-templates/src/processors/preprocessor.ts b/packages/string-templates/src/processors/preprocessor.ts index ee76b5dbcf..95a4d5500d 100644 --- a/packages/string-templates/src/processors/preprocessor.ts +++ b/packages/string-templates/src/processors/preprocessor.ts @@ -13,12 +13,10 @@ export const PreprocessorNames = { class Preprocessor { name: string private fn: any - private helperNames: string[] constructor(name: string, fn: any) { this.name = name this.fn = fn - this.helperNames = HelperNames() } process(fullString: string, statement: string, opts: Object) { @@ -81,7 +79,7 @@ export const processors = [ if ( !noHelpers && !opts.disabledHelpers?.includes(testHelper) && - this.helperNames.some(option => testHelper === option.toLowerCase()) + HelperNames().some(option => testHelper === option.toLowerCase()) ) { insideStatement = `(${insideStatement})` } diff --git a/packages/string-templates/test/helpers.spec.ts b/packages/string-templates/test/helpers.spec.ts index 5f1855535d..566be2b02b 100644 --- a/packages/string-templates/test/helpers.spec.ts +++ b/packages/string-templates/test/helpers.spec.ts @@ -483,3 +483,26 @@ describe("uuid", () => { expect(output).toMatch(UUID_REGEX) }) }) + +describe("helper overlap", () => { + it("should use context over helpers (regex test helper)", async () => { + const output = await processString("{{ test }}", { test: "a" }) + expect(output).toEqual("a") + }) + + it("should use helper if no sum in context, return the context value otherwise", async () => { + const hbs = "{{ sum 1 2 }}" + const output = await processString(hbs, {}) + expect(output).toEqual("3") + const secondaryOutput = await processString(hbs, { sum: "a" }) + expect(secondaryOutput).toEqual("a") + }) + + it("should handle multiple cases", async () => { + const output = await processString("{{ literal (split test sum) }}", { + test: "a-b", + sum: "-", + }) + expect(output).toEqual(["a", "b"]) + }) +}) From 4a1fd6481fe38ed71ed470a19fca7dcb112f8a65 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 Aug 2024 10:53:43 +0100 Subject: [PATCH 04/59] Remove old, unused feature flags. --- charts/budibase/README.md | 2 +- charts/budibase/values.yaml | 2 +- hosting/single/runner.sh | 1 - packages/backend-core/src/features/index.ts | 4 - .../DataTable/modals/CreateEditColumn.svelte | 5 +- .../src/components/common/HelpMenu.svelte | 42 +- .../portal/licensing/LicensingOverlays.svelte | 4 +- packages/builder/src/helpers/featureFlags.js | 8 - .../builder/app/[application]/_layout.svelte | 19 +- .../portal/_components/UpgradeButton.svelte | 3 +- .../builder/src/stores/portal/licensing.js | 167 +- packages/builder/src/stores/portal/menu.js | 85 +- packages/server/Dockerfile | 1 - packages/server/scripts/dev/manage.js | 1 - packages/types/src/sdk/featureFlag.ts | 1 - packages/worker/Dockerfile | 1 - packages/worker/scripts/dev/manage.js | 1 - yarn.lock | 1372 +---------------- 18 files changed, 211 insertions(+), 1508 deletions(-) diff --git a/charts/budibase/README.md b/charts/budibase/README.md index c944ed1397..c10b42d75b 100644 --- a/charts/budibase/README.md +++ b/charts/budibase/README.md @@ -139,7 +139,7 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | globals.smtp.user | string | `""` | The username to use when authenticating with your SMTP server. | | globals.sqs.enabled | bool | `false` | Whether to use the CouchDB "structured query service" or not. This is disabled by default for now, but will become the default in a future release. | | globals.tempBucketName | string | `""` | | -| globals.tenantFeatureFlags | string | `"*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"` | Sets what feature flags are enabled and for which tenants. Should not ordinarily need to be changed. | +| globals.tenantFeatureFlags | string | `` | Sets what feature flags are enabled and for which tenants. Should not ordinarily need to be changed. | | imagePullSecrets | list | `[]` | Passed to all pods created by this chart. Should not ordinarily need to be changed. | | ingress.className | string | `""` | What ingress class to use. | | ingress.enabled | bool | `true` | Whether to create an Ingress resource pointing to the Budibase proxy. | diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 4e80be7322..dde912410c 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -62,7 +62,7 @@ globals: budibaseEnv: PRODUCTION # -- Sets what feature flags are enabled and for which tenants. Should not ordinarily need to be # changed. - tenantFeatureFlags: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR" + tenantFeatureFlags: "" # -- Whether to enable analytics or not. You can read more about our analytics here: # . enableAnalytics: "1" diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index b89bf8ffd4..89c5540370 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -10,7 +10,6 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME [[ -z "${MINIO_URL}" ]] && [[ -z "${USE_S3}" ]] && export MINIO_URL=http://127.0.0.1:9000 [[ -z "${NODE_ENV}" ]] && export NODE_ENV=production [[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU -[[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR" [[ -z "${ACCOUNT_PORTAL_URL}" ]] && export ACCOUNT_PORTAL_URL=https://account.budibase.app [[ -z "${REDIS_URL}" ]] && export REDIS_URL=127.0.0.1:6379 [[ -z "${SELF_HOSTED}" ]] && export SELF_HOSTED=1 diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts index 67084c8486..0063830417 100644 --- a/packages/backend-core/src/features/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -265,9 +265,5 @@ export class FlagSet, T extends { [key: string]: V }> { // All of the machinery in this file is to make sure that flags have their // default values set correctly and their types flow through the system. export const flags = new FlagSet({ - LICENSING: Flag.boolean(false), - GOOGLE_SHEETS: Flag.boolean(false), - USER_GROUPS: Flag.boolean(false), - ONBOARDING_TOUR: Flag.boolean(false), DEFAULT_VALUES: Flag.boolean(false), }) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 98243c6e2f..1d66a0ba00 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -46,7 +46,7 @@ import { RowUtils } from "@budibase/frontend-core" import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte" import OptionsEditor from "./OptionsEditor.svelte" - import { isEnabled, TENANT_FEATURE_FLAGS } from "helpers/featureFlags" + import { isEnabled } from "helpers/featureFlags" const AUTO_TYPE = FieldType.AUTO const FORMULA_TYPE = FieldType.FORMULA @@ -168,8 +168,7 @@ $: canBeDisplay = canBeDisplayColumn(editableColumn.type) && !editableColumn.autocolumn $: canHaveDefault = - isEnabled(TENANT_FEATURE_FLAGS.DEFAULT_VALUES) && - canHaveDefaultColumn(editableColumn.type) + isEnabled("DEFAULT_VALUES") && canHaveDefaultColumn(editableColumn.type) $: canBeRequired = editableColumn?.type !== LINK_TYPE && !uneditable && diff --git a/packages/builder/src/components/common/HelpMenu.svelte b/packages/builder/src/components/common/HelpMenu.svelte index 0ce5ab9c8e..bba55e88dc 100644 --- a/packages/builder/src/components/common/HelpMenu.svelte +++ b/packages/builder/src/components/common/HelpMenu.svelte @@ -1,7 +1,6 @@ -{#if isEnabled(TENANT_FEATURE_FLAGS.LICENSING) && !$licensing.isEnterprisePlan && !$licensing.isEnterpriseTrial} +{#if !$licensing.isEnterprisePlan && !$licensing.isEnterpriseTrial} {#if $admin.cloud && $auth?.user?.accountPortalAccess}