Merge branch 'master' into builder-store-conversions-pc

packages/backend-core/src/context/mainContext.ts

@@ -385,17 +385,17 @@ export function getCurrentContext(): ContextMap | undefined {
   }
 }
 
-export function getFeatureFlags<T extends Record<string, any>>(
+export function getFeatureFlags(
   key: string
-): T | undefined {
+): Record<string, boolean> | undefined {
   const context = getCurrentContext()
   if (!context) {
     return undefined
   }
-  return context.featureFlagCache?.[key] as T
+  return context.featureFlagCache?.[key]
 }
 
-export function setFeatureFlags(key: string, value: Record<string, any>) {
+export function setFeatureFlags(key: string, value: Record<string, boolean>) {
   const context = getCurrentContext()
   if (!context) {
     return

packages/backend-core/src/context/types.ts

@@ -20,7 +20,7 @@ export type ContextMap = {
     clients: Record<string, GoogleSpreadsheet>
   }
   featureFlagCache?: {
-    [key: string]: Record<string, any>
+    [key: string]: Record<string, boolean>
   }
   viewToTableCache?: Record<string, Table>
 }

packages/backend-core/src/features/features.ts

@@ -2,9 +2,10 @@ import env from "../environment"
 import * as crypto from "crypto"
 import * as context from "../context"
 import { PostHog, PostHogOptions } from "posthog-node"
-import { FeatureFlag } from "@budibase/types"
 import tracer from "dd-trace"
 import { Duration } from "../utils"
+import { cloneDeep } from "lodash"
+import { FeatureFlagDefaults } from "@budibase/types"
 
 let posthog: PostHog | undefined
 export function init(opts?: PostHogOptions) {
@@ -30,74 +31,6 @@ export function shutdown() {
   posthog?.shutdown()
 }
 
-export abstract class Flag<T> {
-  static boolean(defaultValue: boolean): Flag<boolean> {
-    return new BooleanFlag(defaultValue)
-  }
-
-  static string(defaultValue: string): Flag<string> {
-    return new StringFlag(defaultValue)
-  }
-
-  static number(defaultValue: number): Flag<number> {
-    return new NumberFlag(defaultValue)
-  }
-
-  protected constructor(public defaultValue: T) {}
-
-  abstract parse(value: any): T
-}
-
-type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
-
-export type FlagValues<T> = {
-  [K in keyof T]: UnwrapFlag<T[K]>
-}
-
-type KeysOfType<T, U> = {
-  [K in keyof T]: T[K] extends Flag<U> ? K : never
-}[keyof T]
-
-class BooleanFlag extends Flag<boolean> {
-  parse(value: any) {
-    if (typeof value === "string") {
-      return ["true", "t", "1"].includes(value.toLowerCase())
-    }
-
-    if (typeof value === "boolean") {
-      return value
-    }
-
-    throw new Error(`could not parse value "${value}" as boolean`)
-  }
-}
-
-class StringFlag extends Flag<string> {
-  parse(value: any) {
-    if (typeof value === "string") {
-      return value
-    }
-    throw new Error(`could not parse value "${value}" as string`)
-  }
-}
-
-class NumberFlag extends Flag<number> {
-  parse(value: any) {
-    if (typeof value === "number") {
-      return value
-    }
-
-    if (typeof value === "string") {
-      const parsed = parseFloat(value)
-      if (!isNaN(parsed)) {
-        return parsed
-      }
-    }
-
-    throw new Error(`could not parse value "${value}" as number`)
-  }
-}
-
 export interface EnvFlagEntry {
   tenantId: string
   key: string
@@ -120,7 +53,7 @@ export function parseEnvFlags(flags: string): EnvFlagEntry[] {
   return result
 }
 
-export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
+export class FlagSet<T extends { [name: string]: boolean }> {
   // This is used to safely cache flags sets in the current request context.
   // Because multiple sets could theoretically exist, we don't want the cache of
   // one to leak into another.
@@ -130,34 +63,25 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
     this.setId = crypto.randomUUID()
   }
 
-  defaults(): FlagValues<T> {
-    return Object.keys(this.flagSchema).reduce((acc, key) => {
-      const typedKey = key as keyof T
-      acc[typedKey] = this.flagSchema[key].defaultValue
-      return acc
-    }, {} as FlagValues<T>)
+  defaults(): T {
+    return cloneDeep(this.flagSchema)
   }
 
   isFlagName(name: string | number | symbol): name is keyof T {
     return this.flagSchema[name as keyof T] !== undefined
   }
 
-  async get<K extends keyof T>(key: K): Promise<FlagValues<T>[K]> {
+  async isEnabled<K extends keyof T>(key: K): Promise<T[K]> {
     const flags = await this.fetch()
     return flags[key]
   }
 
-  async isEnabled<K extends KeysOfType<T, boolean>>(key: K): Promise<boolean> {
-    const flags = await this.fetch()
-    return flags[key]
-  }
-
-  async fetch(): Promise<FlagValues<T>> {
+  async fetch(): Promise<T> {
     return await tracer.trace("features.fetch", async span => {
-      const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
+      const cachedFlags = context.getFeatureFlags(this.setId)
       if (cachedFlags) {
         span?.addTags({ fromCache: true })
-        return cachedFlags
+        return cachedFlags as T
       }
 
       const tags: Record<string, any> = {}
@@ -189,7 +113,7 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
 
         // @ts-expect-error - TS does not like you writing into a generic type,
         // but we know that it's okay in this case because it's just an object.
-        flagValues[key as keyof FlagValues] = value
+        flagValues[key as keyof T] = value
         tags[`flags.${key}.source`] = "environment"
       }
 
@@ -217,11 +141,11 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
         tags[`readFromPostHog`] = true
 
         const personProperties: Record<string, string> = { tenantId }
-        const posthogFlags = await posthog.getAllFlagsAndPayloads(userId, {
+        const posthogFlags = await posthog.getAllFlags(userId, {
           personProperties,
         })
 
-        for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
+        for (const [name, value] of Object.entries(posthogFlags)) {
           if (!this.isFlagName(name)) {
             // We don't want an unexpected PostHog flag to break the app, so we
             // just log it and continue.
@@ -229,19 +153,20 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
             continue
           }
 
+          if (typeof value !== "boolean") {
+            console.warn(`Invalid value for posthog flag "${name}": ${value}`)
+            continue
+          }
+
           if (flagValues[name] === true || specificallySetFalse.has(name)) {
             // If the flag is already set to through environment variables, we
             // don't want to override it back to false here.
             continue
           }
 
-          const payload = posthogFlags.featureFlagPayloads?.[name]
-          const flag = this.flagSchema[name]
           try {
-            // @ts-expect-error - TS does not like you writing into a generic
-            // type, but we know that it's okay in this case because it's just
-            // an object.
-            flagValues[name] = flag.parse(payload || value)
+            // @ts-expect-error - TS does not like you writing into a generic type.
+            flagValues[name] = value
             tags[`flags.${name}.source`] = "posthog"
           } catch (err) {
             // We don't want an invalid PostHog flag to break the app, so we just
@@ -262,18 +187,12 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
   }
 }
 
-// This is the primary source of truth for feature flags. If you want to add a
-// new flag, add it here and use the `fetch` and `get` functions to access it.
-// All of the machinery in this file is to make sure that flags have their
-// default values set correctly and their types flow through the system.
-const flagsConfig: Record<FeatureFlag, Flag<any>> = {
-  [FeatureFlag.DEFAULT_VALUES]: Flag.boolean(true),
-  [FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(true),
-  [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(true),
-  [FeatureFlag.BUDIBASE_AI]: Flag.boolean(true),
-  [FeatureFlag.USE_ZOD_VALIDATOR]: Flag.boolean(env.isDev()),
-}
-export const flags = new FlagSet(flagsConfig)
+export const flags = new FlagSet(FeatureFlagDefaults)
 
-type UnwrapPromise<T> = T extends Promise<infer U> ? U : T
-export type FeatureFlags = UnwrapPromise<ReturnType<typeof flags.fetch>>
+export async function isEnabled(flag: keyof typeof FeatureFlagDefaults) {
+  return await flags.isEnabled(flag)
+}
+
+export async function all() {
+  return await flags.fetch()
+}

packages/backend-core/src/features/tests/features.spec.ts

@@ -1,5 +1,5 @@
 import { IdentityContext, IdentityType } from "@budibase/types"
-import { Flag, FlagSet, FlagValues, init, shutdown } from "../"
+import { FlagSet, init, shutdown } from "../"
 import * as context from "../../context"
 import environment, { withEnv } from "../../environment"
 import nodeFetch from "node-fetch"
@@ -7,10 +7,8 @@ import nock from "nock"
 import * as crypto from "crypto"
 
 const schema = {
-  TEST_BOOLEAN: Flag.boolean(false),
-  TEST_STRING: Flag.string("default value"),
-  TEST_NUMBER: Flag.number(0),
-  TEST_BOOLEAN_DEFAULT_TRUE: Flag.boolean(true),
+  TEST_BOOLEAN: false,
+  TEST_BOOLEAN_DEFAULT_TRUE: true,
 }
 const flags = new FlagSet(schema)
 
@@ -19,7 +17,7 @@ interface TestCase {
   identity?: Partial<IdentityContext>
   environmentFlags?: string
   posthogFlags?: PostHogFlags
-  expected?: Partial<FlagValues<typeof schema>>
+  expected?: Partial<typeof schema>
   errorMessage?: string | RegExp
 }
 
@@ -83,22 +81,6 @@ describe("feature flags", () => {
       },
       expected: { TEST_BOOLEAN: true },
     },
-    {
-      it: "should be able to read string flags from PostHog",
-      posthogFlags: {
-        featureFlags: { TEST_STRING: true },
-        featureFlagPayloads: { TEST_STRING: "test" },
-      },
-      expected: { TEST_STRING: "test" },
-    },
-    {
-      it: "should be able to read numeric flags from PostHog",
-      posthogFlags: {
-        featureFlags: { TEST_NUMBER: true },
-        featureFlagPayloads: { TEST_NUMBER: "123" },
-      },
-      expected: { TEST_NUMBER: 123 },
-    },
     {
       it: "should not be able to override a negative environment flag from PostHog",
       environmentFlags: "default:!TEST_BOOLEAN",
@@ -177,7 +159,7 @@ describe("feature flags", () => {
             expect(values).toMatchObject(expected)
 
             for (const [key, expectedValue] of Object.entries(expected)) {
-              const value = await flags.get(key as keyof typeof schema)
+              const value = await flags.isEnabled(key as keyof typeof schema)
               expect(value).toBe(expectedValue)
             }
           } else {

packages/backend-core/src/features/tests/utils.ts

@@ -1,5 +1,6 @@
-import { FeatureFlags, parseEnvFlags } from ".."
+import { FeatureFlags } from "@budibase/types"
 import { setEnv } from "../../environment"
+import { parseEnvFlags } from "../features"
 
 function getCurrentFlags(): Record<string, Record<string, boolean>> {
   const result: Record<string, Record<string, boolean>> = {}

packages/pro

@@ -1 +1 @@
-Subproject commit ae786121d923449b0ad5fcbd123d0a9fec28f65e
+Subproject commit 32d84f109d4edc526145472a7446327312151442

packages/server/src/api/controllers/row/staticFormula.ts

@@ -163,9 +163,9 @@ export async function finaliseRow(
     contextRows: [enrichedRow],
   })
   const aiEnabled =
-    ((await features.flags.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
+    ((await features.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
       (await pro.features.isBudibaseAIEnabled())) ||
-    ((await features.flags.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) &&
+    ((await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) &&
       (await pro.features.isAICustomConfigsEnabled()))
   if (aiEnabled) {
     row = await processAIColumns(table, row, {

packages/server/src/automations/actions.ts

@@ -105,13 +105,13 @@ if (env.SELF_HOSTED) {
 export async function getActionDefinitions(): Promise<
   Record<keyof typeof AutomationActionStepId, AutomationStepDefinition>
 > {
-  if (await features.flags.isEnabled(FeatureFlag.AUTOMATION_BRANCHING)) {
+  if (await features.isEnabled(FeatureFlag.AUTOMATION_BRANCHING)) {
     BUILTIN_ACTION_DEFINITIONS["BRANCH"] = branch.definition
   }
   if (
     env.SELF_HOSTED ||
-    (await features.flags.isEnabled(FeatureFlag.BUDIBASE_AI)) ||
-    (await features.flags.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS))
+    (await features.isEnabled(FeatureFlag.BUDIBASE_AI)) ||
+    (await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS))
   ) {
     BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition
   }

packages/server/src/automations/steps/openai.ts

@@ -100,10 +100,10 @@ export async function run({
   try {
     let response
     const customConfigsEnabled =
-      (await features.flags.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) &&
+      (await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) &&
       (await pro.features.isAICustomConfigsEnabled())
     const budibaseAIEnabled =
-      (await features.flags.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
+      (await features.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
       (await pro.features.isBudibaseAIEnabled())
 
     let llmWrapper

packages/server/src/middleware/zod-validator.ts

@@ -7,7 +7,7 @@ import { fromZodError } from "zod-validation-error"
 function validate(schema: AnyZodObject, property: "body" | "params") {
   // Return a Koa middleware function
   return async (ctx: Ctx, next: any) => {
-    if (!(await features.flags.isEnabled(FeatureFlag.USE_ZOD_VALIDATOR))) {
+    if (!(await features.isEnabled(FeatureFlag.USE_ZOD_VALIDATOR))) {
       return next()
     }
 

packages/types/src/api/web/global/self.ts

@@ -1,3 +1,4 @@
+import { FeatureFlags } from "@budibase/types"
 import { DevInfo, User } from "../../../documents"
 
 export interface GenerateAPIKeyRequest {
@@ -8,5 +9,5 @@ export interface GenerateAPIKeyResponse extends DevInfo {}
 export interface FetchAPIKeyResponse extends DevInfo {}
 
 export interface GetGlobalSelfResponse extends User {
-  flags?: Record<string, string>
+  flags?: FeatureFlags
 }

packages/types/src/sdk/featureFlag.ts

@@ -6,6 +6,12 @@ export enum FeatureFlag {
   USE_ZOD_VALIDATOR = "USE_ZOD_VALIDATOR",
 }
 
-export interface TenantFeatureFlags {
-  [key: string]: FeatureFlag[]
+export const FeatureFlagDefaults = {
+  [FeatureFlag.DEFAULT_VALUES]: true,
+  [FeatureFlag.AUTOMATION_BRANCHING]: true,
+  [FeatureFlag.AI_CUSTOM_CONFIGS]: true,
+  [FeatureFlag.BUDIBASE_AI]: true,
+  [FeatureFlag.USE_ZOD_VALIDATOR]: false,
 }
+
+export type FeatureFlags = typeof FeatureFlagDefaults

packages/worker/src/api/controllers/global/self.ts

@@ -111,8 +111,7 @@ export async function getSelf(ctx: UserCtx<void, GetGlobalSelfResponse>) {
   ctx.body = await groups.enrichUserRolesFromGroups(user)
 
   // add the feature flags for this tenant
-  const flags = await features.flags.fetch()
-  ctx.body.flags = flags
+  ctx.body.flags = await features.flags.fetch()
 
   addSessionAttributesToUser(ctx)
 }