Merge master.

packages/backend-core/src/features/features.ts

@@ -0,0 +1,300 @@
+import env from "../environment"
+import * as context from "../context"
+import { PostHog, PostHogOptions } from "posthog-node"
+import { FeatureFlag, IdentityType, UserCtx } from "@budibase/types"
+import tracer from "dd-trace"
+import { Duration } from "../utils"
+
+let posthog: PostHog | undefined
+export function init(opts?: PostHogOptions) {
+  if (
+    env.POSTHOG_TOKEN &&
+    env.POSTHOG_API_HOST &&
+    !env.SELF_HOSTED &&
+    env.POSTHOG_FEATURE_FLAGS_ENABLED
+  ) {
+    console.log("initializing posthog client...")
+    posthog = new PostHog(env.POSTHOG_TOKEN, {
+      host: env.POSTHOG_API_HOST,
+      personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
+      featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(),
+      ...opts,
+    })
+  } else {
+    console.log("posthog disabled")
+  }
+}
+
+export function shutdown() {
+  posthog?.shutdown()
+}
+
+export abstract class Flag<T> {
+  static boolean(defaultValue: boolean): Flag<boolean> {
+    return new BooleanFlag(defaultValue)
+  }
+
+  static string(defaultValue: string): Flag<string> {
+    return new StringFlag(defaultValue)
+  }
+
+  static number(defaultValue: number): Flag<number> {
+    return new NumberFlag(defaultValue)
+  }
+
+  protected constructor(public defaultValue: T) {}
+
+  abstract parse(value: any): T
+}
+
+type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
+
+export type FlagValues<T> = {
+  [K in keyof T]: UnwrapFlag<T[K]>
+}
+
+type KeysOfType<T, U> = {
+  [K in keyof T]: T[K] extends Flag<U> ? K : never
+}[keyof T]
+
+class BooleanFlag extends Flag<boolean> {
+  parse(value: any) {
+    if (typeof value === "string") {
+      return ["true", "t", "1"].includes(value.toLowerCase())
+    }
+
+    if (typeof value === "boolean") {
+      return value
+    }
+
+    throw new Error(`could not parse value "${value}" as boolean`)
+  }
+}
+
+class StringFlag extends Flag<string> {
+  parse(value: any) {
+    if (typeof value === "string") {
+      return value
+    }
+    throw new Error(`could not parse value "${value}" as string`)
+  }
+}
+
+class NumberFlag extends Flag<number> {
+  parse(value: any) {
+    if (typeof value === "number") {
+      return value
+    }
+
+    if (typeof value === "string") {
+      const parsed = parseFloat(value)
+      if (!isNaN(parsed)) {
+        return parsed
+      }
+    }
+
+    throw new Error(`could not parse value "${value}" as number`)
+  }
+}
+
+export interface EnvFlagEntry {
+  tenantId: string
+  key: string
+  value: boolean
+}
+
+export function parseEnvFlags(flags: string): EnvFlagEntry[] {
+  const split = flags.split(",").map(x => x.split(":"))
+  const result: EnvFlagEntry[] = []
+  for (const [tenantId, ...features] of split) {
+    for (let feature of features) {
+      let value = true
+      if (feature.startsWith("!")) {
+        feature = feature.slice(1)
+        value = false
+      }
+      result.push({ tenantId, key: feature, value })
+    }
+  }
+  return result
+}
+
+export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
+  // This is used to safely cache flags sets in the current request context.
+  // Because multiple sets could theoretically exist, we don't want the cache of
+  // one to leak into another.
+  private readonly setId: string
+
+  constructor(private readonly flagSchema: T) {
+    this.setId = crypto.randomUUID()
+  }
+
+  defaults(): FlagValues<T> {
+    return Object.keys(this.flagSchema).reduce((acc, key) => {
+      const typedKey = key as keyof T
+      acc[typedKey] = this.flagSchema[key].defaultValue
+      return acc
+    }, {} as FlagValues<T>)
+  }
+
+  isFlagName(name: string | number | symbol): name is keyof T {
+    return this.flagSchema[name as keyof T] !== undefined
+  }
+
+  async get<K extends keyof T>(
+    key: K,
+    ctx?: UserCtx
+  ): Promise<FlagValues<T>[K]> {
+    const flags = await this.fetch(ctx)
+    return flags[key]
+  }
+
+  async isEnabled<K extends KeysOfType<T, boolean>>(
+    key: K,
+    ctx?: UserCtx
+  ): Promise<boolean> {
+    const flags = await this.fetch(ctx)
+    return flags[key]
+  }
+
+  async fetch(ctx?: UserCtx): Promise<FlagValues<T>> {
+    return await tracer.trace("features.fetch", async span => {
+      const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
+      if (cachedFlags) {
+        span?.addTags({ fromCache: true })
+        return cachedFlags
+      }
+
+      const tags: Record<string, any> = {}
+      const flagValues = this.defaults()
+      const currentTenantId = context.getTenantId()
+      const specificallySetFalse = new Set<string>()
+
+      for (const { tenantId, key, value } of parseEnvFlags(
+        env.TENANT_FEATURE_FLAGS || ""
+      )) {
+        if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
+          continue
+        }
+
+        tags[`readFromEnvironmentVars`] = true
+
+        if (value === false) {
+          specificallySetFalse.add(key)
+        }
+
+        // ignore unknown flags
+        if (!this.isFlagName(key)) {
+          continue
+        }
+
+        if (typeof flagValues[key] !== "boolean") {
+          throw new Error(`Feature: ${key} is not a boolean`)
+        }
+
+        // @ts-expect-error - TS does not like you writing into a generic type,
+        // but we know that it's okay in this case because it's just an object.
+        flagValues[key as keyof FlagValues] = value
+        tags[`flags.${key}.source`] = "environment"
+      }
+
+      const license = ctx?.user?.license
+      if (license) {
+        tags[`readFromLicense`] = true
+
+        for (const feature of license.features) {
+          if (!this.isFlagName(feature)) {
+            continue
+          }
+
+          if (
+            flagValues[feature] === true ||
+            specificallySetFalse.has(feature)
+          ) {
+            // If the flag is already set to through environment variables, we
+            // don't want to override it back to false here.
+            continue
+          }
+
+          // @ts-expect-error - TS does not like you writing into a generic type,
+          // but we know that it's okay in this case because it's just an object.
+          flagValues[feature] = true
+          tags[`flags.${feature}.source`] = "license"
+        }
+      }
+
+      const identity = context.getIdentity()
+      tags[`identity.type`] = identity?.type
+      tags[`identity.tenantId`] = identity?.tenantId
+      tags[`identity._id`] = identity?._id
+
+      if (posthog && identity?.type === IdentityType.USER) {
+        tags[`readFromPostHog`] = true
+
+        const personProperties: Record<string, string> = {}
+        if (identity.tenantId) {
+          personProperties.tenantId = identity.tenantId
+        }
+
+        const posthogFlags = await posthog.getAllFlagsAndPayloads(
+          identity._id,
+          {
+            personProperties,
+          }
+        )
+
+        for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
+          if (!this.isFlagName(name)) {
+            // We don't want an unexpected PostHog flag to break the app, so we
+            // just log it and continue.
+            console.warn(`Unexpected posthog flag "${name}": ${value}`)
+            continue
+          }
+
+          if (flagValues[name] === true || specificallySetFalse.has(name)) {
+            // If the flag is already set to through environment variables, we
+            // don't want to override it back to false here.
+            continue
+          }
+
+          const payload = posthogFlags.featureFlagPayloads?.[name]
+          const flag = this.flagSchema[name]
+          try {
+            // @ts-expect-error - TS does not like you writing into a generic
+            // type, but we know that it's okay in this case because it's just
+            // an object.
+            flagValues[name] = flag.parse(payload || value)
+            tags[`flags.${name}.source`] = "posthog"
+          } catch (err) {
+            // We don't want an invalid PostHog flag to break the app, so we just
+            // log it and continue.
+            console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
+          }
+        }
+      }
+
+      context.setFeatureFlags(this.setId, flagValues)
+      for (const [key, value] of Object.entries(flagValues)) {
+        tags[`flags.${key}.value`] = value
+      }
+      span?.addTags(tags)
+
+      return flagValues
+    })
+  }
+}
+
+// This is the primary source of truth for feature flags. If you want to add a
+// new flag, add it here and use the `fetch` and `get` functions to access it.
+// All of the machinery in this file is to make sure that flags have their
+// default values set correctly and their types flow through the system.
+export const flags = new FlagSet({
+  DEFAULT_VALUES: Flag.boolean(env.isDev()),
+  AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
+  SQS: Flag.boolean(env.isDev()),
+  [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
+  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
+})
+
+type UnwrapPromise<T> = T extends Promise<infer U> ? U : T
+export type FeatureFlags = UnwrapPromise<ReturnType<typeof flags.fetch>>

packages/backend-core/src/features/index.ts

@@ -1,281 +1,2 @@
-import env from "../environment"
+export * from "./features"
-import * as context from "../context"
+export * as testutils from "./tests/utils"
-import { PostHog, PostHogOptions } from "posthog-node"
-import { FeatureFlag, IdentityType, UserCtx } from "@budibase/types"
-import tracer from "dd-trace"
-import { Duration } from "../utils"
-
-let posthog: PostHog | undefined
-export function init(opts?: PostHogOptions) {
-  if (
-    env.POSTHOG_TOKEN &&
-    env.POSTHOG_API_HOST &&
-    !env.SELF_HOSTED &&
-    env.POSTHOG_FEATURE_FLAGS_ENABLED
-  ) {
-    console.log("initializing posthog client...")
-    posthog = new PostHog(env.POSTHOG_TOKEN, {
-      host: env.POSTHOG_API_HOST,
-      personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
-      featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(),
-      ...opts,
-    })
-  } else {
-    console.log("posthog disabled")
-  }
-}
-
-export function shutdown() {
-  posthog?.shutdown()
-}
-
-export abstract class Flag<T> {
-  static boolean(defaultValue: boolean): Flag<boolean> {
-    return new BooleanFlag(defaultValue)
-  }
-
-  static string(defaultValue: string): Flag<string> {
-    return new StringFlag(defaultValue)
-  }
-
-  static number(defaultValue: number): Flag<number> {
-    return new NumberFlag(defaultValue)
-  }
-
-  protected constructor(public defaultValue: T) {}
-
-  abstract parse(value: any): T
-}
-
-type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
-
-export type FlagValues<T> = {
-  [K in keyof T]: UnwrapFlag<T[K]>
-}
-
-type KeysOfType<T, U> = {
-  [K in keyof T]: T[K] extends Flag<U> ? K : never
-}[keyof T]
-
-class BooleanFlag extends Flag<boolean> {
-  parse(value: any) {
-    if (typeof value === "string") {
-      return ["true", "t", "1"].includes(value.toLowerCase())
-    }
-
-    if (typeof value === "boolean") {
-      return value
-    }
-
-    throw new Error(`could not parse value "${value}" as boolean`)
-  }
-}
-
-class StringFlag extends Flag<string> {
-  parse(value: any) {
-    if (typeof value === "string") {
-      return value
-    }
-    throw new Error(`could not parse value "${value}" as string`)
-  }
-}
-
-class NumberFlag extends Flag<number> {
-  parse(value: any) {
-    if (typeof value === "number") {
-      return value
-    }
-
-    if (typeof value === "string") {
-      const parsed = parseFloat(value)
-      if (!isNaN(parsed)) {
-        return parsed
-      }
-    }
-
-    throw new Error(`could not parse value "${value}" as number`)
-  }
-}
-
-export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
-  // This is used to safely cache flags sets in the current request context.
-  // Because multiple sets could theoretically exist, we don't want the cache of
-  // one to leak into another.
-  private readonly setId: string
-
-  constructor(private readonly flagSchema: T) {
-    this.setId = crypto.randomUUID()
-  }
-
-  defaults(): FlagValues<T> {
-    return Object.keys(this.flagSchema).reduce((acc, key) => {
-      const typedKey = key as keyof T
-      acc[typedKey] = this.flagSchema[key].defaultValue
-      return acc
-    }, {} as FlagValues<T>)
-  }
-
-  isFlagName(name: string | number | symbol): name is keyof T {
-    return this.flagSchema[name as keyof T] !== undefined
-  }
-
-  async get<K extends keyof T>(
-    key: K,
-    ctx?: UserCtx
-  ): Promise<FlagValues<T>[K]> {
-    const flags = await this.fetch(ctx)
-    return flags[key]
-  }
-
-  async isEnabled<K extends KeysOfType<T, boolean>>(
-    key: K,
-    ctx?: UserCtx
-  ): Promise<boolean> {
-    const flags = await this.fetch(ctx)
-    return flags[key]
-  }
-
-  async fetch(ctx?: UserCtx): Promise<FlagValues<T>> {
-    return await tracer.trace("features.fetch", async span => {
-      const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
-      if (cachedFlags) {
-        span?.addTags({ fromCache: true })
-        return cachedFlags
-      }
-
-      const tags: Record<string, any> = {}
-      const flagValues = this.defaults()
-      const currentTenantId = context.getTenantId()
-      const specificallySetFalse = new Set<string>()
-
-      const split = (env.TENANT_FEATURE_FLAGS || "")
-        .split(",")
-        .map(x => x.split(":"))
-      for (const [tenantId, ...features] of split) {
-        if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
-          continue
-        }
-
-        tags[`readFromEnvironmentVars`] = true
-
-        for (let feature of features) {
-          let value = true
-          if (feature.startsWith("!")) {
-            feature = feature.slice(1)
-            value = false
-            specificallySetFalse.add(feature)
-          }
-
-          // ignore unknown flags
-          if (!this.isFlagName(feature)) {
-            continue
-          }
-
-          if (typeof flagValues[feature] !== "boolean") {
-            throw new Error(`Feature: ${feature} is not a boolean`)
-          }
-
-          // @ts-expect-error - TS does not like you writing into a generic type,
-          // but we know that it's okay in this case because it's just an object.
-          flagValues[feature as keyof FlagValues] = value
-          tags[`flags.${feature}.source`] = "environment"
-        }
-      }
-
-      const license = ctx?.user?.license
-      if (license) {
-        tags[`readFromLicense`] = true
-
-        for (const feature of license.features) {
-          if (!this.isFlagName(feature)) {
-            continue
-          }
-
-          if (
-            flagValues[feature] === true ||
-            specificallySetFalse.has(feature)
-          ) {
-            // If the flag is already set to through environment variables, we
-            // don't want to override it back to false here.
-            continue
-          }
-
-          // @ts-expect-error - TS does not like you writing into a generic type,
-          // but we know that it's okay in this case because it's just an object.
-          flagValues[feature] = true
-          tags[`flags.${feature}.source`] = "license"
-        }
-      }
-
-      const identity = context.getIdentity()
-      tags[`identity.type`] = identity?.type
-      tags[`identity.tenantId`] = identity?.tenantId
-      tags[`identity._id`] = identity?._id
-
-      if (posthog && identity?.type === IdentityType.USER) {
-        tags[`readFromPostHog`] = true
-
-        const personProperties: Record<string, string> = {}
-        if (identity.tenantId) {
-          personProperties.tenantId = identity.tenantId
-        }
-
-        const posthogFlags = await posthog.getAllFlagsAndPayloads(
-          identity._id,
-          {
-            personProperties,
-          }
-        )
-
-        for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
-          if (!this.isFlagName(name)) {
-            // We don't want an unexpected PostHog flag to break the app, so we
-            // just log it and continue.
-            console.warn(`Unexpected posthog flag "${name}": ${value}`)
-            continue
-          }
-
-          if (flagValues[name] === true || specificallySetFalse.has(name)) {
-            // If the flag is already set to through environment variables, we
-            // don't want to override it back to false here.
-            continue
-          }
-
-          const payload = posthogFlags.featureFlagPayloads?.[name]
-          const flag = this.flagSchema[name]
-          try {
-            // @ts-expect-error - TS does not like you writing into a generic
-            // type, but we know that it's okay in this case because it's just
-            // an object.
-            flagValues[name] = flag.parse(payload || value)
-            tags[`flags.${name}.source`] = "posthog"
-          } catch (err) {
-            // We don't want an invalid PostHog flag to break the app, so we just
-            // log it and continue.
-            console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
-          }
-        }
-      }
-
-      context.setFeatureFlags(this.setId, flagValues)
-      for (const [key, value] of Object.entries(flagValues)) {
-        tags[`flags.${key}.value`] = value
-      }
-      span?.addTags(tags)
-
-      return flagValues
-    })
-  }
-}
-
-// This is the primary source of truth for feature flags. If you want to add a
-// new flag, add it here and use the `fetch` and `get` functions to access it.
-// All of the machinery in this file is to make sure that flags have their
-// default values set correctly and their types flow through the system.
-export const flags = new FlagSet({
-  DEFAULT_VALUES: Flag.boolean(env.isDev()),
-  AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
-  SQS: Flag.boolean(env.isDev()),
-  [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
-  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
-})

packages/backend-core/src/features/tests/utils.ts

@@ -0,0 +1,64 @@
+import { FeatureFlags, parseEnvFlags } from ".."
+import { setEnv } from "../../environment"
+
+function getCurrentFlags(): Record<string, Record<string, boolean>> {
+  const result: Record<string, Record<string, boolean>> = {}
+  for (const { tenantId, key, value } of parseEnvFlags(
+    process.env.TENANT_FEATURE_FLAGS || ""
+  )) {
+    const tenantFlags = result[tenantId] || {}
+    // Don't allow overwriting specifically false flags, to match the beheaviour
+    // of FlagSet.
+    if (tenantFlags[key] === false) {
+      continue
+    }
+    tenantFlags[key] = value
+    result[tenantId] = tenantFlags
+  }
+  return result
+}
+
+function buildFlagString(
+  flags: Record<string, Record<string, boolean>>
+): string {
+  const parts: string[] = []
+  for (const [tenantId, tenantFlags] of Object.entries(flags)) {
+    for (const [key, value] of Object.entries(tenantFlags)) {
+      if (value === false) {
+        parts.push(`${tenantId}:!${key}`)
+      } else {
+        parts.push(`${tenantId}:${key}`)
+      }
+    }
+  }
+  return parts.join(",")
+}
+
+export function setFeatureFlags(
+  tenantId: string,
+  flags: Partial<FeatureFlags>
+): () => void {
+  const current = getCurrentFlags()
+  for (const [key, value] of Object.entries(flags)) {
+    const tenantFlags = current[tenantId] || {}
+    tenantFlags[key] = value
+    current[tenantId] = tenantFlags
+  }
+  const flagString = buildFlagString(current)
+  return setEnv({ TENANT_FEATURE_FLAGS: flagString })
+}
+
+export function withFeatureFlags<T>(
+  tenantId: string,
+  flags: Partial<FeatureFlags>,
+  f: () => T
+) {
+  const cleanup = setFeatureFlags(tenantId, flags)
+  const result = f()
+  if (result instanceof Promise) {
+    return result.finally(cleanup)
+  } else {
+    cleanup()
+    return result
+  }
+}

packages/backend-core/src/middleware/passport/sso/sso.ts

@@ -2,7 +2,6 @@ import { generateGlobalUserID } from "../../../db"
 import { authError } from "../utils"
 import * as users from "../../../users"
 import * as context from "../../../context"
-import fetch from "node-fetch"
 import {
   SaveSSOUserFunction,
   SSOAuthDetails,
@@ -97,28 +96,13 @@ export async function authenticate(
   return done(null, ssoUser)
 }
 
-async function getProfilePictureUrl(user: User, details: SSOAuthDetails) {
-  const pictureUrl = details.profile?._json.picture
-  if (pictureUrl) {
-    const response = await fetch(pictureUrl)
-    if (response.status === 200) {
-      const type = response.headers.get("content-type") as string
-      if (type.startsWith("image/")) {
-        return pictureUrl
-      }
-    }
-  }
-}
-
 /**
  * @returns a user that has been sync'd with third party information
  */
 async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
   let firstName
   let lastName
-  let pictureUrl
   let oauth2
-  let thirdPartyProfile
 
   if (details.profile) {
     const profile = details.profile
@@ -134,12 +118,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
         lastName = name.familyName
       }
     }
-
-    pictureUrl = await getProfilePictureUrl(user, details)
-
-    thirdPartyProfile = {
-      ...profile._json,
-    }
   }
 
   // oauth tokens for future use
@@ -155,8 +133,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
     providerType: details.providerType,
     firstName,
     lastName,
-    thirdPartyProfile,
-    pictureUrl,
     oauth2,
   }
 }

packages/backend-core/tests/core/utilities/structures/accounts.ts

@@ -59,10 +59,8 @@ export function ssoAccount(account: Account = cloudAccount()): SSOAccount {
       accessToken: generator.string(),
       refreshToken: generator.string(),
     },
-    pictureUrl: generator.url(),
     provider: provider(),
     providerType: providerType(),
-    thirdPartyProfile: {},
   }
 }
 
@@ -76,9 +74,7 @@ export function verifiableSsoAccount(
       accessToken: generator.string(),
       refreshToken: generator.string(),
     },
-    pictureUrl: generator.url(),
     provider: AccountSSOProvider.MICROSOFT,
     providerType: AccountSSOProviderType.MICROSOFT,
-    thirdPartyProfile: { id: "abc123" },
   }
 }

packages/backend-core/tests/core/utilities/structures/users.ts

@@ -25,7 +25,6 @@ export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
     roles: { app_test: "admin" },
     firstName: generator.first(),
     lastName: generator.last(),
-    pictureUrl: "http://example.com",
     tenantId: tenant.id(),
     ...userProps,
   }
@@ -86,9 +85,5 @@ export function ssoUser(
     oauth2: opts.details?.oauth2,
     provider: opts.details?.provider!,
     providerType: opts.details?.providerType!,
-    thirdPartyProfile: {
-      email: base.email,
-      picture: base.pictureUrl,
-    },
   }
 }

packages/server/src/api/controllers/view/viewsV2.ts

@@ -127,13 +127,13 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
 
   const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version"> = {
     name: view.name,
+    type: view.type,
     tableId: view.tableId,
     query: view.query,
     queryUI: view.queryUI,
     sort: view.sort,
     schema,
     primaryDisplay: view.primaryDisplay,
-    uiMetadata: view.uiMetadata,
   }
   const result = await sdk.views.create(tableId, parsedView)
   ctx.status = 201
@@ -163,6 +163,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
   const parsedView: RequiredKeys<ViewV2> = {
     id: view.id,
     name: view.name,
+    type: view.type,
     version: view.version,
     tableId: view.tableId,
     query: view.query,
@@ -170,7 +171,6 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
     sort: view.sort,
     schema,
     primaryDisplay: view.primaryDisplay,
-    uiMetadata: view.uiMetadata,
   }
 
   const result = await sdk.views.update(tableId, parsedView)

packages/server/src/api/routes/tests/application.spec.ts

@@ -14,12 +14,7 @@ jest.mock("../../../utilities/redis", () => ({
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 import { AppStatus } from "../../../db/utils"
-import {
+import { events, utils, context, features } from "@budibase/backend-core"
-  events,
-  utils,
-  context,
-  withEnv as withCoreEnv,
-} from "@budibase/backend-core"
 import env from "../../../environment"
 import { type App } from "@budibase/types"
 import tk from "timekeeper"
@@ -358,9 +353,13 @@ describe("/applications", () => {
         .delete(`/api/global/roles/${prodAppId}`)
         .reply(200, {})
 
-      await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, async () => {
+      await features.testutils.withFeatureFlags(
+        "*",
+        { SQS: true },
+        async () => {
           await config.api.application.delete(app.appId)
-      })
+        }
+      )
     })
   })
 

packages/server/src/api/routes/tests/row.spec.ts

@@ -13,8 +13,7 @@ import {
   context,
   InternalTable,
   tenancy,
-  withEnv as withCoreEnv,
+  features,
-  setEnv as setCoreEnv,
 } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
@@ -40,7 +39,6 @@ import {
   TableSchema,
   JsonFieldSubType,
   RowExportFormat,
-  FeatureFlag,
   RelationSchemaField,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
@@ -98,12 +96,12 @@ describe.each([
   let envCleanup: (() => void) | undefined
 
   beforeAll(async () => {
-    await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init())
+    await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
-    if (isLucene) {
+      config.init()
-      envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" })
+    )
-    } else if (isSqs) {
+    envCleanup = features.testutils.setFeatureFlags("*", {
-      envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" })
+      SQS: isSqs,
-    }
+    })
 
     if (dsProvider) {
       const rawDatasource = await dsProvider
@@ -2517,15 +2515,9 @@ describe.each([
       let flagCleanup: (() => void) | undefined
 
       beforeAll(async () => {
-        const env = {
+        flagCleanup = features.testutils.setFeatureFlags("*", {
-          TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
+          ENRICHED_RELATIONSHIPS: true,
-        }
+        })
-        if (isSqs) {
-          env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:SQS`
-        } else {
-          env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:!SQS`
-        }
-        flagCleanup = setCoreEnv(env)
 
         const aux2Table = await config.api.table.save(saveTableRequest())
         const aux2Data = await config.api.row.save(aux2Table._id!, {})
@@ -2752,9 +2744,10 @@ describe.each([
       it.each(testScenarios)(
         "does not enrich relationships when not enabled (via %s)",
         async (__, retrieveDelegate) => {
-          await withCoreEnv(
+          await features.testutils.withFeatureFlags(
+            "*",
             {
-              TENANT_FEATURE_FLAGS: `*:!${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
+              ENRICHED_RELATIONSHIPS: false,
             },
             async () => {
               const otherRows = _.sampleSize(auxData, 5)

packages/server/src/api/routes/tests/search.spec.ts

@@ -7,9 +7,9 @@ import {
 import {
   context,
   db as dbCore,
+  features,
   MAX_VALID_DATE,
   MIN_VALID_DATE,
-  setEnv as setCoreEnv,
   SQLITE_DESIGN_DOC_ID,
   utils,
   withEnv as withCoreEnv,
@@ -94,16 +94,12 @@ describe.each([
   }
 
   beforeAll(async () => {
-    await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init())
+    await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
-    if (isLucene) {
+      config.init()
-      envCleanup = setCoreEnv({
+    )
-        TENANT_FEATURE_FLAGS: "*:!SQS",
+    envCleanup = features.testutils.setFeatureFlags("*", {
+      SQS: isSqs,
     })
-    } else if (isSqs) {
-      envCleanup = setCoreEnv({
-        TENANT_FEATURE_FLAGS: "*:SQS",
-      })
-    }
 
     if (config.app?.appId) {
       config.app = await config.api.application.update(config.app?.appId, {

packages/server/src/api/routes/tests/templates.spec.ts

@@ -2,7 +2,7 @@ import * as setup from "./utilities"
 import path from "path"
 import nock from "nock"
 import { generator } from "@budibase/backend-core/tests"
-import { withEnv as withCoreEnv, env as coreEnv } from "@budibase/backend-core"
+import { features } from "@budibase/backend-core"
 
 interface App {
   background: string
@@ -85,11 +85,10 @@ describe("/templates", () => {
     it.each(["sqs", "lucene"])(
       `should be able to create an app from a template (%s)`,
       async source => {
-        const env: Partial<typeof coreEnv> = {
+        await features.testutils.withFeatureFlags(
-          TENANT_FEATURE_FLAGS: source === "sqs" ? "*:SQS" : "",
+          "*",
-        }
+          { SQS: source === "sqs" },
-
+          async () => {
-        await withCoreEnv(env, async () => {
             const name = generator.guid().replaceAll("-", "")
             const url = `/${name}`
 
@@ -112,14 +111,18 @@ describe("/templates", () => {
               expect(agencyProjects.name).toBe("Agency Projects")
               expect(users.name).toBe("Users")
 
-            const { rows } = await config.api.row.search(agencyProjects._id!, {
+              const { rows } = await config.api.row.search(
+                agencyProjects._id!,
+                {
                   tableId: agencyProjects._id!,
                   query: {},
-            })
+                }
+              )
 
               expect(rows).toHaveLength(3)
             })
-        })
+          }
+        )
       }
     )
   })

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -22,7 +22,6 @@ import {
   RelationshipType,
   TableSchema,
   RenameColumn,
-  FeatureFlag,
   BBReferenceFieldSubType,
   NumericCalculationFieldMetadata,
   ViewV2Schema,
@@ -33,13 +32,7 @@ import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
 import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
-import {
+import { db, roles, features } from "@budibase/backend-core"
-  db,
-  roles,
-  withEnv as withCoreEnv,
-  setEnv as setCoreEnv,
-  env,
-} from "@budibase/backend-core"
 
 describe.each([
   ["lucene", undefined],
@@ -104,18 +97,13 @@ describe.each([
   }
 
   beforeAll(async () => {
-    await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () =>
+    await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
       config.init()
     )
-    if (isLucene) {
+
-      envCleanup = setCoreEnv({
+    envCleanup = features.testutils.setFeatureFlags("*", {
-        TENANT_FEATURE_FLAGS: "*:!SQS",
+      SQS: isSqs,
     })
-    } else if (isSqs) {
-      envCleanup = setCoreEnv({
-        TENANT_FEATURE_FLAGS: "*:SQS",
-      })
-    }
 
     if (dsProvider) {
       datasource = await config.createDatasource({
@@ -157,7 +145,7 @@ describe.each([
       })
 
       it("can persist views with all fields", async () => {
-        const newView: Required<Omit<CreateViewRequest, "queryUI">> = {
+        const newView: Required<Omit<CreateViewRequest, "queryUI" | "type">> = {
           name: generator.name(),
           tableId: table._id!,
           primaryDisplay: "id",
@@ -179,9 +167,6 @@ describe.each([
               visible: true,
             },
           },
-          uiMetadata: {
-            foo: "bar",
-          },
         }
         const res = await config.api.viewV2.create(newView)
 
@@ -551,6 +536,7 @@ describe.each([
         let view = await config.api.viewV2.create({
           tableId: table._id!,
           name: generator.guid(),
+          type: ViewV2Type.CALCULATION,
           schema: {
             sum: {
               visible: true,
@@ -574,11 +560,35 @@ describe.each([
         expect(sum.field).toEqual("Price")
       })
 
+      it("cannot create a view with calculation fields unless it has the right type", async () => {
+        await config.api.viewV2.create(
+          {
+            tableId: table._id!,
+            name: generator.guid(),
+            schema: {
+              sum: {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "Price",
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              message:
+                "Calculation fields are not allowed in non-calculation views",
+            },
+          }
+        )
+      })
+
       it("cannot create a calculation view with more than 5 aggregations", async () => {
         await config.api.viewV2.create(
           {
             tableId: table._id!,
             name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
             schema: {
               sum: {
                 visible: true,
@@ -626,6 +636,7 @@ describe.each([
           {
             tableId: table._id!,
             name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
             schema: {
               sum: {
                 visible: true,
@@ -654,6 +665,7 @@ describe.each([
           {
             tableId: table._id!,
             name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
             schema: {
               count: {
                 visible: true,
@@ -680,6 +692,7 @@ describe.each([
           {
             tableId: table._id!,
             name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
             schema: {
               count: {
                 visible: true,
@@ -709,6 +722,7 @@ describe.each([
         await config.api.viewV2.create({
           tableId: table._id!,
           name: generator.guid(),
+          type: ViewV2Type.CALCULATION,
           schema: {
             count: {
               visible: true,
@@ -795,7 +809,9 @@ describe.each([
       it("can update all fields", async () => {
         const tableId = table._id!
 
-        const updatedData: Required<Omit<UpdateViewRequest, "queryUI">> = {
+        const updatedData: Required<
+          Omit<UpdateViewRequest, "queryUI" | "type">
+        > = {
           version: view.version,
           id: view.id,
           tableId,
@@ -823,9 +839,6 @@ describe.each([
               readonly: true,
             },
           },
-          uiMetadata: {
-            foo: "bar",
-          },
         }
         await config.api.viewV2.update(updatedData)
 
@@ -1010,6 +1023,32 @@ describe.each([
         )
       })
 
+      it("cannot update view type after creation", async () => {
+        const view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          schema: {
+            id: { visible: true },
+            Price: {
+              visible: true,
+            },
+          },
+        })
+
+        await config.api.viewV2.update(
+          {
+            ...view,
+            type: ViewV2Type.CALCULATION,
+          },
+          {
+            status: 400,
+            body: {
+              message: "Cannot update view type after creation",
+            },
+          }
+        )
+      })
+
       isInternal &&
         it("updating schema will only validate modified field", async () => {
           let view = await config.api.viewV2.create({
@@ -1082,6 +1121,7 @@ describe.each([
             view = await config.api.viewV2.create({
               tableId: table._id!,
               name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
               schema: {
                 country: {
                   visible: true,
@@ -2643,12 +2683,8 @@ describe.each([
       describe("foreign relationship columns", () => {
         let envCleanup: () => void
         beforeAll(() => {
-          const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`]
+          envCleanup = features.testutils.setFeatureFlags("*", {
-          if (env.TENANT_FEATURE_FLAGS) {
+            ENRICHED_RELATIONSHIPS: true,
-            flags.push(...env.TENANT_FEATURE_FLAGS.split(","))
-          }
-          envCleanup = setCoreEnv({
-            TENANT_FEATURE_FLAGS: flags.join(","),
           })
         })
 
@@ -2839,6 +2875,7 @@ describe.each([
           it("should be able to search by calculations", async () => {
             const view = await config.api.viewV2.create({
               tableId: table._id!,
+              type: ViewV2Type.CALCULATION,
               name: generator.guid(),
               schema: {
                 "Quantity Sum": {
@@ -2873,6 +2910,7 @@ describe.each([
             const view = await config.api.viewV2.create({
               tableId: table._id!,
               name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
               schema: {
                 quantity: {
                   visible: true,
@@ -2911,6 +2949,7 @@ describe.each([
             const view = await config.api.viewV2.create({
               tableId: table._id!,
               name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
               schema: {
                 aggregate: {
                   visible: true,
@@ -2971,6 +3010,7 @@ describe.each([
             const view = await config.api.viewV2.create({
               tableId: table._id!,
               name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
               schema: {
                 count: {
                   visible: true,
@@ -3005,6 +3045,7 @@ describe.each([
               {
                 tableId: table._id!,
                 name: generator.guid(),
+                type: ViewV2Type.CALCULATION,
                 schema: {
                   count: {
                     visible: true,
@@ -3053,6 +3094,7 @@ describe.each([
           const view = await config.api.viewV2.create({
             tableId: table._id!,
             name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
             schema: {
               sum: {
                 visible: true,

packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts

@@ -2,8 +2,8 @@ import * as setup from "../../../api/routes/tests/utilities"
 import { basicTable } from "../../../tests/utilities/structures"
 import {
   db as dbCore,
+  features,
   SQLITE_DESIGN_DOC_ID,
-  withEnv as withCoreEnv,
 } from "@budibase/backend-core"
 import {
   LinkDocument,
@@ -71,11 +71,11 @@ function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
 }
 
 async function sqsDisabled(cb: () => Promise<void>) {
-  await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" }, cb)
+  await features.testutils.withFeatureFlags("*", { SQS: false }, cb)
 }
 
 async function sqsEnabled(cb: () => Promise<void>) {
-  await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, cb)
+  await features.testutils.withFeatureFlags("*", { SQS: true }, cb)
 }
 
 describe("SQS migration", () => {

packages/server/src/sdk/app/rows/search/tests/search.spec.ts

@@ -10,10 +10,7 @@ import {
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
 import { search } from "../../../../../sdk/app/rows/search"
 import { generator } from "@budibase/backend-core/tests"
-import {
+import { features } from "@budibase/backend-core"
-  withEnv as withCoreEnv,
-  setEnv as setCoreEnv,
-} from "@budibase/backend-core"
 import {
   DatabaseName,
   getDatasource,
@@ -41,19 +38,13 @@ describe.each([
   let table: Table
 
   beforeAll(async () => {
-    await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () =>
+    await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
       config.init()
     )
 
-    if (isLucene) {
+    envCleanup = features.testutils.setFeatureFlags("*", {
-      envCleanup = setCoreEnv({
+      SQS: isSqs,
-        TENANT_FEATURE_FLAGS: "*:!SQS",
     })
-    } else if (isSqs) {
-      envCleanup = setCoreEnv({
-        TENANT_FEATURE_FLAGS: "*:SQS",
-      })
-    }
 
     if (dsProvider) {
       datasource = await config.createDatasource({

packages/server/src/sdk/app/views/external.ts

@@ -70,6 +70,9 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
   if (!existingView || !existingView.name) {
     throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404)
   }
+  if (isV2(existingView) && existingView.type !== view.type) {
+    throw new HTTPError(`Cannot update view type after creation`, 400)
+  }
 
   delete views[existingView.name]
   views[view.name] = view

packages/server/src/sdk/app/views/index.ts

@@ -132,6 +132,13 @@ async function guardViewSchema(
 
   if (helpers.views.isCalculationView(view)) {
     await guardCalculationViewSchema(table, view)
+  } else {
+    if (helpers.views.hasCalculationFields(view)) {
+      throw new HTTPError(
+        "Calculation fields are not allowed in non-calculation views",
+        400
+      )
+    }
   }
 
   await checkReadonlyFields(table, view)

packages/server/src/sdk/app/views/internal.ts

@@ -59,6 +59,10 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
     throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404)
   }
 
+  if (isV2(existingView) && existingView.type !== view.type) {
+    throw new HTTPError(`Cannot update view type after creation`, 400)
+  }
+
   delete table.views[existingView.name]
   table.views[view.name] = view
   await db.put(table)

packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts

@@ -8,7 +8,7 @@ import {
 } from "@budibase/types"
 import { outputProcessing } from ".."
 import { generator, structures } from "@budibase/backend-core/tests"
-import { setEnv as setCoreEnv } from "@budibase/backend-core"
+import { features } from "@budibase/backend-core"
 import * as bbReferenceProcessor from "../bbReferenceProcessor"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 
@@ -21,7 +21,7 @@ jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
 
 describe("rowProcessor - outputProcessing", () => {
   const config = new TestConfiguration()
-  let cleanupEnv: () => void = () => {}
+  let cleanupFlags: () => void = () => {}
 
   beforeAll(async () => {
     await config.init()
@@ -33,11 +33,11 @@ describe("rowProcessor - outputProcessing", () => {
 
   beforeEach(() => {
     jest.resetAllMocks()
-    cleanupEnv = setCoreEnv({ TENANT_FEATURE_FLAGS: "*SQS" })
+    cleanupFlags = features.testutils.setFeatureFlags("*", { SQS: true })
   })
 
   afterEach(() => {
-    cleanupEnv()
+    cleanupFlags()
   })
 
   const processOutputBBReferenceMock =

packages/shared-core/src/helpers/views.ts

@@ -3,6 +3,7 @@ import {
   ViewCalculationFieldMetadata,
   ViewFieldMetadata,
   ViewV2,
+  ViewV2Type,
 } from "@budibase/types"
 import { pickBy } from "lodash"
 
@@ -21,6 +22,10 @@ export function isBasicViewField(
 type UnsavedViewV2 = Omit<ViewV2, "id" | "version">
 
 export function isCalculationView(view: UnsavedViewV2) {
+  return view.type === ViewV2Type.CALCULATION
+}
+
+export function hasCalculationFields(view: UnsavedViewV2) {
   return Object.values(view.schema || {}).some(isCalculationField)
 }
 

packages/types/src/api/account/accounts.ts

@@ -12,7 +12,6 @@ export interface CreateAccountRequest {
   name?: string
   password: string
   provider?: AccountSSOProvider
-  thirdPartyProfile: object
 }
 
 export interface SearchAccountsRequest {

packages/types/src/documents/account/account.ts

@@ -98,8 +98,6 @@ export interface AccountSSO {
   provider: AccountSSOProvider
   providerType: AccountSSOProviderType
   oauth2?: OAuthTokens
-  pictureUrl?: string
-  thirdPartyProfile: any // TODO: define what the google profile looks like
 }
 
 export type SSOAccount = (Account | CloudAccount) & AccountSSO

packages/types/src/documents/app/view.ts

@@ -79,10 +79,15 @@ export enum CalculationType {
   MAX = "max",
 }
 
+export enum ViewV2Type {
+  CALCULATION = "calculation",
+}
+
 export interface ViewV2 {
   version: 2
   id: string
   name: string
+  type?: ViewV2Type
   primaryDisplay?: string
   tableId: string
   query?: LegacyFilter[] | SearchFilters
@@ -94,7 +99,6 @@ export interface ViewV2 {
     type?: SortType
   }
   schema?: ViewV2Schema
-  uiMetadata?: Record<string, any>
 }
 
 export type ViewV2Schema = Record<string, ViewFieldMetadata>

packages/types/src/documents/global/user.ts

@@ -21,7 +21,6 @@ export interface UserSSO {
   provider: string // the individual provider e.g. Okta, Auth0, Google
   providerType: SSOProviderType
   oauth2?: OAuth2
-  thirdPartyProfile?: SSOProfileJson
   profile?: {
     displayName?: string
     name?: {
@@ -45,7 +44,6 @@ export interface User extends Document {
   userId?: string
   firstName?: string
   lastName?: string
-  pictureUrl?: string
   forceResetPassword?: boolean
   roles: UserRoles
   builder?: {

packages/worker/src/api/routes/global/tests/auditLogs.spec.ts

@@ -1,5 +1,5 @@
 import { mocks, structures } from "@budibase/backend-core/tests"
-import { context, events, setEnv as setCoreEnv } from "@budibase/backend-core"
+import { context, events, features } from "@budibase/backend-core"
 import { Event, IdentityType } from "@budibase/types"
 import { TestConfiguration } from "../../../../tests"
 
@@ -17,11 +17,9 @@ describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => {
   let envCleanup: (() => void) | undefined
 
   beforeAll(async () => {
-    if (method === "lucene") {
+    envCleanup = features.testutils.setFeatureFlags("*", {
-      envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" })
+      SQS: method === "sql",
-    } else if (method === "sql") {
+    })
-      envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" })
-    }
     await config.beforeAll()
   })