Merge branch 'master' into type-portal-org-store

.github/workflows/readme-openapi.yml

@@ -19,5 +19,8 @@ jobs:
           cache: yarn
       - run: yarn --frozen-lockfile
 
+      - name: Install OpenAPI pkg
+        run: yarn global add openapi
+
       - name: update specs
         run: cd packages/server && yarn specs && openapi specs/openapi.yaml --key=${{ secrets.README_API_KEY }} --id=6728a74f5918b50036c61841

examples/nextjs-api-sales/package.json

@@ -10,7 +10,7 @@
   },
   "dependencies": {
     "bulma": "^0.9.3",
-    "next": "14.2.15",
+    "next": "14.2.21",
     "node-fetch": "^3.2.10",
     "sass": "^1.52.3",
     "react": "17.0.2",

examples/nextjs-api-sales/yarn.lock

@@ -46,10 +46,10 @@
   resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45"
   integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==
 
-"@next/env@14.2.15":
+"@next/env@14.2.21":
-  version "14.2.15"
+  version "14.2.21"
-  resolved "https://registry.yarnpkg.com/@next/env/-/env-14.2.15.tgz#06d984e37e670d93ddd6790af1844aeb935f332f"
+  resolved "https://registry.yarnpkg.com/@next/env/-/env-14.2.21.tgz#09ff0813d29c596397e141205d4f5fd5c236bdd0"
-  integrity sha512-S1qaj25Wru2dUpcIZMjxeMVSwkt8BK4dmWHHiBuRstcIyOsMapqT4A4jSB6onvqeygkSSmOkyny9VVx8JIGamQ==
+  integrity sha512-lXcwcJd5oR01tggjWJ6SrNNYFGuOOMB9c251wUNkjCpkoXOPkDeF/15c3mnVlBqrW4JJXb2kVxDFhC4GduJt2A==
 
 "@next/eslint-plugin-next@12.1.0":
   version "12.1.0"
@@ -58,50 +58,50 @@
   dependencies:
     glob "7.1.7"
 
-"@next/swc-darwin-arm64@14.2.15":
+"@next/swc-darwin-arm64@14.2.21":
-  version "14.2.15"
+  version "14.2.21"
-  resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.15.tgz#6386d585f39a1c490c60b72b1f76612ba4434347"
+  resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.21.tgz#32a31992aace1440981df9cf7cb3af7845d94fec"
-  integrity sha512-Rvh7KU9hOUBnZ9TJ28n2Oa7dD9cvDBKua9IKx7cfQQ0GoYUwg9ig31O2oMwH3wm+pE3IkAQ67ZobPfEgurPZIA==
+  integrity sha512-HwEjcKsXtvszXz5q5Z7wCtrHeTTDSTgAbocz45PHMUjU3fBYInfvhR+ZhavDRUYLonm53aHZbB09QtJVJj8T7g==
 
-"@next/swc-darwin-x64@14.2.15":
+"@next/swc-darwin-x64@14.2.21":
-  version "14.2.15"
+  version "14.2.21"
-  resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.15.tgz#b7baeedc6a28f7545ad2bc55adbab25f7b45cb89"
+  resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.21.tgz#5ab4b3f6685b6b52f810d0f5cf6e471480ddffdb"
-  integrity sha512-5TGyjFcf8ampZP3e+FyCax5zFVHi+Oe7sZyaKOngsqyaNEpOgkKB3sqmymkZfowy3ufGA/tUgDPPxpQx931lHg==
+  integrity sha512-TSAA2ROgNzm4FhKbTbyJOBrsREOMVdDIltZ6aZiKvCi/v0UwFmwigBGeqXDA97TFMpR3LNNpw52CbVelkoQBxA==
 
-"@next/swc-linux-arm64-gnu@14.2.15":
+"@next/swc-linux-arm64-gnu@14.2.21":
-  version "14.2.15"
+  version "14.2.21"
-  resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.15.tgz#fa13c59d3222f70fb4cb3544ac750db2c6e34d02"
+  resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.21.tgz#8a0e1fa887aef19ca218af2af515d0a5ee67ba3f"
-  integrity sha512-3Bwv4oc08ONiQ3FiOLKT72Q+ndEMyLNsc/D3qnLMbtUYTQAmkx9E/JRu0DBpHxNddBmNT5hxz1mYBphJ3mfrrw==
+  integrity sha512-0Dqjn0pEUz3JG+AImpnMMW/m8hRtl1GQCNbO66V1yp6RswSTiKmnHf3pTX6xMdJYSemf3O4Q9ykiL0jymu0TuA==
 
-"@next/swc-linux-arm64-musl@14.2.15":
+"@next/swc-linux-arm64-musl@14.2.21":
-  version "14.2.15"
+  version "14.2.21"
-  resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.15.tgz#30e45b71831d9a6d6d18d7ac7d611a8d646a17f9"
+  resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.21.tgz#ddad844406b42fa8965fe11250abc85c1fe0fd05"
-  integrity sha512-k5xf/tg1FBv/M4CMd8S+JL3uV9BnnRmoe7F+GWC3DxkTCD9aewFRH1s5rJ1zkzDa+Do4zyN8qD0N8c84Hu96FQ==
+  integrity sha512-Ggfw5qnMXldscVntwnjfaQs5GbBbjioV4B4loP+bjqNEb42fzZlAaK+ldL0jm2CTJga9LynBMhekNfV8W4+HBw==
 
-"@next/swc-linux-x64-gnu@14.2.15":
+"@next/swc-linux-x64-gnu@14.2.21":
-  version "14.2.15"
+  version "14.2.21"
-  resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.15.tgz#5065db17fc86f935ad117483f21f812dc1b39254"
+  resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.21.tgz#db55fd666f9ba27718f65caa54b622a912cdd16b"
-  integrity sha512-kE6q38hbrRbKEkkVn62reLXhThLRh6/TvgSP56GkFNhU22TbIrQDEMrO7j0IcQHcew2wfykq8lZyHFabz0oBrA==
+  integrity sha512-uokj0lubN1WoSa5KKdThVPRffGyiWlm/vCc/cMkWOQHw69Qt0X1o3b2PyLLx8ANqlefILZh1EdfLRz9gVpG6tg==
 
-"@next/swc-linux-x64-musl@14.2.15":
+"@next/swc-linux-x64-musl@14.2.21":
-  version "14.2.15"
+  version "14.2.21"
-  resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.15.tgz#3c4a4568d8be7373a820f7576cf33388b5dab47e"
+  resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.21.tgz#dddb850353624efcd58c4c4e30ad8a1aab379642"
-  integrity sha512-PZ5YE9ouy/IdO7QVJeIcyLn/Rc4ml9M2G4y3kCM9MNf1YKvFY4heg3pVa/jQbMro+tP6yc4G2o9LjAz1zxD7tQ==
+  integrity sha512-iAEBPzWNbciah4+0yI4s7Pce6BIoxTQ0AGCkxn/UBuzJFkYyJt71MadYQkjPqCQCJAFQ26sYh7MOKdU+VQFgPg==
 
-"@next/swc-win32-arm64-msvc@14.2.15":
+"@next/swc-win32-arm64-msvc@14.2.21":
-  version "14.2.15"
+  version "14.2.21"
-  resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.15.tgz#fb812cc4ca0042868e32a6a021da91943bb08b98"
+  resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.21.tgz#290012ee57b196d3d2d04853e6bf0179cae9fbaf"
-  integrity sha512-2raR16703kBvYEQD9HNLyb0/394yfqzmIeyp2nDzcPV4yPjqNUG3ohX6jX00WryXz6s1FXpVhsCo3i+g4RUX+g==
+  integrity sha512-plykgB3vL2hB4Z32W3ktsfqyuyGAPxqwiyrAi2Mr8LlEUhNn9VgkiAl5hODSBpzIfWweX3er1f5uNpGDygfQVQ==
 
-"@next/swc-win32-ia32-msvc@14.2.15":
+"@next/swc-win32-ia32-msvc@14.2.21":
-  version "14.2.15"
+  version "14.2.21"
-  resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.15.tgz#ec26e6169354f8ced240c1427be7fd485c5df898"
+  resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.21.tgz#c959135a78cab18cca588d11d1e33bcf199590d4"
-  integrity sha512-fyTE8cklgkyR1p03kJa5zXEaZ9El+kDNM5A+66+8evQS5e/6v0Gk28LqA0Jet8gKSOyP+OTm/tJHzMlGdQerdQ==
+  integrity sha512-w5bacz4Vxqrh06BjWgua3Yf7EMDb8iMcVhNrNx8KnJXt8t+Uu0Zg4JHLDL/T7DkTCEEfKXO/Er1fcfWxn2xfPA==
 
-"@next/swc-win32-x64-msvc@14.2.15":
+"@next/swc-win32-x64-msvc@14.2.21":
-  version "14.2.15"
+  version "14.2.21"
-  resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.15.tgz#18d68697002b282006771f8d92d79ade9efd35c4"
+  resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.21.tgz#21ff892286555b90538a7d1b505ea21a005d6ead"
-  integrity sha512-SzqGbsLsP9OwKNUG9nekShTwhj6JSB9ZLMWQ8g1gG6hdE5gQLncbnbymrwy2yVmH9nikSLYRYxYMFu78Ggp7/g==
+  integrity sha512-sT6+llIkzpsexGYZq8cjjthRyRGe5cJVhqh12FmlbxHqna6zsDDK8UNaV7g41T6atFHCJUPeLb3uyAwrBwy0NA==
 
 "@nodelib/fs.scandir@2.1.5":
   version "2.1.5"
@@ -1253,12 +1253,12 @@ natural-compare@^1.4.0:
   resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
   integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=
 
-next@14.2.15:
+next@14.2.21:
-  version "14.2.15"
+  version "14.2.21"
-  resolved "https://registry.yarnpkg.com/next/-/next-14.2.15.tgz#348e5603e22649775d19c785c09a89c9acb5189a"
+  resolved "https://registry.yarnpkg.com/next/-/next-14.2.21.tgz#f6da9e2abba1a0e4ca7a5273825daf06632554ba"
-  integrity sha512-h9ctmOokpoDphRvMGnwOJAedT6zKhwqyZML9mDtspgf4Rh3Pn7UTYKqePNoDvhsWBAO5GoPNYshnAUGIazVGmw==
+  integrity sha512-rZmLwucLHr3/zfDMYbJXbw0ZeoBpirxkXuvsJbk7UPorvPYZhP7vq7aHbKnU7dQNCYIimRrbB2pp3xmf+wsYUg==
   dependencies:
-    "@next/env" "14.2.15"
+    "@next/env" "14.2.21"
     "@swc/helpers" "0.5.5"
     busboy "1.6.0"
     caniuse-lite "^1.0.30001579"
@@ -1266,15 +1266,15 @@ next@14.2.15:
     postcss "8.4.31"
     styled-jsx "5.1.1"
   optionalDependencies:
-    "@next/swc-darwin-arm64" "14.2.15"
+    "@next/swc-darwin-arm64" "14.2.21"
-    "@next/swc-darwin-x64" "14.2.15"
+    "@next/swc-darwin-x64" "14.2.21"
-    "@next/swc-linux-arm64-gnu" "14.2.15"
+    "@next/swc-linux-arm64-gnu" "14.2.21"
-    "@next/swc-linux-arm64-musl" "14.2.15"
+    "@next/swc-linux-arm64-musl" "14.2.21"
-    "@next/swc-linux-x64-gnu" "14.2.15"
+    "@next/swc-linux-x64-gnu" "14.2.21"
-    "@next/swc-linux-x64-musl" "14.2.15"
+    "@next/swc-linux-x64-musl" "14.2.21"
-    "@next/swc-win32-arm64-msvc" "14.2.15"
+    "@next/swc-win32-arm64-msvc" "14.2.21"
-    "@next/swc-win32-ia32-msvc" "14.2.15"
+    "@next/swc-win32-ia32-msvc" "14.2.21"
-    "@next/swc-win32-x64-msvc" "14.2.15"
+    "@next/swc-win32-x64-msvc" "14.2.21"
 
 node-domexception@^1.0.0:
   version "1.0.0"

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.29",
+  "version": "3.2.37",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/src/context/mainContext.ts

@@ -385,17 +385,17 @@ export function getCurrentContext(): ContextMap | undefined {
   }
 }
 
-export function getFeatureFlags<T extends Record<string, any>>(
+export function getFeatureFlags(
   key: string
-): T | undefined {
+): Record<string, boolean> | undefined {
   const context = getCurrentContext()
   if (!context) {
     return undefined
   }
-  return context.featureFlagCache?.[key] as T
+  return context.featureFlagCache?.[key]
 }
 
-export function setFeatureFlags(key: string, value: Record<string, any>) {
+export function setFeatureFlags(key: string, value: Record<string, boolean>) {
   const context = getCurrentContext()
   if (!context) {
     return

packages/backend-core/src/context/types.ts

@@ -20,7 +20,7 @@ export type ContextMap = {
     clients: Record<string, GoogleSpreadsheet>
   }
   featureFlagCache?: {
-    [key: string]: Record<string, any>
+    [key: string]: Record<string, boolean>
   }
   viewToTableCache?: Record<string, Table>
 }

packages/backend-core/src/features/features.ts

@@ -2,9 +2,10 @@ import env from "../environment"
 import * as crypto from "crypto"
 import * as context from "../context"
 import { PostHog, PostHogOptions } from "posthog-node"
-import { FeatureFlag } from "@budibase/types"
 import tracer from "dd-trace"
 import { Duration } from "../utils"
+import { cloneDeep } from "lodash"
+import { FeatureFlagDefaults } from "@budibase/types"
 
 let posthog: PostHog | undefined
 export function init(opts?: PostHogOptions) {
@@ -30,74 +31,6 @@ export function shutdown() {
   posthog?.shutdown()
 }
 
-export abstract class Flag<T> {
-  static boolean(defaultValue: boolean): Flag<boolean> {
-    return new BooleanFlag(defaultValue)
-  }
-
-  static string(defaultValue: string): Flag<string> {
-    return new StringFlag(defaultValue)
-  }
-
-  static number(defaultValue: number): Flag<number> {
-    return new NumberFlag(defaultValue)
-  }
-
-  protected constructor(public defaultValue: T) {}
-
-  abstract parse(value: any): T
-}
-
-type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
-
-export type FlagValues<T> = {
-  [K in keyof T]: UnwrapFlag<T[K]>
-}
-
-type KeysOfType<T, U> = {
-  [K in keyof T]: T[K] extends Flag<U> ? K : never
-}[keyof T]
-
-class BooleanFlag extends Flag<boolean> {
-  parse(value: any) {
-    if (typeof value === "string") {
-      return ["true", "t", "1"].includes(value.toLowerCase())
-    }
-
-    if (typeof value === "boolean") {
-      return value
-    }
-
-    throw new Error(`could not parse value "${value}" as boolean`)
-  }
-}
-
-class StringFlag extends Flag<string> {
-  parse(value: any) {
-    if (typeof value === "string") {
-      return value
-    }
-    throw new Error(`could not parse value "${value}" as string`)
-  }
-}
-
-class NumberFlag extends Flag<number> {
-  parse(value: any) {
-    if (typeof value === "number") {
-      return value
-    }
-
-    if (typeof value === "string") {
-      const parsed = parseFloat(value)
-      if (!isNaN(parsed)) {
-        return parsed
-      }
-    }
-
-    throw new Error(`could not parse value "${value}" as number`)
-  }
-}
-
 export interface EnvFlagEntry {
   tenantId: string
   key: string
@@ -120,7 +53,7 @@ export function parseEnvFlags(flags: string): EnvFlagEntry[] {
   return result
 }
 
-export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
+export class FlagSet<T extends { [name: string]: boolean }> {
   // This is used to safely cache flags sets in the current request context.
   // Because multiple sets could theoretically exist, we don't want the cache of
   // one to leak into another.
@@ -130,34 +63,25 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
     this.setId = crypto.randomUUID()
   }
 
-  defaults(): FlagValues<T> {
+  defaults(): T {
-    return Object.keys(this.flagSchema).reduce((acc, key) => {
+    return cloneDeep(this.flagSchema)
-      const typedKey = key as keyof T
-      acc[typedKey] = this.flagSchema[key].defaultValue
-      return acc
-    }, {} as FlagValues<T>)
   }
 
   isFlagName(name: string | number | symbol): name is keyof T {
     return this.flagSchema[name as keyof T] !== undefined
   }
 
-  async get<K extends keyof T>(key: K): Promise<FlagValues<T>[K]> {
+  async isEnabled<K extends keyof T>(key: K): Promise<T[K]> {
     const flags = await this.fetch()
     return flags[key]
   }
 
-  async isEnabled<K extends KeysOfType<T, boolean>>(key: K): Promise<boolean> {
+  async fetch(): Promise<T> {
-    const flags = await this.fetch()
-    return flags[key]
-  }
-
-  async fetch(): Promise<FlagValues<T>> {
     return await tracer.trace("features.fetch", async span => {
-      const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
+      const cachedFlags = context.getFeatureFlags(this.setId)
       if (cachedFlags) {
         span?.addTags({ fromCache: true })
-        return cachedFlags
+        return cachedFlags as T
       }
 
       const tags: Record<string, any> = {}
@@ -189,7 +113,7 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
 
         // @ts-expect-error - TS does not like you writing into a generic type,
         // but we know that it's okay in this case because it's just an object.
-        flagValues[key as keyof FlagValues] = value
+        flagValues[key as keyof T] = value
         tags[`flags.${key}.source`] = "environment"
       }
 
@@ -217,11 +141,11 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
         tags[`readFromPostHog`] = true
 
         const personProperties: Record<string, string> = { tenantId }
-        const posthogFlags = await posthog.getAllFlagsAndPayloads(userId, {
+        const posthogFlags = await posthog.getAllFlags(userId, {
           personProperties,
         })
 
-        for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
+        for (const [name, value] of Object.entries(posthogFlags)) {
           if (!this.isFlagName(name)) {
             // We don't want an unexpected PostHog flag to break the app, so we
             // just log it and continue.
@@ -229,19 +153,20 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
             continue
           }
 
+          if (typeof value !== "boolean") {
+            console.warn(`Invalid value for posthog flag "${name}": ${value}`)
+            continue
+          }
+
           if (flagValues[name] === true || specificallySetFalse.has(name)) {
             // If the flag is already set to through environment variables, we
             // don't want to override it back to false here.
             continue
           }
 
-          const payload = posthogFlags.featureFlagPayloads?.[name]
-          const flag = this.flagSchema[name]
           try {
-            // @ts-expect-error - TS does not like you writing into a generic
+            // @ts-expect-error - TS does not like you writing into a generic type.
-            // type, but we know that it's okay in this case because it's just
+            flagValues[name] = value
-            // an object.
-            flagValues[name] = flag.parse(payload || value)
             tags[`flags.${name}.source`] = "posthog"
           } catch (err) {
             // We don't want an invalid PostHog flag to break the app, so we just
@@ -262,18 +187,12 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
   }
 }
 
-// This is the primary source of truth for feature flags. If you want to add a
+export const flags = new FlagSet(FeatureFlagDefaults)
-// new flag, add it here and use the `fetch` and `get` functions to access it.
-// All of the machinery in this file is to make sure that flags have their
-// default values set correctly and their types flow through the system.
-const flagsConfig: Record<FeatureFlag, Flag<any>> = {
-  [FeatureFlag.DEFAULT_VALUES]: Flag.boolean(true),
-  [FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(true),
-  [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(true),
-  [FeatureFlag.BUDIBASE_AI]: Flag.boolean(true),
-  [FeatureFlag.USE_ZOD_VALIDATOR]: Flag.boolean(env.isDev()),
-}
-export const flags = new FlagSet(flagsConfig)
 
-type UnwrapPromise<T> = T extends Promise<infer U> ? U : T
+export async function isEnabled(flag: keyof typeof FeatureFlagDefaults) {
-export type FeatureFlags = UnwrapPromise<ReturnType<typeof flags.fetch>>
+  return await flags.isEnabled(flag)
+}
+
+export async function all() {
+  return await flags.fetch()
+}

packages/backend-core/src/features/tests/features.spec.ts

@@ -1,5 +1,5 @@
 import { IdentityContext, IdentityType } from "@budibase/types"
-import { Flag, FlagSet, FlagValues, init, shutdown } from "../"
+import { FlagSet, init, shutdown } from "../"
 import * as context from "../../context"
 import environment, { withEnv } from "../../environment"
 import nodeFetch from "node-fetch"
@@ -7,10 +7,8 @@ import nock from "nock"
 import * as crypto from "crypto"
 
 const schema = {
-  TEST_BOOLEAN: Flag.boolean(false),
+  TEST_BOOLEAN: false,
-  TEST_STRING: Flag.string("default value"),
+  TEST_BOOLEAN_DEFAULT_TRUE: true,
-  TEST_NUMBER: Flag.number(0),
-  TEST_BOOLEAN_DEFAULT_TRUE: Flag.boolean(true),
 }
 const flags = new FlagSet(schema)
 
@@ -19,7 +17,7 @@ interface TestCase {
   identity?: Partial<IdentityContext>
   environmentFlags?: string
   posthogFlags?: PostHogFlags
-  expected?: Partial<FlagValues<typeof schema>>
+  expected?: Partial<typeof schema>
   errorMessage?: string | RegExp
 }
 
@@ -83,22 +81,6 @@ describe("feature flags", () => {
       },
       expected: { TEST_BOOLEAN: true },
     },
-    {
-      it: "should be able to read string flags from PostHog",
-      posthogFlags: {
-        featureFlags: { TEST_STRING: true },
-        featureFlagPayloads: { TEST_STRING: "test" },
-      },
-      expected: { TEST_STRING: "test" },
-    },
-    {
-      it: "should be able to read numeric flags from PostHog",
-      posthogFlags: {
-        featureFlags: { TEST_NUMBER: true },
-        featureFlagPayloads: { TEST_NUMBER: "123" },
-      },
-      expected: { TEST_NUMBER: 123 },
-    },
     {
       it: "should not be able to override a negative environment flag from PostHog",
       environmentFlags: "default:!TEST_BOOLEAN",
@@ -177,7 +159,7 @@ describe("feature flags", () => {
             expect(values).toMatchObject(expected)
 
             for (const [key, expectedValue] of Object.entries(expected)) {
-              const value = await flags.get(key as keyof typeof schema)
+              const value = await flags.isEnabled(key as keyof typeof schema)
               expect(value).toBe(expectedValue)
             }
           } else {

packages/backend-core/src/features/tests/utils.ts

@@ -1,5 +1,6 @@
-import { FeatureFlags, parseEnvFlags } from ".."
+import { FeatureFlags } from "@budibase/types"
 import { setEnv } from "../../environment"
+import { parseEnvFlags } from "../features"
 
 function getCurrentFlags(): Record<string, Record<string, boolean>> {
   const result: Record<string, Record<string, boolean>> = {}

packages/bbui/src/helpers.d.ts

@@ -0,0 +1,3 @@
+declare module "./helpers" {
+  export const cloneDeep: <T>(obj: T) => T
+}

packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte

@@ -43,7 +43,6 @@
   export let showDataProviders = true
 
   const dispatch = createEventDispatcher()
-  const arrayTypes = ["attachment", "array"]
 
   let anchorRight, dropdownRight
   let drawer
@@ -116,8 +115,11 @@
       }
     })
   $: fields = bindings
-    .filter(x => arrayTypes.includes(x.fieldSchema?.type))
+    .filter(
-    .filter(x => x.fieldSchema?.tableId != null)
+      x =>
+        x.fieldSchema?.type === "attachment" ||
+        (x.fieldSchema?.type === "array" && x.tableId)
+    )
     .map(binding => {
       const { providerId, readableBinding, runtimeBinding } = binding
       const { name, type, tableId } = binding.fieldSchema

packages/builder/src/pages/builder/portal/users/users/[userId].svelte

@@ -87,6 +87,7 @@
   let popover
   let user, tenantOwner
   let loaded = false
+  let userFieldsToUpdate = {}
 
   $: internalGroups = $groups?.filter(g => !g?.scimInfo?.isSync)
 
@@ -164,40 +165,45 @@
     return label
   }
 
-  async function updateUserFirstName(evt) {
+  async function saveUser() {
     try {
-      await users.save({ ...user, firstName: evt.target.value })
+      await users.save({ ...user, ...userFieldsToUpdate })
+      userFieldsToUpdate = {}
       await fetchUser()
     } catch (error) {
       notifications.error("Error updating user")
     }
   }
 
+  async function updateUserFirstName(evt) {
+    userFieldsToUpdate.firstName = evt.target.value
+  }
+
   async function updateUserLastName(evt) {
-    try {
+    userFieldsToUpdate.lastName = evt.target.value
-      await users.save({ ...user, lastName: evt.target.value })
-      await fetchUser()
-    } catch (error) {
-      notifications.error("Error updating user")
-    }
   }
 
   async function updateUserRole({ detail }) {
+    let flags = {}
     if (detail === Constants.BudibaseRoles.Developer) {
-      toggleFlags({ admin: { global: false }, builder: { global: true } })
+      flags = { admin: { global: false }, builder: { global: true } }
     } else if (detail === Constants.BudibaseRoles.Admin) {
-      toggleFlags({ admin: { global: true }, builder: { global: true } })
+      flags = { admin: { global: true }, builder: { global: true } }
     } else if (detail === Constants.BudibaseRoles.AppUser) {
-      toggleFlags({ admin: { global: false }, builder: { global: false } })
+      flags = { admin: { global: false }, builder: { global: false } }
     } else if (detail === Constants.BudibaseRoles.Creator) {
-      toggleFlags({
+      flags = {
         admin: { global: false },
         builder: {
           global: false,
           creator: true,
           apps: user?.builder?.apps || [],
         },
-      })
+      }
+    }
+    userFieldsToUpdate = {
+      ...userFieldsToUpdate,
+      ...flags,
     }
   }
 
@@ -209,15 +215,6 @@
     tenantOwner = await users.getAccountHolder()
   }
 
-  async function toggleFlags(detail) {
-    try {
-      await users.save({ ...user, ...detail })
-      await fetchUser()
-    } catch (error) {
-      notifications.error("Error updating user")
-    }
-  }
-
   const addGroup = async groupId => {
     await groups.addUser(groupId, userId)
     await fetchUser()
@@ -296,7 +293,7 @@
           <Input
             disabled={readonly}
             value={user?.firstName}
-            on:blur={updateUserFirstName}
+            on:input={updateUserFirstName}
           />
         </div>
         <div class="field">
@@ -304,7 +301,7 @@
           <Input
             disabled={readonly}
             value={user?.lastName}
-            on:blur={updateUserLastName}
+            on:input={updateUserLastName}
           />
         </div>
         <!-- don't let a user remove the privileges that let them be here -->
@@ -325,6 +322,13 @@
         {/if}
       </div>
     </Layout>
+    <div>
+      <Button
+        cta
+        disabled={Object.keys(userFieldsToUpdate).length === 0}
+        on:click={saveUser}>Save</Button
+      >
+    </div>
 
     {#if $licensing.groupsEnabled}
       <!-- User groups -->

packages/builder/src/stores/builder/automations.ts

@@ -291,8 +291,8 @@ const automationActions = (store: AutomationStore) => ({
     let result: (AutomationStep | AutomationTrigger)[] = []
     pathWay.forEach(path => {
       const { stepIdx, branchIdx } = path
-      let last = result ? result[result.length - 1] : []
+      let last = result.length ? result[result.length - 1] : []
-      if (!result) {
+      if (!result.length) {
         // Preceeding steps.
         result = steps.slice(0, stepIdx + 1)
         return

packages/builder/src/stores/builder/flags.js

@@ -1,27 +0,0 @@
-import { writable } from "svelte/store"
-import { API } from "@/api"
-
-export function createFlagsStore() {
-  const { subscribe, set } = writable({})
-
-  const actions = {
-    fetch: async () => {
-      const flags = await API.getFlags()
-      set(flags)
-    },
-    updateFlag: async (flag, value) => {
-      await API.updateFlag(flag, value)
-      await actions.fetch()
-    },
-    toggleUiFeature: async feature => {
-      await API.toggleUiFeature(feature)
-    },
-  }
-
-  return {
-    subscribe,
-    ...actions,
-  }
-}
-
-export const flags = createFlagsStore()

packages/builder/src/stores/builder/flags.ts

@@ -0,0 +1,40 @@
+import { API } from "@/api"
+import { GetUserFlagsResponse } from "@budibase/types"
+import { BudiStore } from "../BudiStore"
+
+interface FlagsState {
+  flags: GetUserFlagsResponse
+}
+
+const INITIAL_FLAGS_STATE: FlagsState = {
+  flags: {},
+}
+
+export class FlagsStore extends BudiStore<FlagsState> {
+  constructor() {
+    super(INITIAL_FLAGS_STATE)
+
+    this.fetch = this.fetch.bind(this)
+    this.updateFlag = this.updateFlag.bind(this)
+    this.toggleUiFeature = this.toggleUiFeature.bind(this)
+  }
+
+  async fetch() {
+    const flags = await API.getFlags()
+    this.update(state => ({
+      ...state,
+      flags,
+    }))
+  }
+
+  async updateFlag(flag: string, value: any) {
+    await API.updateFlag(flag, value)
+    await this.fetch()
+  }
+
+  async toggleUiFeature(feature: string) {
+    await API.toggleUiFeature(feature)
+  }
+}
+
+export const flags = new FlagsStore()

packages/builder/src/stores/builder/hover.ts

@@ -2,19 +2,24 @@ import { get } from "svelte/store"
 import { BudiStore } from "../BudiStore"
 import { previewStore } from "@/stores/builder"
 
+interface BuilderHoverStore {
+  hoverTimeout?: NodeJS.Timeout
+  componentId: string | null
+}
+
 export const INITIAL_HOVER_STATE = {
   componentId: null,
 }
 
-export class HoverStore extends BudiStore {
+export class HoverStore extends BudiStore<BuilderHoverStore> {
-  hoverTimeout
+  hoverTimeout?: NodeJS.Timeout
 
   constructor() {
     super({ ...INITIAL_HOVER_STATE })
     this.hover = this.hover.bind(this)
   }
 
-  hover(componentId, notifyClient = true) {
+  hover(componentId: string, notifyClient = true) {
     clearTimeout(this.hoverTimeout)
     if (componentId) {
       this.processHover(componentId, notifyClient)
@@ -25,7 +30,7 @@ export class HoverStore extends BudiStore {
     }
   }
 
-  processHover(componentId, notifyClient) {
+  processHover(componentId: string, notifyClient?: boolean) {
     if (componentId === get(this.store).componentId) {
       return
     }

packages/builder/src/stores/builder/layouts.ts

@@ -2,13 +2,19 @@ import { derived, get } from "svelte/store"
 import { componentStore } from "@/stores/builder"
 import { API } from "@/api"
 import { BudiStore } from "../BudiStore"
+import { Layout } from "@budibase/types"
 
-export const INITIAL_LAYOUT_STATE = {
+interface LayoutState {
+  layouts: Layout[]
+  selectedLayoutId: string | null
+}
+
+export const INITIAL_LAYOUT_STATE: LayoutState = {
   layouts: [],
   selectedLayoutId: null,
 }
 
-export class LayoutStore extends BudiStore {
+export class LayoutStore extends BudiStore<LayoutState> {
   constructor() {
     super(INITIAL_LAYOUT_STATE)
 
@@ -22,14 +28,14 @@ export class LayoutStore extends BudiStore {
     this.store.set({ ...INITIAL_LAYOUT_STATE })
   }
 
-  syncAppLayouts(pkg) {
+  syncAppLayouts(pkg: { layouts: Layout[] }) {
     this.update(state => ({
       ...state,
       layouts: [...pkg.layouts],
     }))
   }
 
-  select(layoutId) {
+  select(layoutId: string) {
     // Check this layout exists
     const state = get(this.store)
     const componentState = get(componentStore)
@@ -48,15 +54,15 @@ export class LayoutStore extends BudiStore {
 
     // Select new layout
     this.update(state => {
-      state.selectedLayoutId = layout._id
+      state.selectedLayoutId = layout._id!
       return state
     })
 
     componentStore.select(layout.props?._id)
   }
 
-  async deleteLayout(layout) {
+  async deleteLayout(layout: Layout) {
-    if (!layout?._id) {
+    if (!layout?._id || !layout?._rev) {
       return
     }
     await API.deleteLayout(layout._id, layout._rev)

packages/builder/src/stores/builder/navigation.ts

@@ -2,27 +2,22 @@ import { get } from "svelte/store"
 import { API } from "@/api"
 import { appStore } from "@/stores/builder"
 import { BudiStore } from "../BudiStore"
+import { AppNavigation, AppNavigationLink, UIObject } from "@budibase/types"
+
+interface BuilderNavigationStore extends AppNavigation {}
 
 export const INITIAL_NAVIGATION_STATE = {
   navigation: "Top",
   links: [],
-  title: null,
-  sticky: null,
-  hideLogo: null,
-  logoUrl: null,
-  hideTitle: null,
   textAlign: "Left",
-  navBackground: null,
-  navWidth: null,
-  navTextColor: null,
 }
 
-export class NavigationStore extends BudiStore {
+export class NavigationStore extends BudiStore<BuilderNavigationStore> {
   constructor() {
     super(INITIAL_NAVIGATION_STATE)
   }
 
-  syncAppNavigation(nav) {
+  syncAppNavigation(nav: AppNavigation) {
     this.update(state => ({
       ...state,
       ...nav,
@@ -33,15 +28,17 @@ export class NavigationStore extends BudiStore {
     this.store.set({ ...INITIAL_NAVIGATION_STATE })
   }
 
-  async save(navigation) {
+  async save(navigation: AppNavigation) {
     const appId = get(appStore).appId
     const app = await API.saveAppMetadata(appId, { navigation })
+    if (app.navigation) {
       this.syncAppNavigation(app.navigation)
     }
+  }
 
-  async saveLink(url, title, roleId) {
+  async saveLink(url: string, title: string, roleId: string) {
     const navigation = get(this.store)
-    let links = [...(navigation?.links ?? [])]
+    let links: AppNavigationLink[] = [...(navigation?.links ?? [])]
 
     // Skip if we have an identical link
     if (links.find(link => link.url === url && link.text === title)) {
@@ -60,7 +57,7 @@ export class NavigationStore extends BudiStore {
     })
   }
 
-  async deleteLink(urls) {
+  async deleteLink(urls: string[] | string) {
     const navigation = get(this.store)
     let links = navigation?.links
     if (!links?.length) {
@@ -86,7 +83,7 @@ export class NavigationStore extends BudiStore {
     })
   }
 
-  syncMetadata(metadata) {
+  syncMetadata(metadata: UIObject) {
     const { navigation } = metadata
     this.syncAppNavigation(navigation)
   }

packages/builder/src/stores/builder/preview.js

@@ -1,80 +0,0 @@
-import { writable, get } from "svelte/store"
-
-const INITIAL_PREVIEW_STATE = {
-  previewDevice: "desktop",
-  previewEventHandler: null,
-  showPreview: false,
-  selectedComponentContext: null,
-}
-
-export const createPreviewStore = () => {
-  const store = writable({
-    ...INITIAL_PREVIEW_STATE,
-  })
-
-  const setDevice = device => {
-    store.update(state => {
-      state.previewDevice = device
-      return state
-    })
-  }
-
-  // Potential evt names "eject-block", "dragging-new-component"
-  const sendEvent = (name, payload) => {
-    const { previewEventHandler } = get(store)
-    previewEventHandler?.(name, payload)
-  }
-
-  const registerEventHandler = handler => {
-    store.update(state => {
-      state.previewEventHandler = handler
-      return state
-    })
-  }
-
-  const startDrag = component => {
-    sendEvent("dragging-new-component", {
-      dragging: true,
-      component,
-    })
-  }
-
-  const stopDrag = () => {
-    sendEvent("dragging-new-component", {
-      dragging: false,
-    })
-  }
-
-  //load preview?
-  const showPreview = isVisible => {
-    store.update(state => {
-      state.showPreview = isVisible
-      return state
-    })
-  }
-
-  const setSelectedComponentContext = context => {
-    store.update(state => {
-      state.selectedComponentContext = context
-      return state
-    })
-  }
-
-  const requestComponentContext = () => {
-    sendEvent("request-context")
-  }
-
-  return {
-    subscribe: store.subscribe,
-    setDevice,
-    sendEvent, //may not be required
-    registerEventHandler,
-    startDrag,
-    stopDrag,
-    showPreview,
-    setSelectedComponentContext,
-    requestComponentContext,
-  }
-}
-
-export const previewStore = createPreviewStore()

packages/builder/src/stores/builder/preview.ts

@@ -0,0 +1,90 @@
+import { get } from "svelte/store"
+import { BudiStore } from "../BudiStore"
+
+type PreviewDevice = "desktop" | "tablet" | "mobile"
+type PreviewEventHandler = (name: string, payload?: any) => void
+type ComponentContext = Record<string, any>
+
+interface PreviewState {
+  previewDevice: PreviewDevice
+  previewEventHandler: PreviewEventHandler | null
+  showPreview: boolean
+  selectedComponentContext: ComponentContext | null
+}
+
+const INITIAL_PREVIEW_STATE: PreviewState = {
+  previewDevice: "desktop",
+  previewEventHandler: null,
+  showPreview: false,
+  selectedComponentContext: null,
+}
+
+export class PreviewStore extends BudiStore<PreviewState> {
+  constructor() {
+    super(INITIAL_PREVIEW_STATE)
+
+    this.setDevice = this.setDevice.bind(this)
+    this.sendEvent = this.sendEvent.bind(this)
+    this.registerEventHandler = this.registerEventHandler.bind(this)
+    this.startDrag = this.startDrag.bind(this)
+    this.stopDrag = this.stopDrag.bind(this)
+    this.showPreview = this.showPreview.bind(this)
+    this.setSelectedComponentContext =
+      this.setSelectedComponentContext.bind(this)
+    this.requestComponentContext = this.requestComponentContext.bind(this)
+  }
+
+  setDevice(device: PreviewDevice) {
+    this.update(state => ({
+      ...state,
+      previewDevice: device,
+    }))
+  }
+
+  // Potential evt names "eject-block", "dragging-new-component"
+  sendEvent(name: string, payload?: any) {
+    const { previewEventHandler } = get(this.store)
+    previewEventHandler?.(name, payload)
+  }
+
+  registerEventHandler(handler: PreviewEventHandler) {
+    this.update(state => ({
+      ...state,
+      previewEventHandler: handler,
+    }))
+  }
+
+  startDrag(component: any) {
+    this.sendEvent("dragging-new-component", {
+      dragging: true,
+      component,
+    })
+  }
+
+  stopDrag() {
+    this.sendEvent("dragging-new-component", {
+      dragging: false,
+    })
+  }
+
+  //load preview?
+  showPreview(isVisible: boolean) {
+    this.update(state => ({
+      ...state,
+      showPreview: isVisible,
+    }))
+  }
+
+  setSelectedComponentContext(context: ComponentContext) {
+    this.update(state => ({
+      ...state,
+      selectedComponentContext: context,
+    }))
+  }
+
+  requestComponentContext() {
+    this.sendEvent("request-context")
+  }
+}
+
+export const previewStore = new PreviewStore()

packages/builder/src/stores/builder/published.ts

@@ -1,13 +1,16 @@
 import { appStore } from "./app"
 import { appsStore } from "@/stores/portal/apps"
 import { deploymentStore } from "./deployments"
-import { derived } from "svelte/store"
+import { derived, type Readable } from "svelte/store"
+import { DeploymentProgressResponse, DeploymentStatus } from "@budibase/types"
 
-export const appPublished = derived(
+export const appPublished: Readable<boolean> = derived(
   [appStore, appsStore, deploymentStore],
   ([$appStore, $appsStore, $deploymentStore]) => {
     const app = $appsStore.apps.find(app => app.devId === $appStore.appId)
-    const deployments = $deploymentStore.filter(x => x.status === "SUCCESS")
+    const deployments = $deploymentStore.filter(
+      (x: DeploymentProgressResponse) => x.status === DeploymentStatus.SUCCESS
+    )
     return app?.status === "published" && deployments.length > 0
   }
 )

packages/builder/src/stores/builder/roles.js

@@ -1,88 +0,0 @@
-import { derived, writable, get } from "svelte/store"
-import { API } from "@/api"
-import { RoleUtils } from "@budibase/frontend-core"
-
-export function createRolesStore() {
-  const store = writable([])
-  const enriched = derived(store, $store => {
-    return $store.map(role => ({
-      ...role,
-
-      // Ensure we have new metadata for all roles
-      uiMetadata: {
-        displayName: role.uiMetadata?.displayName || role.name,
-        color:
-          role.uiMetadata?.color || "var(--spectrum-global-color-magenta-400)",
-        description: role.uiMetadata?.description || "Custom role",
-      },
-    }))
-  })
-
-  function setRoles(roles) {
-    store.set(
-      roles.sort((a, b) => {
-        const priorityA = RoleUtils.getRolePriority(a._id)
-        const priorityB = RoleUtils.getRolePriority(b._id)
-        if (priorityA !== priorityB) {
-          return priorityA > priorityB ? -1 : 1
-        }
-        const nameA = a.uiMetadata?.displayName || a.name
-        const nameB = b.uiMetadata?.displayName || b.name
-        return nameA < nameB ? -1 : 1
-      })
-    )
-  }
-
-  const actions = {
-    fetch: async () => {
-      const roles = await API.getRoles()
-      setRoles(roles)
-    },
-    fetchByAppId: async appId => {
-      const { roles } = await API.getRolesForApp(appId)
-      setRoles(roles)
-    },
-    delete: async role => {
-      await API.deleteRole(role._id, role._rev)
-      await actions.fetch()
-    },
-    save: async role => {
-      const savedRole = await API.saveRole(role)
-      await actions.fetch()
-      return savedRole
-    },
-    replace: (roleId, role) => {
-      // Handles external updates of roles
-      if (!roleId) {
-        return
-      }
-
-      // Handle deletion
-      if (!role) {
-        store.update(state => state.filter(x => x._id !== roleId))
-        return
-      }
-
-      // Add new role
-      const index = get(store).findIndex(x => x._id === role._id)
-      if (index === -1) {
-        store.update(state => [...state, role])
-      }
-
-      // Update existing role
-      else if (role) {
-        store.update(state => {
-          state[index] = role
-          return [...state]
-        })
-      }
-    },
-  }
-
-  return {
-    subscribe: enriched.subscribe,
-    ...actions,
-  }
-}
-
-export const roles = createRolesStore()

packages/builder/src/stores/builder/roles.ts

@@ -0,0 +1,94 @@
+import { derived, get, type Writable } from "svelte/store"
+import { API } from "@/api"
+import { RoleUtils } from "@budibase/frontend-core"
+import { DerivedBudiStore } from "../BudiStore"
+import { Role } from "@budibase/types"
+
+export class RoleStore extends DerivedBudiStore<Role[], Role[]> {
+  constructor() {
+    const makeDerivedStore = (store: Writable<Role[]>) =>
+      derived(store, $store => {
+        return $store.map((role: Role) => ({
+          ...role,
+          // Ensure we have new metadata for all roles
+          uiMetadata: {
+            displayName: role.uiMetadata?.displayName || role.name,
+            color:
+              role.uiMetadata?.color ||
+              "var(--spectrum-global-color-magenta-400)",
+            description: role.uiMetadata?.description || "Custom role",
+          },
+        }))
+      })
+
+    super([], makeDerivedStore)
+  }
+
+  setRoles = (roles: Role[]) => {
+    this.set(
+      roles.sort((a, b) => {
+        // Ensure we have valid IDs for priority comparison
+        const priorityA = RoleUtils.getRolePriority(a._id)
+        const priorityB = RoleUtils.getRolePriority(b._id)
+        if (priorityA !== priorityB) {
+          return priorityA > priorityB ? -1 : 1
+        }
+        const nameA = a.uiMetadata?.displayName || a.name
+        const nameB = b.uiMetadata?.displayName || b.name
+        return nameA < nameB ? -1 : 1
+      })
+    )
+  }
+
+  fetch = async () => {
+    const roles = await API.getRoles()
+    this.setRoles(roles)
+  }
+
+  fetchByAppId = async (appId: string) => {
+    const { roles } = await API.getRolesForApp(appId)
+    this.setRoles(roles)
+  }
+
+  delete = async (role: Role) => {
+    if (!role._id || !role._rev) {
+      return
+    }
+    await API.deleteRole(role._id, role._rev)
+    await this.fetch()
+  }
+
+  save = async (role: Role) => {
+    const savedRole = await API.saveRole(role)
+    await this.fetch()
+    return savedRole
+  }
+
+  replace = (roleId: string, role?: Role) => {
+    // Handles external updates of roles
+    if (!roleId) {
+      return
+    }
+
+    // Handle deletion
+    if (!role) {
+      this.update(state => state.filter(x => x._id !== roleId))
+      return
+    }
+
+    // Add new role
+    const index = get(this).findIndex(x => x._id === role._id)
+    if (index === -1) {
+      this.update(state => [...state, role])
+    }
+    // Update existing role
+    else if (role) {
+      this.update(state => {
+        state[index] = role
+        return [...state]
+      })
+    }
+  }
+}
+
+export const roles = new RoleStore()

packages/builder/src/stores/builder/rowActions.ts

@@ -6,18 +6,29 @@ import { automationStore } from "./automations"
 import { API } from "@/api"
 import { getSequentialName } from "@/helpers/duplicate"
 
-const initialState = {}
+interface RowAction {
+  id: string
+  name: string
+  tableId: string
+  allowedSources?: string[]
+}
 
-export class RowActionStore extends BudiStore {
+interface RowActionState {
+  [tableId: string]: RowAction[]
+}
+
+const initialState: RowActionState = {}
+
+export class RowActionStore extends BudiStore<RowActionState> {
   constructor() {
     super(initialState)
   }
 
   reset = () => {
-    this.store.set(initialState)
+    this.set(initialState)
   }
 
-  refreshRowActions = async sourceId => {
+  refreshRowActions = async (sourceId: string) => {
     if (!sourceId) {
       return
     }
@@ -34,26 +45,30 @@ export class RowActionStore extends BudiStore {
 
     // Fetch row actions for this table
     const res = await API.rowActions.fetch(tableId)
-    const actions = Object.values(res || {})
+    const actions = Object.values(res || {}) as RowAction[]
     this.update(state => ({
       ...state,
       [tableId]: actions,
     }))
   }
 
-  createRowAction = async (tableId, viewId, name) => {
+  createRowAction = async (tableId: string, viewId?: string, name?: string) => {
     if (!tableId) {
       return
     }
 
     // Get a unique name for this action
     if (!name) {
-      const existingRowActions = get(this.store)[tableId] || []
+      const existingRowActions = get(this)[tableId] || []
       name = getSequentialName(existingRowActions, "New row action ", {
         getName: x => x.name,
       })
     }
 
+    if (!name) {
+      throw new Error("Failed to generate a unique name for the row action")
+    }
+
     // Create the action
     const res = await API.rowActions.create(tableId, name)
 
@@ -73,41 +88,35 @@ export class RowActionStore extends BudiStore {
     return res
   }
 
-  enableView = async (tableId, rowActionId, viewId) => {
+  enableView = async (tableId: string, rowActionId: string, viewId: string) => {
     await API.rowActions.enableView(tableId, rowActionId, viewId)
     await this.refreshRowActions(tableId)
   }
 
-  disableView = async (tableId, rowActionId, viewId) => {
+  disableView = async (
+    tableId: string,
+    rowActionId: string,
+    viewId: string
+  ) => {
     await API.rowActions.disableView(tableId, rowActionId, viewId)
     await this.refreshRowActions(tableId)
   }
 
-  rename = async (tableId, rowActionId, name) => {
+  delete = async (tableId: string, rowActionId: string) => {
-    await API.rowActions.update({
-      tableId,
-      rowActionId,
-      name,
-    })
-    await this.refreshRowActions(tableId)
-    automationStore.actions.fetch()
-  }
-
-  delete = async (tableId, rowActionId) => {
     await API.rowActions.delete(tableId, rowActionId)
     await this.refreshRowActions(tableId)
     // We don't need to refresh automations as we can only delete row actions
     // from the automations store, so we already handle the state update there
   }
 
-  trigger = async (sourceId, rowActionId, rowId) => {
+  trigger = async (sourceId: string, rowActionId: string, rowId: string) => {
     await API.rowActions.trigger(sourceId, rowActionId, rowId)
   }
 }
 
 const store = new RowActionStore()
-const derivedStore = derived(store, $store => {
+const derivedStore = derived<RowActionStore, RowActionState>(store, $store => {
-  let map = {}
+  const map: RowActionState = {}
 
   // Generate an entry for every view as well
   Object.keys($store || {}).forEach(tableId => {
@@ -115,7 +124,7 @@ const derivedStore = derived(store, $store => {
     map[tableId] = $store[tableId]
     for (let action of $store[tableId]) {
       const otherSources = (action.allowedSources || []).filter(
-        sourceId => sourceId !== tableId
+        (sourceId: string) => sourceId !== tableId
       )
       for (let source of otherSources) {
         map[source] ??= []

packages/builder/src/stores/builder/snippets.js

@@ -1,35 +0,0 @@
-import { writable, get } from "svelte/store"
-import { API } from "@/api"
-import { appStore } from "./app"
-
-const createsnippets = () => {
-  const store = writable([])
-
-  const syncMetadata = metadata => {
-    store.set(metadata?.snippets || [])
-  }
-
-  const saveSnippet = async updatedSnippet => {
-    const snippets = [
-      ...get(store).filter(snippet => snippet.name !== updatedSnippet.name),
-      updatedSnippet,
-    ]
-    const app = await API.saveAppMetadata(get(appStore).appId, { snippets })
-    syncMetadata(app)
-  }
-
-  const deleteSnippet = async snippetName => {
-    const snippets = get(store).filter(snippet => snippet.name !== snippetName)
-    const app = await API.saveAppMetadata(get(appStore).appId, { snippets })
-    syncMetadata(app)
-  }
-
-  return {
-    ...store,
-    syncMetadata,
-    saveSnippet,
-    deleteSnippet,
-  }
-}
-
-export const snippets = createsnippets()

packages/builder/src/stores/builder/snippets.ts

@@ -0,0 +1,32 @@
+import { get } from "svelte/store"
+import { API } from "@/api"
+import { appStore } from "./app"
+import { BudiStore } from "../BudiStore"
+import { Snippet, UpdateAppResponse } from "@budibase/types"
+
+export class SnippetStore extends BudiStore<Snippet[]> {
+  constructor() {
+    super([])
+  }
+
+  syncMetadata = (metadata: UpdateAppResponse) => {
+    this.set(metadata?.snippets || [])
+  }
+
+  saveSnippet = async (updatedSnippet: Snippet) => {
+    const snippets = [
+      ...get(this).filter(snippet => snippet.name !== updatedSnippet.name),
+      updatedSnippet,
+    ]
+    const app = await API.saveAppMetadata(get(appStore).appId, { snippets })
+    this.syncMetadata(app)
+  }
+
+  deleteSnippet = async (snippetName: string) => {
+    const snippets = get(this).filter(snippet => snippet.name !== snippetName)
+    const app = await API.saveAppMetadata(get(appStore).appId, { snippets })
+    this.syncMetadata(app)
+  }
+}
+
+export const snippets = new SnippetStore()

packages/builder/src/stores/builder/theme.js

@@ -1,58 +0,0 @@
-import { writable, get } from "svelte/store"
-import { API } from "@/api"
-import { ensureValidTheme, DefaultAppTheme } from "@budibase/shared-core"
-
-export const createThemeStore = () => {
-  const store = writable({
-    theme: DefaultAppTheme,
-    customTheme: {},
-  })
-
-  const syncAppTheme = app => {
-    store.update(state => {
-      const theme = ensureValidTheme(app.theme, DefaultAppTheme)
-      return {
-        ...state,
-        theme,
-        customTheme: app.customTheme,
-      }
-    })
-  }
-
-  const save = async (theme, appId) => {
-    const app = await API.saveAppMetadata(appId, { theme })
-    store.update(state => {
-      state.theme = app.theme
-      return state
-    })
-  }
-
-  const saveCustom = async (theme, appId) => {
-    const updated = { ...get(store).customTheme, ...theme }
-    const app = await API.saveAppMetadata(appId, { customTheme: updated })
-    store.update(state => {
-      state.customTheme = app.customTheme
-      return state
-    })
-  }
-
-  const syncMetadata = metadata => {
-    const { theme, customTheme } = metadata
-    store.update(state => ({
-      ...state,
-      theme: ensureValidTheme(theme, DefaultAppTheme),
-      customTheme,
-    }))
-  }
-
-  return {
-    subscribe: store.subscribe,
-    update: store.update,
-    syncMetadata,
-    syncAppTheme,
-    save,
-    saveCustom,
-  }
-}
-
-export const themeStore = createThemeStore()

packages/builder/src/stores/builder/theme.ts

@@ -0,0 +1,58 @@
+import { get } from "svelte/store"
+import { API } from "@/api"
+import { BudiStore } from "../BudiStore"
+import { ensureValidTheme, DefaultAppTheme } from "@budibase/shared-core"
+import { App, UpdateAppResponse, Theme, AppCustomTheme } from "@budibase/types"
+
+interface ThemeState {
+  theme: Theme
+  customTheme: AppCustomTheme
+}
+
+export class ThemeStore extends BudiStore<ThemeState> {
+  constructor() {
+    super({
+      theme: DefaultAppTheme,
+      customTheme: {},
+    })
+  }
+
+  syncAppTheme = (app: App) => {
+    this.update(state => {
+      const theme = ensureValidTheme(app.theme, DefaultAppTheme)
+      return {
+        ...state,
+        theme,
+        customTheme: app.customTheme || {},
+      }
+    })
+  }
+
+  save = async (theme: Theme, appId: string) => {
+    const app = await API.saveAppMetadata(appId, { theme })
+    this.update(state => ({
+      ...state,
+      theme: ensureValidTheme(app.theme, DefaultAppTheme),
+    }))
+  }
+
+  saveCustom = async (theme: Partial<AppCustomTheme>, appId: string) => {
+    const updated = { ...get(this).customTheme, ...theme }
+    const app = await API.saveAppMetadata(appId, { customTheme: updated })
+    this.update(state => ({
+      ...state,
+      customTheme: app.customTheme || {},
+    }))
+  }
+
+  syncMetadata = (metadata: UpdateAppResponse) => {
+    const { theme, customTheme } = metadata
+    this.update(state => ({
+      ...state,
+      theme: ensureValidTheme(theme, DefaultAppTheme),
+      customTheme: customTheme || {},
+    }))
+  }
+}
+
+export const themeStore = new ThemeStore()

packages/builder/src/stores/builder/users.js

@@ -1,62 +0,0 @@
-import { writable, get, derived } from "svelte/store"
-
-export const createUserStore = () => {
-  const store = writable([])
-
-  const init = users => {
-    store.set(users)
-  }
-
-  const updateUser = user => {
-    const $users = get(store)
-    if (!$users.some(x => x.sessionId === user.sessionId)) {
-      store.set([...$users, user])
-    } else {
-      store.update(state => {
-        const index = state.findIndex(x => x.sessionId === user.sessionId)
-        state[index] = user
-        return state.slice()
-      })
-    }
-  }
-
-  const removeUser = sessionId => {
-    store.update(state => {
-      return state.filter(x => x.sessionId !== sessionId)
-    })
-  }
-
-  const reset = () => {
-    store.set([])
-  }
-
-  return {
-    ...store,
-    actions: {
-      init,
-      updateUser,
-      removeUser,
-      reset,
-    },
-  }
-}
-
-export const userStore = createUserStore()
-
-export const userSelectedResourceMap = derived(userStore, $userStore => {
-  let map = {}
-  $userStore.forEach(user => {
-    const resource = user.builderMetadata?.selectedResourceId
-    if (resource) {
-      if (!map[resource]) {
-        map[resource] = []
-      }
-      map[resource].push(user)
-    }
-  })
-  return map
-})
-
-export const isOnlyUser = derived(userStore, $userStore => {
-  return $userStore.length < 2
-})

packages/builder/src/stores/builder/users.ts

@@ -0,0 +1,59 @@
+import { get, derived } from "svelte/store"
+import { BudiStore } from "../BudiStore"
+import { UIUser } from "@budibase/types"
+
+export class UserStore extends BudiStore<UIUser[]> {
+  constructor() {
+    super([])
+  }
+
+  init(users: UIUser[]) {
+    this.set(users)
+  }
+
+  updateUser(user: UIUser) {
+    const $users = get(this)
+    if (!$users.some(x => x.sessionId === user.sessionId)) {
+      this.set([...$users, user])
+    } else {
+      this.update(state => {
+        const index = state.findIndex(x => x.sessionId === user.sessionId)
+        state[index] = user
+        return state.slice()
+      })
+    }
+  }
+
+  removeUser(sessionId: string) {
+    this.update(state => {
+      return state.filter(x => x.sessionId !== sessionId)
+    })
+  }
+
+  reset() {
+    this.set([])
+  }
+}
+
+export const userStore = new UserStore()
+
+export const userSelectedResourceMap = derived(
+  userStore,
+  ($userStore): Record<string, UIUser[]> => {
+    let map: Record<string, UIUser[]> = {}
+    $userStore.forEach(user => {
+      const resource = user.builderMetadata?.selectedResourceId
+      if (resource) {
+        if (!map[resource]) {
+          map[resource] = []
+        }
+        map[resource].push(user)
+      }
+    })
+    return map
+  }
+)
+
+export const isOnlyUser = derived(userStore, $userStore => {
+  return $userStore.length < 2
+})

packages/builder/src/stores/builder/websocket.js

@@ -1,95 +0,0 @@
-import { createWebsocket } from "@budibase/frontend-core"
-import {
-  automationStore,
-  userStore,
-  appStore,
-  themeStore,
-  navigationStore,
-  deploymentStore,
-  snippets,
-  datasources,
-  tables,
-  roles,
-} from "@/stores/builder"
-import { get } from "svelte/store"
-import { auth, appsStore } from "@/stores/portal"
-import { screenStore } from "./screens"
-import { SocketEvent, BuilderSocketEvent, helpers } from "@budibase/shared-core"
-import { notifications } from "@budibase/bbui"
-
-export const createBuilderWebsocket = appId => {
-  const socket = createWebsocket("/socket/builder")
-
-  // Built-in events
-  socket.on("connect", () => {
-    socket.emit(BuilderSocketEvent.SelectApp, { appId }, ({ users }) => {
-      userStore.actions.init(users)
-    })
-  })
-  socket.on("connect_error", err => {
-    console.error("Failed to connect to builder websocket:", err.message)
-  })
-  socket.on("disconnect", () => {
-    userStore.actions.reset()
-  })
-
-  // User events
-  socket.onOther(SocketEvent.UserUpdate, ({ user }) => {
-    userStore.actions.updateUser(user)
-  })
-  socket.onOther(SocketEvent.UserDisconnect, ({ sessionId }) => {
-    userStore.actions.removeUser(sessionId)
-  })
-  socket.onOther(BuilderSocketEvent.LockTransfer, ({ userId }) => {
-    if (userId === get(auth)?.user?._id) {
-      appStore.update(state => ({
-        ...state,
-        hasLock: true,
-      }))
-    }
-  })
-
-  // Data section events
-  socket.onOther(BuilderSocketEvent.TableChange, ({ id, table }) => {
-    tables.replaceTable(id, table)
-  })
-  socket.onOther(BuilderSocketEvent.DatasourceChange, ({ id, datasource }) => {
-    datasources.replaceDatasource(id, datasource)
-  })
-
-  // Role events
-  socket.onOther(BuilderSocketEvent.RoleChange, ({ id, role }) => {
-    roles.replace(id, role)
-  })
-
-  // Design section events
-  socket.onOther(BuilderSocketEvent.ScreenChange, ({ id, screen }) => {
-    screenStore.replace(id, screen)
-  })
-
-  // App events
-  socket.onOther(BuilderSocketEvent.AppMetadataChange, ({ metadata }) => {
-    appStore.syncMetadata(metadata)
-    themeStore.syncMetadata(metadata)
-    navigationStore.syncMetadata(metadata)
-    snippets.syncMetadata(metadata)
-  })
-  socket.onOther(
-    BuilderSocketEvent.AppPublishChange,
-    async ({ user, published }) => {
-      await appsStore.load()
-      if (published) {
-        await deploymentStore.load()
-      }
-      const verb = published ? "published" : "unpublished"
-      notifications.success(`${helpers.getUserLabel(user)} ${verb} this app`)
-    }
-  )
-
-  // Automation events
-  socket.onOther(BuilderSocketEvent.AutomationChange, ({ id, automation }) => {
-    automationStore.actions.replace(id, automation)
-  })
-
-  return socket
-}

packages/builder/src/stores/builder/websocket.ts

@@ -0,0 +1,124 @@
+import { createWebsocket } from "@budibase/frontend-core"
+import {
+  automationStore,
+  userStore,
+  appStore,
+  themeStore,
+  navigationStore,
+  deploymentStore,
+  snippets,
+  datasources,
+  tables,
+  roles,
+} from "@/stores/builder"
+import { get } from "svelte/store"
+import { auth, appsStore } from "@/stores/portal"
+import { screenStore } from "./screens"
+import { SocketEvent, BuilderSocketEvent, helpers } from "@budibase/shared-core"
+import { notifications } from "@budibase/bbui"
+import { Automation, Datasource, Role, Table, UIUser } from "@budibase/types"
+
+export const createBuilderWebsocket = (appId: string) => {
+  const socket = createWebsocket("/socket/builder")
+
+  // Built-in events
+  socket.on("connect", () => {
+    socket.emit(
+      BuilderSocketEvent.SelectApp,
+      { appId },
+      ({ users }: { users: UIUser[] }) => {
+        userStore.init(users)
+      }
+    )
+  })
+  socket.on("connect_error", err => {
+    console.error("Failed to connect to builder websocket:", err.message)
+  })
+  socket.on("disconnect", () => {
+    userStore.reset()
+  })
+
+  // User events
+  socket.onOther(SocketEvent.UserUpdate, ({ user }: { user: UIUser }) => {
+    userStore.updateUser(user)
+  })
+  socket.onOther(
+    SocketEvent.UserDisconnect,
+    ({ sessionId }: { sessionId: string }) => {
+      userStore.removeUser(sessionId)
+    }
+  )
+  socket.onOther(
+    BuilderSocketEvent.LockTransfer,
+    ({ userId }: { userId: string }) => {
+      if (userId === get(auth)?.user?._id) {
+        appStore.update(state => ({
+          ...state,
+          hasLock: true,
+        }))
+      }
+    }
+  )
+
+  // Data section events
+  socket.onOther(
+    BuilderSocketEvent.TableChange,
+    ({ id, table }: { id: string; table: Table }) => {
+      tables.replaceTable(id, table)
+    }
+  )
+  socket.onOther(
+    BuilderSocketEvent.DatasourceChange,
+    ({ id, datasource }: { id: string; datasource: Datasource }) => {
+      datasources.replaceDatasource(id, datasource)
+    }
+  )
+
+  // Role events
+  socket.onOther(
+    BuilderSocketEvent.RoleChange,
+    ({ id, role }: { id: string; role: Role }) => {
+      roles.replace(id, role)
+    }
+  )
+
+  // Design section events
+  socket.onOther(
+    BuilderSocketEvent.ScreenChange,
+    ({ id, screen }: { id: string; screen: Screen }) => {
+      screenStore.replace(id, screen)
+    }
+  )
+
+  // App events
+  socket.onOther(
+    BuilderSocketEvent.AppMetadataChange,
+    ({ metadata }: { metadata: any }) => {
+      appStore.syncMetadata(metadata)
+      themeStore.syncMetadata(metadata)
+      navigationStore.syncMetadata(metadata)
+      snippets.syncMetadata(metadata)
+    }
+  )
+  socket.onOther(
+    BuilderSocketEvent.AppPublishChange,
+    async ({ user, published }: { user: UIUser; published: boolean }) => {
+      await appsStore.load()
+      if (published) {
+        await deploymentStore.load()
+      }
+      const verb = published ? "published" : "unpublished"
+      notifications.success(`${helpers.getUserLabel(user)} ${verb} this app`)
+    }
+  )
+
+  // Automation events
+  socket.onOther(
+    BuilderSocketEvent.AutomationChange,
+    ({ id, automation }: { id: string; automation: Automation }) => {
+      automationStore.actions.replace(id, automation)
+    }
+  )
+
+  return socket
+}

packages/builder/src/stores/portal/featureFlags.ts

@@ -1,19 +1,8 @@
 import { derived, Readable } from "svelte/store"
 import { auth } from "@/stores/portal"
+import { FeatureFlags, FeatureFlagDefaults } from "@budibase/types"
 
-export const INITIAL_FEATUREFLAG_STATE = {
+export const featureFlags: Readable<FeatureFlags> = derived(auth, $auth => ({
-  SQS: false,
+  ...FeatureFlagDefaults,
-  DEFAULT_VALUES: false,
-  BUDIBASE_AI: false,
-  AI_CUSTOM_CONFIGS: false,
-}
-
-export const featureFlags: Readable<Record<string, any>> = derived(
-  [auth],
-  ([$auth]) => {
-    return {
-      ...INITIAL_FEATUREFLAG_STATE,
   ...($auth?.user?.flags || {}),
-    }
+}))
-  }
-)

packages/builder/src/stores/portal/groups.ts

@@ -47,9 +47,12 @@ class GroupStore extends BudiStore<UserGroup[]> {
 
   async delete(group: UserGroup) {
     await API.deleteGroup(group._id!, group._rev!)
-    this.update(state => {
+    this.update(groups => {
-      state = state.filter(state => state._id !== group._id)
+      const index = groups.findIndex(g => g._id === group._id)
-      return state
+      if (index !== -1) {
+        groups.splice(index, 1)
+      }
+      return groups
     })
   }
 

packages/builder/src/stores/portal/licensing.ts

@@ -227,7 +227,7 @@ class LicensingStore extends BudiStore<LicensingState> {
       MonthlyQuotaName.AUTOMATIONS,
     ].reduce((acc: MonthlyMetrics, key) => {
       const limit = license.quotas.usage.monthly[key].value
-      const used = (usage.monthly.current?.[key] || 0 / limit) * 100
+      const used = ((usage.monthly.current?.[key] || 0) / limit) * 100
       acc[key] = limit > -1 ? Math.floor(used) : -1
       return acc
     }, {})
@@ -236,7 +236,7 @@ class LicensingStore extends BudiStore<LicensingState> {
     const staticMetrics = [StaticQuotaName.APPS, StaticQuotaName.ROWS].reduce(
       (acc: StaticMetrics, key) => {
         const limit = license.quotas.usage.static[key].value
-        const used = (usage.usageQuota[key] || 0 / limit) * 100
+        const used = ((usage.usageQuota[key] || 0) / limit) * 100
         acc[key] = limit > -1 ? Math.floor(used) : -1
         return acc
       },

packages/builder/src/stores/portal/oidc.ts

@@ -11,14 +11,10 @@ class OIDCStore extends BudiStore<PublicOIDCConfig> {
 
   async init() {
     const tenantId = get(auth).tenantId
-    const config = await API.getOIDCConfig(tenantId)
+    const configs = await API.getOIDCConfigs(tenantId)
-    if (Object.keys(config || {}).length) {
     // Just use the first config for now.
     // We will be support multiple logins buttons later on.
-      this.set(config[0])
+    this.set(configs[0] || {})
-    } else {
-      this.set({})
-    }
   }
 }
 

packages/cli/package.json

@@ -16,6 +16,7 @@
   },
   "dependencies": {
     "@budibase/backend-core": "*",
+    "@budibase/pouchdb-replication-stream": "1.2.11",
     "@budibase/string-templates": "*",
     "@budibase/types": "*",
     "chalk": "4.1.0",
@@ -28,9 +29,9 @@
     "inquirer": "8.0.0",
     "lookpath": "1.1.0",
     "node-fetch": "2.6.7",
+    "open": "8.4.2",
     "posthog-node": "4.0.1",
     "pouchdb": "7.3.0",
-    "@budibase/pouchdb-replication-stream": "1.2.11",
     "randomstring": "1.1.5",
     "tar": "6.2.1",
     "yaml": "^2.1.1"

packages/cli/src/hosting/start.ts

@@ -3,6 +3,8 @@ import { info, success } from "../utils"
 import * as makeFiles from "./makeFiles"
 import compose from "docker-compose"
 import fs from "fs"
+import { confirmation } from "../questions"
+const open = require("open")
 
 export async function start() {
   await checkDockerConfigured()
@@ -22,6 +24,9 @@ export async function start() {
     // need to log as it makes it more clear
     await compose.upAll({ cwd: "./", log: true })
   })
+  if (await confirmation(`Do you wish to open http://localhost:${port} ?`)) {
+    await open(`http://localhost:${port}`)
+  }
   console.log(
     success(
       `Services started, please go to http://localhost:${port} for next steps.`

packages/client/src/api/patches.js

@@ -66,10 +66,9 @@ export const patchAPI = API => {
     }
   }
   const fetchRelationshipData = API.fetchRelationshipData
-  API.fetchRelationshipData = async params => {
+  API.fetchRelationshipData = async (sourceId, rowId, fieldName) => {
-    const tableId = params?.tableId
+    const rows = await fetchRelationshipData(sourceId, rowId, fieldName)
-    const rows = await fetchRelationshipData(params)
+    return await enrichRows(rows, sourceId)
-    return await enrichRows(rows, tableId)
   }
   const fetchTableData = API.fetchTableData
   API.fetchTableData = async tableId => {
@@ -85,9 +84,9 @@ export const patchAPI = API => {
     }
   }
   const fetchViewData = API.fetchViewData
-  API.fetchViewData = async params => {
+  API.fetchViewData = async (viewName, params) => {
     const tableId = params?.tableId
-    const rows = await fetchViewData(params)
+    const rows = await fetchViewData(viewName, params)
     return await enrichRows(rows, tableId)
   }
 

packages/client/src/utils/buttonActions.js

@@ -216,11 +216,11 @@ const deleteRowHandler = async action => {
 const triggerAutomationHandler = async action => {
   const { fields, notificationOverride, timeout } = action.parameters
   try {
-    const result = await API.triggerAutomation({
+    const result = await API.triggerAutomation(
-      automationId: action.parameters.automationId,
+      action.parameters.automationId,
       fields,
-      timeout,
+      timeout
-    })
+    )
 
     // Value will exist if automation is synchronous, so return it.
     if (result.value) {

packages/client/src/utils/schema.js

@@ -1,12 +1,12 @@
 import { API } from "api"
-import TableFetch from "@budibase/frontend-core/src/fetch/TableFetch.js"
+import TableFetch from "@budibase/frontend-core/src/fetch/TableFetch"
-import ViewFetch from "@budibase/frontend-core/src/fetch/ViewFetch.js"
+import ViewFetch from "@budibase/frontend-core/src/fetch/ViewFetch"
-import QueryFetch from "@budibase/frontend-core/src/fetch/QueryFetch.js"
+import QueryFetch from "@budibase/frontend-core/src/fetch/QueryFetch"
-import RelationshipFetch from "@budibase/frontend-core/src/fetch/RelationshipFetch.js"
+import RelationshipFetch from "@budibase/frontend-core/src/fetch/RelationshipFetch"
-import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProviderFetch.js"
+import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProviderFetch"
-import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch.js"
+import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch"
-import JSONArrayFetch from "@budibase/frontend-core/src/fetch/JSONArrayFetch.js"
+import JSONArrayFetch from "@budibase/frontend-core/src/fetch/JSONArrayFetch"
-import ViewV2Fetch from "@budibase/frontend-core/src/fetch/ViewV2Fetch.js"
+import ViewV2Fetch from "@budibase/frontend-core/src/fetch/ViewV2Fetch"
 import QueryArrayFetch from "@budibase/frontend-core/src/fetch/QueryArrayFetch"
 
 /**

packages/frontend-core/src/api/configs.ts

@@ -16,7 +16,7 @@ import { BaseAPIClient } from "./types"
 export interface ConfigEndpoints {
   getConfig: (type: ConfigType) => Promise<FindConfigResponse>
   getTenantConfig: (tentantId: string) => Promise<GetPublicSettingsResponse>
-  getOIDCConfig: (tenantId: string) => Promise<GetPublicOIDCConfigResponse>
+  getOIDCConfigs: (tenantId: string) => Promise<GetPublicOIDCConfigResponse>
   getOIDCLogos: () => Promise<Config<OIDCLogosConfig>>
   saveConfig: (config: SaveConfigRequest) => Promise<SaveConfigResponse>
   deleteConfig: (id: string, rev: string) => Promise<DeleteConfigResponse>
@@ -73,7 +73,7 @@ export const buildConfigEndpoints = (API: BaseAPIClient): ConfigEndpoints => ({
    * Gets the OIDC config for a certain tenant.
    * @param tenantId the tenant ID to get the config for
    */
-  getOIDCConfig: async tenantId => {
+  getOIDCConfigs: async tenantId => {
     return await API.get({
       url: `/api/global/configs/public/oidc?tenantId=${tenantId}`,
     })

packages/frontend-core/src/api/views.ts

@@ -3,7 +3,15 @@ import { BaseAPIClient } from "./types"
 
 export interface ViewEndpoints {
   // Missing request or response types
-  fetchViewData: (name: string, opts: any) => Promise<Row[]>
+  fetchViewData: (
+    name: string,
+    opts: {
+      calculation?: string
+      field?: string
+      groupBy?: string
+      tableId: string
+    }
+  ) => Promise<Row[]>
   exportView: (name: string, format: string) => Promise<any>
   saveView: (view: any) => Promise<any>
   deleteView: (name: string) => Promise<any>
@@ -20,7 +28,9 @@ export const buildViewEndpoints = (API: BaseAPIClient): ViewEndpoints => ({
   fetchViewData: async (name, { field, groupBy, calculation }) => {
     const params = new URLSearchParams()
     if (calculation) {
+      if (field) {
         params.set("field", field)
+      }
       params.set("calculation", calculation)
     }
     if (groupBy) {

packages/frontend-core/src/api/viewsV2.ts

@@ -1,6 +1,7 @@
 import {
   CreateViewRequest,
   CreateViewResponse,
+  PaginatedSearchRowResponse,
   SearchRowResponse,
   SearchViewRowRequest,
   UpdateViewRequest,
@@ -13,10 +14,14 @@ export interface ViewV2Endpoints {
   fetchDefinition: (viewId: string) => Promise<ViewResponseEnriched>
   create: (view: CreateViewRequest) => Promise<CreateViewResponse>
   update: (view: UpdateViewRequest) => Promise<UpdateViewResponse>
-  fetch: (
+  fetch: <T extends SearchViewRowRequest>(
     viewId: string,
-    opts: SearchViewRowRequest
+    opts: T
-  ) => Promise<SearchRowResponse>
+  ) => Promise<
+    T extends { paginate: true }
+      ? PaginatedSearchRowResponse
+      : SearchRowResponse
+  >
   delete: (viewId: string) => Promise<void>
 }
 
@@ -59,7 +64,7 @@ export const buildViewV2Endpoints = (API: BaseAPIClient): ViewV2Endpoints => ({
    * @param viewId the id of the view
    * @param opts the search options
    */
-  fetch: async (viewId, opts) => {
+  fetch: async (viewId, opts: SearchViewRowRequest) => {
     return await API.post({
       url: `/api/v2/views/${encodeURIComponent(viewId)}/search`,
       body: opts,

packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte

@@ -26,7 +26,7 @@
         : RelationshipType.MANY_TO_MANY,
   }
 
-  async function searchFunction(searchParams) {
+  async function searchFunction(_tableId, searchParams) {
     if (
       subtype !== BBReferenceFieldSubType.USER &&
       subtype !== BBReferenceFieldSubType.USERS

packages/frontend-core/src/components/grid/stores/config.ts

@@ -69,7 +69,7 @@ export const deriveStores = (context: StoreContext): ConfigDerivedStore => {
       }
 
       // Disable features for non DS+
-      if (!["table", "viewV2"].includes(type)) {
+      if (type && !["table", "viewV2"].includes(type)) {
         config.canAddRows = false
         config.canEditRows = false
         config.canDeleteRows = false

packages/frontend-core/src/components/grid/stores/datasource.ts

@@ -1,3 +1,5 @@
+// TODO: datasource and defitions are unions of the different implementations. At this point, the datasource does not know what type is being used, and the assignations will cause TS exceptions. Casting it "as any" for now. This should be fixed improving the type usages.
+
 import { derived, get, Readable, Writable } from "svelte/store"
 import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch"
 import { enrichSchemaWithRelColumns, memo } from "../../../utils"
@@ -71,10 +73,10 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => {
   } = context
 
   const schema = derived(definition, $definition => {
-    let schema: Record<string, UIFieldSchema> = getDatasourceSchema({
+    const schema: Record<string, any> | undefined = getDatasourceSchema({
       API,
-      datasource: get(datasource),
+      datasource: get(datasource) as any, // TODO: see line 1
-      definition: $definition,
+      definition: $definition ?? undefined,
     })
     if (!schema) {
       return null
@@ -82,7 +84,7 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => {
 
     // Ensure schema is configured as objects.
     // Certain datasources like queries use primitives.
-    Object.keys(schema || {}).forEach(key => {
+    Object.keys(schema).forEach(key => {
       if (typeof schema[key] !== "object") {
         schema[key] = { name: key, type: schema[key] }
       }
@@ -130,13 +132,13 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => {
     ([$datasource, $definition]) => {
       let type = $datasource?.type
       if (type === "provider") {
-        type = ($datasource as any).value?.datasource?.type
+        type = ($datasource as any).value?.datasource?.type // TODO: see line 1
       }
       // Handle calculation views
       if (type === "viewV2" && $definition?.type === ViewV2Type.CALCULATION) {
         return false
       }
-      return ["table", "viewV2", "link"].includes(type)
+      return !!type && ["table", "viewV2", "link"].includes(type)
     }
   )
 
@@ -184,9 +186,9 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => {
   const refreshDefinition = async () => {
     const def = await getDatasourceDefinition({
       API,
-      datasource: get(datasource),
+      datasource: get(datasource) as any, // TODO: see line 1
     })
-    definition.set(def)
+    definition.set(def as any) // TODO: see line 1
   }
 
   // Saves the datasource definition
@@ -231,7 +233,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => {
     if ("default" in newDefinition.schema[column]) {
       delete newDefinition.schema[column].default
     }
-    return await saveDefinition(newDefinition as any)
+    return await saveDefinition(newDefinition as any) // TODO: see line 1
   }
 
   // Adds a schema mutation for a single field
@@ -307,7 +309,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => {
     await saveDefinition({
       ...$definition,
       schema: newSchema,
-    } as any)
+    } as any) // TODO: see line 1
     resetSchemaMutations()
   }
 

packages/frontend-core/src/components/grid/stores/rows.ts

@@ -10,9 +10,10 @@ import {
 import { tick } from "svelte"
 import { Helpers } from "@budibase/bbui"
 import { sleep } from "../../../utils/utils"
-import { FieldType, Row, UIFetchAPI, UIRow } from "@budibase/types"
+import { FieldType, Row, UIRow } from "@budibase/types"
 import { getRelatedTableValues } from "../../../utils"
 import { Store as StoreContext } from "."
+import DataFetch from "../../../fetch/DataFetch"
 
 interface IndexedUIRow extends UIRow {
   __idx: number
@@ -20,7 +21,7 @@ interface IndexedUIRow extends UIRow {
 
 interface RowStore {
   rows: Writable<UIRow[]>
-  fetch: Writable<UIFetchAPI | null>
+  fetch: Writable<DataFetch<any, any, any> | null> // TODO: type this properly, having a union of all the possible options
   loaded: Writable<boolean>
   refreshing: Writable<boolean>
   loading: Writable<boolean>
@@ -225,7 +226,7 @@ export const createActions = (context: StoreContext): RowActionStore => {
     })
 
     // Subscribe to changes of this fetch model
-    unsubscribe = newFetch.subscribe(async ($fetch: UIFetchAPI) => {
+    unsubscribe = newFetch.subscribe(async $fetch => {
       if ($fetch.error) {
         // Present a helpful error to the user
         let message = "An unknown error occurred"
@@ -253,7 +254,7 @@ export const createActions = (context: StoreContext): RowActionStore => {
 
         // Reset state properties when dataset changes
         if (!$instanceLoaded || resetRows) {
-          definition.set($fetch.definition)
+          definition.set($fetch.definition as any) // TODO: datasource and defitions are unions of the different implementations. At this point, the datasource does not know what type is being used, and the assignations will cause TS exceptions. Casting it "as any" for now. This should be fixed improving the type usages.
         }
 
         // Reset scroll state when data changes

packages/frontend-core/src/constants.ts

@@ -32,8 +32,8 @@ export const Cookies = {
 }
 
 // Table names
-export const TableNames = {
+export const enum TableNames {
-  USERS: "ta_users",
+  USERS = "ta_users",
 }
 
 export const BudibaseRoles = {

packages/frontend-core/src/fetch/CustomFetch.ts

@@ -1,8 +1,17 @@
-import DataFetch from "./DataFetch.js"
+import DataFetch from "./DataFetch"
 
-export default class CustomFetch extends DataFetch {
+interface CustomDatasource {
+  data: any
+}
+
+type CustomDefinition = Record<string, any>
+
+export default class CustomFetch extends DataFetch<
+  CustomDatasource,
+  CustomDefinition
+> {
   // Gets the correct Budibase type for a JS value
-  getType(value) {
+  getType(value: any) {
     if (value == null) {
       return "string"
     }
@@ -22,7 +31,7 @@ export default class CustomFetch extends DataFetch {
   }
 
   // Parses the custom data into an array format
-  parseCustomData(data) {
+  parseCustomData(data: any) {
     if (!data) {
       return []
     }
@@ -55,7 +64,7 @@ export default class CustomFetch extends DataFetch {
   }
 
   // Enriches the custom data to ensure the structure and format is usable
-  enrichCustomData(data) {
+  enrichCustomData(data: (string | any)[]) {
     if (!data?.length) {
       return []
     }
@@ -72,7 +81,7 @@ export default class CustomFetch extends DataFetch {
       // Try parsing strings
       if (typeof value === "string") {
         const split = value.split(",").map(x => x.trim())
-        let obj = {}
+        const obj: Record<string, string> = {}
         for (let i = 0; i < split.length; i++) {
           const suffix = i === 0 ? "" : ` ${i + 1}`
           const key = `Value${suffix}`
@@ -87,27 +96,29 @@ export default class CustomFetch extends DataFetch {
   }
 
   // Extracts and parses the custom data from the datasource definition
-  getCustomData(datasource) {
+  getCustomData(datasource: CustomDatasource) {
     return this.enrichCustomData(this.parseCustomData(datasource?.data))
   }
 
-  async getDefinition(datasource) {
+  async getDefinition() {
+    const { datasource } = this.options
+
     // Try and work out the schema from the array provided
-    let schema = {}
+    const schema: CustomDefinition = {}
     const data = this.getCustomData(datasource)
     if (!data?.length) {
       return { schema }
     }
 
     // Go through every object and extract all valid keys
-    for (let datum of data) {
+    for (const datum of data) {
-      for (let key of Object.keys(datum)) {
+      for (const key of Object.keys(datum)) {
         if (key === "_id") {
           continue
         }
         if (!schema[key]) {
           let type = this.getType(datum[key])
-          let constraints = {}
+          const constraints: any = {}
 
           // Determine whether we should render text columns as options instead
           if (type === "string") {

packages/frontend-core/src/fetch/DataFetch.ts

@@ -1,25 +1,102 @@
-import { writable, derived, get } from "svelte/store"
+import { writable, derived, get, Writable, Readable } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
 import { QueryUtils } from "../utils"
 import { convertJSONSchemaToTableSchema } from "../utils/json"
-import { FieldType, SortOrder, SortType } from "@budibase/types"
+import {
+  FieldType,
+  LegacyFilter,
+  Row,
+  SearchFilters,
+  SortOrder,
+  SortType,
+  TableSchema,
+  UISearchFilter,
+} from "@budibase/types"
+import { APIClient } from "../api/types"
 
 const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils
 
+interface DataFetchStore<TDefinition, TQuery> {
+  rows: Row[]
+  info: any
+  schema: TableSchema | null
+  loading: boolean
+  loaded: boolean
+  query: TQuery
+  pageNumber: number
+  cursor: string | null
+  cursors: string[]
+  resetKey: string
+  error: {
+    message: string
+    status: number
+  } | null
+  definition?: TDefinition | null
+}
+
+interface DataFetchDerivedStore<TDefinition, TQuery>
+  extends DataFetchStore<TDefinition, TQuery> {
+  hasNextPage: boolean
+  hasPrevPage: boolean
+  supportsSearch: boolean
+  supportsSort: boolean
+  supportsPagination: boolean
+}
+
+export interface DataFetchParams<
+  TDatasource,
+  TQuery = SearchFilters | undefined
+> {
+  API: APIClient
+  datasource: TDatasource
+  query: TQuery
+  options?: {}
+}
+
 /**
  * Parent class which handles the implementation of fetching data from an
  * internal table or datasource plus.
  * For other types of datasource, this class is overridden and extended.
  */
-export default class DataFetch {
+export default abstract class DataFetch<
+  TDatasource extends {},
+  TDefinition extends {
+    schema?: Record<string, any> | null
+    primaryDisplay?: string
+  },
+  TQuery extends {} = SearchFilters
+> {
+  API: APIClient
+  features: {
+    supportsSearch: boolean
+    supportsSort: boolean
+    supportsPagination: boolean
+  }
+  options: {
+    datasource: TDatasource
+    limit: number
+    // Search config
+    filter: UISearchFilter | LegacyFilter[] | null
+    query: TQuery
+    // Sorting config
+    sortColumn: string | null
+    sortOrder: SortOrder
+    sortType: SortType | null
+    // Pagination config
+    paginate: boolean
+    // Client side feature customisation
+    clientSideSearching: boolean
+    clientSideSorting: boolean
+    clientSideLimiting: boolean
+  }
+  store: Writable<DataFetchStore<TDefinition, TQuery>>
+  derivedStore: Readable<DataFetchDerivedStore<TDefinition, TQuery>>
+
   /**
    * Constructs a new DataFetch instance.
    * @param opts the fetch options
    */
-  constructor(opts) {
+  constructor(opts: DataFetchParams<TDatasource, TQuery>) {
-    // API client
-    this.API = null
-
     // Feature flags
     this.features = {
       supportsSearch: false,
@@ -29,12 +106,12 @@ export default class DataFetch {
 
     // Config
     this.options = {
-      datasource: null,
+      datasource: opts.datasource,
       limit: 10,
 
       // Search config
       filter: null,
-      query: null,
+      query: opts.query,
 
       // Sorting config
       sortColumn: null,
@@ -57,11 +134,11 @@ export default class DataFetch {
       schema: null,
       loading: false,
       loaded: false,
-      query: null,
+      query: opts.query,
       pageNumber: 0,
       cursor: null,
       cursors: [],
-      resetKey: Math.random(),
+      resetKey: Math.random().toString(),
       error: null,
     })
 
@@ -118,7 +195,10 @@ export default class DataFetch {
   /**
    * Gets the default sort column for this datasource
    */
-  getDefaultSortColumn(definition, schema) {
+  getDefaultSortColumn(
+    definition: { primaryDisplay?: string } | null,
+    schema: Record<string, any>
+  ): string | null {
     if (definition?.primaryDisplay && schema[definition.primaryDisplay]) {
       return definition.primaryDisplay
     } else {
@@ -130,13 +210,13 @@ export default class DataFetch {
    * Fetches a fresh set of data from the server, resetting pagination
    */
   async getInitialData() {
-    const { datasource, filter, paginate } = this.options
+    const { filter, paginate } = this.options
 
     // Fetch datasource definition and extract sort properties if configured
-    const definition = await this.getDefinition(datasource)
+    const definition = await this.getDefinition()
 
     // Determine feature flags
-    const features = this.determineFeatureFlags(definition)
+    const features = await this.determineFeatureFlags()
     this.features = {
       supportsSearch: !!features?.supportsSearch,
       supportsSort: !!features?.supportsSort,
@@ -144,11 +224,11 @@ export default class DataFetch {
     }
 
     // Fetch and enrich schema
-    let schema = this.getSchema(datasource, definition)
+    let schema = this.getSchema(definition)
-    schema = this.enrichSchema(schema)
     if (!schema) {
       return
     }
+    schema = this.enrichSchema(schema)
 
     // If an invalid sort column is specified, delete it
     if (this.options.sortColumn && !schema[this.options.sortColumn]) {
@@ -172,20 +252,25 @@ export default class DataFetch {
       if (
         fieldSchema?.type === FieldType.NUMBER ||
         fieldSchema?.type === FieldType.BIGINT ||
-        fieldSchema?.calculationType
+        ("calculationType" in fieldSchema && fieldSchema?.calculationType)
       ) {
         this.options.sortType = SortType.NUMBER
       }
+
       // If no sort order, default to ascending
       if (!this.options.sortOrder) {
         this.options.sortOrder = SortOrder.ASCENDING
+      } else {
+        // Ensure sortOrder matches the enum
+        this.options.sortOrder =
+          this.options.sortOrder.toLowerCase() as SortOrder
       }
     }
 
     // Build the query
     let query = this.options.query
     if (!query) {
-      query = buildQuery(filter)
+      query = buildQuery(filter ?? undefined) as TQuery
     }
 
     // Update store
@@ -210,7 +295,7 @@ export default class DataFetch {
       info: page.info,
       cursors: paginate && page.hasNextPage ? [null, page.cursor] : [null],
       error: page.error,
-      resetKey: Math.random(),
+      resetKey: Math.random().toString(),
     }))
   }
 
@@ -238,8 +323,8 @@ export default class DataFetch {
     }
 
     // If we don't support sorting, do a client-side sort
-    if (!this.features.supportsSort && clientSideSorting) {
+    if (!this.features.supportsSort && clientSideSorting && sortType) {
-      rows = sort(rows, sortColumn, sortOrder, sortType)
+      rows = sort(rows, sortColumn as any, sortOrder, sortType)
     }
 
     // If we don't support pagination, do a client-side limit
@@ -256,49 +341,28 @@ export default class DataFetch {
     }
   }
 
-  /**
+  abstract getData(): Promise<{
-   * Fetches a single page of data from the remote resource.
+    rows: Row[]
-   * Must be overridden by a datasource specific child class.
+    info?: any
-   */
+    hasNextPage?: boolean
-  async getData() {
+    cursor?: any
-    return {
+    error?: any
-      rows: [],
+  }>
-      info: null,
-      hasNextPage: false,
-      cursor: null,
-    }
-  }
 
   /**
    * Gets the definition for this datasource.
-   * Defaults to fetching a table definition.
+
-   * @param datasource
    * @return {object} the definition
    */
-  async getDefinition(datasource) {
+  abstract getDefinition(): Promise<TDefinition | null>
-    if (!datasource?.tableId) {
-      return null
-    }
-    try {
-      return await this.API.fetchTableDefinition(datasource.tableId)
-    } catch (error) {
-      this.store.update(state => ({
-        ...state,
-        error,
-      }))
-      return null
-    }
-  }
 
   /**
    * Gets the schema definition for a datasource.
-   * Defaults to getting the "schema" property of the definition.
-   * @param datasource the datasource
    * @param definition the datasource definition
    * @return {object} the schema
    */
-  getSchema(datasource, definition) {
+  getSchema(definition: TDefinition | null): Record<string, any> | undefined {
-    return definition?.schema
+    return definition?.schema ?? undefined
   }
 
   /**
@@ -307,32 +371,30 @@ export default class DataFetch {
    * @param schema the datasource schema
    * @return {object} the enriched datasource schema
    */
-  enrichSchema(schema) {
+  private enrichSchema(schema: TableSchema): TableSchema {
-    if (schema == null) {
-      return null
-    }
-
     // Check for any JSON fields so we can add any top level properties
-    let jsonAdditions = {}
+    let jsonAdditions: Record<string, { type: string; nestedJSON: true }> = {}
-    Object.keys(schema).forEach(fieldKey => {
+    for (const fieldKey of Object.keys(schema)) {
       const fieldSchema = schema[fieldKey]
-      if (fieldSchema?.type === FieldType.JSON) {
+      if (fieldSchema.type === FieldType.JSON) {
         const jsonSchema = convertJSONSchemaToTableSchema(fieldSchema, {
           squashObjects: true,
-        })
+        }) as Record<string, { type: string }> | null // TODO: remove when convertJSONSchemaToTableSchema is typed
-        Object.keys(jsonSchema).forEach(jsonKey => {
+        if (jsonSchema) {
+          for (const jsonKey of Object.keys(jsonSchema)) {
             jsonAdditions[`${fieldKey}.${jsonKey}`] = {
               type: jsonSchema[jsonKey].type,
               nestedJSON: true,
             }
-        })
           }
-    })
+        }
-    schema = { ...schema, ...jsonAdditions }
+      }
+    }
 
     // Ensure schema is in the correct structure
-    let enrichedSchema = {}
+    let enrichedSchema: TableSchema = {}
-    Object.entries(schema).forEach(([fieldName, fieldSchema]) => {
+    Object.entries({ ...schema, ...jsonAdditions }).forEach(
+      ([fieldName, fieldSchema]) => {
         if (typeof fieldSchema === "string") {
           enrichedSchema[fieldName] = {
             type: fieldSchema,
@@ -341,19 +403,24 @@ export default class DataFetch {
         } else {
           enrichedSchema[fieldName] = {
             ...fieldSchema,
+            type: fieldSchema.type as any, // TODO: check type union definition conflicts
             name: fieldName,
           }
         }
-    })
+      }
+    )
 
     return enrichedSchema
   }
 
   /**
-   * Determine the feature flag for this datasource definition
+   * Determine the feature flag for this datasource
-   * @param definition
    */
-  determineFeatureFlags(_definition) {
+  async determineFeatureFlags(): Promise<{
+    supportsPagination: boolean
+    supportsSearch?: boolean
+    supportsSort?: boolean
+  }> {
     return {
       supportsSearch: false,
       supportsSort: false,
@@ -365,12 +432,11 @@ export default class DataFetch {
    * Resets the data set and updates options
    * @param newOptions any new options
    */
-  async update(newOptions) {
+  async update(newOptions: any) {
     // Check if any settings have actually changed
     let refresh = false
-    const entries = Object.entries(newOptions || {})
+    for (const [key, value] of Object.entries(newOptions || {})) {
-    for (let [key, value] of entries) {
+      const oldVal = this.options[key as keyof typeof this.options] ?? null
-      const oldVal = this.options[key] == null ? null : this.options[key]
       const newVal = value == null ? null : value
       if (JSON.stringify(newVal) !== JSON.stringify(oldVal)) {
         refresh = true
@@ -437,7 +503,7 @@ export default class DataFetch {
    * @param state the current store state
    * @return {boolean} whether there is a next page of data or not
    */
-  hasNextPage(state) {
+  private hasNextPage(state: DataFetchStore<TDefinition, TQuery>): boolean {
     return state.cursors[state.pageNumber + 1] != null
   }
 
@@ -447,7 +513,7 @@ export default class DataFetch {
    * @param state the current store state
    * @return {boolean} whether there is a previous page of data or not
    */
-  hasPrevPage(state) {
+  private hasPrevPage(state: { pageNumber: number }): boolean {
     return state.pageNumber > 0
   }
 

packages/frontend-core/src/fetch/FieldFetch.ts

@@ -1,7 +1,27 @@
-import DataFetch from "./DataFetch.js"
+import { Row } from "@budibase/types"
+import DataFetch from "./DataFetch"
+
+export interface FieldDatasource {
+  tableId: string
+  fieldType: "attachment" | "array"
+  value: string[] | Row[]
+}
+
+export interface FieldDefinition {
+  schema?: Record<string, { type: string }> | null
+}
+
+function isArrayOfStrings(value: string[] | Row[]): value is string[] {
+  return Array.isArray(value) && !!value[0] && typeof value[0] !== "object"
+}
+
+export default class FieldFetch extends DataFetch<
+  FieldDatasource,
+  FieldDefinition
+> {
+  async getDefinition(): Promise<FieldDefinition | null> {
+    const { datasource } = this.options
 
-export default class FieldFetch extends DataFetch {
-  async getDefinition(datasource) {
     // Field sources have their schema statically defined
     let schema
     if (datasource.fieldType === "attachment") {
@@ -28,8 +48,8 @@ export default class FieldFetch extends DataFetch {
 
     // These sources will be available directly from context
     const data = datasource?.value || []
-    let rows
+    let rows: Row[]
-    if (Array.isArray(data) && data[0] && typeof data[0] !== "object") {
+    if (isArrayOfStrings(data)) {
       rows = data.map(value => ({ value }))
     } else {
       rows = data

packages/frontend-core/src/fetch/GroupUserFetch.ts

@@ -1,9 +1,22 @@
 import { get } from "svelte/store"
-import DataFetch from "./DataFetch.js"
+import DataFetch, { DataFetchParams } from "./DataFetch"
 import { TableNames } from "../constants"
 
-export default class GroupUserFetch extends DataFetch {
+interface GroupUserQuery {
-  constructor(opts) {
+  groupId: string
+  emailSearch: string
+}
+
+interface GroupUserDatasource {
+  tableId: TableNames.USERS
+}
+
+export default class GroupUserFetch extends DataFetch<
+  GroupUserDatasource,
+  {},
+  GroupUserQuery
+> {
+  constructor(opts: DataFetchParams<GroupUserDatasource, GroupUserQuery>) {
     super({
       ...opts,
       datasource: {
@@ -12,7 +25,7 @@ export default class GroupUserFetch extends DataFetch {
     })
   }
 
-  determineFeatureFlags() {
+  async determineFeatureFlags() {
     return {
       supportsSearch: true,
       supportsSort: false,
@@ -28,11 +41,12 @@ export default class GroupUserFetch extends DataFetch {
 
   async getData() {
     const { query, cursor } = get(this.store)
+
     try {
       const res = await this.API.getGroupUsers({
         id: query.groupId,
         emailSearch: query.emailSearch,
-        bookmark: cursor,
+        bookmark: cursor ?? undefined,
       })
 
       return {

packages/frontend-core/src/fetch/JSONArrayFetch.ts

@@ -1,8 +1,10 @@
-import FieldFetch from "./FieldFetch.js"
+import FieldFetch from "./FieldFetch"
 import { getJSONArrayDatasourceSchema } from "../utils/json"
 
 export default class JSONArrayFetch extends FieldFetch {
-  async getDefinition(datasource) {
+  async getDefinition() {
+    const { datasource } = this.options
+
     // JSON arrays need their table definitions fetched.
     // We can then extract their schema as a subset of the table schema.
     try {

packages/frontend-core/src/fetch/NestedProviderFetch.js

@@ -1,21 +0,0 @@
-import DataFetch from "./DataFetch.js"
-
-export default class NestedProviderFetch extends DataFetch {
-  async getDefinition(datasource) {
-    // Nested providers should already have exposed their own schema
-    return {
-      schema: datasource?.value?.schema,
-      primaryDisplay: datasource?.value?.primaryDisplay,
-    }
-  }
-
-  async getData() {
-    const { datasource } = this.options
-    // Pull the rows from the existing data provider
-    return {
-      rows: datasource?.value?.rows || [],
-      hasNextPage: false,
-      cursor: null,
-    }
-  }
-}

packages/frontend-core/src/fetch/NestedProviderFetch.ts

@@ -0,0 +1,39 @@
+import { Row, TableSchema } from "@budibase/types"
+import DataFetch from "./DataFetch"
+
+interface NestedProviderDatasource {
+  value?: {
+    schema: TableSchema
+    primaryDisplay: string
+    rows: Row[]
+  }
+}
+
+interface NestedProviderDefinition {
+  schema?: TableSchema
+  primaryDisplay?: string
+}
+export default class NestedProviderFetch extends DataFetch<
+  NestedProviderDatasource,
+  NestedProviderDefinition
+> {
+  async getDefinition() {
+    const { datasource } = this.options
+
+    // Nested providers should already have exposed their own schema
+    return {
+      schema: datasource?.value?.schema,
+      primaryDisplay: datasource?.value?.primaryDisplay,
+    }
+  }
+
+  async getData() {
+    const { datasource } = this.options
+    // Pull the rows from the existing data provider
+    return {
+      rows: datasource?.value?.rows || [],
+      hasNextPage: false,
+      cursor: null,
+    }
+  }
+}

packages/frontend-core/src/fetch/QueryArrayFetch.ts

@@ -1,11 +1,13 @@
-import FieldFetch from "./FieldFetch.js"
+import FieldFetch from "./FieldFetch"
 import {
   getJSONArrayDatasourceSchema,
   generateQueryArraySchemas,
 } from "../utils/json"
 
 export default class QueryArrayFetch extends FieldFetch {
-  async getDefinition(datasource) {
+  async getDefinition() {
+    const { datasource } = this.options
+
     if (!datasource?.tableId) {
       return null
     }
@@ -14,10 +16,14 @@ export default class QueryArrayFetch extends FieldFetch {
     try {
       const table = await this.API.fetchQueryDefinition(datasource.tableId)
       const schema = generateQueryArraySchemas(
-        table?.schema,
+        table.schema,
-        table?.nestedSchemaFields
+        table.nestedSchemaFields
       )
-      return { schema: getJSONArrayDatasourceSchema(schema, datasource) }
+      const result = {
+        schema: getJSONArrayDatasourceSchema(schema, datasource),
+      }
+
+      return result
     } catch (error) {
       return null
     }

packages/frontend-core/src/fetch/QueryFetch.ts

@@ -1,9 +1,24 @@
-import DataFetch from "./DataFetch.js"
+import DataFetch from "./DataFetch"
 import { Helpers } from "@budibase/bbui"
+import { ExecuteQueryRequest, Query } from "@budibase/types"
 import { get } from "svelte/store"
 
-export default class QueryFetch extends DataFetch {
+interface QueryDatasource {
-  determineFeatureFlags(definition) {
+  _id: string
+  fields: Record<string, any> & {
+    pagination?: {
+      type: string
+      location: string
+      pageParam: string
+    }
+  }
+  queryParams?: Record<string, string>
+  parameters: { name: string; default: string }[]
+}
+
+export default class QueryFetch extends DataFetch<QueryDatasource, Query> {
+  async determineFeatureFlags() {
+    const definition = await this.getDefinition()
     const supportsPagination =
       !!definition?.fields?.pagination?.type &&
       !!definition?.fields?.pagination?.location &&
@@ -11,7 +26,9 @@ export default class QueryFetch extends DataFetch {
     return { supportsPagination }
   }
 
-  async getDefinition(datasource) {
+  async getDefinition() {
+    const { datasource } = this.options
+
     if (!datasource?._id) {
       return null
     }
@@ -40,17 +57,17 @@ export default class QueryFetch extends DataFetch {
     const type = definition?.fields?.pagination?.type
 
     // Set the default query params
-    let parameters = Helpers.cloneDeep(datasource?.queryParams || {})
+    const parameters = Helpers.cloneDeep(datasource.queryParams || {})
-    for (let param of datasource?.parameters || {}) {
+    for (const param of datasource?.parameters || []) {
       if (!parameters[param.name]) {
         parameters[param.name] = param.default
       }
     }
 
     // Add pagination to query if supported
-    let queryPayload = { parameters }
+    const queryPayload: ExecuteQueryRequest = { parameters }
     if (paginate && supportsPagination) {
-      const requestCursor = type === "page" ? parseInt(cursor || 1) : cursor
+      const requestCursor = type === "page" ? parseInt(cursor || "1") : cursor
       queryPayload.pagination = { page: requestCursor, limit }
     }
 
@@ -65,7 +82,7 @@ export default class QueryFetch extends DataFetch {
       if (paginate && supportsPagination) {
         if (type === "page") {
           // For "page number" pagination, increment the existing page number
-          nextCursor = queryPayload.pagination.page + 1
+          nextCursor = queryPayload.pagination!.page! + 1
           hasNextPage = data?.length === limit && limit > 0
         } else {
           // For "cursor" pagination, the cursor should be in the response

packages/frontend-core/src/fetch/RelationshipFetch.js

@@ -1,20 +0,0 @@
-import DataFetch from "./DataFetch.js"
-
-export default class RelationshipFetch extends DataFetch {
-  async getData() {
-    const { datasource } = this.options
-    if (!datasource?.rowId || !datasource?.rowTableId) {
-      return { rows: [] }
-    }
-    try {
-      const res = await this.API.fetchRelationshipData(
-        datasource.rowTableId,
-        datasource.rowId,
-        datasource.fieldName
-      )
-      return { rows: res }
-    } catch (error) {
-      return { rows: [] }
-    }
-  }
-}

packages/frontend-core/src/fetch/RelationshipFetch.ts

@@ -0,0 +1,48 @@
+import { Table } from "@budibase/types"
+import DataFetch from "./DataFetch"
+
+interface RelationshipDatasource {
+  tableId: string
+  rowId: string
+  rowTableId: string
+  fieldName: string
+}
+
+export default class RelationshipFetch extends DataFetch<
+  RelationshipDatasource,
+  Table
+> {
+  async getDefinition() {
+    const { datasource } = this.options
+
+    if (!datasource?.tableId) {
+      return null
+    }
+    try {
+      return await this.API.fetchTableDefinition(datasource.tableId)
+    } catch (error: any) {
+      this.store.update(state => ({
+        ...state,
+        error,
+      }))
+      return null
+    }
+  }
+
+  async getData() {
+    const { datasource } = this.options
+    if (!datasource?.rowId || !datasource?.rowTableId) {
+      return { rows: [] }
+    }
+    try {
+      const res = await this.API.fetchRelationshipData(
+        datasource.rowTableId,
+        datasource.rowId,
+        datasource.fieldName
+      )
+      return { rows: res }
+    } catch (error) {
+      return { rows: [] }
+    }
+  }
+}

packages/frontend-core/src/fetch/TableFetch.ts

@@ -1,9 +1,9 @@
 import { get } from "svelte/store"
-import DataFetch from "./DataFetch.js"
+import DataFetch from "./DataFetch"
-import { SortOrder } from "@budibase/types"
+import { SortOrder, Table, UITable } from "@budibase/types"
 
-export default class TableFetch extends DataFetch {
+export default class TableFetch extends DataFetch<UITable, Table> {
-  determineFeatureFlags() {
+  async determineFeatureFlags() {
     return {
       supportsSearch: true,
       supportsSort: true,
@@ -11,6 +11,23 @@ export default class TableFetch extends DataFetch {
     }
   }
 
+  async getDefinition() {
+    const { datasource } = this.options
+
+    if (!datasource?.tableId) {
+      return null
+    }
+    try {
+      return await this.API.fetchTableDefinition(datasource.tableId)
+    } catch (error: any) {
+      this.store.update(state => ({
+        ...state,
+        error,
+      }))
+      return null
+    }
+  }
+
   async getData() {
     const { datasource, limit, sortColumn, sortOrder, sortType, paginate } =
       this.options
@@ -23,7 +40,7 @@ export default class TableFetch extends DataFetch {
         query,
         limit,
         sort: sortColumn,
-        sortOrder: sortOrder?.toLowerCase() ?? SortOrder.ASCENDING,
+        sortOrder: sortOrder ?? SortOrder.ASCENDING,
         sortType,
         paginate,
         bookmark: cursor,

packages/frontend-core/src/fetch/UserFetch.ts

@@ -1,10 +1,28 @@
 import { get } from "svelte/store"
-import DataFetch from "./DataFetch.js"
+import DataFetch, { DataFetchParams } from "./DataFetch"
 import { TableNames } from "../constants"
 import { utils } from "@budibase/shared-core"
+import {
+  BasicOperator,
+  SearchFilters,
+  SearchUsersRequest,
+} from "@budibase/types"
 
-export default class UserFetch extends DataFetch {
+interface UserFetchQuery {
-  constructor(opts) {
+  appId: string
+  paginated: boolean
+}
+
+interface UserDatasource {
+  tableId: string
+}
+
+export default class UserFetch extends DataFetch<
+  UserDatasource,
+  {},
+  UserFetchQuery
+> {
+  constructor(opts: DataFetchParams<UserDatasource, UserFetchQuery>) {
     super({
       ...opts,
       datasource: {
@@ -13,7 +31,7 @@ export default class UserFetch extends DataFetch {
     })
   }
 
-  determineFeatureFlags() {
+  async determineFeatureFlags() {
     return {
       supportsSearch: true,
       supportsSort: false,
@@ -22,9 +40,7 @@ export default class UserFetch extends DataFetch {
   }
 
   async getDefinition() {
-    return {
+    return { schema: {} }
-      schema: {},
-    }
   }
 
   async getData() {
@@ -32,15 +48,16 @@ export default class UserFetch extends DataFetch {
     const { cursor, query } = get(this.store)
 
     // Convert old format to new one - we now allow use of the lucene format
-    const { appId, paginated, ...rest } = query || {}
+    const { appId, paginated, ...rest } = query
-    const finalQuery = utils.isSupportedUserSearch(rest)
+
-      ? query
+    const finalQuery: SearchFilters = utils.isSupportedUserSearch(rest)
-      : { string: { email: null } }
+      ? rest
+      : { [BasicOperator.EMPTY]: { email: null } }
 
     try {
-      const opts = {
+      const opts: SearchUsersRequest = {
-        bookmark: cursor,
+        bookmark: cursor ?? undefined,
-        query: finalQuery,
+        query: finalQuery ?? undefined,
         appId: appId,
         paginate: paginated || paginate,
         limit,

packages/frontend-core/src/fetch/ViewFetch.js

@@ -1,17 +0,0 @@
-import DataFetch from "./DataFetch.js"
-
-export default class ViewFetch extends DataFetch {
-  getSchema(datasource, definition) {
-    return definition?.views?.[datasource.name]?.schema
-  }
-
-  async getData() {
-    const { datasource } = this.options
-    try {
-      const res = await this.API.fetchViewData(datasource.name)
-      return { rows: res || [] }
-    } catch (error) {
-      return { rows: [] }
-    }
-  }
-}

packages/frontend-core/src/fetch/ViewFetch.ts

@@ -0,0 +1,44 @@
+import { Table, View } from "@budibase/types"
+import DataFetch from "./DataFetch"
+
+type ViewV1 = View & { name: string }
+
+export default class ViewFetch extends DataFetch<ViewV1, Table> {
+  async getDefinition() {
+    const { datasource } = this.options
+
+    if (!datasource?.tableId) {
+      return null
+    }
+    try {
+      return await this.API.fetchTableDefinition(datasource.tableId)
+    } catch (error: any) {
+      this.store.update(state => ({
+        ...state,
+        error,
+      }))
+      return null
+    }
+  }
+
+  getSchema(definition: Table) {
+    const { datasource } = this.options
+    return definition?.views?.[datasource.name]?.schema
+  }
+
+  async getData() {
+    const { datasource } = this.options
+    try {
+      const res = await this.API.fetchViewData(datasource.name, {
+        calculation: datasource.calculation,
+        field: datasource.field,
+        groupBy: datasource.groupBy,
+        tableId: datasource.tableId,
+      })
+      return { rows: res || [] }
+    } catch (error) {
+      console.error(error, { datasource })
+      return { rows: [] }
+    }
+  }
+}

packages/frontend-core/src/fetch/ViewV2Fetch.ts

@@ -1,9 +1,10 @@
-import { ViewV2Type } from "@budibase/types"
+import { SortOrder, UIView, ViewV2, ViewV2Type } from "@budibase/types"
-import DataFetch from "./DataFetch.js"
+import DataFetch from "./DataFetch"
 import { get } from "svelte/store"
+import { helpers } from "@budibase/shared-core"
 
-export default class ViewV2Fetch extends DataFetch {
+export default class ViewV2Fetch extends DataFetch<UIView, ViewV2> {
-  determineFeatureFlags() {
+  async determineFeatureFlags() {
     return {
       supportsSearch: true,
       supportsSort: true,
@@ -11,18 +12,13 @@ export default class ViewV2Fetch extends DataFetch {
     }
   }
 
-  getSchema(datasource, definition) {
+  async getDefinition() {
-    return definition?.schema
+    const { datasource } = this.options
-  }
 
-  async getDefinition(datasource) {
-    if (!datasource?.id) {
-      return null
-    }
     try {
       const res = await this.API.viewV2.fetchDefinition(datasource.id)
       return res?.data
-    } catch (error) {
+    } catch (error: any) {
       this.store.update(state => ({
         ...state,
         error,
@@ -42,8 +38,10 @@ export default class ViewV2Fetch extends DataFetch {
 
     // If this is a calculation view and we have no calculations, return nothing
     if (
-      definition.type === ViewV2Type.CALCULATION &&
+      definition?.type === ViewV2Type.CALCULATION &&
-      !Object.values(definition.schema || {}).some(x => x.calculationType)
+      !Object.values(definition.schema || {}).some(
+        helpers.views.isCalculationField
+      )
     ) {
       return {
         rows: [],
@@ -56,26 +54,42 @@ export default class ViewV2Fetch extends DataFetch {
     // If sort/filter params are not defined, update options to store the
     // params built in to this view. This ensures that we can accurately
     // compare old and new params and skip a redundant API call.
-    if (!sortColumn && definition.sort?.field) {
+    if (!sortColumn && definition?.sort?.field) {
       this.options.sortColumn = definition.sort.field
-      this.options.sortOrder = definition.sort.order
+      this.options.sortOrder = definition.sort.order || SortOrder.ASCENDING
     }
 
     try {
-      const res = await this.API.viewV2.fetch(datasource.id, {
+      const request = {
-        ...(query ? { query } : {}),
+        query,
         paginate,
         limit,
         bookmark: cursor,
         sort: sortColumn,
-        sortOrder: sortOrder?.toLowerCase(),
+        sortOrder: sortOrder,
         sortType,
+      }
+      if (paginate) {
+        const res = await this.API.viewV2.fetch(datasource.id, {
+          ...request,
+          paginate,
         })
         return {
           rows: res?.rows || [],
           hasNextPage: res?.hasNextPage || false,
           cursor: res?.bookmark || null,
         }
+      } else {
+        const res = await this.API.viewV2.fetch(datasource.id, {
+          ...request,
+          paginate,
+        })
+        return {
+          rows: res?.rows || [],
+          hasNextPage: false,
+          cursor: null,
+        }
+      }
     } catch (error) {
       return {
         rows: [],

packages/frontend-core/src/fetch/index.js

@@ -1,57 +0,0 @@
-import TableFetch from "./TableFetch.js"
-import ViewFetch from "./ViewFetch.js"
-import ViewV2Fetch from "./ViewV2Fetch.js"
-import QueryFetch from "./QueryFetch.js"
-import RelationshipFetch from "./RelationshipFetch.js"
-import NestedProviderFetch from "./NestedProviderFetch.js"
-import FieldFetch from "./FieldFetch.js"
-import JSONArrayFetch from "./JSONArrayFetch.js"
-import UserFetch from "./UserFetch.js"
-import GroupUserFetch from "./GroupUserFetch.js"
-import CustomFetch from "./CustomFetch.js"
-import QueryArrayFetch from "./QueryArrayFetch.js"
-
-const DataFetchMap = {
-  table: TableFetch,
-  view: ViewFetch,
-  viewV2: ViewV2Fetch,
-  query: QueryFetch,
-  link: RelationshipFetch,
-  user: UserFetch,
-  groupUser: GroupUserFetch,
-  custom: CustomFetch,
-
-  // Client specific datasource types
-  provider: NestedProviderFetch,
-  field: FieldFetch,
-  jsonarray: JSONArrayFetch,
-  queryarray: QueryArrayFetch,
-}
-
-// Constructs a new fetch model for a certain datasource
-export const fetchData = ({ API, datasource, options }) => {
-  const Fetch = DataFetchMap[datasource?.type] || TableFetch
-  return new Fetch({ API, datasource, ...options })
-}
-
-// Creates an empty fetch instance with no datasource configured, so no data
-// will initially be loaded
-const createEmptyFetchInstance = ({ API, datasource }) => {
-  const handler = DataFetchMap[datasource?.type]
-  if (!handler) {
-    return null
-  }
-  return new handler({ API })
-}
-
-// Fetches the definition of any type of datasource
-export const getDatasourceDefinition = async ({ API, datasource }) => {
-  const instance = createEmptyFetchInstance({ API, datasource })
-  return await instance?.getDefinition(datasource)
-}
-
-// Fetches the schema of any type of datasource
-export const getDatasourceSchema = ({ API, datasource, definition }) => {
-  const instance = createEmptyFetchInstance({ API, datasource })
-  return instance?.getSchema(datasource, definition)
-}

packages/frontend-core/src/fetch/index.ts

@@ -0,0 +1,91 @@
+import TableFetch from "./TableFetch.js"
+import ViewFetch from "./ViewFetch.js"
+import ViewV2Fetch from "./ViewV2Fetch.js"
+import QueryFetch from "./QueryFetch"
+import RelationshipFetch from "./RelationshipFetch"
+import NestedProviderFetch from "./NestedProviderFetch"
+import FieldFetch from "./FieldFetch"
+import JSONArrayFetch from "./JSONArrayFetch"
+import UserFetch from "./UserFetch.js"
+import GroupUserFetch from "./GroupUserFetch"
+import CustomFetch from "./CustomFetch"
+import QueryArrayFetch from "./QueryArrayFetch.js"
+import { APIClient } from "../api/types.js"
+
+const DataFetchMap = {
+  table: TableFetch,
+  view: ViewFetch,
+  viewV2: ViewV2Fetch,
+  query: QueryFetch,
+  link: RelationshipFetch,
+  user: UserFetch,
+  groupUser: GroupUserFetch,
+  custom: CustomFetch,
+
+  // Client specific datasource types
+  provider: NestedProviderFetch,
+  field: FieldFetch,
+  jsonarray: JSONArrayFetch,
+  queryarray: QueryArrayFetch,
+}
+
+// Constructs a new fetch model for a certain datasource
+export const fetchData = ({ API, datasource, options }: any) => {
+  const Fetch =
+    DataFetchMap[datasource?.type as keyof typeof DataFetchMap] || TableFetch
+  return new Fetch({ API, datasource, ...options })
+}
+
+// Creates an empty fetch instance with no datasource configured, so no data
+// will initially be loaded
+const createEmptyFetchInstance = <
+  TDatasource extends {
+    type: keyof typeof DataFetchMap
+  }
+>({
+  API,
+  datasource,
+}: {
+  API: APIClient
+  datasource: TDatasource
+}) => {
+  const handler = DataFetchMap[datasource?.type as keyof typeof DataFetchMap]
+  if (!handler) {
+    return null
+  }
+  return new handler({ API, datasource: null as any, query: null as any })
+}
+
+// Fetches the definition of any type of datasource
+export const getDatasourceDefinition = async <
+  TDatasource extends {
+    type: keyof typeof DataFetchMap
+  }
+>({
+  API,
+  datasource,
+}: {
+  API: APIClient
+  datasource: TDatasource
+}) => {
+  const instance = createEmptyFetchInstance({ API, datasource })
+  return await instance?.getDefinition()
+}
+
+// Fetches the schema of any type of datasource
+export const getDatasourceSchema = <
+  TDatasource extends {
+    type: keyof typeof DataFetchMap
+  }
+>({
+  API,
+  datasource,
+  definition,
+}: {
+  API: APIClient
+  datasource: TDatasource
+  definition?: any
+}) => {
+  const instance = createEmptyFetchInstance({ API, datasource })
+  return instance?.getSchema(definition)
+}

packages/frontend-core/src/utils/json.d.ts

@@ -0,0 +1,23 @@
+import { JsonFieldMetadata, QuerySchema } from "@budibase/types"
+
+type Schema = Record<string, QuerySchema | string>
+
+declare module "./json" {
+  export const getJSONArrayDatasourceSchema: (
+    tableSchema: Schema,
+    datasource: any
+  ) => Record<string, { type: string; name: string; prefixKeys: string }>
+
+  export const generateQueryArraySchemas: (
+    schema: Schema,
+    nestedSchemaFields?: Record<string, Schema>
+  ) => Schema
+
+  export const convertJSONSchemaToTableSchema: (
+    jsonSchema: JsonFieldMetadata,
+    options: {
+      squashObjects?: boolean
+      prefixKeys?: string
+    }
+  ) => Record<string, { type: string; name: string; prefixKeys: string }>
+}

packages/frontend-core/src/utils/websocket.ts

@@ -1,12 +1,18 @@
-import { io } from "socket.io-client"
+import { io, Socket } from "socket.io-client"
 import { SocketEvent, SocketSessionTTL } from "@budibase/shared-core"
 import { APISessionID } from "../api"
 
 const DefaultOptions = {
   heartbeat: true,
 }
+export interface ExtendedSocket extends Socket {
+  onOther: (event: string, callback: (data: any) => void) => void
+}
 
-export const createWebsocket = (path, options = DefaultOptions) => {
+export const createWebsocket = (
+  path: string,
+  options = DefaultOptions
+): ExtendedSocket => {
   if (!path) {
     throw "A websocket path must be provided"
   }
@@ -32,10 +38,10 @@ export const createWebsocket = (path, options = DefaultOptions) => {
     // Disable polling and rely on websocket only, as HTTP transport
     // will only work with sticky sessions which we don't have
     transports: ["websocket"],
-  })
+  }) as ExtendedSocket
 
   // Set up a heartbeat that's half of the session TTL
-  let interval
+  let interval: NodeJS.Timeout | undefined
   if (heartbeat) {
     interval = setInterval(() => {
       socket.emit(SocketEvent.Heartbeat)
@@ -48,7 +54,7 @@ export const createWebsocket = (path, options = DefaultOptions) => {
 
   // Helper utility to ignore events that were triggered due to API
   // changes made by us
-  socket.onOther = (event, callback) => {
+  socket.onOther = (event: string, callback: (data: any) => void) => {
     socket.on(event, data => {
       if (data?.apiSessionId !== APISessionID) {
         callback(data)

packages/pro

@@ -1 +1 @@
-Subproject commit ae786121d923449b0ad5fcbd123d0a9fec28f65e
+Subproject commit 32d84f109d4edc526145472a7446327312151442

packages/server/src/api/controllers/query/index.ts

@@ -355,7 +355,7 @@ async function execute(
     ExecuteQueryRequest,
     ExecuteV2QueryResponse | ExecuteV1QueryResponse
   >,
-  opts: any = { rowsOnly: false, isAutomation: false }
+  opts = { rowsOnly: false, isAutomation: false }
 ) {
   const db = context.getAppDB()
 
@@ -416,7 +416,7 @@ export async function executeV1(
 export async function executeV2(
   ctx: UserCtx<ExecuteQueryRequest, ExecuteV2QueryResponse>
 ) {
-  return execute(ctx, { rowsOnly: false })
+  return execute(ctx, { rowsOnly: false, isAutomation: false })
 }
 
 export async function executeV2AsAutomation(

packages/server/src/api/controllers/row/staticFormula.ts

@@ -163,9 +163,9 @@ export async function finaliseRow(
     contextRows: [enrichedRow],
   })
   const aiEnabled =
-    ((await features.flags.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
+    ((await features.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
       (await pro.features.isBudibaseAIEnabled())) ||
-    ((await features.flags.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) &&
+    ((await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) &&
       (await pro.features.isAICustomConfigsEnabled()))
   if (aiEnabled) {
     row = await processAIColumns(table, row, {

packages/server/src/api/controllers/row/views.ts

@@ -1,16 +1,16 @@
 import {
   UserCtx,
   ViewV2,
-  SearchRowResponse,
   SearchViewRowRequest,
   RequiredKeys,
   RowSearchParams,
+  PaginatedSearchRowResponse,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { context } from "@budibase/backend-core"
 
 export async function searchView(
-  ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
+  ctx: UserCtx<SearchViewRowRequest, PaginatedSearchRowResponse>
 ) {
   const { viewId } = ctx.params
 
@@ -49,7 +49,13 @@ export async function searchView(
     user: sdk.users.getUserContextBindings(ctx.user),
   })
   result.rows.forEach(r => (r._viewId = view.id))
-  ctx.body = result
+
+  ctx.body = {
+    rows: result.rows,
+    bookmark: result.bookmark,
+    hasNextPage: result.hasNextPage,
+    totalRows: result.totalRows,
+  }
 }
 
 function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {

packages/server/src/automations/actions.ts

@@ -105,13 +105,13 @@ if (env.SELF_HOSTED) {
 export async function getActionDefinitions(): Promise<
   Record<keyof typeof AutomationActionStepId, AutomationStepDefinition>
 > {
-  if (await features.flags.isEnabled(FeatureFlag.AUTOMATION_BRANCHING)) {
+  if (await features.isEnabled(FeatureFlag.AUTOMATION_BRANCHING)) {
     BUILTIN_ACTION_DEFINITIONS["BRANCH"] = branch.definition
   }
   if (
     env.SELF_HOSTED ||
-    (await features.flags.isEnabled(FeatureFlag.BUDIBASE_AI)) ||
+    (await features.isEnabled(FeatureFlag.BUDIBASE_AI)) ||
-    (await features.flags.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS))
+    (await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS))
   ) {
     BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition
   }

packages/server/src/automations/steps/openai.ts

@@ -100,10 +100,10 @@ export async function run({
   try {
     let response
     const customConfigsEnabled =
-      (await features.flags.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) &&
+      (await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) &&
       (await pro.features.isAICustomConfigsEnabled())
     const budibaseAIEnabled =
-      (await features.flags.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
+      (await features.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
       (await pro.features.isBudibaseAIEnabled())
 
     let llmWrapper

packages/server/src/middleware/zod-validator.ts

@@ -7,7 +7,7 @@ import { fromZodError } from "zod-validation-error"
 function validate(schema: AnyZodObject, property: "body" | "params") {
   // Return a Koa middleware function
   return async (ctx: Ctx, next: any) => {
-    if (!(await features.flags.isEnabled(FeatureFlag.USE_ZOD_VALIDATOR))) {
+    if (!(await features.isEnabled(FeatureFlag.USE_ZOD_VALIDATOR))) {
       return next()
     }
 

packages/server/src/threads/definitions.ts

@@ -3,7 +3,10 @@ import { Datasource, Row, Query } from "@budibase/types"
 export type WorkerCallback = (error: any, response?: any) => void
 
 export interface QueryEvent
-  extends Omit<Query, "datasourceId" | "name" | "parameters" | "readable"> {
+  extends Omit<
+    Query,
+    "datasourceId" | "name" | "parameters" | "readable" | "nestedSchemaFields"
+  > {
   appId?: string
   datasource: Datasource
   pagination?: any

packages/server/src/threads/query.ts

@@ -174,7 +174,9 @@ class QueryRunner {
     }
 
     // needs to an array for next step
-    if (!Array.isArray(rows)) {
+    if (rows === null) {
+      rows = []
+    } else if (!Array.isArray(rows)) {
       rows = [rows]
     }
 

packages/shared-core/src/filters.ts

@@ -911,8 +911,8 @@ export function sort<T extends Record<string, any>>(
  * @param docs the data
  * @param limit the number of docs to limit to
  */
-export function limit<T>(docs: T[], limit: string): T[] {
+export function limit<T>(docs: T[], limit: string | number): T[] {
-  const numLimit = parseFloat(limit)
+  const numLimit = typeof limit === "number" ? limit : parseFloat(limit)
   if (isNaN(numLimit)) {
     return docs
   }

packages/shared-core/src/utils.ts

@@ -109,7 +109,9 @@ export function trimOtherProps(object: any, allowedProps: string[]) {
   return result
 }
 
-export function isSupportedUserSearch(query: SearchFilters) {
+export function isSupportedUserSearch(
+  query: SearchFilters
+): query is SearchFilters {
   const allowed = [
     { op: BasicOperator.STRING, key: "email" },
     { op: BasicOperator.EQUAL, key: "_id" },

packages/types/src/api/web/app/query.ts

@@ -40,6 +40,10 @@ export interface ExecuteQueryRequest {
 export type ExecuteV1QueryResponse = Record<string, any>[]
 export interface ExecuteV2QueryResponse {
   data: Record<string, any>[]
+  pagination?: {
+    page: number
+    cursor: string
+  }
 }
 
 export interface DeleteQueryResponse {

packages/types/src/api/web/global/self.ts

@@ -1,5 +1,6 @@
 import { License } from "../../../sdk"
 import { Account, DevInfo, User } from "../../../documents"
+import { FeatureFlags } from "@budibase/types"
 
 export interface GenerateAPIKeyRequest {
   userId?: string
@@ -9,7 +10,7 @@ export interface GenerateAPIKeyResponse extends DevInfo {}
 export interface FetchAPIKeyResponse extends DevInfo {}
 
 export interface GetGlobalSelfResponse extends User {
-  flags?: Record<string, any>
+  flags?: FeatureFlags
   account?: Account
   license: License
   budibaseAccess: boolean

packages/types/src/api/web/pagination.ts

@@ -24,4 +24,5 @@ export interface PaginationRequest extends BasicPaginationRequest {
 export interface PaginationResponse {
   bookmark: string | number | undefined
   hasNextPage?: boolean
+  totalRows?: number
 }

packages/types/src/documents/app/app.ts

@@ -1,4 +1,4 @@
-import { User, Document, Plugin, Snippet } from "../"
+import { User, Document, Plugin, Snippet, Theme } from "../"
 import { SocketSession } from "../../sdk"
 
 export type AppMetadataErrors = { [key: string]: string[] }
@@ -14,7 +14,7 @@ export interface App extends Document {
   instance: AppInstance
   tenantId: string
   status: string
-  theme?: string
+  theme?: Theme
   customTheme?: AppCustomTheme
   revertableVersion?: string
   lockedBy?: User
@@ -37,8 +37,8 @@ export interface AppInstance {
 
 export interface AppNavigation {
   navigation: string
-  title: string
+  title?: string
-  navWidth: string
+  navWidth?: string
   sticky?: boolean
   hideLogo?: boolean
   logoUrl?: string
@@ -46,6 +46,7 @@ export interface AppNavigation {
   navBackground?: string
   navTextColor?: string
   links?: AppNavigationLink[]
+  textAlign?: string
 }
 
 export interface AppNavigationLink {
@@ -53,6 +54,8 @@ export interface AppNavigationLink {
   url: string
   id?: string
   roleId?: string
+  type?: string
+  subLinks?: AppNavigationLink[]
 }
 
 export interface AppCustomTheme {

packages/types/src/documents/app/query.ts

@@ -1,4 +1,5 @@
 import { Document } from "../document"
+import { Row } from "./row"
 
 export interface QuerySchema {
   name?: string
@@ -13,6 +14,7 @@ export interface Query extends Document {
   fields: RestQueryFields | any
   transformer: string | null
   schema: Record<string, QuerySchema | string>
+  nestedSchemaFields?: Record<string, Record<string, QuerySchema | string>>
   readable: boolean
   queryVerb: string
   // flag to state whether the default bindings are empty strings (old behaviour) or null
@@ -29,7 +31,7 @@ export interface QueryParameter {
 }
 
 export interface QueryResponse {
-  rows: any[]
+  rows: Row[]
   keys: string[]
   info: any
   extra: any

packages/types/src/documents/app/table/schema.ts

@@ -227,6 +227,7 @@ interface OtherFieldMetadata extends BaseFieldSchema {
     | FieldType.OPTIONS
     | FieldType.BOOLEAN
     | FieldType.BIGINT
+    | FieldType.JSON
   >
 }
 

packages/types/src/sdk/featureFlag.ts

@@ -6,6 +6,12 @@ export enum FeatureFlag {
   USE_ZOD_VALIDATOR = "USE_ZOD_VALIDATOR",
 }
 
-export interface TenantFeatureFlags {
+export const FeatureFlagDefaults = {
-  [key: string]: FeatureFlag[]
+  [FeatureFlag.DEFAULT_VALUES]: true,
+  [FeatureFlag.AUTOMATION_BRANCHING]: true,
+  [FeatureFlag.AI_CUSTOM_CONFIGS]: true,
+  [FeatureFlag.BUDIBASE_AI]: true,
+  [FeatureFlag.USE_ZOD_VALIDATOR]: false,
 }
+
+export type FeatureFlags = typeof FeatureFlagDefaults

packages/types/src/ui/stores/grid/datasource.ts

@@ -1,8 +1,6 @@
 import { UITable, UIView } from "@budibase/types"
 
-export type UIDatasource = (UITable | UIView) & {
+export type UIDatasource = UITable | UIView
-  type: string
-}
 
 export interface UIFieldMutation {
   visible?: boolean

packages/types/src/ui/stores/grid/fetch.ts

@@ -1,38 +0,0 @@
-import {
-  Row,
-  SortOrder,
-  UIDatasource,
-  UILegacyFilter,
-  UISearchFilter,
-} from "@budibase/types"
-
-export interface UIFetchAPI {
-  definition: UIDatasource
-
-  getInitialData: () => Promise<void>
-  loading: any
-  loaded: boolean
-
-  resetKey: string | null
-  error: any
-
-  hasNextPage: boolean
-  nextPage: () => Promise<void>
-
-  rows: Row[]
-
-  options?: {
-    datasource?: {
-      tableId: string
-      id: string
-    }
-  }
-  update: ({
-    sortOrder,
-    sortColumn,
-  }: {
-    sortOrder?: SortOrder
-    sortColumn?: string
-    filter?: UILegacyFilter[] | UISearchFilter
-  }) => any
-}

packages/types/src/ui/stores/grid/index.ts

@@ -6,4 +6,3 @@ export * from "./view"
 export * from "./user"
 export * from "./filters"
 export * from "./rows"
-export * from "./fetch"

packages/types/src/ui/stores/grid/user.ts

@@ -3,4 +3,5 @@ import { User } from "@budibase/types"
 export interface UIUser extends User {
   sessionId: string
   gridMetadata?: { focusedCellId?: string }
+  builderMetadata?: { selectedResourceId?: string }
 }

packages/types/src/ui/stores/index.ts

@@ -1,3 +1,5 @@
 export * from "./integration"
+export * from "./misc"
 export * from "./automations"
 export * from "./grid"
+export * from "./preview"

packages/types/src/ui/stores/misc.ts

@@ -0,0 +1,2 @@
+// type purely to capture structures that the type is unknown, but maybe known later
+export type UIObject = Record<string, any>

packages/types/src/ui/stores/preview.ts

@@ -0,0 +1 @@
+export type PreviewDevice = "desktop" | "tablet" | "mobile"

packages/worker/src/api/controllers/global/users.ts

@@ -72,12 +72,14 @@ export const save = async (ctx: UserCtx<User, SaveUserResponse>) => {
     const requestUser = ctx.request.body
 
     // Do not allow the account holder role to be changed
-    const accountMetadata = await users.getExistingAccounts([requestUser.email])
-    if (accountMetadata?.length > 0) {
     if (
       requestUser.admin?.global !== true ||
       requestUser.builder?.global !== true
     ) {
+      const accountMetadata = await users.getExistingAccounts([
+        requestUser.email,
+      ])
+      if (accountMetadata?.length > 0) {
         throw Error("Cannot set role of account holder")
       }
     }
@@ -441,7 +443,6 @@ export const checkInvite = async (ctx: UserCtx<void, CheckInviteResponse>) => {
   } catch (e) {
     console.warn("Error getting invite from code", e)
     ctx.throw(400, "There was a problem with the invite")
-    return
   }
   ctx.body = {
     email: invite.email,
@@ -472,7 +473,6 @@ export const updateInvite = async (
     invite = await cache.invite.getCode(code)
   } catch (e) {
     ctx.throw(400, "There was a problem with the invite")
-    return
   }
 
   let updated = {

yarn.lock

@@ -2131,9 +2131,9 @@
     through2 "^2.0.0"
 
 "@budibase/pro@npm:@budibase/pro@latest":
-  version "3.2.28"
+  version "3.2.32"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.2.28.tgz#59b5b37225715bb8fbf5b1c5c989140b10b58710"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.2.32.tgz#f6abcd5a5524e7f33d958acb6e610e29995427bb"
-  integrity sha512-eDPeZpYFRZYQhCulcQAUwFoPk68c8+K9mIsB6QD3oMHmHTDA1P2ZcXvLNqDTIqHB94DqnWinqDf4hTuGYApgPA==
+  integrity sha512-bF0pd17IjYugjll2yKYmb0RM+tfKZcCmRBc4XG2NZ4f/I47QaOovm9RqSw6tfqCFuzRewxR3SWmtmSseUc/e0w==
   dependencies:
     "@anthropic-ai/sdk" "^0.27.3"
     "@budibase/backend-core" "*"
@@ -15600,15 +15600,7 @@ only@~0.0.2:
   resolved "https://registry.yarnpkg.com/only/-/only-0.0.2.tgz#2afde84d03e50b9a8edc444e30610a70295edfb4"
   integrity sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ==
 
-open@^7.3.1:
+open@8.4.2, open@^8.0.0, open@^8.4.0, open@~8.4.0:
-  version "7.4.2"
-  resolved "https://registry.yarnpkg.com/open/-/open-7.4.2.tgz#b8147e26dcf3e426316c730089fd71edd29c2321"
-  integrity sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==
-  dependencies:
-    is-docker "^2.0.0"
-    is-wsl "^2.1.1"
-
-open@^8.0.0, open@^8.4.0, open@~8.4.0:
   version "8.4.2"
   resolved "https://registry.yarnpkg.com/open/-/open-8.4.2.tgz#5b5ffe2a8f793dcd2aad73e550cb87b59cb084f9"
   integrity sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==
@@ -15617,6 +15609,14 @@ open@^8.0.0, open@^8.4.0, open@~8.4.0:
     is-docker "^2.1.1"
     is-wsl "^2.2.0"
 
+open@^7.3.1:
+  version "7.4.2"
+  resolved "https://registry.yarnpkg.com/open/-/open-7.4.2.tgz#b8147e26dcf3e426316c730089fd71edd29c2321"
+  integrity sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==
+  dependencies:
+    is-docker "^2.0.0"
+    is-wsl "^2.1.1"
+
 openai@4.59.0:
   version "4.59.0"
   resolved "https://registry.yarnpkg.com/openai/-/openai-4.59.0.tgz#3961d11a9afb5920e1bd475948a87969e244fc08"