Merge branch 'master' into fix/postgres-newlines

.github/workflows/deploy-featurebranch.yml

@@ -23,7 +23,6 @@ jobs:
           PAYLOAD_BRANCH: ${{ github.head_ref }}
           PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
           PAYLOAD_LICENSE_TYPE: "free"
-          PAYLOAD_DEPLOY: true
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-deploy

charts/budibase/templates/app-service-deployment.yaml

@@ -184,6 +184,10 @@ spec:
         - name: NODE_DEBUG
           value: {{ .Values.services.apps.nodeDebug | quote }}
         {{ end }}
+        {{ if .Values.services.apps.xssSafeMode }}
+        - name: XSS_SAFE_MODE
+          value: {{ .Values.services.apps.xssSafeMode | quote }}
+        {{ end }}
         {{ if .Values.globals.datadogApmEnabled }}
         - name: DD_LOGS_INJECTION
           value: {{ .Values.globals.datadogApmEnabled | quote }}

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.11",
+  "version": "2.32.13",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/account-portal

@@ -1 +1 @@
-Subproject commit 3e24f6293ff5ee5f9b42822e001504e3bbf19cc0
+Subproject commit 8cd052ce8288f343812a514d06c5a9459b3ba1a8

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -213,17 +213,21 @@ export class DatabaseImpl implements Database {
 
   async getMultiple<T extends Document>(
     ids: string[],
-    opts?: { allowMissing?: boolean }
+    opts?: { allowMissing?: boolean; excludeDocs?: boolean }
   ): Promise<T[]> {
     // get unique
     ids = [...new Set(ids)]
+    const includeDocs = !opts?.excludeDocs
     const response = await this.allDocs<T>({
       keys: ids,
-      include_docs: true,
+      include_docs: includeDocs,
     })
     const rowUnavailable = (row: RowResponse<T>) => {
       // row is deleted - key lookup can return this
-      if (row.doc == null || ("deleted" in row.value && row.value.deleted)) {
+      if (
+        (includeDocs && row.doc == null) ||
+        (row.value && "deleted" in row.value && row.value.deleted)
+      ) {
         return true
       }
       return row.error === "not_found"
@@ -237,7 +241,7 @@ export class DatabaseImpl implements Database {
       const missingIds = missing.map(row => row.key).join(", ")
       throw new Error(`Unable to get documents: ${missingIds}`)
     }
-    return rows.map(row => row.doc!)
+    return rows.map(row => (includeDocs ? row.doc! : row.value))
   }
 
   async remove(idOrDoc: string | Document, rev?: string) {
@@ -371,11 +375,21 @@ export class DatabaseImpl implements Database {
     return this.performCall(() => {
       return async () => {
         const response = await directCouchUrlCall(args)
-        const json = await response.json()
+        const text = await response.text()
         if (response.status > 300) {
+          let json
+          try {
+            json = JSON.parse(text)
+          } catch (err) {
+            console.error(`SQS error: ${text}`)
+            throw new CouchDBError(
+              "error while running SQS query, please try again later",
+              { name: "sqs_error", status: response.status }
+            )
+          }
           throw json
         }
-        return json as T
+        return JSON.parse(text) as T
       }
     })
   }

packages/backend-core/src/features/features.ts

@@ -0,0 +1,300 @@
+import env from "../environment"
+import * as context from "../context"
+import { PostHog, PostHogOptions } from "posthog-node"
+import { FeatureFlag, IdentityType, UserCtx } from "@budibase/types"
+import tracer from "dd-trace"
+import { Duration } from "../utils"
+
+let posthog: PostHog | undefined
+export function init(opts?: PostHogOptions) {
+  if (
+    env.POSTHOG_TOKEN &&
+    env.POSTHOG_API_HOST &&
+    !env.SELF_HOSTED &&
+    env.POSTHOG_FEATURE_FLAGS_ENABLED
+  ) {
+    console.log("initializing posthog client...")
+    posthog = new PostHog(env.POSTHOG_TOKEN, {
+      host: env.POSTHOG_API_HOST,
+      personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
+      featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(),
+      ...opts,
+    })
+  } else {
+    console.log("posthog disabled")
+  }
+}
+
+export function shutdown() {
+  posthog?.shutdown()
+}
+
+export abstract class Flag<T> {
+  static boolean(defaultValue: boolean): Flag<boolean> {
+    return new BooleanFlag(defaultValue)
+  }
+
+  static string(defaultValue: string): Flag<string> {
+    return new StringFlag(defaultValue)
+  }
+
+  static number(defaultValue: number): Flag<number> {
+    return new NumberFlag(defaultValue)
+  }
+
+  protected constructor(public defaultValue: T) {}
+
+  abstract parse(value: any): T
+}
+
+type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
+
+export type FlagValues<T> = {
+  [K in keyof T]: UnwrapFlag<T[K]>
+}
+
+type KeysOfType<T, U> = {
+  [K in keyof T]: T[K] extends Flag<U> ? K : never
+}[keyof T]
+
+class BooleanFlag extends Flag<boolean> {
+  parse(value: any) {
+    if (typeof value === "string") {
+      return ["true", "t", "1"].includes(value.toLowerCase())
+    }
+
+    if (typeof value === "boolean") {
+      return value
+    }
+
+    throw new Error(`could not parse value "${value}" as boolean`)
+  }
+}
+
+class StringFlag extends Flag<string> {
+  parse(value: any) {
+    if (typeof value === "string") {
+      return value
+    }
+    throw new Error(`could not parse value "${value}" as string`)
+  }
+}
+
+class NumberFlag extends Flag<number> {
+  parse(value: any) {
+    if (typeof value === "number") {
+      return value
+    }
+
+    if (typeof value === "string") {
+      const parsed = parseFloat(value)
+      if (!isNaN(parsed)) {
+        return parsed
+      }
+    }
+
+    throw new Error(`could not parse value "${value}" as number`)
+  }
+}
+
+export interface EnvFlagEntry {
+  tenantId: string
+  key: string
+  value: boolean
+}
+
+export function parseEnvFlags(flags: string): EnvFlagEntry[] {
+  const split = flags.split(",").map(x => x.split(":"))
+  const result: EnvFlagEntry[] = []
+  for (const [tenantId, ...features] of split) {
+    for (let feature of features) {
+      let value = true
+      if (feature.startsWith("!")) {
+        feature = feature.slice(1)
+        value = false
+      }
+      result.push({ tenantId, key: feature, value })
+    }
+  }
+  return result
+}
+
+export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
+  // This is used to safely cache flags sets in the current request context.
+  // Because multiple sets could theoretically exist, we don't want the cache of
+  // one to leak into another.
+  private readonly setId: string
+
+  constructor(private readonly flagSchema: T) {
+    this.setId = crypto.randomUUID()
+  }
+
+  defaults(): FlagValues<T> {
+    return Object.keys(this.flagSchema).reduce((acc, key) => {
+      const typedKey = key as keyof T
+      acc[typedKey] = this.flagSchema[key].defaultValue
+      return acc
+    }, {} as FlagValues<T>)
+  }
+
+  isFlagName(name: string | number | symbol): name is keyof T {
+    return this.flagSchema[name as keyof T] !== undefined
+  }
+
+  async get<K extends keyof T>(
+    key: K,
+    ctx?: UserCtx
+  ): Promise<FlagValues<T>[K]> {
+    const flags = await this.fetch(ctx)
+    return flags[key]
+  }
+
+  async isEnabled<K extends KeysOfType<T, boolean>>(
+    key: K,
+    ctx?: UserCtx
+  ): Promise<boolean> {
+    const flags = await this.fetch(ctx)
+    return flags[key]
+  }
+
+  async fetch(ctx?: UserCtx): Promise<FlagValues<T>> {
+    return await tracer.trace("features.fetch", async span => {
+      const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
+      if (cachedFlags) {
+        span?.addTags({ fromCache: true })
+        return cachedFlags
+      }
+
+      const tags: Record<string, any> = {}
+      const flagValues = this.defaults()
+      const currentTenantId = context.getTenantId()
+      const specificallySetFalse = new Set<string>()
+
+      for (const { tenantId, key, value } of parseEnvFlags(
+        env.TENANT_FEATURE_FLAGS || ""
+      )) {
+        if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
+          continue
+        }
+
+        tags[`readFromEnvironmentVars`] = true
+
+        if (value === false) {
+          specificallySetFalse.add(key)
+        }
+
+        // ignore unknown flags
+        if (!this.isFlagName(key)) {
+          continue
+        }
+
+        if (typeof flagValues[key] !== "boolean") {
+          throw new Error(`Feature: ${key} is not a boolean`)
+        }
+
+        // @ts-expect-error - TS does not like you writing into a generic type,
+        // but we know that it's okay in this case because it's just an object.
+        flagValues[key as keyof FlagValues] = value
+        tags[`flags.${key}.source`] = "environment"
+      }
+
+      const license = ctx?.user?.license
+      if (license) {
+        tags[`readFromLicense`] = true
+
+        for (const feature of license.features) {
+          if (!this.isFlagName(feature)) {
+            continue
+          }
+
+          if (
+            flagValues[feature] === true ||
+            specificallySetFalse.has(feature)
+          ) {
+            // If the flag is already set to through environment variables, we
+            // don't want to override it back to false here.
+            continue
+          }
+
+          // @ts-expect-error - TS does not like you writing into a generic type,
+          // but we know that it's okay in this case because it's just an object.
+          flagValues[feature] = true
+          tags[`flags.${feature}.source`] = "license"
+        }
+      }
+
+      const identity = context.getIdentity()
+      tags[`identity.type`] = identity?.type
+      tags[`identity.tenantId`] = identity?.tenantId
+      tags[`identity._id`] = identity?._id
+
+      if (posthog && identity?.type === IdentityType.USER) {
+        tags[`readFromPostHog`] = true
+
+        const personProperties: Record<string, string> = {}
+        if (identity.tenantId) {
+          personProperties.tenantId = identity.tenantId
+        }
+
+        const posthogFlags = await posthog.getAllFlagsAndPayloads(
+          identity._id,
+          {
+            personProperties,
+          }
+        )
+
+        for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
+          if (!this.isFlagName(name)) {
+            // We don't want an unexpected PostHog flag to break the app, so we
+            // just log it and continue.
+            console.warn(`Unexpected posthog flag "${name}": ${value}`)
+            continue
+          }
+
+          if (flagValues[name] === true || specificallySetFalse.has(name)) {
+            // If the flag is already set to through environment variables, we
+            // don't want to override it back to false here.
+            continue
+          }
+
+          const payload = posthogFlags.featureFlagPayloads?.[name]
+          const flag = this.flagSchema[name]
+          try {
+            // @ts-expect-error - TS does not like you writing into a generic
+            // type, but we know that it's okay in this case because it's just
+            // an object.
+            flagValues[name] = flag.parse(payload || value)
+            tags[`flags.${name}.source`] = "posthog"
+          } catch (err) {
+            // We don't want an invalid PostHog flag to break the app, so we just
+            // log it and continue.
+            console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
+          }
+        }
+      }
+
+      context.setFeatureFlags(this.setId, flagValues)
+      for (const [key, value] of Object.entries(flagValues)) {
+        tags[`flags.${key}.value`] = value
+      }
+      span?.addTags(tags)
+
+      return flagValues
+    })
+  }
+}
+
+// This is the primary source of truth for feature flags. If you want to add a
+// new flag, add it here and use the `fetch` and `get` functions to access it.
+// All of the machinery in this file is to make sure that flags have their
+// default values set correctly and their types flow through the system.
+export const flags = new FlagSet({
+  DEFAULT_VALUES: Flag.boolean(env.isDev()),
+  AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
+  SQS: Flag.boolean(env.isDev()),
+  [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
+  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
+})
+
+type UnwrapPromise<T> = T extends Promise<infer U> ? U : T
+export type FeatureFlags = UnwrapPromise<ReturnType<typeof flags.fetch>>

packages/backend-core/src/features/index.ts

@@ -1,281 +1,2 @@
-import env from "../environment"
+export * from "./features"
-import * as context from "../context"
+export * as testutils from "./tests/utils"
-import { PostHog, PostHogOptions } from "posthog-node"
-import { FeatureFlag, IdentityType, UserCtx } from "@budibase/types"
-import tracer from "dd-trace"
-import { Duration } from "../utils"
-
-let posthog: PostHog | undefined
-export function init(opts?: PostHogOptions) {
-  if (
-    env.POSTHOG_TOKEN &&
-    env.POSTHOG_API_HOST &&
-    !env.SELF_HOSTED &&
-    env.POSTHOG_FEATURE_FLAGS_ENABLED
-  ) {
-    console.log("initializing posthog client...")
-    posthog = new PostHog(env.POSTHOG_TOKEN, {
-      host: env.POSTHOG_API_HOST,
-      personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
-      featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(),
-      ...opts,
-    })
-  } else {
-    console.log("posthog disabled")
-  }
-}
-
-export function shutdown() {
-  posthog?.shutdown()
-}
-
-export abstract class Flag<T> {
-  static boolean(defaultValue: boolean): Flag<boolean> {
-    return new BooleanFlag(defaultValue)
-  }
-
-  static string(defaultValue: string): Flag<string> {
-    return new StringFlag(defaultValue)
-  }
-
-  static number(defaultValue: number): Flag<number> {
-    return new NumberFlag(defaultValue)
-  }
-
-  protected constructor(public defaultValue: T) {}
-
-  abstract parse(value: any): T
-}
-
-type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
-
-export type FlagValues<T> = {
-  [K in keyof T]: UnwrapFlag<T[K]>
-}
-
-type KeysOfType<T, U> = {
-  [K in keyof T]: T[K] extends Flag<U> ? K : never
-}[keyof T]
-
-class BooleanFlag extends Flag<boolean> {
-  parse(value: any) {
-    if (typeof value === "string") {
-      return ["true", "t", "1"].includes(value.toLowerCase())
-    }
-
-    if (typeof value === "boolean") {
-      return value
-    }
-
-    throw new Error(`could not parse value "${value}" as boolean`)
-  }
-}
-
-class StringFlag extends Flag<string> {
-  parse(value: any) {
-    if (typeof value === "string") {
-      return value
-    }
-    throw new Error(`could not parse value "${value}" as string`)
-  }
-}
-
-class NumberFlag extends Flag<number> {
-  parse(value: any) {
-    if (typeof value === "number") {
-      return value
-    }
-
-    if (typeof value === "string") {
-      const parsed = parseFloat(value)
-      if (!isNaN(parsed)) {
-        return parsed
-      }
-    }
-
-    throw new Error(`could not parse value "${value}" as number`)
-  }
-}
-
-export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
-  // This is used to safely cache flags sets in the current request context.
-  // Because multiple sets could theoretically exist, we don't want the cache of
-  // one to leak into another.
-  private readonly setId: string
-
-  constructor(private readonly flagSchema: T) {
-    this.setId = crypto.randomUUID()
-  }
-
-  defaults(): FlagValues<T> {
-    return Object.keys(this.flagSchema).reduce((acc, key) => {
-      const typedKey = key as keyof T
-      acc[typedKey] = this.flagSchema[key].defaultValue
-      return acc
-    }, {} as FlagValues<T>)
-  }
-
-  isFlagName(name: string | number | symbol): name is keyof T {
-    return this.flagSchema[name as keyof T] !== undefined
-  }
-
-  async get<K extends keyof T>(
-    key: K,
-    ctx?: UserCtx
-  ): Promise<FlagValues<T>[K]> {
-    const flags = await this.fetch(ctx)
-    return flags[key]
-  }
-
-  async isEnabled<K extends KeysOfType<T, boolean>>(
-    key: K,
-    ctx?: UserCtx
-  ): Promise<boolean> {
-    const flags = await this.fetch(ctx)
-    return flags[key]
-  }
-
-  async fetch(ctx?: UserCtx): Promise<FlagValues<T>> {
-    return await tracer.trace("features.fetch", async span => {
-      const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
-      if (cachedFlags) {
-        span?.addTags({ fromCache: true })
-        return cachedFlags
-      }
-
-      const tags: Record<string, any> = {}
-      const flagValues = this.defaults()
-      const currentTenantId = context.getTenantId()
-      const specificallySetFalse = new Set<string>()
-
-      const split = (env.TENANT_FEATURE_FLAGS || "")
-        .split(",")
-        .map(x => x.split(":"))
-      for (const [tenantId, ...features] of split) {
-        if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
-          continue
-        }
-
-        tags[`readFromEnvironmentVars`] = true
-
-        for (let feature of features) {
-          let value = true
-          if (feature.startsWith("!")) {
-            feature = feature.slice(1)
-            value = false
-            specificallySetFalse.add(feature)
-          }
-
-          // ignore unknown flags
-          if (!this.isFlagName(feature)) {
-            continue
-          }
-
-          if (typeof flagValues[feature] !== "boolean") {
-            throw new Error(`Feature: ${feature} is not a boolean`)
-          }
-
-          // @ts-expect-error - TS does not like you writing into a generic type,
-          // but we know that it's okay in this case because it's just an object.
-          flagValues[feature as keyof FlagValues] = value
-          tags[`flags.${feature}.source`] = "environment"
-        }
-      }
-
-      const license = ctx?.user?.license
-      if (license) {
-        tags[`readFromLicense`] = true
-
-        for (const feature of license.features) {
-          if (!this.isFlagName(feature)) {
-            continue
-          }
-
-          if (
-            flagValues[feature] === true ||
-            specificallySetFalse.has(feature)
-          ) {
-            // If the flag is already set to through environment variables, we
-            // don't want to override it back to false here.
-            continue
-          }
-
-          // @ts-expect-error - TS does not like you writing into a generic type,
-          // but we know that it's okay in this case because it's just an object.
-          flagValues[feature] = true
-          tags[`flags.${feature}.source`] = "license"
-        }
-      }
-
-      const identity = context.getIdentity()
-      tags[`identity.type`] = identity?.type
-      tags[`identity.tenantId`] = identity?.tenantId
-      tags[`identity._id`] = identity?._id
-
-      if (posthog && identity?.type === IdentityType.USER) {
-        tags[`readFromPostHog`] = true
-
-        const personProperties: Record<string, string> = {}
-        if (identity.tenantId) {
-          personProperties.tenantId = identity.tenantId
-        }
-
-        const posthogFlags = await posthog.getAllFlagsAndPayloads(
-          identity._id,
-          {
-            personProperties,
-          }
-        )
-
-        for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
-          if (!this.isFlagName(name)) {
-            // We don't want an unexpected PostHog flag to break the app, so we
-            // just log it and continue.
-            console.warn(`Unexpected posthog flag "${name}": ${value}`)
-            continue
-          }
-
-          if (flagValues[name] === true || specificallySetFalse.has(name)) {
-            // If the flag is already set to through environment variables, we
-            // don't want to override it back to false here.
-            continue
-          }
-
-          const payload = posthogFlags.featureFlagPayloads?.[name]
-          const flag = this.flagSchema[name]
-          try {
-            // @ts-expect-error - TS does not like you writing into a generic
-            // type, but we know that it's okay in this case because it's just
-            // an object.
-            flagValues[name] = flag.parse(payload || value)
-            tags[`flags.${name}.source`] = "posthog"
-          } catch (err) {
-            // We don't want an invalid PostHog flag to break the app, so we just
-            // log it and continue.
-            console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
-          }
-        }
-      }
-
-      context.setFeatureFlags(this.setId, flagValues)
-      for (const [key, value] of Object.entries(flagValues)) {
-        tags[`flags.${key}.value`] = value
-      }
-      span?.addTags(tags)
-
-      return flagValues
-    })
-  }
-}
-
-// This is the primary source of truth for feature flags. If you want to add a
-// new flag, add it here and use the `fetch` and `get` functions to access it.
-// All of the machinery in this file is to make sure that flags have their
-// default values set correctly and their types flow through the system.
-export const flags = new FlagSet({
-  DEFAULT_VALUES: Flag.boolean(env.isDev()),
-  AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
-  SQS: Flag.boolean(env.isDev()),
-  [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
-  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
-})

packages/backend-core/src/features/tests/utils.ts

@@ -0,0 +1,64 @@
+import { FeatureFlags, parseEnvFlags } from ".."
+import { setEnv } from "../../environment"
+
+function getCurrentFlags(): Record<string, Record<string, boolean>> {
+  const result: Record<string, Record<string, boolean>> = {}
+  for (const { tenantId, key, value } of parseEnvFlags(
+    process.env.TENANT_FEATURE_FLAGS || ""
+  )) {
+    const tenantFlags = result[tenantId] || {}
+    // Don't allow overwriting specifically false flags, to match the beheaviour
+    // of FlagSet.
+    if (tenantFlags[key] === false) {
+      continue
+    }
+    tenantFlags[key] = value
+    result[tenantId] = tenantFlags
+  }
+  return result
+}
+
+function buildFlagString(
+  flags: Record<string, Record<string, boolean>>
+): string {
+  const parts: string[] = []
+  for (const [tenantId, tenantFlags] of Object.entries(flags)) {
+    for (const [key, value] of Object.entries(tenantFlags)) {
+      if (value === false) {
+        parts.push(`${tenantId}:!${key}`)
+      } else {
+        parts.push(`${tenantId}:${key}`)
+      }
+    }
+  }
+  return parts.join(",")
+}
+
+export function setFeatureFlags(
+  tenantId: string,
+  flags: Partial<FeatureFlags>
+): () => void {
+  const current = getCurrentFlags()
+  for (const [key, value] of Object.entries(flags)) {
+    const tenantFlags = current[tenantId] || {}
+    tenantFlags[key] = value
+    current[tenantId] = tenantFlags
+  }
+  const flagString = buildFlagString(current)
+  return setEnv({ TENANT_FEATURE_FLAGS: flagString })
+}
+
+export function withFeatureFlags<T>(
+  tenantId: string,
+  flags: Partial<FeatureFlags>,
+  f: () => T
+) {
+  const cleanup = setFeatureFlags(tenantId, flags)
+  const result = f()
+  if (result instanceof Promise) {
+    return result.finally(cleanup)
+  } else {
+    cleanup()
+    return result
+  }
+}

packages/backend-core/src/middleware/passport/sso/sso.ts

@@ -2,7 +2,6 @@ import { generateGlobalUserID } from "../../../db"
 import { authError } from "../utils"
 import * as users from "../../../users"
 import * as context from "../../../context"
-import fetch from "node-fetch"
 import {
   SaveSSOUserFunction,
   SSOAuthDetails,
@@ -97,28 +96,13 @@ export async function authenticate(
   return done(null, ssoUser)
 }
 
-async function getProfilePictureUrl(user: User, details: SSOAuthDetails) {
-  const pictureUrl = details.profile?._json.picture
-  if (pictureUrl) {
-    const response = await fetch(pictureUrl)
-    if (response.status === 200) {
-      const type = response.headers.get("content-type") as string
-      if (type.startsWith("image/")) {
-        return pictureUrl
-      }
-    }
-  }
-}
-
 /**
  * @returns a user that has been sync'd with third party information
  */
 async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
   let firstName
   let lastName
-  let pictureUrl
   let oauth2
-  let thirdPartyProfile
 
   if (details.profile) {
     const profile = details.profile
@@ -134,12 +118,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
         lastName = name.familyName
       }
     }
-
-    pictureUrl = await getProfilePictureUrl(user, details)
-
-    thirdPartyProfile = {
-      ...profile._json,
-    }
   }
 
   // oauth tokens for future use
@@ -155,8 +133,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
     providerType: details.providerType,
     firstName,
     lastName,
-    thirdPartyProfile,
-    pictureUrl,
     oauth2,
   }
 }

packages/backend-core/src/sql/sql.ts

@@ -139,29 +139,61 @@ class InternalBuilder {
     return this.table.schema[column]
   }
 
-  // Takes a string like foo and returns a quoted string like [foo] for SQL Server
+  private quoteChars(): [string, string] {
-  // and "foo" for Postgres.
-  private quote(str: string): string {
     switch (this.client) {
-      case SqlClient.SQL_LITE:
       case SqlClient.ORACLE:
       case SqlClient.POSTGRES:
-        return `"${str}"`
+        return ['"', '"']
       case SqlClient.MS_SQL:
-        return `[${str}]`
+        return ["[", "]"]
       case SqlClient.MARIADB:
       case SqlClient.MY_SQL:
-        return `\`${str}\``
+      case SqlClient.SQL_LITE:
+        return ["`", "`"]
     }
   }
 
-  // Takes a string like a.b.c and returns a quoted identifier like [a].[b].[c]
+  // Takes a string like foo and returns a quoted string like [foo] for SQL Server
-  // for SQL Server and `a`.`b`.`c` for MySQL.
+  // and "foo" for Postgres.
-  private quotedIdentifier(key: string): string {
+  private quote(str: string): string {
-    return key
+    const [start, end] = this.quoteChars()
-      .split(".")
+    return `${start}${str}${end}`
-      .map(part => this.quote(part))
+  }
-      .join(".")
+
+  private isQuoted(key: string): boolean {
+    const [start, end] = this.quoteChars()
+    return key.startsWith(start) && key.endsWith(end)
+  }
+
+  // Takes a string like a.b.c or an array like ["a", "b", "c"] and returns a
+  // quoted identifier like [a].[b].[c] for SQL Server and `a`.`b`.`c` for
+  // MySQL.
+  private quotedIdentifier(key: string | string[]): string {
+    if (!Array.isArray(key)) {
+      key = this.splitIdentifier(key)
+    }
+    return key.map(part => this.quote(part)).join(".")
+  }
+
+  // Turns an identifier like a.b.c or `a`.`b`.`c` into ["a", "b", "c"]
+  private splitIdentifier(key: string): string[] {
+    const [start, end] = this.quoteChars()
+    if (this.isQuoted(key)) {
+      return key.slice(1, -1).split(`${end}.${start}`)
+    }
+    return key.split(".")
+  }
+
+  private qualifyIdentifier(key: string): string {
+    const tableName = this.getTableName()
+    const parts = this.splitIdentifier(key)
+    if (parts[0] !== tableName) {
+      parts.unshift(tableName)
+    }
+    if (this.isQuoted(key)) {
+      return this.quotedIdentifier(parts)
+    }
+    return parts.join(".")
   }
 
   private isFullSelectStatementRequired(): boolean {
@@ -231,8 +263,13 @@ class InternalBuilder {
   // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
   // so when we use them we need to wrap them in to_char(). This function
   // converts a field name to the appropriate identifier.
-  private convertClobs(field: string): string {
+  private convertClobs(field: string, opts?: { forSelect?: boolean }): string {
-    const parts = field.split(".")
+    if (this.client !== SqlClient.ORACLE) {
+      throw new Error(
+        "you've called convertClobs on a DB that's not Oracle, this is a mistake"
+      )
+    }
+    const parts = this.splitIdentifier(field)
     const col = parts.pop()!
     const schema = this.table.schema[col]
     let identifier = this.quotedIdentifier(field)
@@ -244,7 +281,11 @@ class InternalBuilder {
       schema.type === FieldType.OPTIONS ||
       schema.type === FieldType.BARCODEQR
     ) {
-      identifier = `to_char(${identifier})`
+      if (opts?.forSelect) {
+        identifier = `to_char(${identifier}) as ${this.quotedIdentifier(col)}`
+      } else {
+        identifier = `to_char(${identifier})`
+      }
     }
     return identifier
   }
@@ -859,28 +900,58 @@ class InternalBuilder {
     const fields = this.query.resource?.fields || []
     const tableName = this.getTableName()
     if (fields.length > 0) {
-      query = query.groupBy(fields.map(field => `${tableName}.${field}`))
+      const qualifiedFields = fields.map(field => this.qualifyIdentifier(field))
-      query = query.select(fields.map(field => `${tableName}.${field}`))
+      if (this.client === SqlClient.ORACLE) {
+        const groupByFields = qualifiedFields.map(field =>
+          this.convertClobs(field)
+        )
+        const selectFields = qualifiedFields.map(field =>
+          this.convertClobs(field, { forSelect: true })
+        )
+        query = query
+          .groupByRaw(groupByFields.join(", "))
+          .select(this.knex.raw(selectFields.join(", ")))
+      } else {
+        query = query.groupBy(qualifiedFields).select(qualifiedFields)
+      }
     }
     for (const aggregation of aggregations) {
       const op = aggregation.calculationType
-      const field = `${tableName}.${aggregation.field} as ${aggregation.name}`
+      if (op === CalculationType.COUNT) {
-      switch (op) {
+        if ("distinct" in aggregation && aggregation.distinct) {
-        case CalculationType.COUNT:
+          if (this.client === SqlClient.ORACLE) {
-          query = query.count(field)
+            const field = this.convertClobs(`${tableName}.${aggregation.field}`)
-          break
+            query = query.select(
-        case CalculationType.SUM:
+              this.knex.raw(
-          query = query.sum(field)
+                `COUNT(DISTINCT ${field}) as ${this.quotedIdentifier(
-          break
+                  aggregation.name
-        case CalculationType.AVG:
+                )}`
-          query = query.avg(field)
+              )
-          break
+            )
-        case CalculationType.MIN:
+          } else {
-          query = query.min(field)
+            query = query.countDistinct(
-          break
+              `${tableName}.${aggregation.field} as ${aggregation.name}`
-        case CalculationType.MAX:
+            )
-          query = query.max(field)
+          }
-          break
+        } else {
+          query = query.count(`* as ${aggregation.name}`)
+        }
+      } else {
+        const field = `${tableName}.${aggregation.field} as ${aggregation.name}`
+        switch (op) {
+          case CalculationType.SUM:
+            query = query.sum(field)
+            break
+          case CalculationType.AVG:
+            query = query.avg(field)
+            break
+          case CalculationType.MIN:
+            query = query.min(field)
+            break
+          case CalculationType.MAX:
+            query = query.max(field)
+            break
+        }
       }
     }
     return query

packages/backend-core/src/users/users.ts

@@ -24,6 +24,7 @@ import * as context from "../context"
 import { getGlobalDB } from "../context"
 import { isCreator } from "./utils"
 import { UserDB } from "./db"
+import { dataFilters } from "@budibase/shared-core"
 
 type GetOpts = { cleanup?: boolean }
 
@@ -262,10 +263,17 @@ export async function paginatedUsers({
     userList = await bulkGetGlobalUsersById(query?.oneOf?._id, {
       cleanup: true,
     })
+  } else if (query) {
+    // TODO: this should use SQS search, but the logic is built in the 'server' package. Using the in-memory filtering to get this working meanwhile
+    const response = await db.allDocs<User>(
+      getGlobalUserParams(null, { ...opts, limit: undefined })
+    )
+    userList = response.rows.map(row => row.doc!)
+    userList = dataFilters.search(userList, { query, limit: opts.limit }).rows
   } else {
     // no search, query allDocs
-    const response = await db.allDocs(getGlobalUserParams(null, opts))
+    const response = await db.allDocs<User>(getGlobalUserParams(null, opts))
-    userList = response.rows.map((row: any) => row.doc)
+    userList = response.rows.map(row => row.doc!)
   }
   return pagination(userList, pageSize, {
     paginate: true,

packages/backend-core/tests/core/utilities/structures/accounts.ts

@@ -6,9 +6,6 @@ import {
   AccountSSOProviderType,
   AuthType,
   CloudAccount,
-  CreateAccount,
-  CreatePassswordAccount,
-  CreateVerifiableSSOAccount,
   Hosting,
   SSOAccount,
 } from "@budibase/types"
@@ -19,6 +16,7 @@ export const account = (partial: Partial<Account> = {}): Account => {
     accountId: uuid(),
     tenantId: generator.word(),
     email: generator.email({ domain: "example.com" }),
+    accountName: generator.word(),
     tenantName: generator.word(),
     hosting: Hosting.SELF,
     createdAt: Date.now(),
@@ -61,10 +59,8 @@ export function ssoAccount(account: Account = cloudAccount()): SSOAccount {
       accessToken: generator.string(),
       refreshToken: generator.string(),
     },
-    pictureUrl: generator.url(),
     provider: provider(),
     providerType: providerType(),
-    thirdPartyProfile: {},
   }
 }
 
@@ -78,68 +74,7 @@ export function verifiableSsoAccount(
       accessToken: generator.string(),
       refreshToken: generator.string(),
     },
-    pictureUrl: generator.url(),
     provider: AccountSSOProvider.MICROSOFT,
     providerType: AccountSSOProviderType.MICROSOFT,
-    thirdPartyProfile: { id: "abc123" },
   }
 }
-
-export const cloudCreateAccount: CreatePassswordAccount = {
-  email: "cloud@budibase.com",
-  tenantId: "cloud",
-  hosting: Hosting.CLOUD,
-  authType: AuthType.PASSWORD,
-  password: "Password123!",
-  tenantName: "cloud",
-  name: "Budi Armstrong",
-  size: "10+",
-  profession: "Software Engineer",
-}
-
-export const cloudSSOCreateAccount: CreateAccount = {
-  email: "cloud-sso@budibase.com",
-  tenantId: "cloud-sso",
-  hosting: Hosting.CLOUD,
-  authType: AuthType.SSO,
-  tenantName: "cloudsso",
-  name: "Budi Armstrong",
-  size: "10+",
-  profession: "Software Engineer",
-}
-
-export const cloudVerifiableSSOCreateAccount: CreateVerifiableSSOAccount = {
-  email: "cloud-sso@budibase.com",
-  tenantId: "cloud-sso",
-  hosting: Hosting.CLOUD,
-  authType: AuthType.SSO,
-  tenantName: "cloudsso",
-  name: "Budi Armstrong",
-  size: "10+",
-  profession: "Software Engineer",
-  provider: AccountSSOProvider.MICROSOFT,
-  thirdPartyProfile: { id: "abc123" },
-}
-
-export const selfCreateAccount: CreatePassswordAccount = {
-  email: "self@budibase.com",
-  tenantId: "self",
-  hosting: Hosting.SELF,
-  authType: AuthType.PASSWORD,
-  password: "Password123!",
-  tenantName: "self",
-  name: "Budi Armstrong",
-  size: "10+",
-  profession: "Software Engineer",
-}
-
-export const selfSSOCreateAccount: CreateAccount = {
-  email: "self-sso@budibase.com",
-  tenantId: "self-sso",
-  hosting: Hosting.SELF,
-  authType: AuthType.SSO,
-  tenantName: "selfsso",
-  name: "Budi Armstrong",
-  size: "10+",
-  profession: "Software Engineer",
-}

packages/backend-core/tests/core/utilities/structures/users.ts

@@ -25,7 +25,6 @@ export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
     roles: { app_test: "admin" },
     firstName: generator.first(),
     lastName: generator.last(),
-    pictureUrl: "http://example.com",
     tenantId: tenant.id(),
     ...userProps,
   }
@@ -86,9 +85,5 @@ export function ssoUser(
     oauth2: opts.details?.oauth2,
     provider: opts.details?.provider!,
     providerType: opts.details?.providerType!,
-    thirdPartyProfile: {
-      email: base.email,
-      picture: base.pictureUrl,
-    },
   }
 }

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -66,6 +66,7 @@
   let insertAtPos
   let targetMode = null
   let expressionResult
+  let expressionError
   let evaluating = false
 
   $: useSnippets = allowSnippets && !$licensing.isFreePlan
@@ -142,10 +143,22 @@
   }
 
   const debouncedEval = Utils.debounce((expression, context, snippets) => {
-    expressionResult = processStringSync(expression || "", {
+    try {
-      ...context,
+      expressionError = null
-      snippets,
+      expressionResult = processStringSync(
-    })
+        expression || "",
+        {
+          ...context,
+          snippets,
+        },
+        {
+          noThrow: false,
+        }
+      )
+    } catch (err) {
+      expressionResult = null
+      expressionError = err
+    }
     evaluating = false
   }, 260)
 
@@ -370,6 +383,7 @@
       {:else if sidePanel === SidePanels.Evaluation}
         <EvaluationSidePanel
           {expressionResult}
+          {expressionError}
           {evaluating}
           expression={editorValue}
         />

packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte

@@ -3,26 +3,37 @@
   import { Icon, ProgressCircle, notifications } from "@budibase/bbui"
   import { copyToClipboard } from "@budibase/bbui/helpers"
   import { fade } from "svelte/transition"
+  import { UserScriptError } from "@budibase/string-templates"
 
   export let expressionResult
+  export let expressionError
   export let evaluating = false
   export let expression = null
 
-  $: error = expressionResult === "Error while executing JS"
+  $: error = expressionError != null
   $: empty = expression == null || expression?.trim() === ""
   $: success = !error && !empty
   $: highlightedResult = highlight(expressionResult)
 
+  const formatError = err => {
+    if (err.code === UserScriptError.code) {
+      return err.userScriptError.toString()
+    }
+    return err.toString()
+  }
+
   const highlight = json => {
     if (json == null) {
       return ""
     }
-    // Attempt to parse and then stringify, in case this is valid JSON
+
+    // Attempt to parse and then stringify, in case this is valid result
     try {
       json = JSON.stringify(JSON.parse(json), null, 2)
     } catch (err) {
       // Ignore
     }
+
     return formatHighlight(json, {
       keyColor: "#e06c75",
       numberColor: "#e5c07b",
@@ -34,7 +45,7 @@
   }
 
   const copy = () => {
-    let clipboardVal = expressionResult
+    let clipboardVal = expressionResult.result
     if (typeof clipboardVal === "object") {
       clipboardVal = JSON.stringify(clipboardVal, null, 2)
     }
@@ -73,6 +84,8 @@
   <div class="body">
     {#if empty}
       Your expression will be evaluated here
+    {:else if error}
+      {formatError(expressionError)}
     {:else}
       <!-- eslint-disable-next-line svelte/no-at-html-tags-->
       {@html highlightedResult}

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -26,6 +26,7 @@
     licensing,
     environment,
     enrichedApps,
+    sortBy,
   } from "stores/portal"
   import { goto } from "@roxi/routify"
   import AppRow from "components/start/AppRow.svelte"
@@ -247,7 +248,7 @@
             <div class="app-actions">
               <Select
                 autoWidth
-                value={$appsStore.sortBy}
+                value={$sortBy}
                 on:change={e => {
                   appsStore.updateSort(e.detail)
                 }}

packages/builder/src/stores/portal/apps.js

@@ -9,7 +9,6 @@ const DEV_PROPS = ["updatedBy", "updatedAt"]
 
 export const INITIAL_APPS_STATE = {
   apps: [],
-  sortBy: "name",
 }
 
 export class AppsStore extends BudiStore {
@@ -53,6 +52,15 @@ export class AppsStore extends BudiStore {
       ...state,
       sortBy,
     }))
+    this.updateUserSort(sortBy)
+  }
+
+  async updateUserSort(sortBy) {
+    try {
+      await auth.updateSelf({ appSort: sortBy })
+    } catch (err) {
+      console.error("couldn't save user sort: ", err)
+    }
   }
 
   async load() {
@@ -140,43 +148,50 @@ export class AppsStore extends BudiStore {
 
 export const appsStore = new AppsStore()
 
-// Centralise any logic that enriches the apps list
+export const sortBy = derived([appsStore, auth], ([$store, $auth]) => {
-export const enrichedApps = derived([appsStore, auth], ([$store, $auth]) => {
+  return $store.sortBy || $auth.user?.appSort || "name"
-  const enrichedApps = $store.apps
+})
-    ? $store.apps.map(app => ({
-        ...app,
-        deployed: app.status === AppStatus.DEPLOYED,
-        lockedYou: app.lockedBy && app.lockedBy.email === $auth.user?.email,
-        lockedOther: app.lockedBy && app.lockedBy.email !== $auth.user?.email,
-        favourite: $auth.user?.appFavourites?.includes(app.appId),
-      }))
-    : []
 
-  if ($store.sortBy === "status") {
+// Centralise any logic that enriches the apps list
-    return enrichedApps.sort((a, b) => {
+export const enrichedApps = derived(
-      if (a.favourite === b.favourite) {
+  [appsStore, auth, sortBy],
-        if (a.status === b.status) {
+  ([$store, $auth, $sortBy]) => {
+    const enrichedApps = $store.apps
+      ? $store.apps.map(app => ({
+          ...app,
+          deployed: app.status === AppStatus.DEPLOYED,
+          lockedYou: app.lockedBy && app.lockedBy.email === $auth.user?.email,
+          lockedOther: app.lockedBy && app.lockedBy.email !== $auth.user?.email,
+          favourite: $auth.user?.appFavourites?.includes(app.appId),
+        }))
+      : []
+
+    if ($sortBy === "status") {
+      return enrichedApps.sort((a, b) => {
+        if (a.favourite === b.favourite) {
+          if (a.status === b.status) {
+            return a.name?.toLowerCase() < b.name?.toLowerCase() ? -1 : 1
+          }
+          return a.status === AppStatus.DEPLOYED ? -1 : 1
+        }
+        return a.favourite ? -1 : 1
+      })
+    } else if ($sortBy === "updated") {
+      return enrichedApps?.sort((a, b) => {
+        if (a.favourite === b.favourite) {
+          const aUpdated = a.updatedAt || "9999"
+          const bUpdated = b.updatedAt || "9999"
+          return aUpdated < bUpdated ? 1 : -1
+        }
+        return a.favourite ? -1 : 1
+      })
+    } else {
+      return enrichedApps?.sort((a, b) => {
+        if (a.favourite === b.favourite) {
           return a.name?.toLowerCase() < b.name?.toLowerCase() ? -1 : 1
         }
-        return a.status === AppStatus.DEPLOYED ? -1 : 1
+        return a.favourite ? -1 : 1
-      }
+      })
-      return a.favourite ? -1 : 1
+    }
-    })
-  } else if ($store.sortBy === "updated") {
-    return enrichedApps?.sort((a, b) => {
-      if (a.favourite === b.favourite) {
-        const aUpdated = a.updatedAt || "9999"
-        const bUpdated = b.updatedAt || "9999"
-        return aUpdated < bUpdated ? 1 : -1
-      }
-      return a.favourite ? -1 : 1
-    })
-  } else {
-    return enrichedApps?.sort((a, b) => {
-      if (a.favourite === b.favourite) {
-        return a.name?.toLowerCase() < b.name?.toLowerCase() ? -1 : 1
-      }
-      return a.favourite ? -1 : 1
-    })
   }
-})
+)

packages/builder/src/stores/portal/index.js

@@ -3,7 +3,7 @@ import { writable } from "svelte/store"
 export { organisation } from "./organisation"
 export { users } from "./users"
 export { admin } from "./admin"
-export { appsStore, enrichedApps } from "./apps"
+export { appsStore, enrichedApps, sortBy } from "./apps"
 export { email } from "./email"
 export { auth } from "./auth"
 export { oidc } from "./oidc"

packages/frontend-core/src/api/tables.js

@@ -40,7 +40,7 @@ export const buildTableEndpoints = API => ({
     sortType,
     paginate,
   }) => {
-    if (!tableId || !query) {
+    if (!tableId) {
       return {
         rows: [],
       }

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -696,9 +696,8 @@ export class ExternalRequest<T extends Operation> {
       const calculationFields = helpers.views.calculationFields(this.source)
       for (const [key, field] of Object.entries(calculationFields)) {
         aggregations.push({
+          ...field,
           name: key,
-          field: field.field,
-          calculationType: field.calculationType,
         })
       }
     }

packages/server/src/api/controllers/rowAction/crud.ts

@@ -1,6 +1,7 @@
 import {
   CreateRowActionRequest,
   Ctx,
+  RowActionPermissions,
   RowActionResponse,
   RowActionsResponse,
   UpdateRowActionRequest,
@@ -18,25 +19,26 @@ async function getTable(ctx: Ctx) {
 
 export async function find(ctx: Ctx<void, RowActionsResponse>) {
   const table = await getTable(ctx)
+  const tableId = table._id!
 
-  if (!(await sdk.rowActions.docExists(table._id!))) {
+  if (!(await sdk.rowActions.docExists(tableId))) {
     ctx.body = {
       actions: {},
     }
     return
   }
 
-  const { actions } = await sdk.rowActions.getAll(table._id!)
+  const { actions } = await sdk.rowActions.getAll(tableId)
   const result: RowActionsResponse = {
     actions: Object.entries(actions).reduce<Record<string, RowActionResponse>>(
       (acc, [key, action]) => ({
         ...acc,
         [key]: {
           id: key,
-          tableId: table._id!,
+          tableId,
           name: action.name,
           automationId: action.automationId,
-          allowedViews: flattenAllowedViews(action.permissions.views),
+          allowedSources: flattenAllowedSources(tableId, action.permissions),
         },
       }),
       {}
@@ -49,17 +51,18 @@ export async function create(
   ctx: Ctx<CreateRowActionRequest, RowActionResponse>
 ) {
   const table = await getTable(ctx)
+  const tableId = table._id!
 
-  const createdAction = await sdk.rowActions.create(table._id!, {
+  const createdAction = await sdk.rowActions.create(tableId, {
     name: ctx.request.body.name,
   })
 
   ctx.body = {
-    tableId: table._id!,
+    tableId,
     id: createdAction.id,
     name: createdAction.name,
     automationId: createdAction.automationId,
-    allowedViews: undefined,
+    allowedSources: flattenAllowedSources(tableId, createdAction.permissions),
   }
   ctx.status = 201
 }
@@ -68,18 +71,19 @@ export async function update(
   ctx: Ctx<UpdateRowActionRequest, RowActionResponse>
 ) {
   const table = await getTable(ctx)
+  const tableId = table._id!
   const { actionId } = ctx.params
 
-  const action = await sdk.rowActions.update(table._id!, actionId, {
+  const action = await sdk.rowActions.update(tableId, actionId, {
     name: ctx.request.body.name,
   })
 
   ctx.body = {
-    tableId: table._id!,
+    tableId,
     id: action.id,
     name: action.name,
     automationId: action.automationId,
-    allowedViews: undefined,
+    allowedSources: flattenAllowedSources(tableId, action.permissions),
   }
 }
 
@@ -91,52 +95,89 @@ export async function remove(ctx: Ctx<void, void>) {
   ctx.status = 204
 }
 
+export async function setTablePermission(ctx: Ctx<void, RowActionResponse>) {
+  const table = await getTable(ctx)
+  const tableId = table._id!
+  const { actionId } = ctx.params
+
+  const action = await sdk.rowActions.setTablePermission(tableId, actionId)
+  ctx.body = {
+    tableId,
+    id: action.id,
+    name: action.name,
+    automationId: action.automationId,
+    allowedSources: flattenAllowedSources(tableId, action.permissions),
+  }
+}
+
+export async function unsetTablePermission(ctx: Ctx<void, RowActionResponse>) {
+  const table = await getTable(ctx)
+  const tableId = table._id!
+  const { actionId } = ctx.params
+
+  const action = await sdk.rowActions.unsetTablePermission(tableId, actionId)
+
+  ctx.body = {
+    tableId,
+    id: action.id,
+    name: action.name,
+    automationId: action.automationId,
+    allowedSources: flattenAllowedSources(tableId, action.permissions),
+  }
+}
+
 export async function setViewPermission(ctx: Ctx<void, RowActionResponse>) {
   const table = await getTable(ctx)
+  const tableId = table._id!
   const { actionId, viewId } = ctx.params
 
   const action = await sdk.rowActions.setViewPermission(
-    table._id!,
+    tableId,
     actionId,
     viewId
   )
   ctx.body = {
-    tableId: table._id!,
+    tableId,
     id: action.id,
     name: action.name,
     automationId: action.automationId,
-    allowedViews: flattenAllowedViews(action.permissions.views),
+    allowedSources: flattenAllowedSources(tableId, action.permissions),
   }
 }
 
 export async function unsetViewPermission(ctx: Ctx<void, RowActionResponse>) {
   const table = await getTable(ctx)
+  const tableId = table._id!
   const { actionId, viewId } = ctx.params
 
   const action = await sdk.rowActions.unsetViewPermission(
-    table._id!,
+    tableId,
     actionId,
     viewId
   )
 
   ctx.body = {
-    tableId: table._id!,
+    tableId,
     id: action.id,
     name: action.name,
     automationId: action.automationId,
-    allowedViews: flattenAllowedViews(action.permissions.views),
+    allowedSources: flattenAllowedSources(tableId, action.permissions),
   }
 }
 
-function flattenAllowedViews(
+function flattenAllowedSources(
-  permissions: Record<string, { runAllowed: boolean }>
+  tableId: string,
+  permissions: RowActionPermissions
 ) {
-  const allowedPermissions = Object.entries(permissions || {})
+  const allowedPermissions = []
-    .filter(([_, p]) => p.runAllowed)
+  if (permissions.table.runAllowed) {
-    .map(([viewId]) => viewId)
+    allowedPermissions.push(tableId)
-  if (!allowedPermissions.length) {
-    return undefined
   }
+  allowedPermissions.push(
+    ...Object.keys(permissions.views || {}).filter(
+      viewId => permissions.views[viewId].runAllowed
+    )
+  )
 
   return allowedPermissions
 }

packages/server/src/api/controllers/script.ts

@@ -1,11 +1,18 @@
 import { Ctx } from "@budibase/types"
 import { IsolatedVM } from "../../jsRunner/vm"
-import { iifeWrapper } from "@budibase/string-templates"
+import { iifeWrapper, UserScriptError } from "@budibase/string-templates"
 
 export async function execute(ctx: Ctx) {
   const { script, context } = ctx.request.body
   const vm = new IsolatedVM()
-  ctx.body = vm.withContext(context, () => vm.execute(iifeWrapper(script)))
+  try {
+    ctx.body = vm.withContext(context, () => vm.execute(iifeWrapper(script)))
+  } catch (err: any) {
+    if (err.code === UserScriptError.code) {
+      throw err.userScriptError
+    }
+    throw err
+  }
 }
 
 export async function save(ctx: Ctx) {

packages/server/src/api/controllers/view/viewsV2.ts

@@ -11,14 +11,40 @@ import {
   ViewCalculationFieldMetadata,
   RelationSchemaField,
   ViewFieldMetadata,
+  CalculationType,
 } from "@budibase/types"
 import { builderSocket, gridSocket } from "../../../websockets"
 import { helpers } from "@budibase/shared-core"
 
 function stripUnknownFields(
-  field: BasicViewFieldMetadata
+  field: ViewFieldMetadata
-): RequiredKeys<BasicViewFieldMetadata> {
+): RequiredKeys<ViewFieldMetadata> {
   if (helpers.views.isCalculationField(field)) {
+    if (field.calculationType === CalculationType.COUNT) {
+      if ("distinct" in field && field.distinct) {
+        return {
+          order: field.order,
+          width: field.width,
+          visible: field.visible,
+          readonly: field.readonly,
+          icon: field.icon,
+          distinct: field.distinct,
+          calculationType: field.calculationType,
+          field: field.field,
+          columns: field.columns,
+        }
+      } else {
+        return {
+          order: field.order,
+          width: field.width,
+          visible: field.visible,
+          readonly: field.readonly,
+          icon: field.icon,
+          calculationType: field.calculationType,
+          columns: field.columns,
+        }
+      }
+    }
     const strippedField: RequiredKeys<ViewCalculationFieldMetadata> = {
       order: field.order,
       width: field.width,
@@ -101,6 +127,7 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
 
   const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version"> = {
     name: view.name,
+    type: view.type,
     tableId: view.tableId,
     query: view.query,
     queryUI: view.queryUI,
@@ -136,6 +163,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
   const parsedView: RequiredKeys<ViewV2> = {
     id: view.id,
     name: view.name,
+    type: view.type,
     version: view.version,
     tableId: view.tableId,
     query: view.query,

packages/server/src/api/routes/rowAction.ts

@@ -51,6 +51,16 @@ router
     authorized(BUILDER),
     rowActionController.remove
   )
+  .post(
+    "/api/tables/:tableId/actions/:actionId/permissions",
+    authorized(BUILDER),
+    rowActionController.setTablePermission
+  )
+  .delete(
+    "/api/tables/:tableId/actions/:actionId/permissions",
+    authorized(BUILDER),
+    rowActionController.unsetTablePermission
+  )
   .post(
     "/api/tables/:tableId/actions/:actionId/permissions/:viewId",
     authorized(BUILDER),

packages/server/src/api/routes/tests/application.spec.ts

@@ -14,12 +14,7 @@ jest.mock("../../../utilities/redis", () => ({
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 import { AppStatus } from "../../../db/utils"
-import {
+import { events, utils, context, features } from "@budibase/backend-core"
-  events,
-  utils,
-  context,
-  withEnv as withCoreEnv,
-} from "@budibase/backend-core"
 import env from "../../../environment"
 import { type App } from "@budibase/types"
 import tk from "timekeeper"
@@ -358,9 +353,13 @@ describe("/applications", () => {
         .delete(`/api/global/roles/${prodAppId}`)
         .reply(200, {})
 
-      await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, async () => {
+      await features.testutils.withFeatureFlags(
-        await config.api.application.delete(app.appId)
+        "*",
-      })
+        { SQS: true },
+        async () => {
+          await config.api.application.delete(app.appId)
+        }
+      )
     })
   })
 

packages/server/src/api/routes/tests/row.spec.ts

@@ -13,8 +13,7 @@ import {
   context,
   InternalTable,
   tenancy,
-  withEnv as withCoreEnv,
+  features,
-  setEnv as setCoreEnv,
 } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
@@ -40,7 +39,6 @@ import {
   TableSchema,
   JsonFieldSubType,
   RowExportFormat,
-  FeatureFlag,
   RelationSchemaField,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
@@ -49,6 +47,7 @@ import * as uuid from "uuid"
 import { Knex } from "knex"
 import { InternalTables } from "../../../db/utils"
 import { withEnv } from "../../../environment"
+import { JsTimeoutError } from "@budibase/string-templates"
 
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
@@ -97,12 +96,12 @@ describe.each([
   let envCleanup: (() => void) | undefined
 
   beforeAll(async () => {
-    await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init())
+    await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
-    if (isLucene) {
+      config.init()
-      envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" })
+    )
-    } else if (isSqs) {
+    envCleanup = features.testutils.setFeatureFlags("*", {
-      envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" })
+      SQS: isSqs,
-    }
+    })
 
     if (dsProvider) {
       const rawDatasource = await dsProvider
@@ -1847,7 +1846,7 @@ describe.each([
     })
 
   describe("exportRows", () => {
-    beforeAll(async () => {
+    beforeEach(async () => {
       table = await config.api.table.save(defaultTable())
     })
 
@@ -1884,6 +1883,16 @@ describe.each([
         })
       })
 
+    it("should allow exporting without filtering", async () => {
+      const existing = await config.api.row.save(table._id!, {})
+      const res = await config.api.row.exportRows(table._id!)
+      const results = JSON.parse(res)
+      expect(results.length).toEqual(1)
+      const row = results[0]
+
+      expect(row._id).toEqual(existing._id)
+    })
+
     it("should allow exporting only certain columns", async () => {
       const existing = await config.api.row.save(table._id!, {})
       const res = await config.api.row.exportRows(table._id!, {
@@ -2516,15 +2525,9 @@ describe.each([
       let flagCleanup: (() => void) | undefined
 
       beforeAll(async () => {
-        const env = {
+        flagCleanup = features.testutils.setFeatureFlags("*", {
-          TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
+          ENRICHED_RELATIONSHIPS: true,
-        }
+        })
-        if (isSqs) {
-          env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:SQS`
-        } else {
-          env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:!SQS`
-        }
-        flagCleanup = setCoreEnv(env)
 
         const aux2Table = await config.api.table.save(saveTableRequest())
         const aux2Data = await config.api.row.save(aux2Table._id!, {})
@@ -2751,9 +2754,10 @@ describe.each([
       it.each(testScenarios)(
         "does not enrich relationships when not enabled (via %s)",
         async (__, retrieveDelegate) => {
-          await withCoreEnv(
+          await features.testutils.withFeatureFlags(
+            "*",
             {
-              TENANT_FEATURE_FLAGS: `*:!${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
+              ENRICHED_RELATIONSHIPS: false,
             },
             async () => {
               const otherRows = _.sampleSize(auxData, 5)
@@ -3013,7 +3017,7 @@ describe.each([
             let i = 0
             for (; i < 10; i++) {
               const row = rows[i]
-              if (row.formula !== "Timed out while executing JS") {
+              if (row.formula !== JsTimeoutError.message) {
                 break
               }
             }
@@ -3027,7 +3031,7 @@ describe.each([
             for (; i < 10; i++) {
               const row = rows[i]
               expect(row.text).toBe("foo")
-              expect(row.formula).toBe("Request JS execution limit hit")
+              expect(row.formula).toStartWith("CPU time limit exceeded ")
             }
           }
         }

packages/server/src/api/routes/tests/rowAction.spec.ts

@@ -133,6 +133,7 @@ describe("/rowsActions", () => {
         id: expect.stringMatching(/^row_action_\w+/),
         tableId: tableId,
         automationId: expectAutomationId(),
+        allowedSources: [tableId],
       })
 
       expect(await config.api.rowAction.find(tableId)).toEqual({
@@ -142,6 +143,7 @@ describe("/rowsActions", () => {
             id: res.id,
             tableId: tableId,
             automationId: expectAutomationId(),
+            allowedSources: [tableId],
           },
         },
       })
@@ -180,18 +182,21 @@ describe("/rowsActions", () => {
             id: responses[0].id,
             tableId,
             automationId: expectAutomationId(),
+            allowedSources: [tableId],
           },
           [responses[1].id]: {
             name: rowActions[1].name,
             id: responses[1].id,
             tableId,
             automationId: expectAutomationId(),
+            allowedSources: [tableId],
           },
           [responses[2].id]: {
             name: rowActions[2].name,
             id: responses[2].id,
             tableId,
             automationId: expectAutomationId(),
+            allowedSources: [tableId],
           },
         },
       })
@@ -224,6 +229,7 @@ describe("/rowsActions", () => {
         id: expect.any(String),
         tableId,
         automationId: expectAutomationId(),
+        allowedSources: [tableId],
       })
 
       expect(await config.api.rowAction.find(tableId)).toEqual({
@@ -233,6 +239,7 @@ describe("/rowsActions", () => {
             id: res.id,
             tableId: tableId,
             automationId: expectAutomationId(),
+            allowedSources: [tableId],
           },
         },
       })
@@ -354,6 +361,7 @@ describe("/rowsActions", () => {
         tableId,
         name: updatedName,
         automationId: actionData.automationId,
+        allowedSources: [tableId],
       })
 
       expect(await config.api.rowAction.find(tableId)).toEqual(
@@ -364,6 +372,7 @@ describe("/rowsActions", () => {
               id: actionData.id,
               tableId: actionData.tableId,
               automationId: actionData.automationId,
+              allowedSources: [tableId],
             },
           }),
         })
@@ -515,6 +524,81 @@ describe("/rowsActions", () => {
     })
   })
 
+  describe("set/unsetTablePermission", () => {
+    describe.each([
+      ["setTablePermission", config.api.rowAction.setTablePermission],
+      ["unsetTablePermission", config.api.rowAction.unsetTablePermission],
+    ])("unauthorisedTests for %s", (__, delegateTest) => {
+      unauthorisedTests((expectations, testConfig) =>
+        delegateTest(tableId, generator.guid(), expectations, testConfig)
+      )
+    })
+
+    let actionId1: string, actionId2: string
+
+    beforeEach(async () => {
+      for (const rowAction of createRowActionRequests(3)) {
+        await createRowAction(tableId, rowAction)
+      }
+      const persisted = await config.api.rowAction.find(tableId)
+
+      const actions = _.sampleSize(Object.keys(persisted.actions), 2)
+      actionId1 = actions[0]
+      actionId2 = actions[1]
+    })
+
+    it("can set table permission", async () => {
+      await config.api.rowAction.unsetTablePermission(tableId, actionId1)
+      await config.api.rowAction.unsetTablePermission(tableId, actionId2)
+      const actionResult = await config.api.rowAction.setTablePermission(
+        tableId,
+        actionId1
+      )
+      const expectedAction1 = expect.objectContaining({
+        allowedSources: [tableId],
+      })
+
+      const expectedActions = expect.objectContaining({
+        [actionId1]: expectedAction1,
+        [actionId2]: expect.objectContaining({
+          allowedSources: [],
+        }),
+      })
+      expect(actionResult).toEqual(expectedAction1)
+      expect((await config.api.rowAction.find(tableId)).actions).toEqual(
+        expectedActions
+      )
+    })
+
+    it("can unset table permission", async () => {
+      const actionResult = await config.api.rowAction.unsetTablePermission(
+        tableId,
+        actionId1
+      )
+
+      const expectedAction = expect.objectContaining({
+        allowedSources: [],
+      })
+      expect(actionResult).toEqual(expectedAction)
+      expect(
+        (await config.api.rowAction.find(tableId)).actions[actionId1]
+      ).toEqual(expectedAction)
+    })
+
+    it.each([
+      ["setTablePermission", config.api.rowAction.setTablePermission],
+      ["unsetTablePermission", config.api.rowAction.unsetTablePermission],
+    ])(
+      "cannot update permission for unexisting tables (%s)",
+      async (__, delegateTest) => {
+        const tableId = generator.guid()
+        await delegateTest(tableId, actionId1, {
+          status: 404,
+        })
+      }
+    )
+  })
+
   describe("set/unsetViewPermission", () => {
     describe.each([
       ["setViewPermission", config.api.rowAction.setViewPermission],
@@ -531,11 +615,9 @@ describe("/rowsActions", () => {
       )
     })
 
-    let tableIdForDescribe: string
     let actionId1: string, actionId2: string
     let viewId1: string, viewId2: string
-    beforeAll(async () => {
+    beforeEach(async () => {
-      tableIdForDescribe = tableId
       for (const rowAction of createRowActionRequests(3)) {
         await createRowAction(tableId, rowAction)
       }
@@ -557,11 +639,6 @@ describe("/rowsActions", () => {
       ).id
     })
 
-    beforeEach(() => {
-      // Hack to reuse tables for these given tests
-      tableId = tableIdForDescribe
-    })
-
     it("can set permission views", async () => {
       await config.api.rowAction.setViewPermission(tableId, viewId1, actionId1)
       const action1Result = await config.api.rowAction.setViewPermission(
@@ -576,10 +653,10 @@ describe("/rowsActions", () => {
       )
 
       const expectedAction1 = expect.objectContaining({
-        allowedViews: [viewId1, viewId2],
+        allowedSources: [tableId, viewId1, viewId2],
       })
       const expectedAction2 = expect.objectContaining({
-        allowedViews: [viewId1],
+        allowedSources: [tableId, viewId1],
       })
 
       const expectedActions = expect.objectContaining({
@@ -594,6 +671,8 @@ describe("/rowsActions", () => {
     })
 
     it("can unset permission views", async () => {
+      await config.api.rowAction.setViewPermission(tableId, viewId2, actionId1)
+      await config.api.rowAction.setViewPermission(tableId, viewId1, actionId2)
       const actionResult = await config.api.rowAction.unsetViewPermission(
         tableId,
         viewId1,
@@ -601,7 +680,7 @@ describe("/rowsActions", () => {
       )
 
       const expectedAction = expect.objectContaining({
-        allowedViews: [viewId2],
+        allowedSources: [tableId, viewId2],
       })
       expect(actionResult).toEqual(expectedAction)
       expect(
@@ -672,6 +751,7 @@ describe("/rowsActions", () => {
       )
 
       await config.publish()
+      // Travel time in order to "trim" the selected `getAutomationLogs`
       tk.travel(Date.now() + 100)
     })
 
@@ -712,6 +792,38 @@ describe("/rowsActions", () => {
       ])
     })
 
+    it("triggers from an allowed table", async () => {
+      expect(await getAutomationLogs()).toBeEmpty()
+      await config.api.rowAction.trigger(tableId, rowAction.id, { rowId })
+
+      const automationLogs = await getAutomationLogs()
+      expect(automationLogs).toEqual([
+        expect.objectContaining({
+          automationId: rowAction.automationId,
+        }),
+      ])
+    })
+
+    it("rejects triggering from a non-allowed table", async () => {
+      await config.api.rowAction.unsetTablePermission(tableId, rowAction.id)
+      await config.publish()
+
+      await config.api.rowAction.trigger(
+        tableId,
+        rowAction.id,
+        { rowId },
+        {
+          status: 403,
+          body: {
+            message: `Row action '${rowAction.id}' is not enabled for table '${tableId}'`,
+          },
+        }
+      )
+
+      const automationLogs = await getAutomationLogs()
+      expect(automationLogs).toEqual([])
+    })
+
     it("rejects triggering from a non-allowed view", async () => {
       const viewId = (
         await config.api.viewV2.create(
@@ -901,7 +1013,7 @@ describe("/rowsActions", () => {
       })
 
       it.each(allowedRoleConfig)(
-        "does not allow running row actions for tables by default even",
+        "allow running row actions for tables by default",
         async (userRole, resourcePermission) => {
           await config.api.permission.add({
             level: PermissionLevel.READ,
@@ -918,15 +1030,12 @@ describe("/rowsActions", () => {
               rowAction.id,
               { rowId },
               {
-                status: 403,
+                status: 200,
-                body: {
-                  message: `Row action '${rowAction.id}' is not enabled for table '${tableId}'`,
-                },
               }
             )
 
             const automationLogs = await getAutomationLogs()
-            expect(automationLogs).toBeEmpty()
+            expect(automationLogs).toHaveLength(1)
           })
         }
       )

packages/server/src/api/routes/tests/search.spec.ts

@@ -7,9 +7,9 @@ import {
 import {
   context,
   db as dbCore,
+  features,
   MAX_VALID_DATE,
   MIN_VALID_DATE,
-  setEnv as setCoreEnv,
   SQLITE_DESIGN_DOC_ID,
   utils,
   withEnv as withCoreEnv,
@@ -94,16 +94,12 @@ describe.each([
   }
 
   beforeAll(async () => {
-    await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init())
+    await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
-    if (isLucene) {
+      config.init()
-      envCleanup = setCoreEnv({
+    )
-        TENANT_FEATURE_FLAGS: "*:!SQS",
+    envCleanup = features.testutils.setFeatureFlags("*", {
-      })
+      SQS: isSqs,
-    } else if (isSqs) {
+    })
-      envCleanup = setCoreEnv({
-        TENANT_FEATURE_FLAGS: "*:SQS",
-      })
-    }
 
     if (config.app?.appId) {
       config.app = await config.api.application.update(config.app?.appId, {
@@ -191,7 +187,6 @@ describe.each([
         if (isInMemory) {
           return dataFilters.search(_.cloneDeep(rows), {
             ...this.query,
-            tableId: tableOrViewId,
           })
         } else {
           return config.api.row.search(tableOrViewId, this.query)

packages/server/src/api/routes/tests/templates.spec.ts

@@ -2,7 +2,7 @@ import * as setup from "./utilities"
 import path from "path"
 import nock from "nock"
 import { generator } from "@budibase/backend-core/tests"
-import { withEnv as withCoreEnv, env as coreEnv } from "@budibase/backend-core"
+import { features } from "@budibase/backend-core"
 
 interface App {
   background: string
@@ -85,41 +85,44 @@ describe("/templates", () => {
     it.each(["sqs", "lucene"])(
       `should be able to create an app from a template (%s)`,
       async source => {
-        const env: Partial<typeof coreEnv> = {
+        await features.testutils.withFeatureFlags(
-          TENANT_FEATURE_FLAGS: source === "sqs" ? "*:SQS" : "",
+          "*",
-        }
+          { SQS: source === "sqs" },
+          async () => {
+            const name = generator.guid().replaceAll("-", "")
+            const url = `/${name}`
 
-        await withCoreEnv(env, async () => {
+            const app = await config.api.application.create({
-          const name = generator.guid().replaceAll("-", "")
+              name,
-          const url = `/${name}`
+              url,
-
+              useTemplate: "true",
-          const app = await config.api.application.create({
+              templateName: "Agency Client Portal",
-            name,
+              templateKey: "app/agency-client-portal",
-            url,
-            useTemplate: "true",
-            templateName: "Agency Client Portal",
-            templateKey: "app/agency-client-portal",
-          })
-          expect(app.name).toBe(name)
-          expect(app.url).toBe(url)
-
-          await config.withApp(app, async () => {
-            const tables = await config.api.table.fetch()
-            expect(tables).toHaveLength(2)
-
-            tables.sort((a, b) => a.name.localeCompare(b.name))
-            const [agencyProjects, users] = tables
-            expect(agencyProjects.name).toBe("Agency Projects")
-            expect(users.name).toBe("Users")
-
-            const { rows } = await config.api.row.search(agencyProjects._id!, {
-              tableId: agencyProjects._id!,
-              query: {},
             })
+            expect(app.name).toBe(name)
+            expect(app.url).toBe(url)
 
-            expect(rows).toHaveLength(3)
+            await config.withApp(app, async () => {
-          })
+              const tables = await config.api.table.fetch()
-        })
+              expect(tables).toHaveLength(2)
+
+              tables.sort((a, b) => a.name.localeCompare(b.name))
+              const [agencyProjects, users] = tables
+              expect(agencyProjects.name).toBe("Agency Projects")
+              expect(users.name).toBe("Users")
+
+              const { rows } = await config.api.row.search(
+                agencyProjects._id!,
+                {
+                  tableId: agencyProjects._id!,
+                  query: {},
+                }
+              )
+
+              expect(rows).toHaveLength(3)
+            })
+          }
+        )
       }
     )
   })

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -22,22 +22,16 @@ import {
   RelationshipType,
   TableSchema,
   RenameColumn,
-  FeatureFlag,
   BBReferenceFieldSubType,
+  NumericCalculationFieldMetadata,
   ViewV2Schema,
-  ViewCalculationFieldMetadata,
+  ViewV2Type,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
 import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
-import {
+import { db, roles, features } from "@budibase/backend-core"
-  db,
-  roles,
-  withEnv as withCoreEnv,
-  setEnv as setCoreEnv,
-  env,
-} from "@budibase/backend-core"
 
 describe.each([
   ["lucene", undefined],
@@ -102,18 +96,13 @@ describe.each([
   }
 
   beforeAll(async () => {
-    await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () =>
+    await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
       config.init()
     )
-    if (isLucene) {
+
-      envCleanup = setCoreEnv({
+    envCleanup = features.testutils.setFeatureFlags("*", {
-        TENANT_FEATURE_FLAGS: "*:!SQS",
+      SQS: isSqs,
-      })
+    })
-    } else if (isSqs) {
-      envCleanup = setCoreEnv({
-        TENANT_FEATURE_FLAGS: "*:SQS",
-      })
-    }
 
     if (dsProvider) {
       datasource = await config.createDatasource({
@@ -155,7 +144,7 @@ describe.each([
       })
 
       it("can persist views with all fields", async () => {
-        const newView: Required<Omit<CreateViewRequest, "queryUI">> = {
+        const newView: Required<Omit<CreateViewRequest, "queryUI" | "type">> = {
           name: generator.name(),
           tableId: table._id!,
           primaryDisplay: "id",
@@ -546,6 +535,7 @@ describe.each([
         let view = await config.api.viewV2.create({
           tableId: table._id!,
           name: generator.guid(),
+          type: ViewV2Type.CALCULATION,
           schema: {
             sum: {
               visible: true,
@@ -557,17 +547,195 @@ describe.each([
 
         expect(Object.keys(view.schema!)).toHaveLength(1)
 
-        let sum = view.schema!.sum as ViewCalculationFieldMetadata
+        let sum = view.schema!.sum as NumericCalculationFieldMetadata
         expect(sum).toBeDefined()
         expect(sum.calculationType).toEqual(CalculationType.SUM)
         expect(sum.field).toEqual("Price")
 
         view = await config.api.viewV2.get(view.id)
-        sum = view.schema!.sum as ViewCalculationFieldMetadata
+        sum = view.schema!.sum as NumericCalculationFieldMetadata
         expect(sum).toBeDefined()
         expect(sum.calculationType).toEqual(CalculationType.SUM)
         expect(sum.field).toEqual("Price")
       })
+
+      it("cannot create a view with calculation fields unless it has the right type", async () => {
+        await config.api.viewV2.create(
+          {
+            tableId: table._id!,
+            name: generator.guid(),
+            schema: {
+              sum: {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "Price",
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              message:
+                "Calculation fields are not allowed in non-calculation views",
+            },
+          }
+        )
+      })
+
+      it("cannot create a calculation view with more than 5 aggregations", async () => {
+        await config.api.viewV2.create(
+          {
+            tableId: table._id!,
+            name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
+            schema: {
+              sum: {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "Price",
+              },
+              count: {
+                visible: true,
+                calculationType: CalculationType.COUNT,
+                field: "Price",
+              },
+              min: {
+                visible: true,
+                calculationType: CalculationType.MIN,
+                field: "Price",
+              },
+              max: {
+                visible: true,
+                calculationType: CalculationType.MAX,
+                field: "Price",
+              },
+              avg: {
+                visible: true,
+                calculationType: CalculationType.AVG,
+                field: "Price",
+              },
+              sum2: {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "Price",
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              message: "Calculation views can only have a maximum of 5 fields",
+            },
+          }
+        )
+      })
+
+      it("cannot create a calculation view with duplicate calculations", async () => {
+        await config.api.viewV2.create(
+          {
+            tableId: table._id!,
+            name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
+            schema: {
+              sum: {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "Price",
+              },
+              sum2: {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "Price",
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              message:
+                'Duplicate calculation on field "Price", calculation type "sum"',
+            },
+          }
+        )
+      })
+
+      it("finds duplicate counts", async () => {
+        await config.api.viewV2.create(
+          {
+            tableId: table._id!,
+            name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
+            schema: {
+              count: {
+                visible: true,
+                calculationType: CalculationType.COUNT,
+              },
+              count2: {
+                visible: true,
+                calculationType: CalculationType.COUNT,
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              message:
+                'Duplicate calculation on field "*", calculation type "count"',
+            },
+          }
+        )
+      })
+
+      it("finds duplicate count distincts", async () => {
+        await config.api.viewV2.create(
+          {
+            tableId: table._id!,
+            name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
+            schema: {
+              count: {
+                visible: true,
+                calculationType: CalculationType.COUNT,
+                distinct: true,
+                field: "Price",
+              },
+              count2: {
+                visible: true,
+                calculationType: CalculationType.COUNT,
+                distinct: true,
+                field: "Price",
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              message:
+                'Duplicate calculation on field "Price", calculation type "count"',
+            },
+          }
+        )
+      })
+
+      it("does not confuse counts and count distincts in the duplicate check", async () => {
+        await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          type: ViewV2Type.CALCULATION,
+          schema: {
+            count: {
+              visible: true,
+              calculationType: CalculationType.COUNT,
+            },
+            count2: {
+              visible: true,
+              calculationType: CalculationType.COUNT,
+              distinct: true,
+              field: "Price",
+            },
+          },
+        })
+      })
     })
 
     describe("update", () => {
@@ -612,7 +780,9 @@ describe.each([
       it("can update all fields", async () => {
         const tableId = table._id!
 
-        const updatedData: Required<Omit<UpdateViewRequest, "queryUI">> = {
+        const updatedData: Required<
+          Omit<UpdateViewRequest, "queryUI" | "type">
+        > = {
           version: view.version,
           id: view.id,
           tableId,
@@ -824,6 +994,32 @@ describe.each([
         )
       })
 
+      it("cannot update view type after creation", async () => {
+        const view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          schema: {
+            id: { visible: true },
+            Price: {
+              visible: true,
+            },
+          },
+        })
+
+        await config.api.viewV2.update(
+          {
+            ...view,
+            type: ViewV2Type.CALCULATION,
+          },
+          {
+            status: 400,
+            body: {
+              message: "Cannot update view type after creation",
+            },
+          }
+        )
+      })
+
       isInternal &&
         it("updating schema will only validate modified field", async () => {
           let view = await config.api.viewV2.create({
@@ -864,6 +1060,206 @@ describe.each([
             }
           )
         })
+
+      !isLucene &&
+        describe("calculation views", () => {
+          let table: Table
+          let view: ViewV2
+
+          beforeEach(async () => {
+            table = await config.api.table.save(
+              saveTableRequest({
+                schema: {
+                  name: {
+                    name: "name",
+                    type: FieldType.STRING,
+                    constraints: {
+                      presence: true,
+                    },
+                  },
+                  country: {
+                    name: "country",
+                    type: FieldType.STRING,
+                  },
+                  age: {
+                    name: "age",
+                    type: FieldType.NUMBER,
+                  },
+                },
+              })
+            )
+
+            view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
+              schema: {
+                country: {
+                  visible: true,
+                },
+                age: {
+                  visible: true,
+                  calculationType: CalculationType.SUM,
+                  field: "age",
+                },
+              },
+            })
+
+            await config.api.row.bulkImport(table._id!, {
+              rows: [
+                {
+                  name: "Steve",
+                  age: 30,
+                  country: "UK",
+                },
+                {
+                  name: "Jane",
+                  age: 31,
+                  country: "UK",
+                },
+                {
+                  name: "Ruari",
+                  age: 32,
+                  country: "USA",
+                },
+                {
+                  name: "Alice",
+                  age: 33,
+                  country: "USA",
+                },
+              ],
+            })
+          })
+
+          it("returns the expected rows prior to modification", async () => {
+            const { rows } = await config.api.row.search(view.id)
+            expect(rows).toHaveLength(2)
+            expect(rows).toEqual(
+              expect.arrayContaining([
+                {
+                  country: "USA",
+                  age: 65,
+                },
+                {
+                  country: "UK",
+                  age: 61,
+                },
+              ])
+            )
+          })
+
+          it("can remove a group by field", async () => {
+            delete view.schema!.country
+            await config.api.viewV2.update(view)
+
+            const { rows } = await config.api.row.search(view.id)
+            expect(rows).toHaveLength(1)
+            expect(rows).toEqual(
+              expect.arrayContaining([
+                {
+                  age: 126,
+                },
+              ])
+            )
+          })
+
+          it("can remove a calculation field", async () => {
+            delete view.schema!.age
+            await config.api.viewV2.update(view)
+
+            const { rows } = await config.api.row.search(view.id)
+            expect(rows).toHaveLength(4)
+
+            // Because the removal of the calculation field actually makes this
+            // no longer a calculation view, these rows will now have _id and
+            // _rev fields.
+            expect(rows).toEqual(
+              expect.arrayContaining([
+                expect.objectContaining({ country: "UK" }),
+                expect.objectContaining({ country: "UK" }),
+                expect.objectContaining({ country: "USA" }),
+                expect.objectContaining({ country: "USA" }),
+              ])
+            )
+          })
+
+          it("can add a new group by field", async () => {
+            view.schema!.name = { visible: true }
+            await config.api.viewV2.update(view)
+
+            const { rows } = await config.api.row.search(view.id)
+            expect(rows).toHaveLength(4)
+            expect(rows).toEqual(
+              expect.arrayContaining([
+                {
+                  name: "Steve",
+                  age: 30,
+                  country: "UK",
+                },
+                {
+                  name: "Jane",
+                  age: 31,
+                  country: "UK",
+                },
+                {
+                  name: "Ruari",
+                  age: 32,
+                  country: "USA",
+                },
+                {
+                  name: "Alice",
+                  age: 33,
+                  country: "USA",
+                },
+              ])
+            )
+          })
+
+          it("can add a new group by field that is invisible, even if required on the table", async () => {
+            view.schema!.name = { visible: false }
+            await config.api.viewV2.update(view)
+
+            const { rows } = await config.api.row.search(view.id)
+            expect(rows).toHaveLength(2)
+            expect(rows).toEqual(
+              expect.arrayContaining([
+                {
+                  country: "USA",
+                  age: 65,
+                },
+                {
+                  country: "UK",
+                  age: 61,
+                },
+              ])
+            )
+          })
+
+          it("can add a new calculation field", async () => {
+            view.schema!.count = {
+              visible: true,
+              calculationType: CalculationType.COUNT,
+            }
+            await config.api.viewV2.update(view)
+
+            const { rows } = await config.api.row.search(view.id)
+            expect(rows).toHaveLength(2)
+            expect(rows).toEqual(
+              expect.arrayContaining([
+                {
+                  country: "USA",
+                  age: 65,
+                  count: 2,
+                },
+                {
+                  country: "UK",
+                  age: 61,
+                  count: 2,
+                },
+              ])
+            )
+          })
+        })
     })
 
     describe("delete", () => {
@@ -2258,12 +2654,8 @@ describe.each([
       describe("foreign relationship columns", () => {
         let envCleanup: () => void
         beforeAll(() => {
-          const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`]
+          envCleanup = features.testutils.setFeatureFlags("*", {
-          if (env.TENANT_FEATURE_FLAGS) {
+            ENRICHED_RELATIONSHIPS: true,
-            flags.push(...env.TENANT_FEATURE_FLAGS.split(","))
-          }
-          envCleanup = setCoreEnv({
-            TENANT_FEATURE_FLAGS: flags.join(","),
           })
         })
 
@@ -2454,6 +2846,7 @@ describe.each([
           it("should be able to search by calculations", async () => {
             const view = await config.api.viewV2.create({
               tableId: table._id!,
+              type: ViewV2Type.CALCULATION,
               name: generator.guid(),
               schema: {
                 "Quantity Sum": {
@@ -2488,6 +2881,7 @@ describe.each([
             const view = await config.api.viewV2.create({
               tableId: table._id!,
               name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
               schema: {
                 quantity: {
                   visible: true,
@@ -2526,6 +2920,7 @@ describe.each([
             const view = await config.api.viewV2.create({
               tableId: table._id!,
               name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
               schema: {
                 aggregate: {
                   visible: true,
@@ -2570,6 +2965,76 @@ describe.each([
               expect(actual).toEqual(expected)
             }
           })
+
+          it("should be able to do a COUNT(DISTINCT)", async () => {
+            const table = await config.api.table.save(
+              saveTableRequest({
+                schema: {
+                  name: {
+                    name: "name",
+                    type: FieldType.STRING,
+                  },
+                },
+              })
+            )
+
+            const view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
+              schema: {
+                count: {
+                  visible: true,
+                  calculationType: CalculationType.COUNT,
+                  distinct: true,
+                  field: "name",
+                },
+              },
+            })
+
+            await config.api.row.bulkImport(table._id!, {
+              rows: [
+                {
+                  name: "John",
+                },
+                {
+                  name: "John",
+                },
+                {
+                  name: "Sue",
+                },
+              ],
+            })
+
+            const { rows } = await config.api.row.search(view.id)
+            expect(rows).toHaveLength(1)
+            expect(rows[0].count).toEqual(2)
+          })
+
+          it("should not be able to COUNT(DISTINCT ...) against a non-existent field", async () => {
+            await config.api.viewV2.create(
+              {
+                tableId: table._id!,
+                name: generator.guid(),
+                type: ViewV2Type.CALCULATION,
+                schema: {
+                  count: {
+                    visible: true,
+                    calculationType: CalculationType.COUNT,
+                    distinct: true,
+                    field: "does not exist oh no",
+                  },
+                },
+              },
+              {
+                status: 400,
+                body: {
+                  message:
+                    'Calculation field "count" references field "does not exist oh no" which does not exist in the table schema',
+                },
+              }
+            )
+          })
         })
 
       !isLucene &&
@@ -2600,6 +3065,7 @@ describe.each([
           const view = await config.api.viewV2.create({
             tableId: table._id!,
             name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
             schema: {
               sum: {
                 visible: true,

packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts

@@ -2,8 +2,8 @@ import * as setup from "../../../api/routes/tests/utilities"
 import { basicTable } from "../../../tests/utilities/structures"
 import {
   db as dbCore,
+  features,
   SQLITE_DESIGN_DOC_ID,
-  withEnv as withCoreEnv,
 } from "@budibase/backend-core"
 import {
   LinkDocument,
@@ -71,11 +71,11 @@ function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
 }
 
 async function sqsDisabled(cb: () => Promise<void>) {
-  await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" }, cb)
+  await features.testutils.withFeatureFlags("*", { SQS: false }, cb)
 }
 
 async function sqsEnabled(cb: () => Promise<void>) {
-  await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, cb)
+  await features.testutils.withFeatureFlags("*", { SQS: true }, cb)
 }
 
 describe("SQS migration", () => {

packages/server/src/db/linkedRows/LinkController.ts

@@ -221,9 +221,15 @@ class LinkController {
                 link.id !== row._id && link.fieldName === linkedSchema.name
             )
 
+            // check all the related rows exist
+            const foundRecords = await this._db.getMultiple(
+              links.map(l => l.id),
+              { allowMissing: true, excludeDocs: true }
+            )
+
             // The 1 side of 1:N is already related to something else
             // You must remove the existing relationship
-            if (links.length > 0) {
+            if (foundRecords.length > 0) {
               throw new Error(
                 `1:N Relationship Error: Record already linked to another.`
               )

packages/server/src/environment.ts

@@ -83,6 +83,7 @@ const environment = {
   PLUGINS_DIR: process.env.PLUGINS_DIR || DEFAULTS.PLUGINS_DIR,
   MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB,
   SESSION_EXPIRY_SECONDS: process.env.SESSION_EXPIRY_SECONDS,
+  XSS_SAFE_MODE: process.env.XSS_SAFE_MODE,
   // SQL
   SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
   SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE,

packages/server/src/jsRunner/index.ts

@@ -1,7 +1,7 @@
 import { serializeError } from "serialize-error"
 import env from "../environment"
 import {
-  JsErrorTimeout,
+  JsTimeoutError,
   setJSRunner,
   setOnErrorLog,
 } from "@budibase/string-templates"
@@ -40,7 +40,7 @@ export function init() {
         return vm.withContext(rest, () => vm.execute(js))
       } catch (error: any) {
         if (error.message === "Script execution timed out.") {
-          throw new JsErrorTimeout()
+          throw new JsTimeoutError()
         }
         throw error
       }

packages/server/src/jsRunner/tests/jsRunner.spec.ts

@@ -8,7 +8,6 @@ import { init } from ".."
 import TestConfiguration from "../../tests/utilities/TestConfiguration"
 
 const DATE = "2021-01-21T12:00:00"
-
 tk.freeze(DATE)
 
 describe("jsRunner (using isolated-vm)", () => {
@@ -41,10 +40,38 @@ describe("jsRunner (using isolated-vm)", () => {
   })
 
   it("should prevent sandbox escape", async () => {
-    const output = await processJS(
+    expect(
-      `return this.constructor.constructor("return process.env")()`
+      await processJS(
+        `return this.constructor.constructor("return process.env")()`
+      )
+    ).toEqual("ReferenceError: process is not defined")
+  })
+
+  it("should not allow the context to be mutated", async () => {
+    const context = { array: [1] }
+    const result = await processJS(
+      `
+        const array = $("array");
+        array.push(2);
+        return array[1]
+      `,
+      context
     )
-    expect(output).toBe("Error while executing JS")
+    expect(result).toEqual(2)
+    expect(context.array).toEqual([1])
+  })
+
+  it("should copy values whenever returning them from $", async () => {
+    const context = { array: [1] }
+    const result = await processJS(
+      `
+        $("array").push(2);
+        return $("array")[1];
+      `,
+      context
+    )
+    expect(result).toEqual(undefined)
+    expect(context.array).toEqual([1])
   })
 
   describe("helpers", () => {

packages/server/src/jsRunner/vm/isolated-vm.ts

@@ -7,14 +7,12 @@ import querystring from "querystring"
 
 import { BundleType, loadBundle } from "../bundles"
 import { Snippet, VM } from "@budibase/types"
-import { iifeWrapper } from "@budibase/string-templates"
+import { iifeWrapper, UserScriptError } from "@budibase/string-templates"
 import environment from "../../environment"
 
-class ExecutionTimeoutError extends Error {
+export class JsRequestTimeoutError extends Error {
-  constructor(message: string) {
+  static code = "JS_REQUEST_TIMEOUT_ERROR"
-    super(message)
+  code = JsRequestTimeoutError.code
-    this.name = "ExecutionTimeoutError"
-  }
 }
 
 export class IsolatedVM implements VM {
@@ -29,6 +27,7 @@ export class IsolatedVM implements VM {
 
   private readonly resultKey = "results"
   private runResultKey: string
+  private runErrorKey: string
 
   constructor({
     memoryLimit,
@@ -47,6 +46,7 @@ export class IsolatedVM implements VM {
     this.jail.setSync("global", this.jail.derefInto())
 
     this.runResultKey = crypto.randomUUID()
+    this.runErrorKey = crypto.randomUUID()
     this.addToContext({
       [this.resultKey]: { [this.runResultKey]: "" },
     })
@@ -210,13 +210,19 @@ export class IsolatedVM implements VM {
     if (this.isolateAccumulatedTimeout) {
       const cpuMs = Number(this.isolate.cpuTime) / 1e6
       if (cpuMs > this.isolateAccumulatedTimeout) {
-        throw new ExecutionTimeoutError(
+        throw new JsRequestTimeoutError(
           `CPU time limit exceeded (${cpuMs}ms > ${this.isolateAccumulatedTimeout}ms)`
         )
       }
     }
 
-    code = `results['${this.runResultKey}']=${this.codeWrapper(code)}`
+    code = `
+      try {
+        results['${this.runResultKey}']=${this.codeWrapper(code)}
+      } catch (e) {
+        results['${this.runErrorKey}']=e
+      }
+    `
 
     const script = this.isolate.compileScriptSync(code)
 
@@ -227,6 +233,9 @@ export class IsolatedVM implements VM {
 
     // We can't rely on the script run result as it will not work for non-transferable values
     const result = this.getFromContext(this.resultKey)
+    if (result[this.runErrorKey]) {
+      throw new UserScriptError(result[this.runErrorKey])
+    }
     return result[this.runResultKey]
   }
 

packages/server/src/sdk/app/rowActions.ts

@@ -75,7 +75,7 @@ export async function create(tableId: string, rowAction: { name: string }) {
     name: action.name,
     automationId: automation._id!,
     permissions: {
-      table: { runAllowed: false },
+      table: { runAllowed: true },
       views: {},
     },
   }
@@ -163,6 +163,23 @@ async function guardView(tableId: string, viewId: string) {
   }
 }
 
+export async function setTablePermission(tableId: string, rowActionId: string) {
+  return await updateDoc(tableId, rowActionId, async actionsDoc => {
+    actionsDoc.actions[rowActionId].permissions.table.runAllowed = true
+    return actionsDoc
+  })
+}
+
+export async function unsetTablePermission(
+  tableId: string,
+  rowActionId: string
+) {
+  return await updateDoc(tableId, rowActionId, async actionsDoc => {
+    actionsDoc.actions[rowActionId].permissions.table.runAllowed = false
+    return actionsDoc
+  })
+}
+
 export async function setViewPermission(
   tableId: string,
   rowActionId: string,

packages/server/src/sdk/app/rows/search/internal/internal.ts

@@ -62,10 +62,10 @@ export async function exportRows(
     ).rows.map(row => row.doc!)
 
     result = await outputProcessing(table, response)
-  } else if (query) {
+  } else {
     let searchResponse = await sdk.rows.search({
       tableId,
-      query,
+      query: query || {},
       sort,
       sortOrder,
     })

packages/server/src/sdk/app/rows/search/internal/sqs.ts

@@ -1,5 +1,6 @@
 import {
   Aggregation,
+  CalculationType,
   Datasource,
   DocumentType,
   FieldType,
@@ -369,11 +370,27 @@ export async function search(
         continue
       }
 
-      aggregations.push({
+      if (field.calculationType === CalculationType.COUNT) {
-        name: key,
+        if ("distinct" in field && field.distinct) {
-        field: mapToUserColumn(field.field),
+          aggregations.push({
-        calculationType: field.calculationType,
+            name: key,
-      })
+            distinct: true,
+            field: mapToUserColumn(field.field),
+            calculationType: field.calculationType,
+          })
+        } else {
+          aggregations.push({
+            name: key,
+            calculationType: field.calculationType,
+          })
+        }
+      } else {
+        aggregations.push({
+          name: key,
+          field: mapToUserColumn(field.field),
+          calculationType: field.calculationType,
+        })
+      }
     }
   }
 

packages/server/src/sdk/app/rows/search/tests/search.spec.ts

@@ -10,10 +10,7 @@ import {
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
 import { search } from "../../../../../sdk/app/rows/search"
 import { generator } from "@budibase/backend-core/tests"
-import {
+import { features } from "@budibase/backend-core"
-  withEnv as withCoreEnv,
-  setEnv as setCoreEnv,
-} from "@budibase/backend-core"
 import {
   DatabaseName,
   getDatasource,
@@ -41,19 +38,13 @@ describe.each([
   let table: Table
 
   beforeAll(async () => {
-    await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () =>
+    await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
       config.init()
     )
 
-    if (isLucene) {
+    envCleanup = features.testutils.setFeatureFlags("*", {
-      envCleanup = setCoreEnv({
+      SQS: isSqs,
-        TENANT_FEATURE_FLAGS: "*:!SQS",
+    })
-      })
-    } else if (isSqs) {
-      envCleanup = setCoreEnv({
-        TENANT_FEATURE_FLAGS: "*:SQS",
-      })
-    }
 
     if (dsProvider) {
       datasource = await config.createDatasource({

packages/server/src/sdk/app/rows/tests/utils.spec.ts

@@ -8,6 +8,7 @@ import {
 import { generateTableID } from "../../../../db/utils"
 import { validate } from "../utils"
 import { generator } from "@budibase/backend-core/tests"
+import { withEnv } from "../../../../environment"
 
 describe("validate", () => {
   const hour = () => generator.hour().toString().padStart(2, "0")
@@ -332,4 +333,46 @@ describe("validate", () => {
       })
     })
   })
+
+  describe("XSS Safe mode", () => {
+    const getTable = (): Table => ({
+      type: "table",
+      _id: generateTableID(),
+      name: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
+      schema: {
+        text: {
+          name: "sometext",
+          type: FieldType.STRING,
+        },
+      },
+    })
+    it.each([
+      "SELECT * FROM users WHERE username = 'admin' --",
+      "SELECT * FROM users WHERE id = 1; DROP TABLE users;",
+      "1' OR '1' = '1",
+      "' OR 'a' = 'a",
+      "<script>alert('XSS');</script>",
+      '"><img src=x onerror=alert(1)>',
+      "</script><script>alert('test')</script>",
+      "<div onmouseover=\"alert('XSS')\">Hover over me!</div>",
+      "'; EXEC sp_msforeachtable 'DROP TABLE ?'; --",
+      "{alert('Injected')}",
+      "UNION SELECT * FROM users",
+      "INSERT INTO users (username, password) VALUES ('admin', 'password')",
+      "/* This is a comment */ SELECT * FROM users",
+      '<iframe src="http://malicious-site.com"></iframe>',
+    ])("test potentially unsafe input: %s", async input => {
+      await withEnv({ XSS_SAFE_MODE: "1" }, async () => {
+        const table = getTable()
+        const row = { text: input }
+        const output = await validate({ source: table, row })
+        expect(output.valid).toBe(false)
+        expect(output.errors).toStrictEqual({
+          text: ["Input not sanitised - potentially vulnerable to XSS"],
+        })
+      })
+    })
+  })
 })

packages/server/src/sdk/app/rows/utils.ts

@@ -22,6 +22,7 @@ import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils"
 import { isSQL } from "../../../integrations/utils"
 import { docIds, sql } from "@budibase/backend-core"
 import { getTableFromSource } from "../../../api/controllers/row/utils"
+import env from "../../../environment"
 
 const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
   [SourceName.POSTGRES]: SqlClient.POSTGRES,
@@ -43,6 +44,9 @@ const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
   [SourceName.BUDIBASE]: undefined,
 }
 
+const XSS_INPUT_REGEX =
+  /[<>;"'(){}]|--|\/\*|\*\/|union|select|insert|drop|delete|update|exec|script/i
+
 export function getSQLClient(datasource: Datasource): SqlClient {
   if (!isSQL(datasource)) {
     throw new Error("Cannot get SQL Client for non-SQL datasource")
@@ -222,6 +226,15 @@ export async function validate({
     } else {
       res = validateJs.single(row[fieldName], constraints)
     }
+
+    if (env.XSS_SAFE_MODE && typeof row[fieldName] === "string") {
+      if (XSS_INPUT_REGEX.test(row[fieldName])) {
+        errors[fieldName] = [
+          "Input not sanitised - potentially vulnerable to XSS",
+        ]
+      }
+    }
+
     if (res) errors[fieldName] = res
   }
   return { valid: Object.keys(errors).length === 0, errors }

packages/server/src/sdk/app/views/external.ts

@@ -70,6 +70,9 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
   if (!existingView || !existingView.name) {
     throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404)
   }
+  if (isV2(existingView) && existingView.type !== view.type) {
+    throw new HTTPError(`Cannot update view type after creation`, 400)
+  }
 
   delete views[existingView.name]
   views[view.name] = view

packages/server/src/sdk/app/views/index.ts

@@ -1,4 +1,5 @@
 import {
+  CalculationType,
   FieldType,
   PermissionLevel,
   RelationSchemaField,
@@ -63,19 +64,48 @@ async function guardCalculationViewSchema(
   view: Omit<ViewV2, "id" | "version">
 ) {
   const calculationFields = helpers.views.calculationFields(view)
-  for (const calculationFieldName of Object.keys(calculationFields)) {
+
-    const schema = calculationFields[calculationFieldName]
+  if (Object.keys(calculationFields).length > 5) {
+    throw new HTTPError(
+      "Calculation views can only have a maximum of 5 fields",
+      400
+    )
+  }
+
+  const seen: Record<string, Record<CalculationType, boolean>> = {}
+
+  for (const name of Object.keys(calculationFields)) {
+    const schema = calculationFields[name]
+    const isCount = schema.calculationType === CalculationType.COUNT
+    const isDistinct = isCount && "distinct" in schema && schema.distinct
+
+    const field = isCount && !isDistinct ? "*" : schema.field
+    if (seen[field]?.[schema.calculationType]) {
+      throw new HTTPError(
+        `Duplicate calculation on field "${field}", calculation type "${schema.calculationType}"`,
+        400
+      )
+    }
+    seen[field] ??= {} as Record<CalculationType, boolean>
+    seen[field][schema.calculationType] = true
+
+    // Count fields that aren't distinct don't need to reference another field,
+    // so we don't validate it.
+    if (isCount && !isDistinct) {
+      continue
+    }
+
     const targetSchema = table.schema[schema.field]
     if (!targetSchema) {
       throw new HTTPError(
-        `Calculation field "${calculationFieldName}" references field "${schema.field}" which does not exist in the table schema`,
+        `Calculation field "${name}" references field "${schema.field}" which does not exist in the table schema`,
         400
       )
     }
 
-    if (!helpers.schema.isNumeric(targetSchema)) {
+    if (!isCount && !helpers.schema.isNumeric(targetSchema)) {
       throw new HTTPError(
-        `Calculation field "${calculationFieldName}" references field "${schema.field}" which is not a numeric field`,
+        `Calculation field "${name}" references field "${schema.field}" which is not a numeric field`,
         400
       )
     }
@@ -101,10 +131,21 @@ async function guardViewSchema(
 
   if (helpers.views.isCalculationView(view)) {
     await guardCalculationViewSchema(table, view)
+  } else {
+    if (helpers.views.hasCalculationFields(view)) {
+      throw new HTTPError(
+        "Calculation fields are not allowed in non-calculation views",
+        400
+      )
+    }
   }
 
   await checkReadonlyFields(table, view)
-  checkRequiredFields(table, view)
+
+  if (!helpers.views.isCalculationView(view)) {
+    checkRequiredFields(table, view)
+  }
+
   checkDisplayField(view)
 }
 

packages/server/src/sdk/app/views/internal.ts

@@ -59,6 +59,10 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
     throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404)
   }
 
+  if (isV2(existingView) && existingView.type !== view.type) {
+    throw new HTTPError(`Cannot update view type after creation`, 400)
+  }
+
   delete table.views[existingView.name]
   table.views[view.name] = view
   await db.put(table)

packages/server/src/tests/utilities/api/row.ts

@@ -105,7 +105,7 @@ export class RowAPI extends TestAPI {
 
   exportRows = async (
     tableId: string,
-    body: ExportRowsRequest,
+    body?: ExportRowsRequest,
     format: RowExportFormat = RowExportFormat.JSON,
     expectations?: Expectations
   ) => {

packages/server/src/tests/utilities/api/rowAction.ts

@@ -72,6 +72,42 @@ export class RowActionAPI extends TestAPI {
     )
   }
 
+  setTablePermission = async (
+    tableId: string,
+    rowActionId: string,
+    expectations?: Expectations,
+    config?: { publicUser?: boolean }
+  ) => {
+    return await this._post<RowActionResponse>(
+      `/api/tables/${tableId}/actions/${rowActionId}/permissions`,
+      {
+        expectations: {
+          status: 200,
+          ...expectations,
+        },
+        ...config,
+      }
+    )
+  }
+
+  unsetTablePermission = async (
+    tableId: string,
+    rowActionId: string,
+    expectations?: Expectations,
+    config?: { publicUser?: boolean }
+  ) => {
+    return await this._delete<RowActionResponse>(
+      `/api/tables/${tableId}/actions/${rowActionId}/permissions`,
+      {
+        expectations: {
+          status: 200,
+          ...expectations,
+        },
+        ...config,
+      }
+    )
+  }
+
   setViewPermission = async (
     tableId: string,
     viewId: string,

packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts

@@ -8,7 +8,7 @@ import {
 } from "@budibase/types"
 import { outputProcessing } from ".."
 import { generator, structures } from "@budibase/backend-core/tests"
-import { setEnv as setCoreEnv } from "@budibase/backend-core"
+import { features } from "@budibase/backend-core"
 import * as bbReferenceProcessor from "../bbReferenceProcessor"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 
@@ -21,7 +21,7 @@ jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
 
 describe("rowProcessor - outputProcessing", () => {
   const config = new TestConfiguration()
-  let cleanupEnv: () => void = () => {}
+  let cleanupFlags: () => void = () => {}
 
   beforeAll(async () => {
     await config.init()
@@ -33,11 +33,11 @@ describe("rowProcessor - outputProcessing", () => {
 
   beforeEach(() => {
     jest.resetAllMocks()
-    cleanupEnv = setCoreEnv({ TENANT_FEATURE_FLAGS: "*SQS" })
+    cleanupFlags = features.testutils.setFeatureFlags("*", { SQS: true })
   })
 
   afterEach(() => {
-    cleanupEnv()
+    cleanupFlags()
   })
 
   const processOutputBBReferenceMock =

packages/shared-core/src/filters.ts

@@ -639,19 +639,19 @@ export function fixupFilterArrays(filters: SearchFilters) {
   return filters
 }
 
-export function search<T>(
+export function search<T extends Record<string, any>>(
-  docs: Record<string, T>[],
+  docs: T[],
-  query: RowSearchParams
+  query: Omit<RowSearchParams, "tableId">
-): SearchResponse<Record<string, T>> {
+): SearchResponse<T> {
   let result = runQuery(docs, query.query)
   if (query.sort) {
     result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING)
   }
-  let totalRows = result.length
+  const totalRows = result.length
   if (query.limit) {
     result = limit(result, query.limit.toString())
   }
-  const response: SearchResponse<Record<string, any>> = { rows: result }
+  const response: SearchResponse<T> = { rows: result }
   if (query.countRows) {
     response.totalRows = totalRows
   }

packages/shared-core/src/helpers/views.ts

@@ -3,6 +3,7 @@ import {
   ViewCalculationFieldMetadata,
   ViewFieldMetadata,
   ViewV2,
+  ViewV2Type,
 } from "@budibase/types"
 import { pickBy } from "lodash"
 
@@ -21,6 +22,10 @@ export function isBasicViewField(
 type UnsavedViewV2 = Omit<ViewV2, "id" | "version">
 
 export function isCalculationView(view: UnsavedViewV2) {
+  return view.type === ViewV2Type.CALCULATION
+}
+
+export function hasCalculationFields(view: UnsavedViewV2) {
   return Object.values(view.schema || {}).some(isCalculationField)
 }
 

packages/shared-core/src/utils.ts

@@ -5,6 +5,7 @@ import {
   SearchFilters,
   BasicOperator,
   ArrayOperator,
+  isLogicalSearchOperator,
 } from "@budibase/types"
 import * as Constants from "./constants"
 import { removeKeyNumbering } from "./filters"
@@ -97,10 +98,20 @@ export function isSupportedUserSearch(query: SearchFilters) {
     { op: BasicOperator.EQUAL, key: "_id" },
     { op: ArrayOperator.ONE_OF, key: "_id" },
   ]
-  for (let [key, operation] of Object.entries(query)) {
+  for (const [key, operation] of Object.entries(query)) {
     if (typeof operation !== "object") {
       return false
     }
+
+    if (isLogicalSearchOperator(key)) {
+      for (const condition of query[key]!.conditions) {
+        if (!isSupportedUserSearch(condition)) {
+          return false
+        }
+      }
+      return true
+    }
+
     const fields = Object.keys(operation || {})
     // this filter doesn't contain options - ignore
     if (fields.length === 0) {

packages/string-templates/src/errors.ts

@@ -1,3 +1,20 @@
-export class JsErrorTimeout extends Error {
+export class JsTimeoutError extends Error {
-  code = "ERR_SCRIPT_EXECUTION_TIMEOUT"
+  static message = "Timed out while executing JS"
+  static code = "JS_TIMEOUT_ERROR"
+  code: string = JsTimeoutError.code
+
+  constructor() {
+    super(JsTimeoutError.message)
+  }
+}
+
+export class UserScriptError extends Error {
+  static code = "USER_SCRIPT_ERROR"
+  code: string = UserScriptError.code
+
+  constructor(readonly userScriptError: Error) {
+    super(
+      `error while running user-supplied JavaScript: ${userScriptError.toString()}`
+    )
+  }
 }

packages/string-templates/src/helpers/javascript.ts

@@ -1,12 +1,14 @@
-import { atob, isJSAllowed } from "../utilities"
+import { atob, isBackendService, isJSAllowed } from "../utilities"
-import cloneDeep from "lodash/fp/cloneDeep"
 import { LITERAL_MARKER } from "../helpers/constants"
 import { getJsHelperList } from "./list"
 import { iifeWrapper } from "../iife"
+import { JsTimeoutError, UserScriptError } from "../errors"
+import { cloneDeep } from "lodash/fp"
 
 // The method of executing JS scripts depends on the bundle being built.
 // This setter is used in the entrypoint (either index.js or index.mjs).
-let runJS: ((js: string, context: any) => any) | undefined = undefined
+let runJS: ((js: string, context: Record<string, any>) => any) | undefined =
+  undefined
 export const setJSRunner = (runner: typeof runJS) => (runJS = runner)
 
 export const removeJSRunner = () => {
@@ -30,9 +32,19 @@ const removeSquareBrackets = (value: string) => {
   return value
 }
 
+const isReservedKey = (key: string) =>
+  key === "snippets" ||
+  key === "helpers" ||
+  key.startsWith("snippets.") ||
+  key.startsWith("helpers.")
+
 // Our context getter function provided to JS code as $.
 // Extracts a value from context.
 const getContextValue = (path: string, context: any) => {
+  // We populate `snippets` ourselves, don't allow access to it.
+  if (isReservedKey(path)) {
+    return undefined
+  }
   const literalStringRegex = /^(["'`]).*\1$/
   let data = context
   // check if it's a literal string - just return path if its quoted
@@ -45,6 +57,7 @@ const getContextValue = (path: string, context: any) => {
     }
     data = data[removeSquareBrackets(key)]
   })
+
   return data
 }
 
@@ -66,10 +79,23 @@ export function processJS(handlebars: string, context: any) {
       snippetMap[snippet.name] = snippet.code
     }
 
-    // Our $ context function gets a value from context.
+    let clonedContext: Record<string, any>
-    // We clone the context to avoid mutation in the binding affecting real
+    if (isBackendService()) {
-    // app context.
+      // On the backned, values are copied across the isolated-vm boundary and
-    const clonedContext = cloneDeep({ ...context, snippets: null })
+      // so we don't need to do any cloning here. This does create a fundamental
+      // difference in how JS executes on the frontend vs the backend, e.g.
+      // consider this snippet:
+      //
+      //   $("array").push(2)
+      //   return $("array")[1]
+      //
+      // With the context of `{ array: [1] }`, the backend will return
+      // `undefined` whereas the frontend will return `2`. We should fix this.
+      clonedContext = context
+    } else {
+      clonedContext = cloneDeep(context)
+    }
+
     const sandboxContext = {
       $: (path: string) => getContextValue(path, clonedContext),
       helpers: getJsHelperList(),
@@ -94,12 +120,49 @@ export function processJS(handlebars: string, context: any) {
   } catch (error: any) {
     onErrorLog && onErrorLog(error)
 
+    const { noThrow = true } = context.__opts || {}
+
+    // The error handling below is quite messy, because it has fallen to
+    // string-templates to handle a variety of types of error specific to usages
+    // above it in the stack. It would be nice some day to refactor this to
+    // allow each user of processStringSync to handle errors in the way they see
+    // fit.
+
+    // This is to catch the error vm.runInNewContext() throws when the timeout
+    // is exceeded.
     if (error.code === "ERR_SCRIPT_EXECUTION_TIMEOUT") {
       return "Timed out while executing JS"
     }
-    if (error.name === "ExecutionTimeoutError") {
+
-      return "Request JS execution limit hit"
+    // This is to catch the JsRequestTimeoutError we throw when we detect a
+    // timeout across an entire request in the backend. We use a magic string
+    // because we can't import from the backend into string-templates.
+    if (error.code === "JS_REQUEST_TIMEOUT_ERROR") {
+      return error.message
     }
+
+    // This is to catch the JsTimeoutError we throw when we detect a timeout in
+    // a single JS execution.
+    if (error.code === JsTimeoutError.code) {
+      return JsTimeoutError.message
+    }
+
+    // This is to catch the error that happens if a user-supplied JS script
+    // throws for reasons introduced by the user.
+    if (error.code === UserScriptError.code) {
+      if (noThrow) {
+        return error.userScriptError.toString()
+      }
+      throw error
+    }
+
+    if (error.name === "SyntaxError") {
+      if (noThrow) {
+        return error.toString()
+      }
+      throw error
+    }
+
     return "Error while executing JS"
   }
 }

packages/string-templates/src/index.ts

@@ -1,4 +1,4 @@
-import { Context, createContext, runInNewContext } from "vm"
+import { createContext, runInNewContext } from "vm"
 import { create, TemplateDelegate } from "handlebars"
 import { registerAll, registerMinimum } from "./helpers/index"
 import { postprocess, preprocess } from "./processors"
@@ -16,6 +16,7 @@ import { removeJSRunner, setJSRunner } from "./helpers/javascript"
 
 import manifest from "./manifest.json"
 import { ProcessOptions } from "./types"
+import { UserScriptError } from "./errors"
 
 export { helpersToRemoveForJs, getJsHelperList } from "./helpers/list"
 export { FIND_ANY_HBS_REGEX } from "./utilities"
@@ -122,7 +123,7 @@ function createTemplate(
 export async function processObject<T extends Record<string, any>>(
   object: T,
   context: object,
-  opts?: { noHelpers?: boolean; escapeNewlines?: boolean; onlyFound?: boolean }
+  opts?: ProcessOptions
 ): Promise<T> {
   testObject(object)
 
@@ -172,7 +173,7 @@ export async function processString(
 export function processObjectSync(
   object: { [x: string]: any },
   context: any,
-  opts: any
+  opts?: ProcessOptions
 ): object | Array<any> {
   testObject(object)
   for (let key of Object.keys(object || {})) {
@@ -229,8 +230,12 @@ export function processStringSync(
     } else {
       return process(string)
     }
-  } catch (err) {
+  } catch (err: any) {
-    return input
+    const { noThrow = true } = opts || {}
+    if (noThrow) {
+      return input
+    }
+    throw err
   }
 }
 
@@ -448,23 +453,41 @@ export function convertToJS(hbs: string) {
   return `${varBlock}${js}`
 }
 
-export { JsErrorTimeout } from "./errors"
+export { JsTimeoutError, UserScriptError } from "./errors"
+
+export function browserJSSetup() {
+  /**
+   * Use polyfilled vm to run JS scripts in a browser Env
+   */
+  setJSRunner((js: string, context: Record<string, any>) => {
+    createContext(context)
+
+    const wrappedJs = `
+        result = {
+          result: null,
+          error: null,
+        };
+
+        try {
+          result.result = ${js};
+        } catch (e) {
+          result.error = e;
+        }
+
+        result;
+      `
+
+    const result = runInNewContext(wrappedJs, context, { timeout: 1000 })
+    if (result.error) {
+      throw new UserScriptError(result.error)
+    }
+    return result.result
+  })
+}
 
 export function defaultJSSetup() {
   if (!isBackendService()) {
-    /**
+    browserJSSetup()
-     * Use polyfilled vm to run JS scripts in a browser Env
-     */
-    setJSRunner((js: string, context: Context) => {
-      context = {
-        ...context,
-        alert: undefined,
-        setInterval: undefined,
-        setTimeout: undefined,
-      }
-      createContext(context)
-      return runInNewContext(js, context, { timeout: 1000 })
-    })
   } else {
     removeJSRunner()
   }

packages/string-templates/src/types.ts

@@ -3,6 +3,7 @@ export interface ProcessOptions {
   noEscaping?: boolean
   noHelpers?: boolean
   noFinalise?: boolean
+  noThrow?: boolean
   escapeNewlines?: boolean
   onlyFound?: boolean
   disabledHelpers?: string[]

packages/string-templates/src/utilities.ts

@@ -4,7 +4,14 @@ export const FIND_HBS_REGEX = /{{([^{].*?)}}/g
 export const FIND_ANY_HBS_REGEX = /{?{{([^{].*?)}}}?/g
 export const FIND_TRIPLE_HBS_REGEX = /{{{([^{].*?)}}}/g
 
+const isJest = () => typeof jest !== "undefined"
+
 export const isBackendService = () => {
+  // We consider the tests for string-templates to be frontend, so that they
+  // test the frontend JS functionality.
+  if (isJest()) {
+    return false
+  }
   return typeof window === "undefined"
 }
 

packages/string-templates/test/javascript.spec.ts

@@ -1,7 +1,13 @@
-import vm from "vm"
+import {
-
+  processStringSync,
-import { processStringSync, encodeJSBinding, setJSRunner } from "../src/index"
+  encodeJSBinding,
+  defaultJSSetup,
+} from "../src/index"
 import { UUID_REGEX } from "./constants"
+import tk from "timekeeper"
+
+const DATE = "2021-01-21T12:00:00"
+tk.freeze(DATE)
 
 const processJS = (js: string, context?: object): any => {
   return processStringSync(encodeJSBinding(js), context)
@@ -9,9 +15,7 @@ const processJS = (js: string, context?: object): any => {
 
 describe("Javascript", () => {
   beforeAll(() => {
-    setJSRunner((js, context) => {
+    defaultJSSetup()
-      return vm.runInNewContext(js, context, { timeout: 1000 })
-    })
   })
 
   describe("Test the JavaScript helper", () => {
@@ -118,8 +122,7 @@ describe("Javascript", () => {
     })
 
     it("should handle errors", () => {
-      const output = processJS(`throw "Error"`)
+      expect(processJS(`throw "Error"`)).toEqual("Error")
-      expect(output).toBe("Error while executing JS")
     })
 
     it("should timeout after one second", () => {
@@ -127,16 +130,18 @@ describe("Javascript", () => {
       expect(output).toBe("Timed out while executing JS")
     })
 
-    it("should prevent access to the process global", () => {
+    it("should prevent access to the process global", async () => {
-      const output = processJS(`return process`)
+      expect(processJS(`return process`)).toEqual(
-      expect(output).toBe("Error while executing JS")
+        "ReferenceError: process is not defined"
+      )
     })
   })
 
   describe("check JS helpers", () => {
     it("should error if using the format helper. not helpers.", () => {
-      const output = processJS(`return helper.toInt(4.3)`)
+      expect(processJS(`return helper.toInt(4.3)`)).toEqual(
-      expect(output).toBe("Error while executing JS")
+        "ReferenceError: helper is not defined"
+      )
     })
 
     it("should be able to use toInt", () => {
@@ -156,4 +161,323 @@ describe("Javascript", () => {
       expect(output).toBe("Custom")
     })
   })
+
+  describe("mutability", () => {
+    it("should not allow the context to be mutated", async () => {
+      const context = { array: [1] }
+      const result = await processJS(
+        `
+        const array = $("array");
+        array.push(2);
+        return array[1]
+      `,
+        context
+      )
+      expect(result).toEqual(2)
+      expect(context.array).toEqual([1])
+    })
+  })
+
+  describe("malice", () => {
+    it("should not be able to call JS functions", () => {
+      expect(processJS(`return alert("hello")`)).toEqual(
+        "ReferenceError: alert is not defined"
+      )
+
+      expect(processJS(`return prompt("hello")`)).toEqual(
+        "ReferenceError: prompt is not defined"
+      )
+
+      expect(processJS(`return confirm("hello")`)).toEqual(
+        "ReferenceError: confirm is not defined"
+      )
+
+      expect(processJS(`return setTimeout(() => {}, 1000)`)).toEqual(
+        "ReferenceError: setTimeout is not defined"
+      )
+
+      expect(processJS(`return setInterval(() => {}, 1000)`)).toEqual(
+        "ReferenceError: setInterval is not defined"
+      )
+    })
+  })
+
+  // the test cases here were extracted from templates/real world examples of JS in Budibase
+  describe("real test cases from Budicloud", () => {
+    const context = {
+      "Unit Value": 2,
+      Quantity: 1,
+    }
+    it("handle test case 1", async () => {
+      const result = await processJS(
+        `
+        var Gross = $("[Unit Value]") * $("[Quantity]")
+        return Gross.toFixed(2)`,
+        context
+      )
+      expect(result).toBeDefined()
+      expect(result).toBe("2.00")
+    })
+
+    it("handle test case 2", async () => {
+      const todayDate = new Date()
+      // add a year and a month
+      todayDate.setMonth(new Date().getMonth() + 1)
+      todayDate.setFullYear(todayDate.getFullYear() + 1)
+      const context = {
+        "Purchase Date": DATE,
+        today: todayDate.toISOString(),
+      }
+      const result = await processJS(
+        `
+        var purchase = new Date($("[Purchase Date]"));
+        let purchaseyear = purchase.getFullYear();
+        let purchasemonth = purchase.getMonth();
+
+        var today = new Date($("today"));
+        let todayyear = today.getFullYear();
+        let todaymonth = today.getMonth();
+
+        var age = todayyear - purchaseyear
+
+        if (((todaymonth - purchasemonth) < 6) == true){
+          return age
+        }
+        `,
+        context
+      )
+      expect(result).toBeDefined()
+      expect(result).toBe(1)
+    })
+
+    it("should handle test case 3", async () => {
+      const context = {
+        Escalate: true,
+        "Budget ($)": 1100,
+      }
+      const result = await processJS(
+        `
+        if ($("[Escalate]") == true) {
+          if ($("Budget ($)") <= 1000)
+            {return 2;}
+          if ($("Budget ($)") > 1000) 
+            {return 3;}
+          }
+          else {
+            if ($("Budget ($)") <= 1000)
+              {return 1;}
+            if ($("Budget ($)") > 1000)
+              if ($("Budget ($)") < 10000) 
+                {return 2;}
+              else 
+                {return 3}
+          }
+        `,
+        context
+      )
+      expect(result).toBeDefined()
+      expect(result).toBe(3)
+    })
+
+    it("should handle test case 4", async () => {
+      const context = {
+        "Time Sheets": ["a", "b"],
+      }
+      const result = await processJS(
+        `
+      let hours = 0
+      if (($("[Time Sheets]") != null) == true){
+        for (i = 0; i < $("[Time Sheets]").length; i++){
+          let hoursLogged = "Time Sheets." + i + ".Hours"
+          hours += $(hoursLogged)
+        }
+        return hours
+      }
+      if (($("[Time Sheets]") != null) == false){
+        return hours
+      }
+      `,
+        context
+      )
+      expect(result).toBeDefined()
+      expect(result).toBe("0ab")
+    })
+
+    it("should handle test case 5", async () => {
+      const context = {
+        change: JSON.stringify({ a: 1, primaryDisplay: "a" }),
+        previous: JSON.stringify({ a: 2, primaryDisplay: "b" }),
+      }
+      const result = await processJS(
+        `
+      let change = $("[change]") ? JSON.parse($("[change]")) : {}
+      let previous = $("[previous]") ? JSON.parse($("[previous]")) : {}
+
+      function simplifyLink(originalKey, value, parent) {
+        if (Array.isArray(value)) {
+          if (value.filter(item => Object.keys(item || {}).includes("primaryDisplay")).length > 0) {
+            parent[originalKey] = value.map(link => link.primaryDisplay)
+          }
+        }
+      }
+
+      for (let entry of Object.entries(change)) {
+        simplifyLink(entry[0], entry[1], change)
+      }
+      for (let entry of Object.entries(previous)) {
+        simplifyLink(entry[0], entry[1], previous)
+      }
+      
+      let diff = Object.fromEntries(Object.entries(change).filter(([k, v]) => previous[k]?.toString() !== v?.toString()))
+      
+      delete diff.audit_change
+      delete diff.audit_previous
+      delete diff._id
+      delete diff._rev
+      delete diff.tableId
+      delete diff.audit
+      
+      for (let entry of Object.entries(diff)) {
+        simplifyLink(entry[0], entry[1], diff)
+      }
+      
+      return JSON.stringify(change)?.replaceAll(",\\"", ",\\n\\t\\"").replaceAll("{\\"", "{\\n\\t\\"").replaceAll("}", "\\n}")
+      `,
+        context
+      )
+      expect(result).toBe(`{\n\t"a":1,\n\t"primaryDisplay":"a"\n}`)
+    })
+
+    it("should handle test case 6", async () => {
+      const context = {
+        "Join Date": DATE,
+      }
+      const result = await processJS(
+        `
+        var rate = 5;
+        var today = new Date();
+        
+        // comment
+        function monthDiff(dateFrom, dateTo) {
+         return dateTo.getMonth() - dateFrom.getMonth() + 
+           (12 * (dateTo.getFullYear() - dateFrom.getFullYear()))
+        }
+        var serviceMonths = monthDiff( new Date($("[Join Date]")), today);
+        var serviceYears = serviceMonths / 12;
+        
+        if (serviceYears >= 1 && serviceYears < 5){
+          rate = 10;
+        }
+        if (serviceYears >= 5 && serviceYears < 10){
+          rate = 15;
+        }
+        if (serviceYears >= 10){
+          rate = 15;
+          rate += 0.5 * (Number(serviceYears.toFixed(0)) - 10);
+        }
+        return rate;
+        `,
+        context
+      )
+      expect(result).toBe(10)
+    })
+
+    it("should handle test case 7", async () => {
+      const context = {
+        "P I": "Pass",
+        "PA I": "Pass",
+        "F I": "Fail",
+        "V I": "Pass",
+      }
+      const result = await processJS(
+        `if (($("[P I]") == "Pass") == true)
+              if (($("[  P I]") == "Pass") == true)
+                if (($("[F I]") == "Pass") == true)
+                  if (($("[V I]") == "Pass") == true)
+                    {return "Pass"}
+            
+            if (($("[PA I]") == "Fail") == true)
+              {return "Fail"}
+            if (($("[  P I]") == "Fail") == true)
+              {return "Fail"}
+            if (($("[F I]") == "Fail") == true)
+              {return "Fail"}
+            if (($("[V I]") == "Fail") == true)
+              {return "Fail"}
+            
+            else
+              {return ""}`,
+        context
+      )
+      expect(result).toBe("Fail")
+    })
+
+    it("should handle test case 8", async () => {
+      const context = {
+        "T L": [{ Hours: 10 }],
+        "B H": 50,
+      }
+      const result = await processJS(
+        `var totalHours = 0;
+            if (($("[T L]") != null) == true){
+            for (let i = 0; i < ($("[T L]").length); i++){
+              var individualHours = "T L." + i + ".Hours";
+              var hoursNum = Number($(individualHours));
+              totalHours += hoursNum;
+            }
+            return totalHours.toFixed(2);
+            }
+            if (($("[T L]") != null) == false) {
+              return totalHours.toFixed(2);
+            }
+        `,
+        context
+      )
+      expect(result).toBe("10.00")
+    })
+
+    it("should handle test case 9", async () => {
+      const context = {
+        "T L": [{ Hours: 10 }],
+        "B H": 50,
+      }
+      const result = await processJS(
+        `var totalHours = 0;
+            if (($("[T L]") != null) == true){
+            for (let i = 0; i < ($("[T L]").length); i++){
+              var individualHours = "T L." + i + ".Hours";
+              var hoursNum = Number($(individualHours));
+              totalHours += hoursNum;
+            }
+            return ($("[B H]") - totalHours).toFixed(2);
+            }
+            if (($("[T L]") != null) == false) {
+              return ($("[B H]") - totalHours).toFixed(2);
+            }`,
+        context
+      )
+      expect(result).toBe("40.00")
+    })
+
+    it("should handle test case 10", async () => {
+      const context = {
+        "F F": [{ "F S": 10 }],
+      }
+      const result = await processJS(
+        `var rating = 0;
+          
+          if ($("[F F]") != null){
+          for (i = 0; i < $("[F F]").length; i++){
+            var individualRating = $("F F." + i + ".F S");
+            rating += individualRating;
+            }
+            rating = (rating / $("[F F]").length);
+          }
+          return rating;
+          `,
+        context
+      )
+      expect(result).toBe(10)
+    })
+  })
 })

packages/string-templates/test/manifest.spec.ts

@@ -1,5 +1,3 @@
-import vm from "vm"
-
 jest.mock("@budibase/handlebars-helpers/lib/math", () => {
   const actual = jest.requireActual("@budibase/handlebars-helpers/lib/math")
 
@@ -17,7 +15,7 @@ jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
   }
 })
 
-import { processString, setJSRunner } from "../src/index"
+import { defaultJSSetup, processString } from "../src/index"
 
 import tk from "timekeeper"
 import { getParsedManifest, runJsHelpersTests } from "./utils"
@@ -32,9 +30,7 @@ describe("manifest", () => {
   const manifest = getParsedManifest()
 
   beforeAll(() => {
-    setJSRunner((js, context) => {
+    defaultJSSetup()
-      return vm.runInNewContext(js, context, { timeout: 1000 })
-    })
   })
 
   describe("examples are valid", () => {

packages/types/src/api/account/accounts.ts

@@ -12,7 +12,6 @@ export interface CreateAccountRequest {
   name?: string
   password: string
   provider?: AccountSSOProvider
-  thirdPartyProfile: object
 }
 
 export interface SearchAccountsRequest {

packages/types/src/api/web/app/rowAction.ts

@@ -8,7 +8,7 @@ export interface RowActionResponse extends RowActionData {
   id: string
   tableId: string
   automationId: string
-  allowedViews: string[] | undefined
+  allowedSources: string[] | undefined
 }
 
 export interface RowActionsResponse {

packages/types/src/api/web/auth.ts

@@ -21,6 +21,7 @@ export interface UpdateSelfRequest {
   freeTrialConfirmedAt?: string
   appFavourites?: string[]
   tours?: Record<string, Date>
+  appSort?: string
 }
 
 export interface UpdateSelfResponse {

packages/types/src/documents/account/account.ts

@@ -7,10 +7,9 @@ export interface CreateAccount {
   tenantId: string
   hosting: Hosting
   authType: AuthType
+  accountName: string
   // optional fields - for sso based sign ups
   registrationStep?: string
-  // profile
-  tenantName?: string
   name?: string
   size?: string
   profession?: string
@@ -20,11 +19,6 @@ export interface CreatePassswordAccount extends CreateAccount {
   password: string
 }
 
-export interface CreateVerifiableSSOAccount extends CreateAccount {
-  provider?: AccountSSOProvider
-  thirdPartyProfile?: any
-}
-
 export const isCreatePasswordAccount = (
   account: CreateAccount
 ): account is CreatePassswordAccount => account.authType === AuthType.PASSWORD
@@ -56,6 +50,7 @@ export interface Account extends CreateAccount {
   providerType?: AccountSSOProviderType
   quotaUsage?: QuotaUsage
   offlineLicenseToken?: string
+  tenantName?: string
 }
 
 export interface PasswordAccount extends Account {
@@ -103,8 +98,6 @@ export interface AccountSSO {
   provider: AccountSSOProvider
   providerType: AccountSSOProviderType
   oauth2?: OAuthTokens
-  pictureUrl?: string
-  thirdPartyProfile: any // TODO: define what the google profile looks like
 }
 
 export type SSOAccount = (Account | CloudAccount) & AccountSSO

packages/types/src/documents/app/rowAction.ts

@@ -8,8 +8,10 @@ export interface TableRowActions extends Document {
 export interface RowActionData {
   name: string
   automationId: string
-  permissions: {
+  permissions: RowActionPermissions
-    table: { runAllowed: boolean }
+}
-    views: Record<string, { runAllowed: boolean }>
+
-  }
+export interface RowActionPermissions {
+  table: { runAllowed: boolean }
+  views: Record<string, { runAllowed: boolean }>
 }

packages/types/src/documents/app/view.ts

@@ -42,11 +42,31 @@ export interface RelationSchemaField extends UIFieldMetadata {
   readonly?: boolean
 }
 
-export interface ViewCalculationFieldMetadata extends BasicViewFieldMetadata {
+export interface NumericCalculationFieldMetadata
-  calculationType: CalculationType
+  extends BasicViewFieldMetadata {
+  calculationType:
+    | CalculationType.MIN
+    | CalculationType.MAX
+    | CalculationType.SUM
+    | CalculationType.AVG
   field: string
 }
 
+export interface CountCalculationFieldMetadata extends BasicViewFieldMetadata {
+  calculationType: CalculationType.COUNT
+}
+
+export interface CountDistinctCalculationFieldMetadata
+  extends CountCalculationFieldMetadata {
+  distinct: true
+  field: string
+}
+
+export type ViewCalculationFieldMetadata =
+  | NumericCalculationFieldMetadata
+  | CountCalculationFieldMetadata
+  | CountDistinctCalculationFieldMetadata
+
 export type ViewFieldMetadata =
   | BasicViewFieldMetadata
   | ViewCalculationFieldMetadata
@@ -59,10 +79,15 @@ export enum CalculationType {
   MAX = "max",
 }
 
+export enum ViewV2Type {
+  CALCULATION = "calculation",
+}
+
 export interface ViewV2 {
   version: 2
   id: string
   name: string
+  type?: ViewV2Type
   primaryDisplay?: string
   tableId: string
   query?: LegacyFilter[] | SearchFilters

packages/types/src/documents/global/user.ts

@@ -21,7 +21,6 @@ export interface UserSSO {
   provider: string // the individual provider e.g. Okta, Auth0, Google
   providerType: SSOProviderType
   oauth2?: OAuth2
-  thirdPartyProfile?: SSOProfileJson
   profile?: {
     displayName?: string
     name?: {
@@ -45,7 +44,6 @@ export interface User extends Document {
   userId?: string
   firstName?: string
   lastName?: string
-  pictureUrl?: string
   forceResetPassword?: boolean
   roles: UserRoles
   builder?: {
@@ -67,6 +65,7 @@ export interface User extends Document {
   scimInfo?: { isSync: true } & Record<string, any>
   appFavourites?: string[]
   ssoId?: string
+  appSort?: string
 }
 
 export enum UserStatus {

packages/types/src/sdk/db.ts

@@ -133,7 +133,7 @@ export interface Database {
   exists(docId: string): Promise<boolean>
   getMultiple<T extends Document>(
     ids: string[],
-    opts?: { allowMissing?: boolean }
+    opts?: { allowMissing?: boolean; excludeDocs?: boolean }
   ): Promise<T[]>
   remove(idOrDoc: Document): Promise<Nano.DocumentDestroyResponse>
   remove(idOrDoc: string, rev?: string): Promise<Nano.DocumentDestroyResponse>

packages/types/src/sdk/row.ts

@@ -3,12 +3,33 @@ import { SearchFilters } from "./search"
 import { CalculationType, Row } from "../documents"
 import { WithRequired } from "../shared"
 
-export interface Aggregation {
+export interface BaseAggregation {
   name: string
-  calculationType: CalculationType
+}
+
+export interface NumericAggregation extends BaseAggregation {
+  calculationType:
+    | CalculationType.AVG
+    | CalculationType.MAX
+    | CalculationType.MIN
+    | CalculationType.SUM
   field: string
 }
 
+export interface CountAggregation extends BaseAggregation {
+  calculationType: CalculationType.COUNT
+}
+
+export interface CountDistinctAggregation extends CountAggregation {
+  distinct: true
+  field: string
+}
+
+export type Aggregation =
+  | NumericAggregation
+  | CountAggregation
+  | CountDistinctAggregation
+
 export interface SearchParams {
   tableId?: string
   viewId?: string

packages/worker/src/api/routes/global/tests/auditLogs.spec.ts

@@ -1,5 +1,5 @@
 import { mocks, structures } from "@budibase/backend-core/tests"
-import { context, events, setEnv as setCoreEnv } from "@budibase/backend-core"
+import { context, events, features } from "@budibase/backend-core"
 import { Event, IdentityType } from "@budibase/types"
 import { TestConfiguration } from "../../../../tests"
 
@@ -17,11 +17,9 @@ describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => {
   let envCleanup: (() => void) | undefined
 
   beforeAll(async () => {
-    if (method === "lucene") {
+    envCleanup = features.testutils.setFeatureFlags("*", {
-      envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" })
+      SQS: method === "sql",
-    } else if (method === "sql") {
+    })
-      envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" })
-    }
     await config.beforeAll()
   })
 

packages/worker/src/api/routes/global/tests/users.spec.ts

@@ -741,6 +741,25 @@ describe("/api/global/users", () => {
     it("should throw an error if public query performed", async () => {
       await config.api.users.searchUsers({}, { status: 403, noHeaders: true })
     })
+
+    it("should be able to search using logical conditions", async () => {
+      const user = await config.createUser()
+      const response = await config.api.users.searchUsers({
+        query: {
+          $and: {
+            conditions: [
+              {
+                $and: {
+                  conditions: [{ string: { email: user.email } }],
+                },
+              },
+            ],
+          },
+        },
+      })
+      expect(response.body.data.length).toBe(1)
+      expect(response.body.data[0].email).toBe(user.email)
+    })
   })
 
   describe("DELETE /api/global/users/:userId", () => {

packages/worker/src/api/routes/validation/users.ts

@@ -29,6 +29,7 @@ export const buildSelfSaveValidation = () => {
     freeTrialConfirmedAt: Joi.string().optional(),
     appFavourites: Joi.array().optional(),
     tours: Joi.object().optional(),
+    appSort: Joi.string().optional(),
   }
   return auth.joiValidator.body(Joi.object(schema).required().unknown(false))
 }