merging with v3-ui branch

.github/workflows/deploy-featurebranch.yml

@@ -14,15 +14,39 @@ jobs:
   release:
     if: |
       (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') &&
-      contains(github.event.pull_request.labels.*.name, 'feature-branch')
+      (
+        contains(github.event.pull_request.labels.*.name, 'feature-branch') ||
+        contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') ||
+        contains(github.event.pull_request.labels.*.name, 'feature-branch-team') ||
+        contains(github.event.pull_request.labels.*.name, 'feature-branch-business') ||
+        contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise')
+      )
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+
+      - name: Set PAYLOAD_LICENSE_TYPE
+        id: set_license_type
+        run: |
+          if [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch') }}" == "true" ]]; then
+            echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV
+          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') }}" == "true" ]]; then
+            echo "PAYLOAD_LICENSE_TYPE=pro" >> $GITHUB_ENV
+          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-team') }}" == "true" ]]; then
+            echo "PAYLOAD_LICENSE_TYPE=team" >> $GITHUB_ENV
+          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-business') }}" == "true" ]]; then
+            echo "PAYLOAD_LICENSE_TYPE=business" >> $GITHUB_ENV
+          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') }}" == "true" ]]; then
+            echo "PAYLOAD_LICENSE_TYPE=enterprise" >> $GITHUB_ENV
+          else
+            echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV
+          fi
+
       - uses: passeidireto/trigger-external-workflow-action@main
         env:
           PAYLOAD_BRANCH: ${{ github.head_ref }}
           PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
-          PAYLOAD_LICENSE_TYPE: "free"
+          PAYLOAD_LICENSE_TYPE: ${{ env.PAYLOAD_LICENSE_TYPE }}
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-deploy

charts/budibase/templates/app-service-deployment.yaml

@@ -184,6 +184,10 @@ spec:
         - name: NODE_DEBUG
           value: {{ .Values.services.apps.nodeDebug | quote }}
         {{ end }}
+        {{ if .Values.services.apps.xssSafeMode }}
+        - name: XSS_SAFE_MODE
+          value: {{ .Values.services.apps.xssSafeMode | quote }}
+        {{ end }}
         {{ if .Values.globals.datadogApmEnabled }}
         - name: DD_LOGS_INJECTION
           value: {{ .Values.globals.datadogApmEnabled | quote }}

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.11",
+  "version": "2.32.17",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/account-portal

@@ -1 +1 @@
-Subproject commit 4094517c07ad2b957895b801a7e1beaa7d894c4a
+Subproject commit 8cd052ce8288f343812a514d06c5a9459b3ba1a8

packages/backend-core/src/context/mainContext.ts

@@ -253,6 +253,11 @@ export function getAppId(): string | undefined {
   }
 }
 
+export function getIP(): string | undefined {
+  const context = Context.get()
+  return context?.ip
+}
+
 export const getProdAppId = () => {
   const appId = getAppId()
   if (!appId) {
@@ -281,6 +286,10 @@ export function doInScimContext(task: any) {
   return newContext(updates, task)
 }
 
+export function doInIPContext(ip: string, task: any) {
+  return newContext({ ip }, task)
+}
+
 export async function ensureSnippetContext(enabled = !env.isTest()) {
   const ctx = getCurrentContext()
 

packages/backend-core/src/context/types.ts

@@ -9,6 +9,7 @@ export type ContextMap = {
   identity?: IdentityContext
   environmentVariables?: Record<string, string>
   isScim?: boolean
+  ip?: string
   automationId?: string
   isMigrating?: boolean
   vm?: VM

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -211,19 +211,34 @@ export class DatabaseImpl implements Database {
     })
   }
 
+  async tryGet<T extends Document>(id?: string): Promise<T | undefined> {
+    try {
+      return await this.get<T>(id)
+    } catch (err: any) {
+      if (err.statusCode === 404) {
+        return undefined
+      }
+      throw err
+    }
+  }
+
   async getMultiple<T extends Document>(
     ids: string[],
-    opts?: { allowMissing?: boolean }
+    opts?: { allowMissing?: boolean; excludeDocs?: boolean }
   ): Promise<T[]> {
     // get unique
     ids = [...new Set(ids)]
+    const includeDocs = !opts?.excludeDocs
     const response = await this.allDocs<T>({
       keys: ids,
-      include_docs: true,
+      include_docs: includeDocs,
     })
     const rowUnavailable = (row: RowResponse<T>) => {
       // row is deleted - key lookup can return this
-      if (row.doc == null || ("deleted" in row.value && row.value.deleted)) {
+      if (
+        (includeDocs && row.doc == null) ||
+        (row.value && "deleted" in row.value && row.value.deleted)
+      ) {
         return true
       }
       return row.error === "not_found"
@@ -237,7 +252,7 @@ export class DatabaseImpl implements Database {
       const missingIds = missing.map(row => row.key).join(", ")
       throw new Error(`Unable to get documents: ${missingIds}`)
     }
-    return rows.map(row => row.doc!)
+    return rows.map(row => (includeDocs ? row.doc! : row.value))
   }
 
   async remove(idOrDoc: string | Document, rev?: string) {
@@ -371,11 +386,21 @@ export class DatabaseImpl implements Database {
     return this.performCall(() => {
       return async () => {
         const response = await directCouchUrlCall(args)
-        const json = await response.json()
+        const text = await response.text()
         if (response.status > 300) {
+          let json
+          try {
+            json = JSON.parse(text)
+          } catch (err) {
+            console.error(`SQS error: ${text}`)
+            throw new CouchDBError(
+              "error while running SQS query, please try again later",
+              { name: "sqs_error", status: response.status }
+            )
+          }
           throw json
         }
-        return json as T
+        return JSON.parse(text) as T
       }
     })
   }

packages/backend-core/src/db/instrumentation.ts

@@ -42,6 +42,13 @@ export class DDInstrumentedDatabase implements Database {
     })
   }
 
+  tryGet<T extends Document>(id?: string | undefined): Promise<T | undefined> {
+    return tracer.trace("db.tryGet", span => {
+      span?.addTags({ db_name: this.name, doc_id: id })
+      return this.db.tryGet(id)
+    })
+  }
+
   getMultiple<T extends Document>(
     ids: string[],
     opts?: { allowMissing?: boolean | undefined } | undefined

packages/backend-core/src/features/features.ts

@@ -0,0 +1,279 @@
+import env from "../environment"
+import * as crypto from "crypto"
+import * as context from "../context"
+import { PostHog, PostHogOptions } from "posthog-node"
+import { FeatureFlag } from "@budibase/types"
+import tracer from "dd-trace"
+import { Duration } from "../utils"
+
+let posthog: PostHog | undefined
+export function init(opts?: PostHogOptions) {
+  if (
+    env.POSTHOG_TOKEN &&
+    env.POSTHOG_API_HOST &&
+    !env.SELF_HOSTED &&
+    env.POSTHOG_FEATURE_FLAGS_ENABLED
+  ) {
+    console.log("initializing posthog client...")
+    posthog = new PostHog(env.POSTHOG_TOKEN, {
+      host: env.POSTHOG_API_HOST,
+      personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
+      featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(),
+      ...opts,
+    })
+  } else {
+    console.log("posthog disabled")
+  }
+}
+
+export function shutdown() {
+  posthog?.shutdown()
+}
+
+export abstract class Flag<T> {
+  static boolean(defaultValue: boolean): Flag<boolean> {
+    return new BooleanFlag(defaultValue)
+  }
+
+  static string(defaultValue: string): Flag<string> {
+    return new StringFlag(defaultValue)
+  }
+
+  static number(defaultValue: number): Flag<number> {
+    return new NumberFlag(defaultValue)
+  }
+
+  protected constructor(public defaultValue: T) {}
+
+  abstract parse(value: any): T
+}
+
+type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
+
+export type FlagValues<T> = {
+  [K in keyof T]: UnwrapFlag<T[K]>
+}
+
+type KeysOfType<T, U> = {
+  [K in keyof T]: T[K] extends Flag<U> ? K : never
+}[keyof T]
+
+class BooleanFlag extends Flag<boolean> {
+  parse(value: any) {
+    if (typeof value === "string") {
+      return ["true", "t", "1"].includes(value.toLowerCase())
+    }
+
+    if (typeof value === "boolean") {
+      return value
+    }
+
+    throw new Error(`could not parse value "${value}" as boolean`)
+  }
+}
+
+class StringFlag extends Flag<string> {
+  parse(value: any) {
+    if (typeof value === "string") {
+      return value
+    }
+    throw new Error(`could not parse value "${value}" as string`)
+  }
+}
+
+class NumberFlag extends Flag<number> {
+  parse(value: any) {
+    if (typeof value === "number") {
+      return value
+    }
+
+    if (typeof value === "string") {
+      const parsed = parseFloat(value)
+      if (!isNaN(parsed)) {
+        return parsed
+      }
+    }
+
+    throw new Error(`could not parse value "${value}" as number`)
+  }
+}
+
+export interface EnvFlagEntry {
+  tenantId: string
+  key: string
+  value: boolean
+}
+
+export function parseEnvFlags(flags: string): EnvFlagEntry[] {
+  const split = flags.split(",").map(x => x.split(":"))
+  const result: EnvFlagEntry[] = []
+  for (const [tenantId, ...features] of split) {
+    for (let feature of features) {
+      let value = true
+      if (feature.startsWith("!")) {
+        feature = feature.slice(1)
+        value = false
+      }
+      result.push({ tenantId, key: feature, value })
+    }
+  }
+  return result
+}
+
+export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
+  // This is used to safely cache flags sets in the current request context.
+  // Because multiple sets could theoretically exist, we don't want the cache of
+  // one to leak into another.
+  private readonly setId: string
+
+  constructor(private readonly flagSchema: T) {
+    this.setId = crypto.randomUUID()
+  }
+
+  defaults(): FlagValues<T> {
+    return Object.keys(this.flagSchema).reduce((acc, key) => {
+      const typedKey = key as keyof T
+      acc[typedKey] = this.flagSchema[key].defaultValue
+      return acc
+    }, {} as FlagValues<T>)
+  }
+
+  isFlagName(name: string | number | symbol): name is keyof T {
+    return this.flagSchema[name as keyof T] !== undefined
+  }
+
+  async get<K extends keyof T>(key: K): Promise<FlagValues<T>[K]> {
+    const flags = await this.fetch()
+    return flags[key]
+  }
+
+  async isEnabled<K extends KeysOfType<T, boolean>>(key: K): Promise<boolean> {
+    const flags = await this.fetch()
+    return flags[key]
+  }
+
+  async fetch(): Promise<FlagValues<T>> {
+    return await tracer.trace("features.fetch", async span => {
+      const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
+      if (cachedFlags) {
+        span?.addTags({ fromCache: true })
+        return cachedFlags
+      }
+
+      const tags: Record<string, any> = {}
+      const flagValues = this.defaults()
+      const currentTenantId = context.getTenantId()
+      const specificallySetFalse = new Set<string>()
+
+      for (const { tenantId, key, value } of parseEnvFlags(
+        env.TENANT_FEATURE_FLAGS || ""
+      )) {
+        if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
+          continue
+        }
+
+        tags[`readFromEnvironmentVars`] = true
+
+        if (value === false) {
+          specificallySetFalse.add(key)
+        }
+
+        // ignore unknown flags
+        if (!this.isFlagName(key)) {
+          continue
+        }
+
+        if (typeof flagValues[key] !== "boolean") {
+          throw new Error(`Feature: ${key} is not a boolean`)
+        }
+
+        // @ts-expect-error - TS does not like you writing into a generic type,
+        // but we know that it's okay in this case because it's just an object.
+        flagValues[key as keyof FlagValues] = value
+        tags[`flags.${key}.source`] = "environment"
+      }
+
+      const identity = context.getIdentity()
+
+      let userId = identity?._id
+      if (!userId) {
+        const ip = context.getIP()
+        if (ip) {
+          userId = crypto.createHash("sha512").update(ip).digest("hex")
+        }
+      }
+
+      let tenantId = identity?.tenantId
+      if (!tenantId) {
+        tenantId = currentTenantId
+      }
+
+      tags[`identity.type`] = identity?.type
+      tags[`identity._id`] = identity?._id
+      tags[`tenantId`] = tenantId
+      tags[`userId`] = userId
+
+      if (posthog && userId) {
+        tags[`readFromPostHog`] = true
+
+        const personProperties: Record<string, string> = { tenantId }
+        const posthogFlags = await posthog.getAllFlagsAndPayloads(userId, {
+          personProperties,
+        })
+
+        for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
+          if (!this.isFlagName(name)) {
+            // We don't want an unexpected PostHog flag to break the app, so we
+            // just log it and continue.
+            console.warn(`Unexpected posthog flag "${name}": ${value}`)
+            continue
+          }
+
+          if (flagValues[name] === true || specificallySetFalse.has(name)) {
+            // If the flag is already set to through environment variables, we
+            // don't want to override it back to false here.
+            continue
+          }
+
+          const payload = posthogFlags.featureFlagPayloads?.[name]
+          const flag = this.flagSchema[name]
+          try {
+            // @ts-expect-error - TS does not like you writing into a generic
+            // type, but we know that it's okay in this case because it's just
+            // an object.
+            flagValues[name] = flag.parse(payload || value)
+            tags[`flags.${name}.source`] = "posthog"
+          } catch (err) {
+            // We don't want an invalid PostHog flag to break the app, so we just
+            // log it and continue.
+            console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
+          }
+        }
+      }
+
+      context.setFeatureFlags(this.setId, flagValues)
+      for (const [key, value] of Object.entries(flagValues)) {
+        tags[`flags.${key}.value`] = value
+      }
+      span?.addTags(tags)
+
+      return flagValues
+    })
+  }
+}
+
+// This is the primary source of truth for feature flags. If you want to add a
+// new flag, add it here and use the `fetch` and `get` functions to access it.
+// All of the machinery in this file is to make sure that flags have their
+// default values set correctly and their types flow through the system.
+export const flags = new FlagSet({
+  DEFAULT_VALUES: Flag.boolean(env.isDev()),
+  AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
+  SQS: Flag.boolean(env.isDev()),
+  [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
+  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
+  [FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()),
+})
+
+type UnwrapPromise<T> = T extends Promise<infer U> ? U : T
+export type FeatureFlags = UnwrapPromise<ReturnType<typeof flags.fetch>>

packages/backend-core/src/features/index.ts

@@ -1,281 +1,2 @@
-import env from "../environment"
-import * as context from "../context"
-import { PostHog, PostHogOptions } from "posthog-node"
-import { FeatureFlag, IdentityType, UserCtx } from "@budibase/types"
-import tracer from "dd-trace"
-import { Duration } from "../utils"
-
-let posthog: PostHog | undefined
-export function init(opts?: PostHogOptions) {
-  if (
-    env.POSTHOG_TOKEN &&
-    env.POSTHOG_API_HOST &&
-    !env.SELF_HOSTED &&
-    env.POSTHOG_FEATURE_FLAGS_ENABLED
-  ) {
-    console.log("initializing posthog client...")
-    posthog = new PostHog(env.POSTHOG_TOKEN, {
-      host: env.POSTHOG_API_HOST,
-      personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
-      featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(),
-      ...opts,
-    })
-  } else {
-    console.log("posthog disabled")
-  }
-}
-
-export function shutdown() {
-  posthog?.shutdown()
-}
-
-export abstract class Flag<T> {
-  static boolean(defaultValue: boolean): Flag<boolean> {
-    return new BooleanFlag(defaultValue)
-  }
-
-  static string(defaultValue: string): Flag<string> {
-    return new StringFlag(defaultValue)
-  }
-
-  static number(defaultValue: number): Flag<number> {
-    return new NumberFlag(defaultValue)
-  }
-
-  protected constructor(public defaultValue: T) {}
-
-  abstract parse(value: any): T
-}
-
-type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
-
-export type FlagValues<T> = {
-  [K in keyof T]: UnwrapFlag<T[K]>
-}
-
-type KeysOfType<T, U> = {
-  [K in keyof T]: T[K] extends Flag<U> ? K : never
-}[keyof T]
-
-class BooleanFlag extends Flag<boolean> {
-  parse(value: any) {
-    if (typeof value === "string") {
-      return ["true", "t", "1"].includes(value.toLowerCase())
-    }
-
-    if (typeof value === "boolean") {
-      return value
-    }
-
-    throw new Error(`could not parse value "${value}" as boolean`)
-  }
-}
-
-class StringFlag extends Flag<string> {
-  parse(value: any) {
-    if (typeof value === "string") {
-      return value
-    }
-    throw new Error(`could not parse value "${value}" as string`)
-  }
-}
-
-class NumberFlag extends Flag<number> {
-  parse(value: any) {
-    if (typeof value === "number") {
-      return value
-    }
-
-    if (typeof value === "string") {
-      const parsed = parseFloat(value)
-      if (!isNaN(parsed)) {
-        return parsed
-      }
-    }
-
-    throw new Error(`could not parse value "${value}" as number`)
-  }
-}
-
-export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
-  // This is used to safely cache flags sets in the current request context.
-  // Because multiple sets could theoretically exist, we don't want the cache of
-  // one to leak into another.
-  private readonly setId: string
-
-  constructor(private readonly flagSchema: T) {
-    this.setId = crypto.randomUUID()
-  }
-
-  defaults(): FlagValues<T> {
-    return Object.keys(this.flagSchema).reduce((acc, key) => {
-      const typedKey = key as keyof T
-      acc[typedKey] = this.flagSchema[key].defaultValue
-      return acc
-    }, {} as FlagValues<T>)
-  }
-
-  isFlagName(name: string | number | symbol): name is keyof T {
-    return this.flagSchema[name as keyof T] !== undefined
-  }
-
-  async get<K extends keyof T>(
-    key: K,
-    ctx?: UserCtx
-  ): Promise<FlagValues<T>[K]> {
-    const flags = await this.fetch(ctx)
-    return flags[key]
-  }
-
-  async isEnabled<K extends KeysOfType<T, boolean>>(
-    key: K,
-    ctx?: UserCtx
-  ): Promise<boolean> {
-    const flags = await this.fetch(ctx)
-    return flags[key]
-  }
-
-  async fetch(ctx?: UserCtx): Promise<FlagValues<T>> {
-    return await tracer.trace("features.fetch", async span => {
-      const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
-      if (cachedFlags) {
-        span?.addTags({ fromCache: true })
-        return cachedFlags
-      }
-
-      const tags: Record<string, any> = {}
-      const flagValues = this.defaults()
-      const currentTenantId = context.getTenantId()
-      const specificallySetFalse = new Set<string>()
-
-      const split = (env.TENANT_FEATURE_FLAGS || "")
-        .split(",")
-        .map(x => x.split(":"))
-      for (const [tenantId, ...features] of split) {
-        if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
-          continue
-        }
-
-        tags[`readFromEnvironmentVars`] = true
-
-        for (let feature of features) {
-          let value = true
-          if (feature.startsWith("!")) {
-            feature = feature.slice(1)
-            value = false
-            specificallySetFalse.add(feature)
-          }
-
-          // ignore unknown flags
-          if (!this.isFlagName(feature)) {
-            continue
-          }
-
-          if (typeof flagValues[feature] !== "boolean") {
-            throw new Error(`Feature: ${feature} is not a boolean`)
-          }
-
-          // @ts-expect-error - TS does not like you writing into a generic type,
-          // but we know that it's okay in this case because it's just an object.
-          flagValues[feature as keyof FlagValues] = value
-          tags[`flags.${feature}.source`] = "environment"
-        }
-      }
-
-      const license = ctx?.user?.license
-      if (license) {
-        tags[`readFromLicense`] = true
-
-        for (const feature of license.features) {
-          if (!this.isFlagName(feature)) {
-            continue
-          }
-
-          if (
-            flagValues[feature] === true ||
-            specificallySetFalse.has(feature)
-          ) {
-            // If the flag is already set to through environment variables, we
-            // don't want to override it back to false here.
-            continue
-          }
-
-          // @ts-expect-error - TS does not like you writing into a generic type,
-          // but we know that it's okay in this case because it's just an object.
-          flagValues[feature] = true
-          tags[`flags.${feature}.source`] = "license"
-        }
-      }
-
-      const identity = context.getIdentity()
-      tags[`identity.type`] = identity?.type
-      tags[`identity.tenantId`] = identity?.tenantId
-      tags[`identity._id`] = identity?._id
-
-      if (posthog && identity?.type === IdentityType.USER) {
-        tags[`readFromPostHog`] = true
-
-        const personProperties: Record<string, string> = {}
-        if (identity.tenantId) {
-          personProperties.tenantId = identity.tenantId
-        }
-
-        const posthogFlags = await posthog.getAllFlagsAndPayloads(
-          identity._id,
-          {
-            personProperties,
-          }
-        )
-
-        for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
-          if (!this.isFlagName(name)) {
-            // We don't want an unexpected PostHog flag to break the app, so we
-            // just log it and continue.
-            console.warn(`Unexpected posthog flag "${name}": ${value}`)
-            continue
-          }
-
-          if (flagValues[name] === true || specificallySetFalse.has(name)) {
-            // If the flag is already set to through environment variables, we
-            // don't want to override it back to false here.
-            continue
-          }
-
-          const payload = posthogFlags.featureFlagPayloads?.[name]
-          const flag = this.flagSchema[name]
-          try {
-            // @ts-expect-error - TS does not like you writing into a generic
-            // type, but we know that it's okay in this case because it's just
-            // an object.
-            flagValues[name] = flag.parse(payload || value)
-            tags[`flags.${name}.source`] = "posthog"
-          } catch (err) {
-            // We don't want an invalid PostHog flag to break the app, so we just
-            // log it and continue.
-            console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
-          }
-        }
-      }
-
-      context.setFeatureFlags(this.setId, flagValues)
-      for (const [key, value] of Object.entries(flagValues)) {
-        tags[`flags.${key}.value`] = value
-      }
-      span?.addTags(tags)
-
-      return flagValues
-    })
-  }
-}
-
-// This is the primary source of truth for feature flags. If you want to add a
-// new flag, add it here and use the `fetch` and `get` functions to access it.
-// All of the machinery in this file is to make sure that flags have their
-// default values set correctly and their types flow through the system.
-export const flags = new FlagSet({
-  DEFAULT_VALUES: Flag.boolean(env.isDev()),
-  AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
-  SQS: Flag.boolean(env.isDev()),
-  [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
-  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
-})
+export * from "./features"
+export * as testutils from "./tests/utils"

packages/backend-core/src/features/tests/features.spec.ts

@@ -1,9 +1,10 @@
-import { IdentityContext, IdentityType, UserCtx } from "@budibase/types"
+import { IdentityContext, IdentityType } from "@budibase/types"
 import { Flag, FlagSet, FlagValues, init, shutdown } from "../"
 import * as context from "../../context"
 import environment, { withEnv } from "../../environment"
 import nodeFetch from "node-fetch"
 import nock from "nock"
+import * as crypto from "crypto"
 
 const schema = {
   TEST_BOOLEAN: Flag.boolean(false),
@@ -17,7 +18,6 @@ interface TestCase {
   identity?: Partial<IdentityContext>
   environmentFlags?: string
   posthogFlags?: PostHogFlags
-  licenseFlags?: Array<string>
   expected?: Partial<FlagValues<typeof schema>>
   errorMessage?: string | RegExp
 }
@@ -27,10 +27,14 @@ interface PostHogFlags {
   featureFlagPayloads?: Record<string, string>
 }
 
-function mockPosthogFlags(flags: PostHogFlags) {
+function mockPosthogFlags(
+  flags: PostHogFlags,
+  opts?: { token?: string; distinct_id?: string }
+) {
+  const { token = "test", distinct_id = "us_1234" } = opts || {}
   nock("https://us.i.posthog.com")
     .post("/decide/?v=3", body => {
-      return body.token === "test" && body.distinct_id === "us_1234"
+      return body.token === token && body.distinct_id === distinct_id
     })
     .reply(200, flags)
     .persist()
@@ -112,17 +116,6 @@ describe("feature flags", () => {
       },
       expected: { TEST_BOOLEAN: true },
     },
-    {
-      it: "should be able to set boolean flags through the license",
-      licenseFlags: ["TEST_BOOLEAN"],
-      expected: { TEST_BOOLEAN: true },
-    },
-    {
-      it: "should not be able to override a negative environment flag from license",
-      environmentFlags: "default:!TEST_BOOLEAN",
-      licenseFlags: ["TEST_BOOLEAN"],
-      expected: { TEST_BOOLEAN: false },
-    },
     {
       it: "should not error on unrecognised PostHog flag",
       posthogFlags: {
@@ -130,18 +123,12 @@ describe("feature flags", () => {
       },
       expected: flags.defaults(),
     },
-    {
-      it: "should not error on unrecognised license flag",
-      licenseFlags: ["UNDEFINED"],
-      expected: flags.defaults(),
-    },
   ])(
     "$it",
     async ({
       identity,
       environmentFlags,
       posthogFlags,
-      licenseFlags,
       expected,
       errorMessage,
     }) => {
@@ -157,8 +144,6 @@ describe("feature flags", () => {
         env.POSTHOG_API_HOST = "https://us.i.posthog.com"
       }
 
-      const ctx = { user: { license: { features: licenseFlags || [] } } }
-
       await withEnv(env, async () => {
         // We need to pass in node-fetch here otherwise nock won't get used
         // because posthog-node uses axios under the hood.
@@ -180,18 +165,13 @@ describe("feature flags", () => {
 
         await context.doInIdentityContext(fullIdentity, async () => {
           if (errorMessage) {
-            await expect(flags.fetch(ctx as UserCtx)).rejects.toThrow(
-              errorMessage
-            )
+            await expect(flags.fetch()).rejects.toThrow(errorMessage)
           } else if (expected) {
-            const values = await flags.fetch(ctx as UserCtx)
+            const values = await flags.fetch()
             expect(values).toMatchObject(expected)
 
             for (const [key, expectedValue] of Object.entries(expected)) {
-              const value = await flags.get(
-                key as keyof typeof schema,
-                ctx as UserCtx
-              )
+              const value = await flags.get(key as keyof typeof schema)
               expect(value).toBe(expectedValue)
             }
           } else {
@@ -214,6 +194,14 @@ describe("feature flags", () => {
       lastName: "User",
     }
 
+    // We need to pass in node-fetch here otherwise nock won't get used
+    // because posthog-node uses axios under the hood.
+    init({
+      fetch: (url, opts) => {
+        return nodeFetch(url, opts)
+      },
+    })
+
     nock("https://us.i.posthog.com")
       .post("/decide/?v=3", body => {
         return body.token === "test" && body.distinct_id === "us_1234"
@@ -230,4 +218,44 @@ describe("feature flags", () => {
       }
     )
   })
+
+  it("should still get flags when user is logged out", async () => {
+    const env: Partial<typeof environment> = {
+      SELF_HOSTED: false,
+      POSTHOG_FEATURE_FLAGS_ENABLED: "true",
+      POSTHOG_API_HOST: "https://us.i.posthog.com",
+      POSTHOG_TOKEN: "test",
+    }
+
+    const ip = "127.0.0.1"
+    const hashedIp = crypto.createHash("sha512").update(ip).digest("hex")
+
+    await withEnv(env, async () => {
+      mockPosthogFlags(
+        {
+          featureFlags: { TEST_BOOLEAN: true },
+        },
+        {
+          distinct_id: hashedIp,
+        }
+      )
+
+      // We need to pass in node-fetch here otherwise nock won't get used
+      // because posthog-node uses axios under the hood.
+      init({
+        fetch: (url, opts) => {
+          return nodeFetch(url, opts)
+        },
+      })
+
+      await context.doInIPContext(ip, async () => {
+        await context.doInTenant("default", async () => {
+          const result = await flags.fetch()
+          expect(result.TEST_BOOLEAN).toBe(true)
+        })
+      })
+
+      shutdown()
+    })
+  })
 })

packages/backend-core/src/features/tests/utils.ts

@@ -0,0 +1,64 @@
+import { FeatureFlags, parseEnvFlags } from ".."
+import { setEnv } from "../../environment"
+
+function getCurrentFlags(): Record<string, Record<string, boolean>> {
+  const result: Record<string, Record<string, boolean>> = {}
+  for (const { tenantId, key, value } of parseEnvFlags(
+    process.env.TENANT_FEATURE_FLAGS || ""
+  )) {
+    const tenantFlags = result[tenantId] || {}
+    // Don't allow overwriting specifically false flags, to match the beheaviour
+    // of FlagSet.
+    if (tenantFlags[key] === false) {
+      continue
+    }
+    tenantFlags[key] = value
+    result[tenantId] = tenantFlags
+  }
+  return result
+}
+
+function buildFlagString(
+  flags: Record<string, Record<string, boolean>>
+): string {
+  const parts: string[] = []
+  for (const [tenantId, tenantFlags] of Object.entries(flags)) {
+    for (const [key, value] of Object.entries(tenantFlags)) {
+      if (value === false) {
+        parts.push(`${tenantId}:!${key}`)
+      } else {
+        parts.push(`${tenantId}:${key}`)
+      }
+    }
+  }
+  return parts.join(",")
+}
+
+export function setFeatureFlags(
+  tenantId: string,
+  flags: Partial<FeatureFlags>
+): () => void {
+  const current = getCurrentFlags()
+  for (const [key, value] of Object.entries(flags)) {
+    const tenantFlags = current[tenantId] || {}
+    tenantFlags[key] = value
+    current[tenantId] = tenantFlags
+  }
+  const flagString = buildFlagString(current)
+  return setEnv({ TENANT_FEATURE_FLAGS: flagString })
+}
+
+export function withFeatureFlags<T>(
+  tenantId: string,
+  flags: Partial<FeatureFlags>,
+  f: () => T
+) {
+  const cleanup = setFeatureFlags(tenantId, flags)
+  const result = f()
+  if (result instanceof Promise) {
+    return result.finally(cleanup)
+  } else {
+    cleanup()
+    return result
+  }
+}

packages/backend-core/src/middleware/index.ts

@@ -20,3 +20,4 @@ export { default as correlation } from "../logging/correlation/middleware"
 export { default as errorHandling } from "./errorHandling"
 export { default as querystringToBody } from "./querystringToBody"
 export * as joiValidator from "./joi-validator"
+export { default as ip } from "./ip"

packages/backend-core/src/middleware/ip.ts

@@ -0,0 +1,12 @@
+import { Ctx } from "@budibase/types"
+import { doInIPContext } from "../context"
+
+export default async (ctx: Ctx, next: any) => {
+  if (ctx.ip) {
+    return await doInIPContext(ctx.ip, () => {
+      return next()
+    })
+  } else {
+    return next()
+  }
+}

packages/backend-core/src/middleware/passport/sso/sso.ts

@@ -2,7 +2,6 @@ import { generateGlobalUserID } from "../../../db"
 import { authError } from "../utils"
 import * as users from "../../../users"
 import * as context from "../../../context"
-import fetch from "node-fetch"
 import {
   SaveSSOUserFunction,
   SSOAuthDetails,
@@ -97,28 +96,13 @@ export async function authenticate(
   return done(null, ssoUser)
 }
 
-async function getProfilePictureUrl(user: User, details: SSOAuthDetails) {
-  const pictureUrl = details.profile?._json.picture
-  if (pictureUrl) {
-    const response = await fetch(pictureUrl)
-    if (response.status === 200) {
-      const type = response.headers.get("content-type") as string
-      if (type.startsWith("image/")) {
-        return pictureUrl
-      }
-    }
-  }
-}
-
 /**
  * @returns a user that has been sync'd with third party information
  */
 async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
   let firstName
   let lastName
-  let pictureUrl
   let oauth2
-  let thirdPartyProfile
 
   if (details.profile) {
     const profile = details.profile
@@ -134,12 +118,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
         lastName = name.familyName
       }
     }
-
-    pictureUrl = await getProfilePictureUrl(user, details)
-
-    thirdPartyProfile = {
-      ...profile._json,
-    }
   }
 
   // oauth tokens for future use
@@ -155,8 +133,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
     providerType: details.providerType,
     firstName,
     lastName,
-    thirdPartyProfile,
-    pictureUrl,
     oauth2,
   }
 }

packages/backend-core/src/security/permissions.ts

@@ -65,7 +65,13 @@ export enum BuiltinPermissionID {
   POWER = "power",
 }
 
-export const BUILTIN_PERMISSIONS = {
+export const BUILTIN_PERMISSIONS: {
+  [key in keyof typeof BuiltinPermissionID]: {
+    _id: (typeof BuiltinPermissionID)[key]
+    name: string
+    permissions: Permission[]
+  }
+} = {
   PUBLIC: {
     _id: BuiltinPermissionID.PUBLIC,
     name: "Public",

packages/backend-core/src/security/roles.ts

@@ -435,7 +435,9 @@ export function getExternalRoleID(roleId: string, version?: string) {
     roleId.startsWith(DocumentType.ROLE) &&
     (isBuiltin(roleId) || version === RoleIDVersion.NAME)
   ) {
-    return roleId.split(`${DocumentType.ROLE}${SEPARATOR}`)[1]
+    const parts = roleId.split(SEPARATOR)
+    parts.shift()
+    return parts.join(SEPARATOR)
   }
   return roleId
 }

packages/backend-core/src/sql/sql.ts

@@ -273,6 +273,7 @@ class InternalBuilder {
     const col = parts.pop()!
     const schema = this.table.schema[col]
     let identifier = this.quotedIdentifier(field)
+
     if (
       schema.type === FieldType.STRING ||
       schema.type === FieldType.LONGFORM ||
@@ -325,7 +326,7 @@ class InternalBuilder {
     return input
   }
 
-  private parseBody(body: any) {
+  private parseBody(body: Record<string, any>) {
     for (let [key, value] of Object.entries(body)) {
       const { column } = this.splitter.run(key)
       const schema = this.table.schema[column]
@@ -957,6 +958,13 @@ class InternalBuilder {
     return query
   }
 
+  isAggregateField(field: string): boolean {
+    const found = this.query.resource?.aggregations?.find(
+      aggregation => aggregation.name === field
+    )
+    return !!found
+  }
+
   addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder {
     let { sort, resource } = this.query
     const primaryKey = this.table.primary
@@ -979,6 +987,9 @@ class InternalBuilder {
           nulls = value.direction === SortOrder.ASCENDING ? "first" : "last"
         }
 
+        if (this.isAggregateField(key)) {
+          query = query.orderBy(key, direction, nulls)
+        } else {
           let composite = `${aliased}.${key}`
           if (this.client === SqlClient.ORACLE) {
             query = query.orderByRaw(
@@ -989,6 +1000,7 @@ class InternalBuilder {
           }
         }
       }
+    }
 
     // add sorting by the primary key if the result isn't already sorted by it,
     // to make sure result is deterministic
@@ -1259,6 +1271,10 @@ class InternalBuilder {
 
   create(opts: QueryOptions): Knex.QueryBuilder {
     const { body } = this.query
+    if (!body) {
+      throw new Error("Cannot create without row body")
+    }
+
     let query = this.qualifiedKnex({ alias: false })
     const parsedBody = this.parseBody(body)
 
@@ -1418,6 +1434,9 @@ class InternalBuilder {
 
   update(opts: QueryOptions): Knex.QueryBuilder {
     const { body, filters } = this.query
+    if (!body) {
+      throw new Error("Cannot update without row body")
+    }
     let query = this.qualifiedKnex()
     const parsedBody = this.parseBody(body)
     query = this.addFilters(query, filters)

packages/backend-core/src/users/users.ts

@@ -24,6 +24,7 @@ import * as context from "../context"
 import { getGlobalDB } from "../context"
 import { isCreator } from "./utils"
 import { UserDB } from "./db"
+import { dataFilters } from "@budibase/shared-core"
 
 type GetOpts = { cleanup?: boolean }
 
@@ -262,10 +263,17 @@ export async function paginatedUsers({
     userList = await bulkGetGlobalUsersById(query?.oneOf?._id, {
       cleanup: true,
     })
+  } else if (query) {
+    // TODO: this should use SQS search, but the logic is built in the 'server' package. Using the in-memory filtering to get this working meanwhile
+    const response = await db.allDocs<User>(
+      getGlobalUserParams(null, { ...opts, limit: undefined })
+    )
+    userList = response.rows.map(row => row.doc!)
+    userList = dataFilters.search(userList, { query, limit: opts.limit }).rows
   } else {
     // no search, query allDocs
-    const response = await db.allDocs(getGlobalUserParams(null, opts))
-    userList = response.rows.map((row: any) => row.doc)
+    const response = await db.allDocs<User>(getGlobalUserParams(null, opts))
+    userList = response.rows.map(row => row.doc!)
   }
   return pagination(userList, pageSize, {
     paginate: true,

packages/backend-core/tests/core/utilities/structures/accounts.ts

@@ -6,9 +6,6 @@ import {
   AccountSSOProviderType,
   AuthType,
   CloudAccount,
-  CreateAccount,
-  CreatePassswordAccount,
-  CreateVerifiableSSOAccount,
   Hosting,
   SSOAccount,
 } from "@budibase/types"
@@ -19,6 +16,7 @@ export const account = (partial: Partial<Account> = {}): Account => {
     accountId: uuid(),
     tenantId: generator.word(),
     email: generator.email({ domain: "example.com" }),
+    accountName: generator.word(),
     tenantName: generator.word(),
     hosting: Hosting.SELF,
     createdAt: Date.now(),
@@ -61,10 +59,8 @@ export function ssoAccount(account: Account = cloudAccount()): SSOAccount {
       accessToken: generator.string(),
       refreshToken: generator.string(),
     },
-    pictureUrl: generator.url(),
     provider: provider(),
     providerType: providerType(),
-    thirdPartyProfile: {},
   }
 }
 
@@ -78,68 +74,7 @@ export function verifiableSsoAccount(
       accessToken: generator.string(),
       refreshToken: generator.string(),
     },
-    pictureUrl: generator.url(),
     provider: AccountSSOProvider.MICROSOFT,
     providerType: AccountSSOProviderType.MICROSOFT,
-    thirdPartyProfile: { id: "abc123" },
   }
 }
-
-export const cloudCreateAccount: CreatePassswordAccount = {
-  email: "cloud@budibase.com",
-  tenantId: "cloud",
-  hosting: Hosting.CLOUD,
-  authType: AuthType.PASSWORD,
-  password: "Password123!",
-  tenantName: "cloud",
-  name: "Budi Armstrong",
-  size: "10+",
-  profession: "Software Engineer",
-}
-
-export const cloudSSOCreateAccount: CreateAccount = {
-  email: "cloud-sso@budibase.com",
-  tenantId: "cloud-sso",
-  hosting: Hosting.CLOUD,
-  authType: AuthType.SSO,
-  tenantName: "cloudsso",
-  name: "Budi Armstrong",
-  size: "10+",
-  profession: "Software Engineer",
-}
-
-export const cloudVerifiableSSOCreateAccount: CreateVerifiableSSOAccount = {
-  email: "cloud-sso@budibase.com",
-  tenantId: "cloud-sso",
-  hosting: Hosting.CLOUD,
-  authType: AuthType.SSO,
-  tenantName: "cloudsso",
-  name: "Budi Armstrong",
-  size: "10+",
-  profession: "Software Engineer",
-  provider: AccountSSOProvider.MICROSOFT,
-  thirdPartyProfile: { id: "abc123" },
-}
-
-export const selfCreateAccount: CreatePassswordAccount = {
-  email: "self@budibase.com",
-  tenantId: "self",
-  hosting: Hosting.SELF,
-  authType: AuthType.PASSWORD,
-  password: "Password123!",
-  tenantName: "self",
-  name: "Budi Armstrong",
-  size: "10+",
-  profession: "Software Engineer",
-}
-
-export const selfSSOCreateAccount: CreateAccount = {
-  email: "self-sso@budibase.com",
-  tenantId: "self-sso",
-  hosting: Hosting.SELF,
-  authType: AuthType.SSO,
-  tenantName: "selfsso",
-  name: "Budi Armstrong",
-  size: "10+",
-  profession: "Software Engineer",
-}

packages/backend-core/tests/core/utilities/structures/users.ts

@@ -25,7 +25,6 @@ export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
     roles: { app_test: "admin" },
     firstName: generator.first(),
     lastName: generator.last(),
-    pictureUrl: "http://example.com",
     tenantId: tenant.id(),
     ...userProps,
   }
@@ -86,9 +85,5 @@ export function ssoUser(
     oauth2: opts.details?.oauth2,
     provider: opts.details?.provider!,
     providerType: opts.details?.providerType!,
-    thirdPartyProfile: {
-      email: base.email,
-      picture: base.pictureUrl,
-    },
   }
 }

packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte

@@ -124,7 +124,7 @@
         subtype: column.subtype,
         visible: column.visible,
         readonly: column.readonly,
-        constraints: column.constraints, // This is needed to properly display "users" column
+        icon: column.icon,
       },
     }
   })

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -52,19 +52,13 @@
   import { isEnabled } from "helpers/featureFlags"
   import { getUserBindings } from "dataBinding"
 
-  const AUTO_TYPE = FieldType.AUTO
-  const FORMULA_TYPE = FieldType.FORMULA
-  const LINK_TYPE = FieldType.LINK
-  const STRING_TYPE = FieldType.STRING
-  const NUMBER_TYPE = FieldType.NUMBER
-  const JSON_TYPE = FieldType.JSON
-  const DATE_TYPE = FieldType.DATETIME
-  const AI_TYPE = FieldType.AI
+  export let field
 
   const dispatch = createEventDispatcher()
   const { dispatch: gridDispatch, rows } = getContext("grid")
-
-  export let field
+  const SafeID = `${makePropSafe("user")}.${makePropSafe("_id")}`
+  const SingleUserDefault = `{{ ${SafeID} }}`
+  const MultiUserDefault = `{{ js "${btoa(`return [$("${SafeID}")]`)}" }}`
 
   let mounted = false
   let originalName
@@ -115,7 +109,7 @@
   $: {
     // this parses any changes the user has made when creating a new internal relationship
     // into what we expect the schema to look like
-    if (editableColumn.type === LINK_TYPE) {
+    if (editableColumn.type === FieldType.LINK) {
       relationshipTableIdPrimary = table._id
       if (relationshipPart1 === PrettyRelationshipDefinitions.ONE) {
         relationshipOpts2 = relationshipOpts2.filter(
@@ -152,7 +146,7 @@
     UNEDITABLE_USER_FIELDS.includes(editableColumn.name)
   $: invalid =
     !editableColumn?.name ||
-    (editableColumn?.type === LINK_TYPE && !editableColumn?.tableId) ||
+    (editableColumn?.type === FieldType.LINK && !editableColumn?.tableId) ||
     Object.keys(errors).length !== 0 ||
     !optionsValid
   $: errors = checkErrors(editableColumn)
@@ -178,9 +172,9 @@
   $: defaultValuesEnabled = isEnabled("DEFAULT_VALUES")
   $: canHaveDefault = !required && canHaveDefaultColumn(editableColumn.type)
   $: canBeRequired =
-    editableColumn?.type !== LINK_TYPE &&
+    editableColumn?.type !== FieldType.LINK &&
     !uneditable &&
-    editableColumn?.type !== AUTO_TYPE &&
+    editableColumn?.type !== FieldType.AUTO &&
     !editableColumn.autocolumn
   $: hasDefault =
     editableColumn?.default != null && editableColumn?.default !== ""
@@ -229,7 +223,7 @@
 
   function makeFieldId(type, subtype, autocolumn) {
     // don't make field IDs for auto types
-    if (type === AUTO_TYPE || autocolumn) {
+    if (type === FieldType.AUTO || autocolumn) {
       return type.toUpperCase()
     } else if (
       type === FieldType.BB_REFERENCE ||
@@ -254,7 +248,7 @@
       // Here we are setting the relationship values based on the editableColumn
       // This part of the code is used when viewing an existing field hence the check
       // for the tableId
-      if (editableColumn.type === LINK_TYPE && editableColumn.tableId) {
+      if (editableColumn.type === FieldType.LINK && editableColumn.tableId) {
         relationshipTableIdPrimary = table._id
         relationshipTableIdSecondary = editableColumn.tableId
         if (editableColumn.relationshipType in relationshipMap) {
@@ -295,14 +289,14 @@
 
     delete saveColumn.fieldId
 
-    if (saveColumn.type === AUTO_TYPE) {
+    if (saveColumn.type === FieldType.AUTO) {
       saveColumn = buildAutoColumn(
         $tables.selected.name,
         saveColumn.name,
         saveColumn.subtype
       )
     }
-    if (saveColumn.type !== LINK_TYPE) {
+    if (saveColumn.type !== FieldType.LINK) {
       delete saveColumn.fieldName
     }
 
@@ -389,9 +383,9 @@
     editableColumn.subtype = definition.subtype
 
     // Default relationships many to many
-    if (editableColumn.type === LINK_TYPE) {
+    if (editableColumn.type === FieldType.LINK) {
       editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
-    } else if (editableColumn.type === FORMULA_TYPE) {
+    } else if (editableColumn.type === FieldType.FORMULA) {
       editableColumn.formulaType = "dynamic"
     }
   }
@@ -511,17 +505,23 @@
       fieldToCheck.constraints = {}
     }
     // some string types may have been built by server, may not always have constraints
-    if (fieldToCheck.type === STRING_TYPE && !fieldToCheck.constraints.length) {
+    if (
+      fieldToCheck.type === FieldType.STRING &&
+      !fieldToCheck.constraints.length
+    ) {
       fieldToCheck.constraints.length = {}
     }
     // some number types made server-side will be missing constraints
     if (
-      fieldToCheck.type === NUMBER_TYPE &&
+      fieldToCheck.type === FieldType.NUMBER &&
       !fieldToCheck.constraints.numericality
     ) {
       fieldToCheck.constraints.numericality = {}
     }
-    if (fieldToCheck.type === DATE_TYPE && !fieldToCheck.constraints.datetime) {
+    if (
+      fieldToCheck.type === FieldType.DATETIME &&
+      !fieldToCheck.constraints.datetime
+    ) {
       fieldToCheck.constraints.datetime = {}
     }
   }
@@ -596,13 +596,13 @@
       on:input={e => {
         if (
           !uneditable &&
-          !(linkEditDisabled && editableColumn.type === LINK_TYPE)
+          !(linkEditDisabled && editableColumn.type === FieldType.LINK)
         ) {
           editableColumn.name = e.target.value
         }
       }}
       disabled={uneditable ||
-        (linkEditDisabled && editableColumn.type === LINK_TYPE)}
+        (linkEditDisabled && editableColumn.type === FieldType.LINK)}
       error={errors?.name}
     />
   {/if}
@@ -616,7 +616,7 @@
     getOptionValue={field => field.fieldId}
     getOptionIcon={field => field.icon}
     isOptionEnabled={option => {
-      if (option.type === AUTO_TYPE) {
+      if (option.type === FieldType.AUTO) {
         return availableAutoColumnKeys?.length > 0
       }
       return true
@@ -659,7 +659,7 @@
       bind:optionColors={editableColumn.optionColors}
       bind:valid={optionsValid}
     />
-  {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn}
+  {:else if editableColumn.type === FieldType.DATETIME && !editableColumn.autocolumn}
     <div class="split-label">
       <div class="label-length">
         <Label size="M">Earliest</Label>
@@ -746,7 +746,7 @@
       {tableOptions}
       {errors}
     />
-  {:else if editableColumn.type === FORMULA_TYPE}
+  {:else if editableColumn.type === FieldType.FORMULA}
     {#if !externalTable}
       <div class="split-label">
         <div class="label-length">
@@ -789,19 +789,19 @@
         />
       </div>
     </div>
-  {:else if editableColumn.type === AI_TYPE}
+  {:else if editableColumn.type === FieldType.AI}
     <AIFieldConfiguration
       aiField={editableColumn}
       context={rowGoldenSample}
       bindings={getBindings({ table })}
       schema={table.schema}
     />
-  {:else if editableColumn.type === JSON_TYPE}
+  {:else if editableColumn.type === FieldType.JSON}
     <Button primary text on:click={openJsonSchemaEditor}>
       Open schema editor
     </Button>
   {/if}
-  {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
+  {#if editableColumn.type === FieldType.AUTO || editableColumn.autocolumn}
     <Select
       label="Auto column type"
       value={editableColumn.subtype}
@@ -848,6 +848,18 @@
           (editableColumn.default = e.detail?.length ? e.detail : undefined)}
         placeholder="None"
       />
+    {:else if editableColumn.subtype === BBReferenceFieldSubType.USER}
+      {@const defaultValue =
+        editableColumn.type === FieldType.BB_REFERENCE_SINGLE
+          ? SingleUserDefault
+          : MultiUserDefault}
+      <Toggle
+        disabled={!canHaveDefault}
+        text="Default to current user"
+        value={editableColumn.default === defaultValue}
+        on:change={e =>
+          (editableColumn.default = e.detail ? defaultValue : undefined)}
+      />
     {:else}
       <ModalBindableInput
         disabled={!canHaveDefault}

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte

@@ -33,7 +33,7 @@
 
   const getSchemaFields = resourceId => {
     const { schema } = getSchemaForDatasourcePlus(resourceId)
-    return Object.values(schema || {})
+    return Object.values(schema || {}).filter(field => !field.readonly)
   }
 
   const onFieldsChanged = e => {

packages/builder/src/components/portal/licensing/EnterpriseBasicTrialBanner.svelte

@@ -14,7 +14,13 @@
   function daysUntilCancel() {
     const cancelAt = license?.billing?.subscription?.cancelAt
     const diffTime = Math.abs(cancelAt - new Date().getTime()) / 1000
-    return Math.floor(diffTime / oneDayInSeconds)
+    const days = Math.floor(diffTime / oneDayInSeconds)
+    if (days === 1) {
+      return "tomorrow."
+    } else if (days === 0) {
+      return "today."
+    }
+    return `in ${days} days.`
   }
 </script>
 
@@ -28,7 +34,7 @@
           extraLinkAction={$licensing.goToUpgradePage}
           showCloseButton={false}
         >
-          Your free trial will end in {daysUntilCancel()} days.
+          Your free trial will end {daysUntilCancel()}
         </Banner>
       </div>
     {/if}

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Tables/CreateExternalTableModal.svelte

@@ -23,6 +23,7 @@
       sourceType: DB_TYPE_EXTERNAL,
       schema: {
         id: {
+          name: "id",
           autocolumn: true,
           type: "number",
         },

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/DeleteViewModal.svelte

@@ -20,6 +20,7 @@
       }
       notifications.success("View deleted")
     } catch (error) {
+      console.error(error)
       notifications.error("Error deleting view")
     }
   }

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte

@@ -196,7 +196,7 @@
     on:contextmenu={openTableContextMenu}
   >
     <div class="nav-item__title">
-      {table._id === TableNames.USERS ? "App users" : table.name}
+      {table?._id === TableNames.USERS ? "App users" : table?.name || ""}
     </div>
     {#if tableSelectedBy}
       <UserAvatars size="XS" users={tableSelectedBy} />

packages/builder/src/stores/builder/viewsV2.js

@@ -5,17 +5,23 @@ import { dataFilters } from "@budibase/shared-core"
 
 function convertToSearchFilters(view) {
   // convert from SearchFilterGroup type
-  if (view.query) {
-    view.queryUI = view.query
-    view.query = dataFilters.buildQuery(view.query)
+  if (view?.query) {
+    return {
+      ...view,
+      queryUI: view.query,
+      query: dataFilters.buildQuery(view.query),
+    }
   }
   return view
 }
 
 function convertToSearchFilterGroup(view) {
-  if (view.queryUI) {
-    view.query = view.queryUI
-    delete view.queryUI
+  if (view?.queryUI) {
+    return {
+      ...view,
+      query: view.queryUI,
+      queryUI: undefined,
+    }
   }
   return view
 }

packages/client/src/components/app/container/GridContainer.svelte

@@ -4,6 +4,8 @@
   import { GridRowHeight, GridColumns } from "constants"
   import { memo } from "@budibase/frontend-core"
 
+  export let onClick
+
   const component = getContext("component")
   const { styleable, builderStore } = getContext("sdk")
   const context = getContext("context")
@@ -121,15 +123,19 @@
   })
 </script>
 
+<!-- svelte-ignore a11y-no-static-element-interactions -->
+<!-- svelte-ignore a11y-click-events-have-key-events -->
 <div
   bind:this={ref}
   class="grid"
   class:mobile
+  class:clickable={!!onClick}
   bind:clientWidth={width}
   bind:clientHeight={height}
   use:styleable={$styles}
   data-cols={GridColumns}
   data-col-size={colSize}
+  on:click={onClick}
 >
   {#if inBuilder}
     <div class="underlay">
@@ -176,6 +182,9 @@
   .placeholder.first-col {
     border-left: 1px solid var(--spectrum-global-color-gray-900);
   }
+  .clickable {
+    cursor: pointer;
+  }
 
   /* Highlight grid lines when resizing children */
   :global(.grid.highlight > .underlay) {

packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte

@@ -27,10 +27,7 @@
   let candidateIndex
   let lastSearchId
   let searching = false
-  let container
   let anchor
-  let relationshipAnchor
-  let relationshipFields
 
   $: fieldValue = parseValue(value)
   $: oneRowOnly = schema?.relationshipType === "one-to-many"
@@ -57,13 +54,6 @@
     return acc
   }, {})
 
-  $: showRelationshipFields =
-    relationshipAnchor &&
-    relationshipFields &&
-    Object.keys(relationshipFields).length &&
-    focused &&
-    !isOpen
-
   const parseValue = value => {
     if (Array.isArray(value) && value.every(x => x?._id)) {
       return value
@@ -208,7 +198,6 @@
 
   // Toggles whether a row is included in the relationship or not
   const toggleRow = async row => {
-    hideRelationshipFields()
     if (fieldValue?.some(x => x._id === row._id)) {
       // If the row is already included, remove it and update the candidate
       // row to be the same position if possible
@@ -245,16 +234,6 @@
     return value
   }
 
-  const displayRelationshipFields = (e, relationship) => {
-    relationshipAnchor = e.target
-    relationshipFields = relationFields[relationship._id]
-  }
-
-  const hideRelationshipFields = () => {
-    relationshipAnchor = null
-    relationshipFields = undefined
-  }
-
   onMount(() => {
     api = {
       focus: open,
@@ -274,7 +253,7 @@
   style="--color:{color};"
   bind:this={anchor}
 >
-  <div class="container" bind:this={container}>
+  <div class="container">
     <div
       class="values"
       class:wrap={editable || contentLines > 1}
@@ -286,9 +265,7 @@
           <div
             class="badge"
             class:extra-info={!!relationFields[relationship._id]}
-            on:mouseenter={e => displayRelationshipFields(e, relationship)}
             on:focus={() => {}}
-            on:mouseleave={() => hideRelationshipFields()}
           >
             <span>
               {readable(
@@ -363,26 +340,6 @@
   </GridPopover>
 {/if}
 
-{#if showRelationshipFields}
-  <GridPopover
-    anchor={relationshipAnchor}
-    maxWidth={400}
-    offset={4}
-    clickOutsideOverride
-  >
-    <div class="relationship-fields">
-      {#each Object.entries(relationshipFields) as [fieldName, fieldValue]}
-        <div class="relationship-field-name">
-          {fieldName}
-        </div>
-        <div class="relationship-field-value">
-          {fieldValue}
-        </div>
-      {/each}
-    </div>
-  </GridPopover>
-{/if}
-
 <style>
   .wrapper {
     flex: 1 1 auto;
@@ -547,29 +504,4 @@
   .search :global(.spectrum-Form-item) {
     flex: 1 1 auto;
   }
-
-  .relationship-fields {
-    margin: var(--spacing-m) var(--spacing-l);
-    display: grid;
-    grid-template-columns: auto 1fr;
-    grid-row-gap: var(--spacing-m);
-    grid-column-gap: var(--spacing-xl);
-  }
-
-  .relationship-field-name {
-    text-transform: uppercase;
-    color: var(--spectrum-global-color-gray-600);
-    font-size: 12px;
-    overflow: hidden;
-    white-space: nowrap;
-    text-overflow: ellipsis;
-    max-width: 120px;
-  }
-  .relationship-field-value {
-    overflow: hidden;
-    display: -webkit-box;
-    -webkit-box-orient: vertical;
-    -webkit-line-clamp: 3;
-    line-clamp: 3;
-  }
 </style>

packages/frontend-core/src/components/grid/stores/datasources/viewV2.js

@@ -3,9 +3,12 @@ import { dataFilters } from "@budibase/shared-core"
 
 function convertToSearchFilters(view) {
   // convert from SearchFilterGroup type
-  if (view.query) {
-    view.queryUI = view.query
-    view.query = dataFilters.buildQuery(view.query)
+  if (view?.query) {
+    return {
+      ...view,
+      queryUI: view.query,
+      query: dataFilters.buildQuery(view.query),
+    }
   }
   return view
 }
@@ -191,7 +194,7 @@ export const initialise = context => {
         if ($view?.id !== $datasource.id) {
           return
         }
-        if (JSON.stringify($filter) !== JSON.stringify($view.query)) {
+        if (JSON.stringify($filter) !== JSON.stringify($view.queryUI)) {
           await datasource.actions.saveDefinition({
             ...$view,
             query: $filter,

packages/frontend-core/src/utils/relatedColumns.js

@@ -24,9 +24,9 @@ const columnTypeManyParser = {
       return parsed
     }
 
-    return value?.map(v => parseDate(v))
+    return value.map(v => parseDate(v))
   },
-  [FieldType.BOOLEAN]: value => value?.map(v => !!v),
+  [FieldType.BOOLEAN]: value => value.map(v => !!v),
   [FieldType.BB_REFERENCE_SINGLE]: value => [
     ...new Map(value.map(i => [i._id, i])).values(),
   ],
@@ -80,14 +80,10 @@ export function getRelatedTableValues(row, field, fromField) {
     result = row[field.related.field]?.[0]?.[field.related.subField]
   } else {
     const parser = columnTypeManyParser[field.type] || (value => value)
-
-    result = parser(
-      row[field.related.field]
+    const value = row[field.related.field]
       ?.flatMap(r => r[field.related.subField])
-        ?.filter(i => i !== undefined && i !== null),
-      field
-    )
-
+      ?.filter(i => i !== undefined && i !== null)
+    result = parser(value || [], field)
     if (
       [
         FieldType.STRING,

packages/frontend-core/src/utils/schema.js

@@ -2,6 +2,10 @@ import { helpers } from "@budibase/shared-core"
 import { TypeIconMap } from "../constants"
 
 export const getColumnIcon = column => {
+  if (column.schema.icon) {
+    return column.schema.icon
+  }
+
   if (column.schema.autocolumn) {
     return "MagicWand"
   }

packages/server/src/api/controllers/permission.ts

@@ -1,7 +1,6 @@
 import { permissions, roles, context } from "@budibase/backend-core"
 import {
   UserCtx,
-  Role,
   GetResourcePermsResponse,
   ResourcePermissionInfo,
   GetDependantResourcesResponse,
@@ -9,6 +8,7 @@ import {
   AddPermissionRequest,
   RemovePermissionRequest,
   RemovePermissionResponse,
+  FetchResourcePermissionInfoResponse,
 } from "@budibase/types"
 import {
   CURRENTLY_SUPPORTED_LEVELS,
@@ -28,10 +28,12 @@ export function fetchLevels(ctx: UserCtx) {
   ctx.body = SUPPORTED_LEVELS
 }
 
-export async function fetch(ctx: UserCtx) {
+export async function fetch(
+  ctx: UserCtx<void, FetchResourcePermissionInfoResponse>
+) {
   const db = context.getAppDB()
-  const dbRoles: Role[] = await sdk.permissions.getAllDBRoles(db)
-  let permissions: any = {}
+  const dbRoles = await sdk.permissions.getAllDBRoles(db)
+  let permissions: Record<string, Record<string, string>> = {}
   // create an object with structure role ID -> resource ID -> level
   for (let role of dbRoles) {
     if (!role.permissions) {
@@ -43,13 +45,13 @@ export async function fetch(ctx: UserCtx) {
     }
     for (let [resource, levelArr] of Object.entries(role.permissions)) {
       const levels: string[] = Array.isArray(levelArr) ? levelArr : [levelArr]
-      const perms: Record<string, string> = {}
+      const perms: Record<string, string> = permissions[resource] || {}
       levels.forEach(level => (perms[level] = roleId!))
       permissions[resource] = perms
     }
   }
   // apply the base permissions
-  const finalPermissions: Record<string, Record<string, string>> = {}
+  const finalPermissions: FetchResourcePermissionInfoResponse = {}
   for (let [resource, permission] of Object.entries(permissions)) {
     const basePerms = getBasePermissions(resource)
     finalPermissions[resource] = Object.assign(basePerms, permission)
@@ -92,18 +94,17 @@ export async function getDependantResources(
 
 export async function addPermission(ctx: UserCtx<void, AddPermissionResponse>) {
   const params: AddPermissionRequest = ctx.params
-  ctx.body = await sdk.permissions.updatePermissionOnRole(
-    params,
-    PermissionUpdateType.ADD
-  )
+  await sdk.permissions.updatePermissionOnRole(params, PermissionUpdateType.ADD)
+  ctx.status = 200
 }
 
 export async function removePermission(
   ctx: UserCtx<void, RemovePermissionResponse>
 ) {
   const params: RemovePermissionRequest = ctx.params
-  ctx.body = await sdk.permissions.updatePermissionOnRole(
+  await sdk.permissions.updatePermissionOnRole(
     params,
     PermissionUpdateType.REMOVE
   )
+  ctx.status = 200
 }

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -4,6 +4,7 @@ import {
   AutoFieldSubType,
   AutoReason,
   Datasource,
+  DatasourcePlusQueryResponse,
   FieldSchema,
   FieldType,
   FilterType,
@@ -270,18 +271,13 @@ export class ExternalRequest<T extends Operation> {
     }
   }
 
-  private async removeManyToManyRelationships(
-    rowId: string,
-    table: Table,
-    colName: string
-  ) {
+  private async removeManyToManyRelationships(rowId: string, table: Table) {
     const tableId = table._id!
     const filters = this.prepareFilters(rowId, {}, table)
     // safety check, if there are no filters on deletion bad things happen
     if (Object.keys(filters).length !== 0) {
       return getDatasourceAndQuery({
         endpoint: getEndpoint(tableId, Operation.DELETE),
-        body: { [colName]: null },
         filters,
         meta: {
           table,
@@ -292,13 +288,18 @@ export class ExternalRequest<T extends Operation> {
     }
   }
 
-  private async removeOneToManyRelationships(rowId: string, table: Table) {
+  private async removeOneToManyRelationships(
+    rowId: string,
+    table: Table,
+    colName: string
+  ) {
     const tableId = table._id!
     const filters = this.prepareFilters(rowId, {}, table)
     // safety check, if there are no filters on deletion bad things happen
     if (Object.keys(filters).length !== 0) {
       return getDatasourceAndQuery({
         endpoint: getEndpoint(tableId, Operation.UPDATE),
+        body: { [colName]: null },
         filters,
         meta: {
           table,
@@ -558,8 +559,9 @@ export class ExternalRequest<T extends Operation> {
           return matchesPrimaryLink
         }
 
-        const matchesSecondayLink = row[linkSecondary] === body?.[linkSecondary]
-        return matchesPrimaryLink && matchesSecondayLink
+        const matchesSecondaryLink =
+          row[linkSecondary] === body?.[linkSecondary]
+        return matchesPrimaryLink && matchesSecondaryLink
       }
 
       const existingRelationship = rows.find((row: { [key: string]: any }) =>
@@ -596,8 +598,8 @@ export class ExternalRequest<T extends Operation> {
       for (let row of rows) {
         const rowId = generateIdForRow(row, table)
         const promise: Promise<any> = isMany
-          ? this.removeManyToManyRelationships(rowId, table, colName)
-          : this.removeOneToManyRelationships(rowId, table)
+          ? this.removeManyToManyRelationships(rowId, table)
+          : this.removeOneToManyRelationships(rowId, table, colName)
         if (promise) {
           promises.push(promise)
         }
@@ -620,12 +622,12 @@ export class ExternalRequest<T extends Operation> {
         rows.map(row => {
           const rowId = generateIdForRow(row, table)
           return isMany
-            ? this.removeManyToManyRelationships(
+            ? this.removeManyToManyRelationships(rowId, table)
+            : this.removeOneToManyRelationships(
                 rowId,
                 table,
                 relationshipColumn.fieldName
               )
-            : this.removeOneToManyRelationships(rowId, table)
         })
       )
     }
@@ -670,6 +672,7 @@ export class ExternalRequest<T extends Operation> {
       config.includeSqlRelationships === IncludeRelationship.INCLUDE
 
     // clean up row on ingress using schema
+    const unprocessedRow = config.row
     const processed = this.inputProcessing(row, table)
     row = processed.row
     let manyRelationships = processed.manyRelationships
@@ -744,9 +747,20 @@ export class ExternalRequest<T extends Operation> {
 
     // aliasing can be disabled fully if desired
     const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables))
-    let response = env.SQL_ALIASING_DISABLE
+    let response: DatasourcePlusQueryResponse
+    // there's a chance after input processing nothing needs updated, so pass over the call
+    // we might still need to perform other operations like updating the foreign keys on other rows
+    if (
+      this.operation === Operation.UPDATE &&
+      Object.keys(row || {}).length === 0 &&
+      unprocessedRow
+    ) {
+      response = [unprocessedRow]
+    } else {
+      response = env.SQL_ALIASING_DISABLE
         ? await getDatasourceAndQuery(json)
         : await aliasing.queryWithAliasing(json, makeExternalQuery)
+    }
 
     // if it's a counting operation there will be no more processing, just return the number
     if (this.operation === Operation.COUNT) {

packages/server/src/api/controllers/row/external.ts

@@ -27,6 +27,8 @@ import {
 } from "../../../utilities/rowProcessor"
 import { cloneDeep } from "lodash"
 import { generateIdForRow } from "./utils"
+import { helpers } from "@budibase/shared-core"
+import { HTTPError } from "@budibase/backend-core"
 
 export async function handleRequest<T extends Operation>(
   operation: T,
@@ -42,6 +44,11 @@ export async function handleRequest<T extends Operation>(
 
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const source = await utils.getSource(ctx)
+
+  if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
+    ctx.throw(400, "Cannot update rows through a calculation view")
+  }
+
   const table = await utils.getTableFromSource(source)
   const { _id, ...rowData } = ctx.request.body
 
@@ -96,6 +103,11 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
 
 export async function destroy(ctx: UserCtx) {
   const source = await utils.getSource(ctx)
+
+  if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
+    throw new HTTPError("Cannot delete rows through a calculation view", 400)
+  }
+
   const _id = ctx.request.body._id
   const { row } = await handleRequest(Operation.DELETE, source, {
     id: breakRowIdField(_id),

packages/server/src/api/controllers/row/internal.ts

@@ -8,7 +8,7 @@ import {
 } from "../../../utilities/rowProcessor"
 import * as utils from "./utils"
 import { cloneDeep } from "lodash/fp"
-import { context } from "@budibase/backend-core"
+import { context, HTTPError } from "@budibase/backend-core"
 import { finaliseRow, updateRelatedFormula } from "./staticFormula"
 import {
   FieldType,
@@ -16,19 +16,27 @@ import {
   PatchRowRequest,
   PatchRowResponse,
   Row,
+  Table,
   UserCtx,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils"
 import { flatten } from "lodash"
 import { findRow } from "../../../sdk/app/rows/internal"
+import { helpers } from "@budibase/shared-core"
 
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const { tableId } = utils.getSourceId(ctx)
   const source = await utils.getSource(ctx)
+
+  if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
+    ctx.throw(400, "Cannot update rows through a calculation view")
+  }
+
   const table = sdk.views.isView(source)
     ? await sdk.views.getTable(source.id)
     : source
+
   const inputs = ctx.request.body
   const isUserTable = tableId === InternalTables.USER_METADATA
   let oldRow
@@ -90,15 +98,26 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
 
 export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
-  const { tableId } = utils.getSourceId(ctx)
+  const source = await utils.getSource(ctx)
+
+  if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
+    throw new HTTPError("Cannot delete rows through a calculation view", 400)
+  }
+
+  let table: Table
+  if (sdk.views.isView(source)) {
+    table = await sdk.views.getTable(source.id)
+  } else {
+    table = source
+  }
+
   const { _id } = ctx.request.body
   let row = await db.get<Row>(_id)
   let _rev = ctx.request.body._rev || row._rev
 
-  if (row.tableId !== tableId) {
+  if (row.tableId !== table._id) {
     throw "Supplied tableId doesn't match the row's tableId"
   }
-  const table = await sdk.tables.getTable(tableId)
   // update the row to include full relationships before deleting them
   row = await outputProcessing(table, row, {
     squash: false,
@@ -108,7 +127,7 @@ export async function destroy(ctx: UserCtx) {
   await linkRows.updateLinks({
     eventType: linkRows.EventType.ROW_DELETE,
     row,
-    tableId,
+    tableId: table._id!,
   })
   // remove any attachments that were on the row from object storage
   await AttachmentCleanup.rowDelete(table, [row])
@@ -116,7 +135,7 @@ export async function destroy(ctx: UserCtx) {
   await updateRelatedFormula(table, row)
 
   let response
-  if (tableId === InternalTables.USER_METADATA) {
+  if (table._id === InternalTables.USER_METADATA) {
     ctx.params = {
       id: _id,
     }

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -134,9 +134,7 @@ export async function buildSqlFieldList(
 
   let fields: string[] = []
   if (sdk.views.isView(source)) {
-    fields = Object.keys(helpers.views.basicFields(source)).filter(
-      key => source.schema?.[key]?.visible !== false
-    )
+    fields = Object.keys(helpers.views.basicFields(source))
   } else {
     fields = extractRealFields(source)
   }

packages/server/src/api/controllers/rowAction/crud.ts

@@ -21,14 +21,15 @@ export async function find(ctx: Ctx<void, RowActionsResponse>) {
   const table = await getTable(ctx)
   const tableId = table._id!
 
-  if (!(await sdk.rowActions.docExists(tableId))) {
+  const rowActions = await sdk.rowActions.getAll(tableId)
+  if (!rowActions) {
     ctx.body = {
       actions: {},
     }
     return
   }
 
-  const { actions } = await sdk.rowActions.getAll(tableId)
+  const { actions } = rowActions
   const result: RowActionsResponse = {
     actions: Object.entries(actions).reduce<Record<string, RowActionResponse>>(
       (acc, [key, action]) => ({

packages/server/src/api/controllers/table/external.ts

@@ -31,7 +31,7 @@ function getDatasourceId(table: Table) {
   return breakExternalTableId(table._id).datasourceId
 }
 
-export async function save(
+export async function updateTable(
   ctx: UserCtx<SaveTableRequest, SaveTableResponse>,
   renaming?: RenameColumn
 ) {

packages/server/src/api/controllers/table/index.ts

@@ -71,19 +71,20 @@ export async function fetch(ctx: UserCtx<void, FetchTablesResponse>) {
 
   const datasources = await sdk.datasources.getExternalDatasources()
 
-  const external = datasources.flatMap(datasource => {
+  const external: Table[] = []
+  for (const datasource of datasources) {
     let entities = datasource.entities
     if (entities) {
-      return Object.values(entities).map<Table>((entity: Table) => ({
-        ...entity,
+      for (const entity of Object.values(entities)) {
+        external.push({
+          ...(await processTable(entity)),
           sourceType: TableSourceType.EXTERNAL,
           sourceId: datasource._id!,
           sql: isSQL(datasource),
-      }))
-    } else {
-      return []
-    }
         })
+      }
+    }
+  }
 
   const result: FetchTablesResponse = []
   for (const table of [...internal, ...external]) {
@@ -102,18 +103,22 @@ export async function find(ctx: UserCtx<void, TableResponse>) {
 
 export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   const appId = ctx.appId
-  const table = ctx.request.body
-  const isImport = table.rows
+  const { rows, ...table } = ctx.request.body
+  const isImport = rows
   const renaming = ctx.request.body._rename
 
+  const isCreate = !table._id
+
   checkDefaultFields(table)
 
-  const api = pickApi({ table })
-  let savedTable = await api.save(ctx, renaming)
-  if (!table._id) {
+  let savedTable: Table
+  if (isCreate) {
+    savedTable = await sdk.tables.create(table, rows, ctx.user._id)
     savedTable = await sdk.tables.enrichViewSchemas(savedTable)
     await events.table.created(savedTable)
   } else {
+    const api = pickApi({ table })
+    savedTable = await api.updateTable(ctx, renaming)
     await events.table.updated(savedTable)
   }
   if (renaming) {
@@ -135,6 +140,7 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
 export async function destroy(ctx: UserCtx) {
   const appId = ctx.appId
   const tableId = ctx.params.tableId
+  await sdk.rowActions.deleteAll(tableId)
   const deletedTable = await pickApi({ tableId }).destroy(ctx)
   await events.table.deleted(deletedTable)
   ctx.eventEmitter &&

packages/server/src/api/controllers/table/internal.ts

@@ -12,7 +12,7 @@ import {
 } from "@budibase/types"
 import sdk from "../../../sdk"
 
-export async function save(
+export async function updateTable(
   ctx: UserCtx<SaveTableRequest, SaveTableResponse>,
   renaming?: RenameColumn
 ) {
@@ -25,19 +25,16 @@ export async function save(
     sourceType: rest.sourceType || TableSourceType.INTERNAL,
   }
 
-  const isImport = !!rows
-
   if (!tableToSave.views) {
     tableToSave.views = {}
   }
 
   try {
     const { table } = await sdk.tables.internal.save(tableToSave, {
-      user: ctx.user,
+      userId: ctx.user._id,
       rowsToImport: rows,
       tableId: ctx.request.body._id,
       renaming,
-      isImport,
     })
 
     return table
@@ -72,7 +69,7 @@ export async function bulkImport(
   await handleDataImport(table, {
     importRows: rows,
     identifierFields,
-    user: ctx.user,
+    userId: ctx.user._id,
   })
   return table
 }

packages/server/src/api/controllers/table/tests/utils.spec.ts

@@ -41,7 +41,7 @@ describe("utils", () => {
 
         const data = [{ name: "Alice" }, { name: "Bob" }, { name: "Claire" }]
 
-        const result = await importToRows(data, table, config.user)
+        const result = await importToRows(data, table, config.user?._id)
         expect(result).toEqual([
           expect.objectContaining({
             autoId: 1,

packages/server/src/api/controllers/table/utils.ts

@@ -18,7 +18,6 @@ import { quotas } from "@budibase/pro"
 import { events, context, features } from "@budibase/backend-core"
 import {
   AutoFieldSubType,
-  ContextUser,
   Datasource,
   Row,
   SourceName,
@@ -122,7 +121,7 @@ export function makeSureTableUpToDate(table: Table, tableToSave: Table) {
 export async function importToRows(
   data: Row[],
   table: Table,
-  user?: ContextUser,
+  userId?: string,
   opts?: { keepCouchId: boolean }
 ) {
   const originalTable = table
@@ -136,7 +135,7 @@ export async function importToRows(
 
     // We use a reference to table here and update it after input processing,
     // so that we can auto increment auto IDs in imported data properly
-    const processed = await inputProcessing(user?._id, table, row, {
+    const processed = await inputProcessing(userId, table, row, {
       noAutoRelationships: true,
     })
     row = processed
@@ -167,11 +166,10 @@ export async function importToRows(
 
 export async function handleDataImport(
   table: Table,
-  opts?: { identifierFields?: string[]; user?: ContextUser; importRows?: Row[] }
+  opts?: { identifierFields?: string[]; userId?: string; importRows?: Row[] }
 ) {
   const schema = table.schema
   const identifierFields = opts?.identifierFields || []
-  const user = opts?.user
   const importRows = opts?.importRows
 
   if (!importRows || !isRows(importRows) || !isSchema(schema)) {
@@ -181,7 +179,7 @@ export async function handleDataImport(
   const db = context.getAppDB()
   const data = parse(importRows, table)
 
-  const finalData = await importToRows(data, table, user, {
+  const finalData = await importToRows(data, table, opts?.userId, {
     keepCouchId: identifierFields.includes("_id"),
   })
 
@@ -282,22 +280,22 @@ export function checkStaticTables(table: Table) {
 
 class TableSaveFunctions {
   db: Database
-  user?: ContextUser
+  userId?: string
   oldTable?: Table
   importRows?: Row[]
   rows: Row[]
 
   constructor({
-    user,
+    userId,
     oldTable,
     importRows,
   }: {
-    user?: ContextUser
+    userId?: string
     oldTable?: Table
     importRows?: Row[]
   }) {
     this.db = context.getAppDB()
-    this.user = user
+    this.userId = userId
     this.oldTable = oldTable
     this.importRows = importRows
     // any rows that need updated
@@ -329,7 +327,7 @@ class TableSaveFunctions {
     table = await handleSearchIndexes(table)
     table = await handleDataImport(table, {
       importRows: this.importRows,
-      user: this.user,
+      userId: this.userId,
     })
     if (await features.flags.isEnabled("SQS")) {
       await sdk.tables.sqs.addTable(table)

packages/server/src/api/controllers/view/viewsV2.ts

@@ -127,13 +127,13 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
 
   const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version"> = {
     name: view.name,
+    type: view.type,
     tableId: view.tableId,
     query: view.query,
     queryUI: view.queryUI,
     sort: view.sort,
     schema,
     primaryDisplay: view.primaryDisplay,
-    uiMetadata: view.uiMetadata,
   }
   const result = await sdk.views.create(tableId, parsedView)
   ctx.status = 201
@@ -163,6 +163,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
   const parsedView: RequiredKeys<ViewV2> = {
     id: view.id,
     name: view.name,
+    type: view.type,
     version: view.version,
     tableId: view.tableId,
     query: view.query,
@@ -170,7 +171,6 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
     sort: view.sort,
     schema,
     primaryDisplay: view.primaryDisplay,
-    uiMetadata: view.uiMetadata,
   }
 
   const result = await sdk.views.update(tableId, parsedView)

packages/server/src/api/routes/tests/application.spec.ts

@@ -14,12 +14,7 @@ jest.mock("../../../utilities/redis", () => ({
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 import { AppStatus } from "../../../db/utils"
-import {
-  events,
-  utils,
-  context,
-  withEnv as withCoreEnv,
-} from "@budibase/backend-core"
+import { events, utils, context, features } from "@budibase/backend-core"
 import env from "../../../environment"
 import { type App } from "@budibase/types"
 import tk from "timekeeper"
@@ -358,9 +353,13 @@ describe("/applications", () => {
         .delete(`/api/global/roles/${prodAppId}`)
         .reply(200, {})
 
-      await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, async () => {
+      await features.testutils.withFeatureFlags(
+        "*",
+        { SQS: true },
+        async () => {
           await config.api.application.delete(app.appId)
-      })
+        }
+      )
     })
   })
 

packages/server/src/api/routes/tests/permissions.spec.ts

@@ -1,5 +1,5 @@
 import { roles } from "@budibase/backend-core"
-import { Document, PermissionLevel, Row, Table, ViewV2 } from "@budibase/types"
+import { Document, PermissionLevel, Row } from "@budibase/types"
 import * as setup from "./utilities"
 import { generator, mocks } from "@budibase/backend-core/tests"
 
@@ -9,13 +9,11 @@ const { BUILTIN_ROLE_IDS } = roles
 const HIGHER_ROLE_ID = BUILTIN_ROLE_IDS.BASIC
 const STD_ROLE_ID = BUILTIN_ROLE_IDS.PUBLIC
 
+const DEFAULT_TABLE_ROLE_ID = BUILTIN_ROLE_IDS.ADMIN
+
 describe("/permission", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
-  let table: Table & { _id: string }
-  let perms: Document[]
-  let row: Row
-  let view: ViewV2
 
   afterAll(setup.afterAll)
 
@@ -25,18 +23,6 @@ describe("/permission", () => {
 
   beforeEach(async () => {
     mocks.licenses.useCloudFree()
-
-    table = (await config.createTable()) as typeof table
-    row = await config.createRow()
-    view = await config.api.viewV2.create({
-      tableId: table._id!,
-      name: generator.guid(),
-    })
-    perms = await config.api.permission.add({
-      roleId: STD_ROLE_ID,
-      resourceId: table._id,
-      level: PermissionLevel.READ,
-    })
   })
 
   describe("levels", () => {
@@ -54,33 +40,56 @@ describe("/permission", () => {
     })
   })
 
-  describe("add", () => {
-    it("should be able to add permission to a role for the table", async () => {
-      expect(perms.length).toEqual(1)
-      expect(perms[0]._id).toEqual(`${STD_ROLE_ID}`)
+  describe("table permissions", () => {
+    let tableId: string
+
+    beforeEach(async () => {
+      const table = await config.createTable()
+      tableId = table._id!
+      await config.api.permission.add({
+        roleId: STD_ROLE_ID,
+        resourceId: tableId,
+        level: PermissionLevel.READ,
+      })
     })
 
-    it("should get the resource permissions", async () => {
+    it("tables should be defaulted to admin", async () => {
+      const table = await config.createTable()
+      const { permissions } = await config.api.permission.get(table._id!)
+      expect(permissions).toEqual({
+        read: {
+          permissionType: "EXPLICIT",
+          role: DEFAULT_TABLE_ROLE_ID,
+        },
+        write: {
+          permissionType: "EXPLICIT",
+          role: DEFAULT_TABLE_ROLE_ID,
+        },
+      })
+    })
+
+    describe("add", () => {
+      it("should be able to add permission to a role for the table", async () => {
         const res = await request
-        .get(`/api/permission/${table._id}`)
+          .get(`/api/permission/${tableId}`)
           .set(config.defaultHeaders())
           .expect("Content-Type", /json/)
           .expect(200)
         expect(res.body).toEqual({
           permissions: {
             read: { permissionType: "EXPLICIT", role: STD_ROLE_ID },
-          write: { permissionType: "BASE", role: HIGHER_ROLE_ID },
+            write: { permissionType: "EXPLICIT", role: DEFAULT_TABLE_ROLE_ID },
           },
         })
       })
 
       it("should get resource permissions with multiple roles", async () => {
-      perms = await config.api.permission.add({
+        await config.api.permission.add({
           roleId: HIGHER_ROLE_ID,
-        resourceId: table._id,
+          resourceId: tableId,
           level: PermissionLevel.WRITE,
         })
-      const res = await config.api.permission.get(table._id)
+        const res = await config.api.permission.get(tableId)
         expect(res).toEqual({
           permissions: {
             read: { permissionType: "EXPLICIT", role: STD_ROLE_ID },
@@ -93,31 +102,43 @@ describe("/permission", () => {
           .set(config.defaultHeaders())
           .expect("Content-Type", /json/)
           .expect(200)
-      expect(allRes.body[table._id]["read"]).toEqual(STD_ROLE_ID)
-      expect(allRes.body[table._id]["write"]).toEqual(HIGHER_ROLE_ID)
+        expect(allRes.body[tableId]["read"]).toEqual(STD_ROLE_ID)
+        expect(allRes.body[tableId]["write"]).toEqual(HIGHER_ROLE_ID)
       })
     })
 
     describe("remove", () => {
       it("should be able to remove the permission", async () => {
-      const res = await config.api.permission.revoke({
+        await config.api.permission.revoke({
           roleId: STD_ROLE_ID,
-        resourceId: table._id,
+          resourceId: tableId,
           level: PermissionLevel.READ,
         })
-      expect(res[0]._id).toEqual(STD_ROLE_ID)
-      const permsRes = await config.api.permission.get(table._id)
+
+        const permsRes = await config.api.permission.get(tableId)
         expect(permsRes.permissions[STD_ROLE_ID]).toBeUndefined()
       })
     })
 
     describe("check public user allowed", () => {
+      let viewId: string
+      let row: Row
+
+      beforeEach(async () => {
+        const view = await config.api.viewV2.create({
+          tableId,
+          name: generator.guid(),
+        })
+        viewId = view.id
+        row = await config.createRow()
+      })
+
       it("should be able to read the row", async () => {
         // replicate changes before checking permissions
         await config.publish()
 
         const res = await request
-        .get(`/api/${table._id}/rows`)
+          .get(`/api/${tableId}/rows`)
           .set(config.publicHeaders())
           .expect("Content-Type", /json/)
           .expect(200)
@@ -128,62 +149,144 @@ describe("/permission", () => {
         // Make view inherit table permissions. Needed for backwards compatibility with existing views.
         await config.api.permission.revoke({
           roleId: STD_ROLE_ID,
-        resourceId: view.id,
+          resourceId: viewId,
           level: PermissionLevel.READ,
         })
+
         // replicate changes before checking permissions
         await config.publish()
 
-      const res = await config.api.viewV2.publicSearch(view.id)
+        const res = await config.api.viewV2.publicSearch(viewId)
         expect(res.rows[0]._id).toEqual(row._id)
       })
 
       it("should not be able to access the view data when the table is not public and there are no view permissions overrides", async () => {
         await config.api.permission.revoke({
           roleId: STD_ROLE_ID,
-        resourceId: table._id,
+          resourceId: tableId,
           level: PermissionLevel.READ,
         })
+
         // Make view inherit table permissions. Needed for backwards compatibility with existing views.
         await config.api.permission.revoke({
           roleId: STD_ROLE_ID,
-        resourceId: view.id,
+          resourceId: viewId,
           level: PermissionLevel.READ,
         })
+
         // replicate changes before checking permissions
         await config.publish()
 
-      await config.api.viewV2.publicSearch(view.id, undefined, { status: 401 })
+        await config.api.viewV2.publicSearch(viewId, undefined, {
+          status: 401,
+        })
       })
 
       it("should use the view permissions", async () => {
         await config.api.permission.add({
           roleId: STD_ROLE_ID,
-        resourceId: view.id,
+          resourceId: viewId,
           level: PermissionLevel.READ,
         })
         await config.api.permission.revoke({
           roleId: STD_ROLE_ID,
-        resourceId: table._id,
+          resourceId: tableId,
           level: PermissionLevel.READ,
         })
         // replicate changes before checking permissions
         await config.publish()
 
-      const res = await config.api.viewV2.publicSearch(view.id)
+        const res = await config.api.viewV2.publicSearch(viewId)
         expect(res.rows[0]._id).toEqual(row._id)
       })
 
       it("shouldn't allow writing from a public user", async () => {
         const res = await request
-        .post(`/api/${table._id}/rows`)
-        .send(basicRow(table._id))
+          .post(`/api/${tableId}/rows`)
+          .send(basicRow(tableId))
           .set(config.publicHeaders())
           .expect("Content-Type", /json/)
           .expect(401)
         expect(res.status).toEqual(401)
       })
     })
+  })
+
+  describe("view permissions", () => {
+    let tableId: string
+    let viewId: string
+
+    beforeEach(async () => {
+      const table = await config.createTable()
+      tableId = table._id!
+
+      const view = await config.api.viewV2.create({
+        tableId,
+        name: generator.guid(),
+      })
+      viewId = view.id
+    })
+
+    it("default permissions inherits and persists the table default value", async () => {
+      const { permissions } = await config.api.permission.get(viewId)
+      expect(permissions).toEqual({
+        read: {
+          permissionType: "EXPLICIT",
+          role: DEFAULT_TABLE_ROLE_ID,
+          inheritablePermission: DEFAULT_TABLE_ROLE_ID,
+        },
+        write: {
+          permissionType: "EXPLICIT",
+          role: DEFAULT_TABLE_ROLE_ID,
+          inheritablePermission: DEFAULT_TABLE_ROLE_ID,
+        },
+      })
+    })
+
+    it("does not update view permissions once persisted, even if table permissions change", async () => {
+      await config.api.permission.add({
+        roleId: STD_ROLE_ID,
+        resourceId: tableId,
+        level: PermissionLevel.READ,
+      })
+
+      const { permissions } = await config.api.permission.get(viewId)
+      expect(permissions).toEqual({
+        read: {
+          permissionType: "EXPLICIT",
+          role: DEFAULT_TABLE_ROLE_ID,
+          inheritablePermission: STD_ROLE_ID,
+        },
+        write: {
+          permissionType: "EXPLICIT",
+          role: DEFAULT_TABLE_ROLE_ID,
+          inheritablePermission: DEFAULT_TABLE_ROLE_ID,
+        },
+      })
+    })
+
+    it("can sets permissions inherits explicit view permissions", async () => {
+      await config.api.permission.add({
+        roleId: HIGHER_ROLE_ID,
+        resourceId: viewId,
+        level: PermissionLevel.WRITE,
+      })
+
+      const { permissions } = await config.api.permission.get(viewId)
+      expect(permissions).toEqual({
+        read: {
+          permissionType: "EXPLICIT",
+          role: DEFAULT_TABLE_ROLE_ID,
+          inheritablePermission: DEFAULT_TABLE_ROLE_ID,
+        },
+        write: {
+          permissionType: "EXPLICIT",
+          role: HIGHER_ROLE_ID,
+          inheritablePermission: DEFAULT_TABLE_ROLE_ID,
+        },
+      })
+    })
+  })
 
   describe("fetch builtins", () => {
     it("should be able to fetch builtin definitions", async () => {

packages/server/src/api/routes/tests/queries/generic-sql.spec.ts

@@ -28,6 +28,7 @@ describe.each(
   const config = setup.getConfig()
   const isOracle = dbName === DatabaseName.ORACLE
   const isMsSQL = dbName === DatabaseName.SQL_SERVER
+  const isPostgres = dbName === DatabaseName.POSTGRES
 
   let rawDatasource: Datasource
   let datasource: Datasource
@@ -47,6 +48,9 @@ describe.each(
       transformer: "return data",
       readable: true,
     }
+    if (query.fields?.sql && typeof query.fields.sql !== "string") {
+      throw new Error("Unable to create with knex structure in 'sql' field")
+    }
     return await config.api.query.save(
       { ...defaultQuery, ...query },
       expectations
@@ -207,6 +211,31 @@ describe.each(
         expect(prodQuery.parameters).toBeUndefined()
         expect(prodQuery.schema).toBeDefined()
       })
+
+      isPostgres &&
+        it("should be able to handle a JSON aggregate with newlines", async () => {
+          const jsonStatement = `COALESCE(json_build_object('name', name),'{"name":{}}'::json)`
+          const query = await createQuery({
+            fields: {
+              sql: client("test_table")
+                .select([
+                  "*",
+                  client.raw(
+                    `${jsonStatement} as json,\n${jsonStatement} as json2`
+                  ),
+                ])
+                .toString(),
+            },
+          })
+          const res = await config.api.query.execute(
+            query._id!,
+            {},
+            {
+              status: 200,
+            }
+          )
+          expect(res).toBeDefined()
+        })
     })
   })
 

packages/server/src/api/routes/tests/role.spec.js

@@ -161,7 +161,7 @@ describe("/roles", () => {
 
     it("should not fetch higher level accessible roles when a custom role header is provided", async () => {
       await createRole({
-        name: `CUSTOM_ROLE`,
+        name: `custom_role_1`,
         inherits: roles.BUILTIN_ROLE_IDS.BASIC,
         permissionId: permissions.BuiltinPermissionID.READ_ONLY,
         version: "name",
@@ -170,11 +170,11 @@ describe("/roles", () => {
         .get("/api/roles/accessible")
         .set({
           ...config.defaultHeaders(),
-          "x-budibase-role": "CUSTOM_ROLE",
+          "x-budibase-role": "custom_role_1",
         })
         .expect(200)
       expect(res.body.length).toBe(3)
-      expect(res.body[0]).toBe("CUSTOM_ROLE")
+      expect(res.body[0]).toBe("custom_role_1")
       expect(res.body[1]).toBe("BASIC")
       expect(res.body[2]).toBe("PUBLIC")
     })

packages/server/src/api/routes/tests/row.spec.ts

@@ -1,6 +1,10 @@
 import * as setup from "./utilities"
 
-import { DatabaseName, knexClient } from "../../../integrations/tests/utils"
+import {
+  DatabaseName,
+  getDatasource,
+  knexClient,
+} from "../../../integrations/tests/utils"
 
 import tk from "timekeeper"
 import emitter from "../../../../src/events"
@@ -8,37 +12,36 @@ import { outputProcessing } from "../../../utilities/rowProcessor"
 import {
   context,
   InternalTable,
-  setEnv as setCoreEnv,
   tenancy,
-  withEnv as withCoreEnv,
+  features,
+  utils,
 } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
   AIOperationEnum,
   AttachmentFieldMetadata,
   AutoFieldSubType,
-  BBReferenceFieldSubType,
   Datasource,
   DateFieldMetadata,
   DeleteRow,
-  FeatureFlag,
   FieldSchema,
   FieldType,
+  BBReferenceFieldSubType,
   FormulaType,
   INTERNAL_TABLE_SOURCE_ID,
-  JsonFieldSubType,
   NumberFieldMetadata,
   QuotaUsageType,
-  RelationSchemaField,
   RelationshipType,
   Row,
-  RowExportFormat,
   SaveTableRequest,
   StaticQuotaName,
   Table,
-  TableSchema,
   TableSourceType,
   UpdatedRowEventEmitter,
+  TableSchema,
+  JsonFieldSubType,
+  RowExportFormat,
+  RelationSchemaField,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import _, { merge } from "lodash"
@@ -86,13 +89,13 @@ async function waitForEvent(
 }
 
 describe.each([
-  // ["lucene", undefined],
+  ["lucene", undefined],
   ["sqs", undefined],
-  // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
-  // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
+  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("/rows (%s)", (providerType, dsProvider) => {
   const isInternal = dsProvider === undefined
   const isLucene = providerType === "lucene"
@@ -107,12 +110,12 @@ describe.each([
   let envCleanup: (() => void) | undefined
 
   beforeAll(async () => {
-    await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init())
-    if (isLucene) {
-      envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" })
-    } else if (isSqs) {
-      envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" })
-    }
+    await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
+      config.init()
+    )
+    envCleanup = features.testutils.setFeatureFlags("*", {
+      SQS: isSqs,
+    })
 
     if (dsProvider) {
       const rawDatasource = await dsProvider
@@ -768,6 +771,70 @@ describe.each([
         })
       })
 
+      describe("user column", () => {
+        beforeAll(async () => {
+          table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                user: {
+                  name: "user",
+                  type: FieldType.BB_REFERENCE_SINGLE,
+                  subtype: BBReferenceFieldSubType.USER,
+                  default: "{{ [Current User]._id }}",
+                },
+              },
+            })
+          )
+        })
+
+        it("creates a new row with a default value successfully", async () => {
+          const row = await config.api.row.save(table._id!, {})
+          expect(row.user._id).toEqual(config.getUser()._id)
+        })
+
+        it("does not use default value if value specified", async () => {
+          const id = `us_${utils.newid()}`
+          await config.createUser({ _id: id })
+          const row = await config.api.row.save(table._id!, {
+            user: id,
+          })
+          expect(row.user._id).toEqual(id)
+        })
+      })
+
+      describe("multi-user column", () => {
+        beforeAll(async () => {
+          table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                users: {
+                  name: "users",
+                  type: FieldType.BB_REFERENCE,
+                  subtype: BBReferenceFieldSubType.USER,
+                  default: ["{{ [Current User]._id }}"],
+                },
+              },
+            })
+          )
+        })
+
+        it("creates a new row with a default value successfully", async () => {
+          const row = await config.api.row.save(table._id!, {})
+          expect(row.users).toHaveLength(1)
+          expect(row.users[0]._id).toEqual(config.getUser()._id)
+        })
+
+        it("does not use default value if value specified", async () => {
+          const id = `us_${utils.newid()}`
+          await config.createUser({ _id: id })
+          const row = await config.api.row.save(table._id!, {
+            users: [id],
+          })
+          expect(row.users).toHaveLength(1)
+          expect(row.users[0]._id).toEqual(id)
+        })
+      })
+
       describe("bindings", () => {
         describe("string column", () => {
           beforeAll(async () => {
@@ -1125,6 +1192,33 @@ describe.each([
       expect(getResp.user2[0]._id).toEqual(user2._id)
     })
 
+    it("should be able to remove a relationship from many side", async () => {
+      const row = await config.api.row.save(otherTable._id!, {
+        name: "test",
+        description: "test",
+      })
+      const row2 = await config.api.row.save(otherTable._id!, {
+        name: "test",
+        description: "test",
+      })
+      const { _id } = await config.api.row.save(table._id!, {
+        relationship: [{ _id: row._id }, { _id: row2._id }],
+      })
+      const relatedRow = await config.api.row.get(table._id!, _id!, {
+        status: 200,
+      })
+      expect(relatedRow.relationship.length).toEqual(2)
+      await config.api.row.save(table._id!, {
+        ...relatedRow,
+        relationship: [{ _id: row._id }],
+      })
+      const afterRelatedRow = await config.api.row.get(table._id!, _id!, {
+        status: 200,
+      })
+      expect(afterRelatedRow.relationship.length).toEqual(1)
+      expect(afterRelatedRow.relationship[0]._id).toEqual(row._id)
+    })
+
     it("should be able to update relationships when both columns are same name", async () => {
       let row = await config.api.row.save(table._id!, {
         name: "test",
@@ -1857,7 +1951,7 @@ describe.each([
     })
 
   describe("exportRows", () => {
-    beforeAll(async () => {
+    beforeEach(async () => {
       table = await config.api.table.save(defaultTable())
     })
 
@@ -1894,6 +1988,16 @@ describe.each([
         })
       })
 
+    it("should allow exporting without filtering", async () => {
+      const existing = await config.api.row.save(table._id!, {})
+      const res = await config.api.row.exportRows(table._id!)
+      const results = JSON.parse(res)
+      expect(results.length).toEqual(1)
+      const row = results[0]
+
+      expect(row._id).toEqual(existing._id)
+    })
+
     it("should allow exporting only certain columns", async () => {
       const existing = await config.api.row.save(table._id!, {})
       const res = await config.api.row.exportRows(table._id!, {
@@ -2528,15 +2632,9 @@ describe.each([
       let flagCleanup: (() => void) | undefined
 
       beforeAll(async () => {
-        const env = {
-          TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
-        }
-        if (isSqs) {
-          env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:SQS`
-        } else {
-          env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:!SQS`
-        }
-        flagCleanup = setCoreEnv(env)
+        flagCleanup = features.testutils.setFeatureFlags("*", {
+          ENRICHED_RELATIONSHIPS: true,
+        })
 
         const aux2Table = await config.api.table.save(saveTableRequest())
         const aux2Data = await config.api.row.save(aux2Table._id!, {})
@@ -2763,9 +2861,10 @@ describe.each([
       it.each(testScenarios)(
         "does not enrich relationships when not enabled (via %s)",
         async (__, retrieveDelegate) => {
-          await withCoreEnv(
+          await features.testutils.withFeatureFlags(
+            "*",
             {
-              TENANT_FEATURE_FLAGS: `*:!${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
+              ENRICHED_RELATIONSHIPS: false,
             },
             async () => {
               const otherRows = _.sampleSize(auxData, 5)

packages/server/src/api/routes/tests/rowAction.spec.ts

@@ -1043,4 +1043,44 @@ describe("/rowsActions", () => {
       )
     })
   })
+
+  describe("scenarios", () => {
+    // https://linear.app/budibase/issue/BUDI-8717/
+    it("should not brick the app when deleting a table with row actions", async () => {
+      const view = await config.api.viewV2.create({
+        tableId,
+        name: generator.guid(),
+        schema: {
+          name: { visible: true },
+        },
+      })
+
+      const rowAction = await config.api.rowAction.save(tableId, {
+        name: generator.guid(),
+      })
+
+      await config.api.rowAction.setViewPermission(
+        tableId,
+        view.id,
+        rowAction.id
+      )
+
+      let actionsResp = await config.api.rowAction.find(tableId)
+      expect(actionsResp.actions[rowAction.id]).toEqual({
+        ...rowAction,
+        allowedSources: [tableId, view.id],
+      })
+
+      const table = await config.api.table.get(tableId)
+      await config.api.table.destroy(table._id!, table._rev!)
+
+      // In the bug reported by Conor, when a delete got deleted its row action
+      // document was not being cleaned up. This meant there existed code paths
+      // that would find it and try to reference the tables within it, resulting
+      // in errors.
+      await config.api.automation.fetchEnriched({
+        status: 200,
+      })
+    })
+  })
 })

packages/server/src/api/routes/tests/search.spec.ts

@@ -7,9 +7,9 @@ import {
 import {
   context,
   db as dbCore,
+  features,
   MAX_VALID_DATE,
   MIN_VALID_DATE,
-  setEnv as setCoreEnv,
   SQLITE_DESIGN_DOC_ID,
   utils,
   withEnv as withCoreEnv,
@@ -107,16 +107,12 @@ describe.each([
   }
 
   beforeAll(async () => {
-    await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init())
-    if (isLucene) {
-      envCleanup = setCoreEnv({
-        TENANT_FEATURE_FLAGS: "*:!SQS",
+    await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
+      config.init()
+    )
+    envCleanup = features.testutils.setFeatureFlags("*", {
+      SQS: isSqs,
     })
-    } else if (isSqs) {
-      envCleanup = setCoreEnv({
-        TENANT_FEATURE_FLAGS: "*:SQS",
-      })
-    }
 
     if (config.app?.appId) {
       config.app = await config.api.application.update(config.app?.appId, {
@@ -204,7 +200,6 @@ describe.each([
         if (isInMemory) {
           return dataFilters.search(_.cloneDeep(rows), {
             ...this.query,
-            tableId: tableOrViewId,
           })
         } else {
           return config.api.row.search(tableOrViewId, this.query)

packages/server/src/api/routes/tests/table.spec.ts

@@ -21,6 +21,7 @@ import {
   ViewCalculation,
   ViewV2Enriched,
   RowExportFormat,
+  PermissionLevel,
 } from "@budibase/types"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
@@ -191,6 +192,55 @@ describe.each([
     )
   })
 
+  describe("permissions", () => {
+    it("get the base permissions for the table", async () => {
+      const table = await config.api.table.save(
+        tableForDatasource(datasource, {
+          schema: {
+            name: {
+              type: FieldType.STRING,
+              name: "name",
+            },
+          },
+        })
+      )
+
+      // get the explicit permissions
+      const { permissions } = await config.api.permission.get(table._id!, {
+        status: 200,
+      })
+      const explicitPermissions = {
+        role: "ADMIN",
+        permissionType: "EXPLICIT",
+      }
+      expect(permissions.write).toEqual(explicitPermissions)
+      expect(permissions.read).toEqual(explicitPermissions)
+
+      // revoke the explicit permissions
+      for (let level of [PermissionLevel.WRITE, PermissionLevel.READ]) {
+        await config.api.permission.revoke(
+          {
+            roleId: permissions[level].role,
+            resourceId: table._id!,
+            level,
+          },
+          { status: 200 }
+        )
+      }
+
+      // check base permissions
+      const { permissions: basePermissions } = await config.api.permission.get(
+        table._id!,
+        {
+          status: 200,
+        }
+      )
+      const basePerms = { role: "BASIC", permissionType: "BASE" }
+      expect(basePermissions.write).toEqual(basePerms)
+      expect(basePermissions.read).toEqual(basePerms)
+    })
+  })
+
   describe("update", () => {
     it("updates a table", async () => {
       const table = await config.api.table.save(

packages/server/src/api/routes/tests/templates.spec.ts

@@ -2,7 +2,7 @@ import * as setup from "./utilities"
 import path from "path"
 import nock from "nock"
 import { generator } from "@budibase/backend-core/tests"
-import { withEnv as withCoreEnv, env as coreEnv } from "@budibase/backend-core"
+import { features } from "@budibase/backend-core"
 
 interface App {
   background: string
@@ -85,11 +85,10 @@ describe("/templates", () => {
     it.each(["sqs", "lucene"])(
       `should be able to create an app from a template (%s)`,
       async source => {
-        const env: Partial<typeof coreEnv> = {
-          TENANT_FEATURE_FLAGS: source === "sqs" ? "*:SQS" : "",
-        }
-
-        await withCoreEnv(env, async () => {
+        await features.testutils.withFeatureFlags(
+          "*",
+          { SQS: source === "sqs" },
+          async () => {
             const name = generator.guid().replaceAll("-", "")
             const url = `/${name}`
 
@@ -112,14 +111,18 @@ describe("/templates", () => {
               expect(agencyProjects.name).toBe("Agency Projects")
               expect(users.name).toBe("Users")
 
-            const { rows } = await config.api.row.search(agencyProjects._id!, {
+              const { rows } = await config.api.row.search(
+                agencyProjects._id!,
+                {
                   tableId: agencyProjects._id!,
                   query: {},
-            })
+                }
+              )
 
               expect(rows).toHaveLength(3)
             })
-        })
+          }
+        )
       }
     )
   })

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -22,22 +22,17 @@ import {
   RelationshipType,
   TableSchema,
   RenameColumn,
-  FeatureFlag,
   BBReferenceFieldSubType,
   NumericCalculationFieldMetadata,
   ViewV2Schema,
+  ViewV2Type,
+  JsonTypes,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
 import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
-import {
-  db,
-  roles,
-  withEnv as withCoreEnv,
-  setEnv as setCoreEnv,
-  env,
-} from "@budibase/backend-core"
+import { db, roles, features } from "@budibase/backend-core"
 
 describe.each([
   ["lucene", undefined],
@@ -102,18 +97,13 @@ describe.each([
   }
 
   beforeAll(async () => {
-    await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () =>
+    await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
       config.init()
     )
-    if (isLucene) {
-      envCleanup = setCoreEnv({
-        TENANT_FEATURE_FLAGS: "*:!SQS",
+
+    envCleanup = features.testutils.setFeatureFlags("*", {
+      SQS: isSqs,
     })
-    } else if (isSqs) {
-      envCleanup = setCoreEnv({
-        TENANT_FEATURE_FLAGS: "*:SQS",
-      })
-    }
 
     if (dsProvider) {
       datasource = await config.createDatasource({
@@ -155,7 +145,7 @@ describe.each([
       })
 
       it("can persist views with all fields", async () => {
-        const newView: Required<Omit<CreateViewRequest, "queryUI">> = {
+        const newView: Required<Omit<CreateViewRequest, "queryUI" | "type">> = {
           name: generator.name(),
           tableId: table._id!,
           primaryDisplay: "id",
@@ -177,9 +167,6 @@ describe.each([
               visible: true,
             },
           },
-          uiMetadata: {
-            foo: "bar",
-          },
         }
         const res = await config.api.viewV2.create(newView)
 
@@ -549,6 +536,7 @@ describe.each([
         let view = await config.api.viewV2.create({
           tableId: table._id!,
           name: generator.guid(),
+          type: ViewV2Type.CALCULATION,
           schema: {
             sum: {
               visible: true,
@@ -571,6 +559,275 @@ describe.each([
         expect(sum.calculationType).toEqual(CalculationType.SUM)
         expect(sum.field).toEqual("Price")
       })
+
+      it("cannot create a view with calculation fields unless it has the right type", async () => {
+        await config.api.viewV2.create(
+          {
+            tableId: table._id!,
+            name: generator.guid(),
+            schema: {
+              sum: {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "Price",
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              message:
+                "Calculation fields are not allowed in non-calculation views",
+            },
+          }
+        )
+      })
+
+      it("cannot create a calculation view with more than 5 aggregations", async () => {
+        await config.api.viewV2.create(
+          {
+            tableId: table._id!,
+            name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
+            schema: {
+              sum: {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "Price",
+              },
+              count: {
+                visible: true,
+                calculationType: CalculationType.COUNT,
+                field: "Price",
+              },
+              min: {
+                visible: true,
+                calculationType: CalculationType.MIN,
+                field: "Price",
+              },
+              max: {
+                visible: true,
+                calculationType: CalculationType.MAX,
+                field: "Price",
+              },
+              avg: {
+                visible: true,
+                calculationType: CalculationType.AVG,
+                field: "Price",
+              },
+              sum2: {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "Price",
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              message: "Calculation views can only have a maximum of 5 fields",
+            },
+          }
+        )
+      })
+
+      it("cannot create a calculation view with duplicate calculations", async () => {
+        await config.api.viewV2.create(
+          {
+            tableId: table._id!,
+            name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
+            schema: {
+              sum: {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "Price",
+              },
+              sum2: {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "Price",
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              message:
+                'Duplicate calculation on field "Price", calculation type "sum"',
+            },
+          }
+        )
+      })
+
+      it("finds duplicate counts", async () => {
+        await config.api.viewV2.create(
+          {
+            tableId: table._id!,
+            name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
+            schema: {
+              count: {
+                visible: true,
+                calculationType: CalculationType.COUNT,
+              },
+              count2: {
+                visible: true,
+                calculationType: CalculationType.COUNT,
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              message:
+                'Duplicate calculation on field "*", calculation type "count"',
+            },
+          }
+        )
+      })
+
+      it("finds duplicate count distincts", async () => {
+        await config.api.viewV2.create(
+          {
+            tableId: table._id!,
+            name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
+            schema: {
+              count: {
+                visible: true,
+                calculationType: CalculationType.COUNT,
+                distinct: true,
+                field: "Price",
+              },
+              count2: {
+                visible: true,
+                calculationType: CalculationType.COUNT,
+                distinct: true,
+                field: "Price",
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              message:
+                'Duplicate calculation on field "Price", calculation type "count"',
+            },
+          }
+        )
+      })
+
+      it("does not confuse counts and count distincts in the duplicate check", async () => {
+        await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          type: ViewV2Type.CALCULATION,
+          schema: {
+            count: {
+              visible: true,
+              calculationType: CalculationType.COUNT,
+            },
+            count2: {
+              visible: true,
+              calculationType: CalculationType.COUNT,
+              distinct: true,
+              field: "Price",
+            },
+          },
+        })
+      })
+
+      !isLucene &&
+        it("does not get confused when a calculation field shadows a basic one", async () => {
+          const table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                age: {
+                  name: "age",
+                  type: FieldType.NUMBER,
+                },
+              },
+            })
+          )
+
+          await config.api.row.bulkImport(table._id!, {
+            rows: [{ age: 1 }, { age: 2 }, { age: 3 }],
+          })
+
+          const view = await config.api.viewV2.create({
+            tableId: table._id!,
+            name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
+            schema: {
+              age: {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "age",
+              },
+            },
+          })
+
+          const { rows } = await config.api.row.search(view.id)
+          expect(rows).toHaveLength(1)
+          expect(rows[0].age).toEqual(6)
+        })
+
+      // We don't allow the creation of tables with most JsonTypes when using
+      // external datasources.
+      isInternal &&
+        it("cannot use complex types as group-by fields", async () => {
+          for (const type of JsonTypes) {
+            const field = { name: "field", type } as FieldSchema
+            const table = await config.api.table.save(
+              saveTableRequest({ schema: { field } })
+            )
+            await config.api.viewV2.create(
+              {
+                tableId: table._id!,
+                name: generator.guid(),
+                type: ViewV2Type.CALCULATION,
+                schema: {
+                  field: { visible: true },
+                },
+              },
+              {
+                status: 400,
+                body: {
+                  message: `Grouping by fields of type "${type}" is not supported`,
+                },
+              }
+            )
+          }
+        })
+
+      isInternal &&
+        it("shouldn't trigger a complex type check on a group by field if field is invisible", async () => {
+          const table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                field: {
+                  name: "field",
+                  type: FieldType.JSON,
+                },
+              },
+            })
+          )
+
+          await config.api.viewV2.create(
+            {
+              tableId: table._id!,
+              name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
+              schema: {
+                field: { visible: false },
+              },
+            },
+            {
+              status: 201,
+            }
+          )
+        })
     })
 
     describe("update", () => {
@@ -615,7 +872,9 @@ describe.each([
       it("can update all fields", async () => {
         const tableId = table._id!
 
-        const updatedData: Required<Omit<UpdateViewRequest, "queryUI">> = {
+        const updatedData: Required<
+          Omit<UpdateViewRequest, "queryUI" | "type">
+        > = {
           version: view.version,
           id: view.id,
           tableId,
@@ -643,9 +902,6 @@ describe.each([
               readonly: true,
             },
           },
-          uiMetadata: {
-            foo: "bar",
-          },
         }
         await config.api.viewV2.update(updatedData)
 
@@ -830,6 +1086,32 @@ describe.each([
         )
       })
 
+      it("cannot update view type after creation", async () => {
+        const view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          schema: {
+            id: { visible: true },
+            Price: {
+              visible: true,
+            },
+          },
+        })
+
+        await config.api.viewV2.update(
+          {
+            ...view,
+            type: ViewV2Type.CALCULATION,
+          },
+          {
+            status: 400,
+            body: {
+              message: "Cannot update view type after creation",
+            },
+          }
+        )
+      })
+
       isInternal &&
         it("updating schema will only validate modified field", async () => {
           let view = await config.api.viewV2.create({
@@ -902,6 +1184,7 @@ describe.each([
             view = await config.api.viewV2.create({
               tableId: table._id!,
               name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
               schema: {
                 country: {
                   visible: true,
@@ -1024,6 +1307,26 @@ describe.each([
             )
           })
 
+          it("can add a new group by field that is invisible, even if required on the table", async () => {
+            view.schema!.name = { visible: false }
+            await config.api.viewV2.update(view)
+
+            const { rows } = await config.api.row.search(view.id)
+            expect(rows).toHaveLength(2)
+            expect(rows).toEqual(
+              expect.arrayContaining([
+                {
+                  country: "USA",
+                  age: 65,
+                },
+                {
+                  country: "UK",
+                  age: 61,
+                },
+              ])
+            )
+          })
+
           it("can add a new calculation field", async () => {
             view.schema!.count = {
               visible: true,
@@ -1629,6 +1932,59 @@ describe.each([
         })
       })
     })
+
+    describe("calculation views", () => {
+      it("should not remove calculation columns when modifying table schema", async () => {
+        let table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              name: {
+                name: "name",
+                type: FieldType.STRING,
+              },
+              age: {
+                name: "age",
+                type: FieldType.NUMBER,
+              },
+            },
+          })
+        )
+
+        let view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          type: ViewV2Type.CALCULATION,
+          schema: {
+            sum: {
+              visible: true,
+              calculationType: CalculationType.SUM,
+              field: "age",
+            },
+          },
+        })
+
+        table = await config.api.table.get(table._id!)
+        await config.api.table.save({
+          ...table,
+          schema: {
+            ...table.schema,
+            name: {
+              name: "name",
+              type: FieldType.STRING,
+              constraints: { presence: true },
+            },
+          },
+        })
+
+        view = await config.api.viewV2.get(view.id)
+        expect(Object.keys(view.schema!).sort()).toEqual([
+          "age",
+          "id",
+          "name",
+          "sum",
+        ])
+      })
+    })
   })
 
   describe("row operations", () => {
@@ -1703,6 +2059,30 @@ describe.each([
         expect(newRow.one).toBeUndefined()
         expect(newRow.two).toEqual("bar")
       })
+
+      it("should not be possible to create a row in a calculation view", async () => {
+        const view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          type: ViewV2Type.CALCULATION,
+          schema: {
+            id: { visible: true },
+            one: { visible: true },
+          },
+        })
+
+        await config.api.row.save(
+          view.id,
+          { one: "foo" },
+          {
+            status: 400,
+            body: {
+              message: "Cannot insert rows through a calculation view",
+              status: 400,
+            },
+          }
+        )
+      })
     })
 
     describe("patch", () => {
@@ -1767,6 +2147,40 @@ describe.each([
         expect(row.one).toEqual("foo")
         expect(row.two).toEqual("newBar")
       })
+
+      it("should not be possible to modify a row in a calculation view", async () => {
+        const view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          type: ViewV2Type.CALCULATION,
+          schema: {
+            id: { visible: true },
+            one: { visible: true },
+          },
+        })
+
+        const newRow = await config.api.row.save(table._id!, {
+          one: "foo",
+          two: "bar",
+        })
+
+        await config.api.row.patch(
+          view.id,
+          {
+            tableId: table._id!,
+            _id: newRow._id!,
+            _rev: newRow._rev!,
+            one: "newFoo",
+            two: "newBar",
+          },
+          {
+            status: 400,
+            body: {
+              message: "Cannot update rows through a calculation view",
+            },
+          }
+        )
+      })
     })
 
     describe("destroy", () => {
@@ -1815,6 +2229,32 @@ describe.each([
         })
         await config.api.row.get(table._id!, rows[1]._id!, { status: 200 })
       })
+
+      it("should not be possible to delete a row in a calculation view", async () => {
+        const row = await config.api.row.save(table._id!, {})
+
+        const view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          type: ViewV2Type.CALCULATION,
+          schema: {
+            id: { visible: true },
+            one: { visible: true },
+          },
+        })
+
+        await config.api.row.delete(
+          view.id,
+          { _id: row._id! },
+          {
+            status: 400,
+            body: {
+              message: "Cannot delete rows through a calculation view",
+              status: 400,
+            },
+          }
+        )
+      })
     })
 
     describe("read", () => {
@@ -2443,12 +2883,8 @@ describe.each([
       describe("foreign relationship columns", () => {
         let envCleanup: () => void
         beforeAll(() => {
-          const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`]
-          if (env.TENANT_FEATURE_FLAGS) {
-            flags.push(...env.TENANT_FEATURE_FLAGS.split(","))
-          }
-          envCleanup = setCoreEnv({
-            TENANT_FEATURE_FLAGS: flags.join(","),
+          envCleanup = features.testutils.setFeatureFlags("*", {
+            ENRICHED_RELATIONSHIPS: true,
           })
         })
 
@@ -2639,6 +3075,7 @@ describe.each([
           it("should be able to search by calculations", async () => {
             const view = await config.api.viewV2.create({
               tableId: table._id!,
+              type: ViewV2Type.CALCULATION,
               name: generator.guid(),
               schema: {
                 "Quantity Sum": {
@@ -2673,6 +3110,7 @@ describe.each([
             const view = await config.api.viewV2.create({
               tableId: table._id!,
               name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
               schema: {
                 quantity: {
                   visible: true,
@@ -2711,6 +3149,7 @@ describe.each([
             const view = await config.api.viewV2.create({
               tableId: table._id!,
               name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
               schema: {
                 aggregate: {
                   visible: true,
@@ -2771,6 +3210,7 @@ describe.each([
             const view = await config.api.viewV2.create({
               tableId: table._id!,
               name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
               schema: {
                 count: {
                   visible: true,
@@ -2805,6 +3245,7 @@ describe.each([
               {
                 tableId: table._id!,
                 name: generator.guid(),
+                type: ViewV2Type.CALCULATION,
                 schema: {
                   count: {
                     visible: true,
@@ -2823,6 +3264,241 @@ describe.each([
               }
             )
           })
+
+          it("should be able to filter rows on the view itself", async () => {
+            const table = await config.api.table.save(
+              saveTableRequest({
+                schema: {
+                  quantity: {
+                    type: FieldType.NUMBER,
+                    name: "quantity",
+                  },
+                  price: {
+                    type: FieldType.NUMBER,
+                    name: "price",
+                  },
+                },
+              })
+            )
+
+            const view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
+              query: {
+                equal: {
+                  quantity: 1,
+                },
+              },
+              schema: {
+                sum: {
+                  visible: true,
+                  calculationType: CalculationType.SUM,
+                  field: "price",
+                },
+              },
+            })
+
+            await config.api.row.bulkImport(table._id!, {
+              rows: [
+                {
+                  quantity: 1,
+                  price: 1,
+                },
+                {
+                  quantity: 1,
+                  price: 2,
+                },
+                {
+                  quantity: 2,
+                  price: 10,
+                },
+              ],
+            })
+
+            const { rows } = await config.api.viewV2.search(view.id, {
+              query: {},
+            })
+            expect(rows).toHaveLength(1)
+            expect(rows[0].sum).toEqual(3)
+          })
+
+          it("should be able to filter on group by fields", async () => {
+            const table = await config.api.table.save(
+              saveTableRequest({
+                schema: {
+                  quantity: {
+                    type: FieldType.NUMBER,
+                    name: "quantity",
+                  },
+                  price: {
+                    type: FieldType.NUMBER,
+                    name: "price",
+                  },
+                },
+              })
+            )
+
+            const view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
+              schema: {
+                quantity: { visible: true },
+                sum: {
+                  visible: true,
+                  calculationType: CalculationType.SUM,
+                  field: "price",
+                },
+              },
+            })
+
+            await config.api.row.bulkImport(table._id!, {
+              rows: [
+                {
+                  quantity: 1,
+                  price: 1,
+                },
+                {
+                  quantity: 1,
+                  price: 2,
+                },
+                {
+                  quantity: 2,
+                  price: 10,
+                },
+              ],
+            })
+
+            const { rows } = await config.api.viewV2.search(view.id, {
+              query: {
+                equal: {
+                  quantity: 1,
+                },
+              },
+            })
+
+            expect(rows).toHaveLength(1)
+            expect(rows[0].sum).toEqual(3)
+          })
+
+          it("should be able to sort by group by field", async () => {
+            const table = await config.api.table.save(
+              saveTableRequest({
+                schema: {
+                  quantity: {
+                    type: FieldType.NUMBER,
+                    name: "quantity",
+                  },
+                  price: {
+                    type: FieldType.NUMBER,
+                    name: "price",
+                  },
+                },
+              })
+            )
+
+            const view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
+              schema: {
+                quantity: { visible: true },
+                sum: {
+                  visible: true,
+                  calculationType: CalculationType.SUM,
+                  field: "price",
+                },
+              },
+            })
+
+            await config.api.row.bulkImport(table._id!, {
+              rows: [
+                {
+                  quantity: 1,
+                  price: 1,
+                },
+                {
+                  quantity: 1,
+                  price: 2,
+                },
+                {
+                  quantity: 2,
+                  price: 10,
+                },
+              ],
+            })
+
+            const { rows } = await config.api.viewV2.search(view.id, {
+              query: {},
+              sort: "quantity",
+              sortOrder: SortOrder.DESCENDING,
+            })
+
+            expect(rows).toEqual([
+              expect.objectContaining({ quantity: 2, sum: 10 }),
+              expect.objectContaining({ quantity: 1, sum: 3 }),
+            ])
+          })
+
+          it("should be able to sort by a calculation", async () => {
+            const table = await config.api.table.save(
+              saveTableRequest({
+                schema: {
+                  quantity: {
+                    type: FieldType.NUMBER,
+                    name: "quantity",
+                  },
+                  price: {
+                    type: FieldType.NUMBER,
+                    name: "price",
+                  },
+                },
+              })
+            )
+
+            await config.api.row.bulkImport(table._id!, {
+              rows: [
+                {
+                  quantity: 1,
+                  price: 1,
+                },
+                {
+                  quantity: 1,
+                  price: 2,
+                },
+                {
+                  quantity: 2,
+                  price: 10,
+                },
+              ],
+            })
+
+            const view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
+              schema: {
+                quantity: { visible: true },
+                sum: {
+                  visible: true,
+                  calculationType: CalculationType.SUM,
+                  field: "price",
+                },
+              },
+            })
+
+            const { rows } = await config.api.viewV2.search(view.id, {
+              query: {},
+              sort: "sum",
+              sortOrder: SortOrder.DESCENDING,
+            })
+
+            expect(rows).toEqual([
+              expect.objectContaining({ quantity: 2, sum: 10 }),
+              expect.objectContaining({ quantity: 1, sum: 3 }),
+            ])
+          })
         })
 
       !isLucene &&
@@ -2853,6 +3529,7 @@ describe.each([
           const view = await config.api.viewV2.create({
             tableId: table._id!,
             name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
             schema: {
               sum: {
                 visible: true,
@@ -2869,6 +3546,50 @@ describe.each([
           expect(response.rows).toHaveLength(1)
           expect(response.rows[0].sum).toEqual(61)
         })
+
+      it("should be able to filter on a single user field in both the view query and search query", async () => {
+        const table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              user: {
+                name: "user",
+                type: FieldType.BB_REFERENCE_SINGLE,
+                subtype: BBReferenceFieldSubType.USER,
+              },
+            },
+          })
+        )
+
+        await config.api.row.save(table._id!, {
+          user: config.getUser()._id,
+        })
+
+        const view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          query: {
+            equal: {
+              user: "{{ [user].[_id] }}",
+            },
+          },
+          schema: {
+            user: {
+              visible: true,
+            },
+          },
+        })
+
+        const { rows } = await config.api.viewV2.search(view.id, {
+          query: {
+            equal: {
+              user: "{{ [user].[_id] }}",
+            },
+          },
+        })
+
+        expect(rows).toHaveLength(1)
+        expect(rows[0].user._id).toEqual(config.getUser()._id)
+      })
     })
 
     describe("permissions", () => {

packages/server/src/api/routes/utils/validators.ts

@@ -20,31 +20,30 @@ const OPTIONAL_NUMBER = Joi.number().optional().allow(null)
 const OPTIONAL_BOOLEAN = Joi.boolean().optional().allow(null)
 const APP_NAME_REGEX = /^[\w\s]+$/
 
-const validateViewSchemas: CustomValidator<Table> = (table, helpers) => {
-  if (table.views && Object.entries(table.views).length) {
-    const requiredFields = Object.entries(table.schema)
-      .filter(([_, v]) => isRequired(v.constraints))
-      .map(([key]) => key)
-    if (requiredFields.length) {
+const validateViewSchemas: CustomValidator<Table> = (table, joiHelpers) => {
+  if (!table.views || Object.keys(table.views).length === 0) {
+    return table
+  }
+  const required = Object.keys(table.schema).filter(key =>
+    isRequired(table.schema[key].constraints)
+  )
+  if (required.length === 0) {
+    return table
+  }
   for (const view of Object.values(table.views)) {
-        if (!sdk.views.isV2(view)) {
+    if (!sdk.views.isV2(view) || helpers.views.isCalculationView(view)) {
       continue
     }
-
-        const editableViewFields = Object.entries(view.schema || {})
+    const editable = Object.entries(view.schema || {})
       .filter(([_, f]) => f.visible && !f.readonly)
       .map(([key]) => key)
-        const missingField = requiredFields.find(
-          f => !editableViewFields.includes(f)
-        )
+    const missingField = required.find(f => !editable.includes(f))
     if (missingField) {
-          return helpers.message({
+      return joiHelpers.message({
         custom: `To make field "${missingField}" required, this field must be present and writable in views: ${view.name}.`,
       })
     }
   }
-    }
-  }
   return table
 }
 

packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts

@@ -2,8 +2,8 @@ import * as setup from "../../../api/routes/tests/utilities"
 import { basicTable } from "../../../tests/utilities/structures"
 import {
   db as dbCore,
+  features,
   SQLITE_DESIGN_DOC_ID,
-  withEnv as withCoreEnv,
 } from "@budibase/backend-core"
 import {
   LinkDocument,
@@ -71,11 +71,11 @@ function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
 }
 
 async function sqsDisabled(cb: () => Promise<void>) {
-  await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" }, cb)
+  await features.testutils.withFeatureFlags("*", { SQS: false }, cb)
 }
 
 async function sqsEnabled(cb: () => Promise<void>) {
-  await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, cb)
+  await features.testutils.withFeatureFlags("*", { SQS: true }, cb)
 }
 
 describe("SQS migration", () => {

packages/server/src/db/linkedRows/LinkController.ts

@@ -221,9 +221,15 @@ class LinkController {
                 link.id !== row._id && link.fieldName === linkedSchema.name
             )
 
+            // check all the related rows exist
+            const foundRecords = await this._db.getMultiple(
+              links.map(l => l.id),
+              { allowMissing: true, excludeDocs: true }
+            )
+
             // The 1 side of 1:N is already related to something else
             // You must remove the existing relationship
-            if (links.length > 0) {
+            if (foundRecords.length > 0) {
               throw new Error(
                 `1:N Relationship Error: Record already linked to another.`
               )

packages/server/src/environment.ts

@@ -83,6 +83,7 @@ const environment = {
   PLUGINS_DIR: process.env.PLUGINS_DIR || DEFAULTS.PLUGINS_DIR,
   MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB,
   SESSION_EXPIRY_SECONDS: process.env.SESSION_EXPIRY_SECONDS,
+  XSS_SAFE_MODE: process.env.XSS_SAFE_MODE,
   // SQL
   SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
   SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE,

packages/server/src/integrations/postgres.ts

@@ -41,7 +41,7 @@ if (types) {
   types.setTypeParser(1184, (val: any) => val) // timestampz
 }
 
-const JSON_REGEX = /'{.*}'::json/s
+const JSON_REGEX = /'{\s*.*?\s*}'::json/gs
 const Sql = sql.Sql
 
 interface PostgresConfig {

packages/server/src/jsRunner/tests/jsRunner.spec.ts

@@ -8,7 +8,6 @@ import { init } from ".."
 import TestConfiguration from "../../tests/utilities/TestConfiguration"
 
 const DATE = "2021-01-21T12:00:00"
-
 tk.freeze(DATE)
 
 describe("jsRunner (using isolated-vm)", () => {
@@ -48,6 +47,33 @@ describe("jsRunner (using isolated-vm)", () => {
     ).toEqual("ReferenceError: process is not defined")
   })
 
+  it("should not allow the context to be mutated", async () => {
+    const context = { array: [1] }
+    const result = await processJS(
+      `
+        const array = $("array");
+        array.push(2);
+        return array[1]
+      `,
+      context
+    )
+    expect(result).toEqual(2)
+    expect(context.array).toEqual([1])
+  })
+
+  it("should copy values whenever returning them from $", async () => {
+    const context = { array: [1] }
+    const result = await processJS(
+      `
+        $("array").push(2);
+        return $("array")[1];
+      `,
+      context
+    )
+    expect(result).toEqual(undefined)
+    expect(context.array).toEqual([1])
+  })
+
   describe("helpers", () => {
     runJsHelpersTests({
       funcWrap: (func: any) => config.doInContext(config.getAppId(), func),

packages/server/src/koa.ts

@@ -12,6 +12,7 @@ import { userAgent } from "koa-useragent"
 
 export default function createKoaApp() {
   const app = new Koa()
+  app.proxy = true
 
   let mbNumber = parseInt(env.HTTP_MB_LIMIT || "10")
   if (!mbNumber || isNaN(mbNumber)) {
@@ -35,6 +36,7 @@ export default function createKoaApp() {
 
   app.use(middleware.correlation)
   app.use(middleware.pino)
+  app.use(middleware.ip)
   app.use(userAgent)
 
   const server = http.createServer(app.callback())

packages/server/src/migrations/tests/index.spec.ts

@@ -71,7 +71,7 @@ describe("migrations", () => {
         expect(events.datasource.created).toHaveBeenCalledTimes(2)
         expect(events.layout.created).toHaveBeenCalledTimes(1)
         expect(events.query.created).toHaveBeenCalledTimes(2)
-        expect(events.role.created).toHaveBeenCalledTimes(2)
+        expect(events.role.created).toHaveBeenCalledTimes(3) // created roles + admin (created on table creation)
         expect(events.table.created).toHaveBeenCalledTimes(3)
         expect(events.view.created).toHaveBeenCalledTimes(2)
         expect(events.view.calculationCreated).toHaveBeenCalledTimes(1)
@@ -82,7 +82,7 @@ describe("migrations", () => {
         // to make sure caching is working as expected
         expect(
           events.processors.analyticsProcessor.processEvent
-        ).toHaveBeenCalledTimes(23)
+        ).toHaveBeenCalledTimes(24) // Addtion of of the events above
       })
     })
   })

packages/server/src/sdk/app/automations/utils.ts

@@ -26,13 +26,13 @@ export async function getBuilderData(
     return tableNameCache[tableId]
   }
 
-  const rowActionNameCache: Record<string, TableRowActions> = {}
+  const rowActionNameCache: Record<string, TableRowActions | undefined> = {}
   async function getRowActionName(tableId: string, rowActionId: string) {
     if (!rowActionNameCache[tableId]) {
       rowActionNameCache[tableId] = await sdk.rowActions.getAll(tableId)
     }
 
-    return rowActionNameCache[tableId].actions[rowActionId]?.name
+    return rowActionNameCache[tableId]?.actions[rowActionId]?.name
   }
 
   const result: Record<string, AutomationBuilderData> = {}
@@ -51,6 +51,10 @@ export async function getBuilderData(
     const tableName = await getTableName(tableId)
     const rowActionName = await getRowActionName(tableId, rowActionId)
 
+    if (!rowActionName) {
+      throw new Error(`Row action not found: ${rowActionId}`)
+    }
+
     result[automation._id!] = {
       displayName: rowActionName,
       triggerInfo: {

packages/server/src/sdk/app/permissions/index.ts

@@ -73,8 +73,7 @@ export async function getResourcePerms(
     p[level] = { role, type: PermissionSource.BASE }
     return p
   }, {})
-  const result = Object.assign(basePermissions, permissions)
-  return result
+  return Object.assign(basePermissions, permissions)
 }
 
 export async function getDependantResources(
@@ -185,6 +184,26 @@ export async function updatePermissionOnRole(
   })
 }
 
+export async function setPermissions(
+  resourceId: string,
+  {
+    writeRole,
+    readRole,
+  }: {
+    writeRole: string
+    readRole: string
+  }
+) {
+  await updatePermissionOnRole(
+    { roleId: writeRole, resourceId, level: PermissionLevel.WRITE },
+    PermissionUpdateType.ADD
+  )
+  await updatePermissionOnRole(
+    { roleId: readRole, resourceId, level: PermissionLevel.READ },
+    PermissionUpdateType.ADD
+  )
+}
+
 // utility function to stop this repetition - permissions always stored under roles
 export async function getAllDBRoles(db: Database) {
   const body = await db.allDocs<Role>(

packages/server/src/sdk/app/rowActions.ts

@@ -1,5 +1,6 @@
 import { context, docIds, HTTPError, utils } from "@budibase/backend-core"
 import {
+  Automation,
   AutomationTriggerStepId,
   SEPARATOR,
   TableRowActions,
@@ -102,7 +103,25 @@ export async function get(tableId: string, rowActionId: string) {
 export async function getAll(tableId: string) {
   const db = context.getAppDB()
   const rowActionsId = generateRowActionsID(tableId)
-  return await db.get<TableRowActions>(rowActionsId)
+  return await db.tryGet<TableRowActions>(rowActionsId)
+}
+
+export async function deleteAll(tableId: string) {
+  const db = context.getAppDB()
+
+  const doc = await getAll(tableId)
+  if (!doc) {
+    return
+  }
+
+  const automationIds = Object.values(doc.actions).map(a => a.automationId)
+  const automations = await db.getMultiple<Automation>(automationIds)
+
+  for (const automation of automations) {
+    await sdk.automations.remove(automation._id!, automation._rev!)
+  }
+
+  await db.remove(doc)
 }
 
 export async function docExists(tableId: string) {
@@ -223,9 +242,8 @@ export async function run(tableId: any, rowActionId: any, rowId: string) {
     throw new HTTPError("Table not found", 404)
   }
 
-  const { actions } = await getAll(tableId)
-
-  const rowAction = actions[rowActionId]
+  const rowActions = await getAll(tableId)
+  const rowAction = rowActions?.actions[rowActionId]
   if (!rowAction) {
     throw new HTTPError("Row action not found", 404)
   }

packages/server/src/sdk/app/rows/external.ts

@@ -15,6 +15,7 @@ import {
 } from "../../../utilities/rowProcessor"
 import cloneDeep from "lodash/fp/cloneDeep"
 import { tryExtractingTableAndViewId } from "./utils"
+import { helpers } from "@budibase/shared-core"
 
 export async function getRow(
   sourceId: string | Table | ViewV2,
@@ -54,6 +55,10 @@ export async function save(
     source = await sdk.tables.getTable(tableId)
   }
 
+  if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
+    throw new HTTPError("Cannot insert rows through a calculation view", 400)
+  }
+
   const row = await inputProcessing(userId, cloneDeep(source), inputs)
 
   const validateResult = await sdk.rows.utils.validate({

packages/server/src/sdk/app/rows/internal.ts

@@ -1,4 +1,4 @@
-import { context, db } from "@budibase/backend-core"
+import { context, db, HTTPError } from "@budibase/backend-core"
 import { Row, Table, ViewV2 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { finaliseRow } from "../../../api/controllers/row/staticFormula"
@@ -10,6 +10,7 @@ import * as linkRows from "../../../db/linkedRows"
 import { InternalTables } from "../../../db/utils"
 import { getFullUser } from "../../../utilities/users"
 import { getSource, tryExtractingTableAndViewId } from "./utils"
+import { helpers } from "@budibase/shared-core"
 
 export async function save(
   tableOrViewId: string,
@@ -29,6 +30,10 @@ export async function save(
     table = source
   }
 
+  if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
+    throw new HTTPError("Cannot insert rows through a calculation view", 400)
+  }
+
   if (!inputs._rev && !inputs._id) {
     inputs._id = db.generateRowID(inputs.tableId)
   }

packages/server/src/sdk/app/rows/search.ts

@@ -16,7 +16,7 @@ import * as external from "./search/external"
 import { ExportRowsParams, ExportRowsResult } from "./search/types"
 import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../index"
-import { searchInputMapping } from "./search/utils"
+import { checkFilters, searchInputMapping } from "./search/utils"
 import { db, features } from "@budibase/backend-core"
 import tracer from "dd-trace"
 import { getQueryableFields, removeInvalidFilters } from "./queryUtils"
@@ -81,6 +81,10 @@ export async function search(
       options.query = {}
     }
 
+    if (context) {
+      options.query = await enrichSearchContext(options.query, context)
+    }
+
     // need to make sure filters in correct shape before checking for view
     options = searchInputMapping(table, options)
 
@@ -92,12 +96,15 @@ export async function search(
       // Enrich saved query with ephemeral query params.
       // We prevent searching on any fields that are saved as part of the query, as
       // that could let users find rows they should not be allowed to access.
-      let viewQuery = dataFilters.buildQueryLegacy(view.query) || {}
+      let viewQuery = await enrichSearchContext(view.query || {}, context)
+      viewQuery = dataFilters.buildQueryLegacy(viewQuery) || {}
+      viewQuery = checkFilters(table, viewQuery)
       delete viewQuery?.onEmptyFilter
 
       const sqsEnabled = await features.flags.isEnabled("SQS")
       const supportsLogicalOperators =
         isExternalTableID(view.tableId) || sqsEnabled
+
       if (!supportsLogicalOperators) {
         // In the unlikely event that a Grouped Filter is in a non-SQS environment
         // It needs to be ignored entirely
@@ -113,13 +120,12 @@ export async function search(
             ?.filter(filter => filter.field)
             .map(filter => db.removeKeyNumbering(filter.field)) || []
 
-        viewQuery ??= {}
         // Carry over filters for unused fields
         Object.keys(options.query).forEach(key => {
           const operator = key as Exclude<SearchFilterKey, LogicalOperator>
           Object.keys(options.query[operator] || {}).forEach(field => {
             if (!existingFields.includes(db.removeKeyNumbering(field))) {
-              viewQuery![operator]![field] = options.query[operator]![field]
+              viewQuery[operator]![field] = options.query[operator]![field]
             }
           })
         })
@@ -137,10 +143,6 @@ export async function search(
       }
     }
 
-    if (context) {
-      options.query = await enrichSearchContext(options.query, context)
-    }
-
     options.query = dataFilters.cleanupQuery(options.query)
     options.query = dataFilters.fixupFilterArrays(options.query)
 

packages/server/src/sdk/app/rows/search/internal/internal.ts

@@ -62,10 +62,10 @@ export async function exportRows(
     ).rows.map(row => row.doc!)
 
     result = await outputProcessing(table, response)
-  } else if (query) {
+  } else {
     let searchResponse = await sdk.rows.search({
       tableId,
-      query,
+      query: query || {},
       sort,
       sortOrder,
     })

packages/server/src/sdk/app/rows/search/internal/sqs.ts

@@ -68,9 +68,7 @@ async function buildInternalFieldList(
   const { relationships, allowedFields } = opts || {}
   let schemaFields: string[] = []
   if (sdk.views.isView(source)) {
-    schemaFields = Object.keys(helpers.views.basicFields(source)).filter(
-      key => source.schema?.[key]?.visible !== false
-    )
+    schemaFields = Object.keys(helpers.views.basicFields(source))
   } else {
     schemaFields = Object.keys(source.schema).filter(
       key => source.schema[key].visible !== false
@@ -420,6 +418,16 @@ export async function search(
 
   if (params.sort) {
     const sortField = table.schema[params.sort]
+    const isAggregateField = aggregations.some(agg => agg.name === params.sort)
+
+    if (isAggregateField) {
+      request.sort = {
+        [params.sort]: {
+          direction: params.sortOrder || SortOrder.ASCENDING,
+          type: SortType.NUMBER,
+        },
+      }
+    } else if (sortField) {
       const sortType =
         sortField.type === FieldType.NUMBER ? SortType.NUMBER : SortType.STRING
       request.sort = {
@@ -428,6 +436,9 @@ export async function search(
           type: sortType as SortType,
         },
       }
+    } else {
+      throw new Error(`Unable to sort by ${params.sort}`)
+    }
   }
 
   if (params.bookmark && typeof params.bookmark !== "number") {

packages/server/src/sdk/app/rows/search/tests/search.spec.ts

@@ -10,10 +10,7 @@ import {
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
 import { search } from "../../../../../sdk/app/rows/search"
 import { generator } from "@budibase/backend-core/tests"
-import {
-  withEnv as withCoreEnv,
-  setEnv as setCoreEnv,
-} from "@budibase/backend-core"
+import { features } from "@budibase/backend-core"
 import {
   DatabaseName,
   getDatasource,
@@ -41,19 +38,13 @@ describe.each([
   let table: Table
 
   beforeAll(async () => {
-    await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () =>
+    await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
       config.init()
     )
 
-    if (isLucene) {
-      envCleanup = setCoreEnv({
-        TENANT_FEATURE_FLAGS: "*:!SQS",
+    envCleanup = features.testutils.setFeatureFlags("*", {
+      SQS: isSqs,
     })
-    } else if (isSqs) {
-      envCleanup = setCoreEnv({
-        TENANT_FEATURE_FLAGS: "*:SQS",
-      })
-    }
 
     if (dsProvider) {
       datasource = await config.createDatasource({

packages/server/src/sdk/app/rows/search/utils.ts

@@ -80,11 +80,10 @@ function userColumnMapping(column: string, filters: SearchFilters) {
   })
 }
 
-// maps through the search parameters to check if any of the inputs are invalid
-// based on the table schema, converts them to something that is valid.
-export function searchInputMapping(table: Table, options: RowSearchParams) {
-  // need an internal function to loop over filters, because this takes the full options
-  function checkFilters(filters: SearchFilters) {
+export function checkFilters(
+  table: Table,
+  filters: SearchFilters
+): SearchFilters {
   for (let [key, column] of Object.entries(table.schema || {})) {
     switch (column.type) {
       case FieldType.BB_REFERENCE_SINGLE: {
@@ -105,10 +104,17 @@ export function searchInputMapping(table: Table, options: RowSearchParams) {
       }
     }
   }
-    return dataFilters.recurseLogicalOperators(filters, checkFilters)
+  return dataFilters.recurseLogicalOperators(filters, filters =>
+    checkFilters(table, filters)
+  )
 }
+
+// maps through the search parameters to check if any of the inputs are invalid
+// based on the table schema, converts them to something that is valid.
+export function searchInputMapping(table: Table, options: RowSearchParams) {
+  // need an internal function to loop over filters, because this takes the full options
   if (options.query) {
-    options.query = checkFilters(options.query)
+    options.query = checkFilters(table, options.query)
   }
   return options
 }

packages/server/src/sdk/app/rows/tests/utils.spec.ts

@@ -8,6 +8,7 @@ import {
 import { generateTableID } from "../../../../db/utils"
 import { validate } from "../utils"
 import { generator } from "@budibase/backend-core/tests"
+import { withEnv } from "../../../../environment"
 
 describe("validate", () => {
   const hour = () => generator.hour().toString().padStart(2, "0")
@@ -332,4 +333,46 @@ describe("validate", () => {
       })
     })
   })
+
+  describe("XSS Safe mode", () => {
+    const getTable = (): Table => ({
+      type: "table",
+      _id: generateTableID(),
+      name: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
+      schema: {
+        text: {
+          name: "sometext",
+          type: FieldType.STRING,
+        },
+      },
+    })
+    it.each([
+      "SELECT * FROM users WHERE username = 'admin' --",
+      "SELECT * FROM users WHERE id = 1; DROP TABLE users;",
+      "1' OR '1' = '1",
+      "' OR 'a' = 'a",
+      "<script>alert('XSS');</script>",
+      '"><img src=x onerror=alert(1)>',
+      "</script><script>alert('test')</script>",
+      "<div onmouseover=\"alert('XSS')\">Hover over me!</div>",
+      "'; EXEC sp_msforeachtable 'DROP TABLE ?'; --",
+      "{alert('Injected')}",
+      "UNION SELECT * FROM users",
+      "INSERT INTO users (username, password) VALUES ('admin', 'password')",
+      "/* This is a comment */ SELECT * FROM users",
+      '<iframe src="http://malicious-site.com"></iframe>',
+    ])("test potentially unsafe input: %s", async input => {
+      await withEnv({ XSS_SAFE_MODE: "1" }, async () => {
+        const table = getTable()
+        const row = { text: input }
+        const output = await validate({ source: table, row })
+        expect(output.valid).toBe(false)
+        expect(output.errors).toStrictEqual({
+          text: ["Input not sanitised - potentially vulnerable to XSS"],
+        })
+      })
+    })
+  })
 })

packages/server/src/sdk/app/rows/utils.ts

@@ -22,6 +22,7 @@ import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils"
 import { isSQL } from "../../../integrations/utils"
 import { docIds, sql } from "@budibase/backend-core"
 import { getTableFromSource } from "../../../api/controllers/row/utils"
+import env from "../../../environment"
 
 const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
   [SourceName.POSTGRES]: SqlClient.POSTGRES,
@@ -43,6 +44,9 @@ const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
   [SourceName.BUDIBASE]: undefined,
 }
 
+const XSS_INPUT_REGEX =
+  /[<>;"'(){}]|--|\/\*|\*\/|union|select|insert|drop|delete|update|exec|script/i
+
 export function getSQLClient(datasource: Datasource): SqlClient {
   if (!isSQL(datasource)) {
     throw new Error("Cannot get SQL Client for non-SQL datasource")
@@ -222,6 +226,15 @@ export async function validate({
     } else {
       res = validateJs.single(row[fieldName], constraints)
     }
+
+    if (env.XSS_SAFE_MODE && typeof row[fieldName] === "string") {
+      if (XSS_INPUT_REGEX.test(row[fieldName])) {
+        errors[fieldName] = [
+          "Input not sanitised - potentially vulnerable to XSS",
+        ]
+      }
+    }
+
     if (res) errors[fieldName] = res
   }
   return { valid: Object.keys(errors).length === 0, errors }

packages/server/src/sdk/app/tables/create.ts

@@ -0,0 +1,33 @@
+import { FeatureFlag, Row, Table } from "@budibase/types"
+
+import * as external from "./external"
+import * as internal from "./internal"
+import { isExternal } from "./utils"
+import { setPermissions } from "../permissions"
+import { features, roles } from "@budibase/backend-core"
+
+export async function create(
+  table: Omit<Table, "_id" | "_rev">,
+  rows?: Row[],
+  userId?: string
+): Promise<Table> {
+  let createdTable: Table
+  if (isExternal({ table })) {
+    createdTable = await external.create(table)
+  } else {
+    createdTable = await internal.create(table, rows, userId)
+  }
+
+  const setExplicitPermission = await features.flags.isEnabled(
+    FeatureFlag.TABLES_DEFAULT_ADMIN
+  )
+
+  if (setExplicitPermission) {
+    await setPermissions(createdTable._id!, {
+      writeRole: roles.BUILTIN_ROLE_IDS.ADMIN,
+      readRole: roles.BUILTIN_ROLE_IDS.ADMIN,
+    })
+  }
+
+  return createdTable
+}

packages/server/src/sdk/app/tables/external/index.ts

@@ -8,8 +8,11 @@ import {
   ViewV2,
   AutoFieldSubType,
 } from "@budibase/types"
-import { context } from "@budibase/backend-core"
-import { buildExternalTableId } from "../../../../integrations/utils"
+import { context, HTTPError } from "@budibase/backend-core"
+import {
+  breakExternalTableId,
+  buildExternalTableId,
+} from "../../../../integrations/utils"
 import {
   foreignKeyStructure,
   hasTypeChanged,
@@ -86,6 +89,35 @@ function validate(table: Table, oldTable?: Table) {
   }
 }
 
+function getDatasourceId(table: Table) {
+  if (!table) {
+    throw new Error("No table supplied")
+  }
+  if (table.sourceId) {
+    return table.sourceId
+  }
+  if (!table._id) {
+    throw new Error("No table ID supplied")
+  }
+  return breakExternalTableId(table._id).datasourceId
+}
+
+export async function create(table: Omit<Table, "_id" | "_rev">) {
+  const datasourceId = getDatasourceId(table)
+
+  const tableToCreate = { ...table, created: true }
+  try {
+    const result = await save(datasourceId!, tableToCreate)
+    return result.table
+  } catch (err: any) {
+    if (err instanceof Error) {
+      throw new HTTPError(err.message, 400)
+    } else {
+      throw new HTTPError(err?.message || err, err.status || 500)
+    }
+  }
+}
+
 export async function save(
   datasourceId: string,
   update: Table,

packages/server/src/sdk/app/tables/getters.ts

@@ -20,7 +20,13 @@ export async function processTable(table: Table): Promise<Table> {
   if (!table) {
     return table
   }
+
+  table = { ...table }
   if (table._id && isExternalTableID(table._id)) {
+    // Old created external tables via Budibase might have a missing field name breaking some UI such as filters
+    if (table.schema["id"] && !table.schema["id"].name) {
+      table.schema["id"].name = "id"
+    }
     return {
       ...table,
       type: "table",

packages/server/src/sdk/app/tables/index.ts

@@ -1,5 +1,6 @@
 import { populateExternalTableSchemas } from "./validation"
 import * as getters from "./getters"
+import * as create from "./create"
 import * as updates from "./update"
 import * as utils from "./utils"
 import { migrate } from "./migration"
@@ -7,6 +8,7 @@ import * as sqs from "./internal/sqs"
 
 export default {
   populateExternalTableSchemas,
+  ...create,
   ...updates,
   ...getters,
   ...utils,

packages/server/src/sdk/app/tables/internal/index.ts

@@ -5,7 +5,7 @@ import {
   ViewStatisticsSchema,
   ViewV2,
   Row,
-  ContextUser,
+  TableSourceType,
 } from "@budibase/types"
 import {
   hasTypeChanged,
@@ -16,18 +16,56 @@ import { EventType, updateLinks } from "../../../../db/linkedRows"
 import { cloneDeep } from "lodash/fp"
 import isEqual from "lodash/isEqual"
 import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula"
-import { context } from "@budibase/backend-core"
+import { context, HTTPError } from "@budibase/backend-core"
 import { findDuplicateInternalColumns } from "@budibase/shared-core"
 import { getTable } from "../getters"
 import { checkAutoColumns } from "./utils"
 import * as viewsSdk from "../../views"
-import { getRowParams } from "../../../../db/utils"
+import { generateTableID, getRowParams } from "../../../../db/utils"
 import { quotas } from "@budibase/pro"
 
+export async function create(
+  table: Omit<Table, "_id" | "_rev">,
+  rows?: Row[],
+  userId?: string
+) {
+  const tableId = generateTableID()
+
+  let tableToSave: Table = {
+    _id: tableId,
+    ...table,
+    // Ensure these fields are populated, even if not sent in the request
+    type: table.type || "table",
+    sourceType: TableSourceType.INTERNAL,
+  }
+
+  const isImport = !!rows
+
+  if (!tableToSave.views) {
+    tableToSave.views = {}
+  }
+
+  try {
+    const { table } = await save(tableToSave, {
+      userId,
+      rowsToImport: rows,
+      isImport,
+    })
+
+    return table
+  } catch (err: any) {
+    if (err instanceof Error) {
+      throw new HTTPError(err.message, 400)
+    } else {
+      throw new HTTPError(err.message || err, err.status || 500)
+    }
+  }
+}
+
 export async function save(
   table: Table,
   opts?: {
-    user?: ContextUser
+    userId?: string
     tableId?: string
     rowsToImport?: Row[]
     renaming?: RenameColumn
@@ -63,7 +101,7 @@ export async function save(
   // saving a table is a complex operation, involving many different steps, this
   // has been broken out into a utility to make it more obvious/easier to manipulate
   const tableSaveFunctions = new TableSaveFunctions({
-    user: opts?.user,
+    userId: opts?.userId,
     oldTable,
     importRows: opts?.rowsToImport,
   })

packages/server/src/sdk/app/views/external.ts

@@ -70,6 +70,9 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
   if (!existingView || !existingView.name) {
     throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404)
   }
+  if (isV2(existingView) && existingView.type !== view.type) {
+    throw new HTTPError(`Cannot update view type after creation`, 400)
+  }
 
   delete views[existingView.name]
   views[view.name] = view

packages/server/src/sdk/app/views/index.ts

@@ -1,9 +1,14 @@
 import {
+  BBReferenceFieldSubType,
   CalculationType,
+  canGroupBy,
+  FeatureFlag,
   FieldType,
+  isNumeric,
   PermissionLevel,
   RelationSchemaField,
   RenameColumn,
+  RequiredKeys,
   Table,
   TableSchema,
   View,
@@ -11,7 +16,7 @@ import {
   ViewV2ColumnEnriched,
   ViewV2Enriched,
 } from "@budibase/types"
-import { context, docIds, HTTPError, roles } from "@budibase/backend-core"
+import { context, docIds, features, HTTPError } from "@budibase/backend-core"
 import {
   helpers,
   PROTECTED_EXTERNAL_COLUMNS,
@@ -22,7 +27,6 @@ import { isExternalTableID } from "../../../integrations/utils"
 import * as internal from "./internal"
 import * as external from "./external"
 import sdk from "../../../sdk"
-import { PermissionUpdateType, updatePermissionOnRole } from "../permissions"
 
 function pickApi(tableId: any) {
   if (isExternalTableID(tableId)) {
@@ -62,28 +66,55 @@ async function guardCalculationViewSchema(
   view: Omit<ViewV2, "id" | "version">
 ) {
   const calculationFields = helpers.views.calculationFields(view)
-  for (const calculationFieldName of Object.keys(calculationFields)) {
-    const schema = calculationFields[calculationFieldName]
+
+  if (Object.keys(calculationFields).length > 5) {
+    throw new HTTPError(
+      "Calculation views can only have a maximum of 5 fields",
+      400
+    )
+  }
+
+  const seen: Record<string, Record<CalculationType, boolean>> = {}
+
+  for (const name of Object.keys(calculationFields)) {
+    const schema = calculationFields[name]
     const isCount = schema.calculationType === CalculationType.COUNT
     const isDistinct = isCount && "distinct" in schema && schema.distinct
 
+    const field = isCount && !isDistinct ? "*" : schema.field
+    if (seen[field]?.[schema.calculationType]) {
+      throw new HTTPError(
+        `Duplicate calculation on field "${field}", calculation type "${schema.calculationType}"`,
+        400
+      )
+    }
+    seen[field] ??= {} as Record<CalculationType, boolean>
+    seen[field][schema.calculationType] = true
+
     // Count fields that aren't distinct don't need to reference another field,
     // so we don't validate it.
     if (isCount && !isDistinct) {
       continue
     }
 
-    const targetSchema = table.schema[schema.field]
-    if (!targetSchema) {
+    if (!schema.field) {
       throw new HTTPError(
-        `Calculation field "${calculationFieldName}" references field "${schema.field}" which does not exist in the table schema`,
+        `Calculation field "${name}" is missing a "field" property`,
         400
       )
     }
 
-    if (!isCount && !helpers.schema.isNumeric(targetSchema)) {
+    const targetSchema = table.schema[schema.field]
+    if (!targetSchema) {
       throw new HTTPError(
-        `Calculation field "${calculationFieldName}" references field "${schema.field}" which is not a numeric field`,
+        `Calculation field "${name}" references field "${schema.field}" which does not exist in the table schema`,
+        400
+      )
+    }
+
+    if (!isCount && !isNumeric(targetSchema.type)) {
+      throw new HTTPError(
+        `Calculation field "${name}" references field "${schema.field}" which is not a numeric field`,
         400
       )
     }
@@ -98,6 +129,13 @@ async function guardCalculationViewSchema(
         400
       )
     }
+
+    if (!canGroupBy(targetSchema.type)) {
+      throw new HTTPError(
+        `Grouping by fields of type "${targetSchema.type}" is not supported`,
+        400
+      )
+    }
   }
 }
 
@@ -109,10 +147,21 @@ async function guardViewSchema(
 
   if (helpers.views.isCalculationView(view)) {
     await guardCalculationViewSchema(table, view)
+  } else {
+    if (helpers.views.hasCalculationFields(view)) {
+      throw new HTTPError(
+        "Calculation fields are not allowed in non-calculation views",
+        400
+      )
+    }
   }
 
   await checkReadonlyFields(table, view)
+
+  if (!helpers.views.isCalculationView(view)) {
     checkRequiredFields(table, view)
+  }
+
   checkDisplayField(view)
 }
 
@@ -202,26 +251,17 @@ export async function create(
   await guardViewSchema(tableId, viewRequest)
   const view = await pickApi(tableId).create(tableId, viewRequest)
 
+  const setExplicitPermission = await features.flags.isEnabled(
+    FeatureFlag.TABLES_DEFAULT_ADMIN
+  )
+  if (setExplicitPermission) {
     // Set permissions to be the same as the table
     const tablePerms = await sdk.permissions.getResourcePerms(tableId)
-  const readRole = tablePerms[PermissionLevel.READ]?.role
-  const writeRole = tablePerms[PermissionLevel.WRITE]?.role
-  await updatePermissionOnRole(
-    {
-      roleId: readRole || roles.BUILTIN_ROLE_IDS.BASIC,
-      resourceId: view.id,
-      level: PermissionLevel.READ,
-    },
-    PermissionUpdateType.ADD
-  )
-  await updatePermissionOnRole(
-    {
-      roleId: writeRole || roles.BUILTIN_ROLE_IDS.BASIC,
-      resourceId: view.id,
-      level: PermissionLevel.WRITE,
-    },
-    PermissionUpdateType.ADD
-  )
+    await sdk.permissions.setPermissions(view.id, {
+      writeRole: tablePerms[PermissionLevel.WRITE].role,
+      readRole: tablePerms[PermissionLevel.READ].role,
+    })
+  }
 
   return view
 }
@@ -288,13 +328,26 @@ export async function enrichSchema(
       const viewFieldSchema = viewFields[relTableFieldName]
       const isVisible = !!viewFieldSchema?.visible
       const isReadonly = !!viewFieldSchema?.readonly
-      result[relTableFieldName] = {
-        ...relTableField,
-        ...viewFieldSchema,
-        name: relTableField.name,
+      const enrichedFieldSchema: RequiredKeys<ViewV2ColumnEnriched> = {
         visible: isVisible,
         readonly: isReadonly,
+        order: viewFieldSchema?.order,
+        width: viewFieldSchema?.width,
+
+        icon: relTableField.icon,
+        type: relTableField.type,
+        subtype: relTableField.subtype,
       }
+      if (
+        !enrichedFieldSchema.icon &&
+        relTableField.type === FieldType.BB_REFERENCE &&
+        relTableField.subtype === BBReferenceFieldSubType.USER &&
+        !helpers.schema.isDeprecatedSingleUserColumn(relTableField)
+      ) {
+        // Forcing the icon, otherwise we would need to pass the constraints to show the proper icon
+        enrichedFieldSchema.icon = "UserGroup"
+      }
+      result[relTableFieldName] = enrichedFieldSchema
     }
     return result
   }
@@ -347,7 +400,8 @@ export function syncSchema(
 
   if (view.schema) {
     for (const fieldName of Object.keys(view.schema)) {
-      if (!schema[fieldName]) {
+      const viewSchema = view.schema[fieldName]
+      if (!helpers.views.isCalculationField(viewSchema) && !schema[fieldName]) {
         delete view.schema[fieldName]
       }
     }

packages/server/src/sdk/app/views/internal.ts

@@ -59,6 +59,10 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
     throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404)
   }
 
+  if (isV2(existingView) && existingView.type !== view.type) {
+    throw new HTTPError(`Cannot update view type after creation`, 400)
+  }
+
   delete table.views[existingView.name]
   table.views[view.name] = view
   await db.put(table)

packages/server/src/sdk/app/views/tests/views.spec.ts

@@ -355,13 +355,11 @@ describe("table sdk", () => {
               visible: true,
               columns: {
                 title: {
-                  name: "title",
                   type: "string",
                   visible: true,
                   readonly: true,
                 },
                 age: {
-                  name: "age",
                   type: "number",
                   visible: false,
                   readonly: false,

packages/server/src/tests/utilities/api/automation.ts

@@ -1,4 +1,4 @@
-import { Automation } from "@budibase/types"
+import { Automation, FetchAutomationResponse } from "@budibase/types"
 import { Expectations, TestAPI } from "./base"
 
 export class AutomationAPI extends TestAPI {
@@ -14,6 +14,26 @@ export class AutomationAPI extends TestAPI {
     )
     return result
   }
+
+  fetch = async (
+    expectations?: Expectations
+  ): Promise<FetchAutomationResponse> => {
+    return await this._get<FetchAutomationResponse>(`/api/automations`, {
+      expectations,
+    })
+  }
+
+  fetchEnriched = async (
+    expectations?: Expectations
+  ): Promise<FetchAutomationResponse> => {
+    return await this._get<FetchAutomationResponse>(
+      `/api/automations?enrich=true`,
+      {
+        expectations,
+      }
+    )
+  }
+
   post = async (
     body: Automation,
     expectations?: Expectations

packages/server/src/tests/utilities/api/permission.ts

@@ -1,6 +1,7 @@
 import {
   AddPermissionRequest,
   AddPermissionResponse,
+  FetchResourcePermissionInfoResponse,
   GetResourcePermsResponse,
   RemovePermissionRequest,
   RemovePermissionResponse,
@@ -26,6 +27,15 @@ export class PermissionAPI extends TestAPI {
     )
   }
 
+  fetch = async (
+    expectations?: Expectations
+  ): Promise<FetchResourcePermissionInfoResponse> => {
+    return await this._get<FetchResourcePermissionInfoResponse>(
+      `/api/permission`,
+      { expectations }
+    )
+  }
+
   revoke = async (
     request: RemovePermissionRequest,
     expectations?: Expectations

packages/server/src/tests/utilities/api/row.ts

@@ -105,7 +105,7 @@ export class RowAPI extends TestAPI {
 
   exportRows = async (
     tableId: string,
-    body: ExportRowsRequest,
+    body?: ExportRowsRequest,
     format: RowExportFormat = RowExportFormat.JSON,
     expectations?: Expectations
   ) => {

packages/server/src/utilities/rowProcessor/index.ts

@@ -33,7 +33,7 @@ import {
   PROTECTED_EXTERNAL_COLUMNS,
   PROTECTED_INTERNAL_COLUMNS,
 } from "@budibase/shared-core"
-import { processString } from "@budibase/string-templates"
+import { processStringSync } from "@budibase/string-templates"
 import {
   getTableFromSource,
   isUserMetadataTable,
@@ -134,10 +134,15 @@ async function processDefaultValues(table: Table, row: Row) {
 
   for (const [key, schema] of Object.entries(table.schema)) {
     if ("default" in schema && schema.default != null && row[key] == null) {
-      const processed =
-        typeof schema.default === "string"
-          ? await processString(schema.default, ctx)
-          : schema.default
+      let processed: string | string[]
+      if (Array.isArray(schema.default)) {
+        processed = schema.default.map(val => processStringSync(val, ctx))
+      } else if (typeof schema.default === "string") {
+        processed = processStringSync(schema.default, ctx)
+      } else {
+        processed = schema.default
+      }
+
       try {
         row[key] = coerce(processed, schema.type)
       } catch (err: any) {

packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts

@@ -8,7 +8,7 @@ import {
 } from "@budibase/types"
 import { outputProcessing } from ".."
 import { generator, structures } from "@budibase/backend-core/tests"
-import { setEnv as setCoreEnv } from "@budibase/backend-core"
+import { features } from "@budibase/backend-core"
 import * as bbReferenceProcessor from "../bbReferenceProcessor"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 
@@ -21,7 +21,7 @@ jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
 
 describe("rowProcessor - outputProcessing", () => {
   const config = new TestConfiguration()
-  let cleanupEnv: () => void = () => {}
+  let cleanupFlags: () => void = () => {}
 
   beforeAll(async () => {
     await config.init()
@@ -33,11 +33,11 @@ describe("rowProcessor - outputProcessing", () => {
 
   beforeEach(() => {
     jest.resetAllMocks()
-    cleanupEnv = setCoreEnv({ TENANT_FEATURE_FLAGS: "*SQS" })
+    cleanupFlags = features.testutils.setFeatureFlags("*", { SQS: true })
   })
 
   afterEach(() => {
-    cleanupEnv()
+    cleanupFlags()
   })
 
   const processOutputBBReferenceMock =

packages/server/src/utilities/security.ts

@@ -1,5 +1,6 @@
-import { permissions, roles } from "@budibase/backend-core"
-import { DocumentType, VirtualDocumentType } from "../db/utils"
+import { DocumentType, permissions, roles } from "@budibase/backend-core"
+import { VirtualDocumentType } from "../db/utils"
+import { getDocumentType, getVirtualDocumentType } from "@budibase/types"
 
 export const CURRENTLY_SUPPORTED_LEVELS: string[] = [
   permissions.PermissionLevel.WRITE,
@@ -8,13 +9,17 @@ export const CURRENTLY_SUPPORTED_LEVELS: string[] = [
 ]
 
 export function getPermissionType(resourceId: string) {
-  const docType = Object.values(DocumentType).filter(docType =>
-    resourceId.startsWith(docType)
-  )[0]
-  switch (docType as DocumentType | VirtualDocumentType) {
+  const virtualDocType = getVirtualDocumentType(resourceId)
+  switch (virtualDocType) {
+    case VirtualDocumentType.VIEW:
+      return permissions.PermissionType.TABLE
+  }
+
+  const docType = getDocumentType(resourceId)
+  switch (docType) {
     case DocumentType.TABLE:
     case DocumentType.ROW:
-    case VirtualDocumentType.VIEW:
+    case DocumentType.DATASOURCE_PLUS:
       return permissions.PermissionType.TABLE
     case DocumentType.AUTOMATION:
       return permissions.PermissionType.AUTOMATION
@@ -32,22 +37,25 @@ export function getPermissionType(resourceId: string) {
 /**
  *  works out the basic permissions based on builtin roles for a resource, using its ID
  */
-export function getBasePermissions(resourceId: string) {
+export function getBasePermissions(resourceId: string): Record<string, string> {
   const type = getPermissionType(resourceId)
-  const basePermissions: { [key: string]: string } = {}
+  const basePermissions: Record<string, string> = {}
   for (let [roleId, role] of Object.entries(roles.getBuiltinRoles())) {
     if (!role.permissionId) {
       continue
     }
+
     const perms = permissions.getBuiltinPermissionByID(role.permissionId)
     if (!perms) {
       continue
     }
+
     const typedPermission = perms.permissions.find(perm => perm.type === type)
-    if (
-      typedPermission &&
-      CURRENTLY_SUPPORTED_LEVELS.indexOf(typedPermission.level) !== -1
-    ) {
+    if (!typedPermission) {
+      continue
+    }
+
+    if (CURRENTLY_SUPPORTED_LEVELS.includes(typedPermission.level)) {
       const level = typedPermission.level
       basePermissions[level] = roles.lowerBuiltinRoleID(
         basePermissions[level],

packages/shared-core/src/filters.ts

@@ -644,19 +644,19 @@ export function fixupFilterArrays(filters: SearchFilters) {
   return filters
 }
 
-export function search<T>(
-  docs: Record<string, T>[],
-  query: RowSearchParams
-): SearchResponse<Record<string, T>> {
+export function search<T extends Record<string, any>>(
+  docs: T[],
+  query: Omit<RowSearchParams, "tableId">
+): SearchResponse<T> {
   let result = runQuery(docs, query.query)
   if (query.sort) {
     result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING)
   }
-  let totalRows = result.length
+  const totalRows = result.length
   if (query.limit) {
     result = limit(result, query.limit.toString())
   }
-  const response: SearchResponse<Record<string, any>> = { rows: result }
+  const response: SearchResponse<T> = { rows: result }
   if (query.countRows) {
     response.totalRows = totalRows
   }

packages/shared-core/src/helpers/views.ts

@@ -3,6 +3,7 @@ import {
   ViewCalculationFieldMetadata,
   ViewFieldMetadata,
   ViewV2,
+  ViewV2Type,
 } from "@budibase/types"
 import { pickBy } from "lodash"
 
@@ -21,6 +22,10 @@ export function isBasicViewField(
 type UnsavedViewV2 = Omit<ViewV2, "id" | "version">
 
 export function isCalculationView(view: UnsavedViewV2) {
+  return view.type === ViewV2Type.CALCULATION
+}
+
+export function hasCalculationFields(view: UnsavedViewV2) {
   return Object.values(view.schema || {}).some(isCalculationField)
 }
 
@@ -28,6 +33,13 @@ export function calculationFields(view: UnsavedViewV2) {
   return pickBy(view.schema || {}, isCalculationField)
 }
 
-export function basicFields(view: UnsavedViewV2) {
-  return pickBy(view.schema || {}, field => !isCalculationField(field))
+export function isVisible(field: ViewFieldMetadata) {
+  return field.visible !== false
+}
+
+export function basicFields(view: UnsavedViewV2, opts?: { visible?: boolean }) {
+  const { visible = true } = opts || {}
+  return pickBy(view.schema || {}, field => {
+    return !isCalculationField(field) && (!visible || isVisible(field))
+  })
 }

packages/shared-core/src/table.ts

@@ -69,8 +69,8 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
   [FieldType.ATTACHMENT_SINGLE]: false,
   [FieldType.SIGNATURE_SINGLE]: false,
   [FieldType.LINK]: false,
-  [FieldType.BB_REFERENCE]: false,
-  [FieldType.BB_REFERENCE_SINGLE]: false,
+  [FieldType.BB_REFERENCE]: true,
+  [FieldType.BB_REFERENCE_SINGLE]: true,
 }
 
 export function canBeDisplayColumn(type: FieldType): boolean {

packages/shared-core/src/utils.ts

@@ -5,6 +5,7 @@ import {
   SearchFilters,
   BasicOperator,
   ArrayOperator,
+  isLogicalSearchOperator,
 } from "@budibase/types"
 import * as Constants from "./constants"
 import { removeKeyNumbering } from "./filters"
@@ -97,10 +98,20 @@ export function isSupportedUserSearch(query: SearchFilters) {
     { op: BasicOperator.EQUAL, key: "_id" },
     { op: ArrayOperator.ONE_OF, key: "_id" },
   ]
-  for (let [key, operation] of Object.entries(query)) {
+  for (const [key, operation] of Object.entries(query)) {
     if (typeof operation !== "object") {
       return false
     }
+
+    if (isLogicalSearchOperator(key)) {
+      for (const condition of query[key]!.conditions) {
+        if (!isSupportedUserSearch(condition)) {
+          return false
+        }
+      }
+      return true
+    }
+
     const fields = Object.keys(operation || {})
     // this filter doesn't contain options - ignore
     if (fields.length === 0) {
@@ -161,7 +172,7 @@ export const processSearchFilters = (
         .sort((a, b) => {
           return a.localeCompare(b)
         })
-        .filter(key => key in filter)
+        .filter(key => filter[key])
 
       if (filterPropertyKeys.length == 1) {
         const key = filterPropertyKeys[0],

packages/string-templates/src/helpers/javascript.ts

@@ -1,13 +1,14 @@
-import { atob, isJSAllowed } from "../utilities"
-import cloneDeep from "lodash/fp/cloneDeep"
+import { atob, isBackendService, isJSAllowed } from "../utilities"
 import { LITERAL_MARKER } from "../helpers/constants"
 import { getJsHelperList } from "./list"
 import { iifeWrapper } from "../iife"
 import { JsTimeoutError, UserScriptError } from "../errors"
+import { cloneDeep } from "lodash/fp"
 
 // The method of executing JS scripts depends on the bundle being built.
 // This setter is used in the entrypoint (either index.js or index.mjs).
-let runJS: ((js: string, context: any) => any) | undefined = undefined
+let runJS: ((js: string, context: Record<string, any>) => any) | undefined =
+  undefined
 export const setJSRunner = (runner: typeof runJS) => (runJS = runner)
 
 export const removeJSRunner = () => {
@@ -31,9 +32,19 @@ const removeSquareBrackets = (value: string) => {
   return value
 }
 
+const isReservedKey = (key: string) =>
+  key === "snippets" ||
+  key === "helpers" ||
+  key.startsWith("snippets.") ||
+  key.startsWith("helpers.")
+
 // Our context getter function provided to JS code as $.
 // Extracts a value from context.
 const getContextValue = (path: string, context: any) => {
+  // We populate `snippets` ourselves, don't allow access to it.
+  if (isReservedKey(path)) {
+    return undefined
+  }
   const literalStringRegex = /^(["'`]).*\1$/
   let data = context
   // check if it's a literal string - just return path if its quoted
@@ -46,6 +57,7 @@ const getContextValue = (path: string, context: any) => {
     }
     data = data[removeSquareBrackets(key)]
   })
+
   return data
 }
 
@@ -67,10 +79,23 @@ export function processJS(handlebars: string, context: any) {
       snippetMap[snippet.name] = snippet.code
     }
 
-    // Our $ context function gets a value from context.
-    // We clone the context to avoid mutation in the binding affecting real
-    // app context.
-    const clonedContext = cloneDeep({ ...context, snippets: null })
+    let clonedContext: Record<string, any>
+    if (isBackendService()) {
+      // On the backned, values are copied across the isolated-vm boundary and
+      // so we don't need to do any cloning here. This does create a fundamental
+      // difference in how JS executes on the frontend vs the backend, e.g.
+      // consider this snippet:
+      //
+      //   $("array").push(2)
+      //   return $("array")[1]
+      //
+      // With the context of `{ array: [1] }`, the backend will return
+      // `undefined` whereas the frontend will return `2`. We should fix this.
+      clonedContext = context
+    } else {
+      clonedContext = cloneDeep(context)
+    }
+
     const sandboxContext = {
       $: (path: string) => getContextValue(path, clonedContext),
       helpers: getJsHelperList(),

packages/string-templates/src/index.ts

@@ -1,4 +1,4 @@
-import { Context, createContext, runInNewContext } from "vm"
+import { createContext, runInNewContext } from "vm"
 import { create, TemplateDelegate } from "handlebars"
 import { registerAll, registerMinimum } from "./helpers/index"
 import { postprocess, preprocess } from "./processors"
@@ -455,18 +455,11 @@ export function convertToJS(hbs: string) {
 
 export { JsTimeoutError, UserScriptError } from "./errors"
 
-export function defaultJSSetup() {
-  if (!isBackendService()) {
+export function browserJSSetup() {
   /**
    * Use polyfilled vm to run JS scripts in a browser Env
    */
-    setJSRunner((js: string, context: Context) => {
-      context = {
-        ...context,
-        alert: undefined,
-        setInterval: undefined,
-        setTimeout: undefined,
-      }
+  setJSRunner((js: string, context: Record<string, any>) => {
     createContext(context)
 
     const wrappedJs = `
@@ -490,6 +483,11 @@ export function defaultJSSetup() {
     }
     return result.result
   })
+}
+
+export function defaultJSSetup() {
+  if (!isBackendService()) {
+    browserJSSetup()
   } else {
     removeJSRunner()
   }

packages/string-templates/src/utilities.ts

@@ -4,7 +4,14 @@ export const FIND_HBS_REGEX = /{{([^{].*?)}}/g
 export const FIND_ANY_HBS_REGEX = /{?{{([^{].*?)}}}?/g
 export const FIND_TRIPLE_HBS_REGEX = /{{{([^{].*?)}}}/g
 
+const isJest = () => typeof jest !== "undefined"
+
 export const isBackendService = () => {
+  // We consider the tests for string-templates to be frontend, so that they
+  // test the frontend JS functionality.
+  if (isJest()) {
+    return false
+  }
   return typeof window === "undefined"
 }
 

packages/string-templates/test/javascript.spec.ts

@@ -1,7 +1,13 @@
-import vm from "vm"
-
-import { processStringSync, encodeJSBinding, setJSRunner } from "../src/index"
+import {
+  processStringSync,
+  encodeJSBinding,
+  defaultJSSetup,
+} from "../src/index"
 import { UUID_REGEX } from "./constants"
+import tk from "timekeeper"
+
+const DATE = "2021-01-21T12:00:00"
+tk.freeze(DATE)
 
 const processJS = (js: string, context?: object): any => {
   return processStringSync(encodeJSBinding(js), context)
@@ -9,9 +15,7 @@ const processJS = (js: string, context?: object): any => {
 
 describe("Javascript", () => {
   beforeAll(() => {
-    setJSRunner((js, context) => {
-      return vm.runInNewContext(js, context, { timeout: 1000 })
-    })
+    defaultJSSetup()
   })
 
   describe("Test the JavaScript helper", () => {
@@ -118,8 +122,7 @@ describe("Javascript", () => {
     })
 
     it("should handle errors", () => {
-      const output = processJS(`throw "Error"`)
-      expect(output).toBe("Error while executing JS")
+      expect(processJS(`throw "Error"`)).toEqual("Error")
     })
 
     it("should timeout after one second", () => {
@@ -127,16 +130,18 @@ describe("Javascript", () => {
       expect(output).toBe("Timed out while executing JS")
     })
 
-    it("should prevent access to the process global", () => {
-      const output = processJS(`return process`)
-      expect(output).toBe("Error while executing JS")
+    it("should prevent access to the process global", async () => {
+      expect(processJS(`return process`)).toEqual(
+        "ReferenceError: process is not defined"
+      )
     })
   })
 
   describe("check JS helpers", () => {
     it("should error if using the format helper. not helpers.", () => {
-      const output = processJS(`return helper.toInt(4.3)`)
-      expect(output).toBe("Error while executing JS")
+      expect(processJS(`return helper.toInt(4.3)`)).toEqual(
+        "ReferenceError: helper is not defined"
+      )
     })
 
     it("should be able to use toInt", () => {
@@ -156,4 +161,323 @@ describe("Javascript", () => {
       expect(output).toBe("Custom")
     })
   })
+
+  describe("mutability", () => {
+    it("should not allow the context to be mutated", async () => {
+      const context = { array: [1] }
+      const result = await processJS(
+        `
+        const array = $("array");
+        array.push(2);
+        return array[1]
+      `,
+        context
+      )
+      expect(result).toEqual(2)
+      expect(context.array).toEqual([1])
+    })
+  })
+
+  describe("malice", () => {
+    it("should not be able to call JS functions", () => {
+      expect(processJS(`return alert("hello")`)).toEqual(
+        "ReferenceError: alert is not defined"
+      )
+
+      expect(processJS(`return prompt("hello")`)).toEqual(
+        "ReferenceError: prompt is not defined"
+      )
+
+      expect(processJS(`return confirm("hello")`)).toEqual(
+        "ReferenceError: confirm is not defined"
+      )
+
+      expect(processJS(`return setTimeout(() => {}, 1000)`)).toEqual(
+        "ReferenceError: setTimeout is not defined"
+      )
+
+      expect(processJS(`return setInterval(() => {}, 1000)`)).toEqual(
+        "ReferenceError: setInterval is not defined"
+      )
+    })
+  })
+
+  // the test cases here were extracted from templates/real world examples of JS in Budibase
+  describe("real test cases from Budicloud", () => {
+    const context = {
+      "Unit Value": 2,
+      Quantity: 1,
+    }
+    it("handle test case 1", async () => {
+      const result = await processJS(
+        `
+        var Gross = $("[Unit Value]") * $("[Quantity]")
+        return Gross.toFixed(2)`,
+        context
+      )
+      expect(result).toBeDefined()
+      expect(result).toBe("2.00")
+    })
+
+    it("handle test case 2", async () => {
+      const todayDate = new Date()
+      // add a year and a month
+      todayDate.setMonth(new Date().getMonth() + 1)
+      todayDate.setFullYear(todayDate.getFullYear() + 1)
+      const context = {
+        "Purchase Date": DATE,
+        today: todayDate.toISOString(),
+      }
+      const result = await processJS(
+        `
+        var purchase = new Date($("[Purchase Date]"));
+        let purchaseyear = purchase.getFullYear();
+        let purchasemonth = purchase.getMonth();
+
+        var today = new Date($("today"));
+        let todayyear = today.getFullYear();
+        let todaymonth = today.getMonth();
+
+        var age = todayyear - purchaseyear
+
+        if (((todaymonth - purchasemonth) < 6) == true){
+          return age
+        }
+        `,
+        context
+      )
+      expect(result).toBeDefined()
+      expect(result).toBe(1)
+    })
+
+    it("should handle test case 3", async () => {
+      const context = {
+        Escalate: true,
+        "Budget ($)": 1100,
+      }
+      const result = await processJS(
+        `
+        if ($("[Escalate]") == true) {
+          if ($("Budget ($)") <= 1000)
+            {return 2;}
+          if ($("Budget ($)") > 1000) 
+            {return 3;}
+          }
+          else {
+            if ($("Budget ($)") <= 1000)
+              {return 1;}
+            if ($("Budget ($)") > 1000)
+              if ($("Budget ($)") < 10000) 
+                {return 2;}
+              else 
+                {return 3}
+          }
+        `,
+        context
+      )
+      expect(result).toBeDefined()
+      expect(result).toBe(3)
+    })
+
+    it("should handle test case 4", async () => {
+      const context = {
+        "Time Sheets": ["a", "b"],
+      }
+      const result = await processJS(
+        `
+      let hours = 0
+      if (($("[Time Sheets]") != null) == true){
+        for (i = 0; i < $("[Time Sheets]").length; i++){
+          let hoursLogged = "Time Sheets." + i + ".Hours"
+          hours += $(hoursLogged)
+        }
+        return hours
+      }
+      if (($("[Time Sheets]") != null) == false){
+        return hours
+      }
+      `,
+        context
+      )
+      expect(result).toBeDefined()
+      expect(result).toBe("0ab")
+    })
+
+    it("should handle test case 5", async () => {
+      const context = {
+        change: JSON.stringify({ a: 1, primaryDisplay: "a" }),
+        previous: JSON.stringify({ a: 2, primaryDisplay: "b" }),
+      }
+      const result = await processJS(
+        `
+      let change = $("[change]") ? JSON.parse($("[change]")) : {}
+      let previous = $("[previous]") ? JSON.parse($("[previous]")) : {}
+
+      function simplifyLink(originalKey, value, parent) {
+        if (Array.isArray(value)) {
+          if (value.filter(item => Object.keys(item || {}).includes("primaryDisplay")).length > 0) {
+            parent[originalKey] = value.map(link => link.primaryDisplay)
+          }
+        }
+      }
+
+      for (let entry of Object.entries(change)) {
+        simplifyLink(entry[0], entry[1], change)
+      }
+      for (let entry of Object.entries(previous)) {
+        simplifyLink(entry[0], entry[1], previous)
+      }
+      
+      let diff = Object.fromEntries(Object.entries(change).filter(([k, v]) => previous[k]?.toString() !== v?.toString()))
+      
+      delete diff.audit_change
+      delete diff.audit_previous
+      delete diff._id
+      delete diff._rev
+      delete diff.tableId
+      delete diff.audit
+      
+      for (let entry of Object.entries(diff)) {
+        simplifyLink(entry[0], entry[1], diff)
+      }
+      
+      return JSON.stringify(change)?.replaceAll(",\\"", ",\\n\\t\\"").replaceAll("{\\"", "{\\n\\t\\"").replaceAll("}", "\\n}")
+      `,
+        context
+      )
+      expect(result).toBe(`{\n\t"a":1,\n\t"primaryDisplay":"a"\n}`)
+    })
+
+    it("should handle test case 6", async () => {
+      const context = {
+        "Join Date": DATE,
+      }
+      const result = await processJS(
+        `
+        var rate = 5;
+        var today = new Date();
+        
+        // comment
+        function monthDiff(dateFrom, dateTo) {
+         return dateTo.getMonth() - dateFrom.getMonth() + 
+           (12 * (dateTo.getFullYear() - dateFrom.getFullYear()))
+        }
+        var serviceMonths = monthDiff( new Date($("[Join Date]")), today);
+        var serviceYears = serviceMonths / 12;
+        
+        if (serviceYears >= 1 && serviceYears < 5){
+          rate = 10;
+        }
+        if (serviceYears >= 5 && serviceYears < 10){
+          rate = 15;
+        }
+        if (serviceYears >= 10){
+          rate = 15;
+          rate += 0.5 * (Number(serviceYears.toFixed(0)) - 10);
+        }
+        return rate;
+        `,
+        context
+      )
+      expect(result).toBe(10)
+    })
+
+    it("should handle test case 7", async () => {
+      const context = {
+        "P I": "Pass",
+        "PA I": "Pass",
+        "F I": "Fail",
+        "V I": "Pass",
+      }
+      const result = await processJS(
+        `if (($("[P I]") == "Pass") == true)
+              if (($("[  P I]") == "Pass") == true)
+                if (($("[F I]") == "Pass") == true)
+                  if (($("[V I]") == "Pass") == true)
+                    {return "Pass"}
+            
+            if (($("[PA I]") == "Fail") == true)
+              {return "Fail"}
+            if (($("[  P I]") == "Fail") == true)
+              {return "Fail"}
+            if (($("[F I]") == "Fail") == true)
+              {return "Fail"}
+            if (($("[V I]") == "Fail") == true)
+              {return "Fail"}
+            
+            else
+              {return ""}`,
+        context
+      )
+      expect(result).toBe("Fail")
+    })
+
+    it("should handle test case 8", async () => {
+      const context = {
+        "T L": [{ Hours: 10 }],
+        "B H": 50,
+      }
+      const result = await processJS(
+        `var totalHours = 0;
+            if (($("[T L]") != null) == true){
+            for (let i = 0; i < ($("[T L]").length); i++){
+              var individualHours = "T L." + i + ".Hours";
+              var hoursNum = Number($(individualHours));
+              totalHours += hoursNum;
+            }
+            return totalHours.toFixed(2);
+            }
+            if (($("[T L]") != null) == false) {
+              return totalHours.toFixed(2);
+            }
+        `,
+        context
+      )
+      expect(result).toBe("10.00")
+    })
+
+    it("should handle test case 9", async () => {
+      const context = {
+        "T L": [{ Hours: 10 }],
+        "B H": 50,
+      }
+      const result = await processJS(
+        `var totalHours = 0;
+            if (($("[T L]") != null) == true){
+            for (let i = 0; i < ($("[T L]").length); i++){
+              var individualHours = "T L." + i + ".Hours";
+              var hoursNum = Number($(individualHours));
+              totalHours += hoursNum;
+            }
+            return ($("[B H]") - totalHours).toFixed(2);
+            }
+            if (($("[T L]") != null) == false) {
+              return ($("[B H]") - totalHours).toFixed(2);
+            }`,
+        context
+      )
+      expect(result).toBe("40.00")
+    })
+
+    it("should handle test case 10", async () => {
+      const context = {
+        "F F": [{ "F S": 10 }],
+      }
+      const result = await processJS(
+        `var rating = 0;
+          
+          if ($("[F F]") != null){
+          for (i = 0; i < $("[F F]").length; i++){
+            var individualRating = $("F F." + i + ".F S");
+            rating += individualRating;
+            }
+            rating = (rating / $("[F F]").length);
+          }
+          return rating;
+          `,
+        context
+      )
+      expect(result).toBe(10)
+    })
+  })
 })

packages/string-templates/test/manifest.spec.ts

@@ -1,5 +1,3 @@
-import vm from "vm"
-
 jest.mock("@budibase/handlebars-helpers/lib/math", () => {
   const actual = jest.requireActual("@budibase/handlebars-helpers/lib/math")
 
@@ -17,7 +15,7 @@ jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
   }
 })
 
-import { processString, setJSRunner } from "../src/index"
+import { defaultJSSetup, processString } from "../src/index"
 
 import tk from "timekeeper"
 import { getParsedManifest, runJsHelpersTests } from "./utils"
@@ -32,9 +30,7 @@ describe("manifest", () => {
   const manifest = getParsedManifest()
 
   beforeAll(() => {
-    setJSRunner((js, context) => {
-      return vm.runInNewContext(js, context, { timeout: 1000 })
-    })
+    defaultJSSetup()
   })
 
   describe("examples are valid", () => {