Merge remote-tracking branch 'origin/develop' into test/9339-sqlpostgres-row-api-test-suite

.github/workflows/release-develop.yml

@@ -194,5 +194,5 @@ jobs:
           PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }}
         with:
           repository: budibase/budibase-deploys
-          event: deploy-develop-to-qa
+          event: deploy-budibase-develop-to-qa
           github_pat: ${{ secrets.GH_ACCESS_TOKEN }}

.nvmrc

@@ -0,0 +1 @@
+v14.19.3

.python-version

@@ -0,0 +1 @@
+3.11.1

.tool-versions

@@ -0,0 +1,2 @@
+nodejs 14.19.3
+python 3.11.1

.vscode/extensions.json

@@ -0,0 +1,6 @@
+{
+    "recommendations": [
+        "esbenp.prettier-vscode",
+        "svelte.svelte-vscode"
+    ]
+}

.vscode/settings.json

@@ -1,22 +1,28 @@
 {
-    "editor.formatOnSave": true,
+  "editor.formatOnSave": true,
-    "editor.codeActionsOnSave": {
+  "editor.codeActionsOnSave": {
-        "source.fixAll": true
+    "source.fixAll": true
-    },
+  },
-    "editor.defaultFormatter": "svelte.svelte-vscode",
+  "editor.defaultFormatter": "esbenp.prettier-vscode",
-    "[json]": {
+  "[json]": {
-        "editor.defaultFormatter": "vscode.json-language-features"
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
-    },
+  },
-    "[javascript]": {
+  "[javascript]": {
-        "editor.defaultFormatter": "vscode.typescript-language-features"
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
-    },
+  },
-    "debug.javascript.terminalOptions": {
+  "debug.javascript.terminalOptions": {
-        "skipFiles": [
+    "skipFiles": [
-            "${workspaceFolder}/packages/backend-core/node_modules/**",
+      "${workspaceFolder}/packages/backend-core/node_modules/**",
-            "<node_internals>/**"
+      "<node_internals>/**"
-        ]
+    ]
-    },
+  },
-    "[typescript]": {
+  "[typescript]": {
-        "editor.defaultFormatter": "vscode.typescript-language-features"
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
-    },
+  },
+  "[dockercompose]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[svelte]": {
+    "editor.defaultFormatter": "svelte.svelte-vscode"
+  }
 }

charts/budibase/templates/alb-ingress.yaml

@@ -14,6 +14,9 @@ metadata:
     alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
     alb.ingress.kubernetes.io/certificate-arn: {{ .Values.ingress.certificateArn }} 
     {{- end }}
+    {{- if .Values.ingress.securityGroups }}
+    alb.ingress.kubernetes.io/security-groups: {{ .Values.ingress.securityGroups }}
+    {{- end }}
 spec:
   rules:
     - http:

charts/budibase/values.yaml

@@ -76,7 +76,7 @@ affinity: {}
 globals:
   appVersion: "latest"
   budibaseEnv: PRODUCTION
-  tenantFeatureFlags: "*:LICENSING,*:USER_GROUPS"
+  tenantFeatureFlags: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"
   enableAnalytics: "1"
   sentryDSN: ""
   posthogToken: "phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU"

docs/CONTRIBUTING.md

@@ -9,7 +9,6 @@ From opening a bug report to creating a pull request: every contribution is appr
 - [Glossary of Terms](#glossary-of-terms)
 - [Contributing to Budibase](#contributing-to-budibase)
 
-
 ## Not Sure Where to Start?
 
 Budibase is a low-code web application builder that creates svelte-based web applications.
@@ -22,7 +21,7 @@ Budibase is a monorepo managed by [lerna](https://github.com/lerna/lerna). Lerna
 
 - **packages/server** - The budibase server. This [Koa](https://koajs.com/) app is responsible for serving the JS for the builder and budibase apps, as well as providing the API for interaction with the database and file system.
 
-- **packages/worker** - This [Koa](https://koajs.com/) app is responsible for providing global apis for managing your budibase installation. Authentication, Users, Email, Org and Auth configs are all provided by the worker. 
+- **packages/worker** - This [Koa](https://koajs.com/) app is responsible for providing global apis for managing your budibase installation. Authentication, Users, Email, Org and Auth configs are all provided by the worker.
 
 ## Contributor License Agreement (CLA)
 
@@ -45,7 +44,7 @@ A client represents a single budibase customer. Each budibase client will have 1
 
 ### App
 
-A client can have one or more budibase applications. Budibase applications would be things like "Developer Inventory Management" or "Goat Herder CRM". Think of a budibase application as a tree. 
+A client can have one or more budibase applications. Budibase applications would be things like "Developer Inventory Management" or "Goat Herder CRM". Think of a budibase application as a tree.
 
 ### Database
 
@@ -73,28 +72,55 @@ A component is the basic frontend building block of a budibase app.
 
 ### Component Library
 
-Component libraries are collections of components as well as the definition of their props contained in a file called `components.json`. 
+Component libraries are collections of components as well as the definition of their props contained in a file called `components.json`.
 
 ## Contributing to Budibase
 
-* Please maintain the existing code style. 
+- Please maintain the existing code style.
 
-* Please try to keep your commits small and focused.
+- Please try to keep your commits small and focused.
 
-* Please write tests.
+- Please write tests.
 
-* If the project diverges from your branch, please rebase instead of merging. This makes the commit graph easier to read.
+- If the project diverges from your branch, please rebase instead of merging. This makes the commit graph easier to read.
 
-* Once your work is completed, please raise a PR against the `develop` branch with some information about what has changed and why.
+- Once your work is completed, please raise a PR against the `develop` branch with some information about what has changed and why.
 
 ### Getting Started For Contributors
-#### 1.  Prerequisites
 
-NodeJS Version `14.x.x`
+#### 1. Prerequisites
 
-*yarn -* `npm install -g yarn`
+- NodeJS version `14.x.x`
+- Python version `3.x`
 
-*jest* - `npm install -g jest`
+### Using asdf (recommended)
+
+Asdf is a package manager that allows managing multiple dependencies.
+
+You can install them following any of the steps described below:
+
+- Install using script (only for mac users):
+
+`./scripts/install-contributor-dependencies.sh`
+
+- Or, manually:
+
+  - Installation steps: https://asdf-vm.com/guide/getting-started.html
+  - asdf plugin add nodejs
+  - asdf plugin add python
+  - npm install -g yarn
+
+### Using NVM and pyenv
+
+- NVM:
+  - Install: https://github.com/nvm-sh/nvm#installing-and-updating
+  - Setup: `nvm use`
+- Pyenv:
+
+  - Install: https://github.com/pyenv/pyenv#installation
+  - Setup: `pyenv install -v 3.7.2`
+
+- _yarn -_ `npm install -g yarn`
 
 #### 2. Clone this repository
 
@@ -102,7 +128,7 @@ NodeJS Version `14.x.x`
 
 then `cd ` into your local copy.
 
-#### 3.  Install and Build
+#### 3. Install and Build
 
 | **NOTE**: On Windows, all yarn commands must be executed on a bash shell (e.g. git bash)
 
@@ -134,9 +160,9 @@ This will enable watch mode for both the builder app, server, client library and
 
 #### 5. Debugging using VS Code
 
-To debug the budibase server and worker a VS Code launch configuration has been provided. 
+To debug the budibase server and worker a VS Code launch configuration has been provided.
 
-Visit the debug window and select `Budibase Server` or `Budibase Worker` to debug the respective component. 
+Visit the debug window and select `Budibase Server` or `Budibase Worker` to debug the respective component.
 Alternatively to start both components simultaneously select `Start Budibase`.
 
 In addition to the above, the remaining budibase components may be run in dev mode using: `yarn dev:noserver`.
@@ -156,11 +182,11 @@ For the backend we run [Redis](https://redis.io/), [CouchDB](https://couchdb.apa
 
 When you are running locally, budibase stores data on disk using docker volumes. The volumes and the types of data associated with each are:
 
-- `redis_data` 
+- `redis_data`
   - Sessions, email tokens
-- `couchdb3_data` 
+- `couchdb3_data`
   - Global and app databases
-- `minio_data` 
+- `minio_data`
   - App manifest, budibase client, static assets
 
 ### Development Modes
@@ -172,34 +198,42 @@ A combination of environment variables controls the mode budibase runs in.
 Yarn commands can be used to mimic the different modes as described in the sections below:
 
 #### Self Hosted
-The default mode. A single tenant installation with no usage restrictions. 
+
+The default mode. A single tenant installation with no usage restrictions.
 
 To enable this mode, use:
+
 ```
 yarn mode:self
 ```
 
 #### Cloud
-The cloud mode, with account portal turned off. 
+
+The cloud mode, with account portal turned off.
 
 To enable this mode, use:
+
 ```
 yarn mode:cloud
 ```
-#### Cloud & Account
-The cloud mode, with account portal turned on. This is a replica of the mode that runs at https://budibase.app 
 
+#### Cloud & Account
+
+The cloud mode, with account portal turned on. This is a replica of the mode that runs at https://budibase.app
 
 To enable this mode, use:
+
 ```
 yarn mode:account
 ```
+
 ### CI
- An overview of the CI pipelines can be found [here](../.github/workflows/README.md)
+
+An overview of the CI pipelines can be found [here](../.github/workflows/README.md)
 
 ### Pro
 
-@budibase/pro is the closed source package that supports licensed features in budibase. By default the package will be pulled from NPM and will not normally need to be touched in local development. If you require to update code inside the pro package it can be cloned to the same root level as budibase, e.g. 
+@budibase/pro is the closed source package that supports licensed features in budibase. By default the package will be pulled from NPM and will not normally need to be touched in local development. If you require to update code inside the pro package it can be cloned to the same root level as budibase, e.g.
 
 ```
 .
@@ -207,13 +241,14 @@ yarn mode:account
 |_ budibase-pro
 ```
 
-Note that only budibase maintainers will be able to access the pro repo. 
+Note that only budibase maintainers will be able to access the pro repo.
 
-The `yarn bootstrap` command can be used to replace the NPM supplied dependency with the local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev. 
+The `yarn bootstrap` command can be used to replace the NPM supplied dependency with the local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev.
 
 ### Troubleshooting
 
 Sometimes, things go wrong. This can be due to incompatible updates on the budibase platform. To clear down your development environment and start again follow **Step 6. Cleanup**, then proceed from **Step 3. Install and Build** in the setup guide above to create a fresh Budibase installation.
+
 ### Running tests
 
 #### End-to-end Tests
@@ -226,12 +261,11 @@ yarn test:e2e
 
 Or if you are in the builder you can run `yarn cy:test`.
 
-
 ### Other Useful Information
 
-* The contributors are listed in [AUTHORS.md](https://github.com/Budibase/budibase/blob/master/.github/AUTHORS.md) (add yourself).
+- The contributors are listed in [AUTHORS.md](https://github.com/Budibase/budibase/blob/master/.github/AUTHORS.md) (add yourself).
 
-* This project uses a modified version of the MPLv2 license, see [LICENSE](https://github.com/budibase/server/blob/master/LICENSE).
+- This project uses a modified version of the MPLv2 license, see [LICENSE](https://github.com/budibase/server/blob/master/LICENSE).
 
-* We use the [C4 (Collective Code Construction Contract)](https://rfc.zeromq.org/spec:42/C4/) process for contributions.
+- We use the [C4 (Collective Code Construction Contract)](https://rfc.zeromq.org/spec:42/C4/) process for contributions.
   Please read this if you are unfamiliar with it.

hosting/dependencies/runner.sh

@@ -4,5 +4,5 @@ redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
 /bbcouch-runner.sh &
 /minio/minio server ${DATA_DIR}/minio --console-address ":9001" > /dev/stdout 2>&1 &
 
-echo "Test environment started..."
+echo "Budibase dependencies started..."
 sleep infinity

hosting/docker-compose.dev.yaml

@@ -53,20 +53,6 @@ services:
     volumes:
       - couchdb_data:/data
 
-  couch-init:
-    container_name: budi-couchdb-init-dev
-    image: curlimages/curl
-    environment:
-      PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984"
-    depends_on:
-      - couchdb-service
-    command:
-      [
-        "sh",
-        "-c",
-        "sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;",
-      ]
-
   redis-service:
     container_name: budi-redis-dev
     restart: on-failure

hosting/single/runner.sh

@@ -10,7 +10,7 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME
 [[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://localhost:9000
 [[ -z "${NODE_ENV}" ]] && export NODE_ENV=production
 [[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
-[[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS"
+[[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"
 [[ -z "${ACCOUNT_PORTAL_URL}" ]] && export ACCOUNT_PORTAL_URL=https://account.budibase.app
 [[ -z "${REDIS_URL}" ]] && export REDIS_URL=localhost:6379
 [[ -z "${SELF_HOSTED}" ]] && export SELF_HOSTED=1

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.2.12-alpha.59",
+  "version": "2.2.27-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/jest.config.ts

@@ -1,7 +1,7 @@
-import { Config } from "jest"
+import { Config } from "@jest/types"
 const preset = require("ts-jest/jest-preset")
 
-const testContainersSettings = {
+const baseConfig: Config.InitialProjectOptions = {
   ...preset,
   preset: "@trendyol/jest-testcontainers",
   setupFiles: ["./tests/jestEnv.ts"],
@@ -13,23 +13,23 @@ const testContainersSettings = {
 
 if (!process.env.CI) {
   // use sources when not in CI
-  testContainersSettings.moduleNameMapper = {
+  baseConfig.moduleNameMapper = {
     "@budibase/types": "<rootDir>/../types/src",
   }
 } else {
   console.log("Running tests with compiled dependency sources")
 }
 
-const config: Config = {
+const config: Config.InitialOptions = {
   projects: [
     {
-      ...testContainersSettings,
+      ...baseConfig,
       displayName: "sequential test",
       testMatch: ["<rootDir>/**/*.seq.spec.[jt]s"],
       runner: "jest-serial-runner",
     },
     {
-      ...testContainersSettings,
+      ...baseConfig,
       testMatch: ["<rootDir>/**/!(*.seq).spec.[jt]s"],
     },
   ],

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.2.12-alpha.59",
+  "version": "2.2.27-alpha.0",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -23,7 +23,7 @@
   },
   "dependencies": {
     "@budibase/nano": "10.1.1",
-    "@budibase/types": "2.2.12-alpha.59",
+    "@budibase/types": "2.2.27-alpha.0",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",

packages/backend-core/src/cache/tests/writethrough.spec.js

@@ -1,17 +1,17 @@
-const{generator}=require("../../../tests")
+require("../../../tests")
 const { Writethrough } = require("../writethrough")
 const { getDB } = require("../../db")
 const tk = require("timekeeper")
+const { structures } = require("../../../tests")
 
 const START_DATE = Date.now()
 tk.freeze(START_DATE)
 
-const { newid } = require("../../newid")
 
 const DELAY = 5000
 
-const db = getDB(`db_${newid()}`)
+const db = getDB(structures.db.id())
-const db2 = getDB(`db_${newid()}`)
+const db2 = getDB(structures.db.id())
 const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY)
 
 describe("writethrough", () => {

packages/backend-core/src/db/tests/index.spec.js

@@ -1,19 +1,19 @@
 require("../../../tests")
-const { newid } = require("../../newid")
+const { structures } = require("../../../tests")
 const { getDB } = require("../db")
 
 describe("db", () => {
   describe("getDB", () => {
     it("returns a db", async () => {
       
-      const dbName = `db_${newid()}`
+      const dbName = structures.db.id()
       const db = getDB(dbName)
       expect(db).toBeDefined()
       expect(db.name).toBe(dbName)
     })
 
     it("uses the custom put function", async () => {
-      const db = getDB(`db_${newid()}`)
+      const db = getDB(structures.db.id())
       let doc = { _id: "test" }
       await db.put(doc)
       doc = await db.get(doc._id)

packages/backend-core/src/featureFlags/index.ts

@@ -6,7 +6,7 @@ import * as tenancy from "../tenancy"
  * The env var is formatted as:
  *  tenant1:feature1:feature2,tenant2:feature1
  */
-function getFeatureFlags() {
+export function buildFeatureFlags() {
   if (!env.TENANT_FEATURE_FLAGS) {
     return
   }
@@ -27,8 +27,6 @@ function getFeatureFlags() {
   return tenantFeatureFlags
 }
 
-const TENANT_FEATURE_FLAGS = getFeatureFlags()
-
 export function isEnabled(featureFlag: string) {
   const tenantId = tenancy.getTenantId()
   const flags = getTenantFeatureFlags(tenantId)
@@ -36,18 +34,36 @@ export function isEnabled(featureFlag: string) {
 }
 
 export function getTenantFeatureFlags(tenantId: string) {
-  const flags = []
+  let flags: string[] = []
+  const envFlags = buildFeatureFlags()
+  if (envFlags) {
+    const globalFlags = envFlags["*"]
+    const tenantFlags = envFlags[tenantId] || []
 
-  if (TENANT_FEATURE_FLAGS) {
+    // Explicitly exclude tenants from global features if required.
-    const globalFlags = TENANT_FEATURE_FLAGS["*"]
+    // Prefix the tenant flag with '!'
-    const tenantFlags = TENANT_FEATURE_FLAGS[tenantId]
+    const tenantOverrides = tenantFlags.reduce(
+      (acc: string[], flag: string) => {
+        if (flag.startsWith("!")) {
+          let stripped = flag.substring(1)
+          acc.push(stripped)
+        }
+        return acc
+      },
+      []
+    )
 
     if (globalFlags) {
       flags.push(...globalFlags)
     }
-    if (tenantFlags) {
+    if (tenantFlags.length) {
       flags.push(...tenantFlags)
     }
+
+    // Purge any tenant specific overrides
+    flags = flags.filter(flag => {
+      return tenantOverrides.indexOf(flag) == -1 && !flag.startsWith("!")
+    })
   }
 
   return flags
@@ -57,4 +73,5 @@ export enum TenantFeatureFlag {
   LICENSING = "LICENSING",
   GOOGLE_SHEETS = "GOOGLE_SHEETS",
   USER_GROUPS = "USER_GROUPS",
+  ONBOARDING_TOUR = "ONBOARDING_TOUR",
 }

packages/backend-core/src/featureFlags/tests/featureFlags.spec.ts

@@ -0,0 +1,85 @@
+import {
+  TenantFeatureFlag,
+  buildFeatureFlags,
+  getTenantFeatureFlags,
+} from "../"
+import env from "../../environment"
+
+const { ONBOARDING_TOUR, LICENSING, USER_GROUPS } = TenantFeatureFlag
+
+describe("featureFlags", () => {
+  beforeEach(() => {
+    env._set("TENANT_FEATURE_FLAGS", "")
+  })
+
+  it("Should return no flags when the TENANT_FEATURE_FLAG is empty", async () => {
+    let features = buildFeatureFlags()
+    expect(features).toBeUndefined()
+  })
+
+  it("Should generate a map of global and named tenant feature flags from the env value", async () => {
+    env._set(
+      "TENANT_FEATURE_FLAGS",
+      `*:${ONBOARDING_TOUR},tenant1:!${ONBOARDING_TOUR},tenant2:${USER_GROUPS},tenant1:${LICENSING}`
+    )
+
+    const parsedFlags: Record<string, string[]> = {
+      "*": [ONBOARDING_TOUR],
+      tenant1: [`!${ONBOARDING_TOUR}`, LICENSING],
+      tenant2: [USER_GROUPS],
+    }
+
+    let features = buildFeatureFlags()
+
+    expect(features).toBeDefined()
+    expect(features).toEqual(parsedFlags)
+  })
+
+  it("Should add feature flag flag only to explicitly configured tenant", async () => {
+    env._set(
+      "TENANT_FEATURE_FLAGS",
+      `*:${LICENSING},*:${USER_GROUPS},tenant1:${ONBOARDING_TOUR}`
+    )
+
+    let tenant1Flags = getTenantFeatureFlags("tenant1")
+    let tenant2Flags = getTenantFeatureFlags("tenant2")
+
+    expect(tenant1Flags).toBeDefined()
+    expect(tenant1Flags).toEqual([LICENSING, USER_GROUPS, ONBOARDING_TOUR])
+
+    expect(tenant2Flags).toBeDefined()
+    expect(tenant2Flags).toEqual([LICENSING, USER_GROUPS])
+  })
+})
+
+it("Should exclude tenant1 from global feature flag", async () => {
+  env._set(
+    "TENANT_FEATURE_FLAGS",
+    `*:${LICENSING},*:${ONBOARDING_TOUR},tenant1:!${ONBOARDING_TOUR}`
+  )
+
+  let tenant1Flags = getTenantFeatureFlags("tenant1")
+  let tenant2Flags = getTenantFeatureFlags("tenant2")
+
+  expect(tenant1Flags).toBeDefined()
+  expect(tenant1Flags).toEqual([LICENSING])
+
+  expect(tenant2Flags).toBeDefined()
+  expect(tenant2Flags).toEqual([LICENSING, ONBOARDING_TOUR])
+})
+
+it("Should explicitly add flags to configured tenants only", async () => {
+  env._set(
+    "TENANT_FEATURE_FLAGS",
+    `tenant1:${ONBOARDING_TOUR},tenant1:${LICENSING},tenant2:${LICENSING}`
+  )
+
+  let tenant1Flags = getTenantFeatureFlags("tenant1")
+  let tenant2Flags = getTenantFeatureFlags("tenant2")
+
+  expect(tenant1Flags).toBeDefined()
+  expect(tenant1Flags).toEqual([ONBOARDING_TOUR, LICENSING])
+
+  expect(tenant2Flags).toBeDefined()
+  expect(tenant2Flags).toEqual([LICENSING])
+})

packages/backend-core/src/middleware/logging.ts

@@ -64,7 +64,9 @@ const print = (fn: any, data: any[]) => {
     message = message + ` [identityId=${identityId}]`
   }
 
-  // fn(message, data)
+  if (!process.env.CI) {
+    fn(message, data)
+  }
 }
 
 const logging = (ctx: any, next: any) => {

packages/backend-core/src/middleware/passport/tests/third-party-common.seq.spec.js

@@ -6,8 +6,6 @@ const { DEFAULT_TENANT_ID } = require("../../../constants")
 const { generateGlobalUserID } = require("../../../db/utils")
 const { newid } = require("../../../utils")
 const { doWithGlobalDB, doInTenant } = require("../../../tenancy")
-const { default: environment } = require("../../../environment")
-environment._set("MULTI_TENANCY", 'TRUE')
 
 const done = jest.fn()
 

packages/backend-core/src/migrations/tests/index.spec.js

@@ -2,9 +2,8 @@ require("../../../tests")
 const { runMigrations, getMigrationsDoc } = require("../index")
 const { getGlobalDBName, getDB } = require("../../db")
 
-const { default: environment } = require("../../environment")
+const { structures, testEnv } = require("../../../tests")
-const { newid } = require("../../newid")
+testEnv.multiTenant()
-environment._set("MULTI_TENANCY", 'TRUE')
 
 let db
 
@@ -21,7 +20,7 @@ describe("migrations", () => {
   let tenantId
 
   beforeEach(() => {
-    tenantId = `tenant_${newid()}`
+    tenantId = structures.tenant.id()
     db = getDB(getGlobalDBName(tenantId))
   })
 

packages/backend-core/src/utils/tests/utils.spec.ts

@@ -4,17 +4,12 @@ import * as events from "../../events"
 import * as db from "../../db"
 import { Header } from "../../constants"
 import { doInTenant } from "../../context"
-import environment from "../../environment"
 import { newid } from "../../utils"
 
 describe("utils", () => {
   describe("platformLogout", () => {
-    beforeEach(() => {
-      environment._set("MULTI_TENANCY", "TRUE")
-    })
-
     it("should call platform logout", async () => {
-      await doInTenant(`tenant-${newid()}`, async () => {
+      await doInTenant(structures.tenant.id(), async () => {
         const ctx = structures.koa.newContext()
         await utils.platformLogout({ ctx, userId: "test" })
         expect(events.auth.logout).toBeCalledTimes(1)
@@ -23,10 +18,6 @@ describe("utils", () => {
   })
 
   describe("getAppIdFromCtx", () => {
-    beforeEach(() => {
-      environment._set("MULTI_TENANCY", undefined)
-    })
-
     it("gets appId from header", async () => {
       const ctx = structures.koa.newContext()
       const expected = db.generateAppID()

packages/backend-core/tests/utilities/structures/db.ts

@@ -0,0 +1,5 @@
+import { newid } from "../../../src/newid"
+
+export function id() {
+  return `db_${newid()}`
+}

packages/backend-core/tests/utilities/structures/index.ts

@@ -8,3 +8,5 @@ export * as apps from "./apps"
 export * as koa from "./koa"
 export * as licenses from "./licenses"
 export * as plugins from "./plugins"
+export * as tenant from "./tenants"
+export * as db from "./db"

packages/backend-core/tests/utilities/structures/tenants.ts

@@ -0,0 +1,5 @@
+import { newid } from "../../../src/newid"
+
+export function id() {
+  return `tenant-${newid()}`
+}

packages/backend-core/yarn.lock

@@ -487,11 +487,6 @@
     qs "^6.11.0"
     tough-cookie "^4.1.2"
 
-"@budibase/types@2.2.12-alpha.59":
-  version "2.2.12-alpha.59"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.59.tgz#41635c1e405acfa6162b5ca0f79f0c73f16bc764"
-  integrity sha512-cEcM0nnTEOEan9UYVspwcdgYgIbtY2zQTe1uDdwys+NFplMrbiwGyQbsafOx2IA0jCxmyqqYGmUAC0eF1napKQ==
-
 "@cspotcode/source-map-support@^0.8.0":
   version "0.8.1"
   resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1"
@@ -3126,9 +3121,9 @@ http-assert@^1.3.0:
     http-errors "~1.8.0"
 
 http-cache-semantics@^4.0.0:
-  version "4.1.0"
+  version "4.1.1"
-  resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390"
+  resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a"
-  integrity sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==
+  integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==
 
 http-cookie-agent@^4.0.2:
   version "4.0.2"

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.2.12-alpha.59",
+  "version": "2.2.27-alpha.0",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/string-templates": "2.2.12-alpha.59",
+    "@budibase/string-templates": "2.2.27-alpha.0",
     "@spectrum-css/accordion": "3.0.24",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",

packages/bbui/src/ActionButton/ActionButton.svelte

@@ -86,7 +86,7 @@
   }
   .is-selected:not(.spectrum-ActionButton--emphasized):not(.spectrum-ActionButton--quiet) {
     background: var(--spectrum-global-color-gray-300);
-    border-color: var(--spectrum-global-color-gray-700);
+    border-color: var(--spectrum-global-color-gray-500);
   }
   .noPadding {
     padding: 0;

packages/bbui/src/Actions/position_dropdown.js

@@ -1,11 +1,21 @@
-export default function positionDropdown(
+export default function positionDropdown(element, opts) {
-  element,
+  let resizeObserver
-  { anchor, align, maxWidth, useAnchorWidth, offset = 5 }
+  let latestOpts = opts
-) {
+
-  const update = () => {
+  // We need a static reference to this function so that we can properly
+  // clean up the scroll listener.
+  const scrollUpdate = () => {
+    updatePosition(latestOpts)
+  }
+
+  // Updates the position of the dropdown
+  const updatePosition = opts => {
+    const { anchor, align, maxWidth, useAnchorWidth, offset = 5 } = opts
     if (!anchor) {
       return
     }
+
+    // Compute bounds
     const anchorBounds = anchor.getBoundingClientRect()
     const elementBounds = element.getBoundingClientRect()
     let styles = {
@@ -51,26 +61,47 @@ export default function positionDropdown(
     })
   }
 
+  // The actual svelte action callback which creates observers on the relevant
+  // DOM elements
+  const update = newOpts => {
+    latestOpts = newOpts
+
+    // Cleanup old state
+    if (resizeObserver) {
+      resizeObserver.disconnect()
+    }
+
+    // Do nothing if no anchor
+    const { anchor } = newOpts
+    if (!anchor) {
+      return
+    }
+
+    // Observe both anchor and element and resize the popover as appropriate
+    resizeObserver = new ResizeObserver(() => updatePosition(newOpts))
+    resizeObserver.observe(anchor)
+    resizeObserver.observe(element)
+    resizeObserver.observe(document.body)
+  }
+
   // Apply initial styles which don't need to change
   element.style.position = "absolute"
   element.style.zIndex = "9999"
 
-  // Observe both anchor and element and resize the popover as appropriate
+  // Set up a scroll listener
-  const resizeObserver = new ResizeObserver(entries => {
+  document.addEventListener("scroll", scrollUpdate, true)
-    entries.forEach(update)
-  })
-  if (anchor) {
-    resizeObserver.observe(anchor)
-  }
-  resizeObserver.observe(element)
-  resizeObserver.observe(document.body)
 
-  document.addEventListener("scroll", update, true)
+  // Perform initial update
+  update(opts)
 
   return {
+    update,
     destroy() {
-      resizeObserver.disconnect()
+      // Cleanup
-      document.removeEventListener("scroll", update, true)
+      if (resizeObserver) {
+        resizeObserver.disconnect()
+      }
+      document.removeEventListener("scroll", scrollUpdate, true)
     },
   }
 }

packages/bbui/src/Form/Core/DatePicker.svelte

@@ -76,13 +76,6 @@
     }
     // If time only set date component to 2000-01-01
     if (timeOnly) {
-      // Classic flackpickr causing issues.
-      // When selecting a value for the first time for a "time only" field,
-      // the time is always offset by 1 hour for some reason (regardless of time
-      // zone) so we need to correct it.
-      if (!value && newValue) {
-        newValue = new Date(dates[0].getTime() + 60 * 60 * 1000).toISOString()
-      }
       newValue = `2000-01-01T${newValue.split("T")[1]}`
     }
 
@@ -113,7 +106,7 @@
 
   const clearDateOnBackspace = event => {
     if (["Backspace", "Clear", "Delete"].includes(event.key)) {
-      dispatch("change", null)
+      dispatch("change", "")
       flatpickr.close()
     }
   }

packages/bbui/src/Form/Core/RadioGroup.svelte

@@ -11,14 +11,31 @@
   export let getOptionLabel = option => option
   export let getOptionValue = option => option
   export let getOptionTitle = option => option
+  export let sort = false
 
   const dispatch = createEventDispatcher()
   const onChange = e => dispatch("change", e.target.value)
+
+  const getSortedOptions = (options, getLabel, sort) => {
+    if (!options?.length || !Array.isArray(options)) {
+      return []
+    }
+    if (!sort) {
+      return options
+    }
+    return [...options].sort((a, b) => {
+      const labelA = getLabel(a)
+      const labelB = getLabel(b)
+      return labelA > labelB ? 1 : -1
+    })
+  }
+
+  $: parsedOptions = getSortedOptions(options, getOptionLabel, sort)
 </script>
 
 <div class={`spectrum-FieldGroup spectrum-FieldGroup--${direction}`}>
-  {#if options && Array.isArray(options)}
+  {#if parsedOptions && Array.isArray(parsedOptions)}
-    {#each options as option}
+    {#each parsedOptions as option}
       <div
         title={getOptionTitle(option)}
         class="spectrum-Radio spectrum-FieldGroup-item spectrum-Radio--emphasized"

packages/bbui/src/Popover/Popover.svelte

@@ -57,30 +57,28 @@
 </script>
 
 {#if open}
-  {#key anchor}
+  <Portal {target}>
-    <Portal {target}>
+    <div
-      <div
+      tabindex="0"
-        tabindex="0"
+      use:positionDropdown={{
-        use:positionDropdown={{
+        anchor,
-          anchor,
+        align,
-          align,
+        maxWidth,
-          maxWidth,
+        useAnchorWidth,
-          useAnchorWidth,
+        offset,
-          offset,
+      }}
-        }}
+      use:clickOutside={{
-        use:clickOutside={{
+        callback: dismissible ? handleOutsideClick : () => {},
-          callback: dismissible ? handleOutsideClick : () => {},
+        anchor,
-          anchor,
+      }}
-        }}
+      on:keydown={handleEscape}
-        on:keydown={handleEscape}
+      class="spectrum-Popover is-open"
-        class="spectrum-Popover is-open"
+      role="presentation"
-        role="presentation"
+      transition:fly|local={{ y: -20, duration: 200 }}
-        transition:fly|local={{ y: -20, duration: 200 }}
+    >
-      >
+      <slot />
-        <slot />
+    </div>
-      </div>
+  </Portal>
-    </Portal>
-  {/key}
 {/if}
 
 <style>

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.2.12-alpha.59",
+  "version": "2.2.27-alpha.0",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -58,10 +58,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.2.12-alpha.59",
+    "@budibase/bbui": "2.2.27-alpha.0",
-    "@budibase/client": "2.2.12-alpha.59",
+    "@budibase/client": "2.2.27-alpha.0",
-    "@budibase/frontend-core": "2.2.12-alpha.59",
+    "@budibase/frontend-core": "2.2.27-alpha.0",
-    "@budibase/string-templates": "2.2.12-alpha.59",
+    "@budibase/string-templates": "2.2.27-alpha.0",
     "@fortawesome/fontawesome-svg-core": "^6.2.1",
     "@fortawesome/free-brands-svg-icons": "^6.2.1",
     "@fortawesome/free-solid-svg-icons": "^6.2.1",

packages/builder/src/builderStore/dataBinding.js

@@ -509,21 +509,24 @@ const getSelectedRowsBindings = asset => {
   return bindings
 }
 
+export const makeStateBinding = key => {
+  return {
+    type: "context",
+    runtimeBinding: `${makePropSafe("state")}.${makePropSafe(key)}`,
+    readableBinding: `State.${key}`,
+    category: "State",
+    icon: "AutomatedSegment",
+    display: { name: key },
+  }
+}
+
 /**
  * Gets all state bindings that are globally available.
  */
 const getStateBindings = () => {
   let bindings = []
   if (get(store).clientFeatures?.state) {
-    const safeState = makePropSafe("state")
+    bindings = getAllStateVariables().map(makeStateBinding)
-    bindings = getAllStateVariables().map(key => ({
-      type: "context",
-      runtimeBinding: `${safeState}.${makePropSafe(key)}`,
-      readableBinding: `State.${key}`,
-      category: "State",
-      icon: "AutomatedSegment",
-      display: { name: key },
-    }))
   }
   return bindings
 }

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -74,8 +74,19 @@
   $: schemaFields = Object.values(schema || {})
   $: queryLimit = tableId?.includes("datasource") ? "∞" : "1000"
   $: isTrigger = block?.type === "TRIGGER"
+  $: isUpdateRow = stepId === ActionStepID.UPDATE_ROW
 
   const onChange = Utils.sequential(async (e, key) => {
+    if (e.detail?.tableId) {
+      const tableSchema = getSchemaForTable(e.detail.tableId, {
+        searchableSchema: true,
+      }).schema
+      if (isTestModal) {
+        testData.schema = tableSchema
+      } else {
+        block.inputs.schema = tableSchema
+      }
+    }
     try {
       if (isTestModal) {
         // Special case for webhook, as it requires a body, but the schema already brings back the body's contents
@@ -321,9 +332,17 @@
         <RowSelector
           {block}
           value={inputData[key]}
-          on:change={e => onChange(e, key)}
+          meta={inputData["meta"] || {}}
+          on:change={e => {
+            if (e.detail?.key) {
+              onChange(e, e.detail.key)
+            } else {
+              onChange(e, key)
+            }
+          }}
           {bindings}
           {isTestModal}
+          {isUpdateRow}
         />
       {:else if value.customType === "webhookUrl"}
         <WebhookDisplay

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -1,6 +1,6 @@
 <script>
   import { tables } from "stores/backend"
-  import { Select } from "@budibase/bbui"
+  import { Select, Checkbox } from "@budibase/bbui"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
@@ -10,9 +10,11 @@
   const dispatch = createEventDispatcher()
 
   export let value
+  export let meta
   export let bindings
   export let block
   export let isTestModal
+  export let isUpdateRow
 
   $: parsedBindings = bindings.map(binding => {
     let clone = Object.assign({}, binding)
@@ -97,6 +99,17 @@
     dispatch("change", value)
   }
 
+  const onChangeSetting = (e, field) => {
+    let fields = {}
+    fields[field] = {
+      clearRelationships: e.detail,
+    }
+    dispatch("change", {
+      key: "meta",
+      fields,
+    })
+  }
+
   // Ensure any nullish tableId values get set to empty string so
   // that the select works
   $: if (value?.tableId == null) value = { tableId: "" }
@@ -124,21 +137,33 @@
               {onChange}
             />
           {:else}
-            <svelte:component
+            <div>
-              this={isTestModal ? ModalBindableInput : DrawerBindableInput}
+              <svelte:component
-              placeholder={placeholders[schema.type]}
+                this={isTestModal ? ModalBindableInput : DrawerBindableInput}
-              panel={AutomationBindingPanel}
+                placeholder={placeholders[schema.type]}
-              value={Array.isArray(value[field])
+                panel={AutomationBindingPanel}
-                ? value[field].join(" ")
+                value={Array.isArray(value[field])
-                : value[field]}
+                  ? value[field].join(" ")
-              on:change={e => onChange(e, field, schema.type)}
+                  : value[field]}
-              label={field}
+                on:change={e => onChange(e, field, schema.type)}
-              type="string"
+                label={field}
-              bindings={parsedBindings}
+                type="string"
-              fillWidth={true}
+                bindings={parsedBindings}
-              allowJS={true}
+                fillWidth={true}
-              updateOnChange={false}
+                allowJS={true}
-            />
+                updateOnChange={false}
+              />
+              {#if isUpdateRow && schema.type === "link"}
+                <div class="checkbox-field">
+                  <Checkbox
+                    value={meta.fields?.[field]?.clearRelationships}
+                    text={"Clear relationships if empty?"}
+                    size={"S"}
+                    on:change={e => onChangeSetting(e, field)}
+                  />
+                </div>
+              {/if}
+            </div>
           {/if}
         {/if}
       {/if}
@@ -155,4 +180,12 @@
   .schema-fields :global(label) {
     text-transform: capitalize;
   }
+  .checkbox-field {
+    padding-bottom: var(--spacing-s);
+    padding-left: 1px;
+    padding-top: var(--spacing-s);
+  }
+  .checkbox-field :global(label) {
+    text-transform: none;
+  }
 </style>

packages/builder/src/components/automation/SetupPanel/SchemaSetup.svelte

@@ -58,7 +58,7 @@
       entries = entries.filter(f => f.name !== originalName)
     }
     value = entries.reduce((newVals, current) => {
-      newVals[current.name] = current.type
+      newVals[current.name.trim()] = current.type
       return newVals
     }, {})
     dispatch("change", value)

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -12,7 +12,7 @@
     Modal,
     notifications,
   } from "@budibase/bbui"
-  import { createEventDispatcher, onMount } from "svelte"
+  import { createEventDispatcher } from "svelte"
   import { cloneDeep } from "lodash/fp"
   import { tables, datasources } from "stores/backend"
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
@@ -48,7 +48,22 @@
   const { hide } = getContext(Context.Modal)
   let fieldDefinitions = cloneDeep(FIELDS)
 
-  export let field = {
+  export let field
+
+  let originalName
+  let linkEditDisabled
+  let primaryDisplay
+  let indexes = [...($tables.selected.indexes || [])]
+  let isCreating
+
+  let table = $tables.selected
+  let confirmDeleteDialog
+  let deletion
+  let savingColumn
+  let deleteColName
+  let jsonSchemaModal
+
+  let editableColumn = {
     type: "string",
     constraints: fieldDefinitions.STRING.constraints,
 
@@ -56,48 +71,80 @@
     fieldName: $tables.selected.name,
   }
 
-  let originalName = field.name
+  $: if (primaryDisplay) {
-  const linkEditDisabled = originalName != null
+    editableColumn.constraints.presence = { allowEmpty: false }
-  let primaryDisplay =
+  }
-    $tables.selected.primaryDisplay == null ||
-    $tables.selected.primaryDisplay === field.name
-  let isCreating = originalName == null
 
-  let table = $tables.selected
+  $: if (field && !savingColumn) {
-  let indexes = [...($tables.selected.indexes || [])]
+    editableColumn = cloneDeep(field)
-  let confirmDeleteDialog
+    originalName = editableColumn.name ? editableColumn.name + "" : null
-  let deletion
+    linkEditDisabled = originalName != null
-  let deleteColName
+    isCreating = originalName == null
-  let jsonSchemaModal
+    primaryDisplay =
+      $tables.selected.primaryDisplay == null ||
+      $tables.selected.primaryDisplay === editableColumn.name
+  }
 
-  $: checkConstraints(field)
+  $: checkConstraints(editableColumn)
-  $: required = !!field?.constraints?.presence || primaryDisplay
+  $: required = !!editableColumn?.constraints?.presence || primaryDisplay
   $: uneditable =
     $tables.selected?._id === TableNames.USERS &&
-    UNEDITABLE_USER_FIELDS.includes(field.name)
+    UNEDITABLE_USER_FIELDS.includes(editableColumn.name)
   $: invalid =
-    !field.name ||
+    !editableColumn?.name ||
-    (field.type === LINK_TYPE && !field.tableId) ||
+    (editableColumn?.type === LINK_TYPE && !editableColumn?.tableId) ||
     Object.keys(errors).length !== 0
-  $: errors = checkErrors(field)
+  $: errors = checkErrors(editableColumn)
   $: datasource = $datasources.list.find(
     source => source._id === table?.sourceId
   )
 
+  const getTableAutoColumnTypes = table => {
+    return Object.keys(table?.schema).reduce((acc, key) => {
+      let fieldSchema = table?.schema[key]
+      if (fieldSchema.autocolumn) {
+        acc.push(fieldSchema.subtype)
+      }
+      return acc
+    }, [])
+  }
+
+  let autoColumnInfo = getAutoColumnInformation()
+
+  $: tableAutoColumnsTypes = getTableAutoColumnTypes($tables?.selected)
+  $: availableAutoColumns = Object.keys(autoColumnInfo).reduce((acc, key) => {
+    if (!tableAutoColumnsTypes.includes(key)) {
+      acc[key] = autoColumnInfo[key]
+    }
+    return acc
+  }, {})
+
+  $: availableAutoColumnKeys = availableAutoColumns
+    ? Object.keys(availableAutoColumns)
+    : []
+
+  $: autoColumnOptions = editableColumn.autocolumn
+    ? autoColumnInfo
+    : availableAutoColumns
+
   // used to select what different options can be displayed for column type
   $: canBeSearched =
-    field.type !== LINK_TYPE &&
+    editableColumn?.type !== LINK_TYPE &&
-    field.type !== JSON_TYPE &&
+    editableColumn?.type !== JSON_TYPE &&
-    field.subtype !== AUTO_COLUMN_SUB_TYPES.CREATED_BY &&
+    editableColumn?.subtype !== AUTO_COLUMN_SUB_TYPES.CREATED_BY &&
-    field.subtype !== AUTO_COLUMN_SUB_TYPES.UPDATED_BY &&
+    editableColumn?.subtype !== AUTO_COLUMN_SUB_TYPES.UPDATED_BY &&
-    field.type !== FORMULA_TYPE
+    editableColumn?.type !== FORMULA_TYPE
   $: canBeDisplay =
-    field.type !== LINK_TYPE &&
+    editableColumn?.type !== LINK_TYPE &&
-    field.type !== AUTO_TYPE &&
+    editableColumn?.type !== AUTO_TYPE &&
-    field.type !== JSON_TYPE
+    editableColumn?.type !== JSON_TYPE &&
+    !editableColumn.autocolumn
   $: canBeRequired =
-    field.type !== LINK_TYPE && !uneditable && field.type !== AUTO_TYPE
+    editableColumn?.type !== LINK_TYPE &&
-  $: relationshipOptions = getRelationshipOptions(field)
+    !uneditable &&
+    editableColumn?.type !== AUTO_TYPE &&
+    !editableColumn.autocolumn
+  $: relationshipOptions = getRelationshipOptions(editableColumn)
   $: external = table.type === "external"
   // in the case of internal tables the sourceId will just be undefined
   $: tableOptions = $tables.list.filter(
@@ -108,76 +155,90 @@
   )
   $: typeEnabled =
     !originalName ||
-    (originalName && SWITCHABLE_TYPES.indexOf(field.type) !== -1)
+    (originalName &&
+      SWITCHABLE_TYPES.indexOf(editableColumn.type) !== -1 &&
+      !editableColumn?.autocolumn)
 
   async function saveColumn() {
-    if (field.type === AUTO_TYPE) {
+    savingColumn = true
-      field = buildAutoColumn($tables.selected.name, field.name, field.subtype)
+    if (errors?.length) {
+      return
     }
-    if (field.type !== LINK_TYPE) {
+
-      delete field.fieldName
+    let saveColumn = cloneDeep(editableColumn)
+
+    if (saveColumn.type === AUTO_TYPE) {
+      saveColumn = buildAutoColumn(
+        $tables.draft.name,
+        saveColumn.name,
+        saveColumn.subtype
+      )
+    }
+    if (saveColumn.type !== LINK_TYPE) {
+      delete saveColumn.fieldName
     }
     try {
       await tables.saveField({
         originalName,
-        field,
+        field: saveColumn,
         primaryDisplay,
         indexes,
       })
       dispatch("updatecolumns")
     } catch (err) {
-      notifications.error("Error saving column")
+      console.log(err)
+      notifications.error(`Error saving column: ${err.message}`)
     }
   }
 
   function cancelEdit() {
-    field.name = originalName
+    editableColumn.name = originalName
   }
 
   function deleteColumn() {
     try {
-      field.name = deleteColName
+      editableColumn.name = deleteColName
-      if (field.name === $tables.selected.primaryDisplay) {
+      if (editableColumn.name === $tables.selected.primaryDisplay) {
         notifications.error("You cannot delete the display column")
       } else {
-        tables.deleteField(field)
+        tables.deleteField(editableColumn)
-        notifications.success(`Column ${field.name} deleted.`)
+        notifications.success(`Column ${editableColumn.name} deleted.`)
         confirmDeleteDialog.hide()
         hide()
         deletion = false
         dispatch("updatecolumns")
       }
     } catch (error) {
-      notifications.error("Error deleting column")
+      notifications.error(`Error deleting column: ${error.message}`)
     }
   }
 
   function handleTypeChange(event) {
     // remove any extra fields that may not be related to this type
-    delete field.autocolumn
+    delete editableColumn.autocolumn
-    delete field.subtype
+    delete editableColumn.subtype
-    delete field.tableId
+    delete editableColumn.tableId
-    delete field.relationshipType
+    delete editableColumn.relationshipType
-    delete field.formulaType
+    delete editableColumn.formulaType
 
     // Add in defaults and initial definition
     const definition = fieldDefinitions[event.detail?.toUpperCase()]
     if (definition?.constraints) {
-      field.constraints = definition.constraints
+      editableColumn.constraints = definition.constraints
     }
 
     // Default relationships many to many
-    if (field.type === LINK_TYPE) {
+    if (editableColumn.type === LINK_TYPE) {
-      field.relationshipType = RelationshipTypes.MANY_TO_MANY
+      editableColumn.relationshipType = RelationshipTypes.MANY_TO_MANY
     }
-    if (field.type === FORMULA_TYPE) {
+    if (editableColumn.type === FORMULA_TYPE) {
-      field.formulaType = "dynamic"
+      editableColumn.formulaType = "dynamic"
     }
   }
 
   function onChangeRequired(e) {
     const req = e.detail
-    field.constraints.presence = req ? { allowEmpty: false } : false
+    editableColumn.constraints.presence = req ? { allowEmpty: false } : false
     required = req
   }
 
@@ -185,17 +246,17 @@
     const isPrimary = e.detail
     // primary display is always required
     if (isPrimary) {
-      field.constraints.presence = { allowEmpty: false }
+      editableColumn.constraints.presence = { allowEmpty: false }
     }
   }
 
   function onChangePrimaryIndex(e) {
-    indexes = e.detail ? [field.name] : []
+    indexes = e.detail ? [editableColumn.name] : []
   }
 
   function onChangeSecondaryIndex(e) {
     if (e.detail) {
-      indexes[1] = field.name
+      indexes[1] = editableColumn.name
     } else {
       indexes = indexes.slice(0, 1)
     }
@@ -246,11 +307,14 @@
   }
 
   function getAllowedTypes() {
-    if (originalName && ALLOWABLE_STRING_TYPES.indexOf(field.type) !== -1) {
+    if (
+      originalName &&
+      ALLOWABLE_STRING_TYPES.indexOf(editableColumn.type) !== -1
+    ) {
       return ALLOWABLE_STRING_OPTIONS
     } else if (
       originalName &&
-      ALLOWABLE_NUMBER_TYPES.indexOf(field.type) !== -1
+      ALLOWABLE_NUMBER_TYPES.indexOf(editableColumn.type) !== -1
     ) {
       return ALLOWABLE_NUMBER_OPTIONS
     } else if (!external) {
@@ -275,6 +339,9 @@
   }
 
   function checkConstraints(fieldToCheck) {
+    if (!fieldToCheck) {
+      return
+    }
     // most types need this, just make sure its always present
     if (fieldToCheck && !fieldToCheck.constraints) {
       fieldToCheck.constraints = {}
@@ -296,10 +363,16 @@
   }
 
   function checkErrors(fieldInfo) {
+    if (!editableColumn) {
+      return {}
+    }
     function inUse(tbl, column, ogName = null) {
-      return Object.keys(tbl?.schema || {}).some(
+      const parsedColumn = column ? column.toLowerCase().trim() : column
-        key => key !== ogName && key === column
+
-      )
+      return Object.keys(tbl?.schema || {}).some(key => {
+        let lowerKey = key.toLowerCase()
+        return lowerKey !== ogName?.toLowerCase() && lowerKey === parsedColumn
+      })
     }
     const newError = {}
     if (!external && fieldInfo.name?.startsWith("_")) {
@@ -313,6 +386,11 @@
     } else if (inUse($tables.selected, fieldInfo.name, originalName)) {
       newError.name = `Column name already in use.`
     }
+
+    if (fieldInfo.type == "auto" && !fieldInfo.subtype) {
+      newError.subtype = `Auto Column requires a type`
+    }
+
     if (fieldInfo.fieldName && fieldInfo.tableId) {
       const relatedTable = $tables.list.find(
         tbl => tbl._id === fieldInfo.tableId
@@ -323,12 +401,6 @@
     }
     return newError
   }
-
-  onMount(() => {
-    if (primaryDisplay) {
-      field.constraints.presence = { allowEmpty: false }
-    }
-  })
 </script>
 
 <ModalContent
@@ -340,19 +412,26 @@
 >
   <Input
     label="Name"
-    bind:value={field.name}
+    bind:value={editableColumn.name}
-    disabled={uneditable || (linkEditDisabled && field.type === LINK_TYPE)}
+    disabled={uneditable ||
+      (linkEditDisabled && editableColumn.type === LINK_TYPE)}
     error={errors?.name}
   />
 
   <Select
     disabled={!typeEnabled}
     label="Type"
-    bind:value={field.type}
+    bind:value={editableColumn.type}
     on:change={handleTypeChange}
     options={getAllowedTypes()}
     getOptionLabel={field => field.name}
     getOptionValue={field => field.type}
+    isOptionEnabled={option => {
+      if (option.type == AUTO_TYPE) {
+        return availableAutoColumnKeys?.length > 0
+      }
+      return true
+    }}
   />
 
   {#if canBeRequired || canBeDisplay}
@@ -381,32 +460,32 @@
     <div>
       <Label>Search Indexes</Label>
       <Toggle
-        value={indexes[0] === field.name}
+        value={indexes[0] === editableColumn.name}
-        disabled={indexes[1] === field.name}
+        disabled={indexes[1] === editableColumn.name}
         on:change={onChangePrimaryIndex}
         text="Primary"
       />
       <Toggle
-        value={indexes[1] === field.name}
+        value={indexes[1] === editableColumn.name}
-        disabled={!indexes[0] || indexes[0] === field.name}
+        disabled={!indexes[0] || indexes[0] === editableColumn.name}
         on:change={onChangeSecondaryIndex}
         text="Secondary"
       />
     </div>
   {/if}
 
-  {#if field.type === "string"}
+  {#if editableColumn.type === "string"}
     <Input
       type="number"
       label="Max Length"
-      bind:value={field.constraints.length.maximum}
+      bind:value={editableColumn.constraints.length.maximum}
     />
-  {:else if field.type === "options"}
+  {:else if editableColumn.type === "options"}
     <ValuesList
       label="Options (one per line)"
-      bind:values={field.constraints.inclusion}
+      bind:values={editableColumn.constraints.inclusion}
     />
-  {:else if field.type === "longform"}
+  {:else if editableColumn.type === "longform"}
     <div>
       <Label
         size="M"
@@ -415,21 +494,24 @@
         Formatting
       </Label>
       <Toggle
-        bind:value={field.useRichText}
+        bind:value={editableColumn.useRichText}
         text="Enable rich text support (markdown)"
       />
     </div>
-  {:else if field.type === "array"}
+  {:else if editableColumn.type === "array"}
     <ValuesList
       label="Options (one per line)"
-      bind:values={field.constraints.inclusion}
+      bind:values={editableColumn.constraints.inclusion}
     />
-  {:else if field.type === "datetime"}
+  {:else if editableColumn.type === "datetime" && !editableColumn.autocolumn}
     <DatePicker
       label="Earliest"
-      bind:value={field.constraints.datetime.earliest}
+      bind:value={editableColumn.constraints.datetime.earliest}
+    />
+    <DatePicker
+      label="Latest"
+      bind:value={editableColumn.constraints.datetime.latest}
     />
-    <DatePicker label="Latest" bind:value={field.constraints.datetime.latest} />
     {#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER"}
       <div>
         <Label
@@ -439,25 +521,28 @@
         >
           Time zones
         </Label>
-        <Toggle bind:value={field.ignoreTimezones} text="Ignore time zones" />
+        <Toggle
+          bind:value={editableColumn.ignoreTimezones}
+          text="Ignore time zones"
+        />
       </div>
     {/if}
-  {:else if field.type === "number"}
+  {:else if editableColumn.type === "number" && !editableColumn.autocolumn}
     <Input
       type="number"
       label="Min Value"
-      bind:value={field.constraints.numericality.greaterThanOrEqualTo}
+      bind:value={editableColumn.constraints.numericality.greaterThanOrEqualTo}
     />
     <Input
       type="number"
       label="Max Value"
-      bind:value={field.constraints.numericality.lessThanOrEqualTo}
+      bind:value={editableColumn.constraints.numericality.lessThanOrEqualTo}
     />
-  {:else if field.type === "link"}
+  {:else if editableColumn.type === "link"}
     <Select
       label="Table"
       disabled={linkEditDisabled}
-      bind:value={field.tableId}
+      bind:value={editableColumn.tableId}
       options={tableOptions}
       getOptionLabel={table => table.name}
       getOptionValue={table => table._id}
@@ -466,7 +551,7 @@
       <RadioGroup
         disabled={linkEditDisabled}
         label="Define the relationship"
-        bind:value={field.relationshipType}
+        bind:value={editableColumn.relationshipType}
         options={relationshipOptions}
         getOptionLabel={option => option.name}
         getOptionValue={option => option.value}
@@ -476,14 +561,14 @@
     <Input
       disabled={linkEditDisabled}
       label={`Column name in other table`}
-      bind:value={field.fieldName}
+      bind:value={editableColumn.fieldName}
       error={errors.relatedName}
     />
-  {:else if field.type === FORMULA_TYPE}
+  {:else if editableColumn.type === FORMULA_TYPE}
     {#if !table.sql}
       <Select
         label="Formula type"
-        bind:value={field.formulaType}
+        bind:value={editableColumn.formulaType}
         options={[
           { label: "Dynamic", value: "dynamic" },
           { label: "Static", value: "static" },
@@ -497,25 +582,28 @@
     <ModalBindableInput
       title="Formula"
       label="Formula"
-      value={field.formula}
+      value={editableColumn.formula}
-      on:change={e => (field.formula = e.detail)}
+      on:change={e => (editableColumn.formula = e.detail)}
       bindings={getBindings({ table })}
       allowJS
     />
-  {:else if field.type === AUTO_TYPE}
+  {:else if editableColumn.type === JSON_TYPE}
-    <Select
-      label="Auto column type"
-      value={field.subtype}
-      on:change={e => (field.subtype = e.detail)}
-      options={Object.entries(getAutoColumnInformation())}
-      getOptionLabel={option => option[1].name}
-      getOptionValue={option => option[0]}
-    />
-  {:else if field.type === JSON_TYPE}
     <Button primary text on:click={openJsonSchemaEditor}
       >Open schema editor</Button
     >
   {/if}
+  {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
+    <Select
+      label="Auto column type"
+      value={editableColumn.subtype}
+      on:change={e => (editableColumn.subtype = e.detail)}
+      options={Object.entries(autoColumnOptions)}
+      getOptionLabel={option => option[1].name}
+      getOptionValue={option => option[0]}
+      disabled={!availableAutoColumnKeys?.length || editableColumn.autocolumn}
+      error={errors?.subtype}
+    />
+  {/if}
 
   <div slot="footer">
     {#if !uneditable && originalName != null}
@@ -525,11 +613,11 @@
 </ModalContent>
 <Modal bind:this={jsonSchemaModal}>
   <JSONSchemaModal
-    schema={field.schema}
+    schema={editableColumn.schema}
-    json={field.json}
+    json={editableColumn.json}
     on:save={({ detail }) => {
-      field.schema = detail.schema
+      editableColumn.schema = detail.schema
-      field.json = detail.json
+      editableColumn.json = detail.json
     }}
   />
 </Modal>

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte

@@ -177,7 +177,7 @@
           <EnvDropdown
             showModal={() => showModal(configKey)}
             variables={$environment.variables}
-            type={schema[configKey].type}
+            type={configKey === "port" ? "string" : schema[configKey].type}
             on:change
             bind:value={config[configKey]}
             error={$validation.errors[configKey]}

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte

@@ -82,7 +82,7 @@
 
     let displayString
     if (throughTableName) {
-      displayString = `${fromTableName} through ${throughTableName} → ${toTableName}`
+      displayString = `${fromTableName} ↔ ${toTableName}`
     } else {
       displayString = `${fromTableName} → ${toTableName}`
     }

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -10,17 +10,17 @@
   } from "@budibase/bbui"
   import { tables } from "stores/backend"
   import { Helpers } from "@budibase/bbui"
+  import { RelationshipErrorChecker } from "./relationshipErrors"
+  import { onMount } from "svelte"
 
   export let save
   export let datasource
   export let plusTables = []
   export let fromRelationship = {}
   export let toRelationship = {}
+  export let selectedFromTable
   export let close
 
-  const colNotSet = "Please specify a column name"
-  const relationshipAlreadyExists =
-    "A relationship between these tables already exists."
   const relationshipTypes = [
     {
       label: "One to Many",
@@ -42,63 +42,28 @@
   )
 
   let tableOptions
+  let errorChecker = new RelationshipErrorChecker(
+    invalidThroughTable,
+    relationshipExists
+  )
   let errors = {}
-  let hasClickedSave = !!fromRelationship.relationshipType
+  let fromPrimary, fromForeign, fromColumn, toColumn
-  let fromPrimary,
-    fromForeign,
-    fromTable,
-    toTable,
-    throughTable,
-    fromColumn,
-    toColumn
   let fromId, toId, throughId, throughToKey, throughFromKey
   let isManyToMany, isManyToOne, relationshipType
-
+  let hasValidated = false
-  $: {
-    if (!fromPrimary) {
-      fromPrimary = fromRelationship.foreignKey
-      fromForeign = toRelationship.foreignKey
-    }
-    if (!fromColumn && !errors.fromColumn) {
-      fromColumn = toRelationship.name
-    }
-    if (!toColumn && !errors.toColumn) {
-      toColumn = fromRelationship.name
-    }
-    if (!fromId) {
-      fromId = toRelationship.tableId
-    }
-    if (!toId) {
-      toId = fromRelationship.tableId
-    }
-    if (!throughId) {
-      throughId = fromRelationship.through
-      throughFromKey = fromRelationship.throughFrom
-      throughToKey = fromRelationship.throughTo
-    }
-    if (!relationshipType) {
-      relationshipType = fromRelationship.relationshipType
-    }
-  }
 
   $: tableOptions = plusTables.map(table => ({
     label: table.name,
     value: table._id,
   }))
-  $: valid = getErrorCount(errors) === 0 || !hasClickedSave
+  $: valid = getErrorCount(errors) === 0 && allRequiredAttributesSet()
-
   $: isManyToMany = relationshipType === RelationshipTypes.MANY_TO_MANY
   $: isManyToOne = relationshipType === RelationshipTypes.MANY_TO_ONE
-  $: fromTable = plusTables.find(table => table._id === fromId)
-  $: toTable = plusTables.find(table => table._id === toId)
-  $: throughTable = plusTables.find(table => table._id === throughId)
-
   $: toRelationship.relationshipType = fromRelationship?.relationshipType
 
-  const getErrorCount = errors =>
+  function getTable(id) {
-    Object.entries(errors)
+    return plusTables.find(table => table._id === id)
-      .filter(entry => !!entry[1])
+  }
-      .map(entry => entry[0]).length
 
   function invalidThroughTable() {
     // need to know the foreign key columns to check error
@@ -116,93 +81,103 @@
     }
     return false
   }
-
+  function relationshipExists() {
-  function validate() {
-    const isMany = relationshipType === RelationshipTypes.MANY_TO_MANY
-    const tableNotSet = "Please specify a table"
-    const foreignKeyNotSet = "Please pick a foreign key"
-    const errObj = {}
-    if (!relationshipType) {
-      errObj.relationshipType = "Please specify a relationship type"
-    }
-    if (!fromTable) {
-      errObj.fromTable = tableNotSet
-    }
-    if (!toTable) {
-      errObj.toTable = tableNotSet
-    }
-    if (isMany && !throughTable) {
-      errObj.throughTable = tableNotSet
-    }
-    if (isMany && !throughFromKey) {
-      errObj.throughFromKey = foreignKeyNotSet
-    }
-    if (isMany && !throughToKey) {
-      errObj.throughToKey = foreignKeyNotSet
-    }
-    if (invalidThroughTable()) {
-      errObj.throughTable =
-        "Ensure non-key columns are nullable or auto-generated"
-    }
-    if (!isMany && !fromForeign) {
-      errObj.fromForeign = foreignKeyNotSet
-    }
-    if (!fromColumn) {
-      errObj.fromColumn = colNotSet
-    }
-    if (!toColumn) {
-      errObj.toColumn = colNotSet
-    }
-    if (!isMany && !fromPrimary) {
-      errObj.fromPrimary = "Please pick the primary key"
-    }
-    if (isMany && relationshipExists()) {
-      errObj.fromTable = relationshipAlreadyExists
-      errObj.toTable = relationshipAlreadyExists
-    }
-
-    // currently don't support relationships back onto the table itself, needs to relate out
-    const tableError = "From/to/through tables must be different"
-    if (fromTable && (fromTable === toTable || fromTable === throughTable)) {
-      errObj.fromTable = tableError
-    }
-    if (toTable && (toTable === fromTable || toTable === throughTable)) {
-      errObj.toTable = tableError
-    }
     if (
-      throughTable &&
+      originalFromTable &&
-      (throughTable === fromTable || throughTable === toTable)
+      originalToTable &&
+      originalFromTable === getTable(fromId) &&
+      originalToTable === getTable(toId)
     ) {
-      errObj.throughTable = tableError
-    }
-    const colError = "Column name cannot be an existing column"
-    if (isColumnNameBeingUsed(toTable, fromColumn, originalFromColumnName)) {
-      errObj.fromColumn = colError
-    }
-    if (isColumnNameBeingUsed(fromTable, toColumn, originalToColumnName)) {
-      errObj.toColumn = colError
-    }
-
-    let fromType, toType
-    if (fromPrimary && fromForeign) {
-      fromType = fromTable?.schema[fromPrimary]?.type
-      toType = toTable?.schema[fromForeign]?.type
-    }
-    if (fromType && toType && fromType !== toType) {
-      errObj.fromForeign =
-        "Column type of the foreign key must match the primary key"
-    }
-
-    errors = errObj
-    return getErrorCount(errors) === 0
-  }
-
-  function isColumnNameBeingUsed(table, columnName, originalName) {
-    if (!table || !columnName || columnName === originalName) {
       return false
     }
-    const keys = Object.keys(table.schema).map(key => key.toLowerCase())
+    let fromThroughLinks = Object.values(
-    return keys.indexOf(columnName.toLowerCase()) !== -1
+      datasource.entities[getTable(fromId).name].schema
+    ).filter(value => value.through)
+    let toThroughLinks = Object.values(
+      datasource.entities[getTable(toId).name].schema
+    ).filter(value => value.through)
+
+    const matchAgainstUserInput = (fromTableId, toTableId) =>
+      (fromTableId === fromId && toTableId === toId) ||
+      (fromTableId === toId && toTableId === fromId)
+
+    return !!fromThroughLinks.find(from =>
+      toThroughLinks.find(
+        to =>
+          from.through === to.through &&
+          matchAgainstUserInput(from.tableId, to.tableId)
+      )
+    )
+  }
+
+  function getErrorCount(errors) {
+    return Object.entries(errors).filter(entry => !!entry[1]).length
+  }
+
+  function allRequiredAttributesSet() {
+    const base = getTable(fromId) && getTable(toId) && fromColumn && toColumn
+    if (relationshipType === RelationshipTypes.MANY_TO_ONE) {
+      return base && fromPrimary && fromForeign
+    } else {
+      return base && getTable(throughId) && throughFromKey && throughToKey
+    }
+  }
+
+  function validate() {
+    if (!allRequiredAttributesSet() && !hasValidated) {
+      return
+    }
+    hasValidated = true
+    errorChecker.setType(relationshipType)
+    const fromTable = getTable(fromId),
+      toTable = getTable(toId),
+      throughTable = getTable(throughId)
+    errors = {
+      relationshipType: errorChecker.relationshipTypeSet(relationshipType),
+      fromTable:
+        errorChecker.tableSet(fromTable) ||
+        errorChecker.doesRelationshipExists() ||
+        errorChecker.differentTables(fromId, toId, throughId),
+      toTable:
+        errorChecker.tableSet(toTable) ||
+        errorChecker.doesRelationshipExists() ||
+        errorChecker.differentTables(toId, fromId, throughId),
+      throughTable:
+        errorChecker.throughTableSet(throughTable) ||
+        errorChecker.throughIsNullable() ||
+        errorChecker.differentTables(throughId, fromId, toId),
+      throughFromKey:
+        errorChecker.manyForeignKeySet(throughFromKey) ||
+        errorChecker.manyTypeMismatch(
+          fromTable,
+          throughTable,
+          fromTable.primary[0],
+          throughFromKey
+        ),
+      throughToKey:
+        errorChecker.manyForeignKeySet(throughToKey) ||
+        errorChecker.manyTypeMismatch(
+          toTable,
+          throughTable,
+          toTable.primary[0],
+          throughToKey
+        ),
+      fromForeign:
+        errorChecker.foreignKeySet(fromForeign) ||
+        errorChecker.typeMismatch(fromTable, toTable, fromPrimary, fromForeign),
+      fromPrimary: errorChecker.primaryKeySet(fromPrimary),
+      fromColumn: errorChecker.columnBeingUsed(
+        toTable,
+        fromColumn,
+        originalFromColumnName
+      ),
+      toColumn: errorChecker.columnBeingUsed(
+        fromTable,
+        toColumn,
+        originalToColumnName
+      ),
+    }
+    return getErrorCount(errors) === 0
   }
 
   function buildRelationships() {
@@ -243,13 +218,13 @@
     if (manyToMany) {
       relateFrom = {
         ...relateFrom,
-        through: throughTable._id,
+        through: getTable(throughId)._id,
-        fieldName: toTable.primary[0],
+        fieldName: getTable(toId).primary[0],
       }
       relateTo = {
         ...relateTo,
-        through: throughTable._id,
+        through: getTable(throughId)._id,
-        fieldName: fromTable.primary[0],
+        fieldName: getTable(fromId).primary[0],
         throughFrom: relateFrom.throughTo,
         throughTo: relateFrom.throughFrom,
       }
@@ -277,35 +252,6 @@
     toRelationship = relateTo
   }
 
-  function relationshipExists() {
-    if (
-      originalFromTable &&
-      originalToTable &&
-      originalFromTable === fromTable &&
-      originalToTable === toTable
-    ) {
-      return false
-    }
-    let fromThroughLinks = Object.values(
-      datasource.entities[fromTable.name].schema
-    ).filter(value => value.through)
-    let toThroughLinks = Object.values(
-      datasource.entities[toTable.name].schema
-    ).filter(value => value.through)
-
-    const matchAgainstUserInput = (fromTableId, toTableId) =>
-      (fromTableId === fromId && toTableId === toId) ||
-      (fromTableId === toId && toTableId === fromId)
-
-    return !!fromThroughLinks.find(from =>
-      toThroughLinks.find(
-        to =>
-          from.through === to.through &&
-          matchAgainstUserInput(from.tableId, to.tableId)
-      )
-    )
-  }
-
   function removeExistingRelationship() {
     if (originalFromTable && originalFromColumnName) {
       delete datasource.entities[originalFromTable.name].schema[
@@ -320,7 +266,6 @@
   }
 
   async function saveRelationship() {
-    hasClickedSave = true
     if (!validate()) {
       return false
     }
@@ -328,10 +273,10 @@
     removeExistingRelationship()
 
     // source of relationship
-    datasource.entities[fromTable.name].schema[fromRelationship.name] =
+    datasource.entities[getTable(fromId).name].schema[fromRelationship.name] =
       fromRelationship
     // save other side of relationship in the other schema
-    datasource.entities[toTable.name].schema[toRelationship.name] =
+    datasource.entities[getTable(toId).name].schema[toRelationship.name] =
       toRelationship
 
     await save()
@@ -342,6 +287,36 @@
     await tables.fetch()
     close()
   }
+
+  function changed(fn) {
+    if (typeof fn === "function") {
+      fn()
+    }
+    validate()
+  }
+
+  onMount(() => {
+    if (fromRelationship) {
+      fromPrimary = fromRelationship.foreignKey
+      toId = fromRelationship.tableId
+      throughId = fromRelationship.through
+      throughFromKey = fromRelationship.throughFrom
+      throughToKey = fromRelationship.throughTo
+      toColumn = fromRelationship.name
+    }
+    if (toRelationship) {
+      fromForeign = toRelationship.foreignKey
+      fromId = toRelationship.tableId
+      fromColumn = toRelationship.name
+    }
+    relationshipType =
+      fromRelationship.relationshipType || RelationshipTypes.MANY_TO_ONE
+    if (selectedFromTable) {
+      fromId = selectedFromTable._id
+      fromColumn = selectedFromTable.name
+      fromPrimary = selectedFromTable?.primary[0] || null
+    }
+  })
 </script>
 
 <ModalContent
@@ -355,34 +330,35 @@
     options={relationshipTypes}
     bind:value={relationshipType}
     bind:error={errors.relationshipType}
-    on:change={() => (errors.relationshipType = null)}
+    on:change={() =>
+      changed(() => {
+        hasValidated = false
+      })}
   />
   <div class="headings">
     <Detail>Tables</Detail>
   </div>
-  <Select
+  {#if !selectedFromTable}
-    label="Select from table"
-    options={tableOptions}
-    bind:value={fromId}
-    bind:error={errors.fromTable}
-    on:change={e => {
-      fromColumn = tableOptions.find(opt => opt.value === e.detail)?.label || ""
-      if (errors.fromTable === relationshipAlreadyExists) {
-        errors.toColumn = null
-      }
-      errors.fromTable = null
-      errors.fromColumn = null
-      errors.toTable = null
-      errors.throughTable = null
-    }}
-  />
-  {#if isManyToOne && fromTable}
     <Select
-      label={`Primary Key (${fromTable.name})`}
+      label="Select from table"
-      options={Object.keys(fromTable.schema)}
+      options={tableOptions}
+      bind:value={fromId}
+      bind:error={errors.fromTable}
+      on:change={e =>
+        changed(() => {
+          const table = plusTables.find(tbl => tbl._id === e.detail)
+          fromColumn = table?.name || ""
+          fromPrimary = table?.primary?.[0]
+        })}
+    />
+  {/if}
+  {#if isManyToOne && fromId}
+    <Select
+      label={`Primary Key (${getTable(fromId).name})`}
+      options={Object.keys(getTable(fromId).schema)}
       bind:value={fromPrimary}
       bind:error={errors.fromPrimary}
-      on:change={() => (errors.fromPrimary = null)}
+      on:change={changed}
     />
   {/if}
   <Select
@@ -390,16 +366,12 @@
     options={tableOptions}
     bind:value={toId}
     bind:error={errors.toTable}
-    on:change={e => {
+    on:change={e =>
-      toColumn = tableOptions.find(opt => opt.value === e.detail)?.label || ""
+      changed(() => {
-      if (errors.toTable === relationshipAlreadyExists) {
+        const table = plusTables.find(tbl => tbl._id === e.detail)
-        errors.fromColumn = null
+        toColumn = table.name || ""
-      }
+        fromForeign = null
-      errors.toTable = null
+      })}
-      errors.toColumn = null
-      errors.fromTable = null
-      errors.throughTable = null
-    }}
   />
   {#if isManyToMany}
     <Select
@@ -407,45 +379,45 @@
       options={tableOptions}
       bind:value={throughId}
       bind:error={errors.throughTable}
-      on:change={() => {
+      on:change={() =>
-        errors.fromTable = null
+        changed(() => {
-        errors.toTable = null
+          throughToKey = null
-        errors.throughTable = null
+          throughFromKey = null
-      }}
+        })}
     />
-    {#if fromTable && toTable && throughTable}
+    {#if fromId && toId && throughId}
       <Select
-        label={`Foreign Key (${fromTable?.name})`}
+        label={`Foreign Key (${getTable(fromId)?.name})`}
-        options={Object.keys(throughTable?.schema)}
+        options={Object.keys(getTable(throughId)?.schema)}
         bind:value={throughToKey}
         bind:error={errors.throughToKey}
-        on:change={e => {
+        on:change={e =>
-          if (throughFromKey === e.detail) {
+          changed(() => {
-            throughFromKey = null
+            if (throughFromKey === e.detail) {
-          }
+              throughFromKey = null
-          errors.throughToKey = null
+            }
-        }}
+          })}
       />
       <Select
-        label={`Foreign Key (${toTable?.name})`}
+        label={`Foreign Key (${getTable(toId)?.name})`}
-        options={Object.keys(throughTable?.schema)}
+        options={Object.keys(getTable(throughId)?.schema)}
         bind:value={throughFromKey}
         bind:error={errors.throughFromKey}
-        on:change={e => {
+        on:change={e =>
-          if (throughToKey === e.detail) {
+          changed(() => {
-            throughToKey = null
+            if (throughToKey === e.detail) {
-          }
+              throughToKey = null
-          errors.throughFromKey = null
+            }
-        }}
+          })}
       />
     {/if}
-  {:else if isManyToOne && toTable}
+  {:else if isManyToOne && toId}
     <Select
-      label={`Foreign Key (${toTable?.name})`}
+      label={`Foreign Key (${getTable(toId)?.name})`}
-      options={Object.keys(toTable?.schema)}
+      options={Object.keys(getTable(toId)?.schema)}
       bind:value={fromForeign}
       bind:error={errors.fromForeign}
-      on:change={() => (errors.fromForeign = null)}
+      on:change={changed}
     />
   {/if}
   <div class="headings">
@@ -459,15 +431,13 @@
     label="From table column"
     bind:value={fromColumn}
     bind:error={errors.fromColumn}
-    on:change={e => {
+    on:change={changed}
-      errors.fromColumn = e.detail?.length > 0 ? null : colNotSet
-    }}
   />
   <Input
     label="To table column"
     bind:value={toColumn}
     bind:error={errors.toColumn}
-    on:change={e => (errors.toColumn = e.detail?.length > 0 ? null : colNotSet)}
+    on:change={changed}
   />
   <div slot="footer">
     {#if originalFromColumnName != null}

packages/builder/src/components/backend/Datasources/relationshipErrors.js

@@ -0,0 +1,103 @@
+import { RelationshipTypes } from "constants/backend"
+
+const typeMismatch = "Column type of the foreign key must match the primary key"
+const columnBeingUsed = "Column name cannot be an existing column"
+const mustBeDifferentTables = "From/to/through tables must be different"
+const primaryKeyNotSet = "Please pick the primary key"
+const throughNotNullable =
+  "Ensure non-key columns are nullable or auto-generated"
+const noRelationshipType = "Please specify a relationship type"
+const tableNotSet = "Please specify a table"
+const foreignKeyNotSet = "Please pick a foreign key"
+const relationshipAlreadyExists =
+  "A relationship between these tables already exists"
+
+function isColumnNameBeingUsed(table, columnName, originalName) {
+  if (!table || !columnName || columnName === originalName) {
+    return false
+  }
+  const keys = Object.keys(table.schema).map(key => key.toLowerCase())
+  return keys.indexOf(columnName.toLowerCase()) !== -1
+}
+
+function typeMismatchCheck(fromTable, toTable, primary, foreign) {
+  let fromType, toType
+  if (primary && foreign) {
+    fromType = fromTable?.schema[primary]?.type
+    toType = toTable?.schema[foreign]?.type
+  }
+  return fromType && toType && fromType !== toType ? typeMismatch : null
+}
+
+export class RelationshipErrorChecker {
+  constructor(invalidThroughTableFn, relationshipExistsFn) {
+    this.invalidThroughTable = invalidThroughTableFn
+    this.relationshipExists = relationshipExistsFn
+  }
+
+  setType(type) {
+    this.type = type
+  }
+
+  isMany() {
+    return this.type === RelationshipTypes.MANY_TO_MANY
+  }
+
+  relationshipTypeSet(type) {
+    return !type ? noRelationshipType : null
+  }
+
+  tableSet(table) {
+    return !table ? tableNotSet : null
+  }
+
+  throughTableSet(table) {
+    return this.isMany() && !table ? tableNotSet : null
+  }
+
+  manyForeignKeySet(key) {
+    return this.isMany() && !key ? foreignKeyNotSet : null
+  }
+
+  foreignKeySet(key) {
+    return !this.isMany() && !key ? foreignKeyNotSet : null
+  }
+
+  primaryKeySet(key) {
+    return !this.isMany() && !key ? primaryKeyNotSet : null
+  }
+
+  throughIsNullable() {
+    return this.invalidThroughTable() ? throughNotNullable : null
+  }
+
+  doesRelationshipExists() {
+    return this.isMany() && this.relationshipExists()
+      ? relationshipAlreadyExists
+      : null
+  }
+
+  differentTables(table1, table2, table3) {
+    // currently don't support relationships back onto the table itself, needs to relate out
+    const error = table1 && (table1 === table2 || (table3 && table1 === table3))
+    return error ? mustBeDifferentTables : null
+  }
+
+  columnBeingUsed(table, column, ogName) {
+    return isColumnNameBeingUsed(table, column, ogName) ? columnBeingUsed : null
+  }
+
+  typeMismatch(fromTable, toTable, primary, foreign) {
+    if (this.isMany()) {
+      return null
+    }
+    return typeMismatchCheck(fromTable, toTable, primary, foreign)
+  }
+
+  manyTypeMismatch(table, throughTable, primary, foreign) {
+    if (!this.isMany()) {
+      return null
+    }
+    return typeMismatchCheck(table, throughTable, primary, foreign)
+  }
+}

packages/builder/src/components/common/bindings/DrawerBindableCombobox.svelte

@@ -22,6 +22,7 @@
 
   const dispatch = createEventDispatcher()
   let bindingDrawer
+  let valid = true
 
   $: readableValue = runtimeToReadableBinding(bindings, value)
   $: tempValue = readableValue
@@ -76,12 +77,15 @@
   <svelte:fragment slot="description">
     Add the objects on the left to enrich your text.
   </svelte:fragment>
-  <Button cta slot="buttons" on:click={handleClose}>Save</Button>
+  <Button cta slot="buttons" on:click={handleClose} disabled={!valid}>
+    Save
+  </Button>
   <svelte:component
     this={panel}
     slot="body"
     value={readableValue}
     close={handleClose}
+    bind:valid
     on:change={event => (tempValue = event.detail)}
     {bindings}
     {allowJS}

packages/builder/src/components/common/bindings/ModalBindableInput.svelte

@@ -106,4 +106,8 @@
     border: var(--border-light);
     border-radius: 4px;
   }
+
+  .control :global(.spectrum-Textfield-input) {
+    padding-right: 40px;
+  }
 </style>

packages/builder/src/components/design/settings/componentSettings.js

@@ -56,7 +56,7 @@ const componentMap = {
   "field/link": FormFieldSelect,
   "field/array": FormFieldSelect,
   "field/json": FormFieldSelect,
-  "field/barcode/qr": FormFieldSelect,
+  "field/barcodeqr": FormFieldSelect,
   // Some validation types are the same as others, so not all types are
   // explicitly listed here. e.g. options uses string validation
   "validation/string": ValidationEditor,

packages/builder/src/components/design/settings/controls/ButtonActionEditor/ButtonActionDrawer.svelte

@@ -11,7 +11,10 @@
   } from "@budibase/bbui"
   import { getAvailableActions } from "./index"
   import { generate } from "shortid"
-  import { getEventContextBindings } from "builderStore/dataBinding"
+  import {
+    getEventContextBindings,
+    makeStateBinding,
+  } from "builderStore/dataBinding"
   import { currentAsset, store } from "builderStore"
 
   const flipDurationMs = 150
@@ -52,7 +55,7 @@
     actions,
     selectedAction?.id
   )
-  $: allBindings = eventContexBindings.concat(bindings)
+  $: allBindings = getAllBindings(bindings, eventContexBindings, actions)
   $: {
     // Ensure each action has a unique ID
     if (actions) {
@@ -74,8 +77,18 @@
   }
 
   const deleteAction = index => {
+    // Check if we're deleting the selected action
+    const selectedIndex = actions.indexOf(selectedAction)
+    const isSelected = index === selectedIndex
+
+    // Delete the action
     actions.splice(index, 1)
     actions = actions
+
+    // Select a new action if we deleted the selected one
+    if (isSelected) {
+      selectedAction = actions?.length ? actions[0] : null
+    }
   }
 
   const toggleActionList = () => {
@@ -111,6 +124,37 @@
   function handleDndFinalize(e) {
     actions = e.detail.items
   }
+
+  const getAllBindings = (bindings, eventContextBindings, actions) => {
+    let allBindings = eventContextBindings.concat(bindings)
+
+    if (!actions) {
+      return []
+    }
+
+    // Ensure bindings are generated for all "update state" action keys
+    actions
+      .filter(action => {
+        // Find all "Update State" actions which set values
+        return (
+          action[EVENT_TYPE_KEY] === "Update State" &&
+          action.parameters?.type === "set" &&
+          action.parameters.key
+        )
+      })
+      .forEach(action => {
+        // Check we have a binding for this action, and generate one if not
+        const stateBinding = makeStateBinding(action.parameters.key)
+        const hasKey = allBindings.some(binding => {
+          return binding.runtimeBinding === stateBinding.runtimeBinding
+        })
+        if (!hasKey) {
+          allBindings.push(stateBinding)
+        }
+      })
+
+    return allBindings
+  }
 </script>
 
 <DrawerContent>
@@ -186,7 +230,7 @@
         <div class="selected-action-container">
           <svelte:component
             this={selectedActionComponent}
-            parameters={selectedAction.parameters}
+            bind:parameters={selectedAction.parameters}
             bindings={allBindings}
             {nested}
           />

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/TriggerAutomation.svelte

@@ -36,7 +36,13 @@
   $: selectedSchema = selectedAutomation?.schema
 
   const onFieldsChanged = e => {
-    parameters.fields = e.detail
+    parameters.fields = Object.entries(e.detail || {}).reduce(
+      (acc, [key, value]) => {
+        acc[key.trim()] = value
+        return acc
+      },
+      {}
+    )
   }
 
   const setNew = () => {

packages/builder/src/components/design/settings/controls/FormFieldSelect.svelte

@@ -25,7 +25,7 @@
   const getOptions = (schema, type) => {
     let entries = Object.entries(schema ?? {})
     let types = []
-    if (type === "field/options" || type === "field/barcode/qr") {
+    if (type === "field/options") {
       // allow options to be used on both options and string fields
       types = [type, "field/string"]
     } else {
@@ -35,6 +35,7 @@
     types = types.map(type => type.slice(type.indexOf("/") + 1))
 
     entries = entries.filter(entry => types.includes(entry[1].type))
+
     return entries.map(entry => entry[0])
   }
 </script>

packages/builder/src/components/portal/onboarding/TourPopover.svelte

@@ -108,50 +108,52 @@
   }
 </script>
 
-{#key tourStepKey}
+{#if tourKey}
-  <Popover
+  {#key tourStepKey}
-    align={tourStep?.align}
+    <Popover
-    bind:this={popover}
+      align={tourStep?.align}
-    anchor={popoverAnchor}
+      bind:this={popover}
-    maxWidth={300}
+      anchor={popoverAnchor}
-    dismissible={false}
+      maxWidth={300}
-    offset={15}
+      dismissible={false}
-  >
+      offset={15}
-    <div class="tour-content">
+    >
-      <Layout noPadding gap="M">
+      <div class="tour-content">
-        <div class="tour-header">
+        <Layout noPadding gap="M">
-          <Heading size="XS">{tourStep?.title || "-"}</Heading>
+          <div class="tour-header">
-          <div>{`${tourStepIdx + 1}/${tourSteps?.length}`}</div>
+            <Heading size="XS">{tourStep?.title || "-"}</Heading>
-        </div>
+            <div>{`${tourStepIdx + 1}/${tourSteps?.length}`}</div>
-        <Body size="S">
-          <span class="tour-body">
-            {#if tourStep.layout}
-              <svelte:component this={tourStep.layout} />
-            {:else}
-              {tourStep?.body || ""}
-            {/if}
-          </span>
-        </Body>
-        <div class="tour-footer">
-          <div class="tour-navigation">
-            {#if tourStepIdx > 0}
-              <Button
-                secondary
-                on:click={previousStep}
-                disabled={tourStepIdx == 0}
-              >
-                <div>Back</div>
-              </Button>
-            {/if}
-            <Button cta on:click={nextStep}>
-              <div>{lastStep ? "Finish" : "Next"}</div>
-            </Button>
           </div>
-        </div>
+          <Body size="S">
-      </Layout>
+            <span class="tour-body">
-    </div>
+              {#if tourStep.layout}
-  </Popover>
+                <svelte:component this={tourStep.layout} />
-{/key}
+              {:else}
+                {tourStep?.body || ""}
+              {/if}
+            </span>
+          </Body>
+          <div class="tour-footer">
+            <div class="tour-navigation">
+              {#if tourStepIdx > 0}
+                <Button
+                  secondary
+                  on:click={previousStep}
+                  disabled={tourStepIdx == 0}
+                >
+                  <div>Back</div>
+                </Button>
+              {/if}
+              <Button cta on:click={nextStep}>
+                <div>{lastStep ? "Finish" : "Next"}</div>
+              </Button>
+            </div>
+          </div>
+        </Layout>
+      </div>
+    </Popover>
+  {/key}
+{/if}
 
 <style>
   .tour-content {

packages/builder/src/components/portal/onboarding/steps/OnboardingData.svelte

@@ -1,5 +1,5 @@
 <div>
-  In this section you can mange the data for your app:
+  In this section you can manage the data for your app:
   <ul class="feature-list">
     <li>Connect data sources</li>
     <li>Edit data</li>

packages/builder/src/components/start/CreateAppModal.svelte

@@ -138,7 +138,6 @@
       }
 
       $goto(`/builder/app/${createdApp.instance._id}`)
-      // apps.load()
     } catch (error) {
       creating = false
       console.error(error)

packages/builder/src/helpers/featureFlags.js

@@ -4,6 +4,7 @@ import { get } from "svelte/store"
 export const TENANT_FEATURE_FLAGS = {
   LICENSING: "LICENSING",
   USER_GROUPS: "USER_GROUPS",
+  ONBOARDING_TOUR: "ONBOARDING_TOUR",
 }
 
 export const isEnabled = featureFlag => {

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -2,6 +2,7 @@
   import { store, automationStore } from "builderStore"
   import { roles, flags } from "stores/backend"
   import { auth } from "stores/portal"
+  import { TENANT_FEATURE_FLAGS, isEnabled } from "helpers/featureFlags"
   import {
     ActionMenu,
     MenuItem,
@@ -68,7 +69,10 @@
   }
 
   const initTour = async () => {
-    if (!$auth.user?.onboardedAt) {
+    if (
+      !$auth.user?.onboardedAt &&
+      isEnabled(TENANT_FEATURE_FLAGS.ONBOARDING_TOUR)
+    ) {
       // Determine the correct step
       const activeNav = $layout.children.find(c => $isActive(c.path))
       const onboardingTour = TOURS[TOUR_KEYS.TOUR_BUILDER_ONBOARDING]

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte

@@ -1,9 +1,45 @@
 <script>
   import TableDataTable from "components/backend/DataTable/DataTable.svelte"
   import { tables, database } from "stores/backend"
+  import { Banner } from "@budibase/bbui"
+
+  const verifyAutocolumns = table => {
+    // Check for duplicates
+    return Object.values(table?.schema || {}).reduce((acc, fieldSchema) => {
+      if (!fieldSchema.autocolumn || !fieldSchema.subtype) {
+        return acc
+      }
+      let fieldKey = fieldSchema.tableId
+        ? `${fieldSchema.tableId}-${fieldSchema.subtype}`
+        : fieldSchema.subtype
+      acc[fieldKey] = acc[fieldKey] || []
+      acc[fieldKey].push(fieldSchema)
+      return acc
+    }, {})
+  }
+
+  $: autoColumnStatus = verifyAutocolumns($tables?.selected)
+  $: duplicates = Object.values(autoColumnStatus).reduce((acc, status) => {
+    if (status.length > 1) {
+      acc = [...acc, ...status]
+    }
+    return acc
+  }, [])
+  $: invalidColumnText = duplicates.map(entry => {
+    return `${entry.name} (${entry.subtype})`
+  })
 </script>
 
-{#if $database?._id && $tables?.selected}
+{#if $database?._id && $tables?.selected?.name}
+  {#if duplicates?.length}
+    <div class="alert-wrap">
+      <Banner type="warning" showCloseButton={false}>
+        {`Schema Invalid - There are duplicate auto column types defined in this schema. 
+      Please delete the duplicate entries where appropriate: - 
+      ${invalidColumnText.join(", ")}`}
+      </Banner>
+    </div>
+  {/if}
   <TableDataTable />
 {:else}
   <i>Create your first table to start building</i>
@@ -15,4 +51,11 @@
     color: var(--grey-5);
     margin-top: 2px;
   }
+  .alert-wrap {
+    display: flex;
+    width: 100%;
+  }
+  .alert-wrap :global(> *) {
+    flex: 1;
+  }
 </style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/settings/ComponentSettingsPanel.svelte

@@ -30,7 +30,7 @@
 {#if $selectedComponent}
   {#key $selectedComponent._id}
     <Panel {title} icon={componentDefinition?.icon} borderLeft>
-      {#if componentDefinition.info}
+      {#if componentDefinition?.info}
         <ComponentInfoSection {componentDefinition} />
       {/if}
       <ComponentSettingsSection

packages/builder/src/pages/builder/auth/forgot.svelte

@@ -52,7 +52,7 @@
           <span class="back-chev" on:click={() => $goto("../")}>
             <Icon name="ChevronLeft" size="XL" />
           </span>
-          Forgotten your password?
+          Forgot your password?
         </div>
       </Heading>
     </span>
@@ -83,7 +83,12 @@
       </FancyForm>
     </Layout>
     <div>
-      <Button disabled={!email || error || submitted} cta on:click={forgot}>
+      <Button
+        size="L"
+        disabled={!email || error || submitted}
+        cta
+        on:click={forgot}
+      >
         Reset password
       </Button>
     </div>
@@ -92,7 +97,7 @@
 
 <style>
   img {
-    width: 48px;
+    width: 46px;
   }
   .back-chev {
     display: inline-block;
@@ -102,5 +107,6 @@
   .heading-content {
     display: flex;
     align-items: center;
+    gap: var(--spacing-m);
   }
 </style>

packages/builder/src/pages/builder/auth/login.svelte

@@ -66,7 +66,7 @@
 <svelte:window on:keydown={handleKeydown} />
 
 <TestimonialPage>
-  <Layout gap="S" noPadding>
+  <Layout gap="L" noPadding>
     <Layout justifyItems="center" noPadding>
       {#if loaded}
         <img alt="logo" src={$organisation.logoUrl || Logo} />
@@ -124,14 +124,19 @@
       </FancyForm>
     </Layout>
     <Layout gap="XS" noPadding justifyItems="center">
-      <Button cta disabled={Object.keys(errors).length > 0} on:click={login}>
+      <Button
+        size="L"
+        cta
+        disabled={Object.keys(errors).length > 0}
+        on:click={login}
+      >
         Log in to {company}
       </Button>
     </Layout>
     <Layout gap="XS" noPadding justifyItems="center">
       <div class="user-actions">
-        <ActionButton quiet on:click={() => $goto("./forgot")}>
+        <ActionButton size="L" quiet on:click={() => $goto("./forgot")}>
-          Forgot password
+          Forgot password?
         </ActionButton>
       </div>
     </Layout>

packages/builder/src/pages/builder/invite/index.svelte

@@ -68,7 +68,7 @@
 </script>
 
 <TestimonialPage>
-  <Layout gap="S" noPadding>
+  <Layout gap="M" noPadding>
     <img alt="logo" src={$organisation.logoUrl || Logo} />
     <Layout gap="XS" noPadding>
       <Heading size="M">Join {company}</Heading>
@@ -175,6 +175,7 @@
     </Layout>
     <div>
       <Button
+        size="L"
         disabled={Object.keys(errors).length > 0 || onboarding}
         cta
         on:click={acceptInvite}

packages/builder/src/pages/builder/portal/_layout.svelte

@@ -14,7 +14,7 @@
   let activeTab = "Apps"
 
   $: $url(), updateActiveTab($menu)
-  $: fullScreen = !$apps?.length
+  $: fullscreen = !$apps.length
 
   const updateActiveTab = menu => {
     for (let entry of menu) {
@@ -37,7 +37,8 @@
         $redirect("../")
       } else {
         try {
-          await organisation.init()
+          // We need to load apps to know if we need to show onboarding fullscreen
+          await Promise.all([apps.load(), organisation.init()])
         } catch (error) {
           notifications.error("Error getting org config")
         }
@@ -47,37 +48,39 @@
   })
 </script>
 
-{#if fullScreen}
+{#if $auth.user && loaded}
-  <slot />
+  {#if fullscreen}
-{:else if $auth.user && loaded}
+    <slot />
-  <HelpMenu />
+  {:else}
-  <div class="container">
+    <HelpMenu />
-    <div class="nav">
+    <div class="container">
-      <div class="branding">
+      <div class="nav">
-        <Logo />
+        <div class="branding">
+          <Logo />
+        </div>
+        <div class="desktop">
+          <Tabs selected={activeTab}>
+            {#each $menu as { title, href }}
+              <Tab {title} on:click={() => $goto(href)} />
+            {/each}
+          </Tabs>
+        </div>
+        <div class="mobile">
+          <Icon hoverable name="ShowMenu" on:click={showMobileMenu} />
+        </div>
+        <div class="desktop">
+          <UpgradeButton />
+        </div>
+        <div class="dropdown">
+          <UserDropdown />
+        </div>
       </div>
-      <div class="desktop">
+      <div class="main">
-        <Tabs selected={activeTab}>
+        <slot />
-          {#each $menu as { title, href }}
-            <Tab {title} on:click={() => $goto(href)} />
-          {/each}
-        </Tabs>
-      </div>
-      <div class="mobile">
-        <Icon hoverable name="ShowMenu" on:click={showMobileMenu} />
-      </div>
-      <div class="desktop">
-        <UpgradeButton />
-      </div>
-      <div class="dropdown">
-        <UserDropdown />
       </div>
+      <MobileMenu visible={mobileMenuVisible} on:close={hideMobileMenu} />
     </div>
-    <div class="main">
+  {/if}
-      <slot />
-    </div>
-    <MobileMenu visible={mobileMenuVisible} on:close={hideMobileMenu} />
-  </div>
 {/if}
 
 <style>

packages/builder/src/pages/builder/portal/apps/_layout.svelte

@@ -10,13 +10,11 @@
   onMount(async () => {
     try {
       // Always load latest
-      await apps.load()
+      await Promise.all([
-      await licensing.init()
+        licensing.init(),
-      await templates.load()
+        templates.load(),
-
+        groups.actions.init(),
-      if ($licensing.groupsEnabled) {
+      ])
-        await groups.actions.init()
-      }
 
       if ($templates?.length === 0) {
         notifications.error("There was a problem loading quick start templates")

packages/builder/src/pages/builder/portal/apps/onboarding/_components/NamePanel.svelte

@@ -5,6 +5,8 @@
   export let name = ""
   export let url = ""
   export let onNext = () => {}
+
+  const nameRegex = /^[a-zA-Z0-9\s]*$/
   let nameError = null
   let urlError = null
 
@@ -14,6 +16,9 @@
     if (name.length < 1) {
       return "Name must be provided"
     }
+    if (!nameRegex.test(name)) {
+      return "No special characters are allowed"
+    }
   }
 
   const validateUrl = url => {

packages/builder/src/pages/builder/portal/apps/onboarding/index.svelte

@@ -17,8 +17,8 @@
   import createFromScratchScreen from "builderStore/store/screenTemplates/createFromScratchScreen"
   import { Roles } from "constants/backend"
 
-  let name = ""
+  let name = "My first app"
-  let url = ""
+  let url = "my-first-app"
   let stage = "name"
   let appId = null
 
@@ -57,7 +57,7 @@
     defaultScreenTemplate.routing.roldId = Roles.BASIC
     await store.actions.screens.save(defaultScreenTemplate)
 
-    return createdApp.instance._id
+    appId = createdApp.instance._id
   }
 
   const getIntegrations = async () => {
@@ -79,14 +79,14 @@
     }
   }
 
-  const goToApp = appId => {
+  const goToApp = () => {
     $goto(`/builder/app/${appId}`)
     notifications.success(`App created successfully`)
   }
 
   const handleCreateApp = async ({ datasourceConfig, useSampleData }) => {
     try {
-      appId = await createApp(useSampleData)
+      await createApp(useSampleData)
 
       if (datasourceConfig) {
         await saveDatasource({
@@ -99,7 +99,7 @@
         })
       }
 
-      goToApp(appId)
+      goToApp()
     } catch (e) {
       console.log(e)
       notifications.error("There was a problem creating your app")
@@ -111,7 +111,7 @@
   <CreateTableModal
     name="Your Data"
     beforeSave={createApp}
-    afterSave={() => goToApp(appId)}
+    afterSave={goToApp}
   />
 </Modal>
 
@@ -142,7 +142,7 @@
             <div class="dataButtonIcon">
               <FontAwesomeIcon name="fa-solid fa-file-arrow-up" />
             </div>
-            Upload file
+            Upload data (CSV or JSON)
           </div>
         </FancyButton>
       </div>

packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte

@@ -100,8 +100,9 @@
   const deleteApp = async () => {
     try {
       await API.deleteApp(app?.devId)
+      apps.load()
       notifications.success("App deleted successfully")
-      $goto("../")
+      $goto("../../")
     } catch (err) {
       notifications.error("Error deleting app")
     }

packages/builder/src/pages/builder/portal/overview/_layout.svelte

@@ -1,17 +1,12 @@
 <script>
-  import { apps, groups, licensing } from "stores/portal"
+  import { groups } from "stores/portal"
   import { onMount } from "svelte"
 
-  let loaded = !!$apps?.length
+  let loaded = false
 
   onMount(async () => {
-    if (!loaded) {
+    await groups.actions.init()
-      await apps.load()
+    loaded = true
-      if ($licensing.groupsEnabled) {
-        await groups.actions.init()
-      }
-      loaded = true
-    }
   })
 </script>
 

packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte

@@ -146,7 +146,7 @@
 
   onMount(async () => {
     try {
-      await Promise.all([groups.actions.init(), apps.load(), roles.fetch()])
+      await Promise.all([groups.actions.init(), roles.fetch()])
       loaded = true
     } catch (error) {
       notifications.error("Error fetching user group data")

packages/builder/src/pages/builder/portal/users/groups/index.svelte

@@ -80,9 +80,7 @@
     try {
       // always load latest
       await licensing.init()
-      if ($licensing.groupsEnabled) {
+      await groups.actions.init()
-        await groups.actions.init()
-      }
     } catch (error) {
       notifications.error("Error getting user groups")
     }

packages/builder/src/pages/builder/portal/users/users/[userId].svelte

@@ -215,12 +215,7 @@
 
   onMount(async () => {
     try {
-      await Promise.all([
+      await Promise.all([fetchUser(), groups.actions.init(), roles.fetch()])
-        fetchUser(),
-        groups.actions.init(),
-        apps.load(),
-        roles.fetch(),
-      ])
       loaded = true
     } catch (error) {
       notifications.error("Error getting user groups")

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "2.2.12-alpha.59",
+  "version": "2.2.27-alpha.0",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -26,9 +26,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "2.2.12-alpha.59",
+    "@budibase/backend-core": "2.2.27-alpha.0",
-    "@budibase/string-templates": "2.2.12-alpha.59",
+    "@budibase/string-templates": "2.2.27-alpha.0",
-    "@budibase/types": "2.2.12-alpha.59",
+    "@budibase/types": "2.2.27-alpha.0",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",

packages/client/manifest.json

@@ -3241,7 +3241,7 @@
     },
     "settings": [
       {
-        "type": "field/barcode/qr",
+        "type": "field/barcodeqr",
         "label": "Field",
         "key": "field",
         "required": true

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "2.2.12-alpha.59",
+  "version": "2.2.27-alpha.0",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "2.2.12-alpha.59",
+    "@budibase/bbui": "2.2.27-alpha.0",
-    "@budibase/frontend-core": "2.2.12-alpha.59",
+    "@budibase/frontend-core": "2.2.27-alpha.0",
-    "@budibase/string-templates": "2.2.12-alpha.59",
+    "@budibase/string-templates": "2.2.27-alpha.0",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/client/src/components/Component.svelte

@@ -171,6 +171,15 @@
   $: pad = pad || (interactive && hasChildren && inDndPath)
   $: $dndIsDragging, (pad = false)
 
+  // Determine whether we should render a skeleton loader for this component
+  $: showSkeleton =
+    $loading &&
+    definition.name !== "Screenslot" &&
+    children.length === 0 &&
+    !instance._blockElementHasChildren &&
+    !definition.block &&
+    definition.skeleton !== false
+
   // Update component context
   $: store.set({
     id,
@@ -473,14 +482,6 @@
       componentStore.actions.unregisterInstance(id)
     }
   })
-
-  $: showSkeleton =
-    $loading &&
-    definition.name !== "Screenslot" &&
-    children.length === 0 &&
-    !instance._blockElementHasChildren &&
-    !definition.block &&
-    definition.skeleton !== false
 </script>
 
 {#if showSkeleton}

packages/client/src/components/app/DataProvider.svelte

@@ -11,20 +11,23 @@
   export let limit
   export let paginate
 
-  const loading = writable(false)
-
   const { styleable, Provider, ActionTypes, API } = getContext("sdk")
   const component = getContext("component")
 
+  // Update loading state
+  const parentLoading = getContext("loading")
+  const loading = writable(true)
+  setContext("loading", loading)
+
   // We need to manage our lucene query manually as we want to allow components
   // to extend it
   let queryExtensions = {}
   $: defaultQuery = LuceneUtils.buildLuceneQuery(filter)
   $: query = extendQuery(defaultQuery, queryExtensions)
 
-  // Keep our data fetch instance up to date
+  // Fetch data and refresh when needed
-  $: fetch = createFetch(dataSource)
+  $: fetch = createFetch(dataSource, $parentLoading)
-  $: fetch.update({
+  $: updateFetch({
     query,
     sortColumn,
     sortOrder,
@@ -32,6 +35,9 @@
     paginate,
   })
 
+  // Keep loading context updated
+  $: loading.set($parentLoading || !$fetch.loaded)
+
   // Build our action context
   $: actions = [
     {
@@ -80,14 +86,21 @@
       sortColumn: $fetch.sortColumn,
       sortOrder: $fetch.sortOrder,
     },
-    limit: limit,
+    limit,
   }
 
-  const parentLoading = getContext("loading")
+  const createFetch = (datasource, parentLoading) => {
-  setContext("loading", loading)
+    // Return a dummy fetch if parent is still loading. We do this so that we
-  $: loading.set($parentLoading || !$fetch.loaded)
+    // can still properly subscribe to a valid fetch object and check all
+    // properties, but we want to avoid fetching the real data until all parents
+    // have finished loading.
+    // This logic is only needed due to skeleton loaders, as previously we
+    // simply blocked component rendering until data was ready.
+    if (parentLoading) {
+      return fetchData({ API })
+    }
 
-  const createFetch = datasource => {
+    // Otherwise return the real thing
     return fetchData({
       API,
       datasource,
@@ -101,6 +114,14 @@
     })
   }
 
+  const updateFetch = opts => {
+    // Only update fetch if parents have stopped loading. Otherwise we will
+    // trigger a fetch of the real data before parents are ready.
+    if (!$parentLoading) {
+      fetch.update(opts)
+    }
+  }
+
   const addQueryExtension = (key, extension) => {
     if (!key || !extension) {
       return

packages/client/src/components/app/forms/Form.svelte

@@ -1,7 +1,8 @@
 <script>
-  import { getContext } from "svelte"
+  import { getContext, setContext } from "svelte"
   import InnerForm from "./InnerForm.svelte"
   import { Helpers } from "@budibase/bbui"
+  import { writable } from "svelte/store"
 
   export let dataSource
   export let theme
@@ -20,6 +21,12 @@
   const context = getContext("context")
   const { API, fetchDatasourceSchema } = getContext("sdk")
 
+  // Forms also use loading context as they require loading a schema
+  const parentLoading = getContext("loading")
+  const loading = writable(true)
+  setContext("loading", loading)
+
+  let loaded = false
   let schema
   let table
 
@@ -29,6 +36,7 @@
   $: resetKey = Helpers.hashString(
     schemaKey + JSON.stringify(initialValues) + disabled
   )
+  $: loading.set($parentLoading || !loaded)
 
   // Returns the closes data context which isn't a built in context
   const getInitialValues = (type, dataSource, context) => {
@@ -60,6 +68,9 @@
     }
     const res = await fetchDatasourceSchema(dataSource)
     schema = res || {}
+    if (!loaded) {
+      loaded = true
+    }
   }
 
   // Generates a predictable string that uniquely identifies a schema. We can't

packages/client/src/components/app/forms/InnerForm.svelte

@@ -128,21 +128,15 @@
     return fields.find(field => get(field).name === name)
   }
 
-  const getDefault = (defaultValue, schema, type) => {
+  // Sanitises a value by ensuring it doesn't contain any invalid data
-    // Remove any values not present in the field schema
+  const sanitiseValue = (value, schema, type) => {
-    // Convert any values supplied to string
+    // Check arrays - remove any values not present in the field schema and
-    if (Array.isArray(defaultValue) && type == "array" && schema) {
+    // convert any values supplied to strings
-      return defaultValue.reduce((acc, entry) => {
+    if (Array.isArray(value) && type === "array" && schema) {
-        let processedOption = String(entry)
+      const options = schema?.constraints.inclusion || []
-        let schemaOptions = schema.constraints.inclusion
+      return value.map(opt => String(opt)).filter(opt => options.includes(opt))
-        if (schemaOptions.indexOf(processedOption) > -1) {
-          acc.push(processedOption)
-        }
-        return acc
-      }, [])
-    } else {
-      return defaultValue
     }
+    return value
   }
 
   const formApi = {
@@ -160,7 +154,6 @@
 
       // Create validation function based on field schema
       const schemaConstraints = schema?.[field]?.constraints
-
       const validator = disableValidation
         ? null
         : createValidatorFromConstraints(
@@ -170,10 +163,11 @@
             table
           )
 
-      const parsedDefault = getDefault(defaultValue, schema?.[field], type)
+      // Sanitise the default value to ensure it doesn't contain invalid data
+      defaultValue = sanitiseValue(defaultValue, schema?.[field], type)
 
       // If we've already registered this field then keep some existing state
-      let initialValue = Helpers.deepGet(initialValues, field) ?? parsedDefault
+      let initialValue = Helpers.deepGet(initialValues, field) ?? defaultValue
       let initialError = null
       let fieldId = `id-${Helpers.uuid()}`
       const existingField = getField(field)
@@ -183,7 +177,9 @@
 
         // Determine the initial value for this field, reusing the current
         // value if one exists
-        initialValue = fieldState.value ?? initialValue
+        if (fieldState.value != null && fieldState.value !== "") {
+          initialValue = fieldState.value
+        }
 
         // If this field has already been registered and we previously had an
         // error set, then re-run the validator to see if we can unset it
@@ -206,11 +202,11 @@
           error: initialError,
           disabled:
             disabled || fieldDisabled || (isAutoColumn && !editAutoColumns),
-          defaultValue: parsedDefault,
+          defaultValue,
           validator,
           lastUpdate: Date.now(),
         },
-        fieldApi: makeFieldApi(field, parsedDefault),
+        fieldApi: makeFieldApi(field),
         fieldSchema: schema?.[field] ?? {},
       })
 
@@ -225,18 +221,9 @@
       return fieldInfo
     },
     validate: () => {
-      let valid = true
+      return fields
-      let validationFields = fields
+        .filter(field => get(field).step === get(currentStep))
-
+        .every(field => get(field).fieldApi.validate())
-      validationFields = fields.filter(f => get(f).step === get(currentStep))
-
-      // Validate fields and check if any are invalid
-      validationFields.forEach(field => {
-        if (!get(field).fieldApi.validate()) {
-          valid = false
-        }
-      })
-      return valid
     },
     reset: () => {
       // Reset the form by resetting each individual field

packages/client/src/components/app/forms/OptionsField.svelte

@@ -79,6 +79,7 @@
         getOptionLabel={flatOptions ? x => x : x => x.label}
         getOptionTitle={flatOptions ? x => x : x => x.label}
         getOptionValue={flatOptions ? x => x : x => x.value}
+        {sort}
       />
     {/if}
   {/if}

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.2.12-alpha.59",
+  "version": "2.2.27-alpha.0",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "2.2.12-alpha.59",
+    "@budibase/bbui": "2.2.27-alpha.0",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "2.2.12-alpha.59",
+  "version": "2.2.27-alpha.0",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",

packages/server/Dockerfile

@@ -12,7 +12,7 @@ ENV COUCH_DB_URL=https://couchdb.budi.live:5984
 ENV BUDIBASE_ENVIRONMENT=PRODUCTION
 ENV SERVICE=app-service
 ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
-ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS
+ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR
 ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
 
 # copy files and install dependencies

packages/server/jest.config.ts

@@ -1,9 +1,9 @@
-import { Config } from "jest"
+import { Config } from "@jest/types"
 
 import * as fs from "fs"
 const preset = require("ts-jest/jest-preset")
 
-const testContainersSettings = {
+const baseConfig: Config.InitialProjectOptions = {
   ...preset,
   preset: "@trendyol/jest-testcontainers",
   setupFiles: ["./src/tests/jestEnv.ts"],
@@ -15,30 +15,30 @@ const testContainersSettings = {
 
 if (!process.env.CI) {
   // use sources when not in CI
-  testContainersSettings.moduleNameMapper = {
+  baseConfig.moduleNameMapper = {
     "@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
     "@budibase/backend-core": "<rootDir>/../backend-core/src",
     "@budibase/types": "<rootDir>/../types/src",
   }
   // add pro sources if they exist
   if (fs.existsSync("../../../budibase-pro")) {
-    testContainersSettings.moduleNameMapper["@budibase/pro"] =
+    baseConfig.moduleNameMapper["@budibase/pro"] =
       "<rootDir>/../../../budibase-pro/packages/pro/src"
   }
 } else {
   console.log("Running tests with compiled dependency sources")
 }
 
-const config: Config = {
+const config: Config.InitialOptions = {
   projects: [
     {
-      ...testContainersSettings,
+      ...baseConfig,
       displayName: "sequential test",
       testMatch: ["<rootDir>/**/*.seq.spec.[jt]s"],
       runner: "jest-serial-runner",
     },
     {
-      ...testContainersSettings,
+      ...baseConfig,
       testMatch: ["<rootDir>/**/!(*.seq).spec.[jt]s"],
     },
   ],

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.2.12-alpha.59",
+  "version": "2.2.27-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -43,11 +43,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.2.12-alpha.59",
+    "@budibase/backend-core": "2.2.27-alpha.0",
-    "@budibase/client": "2.2.12-alpha.59",
+    "@budibase/client": "2.2.27-alpha.0",
-    "@budibase/pro": "2.2.12-alpha.58",
+    "@budibase/pro": "2.2.27-alpha.0",
-    "@budibase/string-templates": "2.2.12-alpha.59",
+    "@budibase/string-templates": "2.2.27-alpha.0",
-    "@budibase/types": "2.2.12-alpha.59",
+    "@budibase/types": "2.2.27-alpha.0",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",

packages/server/scripts/dev/manage.js

@@ -36,7 +36,7 @@ async function init() {
       COUCH_DB_PASSWORD: "budibase",
       COUCH_DB_USER: "budibase",
       SELF_HOSTED: 1,
-      DISABLE_ACCOUNT_PORTAL: "",
+      DISABLE_ACCOUNT_PORTAL: 1,
       MULTI_TENANCY: "",
       DISABLE_THREADING: 1,
       SERVICE: "app-service",
@@ -44,7 +44,7 @@ async function init() {
       BB_ADMIN_USER_EMAIL: "",
       BB_ADMIN_USER_PASSWORD: "",
       PLUGINS_DIR: "",
-      TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS",
+      TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

packages/server/src/api/controllers/query/import/tests/index.spec.js

@@ -41,7 +41,7 @@ const datasets = {
 describe("Rest Importer", () => {
   const config = new TestConfig(false)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
   })
 

packages/server/src/api/controllers/table/utils.ts

@@ -315,7 +315,13 @@ export async function checkForViewUpdates(
 
     // Update view if required
     if (needsUpdated) {
-      const newViewTemplate = viewTemplate(view.meta)
+      const groupByField: any = Object.values(table.schema).find(
+        (field: any) => field.name == view.groupBy
+      )
+      const newViewTemplate = viewTemplate(
+        view.meta,
+        groupByField?.type === FieldTypes.ARRAY
+      )
       await saveView(null, view.name, newViewTemplate)
       if (!newViewTemplate.meta.schema) {
         newViewTemplate.meta.schema = table.schema

packages/server/src/api/controllers/view/index.ts

@@ -6,8 +6,9 @@ import { fetchView } from "../row"
 import { context, events } from "@budibase/backend-core"
 import { DocumentType } from "../../../db/utils"
 import sdk from "../../../sdk"
+import { FieldTypes } from "../../../constants"
 import {
-  BBContext,
+  Ctx,
   Row,
   Table,
   TableExportFormat,
@@ -18,14 +19,22 @@ import { cleanExportRows } from "../row/utils"
 
 const { cloneDeep, isEqual } = require("lodash")
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: Ctx) {
   ctx.body = await getViews()
 }
 
-export async function save(ctx: BBContext) {
+export async function save(ctx: Ctx) {
   const db = context.getAppDB()
   const { originalName, ...viewToSave } = ctx.request.body
-  const view = viewTemplate(viewToSave)
+
+  const existingTable = await db.get(ctx.request.body.tableId)
+  const table = cloneDeep(existingTable)
+
+  const groupByField: any = Object.values(table.schema).find(
+    (field: any) => field.name == viewToSave.groupBy
+  )
+
+  const view = viewTemplate(viewToSave, groupByField?.type === FieldTypes.ARRAY)
   const viewName = viewToSave.name
 
   if (!viewName) {
@@ -35,8 +44,6 @@ export async function save(ctx: BBContext) {
   await saveView(originalName, viewName, view)
 
   // add views to table document
-  const existingTable = await db.get(ctx.request.body.tableId)
-  const table = cloneDeep(existingTable)
   if (!table.views) table.views = {}
   if (!view.meta.schema) {
     view.meta.schema = table.schema
@@ -111,7 +118,7 @@ async function handleViewEvents(existingView: View, newView: View) {
   await filterEvents(existingView, newView)
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: Ctx) {
   const db = context.getAppDB()
   const viewName = decodeURIComponent(ctx.params.viewName)
   const view = await deleteView(viewName)
@@ -123,7 +130,7 @@ export async function destroy(ctx: BBContext) {
   ctx.body = view
 }
 
-export async function exportView(ctx: BBContext) {
+export async function exportView(ctx: Ctx) {
   const viewName = decodeURIComponent(ctx.query.view as string)
   const view = await getView(viewName)
 

packages/server/src/api/controllers/view/viewBuilder.ts

@@ -6,6 +6,7 @@ type ViewTemplateOpts = {
   groupBy: string
   filters: ViewFilter[]
   calculation: string
+  groupByMulti: boolean
 }
 
 const TOKEN_MAP: Record<string, string> = {
@@ -41,6 +42,12 @@ const GROUP_PROPERTY: Record<string, { type: string }> = {
   },
 }
 
+const GROUP_PROPERTY_MULTI: Record<string, { type: string }> = {
+  group: {
+    type: "array",
+  },
+}
+
 const FIELD_PROPERTY: Record<string, { type: string }> = {
   field: {
     type: "string",
@@ -136,13 +143,10 @@ function parseEmitExpression(field: string, groupBy: string) {
  * filters: Array of filter objects containing predicates that are parsed into a JS expression
  * calculation: an optional calculation to be performed over the view data.
  */
-export default function ({
+export default function (
-  field,
+  { field, tableId, groupBy, filters = [], calculation }: ViewTemplateOpts,
-  tableId,
+  groupByMulti?: boolean
-  groupBy,
+) {
-  filters = [],
-  calculation,
-}: ViewTemplateOpts) {
   // first filter can't have a conjunction
   if (filters && filters.length > 0 && filters[0].conjunction) {
     delete filters[0].conjunction
@@ -151,9 +155,11 @@ export default function ({
   let schema = null,
     statFilter = null
 
+  let groupBySchema = groupByMulti ? GROUP_PROPERTY_MULTI : GROUP_PROPERTY
+
   if (calculation) {
     schema = {
-      ...(groupBy ? GROUP_PROPERTY : FIELD_PROPERTY),
+      ...(groupBy ? groupBySchema : FIELD_PROPERTY),
       ...SCHEMA_MAP[calculation],
     }
     if (

packages/server/src/api/routes/tests/analytics.spec.js

@@ -10,8 +10,11 @@ describe("/static", () => {
 
   afterAll(setup.afterAll)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     app = await config.init()
+  })
+  
+  beforeEach(()=>{
     jest.clearAllMocks()
   })
 

packages/server/src/api/routes/tests/apikeys.spec.js

@@ -7,7 +7,7 @@ describe("/api/keys", () => {
 
   afterAll(setup.afterAll)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
   })
 

packages/server/src/api/routes/tests/application.spec.ts

@@ -14,18 +14,22 @@ jest.mock("../../../utilities/redis", () => ({
 import { clearAllApps, checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 import { AppStatus } from "../../../db/utils"
-import { events } from "@budibase/backend-core"
+import { events, utils } from "@budibase/backend-core"
 import env from "../../../environment"
 
+jest.setTimeout(15000)
+
 describe("/applications", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
 
   afterAll(setup.afterAll)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
-    await clearAllApps()
     await config.init()
+  })
+
+  beforeEach(async () => {
     jest.clearAllMocks()
   })
 
@@ -33,7 +37,7 @@ describe("/applications", () => {
     it("creates empty app", async () => {
       const res = await request
         .post("/api/applications")
-        .field("name", "My App")
+        .field("name", utils.newid())
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
@@ -44,7 +48,7 @@ describe("/applications", () => {
     it("creates app from template", async () => {
       const res = await request
         .post("/api/applications")
-        .field("name", "My App")
+        .field("name", utils.newid())
         .field("useTemplate", "true")
         .field("templateKey", "test")
         .field("templateString", "{}") // override the file download
@@ -59,7 +63,7 @@ describe("/applications", () => {
     it("creates app from file", async () => {
       const res = await request
         .post("/api/applications")
-        .field("name", "My App")
+        .field("name", utils.newid())
         .field("useTemplate", "true")
         .set(config.defaultHeaders())
         .attach("templateFile", "src/api/routes/tests/data/export.txt")
@@ -106,6 +110,11 @@ describe("/applications", () => {
   })
 
   describe("fetch", () => {
+    beforeEach(async () => {
+      // Clean all apps but the onde from config
+      await clearAllApps(config.getTenantId(), [config.getAppId()!])
+    })
+
     it("lists all applications", async () => {
       await config.createApp("app1")
       await config.createApp("app2")
@@ -266,6 +275,11 @@ describe("/applications", () => {
   })
 
   describe("unpublish", () => {
+    beforeEach(async () => {
+      // We want to republish as the unpublish will delete the prod app
+      await config.publish()
+    })
+
     it("should unpublish app with dev app ID", async () => {
       const appId = config.getAppId()
       await request

packages/server/src/api/routes/tests/auth.spec.js

@@ -7,7 +7,7 @@ describe("/authenticate", () => {
 
   afterAll(setup.afterAll)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
   })
 

packages/server/src/api/routes/tests/automation.spec.js

@@ -10,12 +10,16 @@ const MAX_RETRIES = 4
 const { TRIGGER_DEFINITIONS, ACTION_DEFINITIONS } = require("../../../automations")
 const { events } = require("@budibase/backend-core")
 
+
+jest.setTimeout(30000)
+
 describe("/automations", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
 
   afterAll(setup.afterAll)
 
+  // For some reason this cannot be a beforeAll or the test "tests the automation successfully" fail
   beforeEach(async () => {
     await config.init()
   })
@@ -305,7 +309,7 @@ describe("/automations", () => {
         .expect('Content-Type', /json/)
         .expect(200)
 
-        expect(res.body[0]).toEqual(expect.objectContaining(autoConfig))
+      expect(res.body[0]).toEqual(expect.objectContaining(autoConfig))
     })
 
     it("should apply authorization to endpoint", async () => {

packages/server/src/api/routes/tests/component.spec.js

@@ -7,7 +7,7 @@ describe("/component", () => {
 
   afterAll(setup.afterAll)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
   })
 

packages/server/src/api/routes/tests/datasource.spec.ts

@@ -19,11 +19,13 @@ describe("/datasources", () => {
 
   afterAll(setup.afterAll)
 
-  beforeEach(async () => {
+  async function setupTest() {
     await config.init()
     datasource = await config.createDatasource()
     jest.clearAllMocks()
-  })
+  }
+
+  beforeAll(setupTest)
 
   describe("create", () => {
     it("should create a new datasource", async () => {
@@ -102,6 +104,8 @@ describe("/datasources", () => {
   })
 
   describe("fetch", () => {
+    beforeAll(setupTest)
+
     it("returns all the datasources from the server", async () => {
       const res = await request
         .get(`/api/datasources`)
@@ -170,6 +174,8 @@ describe("/datasources", () => {
   })
 
   describe("destroy", () => {
+    beforeAll(setupTest)
+
     it("deletes queries for the datasource after deletion and returns a success message", async () => {
       await config.createQuery()
 

packages/server/src/api/routes/tests/dev.spec.js

@@ -8,7 +8,7 @@ describe("/dev", () => {
 
   afterAll(setup.afterAll)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
     jest.clearAllMocks()
   })

packages/server/src/api/routes/tests/integration.spec.js

@@ -7,7 +7,7 @@ describe("/integrations", () => {
 
   afterAll(setup.afterAll)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
   })
 

packages/server/src/api/routes/tests/layout.spec.js

@@ -10,7 +10,7 @@ describe("/layouts", () => {
 
   afterAll(setup.afterAll)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
     layout = await config.createLayout()
     jest.clearAllMocks()

packages/server/src/api/routes/tests/metadata.spec.js

@@ -9,7 +9,7 @@ describe("/metadata", () => {
 
   afterAll(setup.afterAll)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
     automation = await config.createAutomation()
   })

packages/server/src/api/routes/tests/misc.spec.js

@@ -7,7 +7,7 @@ describe("run misc tests", () => {
 
   afterAll(setup.afterAll)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
   })
 

packages/server/src/api/routes/tests/permissions.spec.js

@@ -15,8 +15,11 @@ describe("/permission", () => {
 
   afterAll(setup.afterAll)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
+  })
+  
+  beforeEach(async () => {
     table = await config.createTable()
     row = await config.createRow()
     perms = await config.addPermission(STD_ROLE_ID, table._id)