Merge pull request #11938 from Budibase/develop

develop -> master

.github/workflows/budibase_ci.yml

@@ -30,8 +30,8 @@ jobs:
         with:
           root-reserve-mb: 35000
           swap-size-mb: 1024
-          remove-android: 'true'
+          remove-android: "true"
-          remove-dotnet: 'true'
+          remove-dotnet: "true"
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
         if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
@@ -121,7 +121,7 @@ jobs:
           name: codecov-umbrella
           verbose: true
 
-  test-services:
+  test-worker:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo and submodules
@@ -143,12 +143,48 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
-      - name: Test worker and server
+      - name: Test worker
         run: |
           if ${{ env.USE_NX_AFFECTED }}; then
-            yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
+            yarn test --scope=@budibase/worker --since=${{ env.NX_BASE_BRANCH }}
           else
-            yarn test --scope=@budibase/worker --scope=@budibase/server
+            yarn test --scope=@budibase/worker
+          fi
+
+      - uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
+          name: codecov-umbrella
+          verbose: true
+
+  test-server:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo and submodules
+        uses: actions/checkout@v3
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
+      - name: Checkout repo only
+        uses: actions/checkout@v3
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
+        with:
+          fetch-depth: 0
+
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18.x
+          cache: "yarn"
+      - run: yarn --frozen-lockfile
+      - name: Test server
+        run: |
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn test --scope=@budibase/server
           fi
 
       - uses: codecov/codecov-action@v3
@@ -259,4 +295,4 @@ jobs:
               process.exit(1);
             } else {
               console.log('All good, the submodule had been merged and setup correctly!')
-            }
+            }

.github/workflows/close-featurebranch.yml

@@ -0,0 +1,21 @@
+name: close-featurebranch
+
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - develop
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: passeidireto/trigger-external-workflow-action@main
+        env:
+          PAYLOAD_BRANCH: ${{ github.head_ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.ref }}
+        with:
+          repository: budibase/budibase-deploys
+          event: featurebranch-qa-close
+          github_pat: ${{ secrets.GH_ACCESS_TOKEN }}

.github/workflows/deploy-featurebranch.yml

@@ -12,7 +12,8 @@ jobs:
       - uses: actions/checkout@v3
       - uses: passeidireto/trigger-external-workflow-action@main
         env:
-            BRANCH: ${{ github.head_ref }}
+          PAYLOAD_BRANCH: ${{ github.head_ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.ref }}
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-deploy

.github/workflows/stale_bot.yml

@@ -2,7 +2,7 @@ name: Close stale issues and PRs # https://github.com/actions/stale
 on:
   workflow_dispatch:
   schedule:
-    - cron: '30 1 * * *' # 1:30 every morning
+    - cron: '*/30 * * * *' # Every 30 mins
 
 jobs:
   stale:

.prettierignore

@@ -9,4 +9,5 @@ packages/backend-core/coverage
 packages/server/client
 packages/server/src/definitions/openapi.ts
 packages/builder/.routify
-packages/sdk/sdk
+packages/sdk/sdk
+packages/pro/coverage

hosting/proxy/nginx.prod.conf

@@ -55,7 +55,7 @@ http {
     set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
     set $csp_object "object-src 'none'";
     set $csp_base_uri "base-uri 'self'";
-    set $csp_connect "connect-src 'self' https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";    
+    set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibaseqa.app https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";    
     set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com  https://js.intercomcdn.com  https://fonts.intercomcdn.com";
     set $csp_frame "frame-src 'self' https:";
     set $csp_img "img-src http: https: data: blob:";

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.10.15",
+  "version": "2.10.16-alpha.19",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -5,7 +5,7 @@
     "@esbuild-plugins/tsconfig-paths": "^0.1.2",
     "@nx/js": "16.4.3",
     "@rollup/plugin-json": "^4.0.2",
-    "@typescript-eslint/parser": "5.45.0",
+    "@typescript-eslint/parser": "6.7.2",
     "esbuild": "^0.18.17",
     "esbuild-node-externals": "^1.8.0",
     "eslint": "^8.44.0",
@@ -21,8 +21,8 @@
     "prettier-plugin-svelte": "^2.3.0",
     "rimraf": "^3.0.2",
     "rollup-plugin-replace": "^2.2.0",
-    "svelte": "^3.38.2",
+    "svelte": "3.49.0",
-    "typescript": "4.7.3",
+    "typescript": "5.2.2",
     "@babel/core": "^7.22.5",
     "@babel/eslint-parser": "^7.22.5",
     "@babel/preset-env": "^7.22.5",

packages/backend-core/package.json

@@ -33,17 +33,14 @@
     "bull": "4.10.1",
     "correlation-id": "4.0.0",
     "dotenv": "16.0.1",
-    "emitter-listener": "1.1.2",
     "ioredis": "5.3.2",
     "joi": "17.6.0",
     "jsonwebtoken": "9.0.0",
     "koa-passport": "4.1.4",
     "koa-pino-logger": "4.0.0",
     "lodash": "4.17.21",
-    "lodash.isarguments": "3.1.0",
     "node-fetch": "2.6.7",
     "passport-google-oauth": "2.0.0",
-    "passport-jwt": "4.0.0",
     "passport-local": "1.0.0",
     "passport-oauth2-refresh": "^2.1.0",
     "pino": "8.11.0",
@@ -59,14 +56,13 @@
     "uuid": "8.3.2"
   },
   "devDependencies": {
-    "@jest/test-sequencer": "29.6.2",
     "@shopify/jest-koa-mocks": "5.1.1",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
     "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/chance": "1.1.3",
+    "@types/cookies": "0.7.8",
     "@types/jest": "29.5.3",
-    "@types/koa": "2.13.4",
     "@types/lodash": "4.14.180",
     "@types/node": "18.17.0",
     "@types/node-fetch": "2.6.4",
@@ -80,14 +76,10 @@
     "jest": "29.6.2",
     "jest-environment-node": "29.6.2",
     "jest-serial-runner": "1.2.1",
-    "koa": "2.13.4",
-    "nodemon": "2.0.16",
     "pino-pretty": "10.0.0",
     "pouchdb-adapter-memory": "7.2.2",
     "timekeeper": "2.2.0",
-    "ts-node": "10.8.1",
+    "typescript": "5.2.2"
-    "tsconfig-paths": "4.0.0",
-    "typescript": "4.7.3"
   },
   "nx": {
     "targets": {

packages/backend-core/src/cache/tests/user.spec.ts

@@ -0,0 +1,145 @@
+import { User } from "@budibase/types"
+import { generator, structures } from "../../../tests"
+import { DBTestConfiguration } from "../../../tests/extra"
+import { getUsers } from "../user"
+import { getGlobalDB } from "../../context"
+import _ from "lodash"
+
+import * as redis from "../../redis/init"
+import { UserDB } from "../../users"
+
+const config = new DBTestConfiguration()
+
+describe("user cache", () => {
+  describe("getUsers", () => {
+    const users: User[] = []
+    beforeAll(async () => {
+      const userCount = 10
+      const userIds = generator.arrayOf(() => generator.guid(), {
+        min: userCount,
+        max: userCount,
+      })
+
+      await config.doInTenant(async () => {
+        const db = getGlobalDB()
+        for (const userId of userIds) {
+          const user = structures.users.user({ _id: userId })
+          await db.put(user)
+          users.push(user)
+        }
+      })
+    })
+
+    beforeEach(async () => {
+      jest.clearAllMocks()
+
+      const redisClient = await redis.getUserClient()
+      await redisClient.clear()
+    })
+
+    it("when no user is in cache, all of them are retrieved from db", async () => {
+      const usersToRequest = _.sampleSize(users, 5)
+
+      const userIdsToRequest = usersToRequest.map(x => x._id!)
+
+      jest.spyOn(UserDB, "bulkGet")
+
+      const results = await config.doInTenant(() => getUsers(userIdsToRequest))
+
+      expect(results.users).toHaveLength(5)
+      expect(results).toEqual({
+        users: usersToRequest.map(u => ({
+          ...u,
+          budibaseAccess: true,
+          _rev: expect.any(String),
+        })),
+      })
+
+      expect(UserDB.bulkGet).toBeCalledTimes(1)
+      expect(UserDB.bulkGet).toBeCalledWith(userIdsToRequest)
+    })
+
+    it("on a second all, all of them are retrieved from cache", async () => {
+      const usersToRequest = _.sampleSize(users, 5)
+
+      const userIdsToRequest = usersToRequest.map(x => x._id!)
+
+      jest.spyOn(UserDB, "bulkGet")
+
+      await config.doInTenant(() => getUsers(userIdsToRequest))
+      const resultsFromCache = await config.doInTenant(() =>
+        getUsers(userIdsToRequest)
+      )
+
+      expect(resultsFromCache.users).toHaveLength(5)
+      expect(resultsFromCache).toEqual({
+        users: expect.arrayContaining(
+          usersToRequest.map(u => ({
+            ...u,
+            budibaseAccess: true,
+            _rev: expect.any(String),
+          }))
+        ),
+      })
+
+      expect(UserDB.bulkGet).toBeCalledTimes(1)
+    })
+
+    it("when some users are cached, only the missing ones are retrieved from db", async () => {
+      const usersToRequest = _.sampleSize(users, 5)
+
+      const userIdsToRequest = usersToRequest.map(x => x._id!)
+
+      jest.spyOn(UserDB, "bulkGet")
+
+      await config.doInTenant(() =>
+        getUsers([userIdsToRequest[0], userIdsToRequest[3]])
+      )
+      ;(UserDB.bulkGet as jest.Mock).mockClear()
+
+      const results = await config.doInTenant(() => getUsers(userIdsToRequest))
+
+      expect(results.users).toHaveLength(5)
+      expect(results).toEqual({
+        users: expect.arrayContaining(
+          usersToRequest.map(u => ({
+            ...u,
+            budibaseAccess: true,
+            _rev: expect.any(String),
+          }))
+        ),
+      })
+
+      expect(UserDB.bulkGet).toBeCalledTimes(1)
+      expect(UserDB.bulkGet).toBeCalledWith([
+        userIdsToRequest[1],
+        userIdsToRequest[2],
+        userIdsToRequest[4],
+      ])
+    })
+
+    it("requesting existing and unexisting ids will return found ones", async () => {
+      const usersToRequest = _.sampleSize(users, 3)
+      const missingIds = [generator.guid(), generator.guid()]
+
+      const userIdsToRequest = _.shuffle([
+        ...missingIds,
+        ...usersToRequest.map(x => x._id!),
+      ])
+
+      const results = await config.doInTenant(() => getUsers(userIdsToRequest))
+
+      expect(results.users).toHaveLength(3)
+      expect(results).toEqual({
+        users: expect.arrayContaining(
+          usersToRequest.map(u => ({
+            ...u,
+            budibaseAccess: true,
+            _rev: expect.any(String),
+          }))
+        ),
+        notFoundIds: expect.arrayContaining(missingIds),
+      })
+    })
+  })
+})

packages/backend-core/src/cache/user.ts

@@ -6,6 +6,7 @@ import env from "../environment"
 import * as accounts from "../accounts"
 import { UserDB } from "../users"
 import { sdk } from "@budibase/shared-core"
+import { User } from "@budibase/types"
 
 const EXPIRY_SECONDS = 3600
 
@@ -27,6 +28,35 @@ async function populateFromDB(userId: string, tenantId: string) {
   return user
 }
 
+async function populateUsersFromDB(
+  userIds: string[]
+): Promise<{ users: User[]; notFoundIds?: string[] }> {
+  const getUsersResponse = await UserDB.bulkGet(userIds)
+
+  // Handle missed user ids
+  const notFoundIds = userIds.filter((uid, i) => !getUsersResponse[i])
+
+  const users = getUsersResponse.filter(x => x)
+
+  await Promise.all(
+    users.map(async (user: any) => {
+      user.budibaseAccess = true
+      if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+        const account = await accounts.getAccount(user.email)
+        if (account) {
+          user.account = account
+          user.accountPortalAccess = true
+        }
+      }
+    })
+  )
+
+  if (notFoundIds.length) {
+    return { users, notFoundIds }
+  }
+  return { users }
+}
+
 /**
  * Get the requested user by id.
  * Use redis cache to first read the user.
@@ -77,6 +107,36 @@ export async function getUser(
   return user
 }
 
+/**
+ * Get the requested users by id.
+ * Use redis cache to first read the users.
+ * If not present fallback to loading the users directly and re-caching.
+ * @param {*} userIds the ids of the user to get
+ * @param {*} tenantId the tenant of the users to get
+ * @returns
+ */
+export async function getUsers(
+  userIds: string[]
+): Promise<{ users: User[]; notFoundIds?: string[] }> {
+  const client = await redis.getUserClient()
+  // try cache
+  let usersFromCache = await client.bulkGet<User>(userIds)
+  const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid])
+  const users = Object.values(usersFromCache)
+  let notFoundIds
+
+  if (missingUsersFromCache.length) {
+    const usersFromDb = await populateUsersFromDB(missingUsersFromCache)
+
+    notFoundIds = usersFromDb.notFoundIds
+    for (const userToCache of usersFromDb.users) {
+      await client.store(userToCache._id!, userToCache, EXPIRY_SECONDS)
+    }
+    users.push(...usersFromDb.users)
+  }
+  return { users, notFoundIds: notFoundIds }
+}
+
 export async function invalidateUser(userId: string) {
   const client = await redis.getUserClient()
   await client.delete(userId)

packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts

@@ -102,6 +102,7 @@ describe("sso", () => {
 
           // modified external id to match user format
           ssoUser._id = "us_" + details.userId
+          delete ssoUser.userId
 
           // new sso user won't have a password
           delete ssoUser.password

packages/backend-core/src/redis/redis.ts

@@ -242,7 +242,7 @@ class RedisWrapper {
     }
   }
 
-  async bulkGet(keys: string[]) {
+  async bulkGet<T>(keys: string[]) {
     const db = this._db
     if (keys.length === 0) {
       return {}
@@ -250,7 +250,7 @@ class RedisWrapper {
     const prefixedKeys = keys.map(key => addDbPrefix(db, key))
     let response = await this.getClient().mget(prefixedKeys)
     if (Array.isArray(response)) {
-      let final: any = {}
+      let final: Record<string, T> = {}
       let count = 0
       for (let result of response) {
         if (result) {

packages/backend-core/src/security/permissions.ts

@@ -1,8 +1,9 @@
-import { PermissionType, PermissionLevel } from "@budibase/types"
+import { PermissionLevel, PermissionType } from "@budibase/types"
-export { PermissionType, PermissionLevel } from "@budibase/types"
 import flatten from "lodash/flatten"
 import cloneDeep from "lodash/fp/cloneDeep"
 
+export { PermissionType, PermissionLevel } from "@budibase/types"
+
 export type RoleHierarchy = {
   permissionId: string
 }[]
@@ -78,6 +79,7 @@ export const BUILTIN_PERMISSIONS = {
     permissions: [
       new Permission(PermissionType.QUERY, PermissionLevel.READ),
       new Permission(PermissionType.TABLE, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
     ],
   },
   WRITE: {
@@ -88,6 +90,7 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
       new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
       new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
     ],
   },
   POWER: {
@@ -99,6 +102,7 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
       new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
       new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
     ],
   },
   ADMIN: {
@@ -111,6 +115,7 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
       new Permission(PermissionType.QUERY, PermissionLevel.ADMIN),
       new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
     ],
   },
 }

packages/backend-core/src/security/roles.ts

@@ -215,21 +215,23 @@ async function getAllUserRoles(userRoleId?: string): Promise<RoleDoc[]> {
   return roles
 }
 
+export async function getUserRoleIdHierarchy(
+  userRoleId?: string
+): Promise<string[]> {
+  const roles = await getUserRoleHierarchy(userRoleId)
+  return roles.map(role => role._id!)
+}
+
 /**
  * Returns an ordered array of the user's inherited role IDs, this can be used
  * to determine if a user can access something that requires a specific role.
  * @param {string} userRoleId The user's role ID, this can be found in their access token.
- * @param {object} opts Various options, such as whether to only retrieve the IDs (default true).
+ * @returns {Promise<object[]>} returns an ordered array of the roles, with the first being their
- * @returns {Promise<string[]|object[]>} returns an ordered array of the roles, with the first being their
  * highest level of access and the last being the lowest level.
  */
-export async function getUserRoleHierarchy(
+export async function getUserRoleHierarchy(userRoleId?: string) {
-  userRoleId?: string,
-  opts = { idOnly: true }
-) {
   // special case, if they don't have a role then they are a public user
-  const roles = await getAllUserRoles(userRoleId)
+  return getAllUserRoles(userRoleId)
-  return opts.idOnly ? roles.map(role => role._id) : roles
 }
 
 // this function checks that the provided permissions are in an array format
@@ -249,6 +251,11 @@ export function checkForRoleResourceArray(
   return rolePerms
 }
 
+export async function getAllRoleIds(appId?: string) {
+  const roles = await getAllRoles(appId)
+  return roles.map(role => role._id)
+}
+
 /**
  * Given an app ID this will retrieve all of the roles that are currently within that app.
  * @return {Promise<object[]>} An array of the role objects that were found.
@@ -332,9 +339,7 @@ export class AccessController {
     }
     let roleIds = userRoleId ? this.userHierarchies[userRoleId] : null
     if (!roleIds && userRoleId) {
-      roleIds = (await getUserRoleHierarchy(userRoleId, {
+      roleIds = await getUserRoleIdHierarchy(userRoleId)
-        idOnly: true,
-      })) as string[]
       this.userHierarchies[userRoleId] = roleIds
     }
 

packages/backend-core/src/utils/utils.ts

@@ -10,7 +10,7 @@ import {
   Event,
   TenantResolutionStrategy,
 } from "@budibase/types"
-import { SetOption } from "cookies"
+import type { SetOption } from "cookies"
 const jwt = require("jsonwebtoken")
 
 const APP_PREFIX = DocumentType.APP + SEPARATOR

packages/backend-core/tests/core/utilities/mocks/licenses.ts

@@ -86,8 +86,8 @@ export const useAuditLogs = () => {
   return useFeature(Feature.AUDIT_LOGS)
 }
 
-export const usePublicApiUserRoles = () => {
+export const useExpandedPublicApi = () => {
-  return useFeature(Feature.USER_ROLE_PUBLIC_API)
+  return useFeature(Feature.EXPANDED_PUBLIC_API)
 }
 
 export const useScimIntegration = () => {

packages/backend-core/tests/core/utilities/structures/shared.ts

@@ -1,19 +0,0 @@
-import { User } from "@budibase/types"
-import { generator } from "./generator"
-import { uuid } from "./common"
-
-export const newEmail = () => {
-  return `${uuid()}@test.com`
-}
-
-export const user = (userProps?: any): User => {
-  return {
-    email: newEmail(),
-    password: "test",
-    roles: { app_test: "admin" },
-    firstName: generator.first(),
-    lastName: generator.last(),
-    pictureUrl: "http://test.com",
-    ...userProps,
-  }
-}

packages/backend-core/tests/core/utilities/structures/sso.ts

@@ -13,8 +13,7 @@ import {
 } from "@budibase/types"
 import { generator } from "./generator"
 import { email, uuid } from "./common"
-import * as shared from "./shared"
+import * as users from "./users"
-import { user } from "./shared"
 import sample from "lodash/sample"
 
 export function OAuth(): OAuth2 {
@@ -26,7 +25,7 @@ export function OAuth(): OAuth2 {
 
 export function authDetails(userDoc?: User): SSOAuthDetails {
   if (!userDoc) {
-    userDoc = user()
+    userDoc = users.user()
   }
 
   const userId = userDoc._id || uuid()
@@ -52,7 +51,7 @@ export function providerType(): SSOProviderType {
 
 export function ssoProfile(user?: User): SSOProfile {
   if (!user) {
-    user = shared.user()
+    user = users.user()
   }
   return {
     id: user._id!,

packages/backend-core/tests/core/utilities/structures/users.ts

@@ -4,11 +4,32 @@ import {
   BuilderUser,
   SSOAuthDetails,
   SSOUser,
+  User,
 } from "@budibase/types"
-import { user } from "./shared"
 import { authDetails } from "./sso"
+import { uuid } from "./common"
+import { generator } from "./generator"
+import { tenant } from "."
 
-export { user, newEmail } from "./shared"
+export const newEmail = () => {
+  return `${uuid()}@test.com`
+}
+
+export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
+  const userId = userProps?._id
+  return {
+    _id: userId,
+    userId,
+    email: newEmail(),
+    password: "test",
+    roles: { app_test: "admin" },
+    firstName: generator.first(),
+    lastName: generator.last(),
+    pictureUrl: "http://test.com",
+    tenantId: tenant.id(),
+    ...userProps,
+  }
+}
 
 export const adminUser = (userProps?: any): AdminUser => {
   return {
@@ -31,7 +52,7 @@ export const adminOnlyUser = (userProps?: any): AdminOnlyUser => {
   }
 }
 
-export const builderUser = (userProps?: any): BuilderUser => {
+export const builderUser = (userProps?: Partial<User>): BuilderUser => {
   return {
     ...user(userProps),
     builder: {

packages/backend-core/tests/extra/DBTestConfiguration.ts

@@ -18,7 +18,7 @@ class DBTestConfiguration {
 
   // TENANCY
 
-  doInTenant(task: any) {
+  doInTenant<T>(task: () => Promise<T>) {
     return context.doInTenant(this.tenantId, () => {
       return task()
     })

packages/backend-core/tests/index.ts

@@ -1 +1,2 @@
 export * from "./core/utilities"
+export * from "./extra"

packages/bbui/package.json

@@ -20,14 +20,12 @@
     "@rollup/plugin-commonjs": "^16.0.0",
     "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-node-resolve": "^11.2.1",
-    "cross-env": "^7.0.2",
-    "nollup": "^0.14.1",
     "postcss": "^8.2.9",
     "rollup": "^2.45.2",
     "rollup-plugin-postcss": "^4.0.0",
     "rollup-plugin-svelte": "^7.1.0",
     "rollup-plugin-terser": "^7.0.2",
-    "svelte": "^3.38.2"
+    "svelte": "3.49.0"
   },
   "keywords": [
     "svelte"
@@ -82,7 +80,7 @@
     "@spectrum-css/typography": "3.0.1",
     "@spectrum-css/underlay": "2.0.9",
     "@spectrum-css/vars": "3.0.1",
-    "dayjs": "^1.10.4",
+    "dayjs": "^1.10.8",
     "easymde": "^2.16.1",
     "svelte-flatpickr": "3.2.3",
     "svelte-portal": "^1.0.0",

packages/bbui/src/Banner/Banner.svelte

@@ -66,6 +66,10 @@
     pointer-events: all;
     width: 100%;
   }
+
+  .spectrum-Toast--neutral {
+    background-color: var(--grey-2);
+  }
   .spectrum-Button {
     border: 1px solid rgba(255, 255, 255, 0.2);
   }

packages/bbui/src/Form/Core/Multiselect.svelte

@@ -14,12 +14,12 @@
   export let autocomplete = false
   export let sort = false
   export let autoWidth = false
-  export let fetchTerm = null
+  export let searchTerm = null
-  export let useFetch = false
   export let customPopoverHeight
   export let customPopoverOffsetBelow
   export let customPopoverMaxHeight
   export let open = false
+  export let loading
 
   const dispatch = createEventDispatcher()
 
@@ -82,6 +82,7 @@
 </script>
 
 <Picker
+  on:loadMore
   {id}
   {error}
   {disabled}
@@ -90,9 +91,8 @@
   {options}
   isPlaceholder={!arrayValue.length}
   {autocomplete}
-  bind:fetchTerm
+  bind:searchTerm
   bind:open
-  {useFetch}
   {isOptionSelected}
   {getOptionLabel}
   {getOptionValue}
@@ -102,4 +102,5 @@
   {customPopoverHeight}
   {customPopoverOffsetBelow}
   {customPopoverMaxHeight}
+  {loading}
 />

packages/bbui/src/Form/Core/Picker.svelte

@@ -2,7 +2,7 @@
   import "@spectrum-css/picker/dist/index-vars.css"
   import "@spectrum-css/popover/dist/index-vars.css"
   import "@spectrum-css/menu/dist/index-vars.css"
-  import { createEventDispatcher } from "svelte"
+  import { createEventDispatcher, onDestroy } from "svelte"
   import clickOutside from "../../Actions/click_outside"
   import Search from "./Search.svelte"
   import Icon from "../../Icon/Icon.svelte"
@@ -10,6 +10,7 @@
   import Popover from "../../Popover/Popover.svelte"
   import Tags from "../../Tags/Tags.svelte"
   import Tag from "../../Tags/Tag.svelte"
+  import ProgressCircle from "../../ProgressCircle/ProgressCircle.svelte"
 
   export let id = null
   export let disabled = false
@@ -35,19 +36,20 @@
   export let autoWidth = false
   export let autocomplete = false
   export let sort = false
-  export let fetchTerm = null
+  export let searchTerm = null
-  export let useFetch = false
   export let customPopoverHeight
   export let customPopoverOffsetBelow
   export let customPopoverMaxHeight
   export let align = "left"
   export let footer = null
   export let customAnchor = null
+  export let loading
+
   const dispatch = createEventDispatcher()
 
-  let searchTerm = null
   let button
   let popover
+  let component
 
   $: sortedOptions = getSortedOptions(options, getOptionLabel, sort)
   $: filteredOptions = getFilteredOptions(
@@ -82,7 +84,7 @@
   }
 
   const getFilteredOptions = (options, term, getLabel) => {
-    if (autocomplete && term && !fetchTerm) {
+    if (autocomplete && term) {
       const lowerCaseTerm = term.toLowerCase()
       return options.filter(option => {
         return `${getLabel(option)}`.toLowerCase().includes(lowerCaseTerm)
@@ -90,6 +92,20 @@
     }
     return options
   }
+
+  const onScroll = e => {
+    const scrollPxThreshold = 100
+    const scrollPositionFromBottom =
+      e.target.scrollHeight - e.target.clientHeight - e.target.scrollTop
+    if (scrollPositionFromBottom < scrollPxThreshold) {
+      dispatch("loadMore")
+    }
+  }
+
+  $: component?.addEventListener("scroll", onScroll)
+  onDestroy(() => {
+    component?.removeEventListener("scroll", null)
+  })
 </script>
 
 <button
@@ -163,14 +179,13 @@
   >
     {#if autocomplete}
       <Search
-        value={useFetch ? fetchTerm : searchTerm}
+        value={searchTerm}
-        on:change={event =>
+        on:change={event => (searchTerm = event.detail)}
-          useFetch ? (fetchTerm = event.detail) : (searchTerm = event.detail)}
         {disabled}
         placeholder="Search"
       />
     {/if}
-    <ul class="spectrum-Menu" role="listbox">
+    <ul class="spectrum-Menu" role="listbox" bind:this={component}>
       {#if placeholderOption}
         <li
           class="spectrum-Menu-item placeholder"
@@ -248,6 +263,12 @@
       {/if}
     </ul>
 
+    {#if loading}
+      <div class="loading" class:loading--withAutocomplete={autocomplete}>
+        <ProgressCircle size="S" />
+      </div>
+    {/if}
+
     {#if footer}
       <div class="footer">
         {footer}
@@ -323,18 +344,19 @@
   /* Search styles inside popover */
   .popover-content :global(.spectrum-Search) {
     margin-top: -1px;
-    margin-left: -1px;
+    width: 100%;
-    width: calc(100% + 2px);
   }
   .popover-content :global(.spectrum-Search input) {
     height: auto;
     border-bottom-left-radius: 0;
     border-bottom-right-radius: 0;
+    border-left: 0;
+    border-right: 0;
     padding-top: var(--spectrum-global-dimension-size-100);
     padding-bottom: var(--spectrum-global-dimension-size-100);
   }
   .popover-content :global(.spectrum-Search .spectrum-ClearButton) {
-    right: 1px;
+    right: 2px;
     top: 2px;
   }
   .popover-content :global(.spectrum-Search .spectrum-Textfield-icon) {
@@ -359,4 +381,14 @@
   .option-tag :global(.spectrum-Tags-item > .spectrum-Icon) {
     margin-top: 2px;
   }
+
+  .loading {
+    position: fixed;
+    justify-content: center;
+    right: var(--spacing-s);
+    top: var(--spacing-s);
+  }
+  .loading--withAutocomplete {
+    top: calc(34px + var(--spacing-m));
+  }
 </style>

packages/bbui/src/Form/Core/Select.svelte

@@ -25,6 +25,8 @@
   export let tag = null
   export let customPopoverOffsetBelow
   export let customPopoverMaxHeight
+  export let searchTerm = null
+  export let loading
 
   const dispatch = createEventDispatcher()
 
@@ -65,6 +67,8 @@
 <Picker
   on:click
   bind:open
+  bind:searchTerm
+  on:loadMore
   {quiet}
   {id}
   {error}
@@ -92,4 +96,5 @@
   placeholderOption={placeholder === false ? null : placeholder}
   isOptionSelected={option => option === value}
   onSelectOption={selectOption}
+  {loading}
 />

packages/bbui/src/Form/Core/TextField.svelte

@@ -96,8 +96,8 @@
     {disabled}
     {readonly}
     {id}
-    value={value || ""}
+    value={value ?? ""}
-    placeholder={placeholder || ""}
+    placeholder={placeholder ?? ""}
     on:click
     on:blur
     on:focus

packages/bbui/src/Form/Multiselect.svelte

@@ -16,8 +16,7 @@
   export let sort = false
   export let autoWidth = false
   export let autocomplete = false
-  export let fetchTerm = null
+  export let searchTerm = null
-  export let useFetch = false
   export let customPopoverHeight
 
   const dispatch = createEventDispatcher()
@@ -41,8 +40,7 @@
     {autoWidth}
     {autocomplete}
     {customPopoverHeight}
-    bind:fetchTerm
+    bind:searchTerm
-    {useFetch}
     on:change={onChange}
     on:click
   />

packages/bbui/src/Notification/Notification.svelte

@@ -27,7 +27,11 @@
   <div class="spectrum-Toast-body" class:actionBody={!!action}>
     <div class="wrap spectrum-Toast-content">{message || ""}</div>
     {#if action}
-      <ActionButton quiet emphasized on:click={action}>
+      <ActionButton
+        quiet
+        emphasized
+        on:click={() => action(() => dispatch("dismiss"))}
+      >
         <div style="color: white; font-weight: 600;">{actionMessage}</div>
       </ActionButton>
     {/if}

packages/bbui/src/Notification/NotificationDisplay.svelte

@@ -8,7 +8,7 @@
 
 <Portal target=".modal-container">
   <div class="notifications">
-    {#each $notifications as { type, icon, message, id, dismissable, action, wide } (id)}
+    {#each $notifications as { type, icon, message, id, dismissable, action, actionMessage, wide } (id)}
       <div transition:fly={{ y: 30 }}>
         <Notification
           {type}
@@ -16,6 +16,7 @@
           {message}
           {dismissable}
           {action}
+          {actionMessage}
           {wide}
           on:dismiss={() => notifications.dismiss(id)}
         />

packages/bbui/src/Stores/banner.js

@@ -1,6 +1,7 @@
 import { writable } from "svelte/store"
 
 export const BANNER_TYPES = {
+  NEUTRAL: "neutral",
   INFO: "info",
   NEGATIVE: "negative",
   WARNING: "warning",

packages/bbui/src/Stores/notifications.js

@@ -27,7 +27,9 @@ export const createNotificationStore = () => {
       icon = "",
       autoDismiss = true,
       action = null,
+      actionMessage = null,
       wide = false,
+      dismissTimeout = NOTIFICATION_TIMEOUT,
     }
   ) => {
     if (block) {
@@ -44,14 +46,16 @@ export const createNotificationStore = () => {
           icon,
           dismissable: !autoDismiss,
           action,
+          actionMessage,
           wide,
+          dismissTimeout,
         },
       ]
     })
     if (autoDismiss) {
       const timeoutId = setTimeout(() => {
         dismissNotification(_id)
-      }, NOTIFICATION_TIMEOUT)
+      }, dismissTimeout)
       timeoutIds.add(timeoutId)
     }
   }

packages/bbui/src/Table/CellRenderer.svelte

@@ -25,6 +25,7 @@
     longform: StringRenderer,
     array: ArrayRenderer,
     internal: InternalRenderer,
+    bb_reference: RelationshipRenderer,
   }
   $: type = getType(schema)
   $: customRenderer = customRenderers?.find(x => x.column === schema?.name)

packages/bbui/src/Tooltip/AbsTooltip.svelte

@@ -126,8 +126,9 @@
     transition: top 130ms ease-out, left 130ms ease-out;
   }
   .spectrum-Tooltip-label {
-    text-overflow: ellipsis;
+    display: -webkit-box;
-    white-space: nowrap;
+    -webkit-line-clamp: 3;
+    -webkit-box-orient: vertical;
     overflow: hidden;
     font-size: 12px;
     font-weight: 600;

packages/builder/package.json

@@ -65,28 +65,26 @@
     "@fortawesome/free-brands-svg-icons": "^6.2.1",
     "@fortawesome/free-solid-svg-icons": "^6.2.1",
     "@sentry/browser": "5.19.1",
-    "@spectrum-css/accordion": "^3.0.24",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
     "codemirror": "^5.59.0",
-    "dayjs": "^1.11.2",
+    "dayjs": "^1.10.8",
     "downloadjs": "1.4.7",
     "fast-json-patch": "^3.1.1",
     "lodash": "4.17.21",
     "posthog-js": "^1.36.0",
     "remixicon": "2.5.0",
+    "sanitize-html": "^2.7.0",
     "shortid": "2.2.15",
     "svelte-dnd-action": "^0.9.8",
     "svelte-loading-spinners": "^0.1.1",
     "svelte-portal": "1.0.0",
-    "uuid": "8.3.1",
     "yup": "0.29.2"
   },
   "devDependencies": {
     "@babel/core": "^7.12.14",
     "@babel/plugin-transform-runtime": "^7.13.10",
     "@babel/preset-env": "^7.13.12",
-    "@babel/runtime": "^7.13.10",
     "@rollup/plugin-replace": "^2.4.2",
     "@roxi/routify": "2.18.5",
     "@sveltejs/vite-plugin-svelte": "1.0.1",
@@ -96,19 +94,10 @@
     "identity-obj-proxy": "^3.0.0",
     "jest": "29.6.2",
     "jsdom": "^21.1.1",
-    "mochawesome": "^7.1.3",
-    "mochawesome-merge": "^4.2.1",
-    "mochawesome-report-generator": "^6.2.0",
     "ncp": "^2.0.0",
-    "rimraf": "^3.0.2",
     "rollup": "^2.44.0",
-    "rollup-plugin-copy": "^3.4.0",
-    "start-server-and-test": "^1.12.1",
     "svelte": "^3.48.0",
     "svelte-jester": "^1.3.2",
-    "ts-node": "10.8.1",
-    "tsconfig-paths": "4.0.0",
-    "typescript": "4.7.3",
     "vite": "^3.0.8",
     "vite-plugin-static-copy": "^0.16.0",
     "vitest": "^0.29.2"

packages/builder/src/analytics/constants.js

@@ -2,6 +2,7 @@ export const Events = {
   COMPONENT_CREATED: "component:created",
   COMPONENT_UPDATED: "component:updated",
   APP_VIEW_PUBLISHED: "app:view_published",
+  BLOCK_EJECTED: "block:ejected",
 }
 
 export const EventSource = {

packages/builder/src/builderStore/store/automation/index.js

@@ -221,18 +221,6 @@ const automationActions = store => ({
     newAutomation.definition.steps.splice(blockIdx, 0, block)
     await store.actions.save(newAutomation)
   },
-  /**
-   * "rowControl" appears to be the name of the flag used to determine whether
-   * a certain automation block uses values or bindings as inputs
-   */
-  toggleRowControl: async (block, rowControl) => {
-    const newBlock = { ...block, rowControl }
-    const newAutomation = store.actions.getUpdatedDefinition(
-      get(selectedAutomation),
-      newBlock
-    )
-    await store.actions.save(newAutomation)
-  },
   deleteAutomationBlock: async block => {
     const automation = get(selectedAutomation)
     let newAutomation = cloneDeep(automation)

packages/builder/src/builderStore/store/frontend.js

@@ -1287,6 +1287,11 @@ export const getFrontendStore = () => {
             return false
           }
 
+          // Log event
+          analytics.captureEvent(Events.BLOCK_EJECTED, {
+            block: block._component,
+          })
+
           // Attach block children back into ejected definition, using the
           // _containsSlot flag to know where to insert them
           const slotContainer = findAllMatchingComponents(

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -7,7 +7,6 @@
     Detail,
     Modal,
     Button,
-    Select,
     ActionButton,
     notifications,
     Label,
@@ -39,9 +38,6 @@
     step => step.stepId === ActionStepID.COLLECT
   )
   $: automationId = $selectedAutomation?._id
-  $: showBindingPicker =
-    block.stepId === ActionStepID.CREATE_ROW ||
-    block.stepId === ActionStepID.UPDATE_ROW
   $: isTrigger = block.type === "TRIGGER"
   $: steps = $selectedAutomation?.definition?.steps ?? []
   $: blockIdx = steps.findIndex(step => step.id === block.id)
@@ -96,15 +92,6 @@
     }
   }
 
-  /**
-   * "rowControl" appears to be the name of the flag used to determine whether
-   * a certain automation block uses values or bindings as inputs
-   */
-  function toggleRowControl(evt) {
-    const rowControl = evt.detail !== "Use values"
-    automationStore.actions.toggleRowControl(block, rowControl)
-  }
-
   async function addLooping() {
     const loopDefinition = $automationStore.blockDefinitions.ACTION.LOOP
     const loopBlock = automationStore.actions.constructBlock(
@@ -189,16 +176,6 @@
                   Add Looping
                 </ActionButton>
               {/if}
-              {#if showBindingPicker}
-                <Select
-                  on:change={toggleRowControl}
-                  defaultValue="Use values"
-                  autoWidth
-                  value={block.rowControl ? "Use bindings" : "Use values"}
-                  options={["Use values", "Use bindings"]}
-                  placeholder={null}
-                />
-              {/if}
               <ActionButton
                 on:click={() => deleteStep()}
                 icon="DeleteOutline"

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -23,6 +23,7 @@
   import { environment, licensing } from "stores/portal"
   import WebhookDisplay from "../Shared/WebhookDisplay.svelte"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
+  import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import CodeEditorModal from "./CodeEditorModal.svelte"
   import QuerySelector from "./QuerySelector.svelte"
@@ -82,33 +83,6 @@
       ? [hbAutocomplete([...bindingsToCompletions(bindings, codeMode)])]
       : []
 
-  /**
-   * TODO - Remove after November 2023
-   * *******************************
-   * Code added to provide backwards compatibility between Values 1,2,3,4,5
-   * and the new JSON body.
-   */
-  let deprecatedSchemaProperties
-  $: {
-    if (block?.stepId === "integromat" || block?.stepId === "zapier") {
-      deprecatedSchemaProperties = schemaProperties.filter(
-        prop => !prop[0].startsWith("value")
-      )
-      if (!deprecatedSchemaProperties.map(entry => entry[0]).includes("body")) {
-        deprecatedSchemaProperties.push([
-          "body",
-          {
-            title: "Payload",
-            type: "json",
-          },
-        ])
-      }
-    } else {
-      deprecatedSchemaProperties = schemaProperties
-    }
-  }
-  /****************************************************/
-
   const getInputData = (testData, blockInputs) => {
     // Test data is not cloned for reactivity
     let newInputData = testData || cloneDeep(blockInputs)
@@ -118,30 +92,6 @@
       newInputData = cloneDeep(blockInputs)
     }
 
-    /**
-     * TODO - Remove after November 2023
-     * *******************************
-     * Code added to provide backwards compatibility between Values 1,2,3,4,5
-     * and the new JSON body.
-     */
-    if (
-      (block?.stepId === "integromat" || block?.stepId === "zapier") &&
-      !newInputData?.body?.value
-    ) {
-      let deprecatedValues = {
-        ...newInputData,
-      }
-      delete deprecatedValues.url
-      delete deprecatedValues.body
-      newInputData = {
-        url: newInputData.url,
-        body: {
-          value: JSON.stringify(deprecatedValues),
-        },
-      }
-    }
-    /**********************************/
-
     inputData = newInputData
     setDefaultEnumValues()
   }
@@ -337,7 +287,7 @@
 </script>
 
 <div class="fields">
-  {#each deprecatedSchemaProperties as [key, value]}
+  {#each schemaProperties as [key, value]}
     {#if canShowField(key, value)}
       <div class="block-field">
         {#if key !== "fields" && value.type !== "boolean"}
@@ -362,18 +312,6 @@
             mode="json"
             value={inputData[key]?.value}
             on:change={e => {
-              /**
-               * TODO - Remove after November 2023
-               * *******************************
-               * Code added to provide backwards compatibility between Values 1,2,3,4,5
-               * and the new JSON body.
-               */
-              delete inputData.value1
-              delete inputData.value2
-              delete inputData.value3
-              delete inputData.value4
-              delete inputData.value5
-              /***********************/
               onChange(e, key)
             }}
           />
@@ -386,10 +324,23 @@
             />
           </div>
         {:else if value.type === "date"}
-          <DatePicker
+          <DrawerBindableSlot
+            fillWidth
+            title={value.title}
+            panel={AutomationBindingPanel}
+            type={"date"}
             value={inputData[key]}
             on:change={e => onChange(e, key)}
-          />
+            {bindings}
+            allowJS={true}
+            updateOnChange={false}
+            drawerLeft="260px"
+          >
+            <DatePicker
+              value={inputData[key]}
+              on:change={e => onChange(e, key)}
+            />
+          </DrawerBindableSlot>
         {:else if value.customType === "column"}
           <Select
             on:change={e => onChange(e, key)}
@@ -469,7 +420,6 @@
           />
         {:else if value.customType === "row"}
           <RowSelector
-            {block}
             value={inputData[key]}
             meta={inputData["meta"] || {}}
             on:change={e => {

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -1,18 +1,14 @@
 <script>
   import { tables } from "stores/backend"
   import { Select, Checkbox } from "@budibase/bbui"
-  import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
-  import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
   import RowSelectorTypes from "./RowSelectorTypes.svelte"
-  import ModalBindableInput from "../../common/bindings/ModalBindableInput.svelte"
 
   const dispatch = createEventDispatcher()
 
   export let value
   export let meta
   export let bindings
-  export let block
   export let isTestModal
   export let isUpdateRow
 
@@ -25,16 +21,6 @@
   let table
   let schemaFields
 
-  let placeholders = {
-    number: 10,
-    boolean: "true",
-    datetime: "2022-02-16T12:00:00.000Z ",
-    options: "1",
-    array: "1 2 3 4",
-    link: "ro_ta_123_456",
-    longform: "long form text",
-  }
-  $: rowControl = block.rowControl
   $: {
     table = $tables.list.find(table => table._id === value?.tableId)
     schemaFields = Object.entries(table?.schema ?? {})
@@ -57,19 +43,13 @@
       return value
     }
 
-    if (type === "boolean") {
-      if (typeof value === "boolean") {
-        return value
-      }
-      return value === "true"
-    }
     if (type === "number") {
       if (typeof value === "number") {
         return value
       }
       return Number(value)
     }
-    if (type === "options") {
+    if (type === "options" || type === "boolean") {
       return value
     }
     if (type === "array") {
@@ -83,8 +63,7 @@
       if (Array.isArray(value)) {
         return value
       }
-
+      return value.split(",").map(x => x.trim())
-      return [value]
     }
 
     if (type === "json") {
@@ -128,47 +107,25 @@
 {#if schemaFields.length}
   <div class="schema-fields">
     {#each schemaFields as [field, schema]}
-      {#if !schema.autocolumn}
+      {#if !schema.autocolumn && schema.type !== "attachment"}
-        {#if schema.type !== "attachment"}
+        <RowSelectorTypes
-          {#if !rowControl}
+          {isTestModal}
-            <RowSelectorTypes
+          {field}
-              {isTestModal}
+          {schema}
-              {field}
+          bindings={parsedBindings}
-              {schema}
+          {value}
-              bindings={parsedBindings}
+          {onChange}
-              {value}
+        />
-              {onChange}
+      {/if}
-            />
+      {#if isUpdateRow && schema.type === "link"}
-          {:else}
+        <div class="checkbox-field">
-            <div>
+          <Checkbox
-              <svelte:component
+            value={meta.fields?.[field]?.clearRelationships}
-                this={isTestModal ? ModalBindableInput : DrawerBindableInput}
+            text={"Clear relationships if empty?"}
-                placeholder={placeholders[schema.type]}
+            size={"S"}
-                panel={AutomationBindingPanel}
+            on:change={e => onChangeSetting(e, field)}
-                value={Array.isArray(value[field])
+          />
-                  ? value[field].join(" ")
+        </div>
-                  : value[field]}
-                on:change={e => onChange(e, field, schema.type)}
-                label={field}
-                type="string"
-                bindings={parsedBindings}
-                fillWidth={true}
-                allowJS={true}
-                updateOnChange={false}
-              />
-              {#if isUpdateRow && schema.type === "link"}
-                <div class="checkbox-field">
-                  <Checkbox
-                    value={meta.fields?.[field]?.clearRelationships}
-                    text={"Clear relationships if empty?"}
-                    size={"S"}
-                    on:change={e => onChangeSetting(e, field)}
-                  />
-                </div>
-              {/if}
-            </div>
-          {/if}
-        {/if}
       {/if}
     {/each}
   </div>

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -1,7 +1,6 @@
 <script>
   import {
     Select,
-    Toggle,
     DatePicker,
     Multiselect,
     TextArea,
@@ -9,6 +8,7 @@
   } from "@budibase/bbui"
   import LinkedRowSelector from "components/common/LinkedRowSelector.svelte"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
+  import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
   import ModalBindableInput from "../../common/bindings/ModalBindableInput.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import Editor from "components/integration/QueryEditor.svelte"
@@ -31,60 +31,88 @@
   }
 </script>
 
-{#if schemaHasOptions(schema) && schema.type !== "array"}
+<DrawerBindableSlot
-  <Select
+  fillWidth
-    on:change={e => onChange(e, field)}
+  title={value.title}
-    label={field}
+  label={field}
-    value={value[field]}
+  panel={AutomationBindingPanel}
-    options={schema.constraints.inclusion}
+  type={schema.type}
-  />
+  {schema}
-{:else if schema.type === "datetime"}
+  value={value[field]}
-  <DatePicker
+  on:change={e => onChange(e, field)}
-    label={field}
+  {bindings}
-    value={value[field]}
+  allowJS={true}
-    on:change={e => onChange(e, field)}
+  updateOnChange={false}
-  />
+  drawerLeft="260px"
-{:else if schema.type === "boolean"}
+>
-  <Toggle
+  {#if schemaHasOptions(schema) && schema.type !== "array"}
-    text={field}
+    <Select
-    value={value[field]}
+      on:change={e => onChange(e, field)}
-    on:change={e => onChange(e, field)}
+      label={field}
-  />
-{:else if schema.type === "array"}
-  <Multiselect
-    bind:value={value[field]}
-    label={field}
-    options={schema.constraints.inclusion}
-  />
-{:else if schema.type === "longform"}
-  <TextArea label={field} bind:value={value[field]} />
-{:else if schema.type === "json"}
-  <span>
-    <Label>{field}</Label>
-    <Editor
-      editorHeight="150"
-      mode="json"
-      on:change={e => {
-        if (e.detail?.value !== value[field]) {
-          onChange(e, field, schema.type)
-        }
-      }}
       value={value[field]}
+      options={schema.constraints.inclusion}
     />
-  </span>
+  {:else if schema.type === "datetime"}
-{:else if schema.type === "link"}
+    <DatePicker
-  <LinkedRowSelector bind:linkedRows={value[field]} {schema} />
+      label={field}
-{:else if schema.type === "string" || schema.type === "number"}
+      value={value[field]}
-  <svelte:component
+      on:change={e => onChange(e, field)}
-    this={isTestModal ? ModalBindableInput : DrawerBindableInput}
+    />
-    panel={AutomationBindingPanel}
+  {:else if schema.type === "boolean"}
-    value={value[field]}
+    <Select
-    on:change={e => onChange(e, field)}
+      on:change={e => onChange(e, field)}
-    label={field}
+      label={field}
-    type="string"
+      value={value[field]}
-    bindings={parsedBindings}
+      options={[
-    fillWidth={true}
+        { label: "True", value: "true" },
-    allowJS={true}
+        { label: "False", value: "false" },
-    updateOnChange={false}
+      ]}
-  />
+    />
-{/if}
+  {:else if schema.type === "array"}
+    <Multiselect
+      bind:value={value[field]}
+      label={field}
+      options={schema.constraints.inclusion}
+      on:change={e => onChange(e, field)}
+    />
+  {:else if schema.type === "longform"}
+    <TextArea
+      label={field}
+      bind:value={value[field]}
+      on:change={e => onChange(e, field)}
+    />
+  {:else if schema.type === "json"}
+    <span>
+      <Label>{field}</Label>
+      <Editor
+        editorHeight="150"
+        mode="json"
+        on:change={e => {
+          if (e.detail?.value !== value[field]) {
+            onChange(e, field, schema.type)
+          }
+        }}
+        value={value[field]}
+      />
+    </span>
+  {:else if schema.type === "link"}
+    <LinkedRowSelector
+      bind:linkedRows={value[field]}
+      {schema}
+      on:change={e => onChange(e, field)}
+    />
+  {:else if schema.type === "string" || schema.type === "number"}
+    <svelte:component
+      this={isTestModal ? ModalBindableInput : DrawerBindableInput}
+      panel={AutomationBindingPanel}
+      value={value[field]}
+      on:change={e => onChange(e, field)}
+      label={field}
+      type="string"
+      bindings={parsedBindings}
+      fillWidth={true}
+      allowJS={true}
+      updateOnChange={false}
+    />
+  {/if}
+</DrawerBindableSlot>

packages/builder/src/components/backend/DataTable/ExternalDataSourceTable.svelte

@@ -1,33 +0,0 @@
-<script>
-  import Table from "./Table.svelte"
-
-  export let query = {}
-  export let data = []
-  export let editRows = false
-
-  let loading = false
-  let error = false
-  let type = "external"
-</script>
-
-{#if error}
-  <div class="errors">{error}</div>
-{/if}
-<Table
-  schema={query.schema}
-  {data}
-  {loading}
-  {type}
-  rowCount={5}
-  allowEditing={editRows}
-/>
-
-<style>
-  .errors {
-    color: var(--red);
-    background: var(--red-light);
-    padding: var(--spacing-m);
-    border-radius: var(--border-radius-m);
-    margin-bottom: var(--spacing-m);
-  }
-</style>

packages/builder/src/components/backend/DataTable/TableDataTable.svelte

@@ -14,13 +14,14 @@
   import GridManageAccessButton from "components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte"
   import GridRelationshipButton from "components/backend/DataTable/buttons/grid/GridRelationshipButton.svelte"
   import GridEditColumnModal from "components/backend/DataTable/modals/grid/GridEditColumnModal.svelte"
+  import GridUsersTableButton from "components/backend/DataTable/modals/grid/GridUsersTableButton.svelte"
 
   const userSchemaOverrides = {
-    firstName: { displayName: "First name", disabled: true },
+    firstName: { displayName: "First name" },
-    lastName: { displayName: "Last name", disabled: true },
+    lastName: { displayName: "Last name" },
-    email: { displayName: "Email", disabled: true },
+    email: { displayName: "Email" },
-    roleId: { displayName: "Role", disabled: true },
+    roleId: { displayName: "Role" },
-    status: { displayName: "Status", disabled: true },
+    status: { displayName: "Status" },
   }
 
   $: id = $tables.selected?._id
@@ -59,22 +60,22 @@
     datasource={gridDatasource}
     canAddRows={!isUsersTable}
     canDeleteRows={!isUsersTable}
+    canEditRows={!isUsersTable}
+    canEditColumns={!isUsersTable}
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
     on:updatedatasource={handleGridTableUpdate}
   >
     <svelte:fragment slot="filter">
+      {#if isUsersTable}
+        <GridUsersTableButton />
+      {/if}
       <GridFilterButton />
     </svelte:fragment>
-    <svelte:fragment slot="edit-column">
-      <GridEditColumnModal />
-    </svelte:fragment>
-    <svelte:fragment slot="add-column">
-      <GridAddColumnModal />
-    </svelte:fragment>
-
     <svelte:fragment slot="controls">
-      <GridCreateViewButton />
+      {#if !isUsersTable}
+        <GridCreateViewButton />
+      {/if}
       <GridManageAccessButton />
       {#if relationshipsEnabled}
         <GridRelationshipButton />
@@ -84,7 +85,6 @@
       {:else}
         <GridImportButton />
       {/if}
-
       <GridExportButton />
       {#if isUsersTable}
         <GridEditUserModal />
@@ -92,6 +92,12 @@
         <GridCreateEditRowModal />
       {/if}
     </svelte:fragment>
+    <svelte:fragment slot="edit-column">
+      <GridEditColumnModal />
+    </svelte:fragment>
+    <svelte:fragment slot="add-column">
+      <GridAddColumnModal />
+    </svelte:fragment>
   </Grid>
 </div>
 

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -5,7 +5,6 @@
     Label,
     Select,
     Toggle,
-    RadioGroup,
     Icon,
     DatePicker,
     Modal,
@@ -26,14 +25,16 @@
     ALLOWABLE_STRING_TYPES,
     ALLOWABLE_NUMBER_TYPES,
     SWITCHABLE_TYPES,
+    PrettyRelationshipDefinitions,
   } from "constants/backend"
   import { getAutoColumnInformation, buildAutoColumn } from "builderStore/utils"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
-  import { truncate } from "lodash"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import { getBindings } from "components/backend/DataTable/formula"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
   import { ValidColumnNameRegex } from "@budibase/shared-core"
+  import { FieldType } from "@budibase/types"
+  import RelationshipSelector from "components/common/RelationshipSelector.svelte"
 
   const AUTO_TYPE = "auto"
   const FORMULA_TYPE = FIELDS.FORMULA.type
@@ -42,6 +43,7 @@
   const NUMBER_TYPE = FIELDS.NUMBER.type
   const JSON_TYPE = FIELDS.JSON.type
   const DATE_TYPE = FIELDS.DATETIME.type
+  const USER_REFRENCE_TYPE = FIELDS.BB_REFERENCE_USER.compositeType
 
   const dispatch = createEventDispatcher()
   const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
@@ -57,6 +59,12 @@
   let indexes = [...($tables.selected.indexes || [])]
   let isCreating = undefined
 
+  let relationshipPart1 = PrettyRelationshipDefinitions.Many
+  let relationshipPart2 = PrettyRelationshipDefinitions.One
+
+  let relationshipTableIdPrimary = null
+  let relationshipTableIdSecondary = null
+
   let table = $tables.selected
   let confirmDeleteDialog
   let savingColumn
@@ -64,18 +72,65 @@
   let jsonSchemaModal
   let allowedTypes = []
   let editableColumn = {
-    type: "string",
+    type: fieldDefinitions.STRING.type,
     constraints: fieldDefinitions.STRING.constraints,
     // Initial value for column name in other table for linked records
     fieldName: $tables.selected.name,
   }
+  let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions)
+  let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions)
 
   $: if (primaryDisplay) {
     editableColumn.constraints.presence = { allowEmpty: false }
   }
 
+  let relationshipMap = {
+    [RelationshipType.ONE_TO_MANY]: {
+      part1: PrettyRelationshipDefinitions.MANY,
+      part2: PrettyRelationshipDefinitions.ONE,
+    },
+    [RelationshipType.MANY_TO_MANY]: {
+      part1: PrettyRelationshipDefinitions.MANY,
+      part2: PrettyRelationshipDefinitions.MANY,
+    },
+    [RelationshipType.MANY_TO_ONE]: {
+      part1: PrettyRelationshipDefinitions.ONE,
+      part2: PrettyRelationshipDefinitions.MANY,
+    },
+  }
+
+  $: {
+    // this parses any changes the user has made when creating a new internal relationship
+    // into what we expect the schema to look like
+    if (editableColumn.type === LINK_TYPE) {
+      relationshipTableIdPrimary = table._id
+      if (relationshipPart1 === PrettyRelationshipDefinitions.ONE) {
+        relationshipOpts2 = relationshipOpts2.filter(
+          opt => opt !== PrettyRelationshipDefinitions.ONE
+        )
+      } else {
+        relationshipOpts2 = Object.values(PrettyRelationshipDefinitions)
+      }
+
+      if (relationshipPart2 === PrettyRelationshipDefinitions.ONE) {
+        relationshipOpts1 = relationshipOpts1.filter(
+          opt => opt !== PrettyRelationshipDefinitions.ONE
+        )
+      } else {
+        relationshipOpts1 = Object.values(PrettyRelationshipDefinitions)
+      }
+      // Determine the relationship type based on the selected values of both parts
+      editableColumn.relationshipType = Object.entries(relationshipMap).find(
+        ([_, parts]) =>
+          parts.part1 === relationshipPart1 && parts.part2 === relationshipPart2
+      )?.[0]
+      // Set the tableId based on the selected table
+      editableColumn.tableId = relationshipTableIdSecondary
+    }
+  }
   const initialiseField = (field, savingColumn) => {
     isCreating = !field
+
     if (field && !savingColumn) {
       editableColumn = cloneDeep(field)
       originalName = editableColumn.name ? editableColumn.name + "" : null
@@ -83,6 +138,23 @@
       primaryDisplay =
         $tables.selected.primaryDisplay == null ||
         $tables.selected.primaryDisplay === editableColumn.name
+
+      if (editableColumn.type === FieldType.BB_REFERENCE) {
+        editableColumn.type = `${editableColumn.type}_${editableColumn.subtype}`
+      }
+      // Here we are setting the relationship values based on the editableColumn
+      // This part of the code is used when viewing an existing field hence the check
+      // for the tableId
+      if (editableColumn.type === LINK_TYPE && editableColumn.tableId) {
+        relationshipTableIdPrimary = table._id
+        relationshipTableIdSecondary = editableColumn.tableId
+        if (editableColumn.relationshipType in relationshipMap) {
+          const { part1, part2 } =
+            relationshipMap[editableColumn.relationshipType]
+          relationshipPart1 = part1
+          relationshipPart2 = part2
+        }
+      }
     } else if (!savingColumn) {
       let highestNumber = 0
       Object.keys(table.schema).forEach(columnName => {
@@ -99,6 +171,7 @@
         editableColumn.name = "Column 01"
       }
     }
+
     allowedTypes = getAllowedTypes()
   }
 
@@ -157,14 +230,10 @@
     !uneditable &&
     editableColumn?.type !== AUTO_TYPE &&
     !editableColumn.autocolumn
-  $: relationshipOptions = getRelationshipOptions(editableColumn)
   $: external = table.type === "external"
   // in the case of internal tables the sourceId will just be undefined
   $: tableOptions = $tables.list.filter(
-    opt =>
+    opt => opt.type === table.type && table.sourceId === opt.sourceId
-      opt._id !== $tables.selected._id &&
-      opt.type === table.type &&
-      table.sourceId === opt.sourceId
   )
   $: typeEnabled =
     !originalName ||
@@ -180,6 +249,14 @@
 
     let saveColumn = cloneDeep(editableColumn)
 
+    // Handle types on composite types
+    const definition = fieldDefinitions[saveColumn.type.toUpperCase()]
+    if (definition && saveColumn.type === definition.compositeType) {
+      saveColumn.type = definition.type
+      saveColumn.subtype = definition.subtype
+      delete saveColumn.compositeType
+    }
+
     if (saveColumn.type === AUTO_TYPE) {
       saveColumn = buildAutoColumn(
         $tables.selected.name,
@@ -258,9 +335,10 @@
     // Default relationships many to many
     if (editableColumn.type === LINK_TYPE) {
       editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
-    }
+    } else if (editableColumn.type === FORMULA_TYPE) {
-    if (editableColumn.type === FORMULA_TYPE) {
       editableColumn.formulaType = "dynamic"
+    } else if (editableColumn.type === USER_REFRENCE_TYPE) {
+      editableColumn.relationshipType = RelationshipType.ONE_TO_MANY
     }
   }
 
@@ -288,35 +366,6 @@
     return match ? parseInt(match[1]) : 0
   }
 
-  function getRelationshipOptions(field) {
-    if (!field || !field.tableId) {
-      return null
-    }
-    const linkTable = tableOptions?.find(table => table._id === field.tableId)
-    if (!linkTable) {
-      return null
-    }
-    const thisName = truncate(table.name, { length: 14 }),
-      linkName = truncate(linkTable.name, { length: 14 })
-    return [
-      {
-        name: `Many ${thisName} rows → many ${linkName} rows`,
-        alt: `Many ${table.name} rows → many ${linkTable.name} rows`,
-        value: RelationshipType.MANY_TO_MANY,
-      },
-      {
-        name: `One ${linkName} row → many ${thisName} rows`,
-        alt: `One ${linkTable.name} rows → many ${table.name} rows`,
-        value: RelationshipType.ONE_TO_MANY,
-      },
-      {
-        name: `One ${thisName} row → many ${linkName} rows`,
-        alt: `One ${table.name} rows → many ${linkTable.name} rows`,
-        value: RelationshipType.MANY_TO_ONE,
-      },
-    ]
-  }
-
   function getAllowedTypes() {
     if (
       originalName &&
@@ -328,7 +377,9 @@
       ALLOWABLE_NUMBER_TYPES.indexOf(editableColumn.type) !== -1
     ) {
       return ALLOWABLE_NUMBER_OPTIONS
-    } else if (!external) {
+    }
+
+    if (!external) {
       return [
         ...Object.values(fieldDefinitions),
         { name: "Auto Column", type: AUTO_TYPE },
@@ -344,6 +395,7 @@
         FIELDS.BOOLEAN,
         FIELDS.FORMULA,
         FIELDS.BIGINT,
+        FIELDS.BB_REFERENCE_USER,
       ]
       // no-sql or a spreadsheet
       if (!external || table.sql) {
@@ -357,8 +409,9 @@
     if (!fieldToCheck) {
       return
     }
+
     // most types need this, just make sure its always present
-    if (fieldToCheck && !fieldToCheck.constraints) {
+    if (!fieldToCheck.constraints) {
       fieldToCheck.constraints = {}
     }
     // some string types may have been built by server, may not always have constraints
@@ -438,7 +491,7 @@
     on:change={handleTypeChange}
     options={allowedTypes}
     getOptionLabel={field => field.name}
-    getOptionValue={field => field.type}
+    getOptionValue={field => field.compositeType || field.type}
     getOptionIcon={field => field.icon}
     isOptionEnabled={option => {
       if (option.type == AUTO_TYPE) {
@@ -500,9 +553,9 @@
         <DatePicker bind:value={editableColumn.constraints.datetime.latest} />
       </div>
     </div>
-    {#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER"}
+    {#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER" && !editableColumn.dateOnly}
       <div>
-        <div>
+        <div class="row">
           <Label>Time zones</Label>
           <AbsTooltip
             position="top"
@@ -520,10 +573,11 @@
         />
       </div>
     {/if}
+    <Toggle bind:value={editableColumn.dateOnly} text="Date only" />
   {:else if editableColumn.type === "number" && !editableColumn.autocolumn}
     <div class="split-label">
       <div class="label-length">
-        <Label size="M">Max Value</Label>
+        <Label size="M">Min Value</Label>
       </div>
       <div class="input-length">
         <Input
@@ -546,30 +600,17 @@
       </div>
     </div>
   {:else if editableColumn.type === "link"}
-    <Select
+    <RelationshipSelector
-      label="Table"
+      bind:relationshipPart1
-      disabled={linkEditDisabled}
+      bind:relationshipPart2
-      bind:value={editableColumn.tableId}
+      bind:relationshipTableIdPrimary
-      options={tableOptions}
+      bind:relationshipTableIdSecondary
-      getOptionLabel={table => table.name}
+      bind:editableColumn
-      getOptionValue={table => table._id}
+      {relationshipOpts1}
-    />
+      {relationshipOpts2}
-    {#if relationshipOptions && relationshipOptions.length > 0}
+      {linkEditDisabled}
-      <RadioGroup
+      {tableOptions}
-        disabled={linkEditDisabled}
+      {errors}
-        label="Define the relationship"
-        bind:value={editableColumn.relationshipType}
-        options={relationshipOptions}
-        getOptionLabel={option => option.name}
-        getOptionValue={option => option.value}
-        getOptionTitle={option => option.alt}
-      />
-    {/if}
-    <Input
-      disabled={linkEditDisabled}
-      label={`Column name in other table`}
-      bind:value={editableColumn.fieldName}
-      error={errors.relatedName}
     />
   {:else if editableColumn.type === FORMULA_TYPE}
     {#if !table.sql}
@@ -616,6 +657,17 @@
     <Button primary text on:click={openJsonSchemaEditor}
       >Open schema editor</Button
     >
+  {:else if editableColumn.type === USER_REFRENCE_TYPE}
+    <Toggle
+      value={editableColumn.relationshipType === RelationshipType.MANY_TO_MANY}
+      on:change={e =>
+        (editableColumn.relationshipType = e.detail
+          ? RelationshipType.MANY_TO_MANY
+          : RelationshipType.ONE_TO_MANY)}
+      disabled={!isCreating}
+      thin
+      text="Allow multiple users"
+    />
   {/if}
   {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
     <Select
@@ -690,18 +742,19 @@
     display: flex;
     align-items: center;
   }
-
   .tooltip-alignment {
     display: flex;
     align-items: center;
     gap: var(--spacing-xs);
   }
-
   .label-length {
     flex-basis: 40%;
   }
-
   .input-length {
     flex-grow: 1;
   }
+  .row {
+    gap: 8px;
+    display: flex;
+  }
 </style>

packages/builder/src/components/backend/DataTable/modals/grid/GridUsersTableButton.svelte

@@ -0,0 +1,54 @@
+<script>
+  import { ActionButton, Popover, Heading, Body, Button } from "@budibase/bbui"
+  import { store } from "builderStore"
+
+  let anchor
+  let open = false
+
+  const openSidePanel = () => {
+    store.update(state => ({
+      ...state,
+      builderSidePanel: true,
+    }))
+    open = false
+  }
+</script>
+
+<div bind:this={anchor}>
+  <ActionButton on:click={() => (open = true)} icon="Help" quiet>
+    Why can't I edit this table?
+  </ActionButton>
+</div>
+
+<Popover bind:open {anchor} align="left">
+  <div class="content">
+    <Heading size="XS">The app users table is read only</Heading>
+    <Body size="S">
+      You can continue to view the users that have access to your application.
+    </Body>
+    <Body size="S">
+      Manage and invite more application users using the user side panel in the
+      top right of your screen.
+    </Body>
+    <div class="button">
+      <Button cta on:click={openSidePanel}>Open users panel</Button>
+    </div>
+  </div>
+</Popover>
+
+<style>
+  .content {
+    width: 300px;
+    padding: var(--spacing-l);
+    display: flex;
+    flex-direction: column;
+    gap: var(--spacing-l);
+  }
+  .content :global(.spectrum-Heading) {
+    font-weight: 400;
+  }
+  .button {
+    display: flex;
+    justify-content: flex-end;
+  }
+</style>

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -180,7 +180,7 @@
   <div class="hierarchy-items-container">
     <NavItem
       icon="UserGroup"
-      text="Users"
+      text="App users"
       selected={$isActive("./table/:tableId") &&
         $tables.selected?._id === TableNames.USERS}
       on:click={() => selectTable(TableNames.USERS)}

packages/builder/src/components/common/LinkedRowSelector.svelte

@@ -3,16 +3,19 @@
   import { API } from "api"
   import { Select, Label, Multiselect } from "@budibase/bbui"
   import { capitalise } from "../../helpers"
+  import { createEventDispatcher } from "svelte"
 
   export let schema
   export let linkedRows = []
 
+  const dispatch = createEventDispatcher()
+
   let rows = []
-  let linkedIds = (Array.isArray(linkedRows) ? linkedRows : [])?.map(
+  let linkedIds = []
+
+  $: linkedIds = (Array.isArray(linkedRows) ? linkedRows : [])?.map(
     row => row?._id || row
   )
-
-  $: linkedRows = linkedIds
   $: label = capitalise(schema.name)
   $: linkedTableId = schema.tableId
   $: linkedTable = $tables.list.find(table => table._id === linkedTableId)
@@ -44,7 +47,10 @@
     options={rows}
     getOptionLabel={getPrettyName}
     getOptionValue={row => row._id}
-    on:change={e => (linkedIds = e.detail ? [e.detail] : [])}
+    on:change={e => {
+      linkedIds = e.detail ? [e.detail] : []
+      dispatch("change", linkedIds)
+    }}
     {label}
     sort
   />
@@ -56,5 +62,6 @@
     getOptionLabel={getPrettyName}
     getOptionValue={row => row._id}
     sort
+    on:change={() => dispatch("change", linkedIds)}
   />
 {/if}

packages/builder/src/components/common/RelationshipSelector.svelte

@@ -0,0 +1,74 @@
+<script>
+  import { Select, Input } from "@budibase/bbui"
+
+  export let relationshipPart1
+  export let relationshipPart2
+  export let relationshipTableIdPrimary
+  export let relationshipTableIdSecondary
+  export let editableColumn
+  export let linkEditDisabled
+  export let tableOptions
+  export let errors
+  export let relationshipOpts1
+  export let relationshipOpts2
+</script>
+
+<div class="relationship-container">
+  <div class="relationship-part">
+    <Select
+      disabled={linkEditDisabled}
+      bind:value={relationshipPart1}
+      options={relationshipOpts1}
+    />
+  </div>
+  <div class="relationship-label">in</div>
+  <div class="relationship-part">
+    <Select
+      disabled
+      options={tableOptions}
+      getOptionLabel={table => table.name}
+      getOptionValue={table => table._id}
+      bind:value={relationshipTableIdPrimary}
+    />
+  </div>
+</div>
+<div class="relationship-container">
+  <div class="relationship-part">
+    <Select
+      disabled={linkEditDisabled}
+      bind:value={relationshipPart2}
+      options={relationshipOpts2}
+      getOptionLabel={option => "To " + option.toLowerCase()}
+    />
+  </div>
+  <div class="relationship-label">in</div>
+  <div class="relationship-part">
+    <Select
+      disabled={linkEditDisabled}
+      bind:value={relationshipTableIdSecondary}
+      options={tableOptions.filter(
+        table => table._id !== relationshipTableIdPrimary
+      )}
+      getOptionLabel={table => table.name}
+      getOptionValue={table => table._id}
+    />
+  </div>
+</div>
+<Input
+  disabled={linkEditDisabled}
+  label={`Column name in other table`}
+  bind:value={editableColumn.fieldName}
+  error={errors.relatedName}
+/>
+
+<style>
+  .relationship-container {
+    display: flex;
+    align-items: center;
+    gap: 20px;
+  }
+
+  .relationship-part {
+    flex-basis: 60%;
+  }
+</style>

packages/builder/src/components/common/UndoRedoControl.svelte

@@ -38,14 +38,12 @@
     hoverable
     on:click={store.undo}
     disabled={!$store.canUndo}
-    tooltip="Undo latest change"
   />
   <Icon
     name="Redo"
     hoverable
     on:click={store.redo}
     disabled={!$store.canRedo}
-    tooltip="Redo latest undo"
   />
 </div>
 

packages/builder/src/components/common/VerificationPromptBanner.svelte

@@ -0,0 +1,102 @@
+<script>
+  import { notifications, Button, Icon, Body } from "@budibase/bbui"
+  import { admin, auth } from "stores/portal"
+
+  $: user = $auth.user
+  let loading = false
+  let complete = false
+
+  const resetPassword = async () => {
+    if (loading || complete) return
+    loading = true
+
+    try {
+      await fetch(`${$admin.accountPortalUrl}/api/auth/reset`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({ email: user.email }),
+      })
+
+      complete = true
+    } catch (e) {
+      notifications.error("There was an issue sending your validation email.")
+    } finally {
+      loading = false
+    }
+  }
+</script>
+
+{#if user?.account?.verified === false}
+  <section class="banner">
+    <div class="icon">
+      <Icon name="Info" />
+    </div>
+    <div class="copy">
+      <Body size="S">
+        Please verify your account. We've sent the verification link to <span
+          class="email">{user.email}</span
+        ></Body
+      >
+    </div>
+    <div class="button" class:disabled={loading || complete}>
+      <Button on:click={resetPassword}
+        >{complete ? "Email sent" : "Resend email"}</Button
+      >
+    </div>
+  </section>
+{/if}
+
+<style>
+  .banner {
+    flex-shrink: 0;
+    display: flex;
+    background-color: var(--grey-2);
+    height: 48px;
+    align-items: center;
+    padding: 0 15px;
+  }
+
+  .icon {
+    margin-right: 15px;
+    color: var(--ink);
+    display: flex;
+  }
+
+  .copy {
+    flex-grow: 1;
+    overflow: hidden;
+  }
+
+  .copy :global(p) {
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
+
+  .email {
+    font-weight: 600;
+  }
+
+  .button {
+    margin-left: 15px;
+  }
+
+  .button :global(button) {
+    color: var(--ink);
+    background-color: var(--grey-3);
+    border: none;
+  }
+
+  .button :global(button):hover {
+    background-color: var(--grey-4);
+  }
+
+  .disabled :global(button) {
+    pointer-events: none;
+    color: var(--ink);
+    background-color: var(--grey-4);
+    border: none;
+  }
+</style>

packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte

@@ -0,0 +1,250 @@
+<script>
+  import { Icon, Input, Drawer, Button } from "@budibase/bbui"
+  import {
+    readableToRuntimeBinding,
+    runtimeToReadableBinding,
+  } from "builderStore/dataBinding"
+
+  import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
+  import { createEventDispatcher, setContext } from "svelte"
+  import { isJSBinding } from "@budibase/string-templates"
+
+  export let panel = ClientBindingPanel
+  export let value = ""
+  export let bindings = []
+  export let title = "Bindings"
+  export let placeholder
+  export let label
+  export let disabled = false
+  export let fillWidth
+  export let allowJS = true
+  export let allowHelpers = true
+  export let updateOnChange = true
+  export let drawerLeft
+  export let type
+  export let schema
+
+  const dispatch = createEventDispatcher()
+  let bindingDrawer
+  let valid = true
+  let currentVal = value
+
+  $: readableValue = runtimeToReadableBinding(bindings, value)
+  $: tempValue = readableValue
+  $: isJS = isJSBinding(value)
+
+  const saveBinding = () => {
+    onChange(tempValue)
+    onBlur()
+    bindingDrawer.hide()
+  }
+
+  setContext("binding-drawer-actions", {
+    save: saveBinding,
+  })
+
+  const onChange = value => {
+    if (type === "link" && value && hasValidLinks(value)) {
+      currentVal = value.split(",")
+    } else if (type === "array" && value && hasValidOptions(value)) {
+      currentVal = value.split(",")
+    } else {
+      currentVal = readableToRuntimeBinding(bindings, value)
+    }
+    dispatch("change", currentVal)
+  }
+
+  const onBlur = () => {
+    dispatch("blur", currentVal)
+  }
+
+  const isValidDate = value => {
+    return !value || !isNaN(new Date(value).valueOf())
+  }
+
+  const hasValidLinks = value => {
+    let links = []
+    if (Array.isArray(value)) {
+      links = value
+    } else if (value && typeof value === "string") {
+      links = value.split(",")
+    } else {
+      return !value
+    }
+
+    return links.every(link => link.startsWith("ro_"))
+  }
+
+  const hasValidOptions = value => {
+    let links = []
+    if (Array.isArray(value)) {
+      links = value
+    } else if (value && typeof value === "string") {
+      links = value.split(",")
+    } else {
+      return !value
+    }
+    return links.every(link => schema?.constraints?.inclusion?.includes(link))
+  }
+
+  const isValidBoolean = value => {
+    return value === "false" || value === "true" || value == ""
+  }
+
+  const validationMap = {
+    date: isValidDate,
+    datetime: isValidDate,
+    link: hasValidLinks,
+    array: hasValidOptions,
+    longform: value => !isJSBinding(value),
+    json: value => !isJSBinding(value),
+    boolean: isValidBoolean,
+  }
+
+  const isValid = value => {
+    const validate = validationMap[type]
+    return validate ? validate(value) : true
+  }
+
+  const getIconClass = (value, type) => {
+    if (type === "longform" && !isJSBinding(value)) {
+      return "text-area-slot-icon"
+    }
+    if (type === "json" && !isJSBinding(value)) {
+      return "json-slot-icon"
+    }
+    if (type !== "string" && type !== "number") {
+      return "slot-icon"
+    }
+    return ""
+  }
+</script>
+
+<div class="control" class:disabled>
+  {#if !isValid(value)}
+    <Input
+      {label}
+      {disabled}
+      readonly={isJS}
+      value={isJS ? "(JavaScript function)" : readableValue}
+      on:change={event => onChange(event.detail)}
+      on:blur={onBlur}
+      {placeholder}
+      {updateOnChange}
+    />
+    <div
+      class="icon"
+      on:click={() => {
+        if (!isJS) {
+          dispatch("change", "")
+        }
+      }}
+    >
+      <Icon disabled={isJS} size="S" name="Close" />
+    </div>
+  {:else}
+    <slot
+      {label}
+      {disabled}
+      readonly={isJS}
+      value={isJS ? "(JavaScript function)" : readableValue}
+      {placeholder}
+      {updateOnChange}
+    />
+  {/if}
+  {#if !disabled && type !== "formula"}
+    <div
+      class={`icon ${getIconClass(value, type)}`}
+      on:click={() => {
+        bindingDrawer.show()
+      }}
+    >
+      <Icon size="S" name="FlashOn" />
+    </div>
+  {/if}
+</div>
+<Drawer
+  on:drawerHide
+  on:drawerShow
+  {fillWidth}
+  bind:this={bindingDrawer}
+  {title}
+  left={drawerLeft}
+  headless
+>
+  <svelte:fragment slot="description">
+    Add the objects on the left to enrich your text.
+  </svelte:fragment>
+  <Button cta slot="buttons" disabled={!valid} on:click={saveBinding}>
+    Save
+  </Button>
+  <svelte:component
+    this={panel}
+    slot="body"
+    bind:valid
+    value={readableValue}
+    on:change={event => (tempValue = event.detail)}
+    {bindings}
+    {allowJS}
+    {allowHelpers}
+  />
+</Drawer>
+
+<style>
+  .control {
+    flex: 1;
+    position: relative;
+  }
+
+  .slot-icon {
+    right: 31px !important;
+    border-right: 1px solid var(--spectrum-alias-border-color);
+    border-top-right-radius: 0px !important;
+    border-bottom-right-radius: 0px !important;
+  }
+
+  .text-area-slot-icon {
+    border-bottom: 1px solid var(--spectrum-alias-border-color);
+    border-bottom-right-radius: 0px !important;
+    top: 26px !important;
+  }
+  .json-slot-icon {
+    border-bottom: 1px solid var(--spectrum-alias-border-color);
+    border-bottom-right-radius: 0px !important;
+    top: 23px !important;
+    right: 0px !important;
+  }
+
+  .icon {
+    right: 1px;
+    bottom: 1px;
+    position: absolute;
+    justify-content: center;
+    align-items: center;
+    display: flex;
+    flex-direction: row;
+    box-sizing: border-box;
+    border-left: 1px solid var(--spectrum-alias-border-color);
+    border-top-right-radius: var(--spectrum-alias-border-radius-regular);
+    border-bottom-right-radius: var(--spectrum-alias-border-radius-regular);
+    width: 31px;
+    color: var(--spectrum-alias-text-color);
+    background-color: var(--spectrum-global-color-gray-75);
+    transition: background-color
+        var(--spectrum-global-animation-duration-100, 130ms),
+      box-shadow var(--spectrum-global-animation-duration-100, 130ms),
+      border-color var(--spectrum-global-animation-duration-100, 130ms);
+    height: calc(var(--spectrum-alias-item-height-m) - 2px);
+  }
+
+  .icon:hover {
+    cursor: pointer;
+    color: var(--spectrum-alias-text-color-hover);
+    background-color: var(--spectrum-global-color-gray-50);
+    border-color: var(--spectrum-alias-border-color-hover);
+  }
+
+  .control:not(.disabled) :global(.spectrum-Textfield-input) {
+    padding-right: 40px;
+  }
+</style>

packages/builder/src/components/deploy/AppActions.svelte

@@ -224,10 +224,10 @@
               <span
                 class="app-link"
                 on:click={() => {
+                  appActionPopover.hide()
                   if (isPublished) {
                     viewApp()
                   } else {
-                    appActionPopover.hide()
                     updateAppModal.show()
                   }
                 }}

packages/builder/src/components/design/Panel.svelte

@@ -12,11 +12,19 @@
   export let borderLeft = false
   export let borderRight = false
   export let wide = false
+  export let extraWide = false
+  export let closeButtonIcon = "Close"
 
   $: customHeaderContent = $$slots["panel-header-content"]
 </script>
 
-<div class="panel" class:wide class:borderLeft class:borderRight>
+<div
+  class="panel"
+  class:wide
+  class:extraWide
+  class:borderLeft
+  class:borderRight
+>
   <div class="header" class:custom={customHeaderContent}>
     {#if showBackButton}
       <Icon name="ArrowLeft" hoverable on:click={onClickBackButton} />
@@ -33,7 +41,7 @@
       </div>
     {/if}
     {#if showCloseButton}
-      <Icon name="Close" hoverable on:click={onClickCloseButton} />
+      <Icon name={closeButtonIcon} hoverable on:click={onClickCloseButton} />
     {/if}
   </div>
 
@@ -70,6 +78,10 @@
     width: 310px;
     flex: 0 0 310px;
   }
+  .panel.extraWide {
+    width: 450px;
+    flex: 0 0 450px;
+  }
   .header {
     flex: 0 0 48px;
     display: flex;

packages/builder/src/components/design/settings/componentSettings.js

@@ -65,6 +65,7 @@ const componentMap = {
   "field/array": FormFieldSelect,
   "field/json": FormFieldSelect,
   "field/barcodeqr": FormFieldSelect,
+  "field/bb_reference": FormFieldSelect,
   // Some validation types are the same as others, so not all types are
   // explicitly listed here. e.g. options uses string validation
   "validation/string": ValidationEditor,
@@ -74,6 +75,7 @@ const componentMap = {
   "validation/datetime": ValidationEditor,
   "validation/attachment": ValidationEditor,
   "validation/link": ValidationEditor,
+  "validation/bb_reference": ValidationEditor,
 }
 
 export const getComponentForSetting = setting => {

packages/builder/src/components/design/settings/controls/ButtonActionEditor/ButtonActionEditor.svelte

@@ -3,7 +3,6 @@
   import { createEventDispatcher } from "svelte"
   import { notifications } from "@budibase/bbui"
   import ButtonActionDrawer from "./ButtonActionDrawer.svelte"
-  import { automationStore } from "builderStore"
   import { cloneDeep } from "lodash/fp"
 
   const dispatch = createEventDispatcher()
@@ -24,47 +23,11 @@
   }
 
   const saveEventData = async () => {
-    // any automations that need created from event triggers
-    const automationsToCreate = tmpValue.filter(
-      action => action["##eventHandlerType"] === "Trigger Automation"
-    )
-    for (let action of automationsToCreate) {
-      await createAutomation(action.parameters)
-    }
-
     dispatch("change", tmpValue)
     notifications.success("Component actions saved.")
     drawer.hide()
   }
 
-  // called by the parent modal when actions are saved
-  const createAutomation = async parameters => {
-    if (parameters.automationId || !parameters.newAutomationName) {
-      return
-    }
-    try {
-      let trigger = automationStore.actions.constructBlock(
-        "TRIGGER",
-        "APP",
-        $automationStore.blockDefinitions.TRIGGER.APP
-      )
-      trigger.inputs = {
-        fields: Object.keys(parameters.fields ?? {}).reduce((fields, key) => {
-          fields[key] = "string"
-          return fields
-        }, {}),
-      }
-      const automation = await automationStore.actions.create(
-        parameters.newAutomationName,
-        trigger
-      )
-      parameters.automationId = automation._id
-      delete parameters.newAutomationName
-    } catch (error) {
-      notifications.error("Error creating automation")
-    }
-  }
-
   $: actionCount = value?.length
   $: actionText = `${actionCount || "No"} action${
     actionCount !== 1 ? "s" : ""

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ScrollTo.svelte

@@ -1,6 +1,5 @@
 <script>
   import { currentAsset, store } from "builderStore"
-  import { onMount } from "svelte"
   import { Label, Combobox, Select } from "@budibase/bbui"
   import {
     getActionProviderComponents,
@@ -10,12 +9,6 @@
 
   export let parameters
 
-  onMount(() => {
-    if (!parameters.type) {
-      parameters.type = "top"
-    }
-  })
-
   $: formComponent = findComponent($currentAsset.props, parameters.componentId)
   $: formSchema = buildFormSchema(formComponent)
   $: fieldOptions = Object.keys(formSchema || {})

packages/builder/src/components/design/settings/controls/FieldConfiguration/utils.js

@@ -43,4 +43,5 @@ export const FieldTypeToComponentMap = {
   link: "relationshipfield",
   json: "jsonfield",
   barcodeqr: "codescanner",
+  bb_reference: "bbreferencefield",
 }

packages/builder/src/components/integration/ExtraQueryConfig.svelte

@@ -18,31 +18,20 @@
 </script>
 
 {#each extraFields as { key, displayName, type }}
-  <div class="config-field">
+  <Label>{displayName}</Label>
-    <Label>{displayName}</Label>
+  {#if type === "string"}
-    {#if type === "string"}
+    <Input
-      <Input
+      on:change={() => populateExtraQuery(extraQueryFields)}
-        on:change={() => populateExtraQuery(extraQueryFields)}
+      bind:value={extraQueryFields[key]}
-        bind:value={extraQueryFields[key]}
+    />
-      />
+  {/if}
-    {/if}
 
-    {#if type === "list"}
+  {#if type === "list"}
-      <Select
+    <Select
-        on:change={() => populateExtraQuery(extraQueryFields)}
+      on:change={() => populateExtraQuery(extraQueryFields)}
-        bind:value={extraQueryFields[key]}
+      bind:value={extraQueryFields[key]}
-        options={config[key].data[query.queryVerb]}
+      options={config[key].data[query.queryVerb]}
-        getOptionLabel={current => current}
+      getOptionLabel={current => current}
-      />
+    />
-    {/if}
+  {/if}
-  </div>
 {/each}
-
-<style>
-  .config-field {
-    display: grid;
-    grid-template-columns: 20% 1fr;
-    grid-gap: var(--spacing-l);
-    align-items: center;
-  }
-</style>

packages/builder/src/components/integration/KeyValueBuilder.svelte

@@ -34,6 +34,7 @@
   export let bindings = []
   export let bindingDrawerLeft
   export let allowHelpers = true
+  export let customButtonText = null
 
   let fields = Object.entries(object || {}).map(([name, value]) => ({
     name,
@@ -158,9 +159,13 @@
 {/if}
 {#if !readOnly && !noAddButton}
   <div>
-    <ActionButton icon="Add" secondary thin outline on:click={addEntry}
+    <ActionButton icon="Add" secondary thin outline on:click={addEntry}>
-      >Add{name ? ` ${lowercase(name)}` : ""}</ActionButton
+      {#if customButtonText}
-    >
+        {customButtonText}
+      {:else}
+        {`Add${name ? ` ${lowercase(name)}` : ""}`}
+      {/if}
+    </ActionButton>
   </div>
 {/if}
 

packages/builder/src/components/integration/QueryViewer.svelte

@@ -1,364 +1,444 @@
 <script>
-  import { goto, beforeUrlChange } from "@roxi/routify"
+  import { goto } from "@roxi/routify"
+  import { datasources, integrations, queries } from "stores/backend"
   import {
     Icon,
     Select,
-    Button,
-    ButtonGroup,
-    Body,
-    Label,
-    Layout,
     Input,
-    Heading,
+    Label,
-    Tabs,
-    Tab,
-    Modal,
-    ModalContent,
     notifications,
+    Heading,
+    Body,
     Divider,
+    Button,
   } from "@budibase/bbui"
-  import ExtraQueryConfig from "./ExtraQueryConfig.svelte"
+  import { capitalise } from "helpers"
-  import IntegrationQueryEditor from "components/integration/index.svelte"
-  import ExternalDataSourceTable from "components/backend/DataTable/ExternalDataSourceTable.svelte"
-  import BindingBuilder from "components/integration/QueryViewerBindingBuilder.svelte"
-  import { datasources, integrations, queries } from "stores/backend"
-  import { capitalise } from "../../helpers"
-  import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
-  import JSONPreview from "./JSONPreview.svelte"
-  import { SchemaTypeOptions } from "constants/backend"
-  import KeyValueBuilder from "./KeyValueBuilder.svelte"
-  import { fieldsToSchema, schemaToFields } from "helpers/data/utils"
   import AccessLevelSelect from "./AccessLevelSelect.svelte"
+  import IntegrationQueryEditor from "components/integration/index.svelte"
+  import QueryViewerSidePanel from "./QueryViewerSidePanel/index.svelte"
+  import { cloneDeep } from "lodash/fp"
+  import BindingBuilder from "components/integration/QueryViewerBindingBuilder.svelte"
+  import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
   import { ValidQueryNameRegex } from "@budibase/shared-core"
+  import ExtraQueryConfig from "./ExtraQueryConfig.svelte"
+  import QueryViewerSavePromptModal from "./QueryViewerSavePromptModal.svelte"
+  import { Utils } from "@budibase/frontend-core"
 
   export let query
+  let queryHash
 
-  const resumeNavigation = () => {
+  let loading = false
-    if (typeof navigateTo == "string") {
+  let modified = false
-      $goto(typeof navigateTo == "string" ? `${navigateTo}` : navigateTo)
+  let scrolling = false
-    }
+  let showSidePanel = false
+  let nameError
+
+  let newQuery
+
+  let datasource
+  let integration
+  let schemaType
+
+  let autoSchema = {}
+  let rows = []
+
+  const parseQuery = query => {
+    modified = false
+
+    datasource = $datasources.list.find(ds => ds._id === query.datasourceId)
+    integration = $integrations[datasource.source]
+    schemaType = integration.query[query.queryVerb].type
+
+    newQuery = cloneDeep(query)
+    // Set the location where the query code will be written to an empty string so that it doesn't
+    // get changed from undefined -> "" by the input, breaking our unsaved changes checks
+    newQuery.fields[schemaType] ??= ""
+
+    queryHash = JSON.stringify(newQuery)
   }
 
-  const transformerDocs = "https://docs.budibase.com/docs/transformers"
+  $: parseQuery(query)
 
-  let fields = query?.schema ? schemaToFields(query.schema) : []
+  const checkIsModified = newQuery => {
-  let parameters
+    const newQueryHash = JSON.stringify(newQuery)
-  let data = []
+    modified = newQueryHash !== queryHash
-  let saveId
-  let currentTab = "JSON"
-  let saveModal
-  let override = false
-  let navigateTo = null
-  let nameError = null
 
-  // seed the transformer
+    return modified
-  if (query && !query.transformer) {
-    query.transformer = "return data"
   }
 
-  // initialise a new empty schema
+  const debouncedCheckIsModified = Utils.debounce(checkIsModified, 1000)
-  if (query && !query.schema) {
-    query.schema = {}
-  }
 
-  let queryStr = JSON.stringify(query)
+  $: debouncedCheckIsModified(newQuery)
 
-  $beforeUrlChange(event => {
+  async function runQuery({ suppressErrors = true }) {
-    const updated = JSON.stringify(query)
-
-    if (updated !== queryStr && !override) {
-      navigateTo = event.type == "pushstate" ? event.url : null
-      saveModal.show()
-      return false
-    } else return true
-  })
-
-  $: datasource = $datasources.list.find(ds => ds._id === query.datasourceId)
-  $: query.schema = fieldsToSchema(fields)
-  $: datasourceType = datasource?.source
-  $: integrationInfo = datasourceType ? $integrations[datasourceType] : null
-  $: queryConfig = integrationInfo?.query
-  $: shouldShowQueryConfig = queryConfig && query.queryVerb
-  $: readQuery = query.queryVerb === "read" || query.readable
-  $: queryInvalid = !query.name || nameError || (readQuery && data.length === 0)
-
-  //Cast field in query preview response to number if specified by schema
-  $: {
-    for (let i = 0; i < data.length; i++) {
-      let row = data[i]
-      for (let fieldName of Object.keys(fields)) {
-        if (fields[fieldName] === "number" && !isNaN(Number(row[fieldName]))) {
-          row[fieldName] = Number(row[fieldName])
-        } else {
-          row[fieldName] = row[fieldName]?.toString()
-        }
-      }
-    }
-  }
-
-  function resetDependentFields() {
-    if (query.fields.extra) {
-      query.fields.extra = {}
-    }
-  }
-
-  function populateExtraQuery(extraQueryFields) {
-    query.fields.extra = extraQueryFields
-  }
-
-  async function previewQuery() {
     try {
-      const response = await queries.preview(query)
+      showSidePanel = true
+      loading = true
+      const response = await queries.preview(newQuery)
       if (response.rows.length === 0) {
         notifications.info(
           "Query results empty. Please execute a query with results to create your schema."
         )
         return
       }
-      data = response.rows
+
-      // need to merge fields that already exist/might have changed
+      if (Object.keys(newQuery.schema).length === 0) {
-      if (fields) {
+        // Assign this to a variable instead of directly to the newQuery.schema so that a user
-        for (let key of Object.keys(response.schema)) {
+        // can change the table they're querying and have the schema update until they first
-          if (fields[key]) {
+        // edit it
-            response.schema[key] = fields[key]
+        autoSchema = response.schema
-          }
-        }
       }
-      fields = response.schema
+
-      currentTab = "JSON"
+      rows = response.rows
+
       notifications.success("Query executed successfully")
     } catch (error) {
       notifications.error(`Query Error: ${error.message}`)
+
+      if (!suppressErrors) {
+        throw error
+      }
+    } finally {
+      loading = false
     }
   }
 
-  // return the query.
   async function saveQuery() {
     try {
-      const response = await queries.save(query.datasourceId, query)
+      showSidePanel = true
-      saveId = response._id
+      loading = true
-
+      const response = await queries.save(newQuery.datasourceId, {
-      if (response?._rev) {
+        ...newQuery,
-        queryStr = JSON.stringify(query)
+        schema:
-      }
+          Object.keys(newQuery.schema).length === 0
+            ? autoSchema
+            : newQuery.schema,
+      })
 
       notifications.success("Query saved successfully")
       return response
     } catch (error) {
       notifications.error(error.message || "Error saving query")
+    } finally {
+      loading = false
     }
   }
+
+  function resetDependentFields() {
+    if (newQuery.fields.extra) {
+      newQuery.fields.extra = {}
+    }
+  }
+
+  function populateExtraQuery(extraQueryFields) {
+    newQuery.fields.extra = extraQueryFields
+  }
+
+  const handleScroll = e => {
+    scrolling = e.target.scrollTop !== 0
+  }
 </script>
 
-<Modal
+<QueryViewerSavePromptModal
-  bind:this={saveModal}
+  checkIsModified={() => checkIsModified(newQuery)}
-  on:hide={() => {
+  attemptSave={() => runQuery({ suppressErrors: false }).then(saveQuery)}
-    navigateTo = null
+/>
-  }}
+<div class="queryViewer">
->
+  <div class="main">
-  <ModalContent
+    <div class="header" class:scrolling>
-    title="You have unsaved changes"
+      <div class="title">
-    confirmText="Save and Continue"
-    cancelText="Discard Changes"
-    size="L"
-    onConfirm={async () => {
-      await saveQuery()
-      override = true
-      resumeNavigation()
-    }}
-    onCancel={async () => {
-      override = true
-      resumeNavigation()
-    }}
-  >
-    <Body>Leaving this section will mean losing and changes to your query</Body>
-  </ModalContent>
-</Modal>
-
-<div class="wrapper">
-  <Layout gap="S" noPadding>
-    <Heading size="M">Query {integrationInfo?.friendlyName}</Heading>
-    <Divider />
-    <Heading size="S">Config</Heading>
-    <div class="config">
-      <div class="config-field">
-        <Label>Query Name</Label>
-        <Input
-          value={query.name}
-          on:input={e => {
-            let newValue = e.target.value || ""
-            if (newValue.match(ValidQueryNameRegex)) {
-              query.name = newValue.trim()
-              nameError = null
-            } else {
-              nameError = "Invalid query name"
-            }
-          }}
-          error={nameError}
-        />
-      </div>
-      {#if queryConfig}
-        <div class="config-field">
-          <Label>Function</Label>
-          <Select
-            bind:value={query.queryVerb}
-            on:change={resetDependentFields}
-            options={Object.keys(queryConfig)}
-            getOptionLabel={verb =>
-              queryConfig[verb]?.displayName || capitalise(verb)}
-          />
-        </div>
-        <div class="config-field">
-          <AccessLevelSelect {saveId} {query} label="Access Level" />
-        </div>
-        {#if integrationInfo?.extra && query.queryVerb}
-          <ExtraQueryConfig
-            {query}
-            {populateExtraQuery}
-            config={integrationInfo.extra}
-          />
-        {/if}
-        {#key query.parameters}
-          <div class="binding-wrap">
-            <BindingBuilder
-              queryBindings={query.parameters}
-              bindable={false}
-              on:change={e => {
-                query.parameters = e.detail.map(binding => {
-                  return {
-                    name: binding.name,
-                    default: binding.value,
-                  }
-                })
-              }}
-            />
-          </div>
-        {/key}
-      {/if}
-    </div>
-    {#if shouldShowQueryConfig}
-      <Divider />
-      <div class="config">
-        <Heading size="S">Fields</Heading>
-        <Body size="S">Fill in the fields specific to this query.</Body>
-        <IntegrationQueryEditor
-          {datasource}
-          {query}
-          height={200}
-          schema={queryConfig[query.queryVerb]}
-          bind:parameters
-        />
-        <Divider />
-      </div>
-      <div class="config">
-        <div class="help-heading">
-          <Heading size="S">Transformer</Heading>
-          <Icon
-            on:click={() => window.open(transformerDocs)}
-            hoverable
-            name="Help"
-            size="L"
-          />
-        </div>
         <Body size="S">
-          Add a JavaScript function to transform the query result.
+          {newQuery.name || "Untitled query"}<span class="unsaved"
+            >{modified ? "*" : ""}</span
+          >
         </Body>
-        <CodeMirrorEditor
-          height={200}
-          label="Transformer"
-          value={query.transformer}
-          resize="vertical"
-          on:change={e => (query.transformer = e.detail)}
-        />
-        <Divider />
       </div>
-      <div class="viewer-controls">
+      <div class="controls">
-        <Heading size="S">Results</Heading>
+        <Button disabled={loading} on:click={runQuery} overBackground>
-        <ButtonGroup gap="S">
+          <Icon size="S" name="Play" />
+          Run query</Button
+        >
+        <div class="tooltip" title="Run your query to enable saving">
           <Button
-            cta
-            disabled={queryInvalid}
             on:click={async () => {
-              await saveQuery()
+              const response = await saveQuery()
-              // Go to the correct URL if we just created a new query
+
-              if (!query._rev) {
+              // When creating a new query the initally passed in query object will have no id.
-                $goto(`../../${query._id}`)
+              if (response._id && !newQuery._id) {
+                // Set the comparison query hash to match the new query so that the user doesn't
+                // get nagged when navigating to the edit view
+                queryHash = JSON.stringify(newQuery)
+                $goto(`../../${response._id}`)
               }
             }}
+            disabled={loading ||
+              !newQuery.name ||
+              nameError ||
+              rows.length === 0}
+            overBackground
           >
-            Save Query
+            <Icon size="S" name="SaveFloppy" />
+            Save
           </Button>
-          <Button secondary on:click={previewQuery}>Run Query</Button>
+        </div>
-        </ButtonGroup>
       </div>
-      <Body size="S">
+    </div>
-        Below, you can preview the results from your query and change the
+
-        schema.
+    <div class="body" on:scroll={handleScroll}>
-      </Body>
+      <div class="bodyInner">
-      <section class="viewer">
+        <div class="configField">
-        {#if data}
+          <Label>Name</Label>
-          <Tabs bind:selected={currentTab}>
+          <Input
-            <Tab title="JSON">
+            value={newQuery.name}
-              <JSONPreview data={data[0]} minHeight="120" />
+            on:input={e => {
-            </Tab>
+              let newValue = e.target.value || ""
-            <Tab title="Schema">
+              if (newValue.match(ValidQueryNameRegex)) {
-              <KeyValueBuilder
+                newQuery.name = newValue.trim()
-                bind:object={fields}
+                nameError = null
-                name="field"
+              } else {
-                headings
+                nameError = "Invalid query name"
-                options={SchemaTypeOptions}
+              }
+            }}
+            error={nameError}
+          />
+          {#if integration.query}
+            <Label>Function</Label>
+            <Select
+              bind:value={newQuery.queryVerb}
+              on:change={resetDependentFields}
+              options={Object.keys(integration.query)}
+              getOptionLabel={verb =>
+                integration.query[verb]?.displayName || capitalise(verb)}
+            />
+            <Label>Access</Label>
+            <AccessLevelSelect query={newQuery} />
+            {#if integration?.extra && newQuery.queryVerb}
+              <ExtraQueryConfig
+                query={newQuery}
+                {populateExtraQuery}
+                config={integration.extra}
               />
-            </Tab>
+            {/if}
-            <Tab title="Preview">
+          {/if}
-              <ExternalDataSourceTable {query} {data} />
+        </div>
-            </Tab>
+
-          </Tabs>
+        <Divider />
-        {/if}
+
-      </section>
+        <div class="heading">
-    {/if}
+          <Heading weight="L" size="XS">Query</Heading>
-  </Layout>
+        </div>
+        <div class="copy">
+          <Body size="S">
+            {#if schemaType === "sql"}
+              Add some SQL to query your data
+            {:else if schemaType === "json"}
+              Add some JSON to query your data
+            {:else if schemaType === "fields"}
+              Add some fields to query your data
+            {:else}
+              Enter your query below
+            {/if}
+          </Body>
+        </div>
+        <IntegrationQueryEditor
+          noLabel
+          {datasource}
+          bind:query={newQuery}
+          height={200}
+          schema={integration.query[newQuery.queryVerb]}
+        />
+
+        <Divider />
+
+        <div class="heading">
+          <Heading weight="L" size="XS">Bindings</Heading>
+        </div>
+        <div class="copy">
+          <Body size="S">
+            Bindings come in two parts: the binding name, and a default/fallback
+            value. These bindings can be used as Handlebars expressions
+            throughout the query.
+          </Body>
+        </div>
+        {#key newQuery.parameters}
+          <BindingBuilder
+            hideHeading
+            queryBindings={newQuery.parameters}
+            on:change={e => {
+              newQuery.parameters = e.detail.map(binding => {
+                return {
+                  name: binding.name,
+                  default: binding.value,
+                }
+              })
+            }}
+          />
+        {/key}
+
+        <Divider />
+        <div class="heading">
+          <Heading weight="L" size="XS">Transformer</Heading>
+        </div>
+        <div class="copy">
+          <Body size="S">
+            Add a JavaScript function to transform the query result.
+          </Body>
+        </div>
+        <CodeMirrorEditor
+          height={200}
+          value={newQuery.transformer}
+          resize="vertical"
+          on:change={e => (newQuery.transformer = e.detail)}
+        />
+      </div>
+    </div>
+  </div>
+
+  <div class:showSidePanel class="sidePanel">
+    <QueryViewerSidePanel
+      onClose={() => (showSidePanel = false)}
+      onSchemaChange={newSchema => {
+        newQuery.schema = newSchema
+      }}
+      {rows}
+      schema={Object.keys(newQuery.schema).length === 0
+        ? autoSchema
+        : newQuery.schema}
+    />
+  </div>
 </div>
 
 <style>
-  .wrapper {
+  .unsaved {
-    width: 640px;
+    color: var(--grey-5);
+    font-style: italic;
+  }
+
+  .queryViewer {
+    height: 100%;
+    margin: -28px -40px -40px -40px;
+    display: flex;
+    flex: 1;
+  }
+
+  .queryViewer :global(.spectrum-Divider) {
+    margin: 35px 0;
+  }
+
+  .main {
+    flex-grow: 1;
+    height: 100%;
+    display: flex;
+    flex-direction: column;
+  }
+
+  .header {
+    align-items: center;
+    padding: 8px 10px 8px 16px;
+    display: flex;
+    border-bottom: 2px solid transparent;
+    transition: border-bottom 130ms ease-out, background 130ms ease-out;
+  }
+
+  .header.scrolling {
+    border-bottom: var(--border-light);
+    background: var(--background);
+  }
+
+  .body {
+    flex-grow: 1;
+    overflow-y: scroll;
+    padding: 23px 23px 80px;
+    box-sizing: border-box;
+  }
+
+  .bodyInner {
+    max-width: 520px;
     margin: auto;
   }
 
-  .config {
+  .title {
-    display: grid;
+    /* width 0 paired with flex-grow necessary here for the truncation to work properly*/
-    grid-gap: var(--spacing-s);
+    width: 0;
-    z-index: 1;
+    flex-grow: 1;
   }
 
-  .config-field {
+  .title :global(p) {
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
+
+  .controls {
+    flex-shrink: 0;
+  }
+
+  .tooltip {
+    display: inline-block;
+  }
+
+  .controls :global(button) {
+    border: none;
+    color: var(--grey-7);
+    font-weight: 300;
+  }
+
+  .controls :global(button):hover {
+    background-color: transparent;
+    color: var(--ink);
+  }
+
+  .controls :global(.is-disabled) {
+    pointer-events: none;
+    background-color: transparent;
+    color: var(--grey-3);
+  }
+
+  .controls :global(span) {
+    display: flex;
+    align-items: center;
+  }
+
+  .controls :global(.icon) {
+    margin-right: 8px;
+  }
+
+  .configField {
     display: grid;
     grid-template-columns: 20% 1fr;
     grid-gap: var(--spacing-l);
     align-items: center;
   }
 
-  .help-heading {
+  .configField :global(label) {
-    display: flex;
+    color: var(--grey-6);
-    justify-content: space-between;
   }
 
-  .viewer {
+  .heading {
-    min-height: 200px;
+    margin-bottom: 8px;
-    width: 640px;
   }
 
-  .viewer-controls {
+  .copy {
-    display: flex;
+    margin-bottom: 14px;
-    flex-direction: row;
-    justify-content: space-between;
-    gap: var(--spacing-m);
-    min-width: 150px;
-    align-items: center;
   }
 
-  .binding-wrap :global(div.container) {
+  .copy :global(p) {
-    padding-left: 0px;
+    color: var(--grey-7);
-    padding-right: 0px;
+  }
+
+  .sidePanel {
+    flex-shrink: 0;
+    height: 100%;
+    width: 0;
+    overflow: hidden;
+    transition: width 150ms;
+  }
+
+  .sidePanel :global(.panel) {
+    height: 100%;
+  }
+
+  .showSidePanel {
+    width: 450px;
   }
 </style>

packages/builder/src/components/integration/QueryViewerBindingBuilder.svelte

@@ -1,9 +1,9 @@
 <script>
-  import { Body, Heading, Layout } from "@budibase/bbui"
   import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
   import { getUserBindings } from "builderStore/dataBinding"
   export let bindable = true
   export let queryBindings = []
+  export let hideHeading = false
 
   const userBindings = getUserBindings()
 
@@ -13,44 +13,16 @@
   }, {})
 </script>
 
-<Layout noPadding={bindable} gap="S">
+<KeyValueBuilder
-  <div class="controls" class:height={!bindable}>
+  bind:object={internalBindings}
-    <Heading size="XS">Bindings</Heading>
+  tooltip="Set the name of the binding which can be used in Handlebars statements throughout your query"
-  </div>
+  name="binding"
-  <Body size="S">
+  customButtonText="Bindings"
-    {#if !bindable}
+  headings
-      Bindings come in two parts: the binding name, and a default/fallback
+  keyPlaceholder="Binding name"
-      value. These bindings can be used as Handlebars expressions throughout the
+  valuePlaceholder="Default"
-      query.
+  bindings={[...userBindings]}
-    {:else}
+  bindingDrawerLeft="260px"
-      Enter a value for each binding. The default values will be used for any
+  allowHelpers={false}
-      values left blank.
+  on:change
-    {/if}
+/>
-  </Body>
-  <div class="bindings" class:bindable>
-    <KeyValueBuilder
-      bind:object={internalBindings}
-      tooltip="Set the name of the binding which can be used in Handlebars statements throughout your query"
-      name="binding"
-      headings
-      keyPlaceholder="Binding name"
-      valuePlaceholder="Default"
-      bindings={[...userBindings]}
-      bindingDrawerLeft="260px"
-      allowHelpers={false}
-      on:change
-    />
-  </div>
-</Layout>
-
-<style>
-  .controls {
-    display: flex;
-    align-items: center;
-    justify-content: space-between;
-  }
-
-  .height {
-    height: 40px;
-  }
-</style>

packages/builder/src/components/integration/QueryViewerSavePromptModal.svelte

@@ -0,0 +1,53 @@
+<script>
+  import { goto, beforeUrlChange } from "@roxi/routify"
+  import { Body, Modal, ModalContent } from "@budibase/bbui"
+
+  export let checkIsModified = () => {}
+  export let attemptSave = () => {}
+  let modal
+  let navigateTo
+  let override = false
+
+  $beforeUrlChange(event => {
+    if (checkIsModified() && !override) {
+      navigateTo = event.type == "pushstate" ? event.url : null
+      modal.show()
+      return false
+    } else return true
+  })
+
+  const resumeNavigation = () => {
+    if (typeof navigateTo == "string") {
+      $goto(typeof navigateTo == "string" ? `${navigateTo}` : navigateTo)
+    }
+  }
+</script>
+
+<Modal
+  bind:this={modal}
+  on:hide={() => {
+    navigateTo = null
+  }}
+>
+  <ModalContent
+    title="You have unsaved changes"
+    confirmText="Save and Continue"
+    cancelText="Discard Changes"
+    size="L"
+    onConfirm={async () => {
+      try {
+        await attemptSave()
+        override = true
+        resumeNavigation()
+      } catch (e) {
+        navigateTo = false
+      }
+    }}
+    onCancel={async () => {
+      override = true
+      resumeNavigation()
+    }}
+  >
+    <Body>Leaving this section will mean losing any changes to your query</Body>
+  </ModalContent>
+</Modal>

packages/builder/src/components/integration/QueryViewerSidePanel/JSONPanel.svelte

@@ -0,0 +1,22 @@
+<script>
+  export let data
+
+  $: string = JSON.stringify(data || {}, null, 2)
+</script>
+
+<textarea class="json" disabled value={string} />
+
+<style>
+  .json {
+    width: 100%;
+    height: 100%;
+    color: var(--ink);
+    padding: 12px;
+    box-sizing: border-box;
+    resize: none;
+    background-color: var(
+      --spectrum-textfield-m-background-color,
+      var(--spectrum-global-color-gray-50)
+    );
+  }
+</style>

packages/builder/src/components/integration/QueryViewerSidePanel/PreviewPanel.svelte

@@ -0,0 +1,33 @@
+<script>
+  import Table from "components/backend/DataTable/Table.svelte"
+  import { cloneDeep } from "lodash/fp"
+
+  export let schema = {}
+  export let rows = []
+
+  $: rowsCopy = cloneDeep(rows)
+
+  // Cast field in query preview response to number if specified by schema
+  $: {
+    for (let i = 0; i < rowsCopy.length; i++) {
+      let row = rowsCopy[i]
+      for (let fieldName of Object.keys(schema)) {
+        if (schema[fieldName] === "number" && !isNaN(Number(row[fieldName]))) {
+          row[fieldName] = Number(row[fieldName])
+        } else {
+          row[fieldName] = row[fieldName]?.toString()
+        }
+      }
+    }
+  }
+</script>
+
+<div class="table">
+  <Table {schema} data={rowsCopy} type="external" allowEditing={false} />
+</div>
+
+<style>
+  .table :global(.spectrum-Table-cell) {
+    min-width: 100px;
+  }
+</style>

packages/builder/src/components/integration/QueryViewerSidePanel/SchemaPanel.svelte

@@ -0,0 +1,29 @@
+<script>
+  import KeyValueBuilder from "../KeyValueBuilder.svelte"
+  import { SchemaTypeOptions } from "constants/backend"
+
+  export let schema
+  export let onSchemaChange = () => {}
+
+  const handleChange = e => {
+    let newSchema = {}
+
+    // KeyValueBuilder on change event returns an array of objects with each object
+    // containing a field
+    e.detail.forEach(({ name, value }) => {
+      newSchema[name] = value
+    })
+
+    onSchemaChange(newSchema)
+  }
+</script>
+
+{#key schema}
+  <KeyValueBuilder
+    on:change={handleChange}
+    object={schema}
+    name="field"
+    headings
+    options={SchemaTypeOptions}
+  />
+{/key}

packages/builder/src/components/integration/QueryViewerSidePanel/index.svelte

@@ -0,0 +1,66 @@
+<script>
+  import Panel from "components/design/Panel.svelte"
+  import { ActionButton } from "@budibase/bbui"
+  import JSONPanel from "./JSONPanel.svelte"
+  import SchemaPanel from "./SchemaPanel.svelte"
+  import PreviewPanel from "./PreviewPanel.svelte"
+
+  export let rows
+  export let schema
+  export let onSchemaChange = () => {}
+  export let onClose = () => {}
+
+  const tabs = ["JSON", "Schema", "Preview"]
+  let activeTab = "JSON"
+</script>
+
+<Panel
+  showCloseButton
+  closeButtonIcon="RailRightClose"
+  onClickCloseButton={onClose}
+  title="Query results"
+  icon={"SQLQuery"}
+  borderLeft
+  extraWide
+>
+  <div slot="panel-header-content">
+    <div class="settings-tabs">
+      {#each tabs as tab}
+        <ActionButton
+          size="M"
+          quiet
+          selected={activeTab === tab}
+          on:click={() => {
+            activeTab = tab
+          }}
+        >
+          {tab}
+        </ActionButton>
+      {/each}
+    </div>
+  </div>
+  <div class="content">
+    {#if activeTab === "JSON"}
+      <JSONPanel data={rows[0] || {}} />
+    {:else if activeTab === "Schema"}
+      <SchemaPanel {onSchemaChange} {schema} />
+    {:else}
+      <PreviewPanel {schema} {rows} />
+    {/if}
+  </div>
+</Panel>
+
+<style>
+  .settings-tabs {
+    display: flex;
+    gap: var(--spacing-s);
+    padding: 0 var(--spacing-l);
+    padding-bottom: var(--spacing-l);
+  }
+
+  .content {
+    padding: 14px;
+    height: 100%;
+    overflow: scroll;
+  }
+</style>

packages/builder/src/components/integration/index.svelte

@@ -23,6 +23,7 @@
   export let schema
   export let editable = true
   export let height = 500
+  export let noLabel = false
 
   let stepEditors = []
 
@@ -75,7 +76,7 @@
     {#if schema.type === QueryTypes.SQL}
       <Editor
         editorHeight={height}
-        label="Query"
+        label={noLabel ? null : "Query"}
         mode="sql"
         on:change={updateQuery}
         readOnly={!editable}
@@ -85,7 +86,7 @@
     {:else if shouldDisplayJsonBox}
       <Editor
         editorHeight={height}
-        label="Query"
+        label={noLabel ? null : "Query"}
         mode="json"
         on:change={updateQuery}
         readOnly={!editable}

packages/builder/src/components/start/ImportAppModal.svelte

@@ -0,0 +1,69 @@
+<script>
+  import {
+    ModalContent,
+    Toggle,
+    Input,
+    Layout,
+    Dropzone,
+    notifications,
+    Body,
+  } from "@budibase/bbui"
+  import { API } from "api"
+  import { automationStore, store } from "../../builderStore"
+
+  export let app
+
+  $: disabled = (encrypted && !password) || !file
+  let encrypted = false,
+    password
+  let file
+
+  async function updateApp() {
+    try {
+      let data = new FormData()
+      data.append("appExport", file)
+      if (encrypted) {
+        data.append("encryptionPassword", password.trim())
+      }
+      const appId = app.devId
+      await API.updateAppFromExport(appId, data)
+      const pkg = await API.fetchAppPackage(appId)
+      await store.actions.initialise(pkg)
+      await automationStore.actions.fetch()
+      notifications.success("App updated successfully")
+    } catch (err) {
+      notifications.error(`Failed to update app - ${err.message || err}`)
+    }
+  }
+</script>
+
+<ModalContent
+  title={`Update ${app.name}`}
+  confirmText="Update"
+  onConfirm={updateApp}
+  bind:disabled
+>
+  <Body size="S"
+    >Updating an app using an app export will replace all tables, datasources,
+    queries, screens and automations. It is recommended to perform a backup
+    before running this operation.</Body
+  >
+  <Layout noPadding gap="XS">
+    <Dropzone
+      gallery={false}
+      label="App export"
+      on:change={e => {
+        file = e.detail?.[0]
+      }}
+    />
+    <Toggle text="Encrypted" bind:value={encrypted} />
+    {#if encrypted}
+      <Input
+        type="password"
+        label="Password"
+        placeholder="Type here..."
+        bind:value={password}
+      />
+    {/if}
+  </Layout>
+</ModalContent>

packages/builder/src/constants/backend/index.js

@@ -120,6 +120,13 @@ export const FIELDS = {
       presence: false,
     },
   },
+  BB_REFERENCE_USER: {
+    name: "User",
+    type: "bb_reference",
+    subtype: "user",
+    compositeType: "bb_reference_user", // Used for working with the subtype on CreateEditColumn as is it was a primary type
+    icon: "User",
+  },
 }
 
 export const AUTO_COLUMN_SUB_TYPES = {
@@ -170,6 +177,11 @@ export const RelationshipType = {
   MANY_TO_ONE: "many-to-one",
 }
 
+export const PrettyRelationshipDefinitions = {
+  MANY: "Many rows",
+  ONE: "One row",
+}
+
 export const ALLOWABLE_STRING_OPTIONS = [
   FIELDS.STRING,
   FIELDS.OPTIONS,

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -22,6 +22,7 @@
   import { isActive, goto, layout, redirect } from "@roxi/routify"
   import { capitalise } from "helpers"
   import { onMount, onDestroy } from "svelte"
+  import VerificationPromptBanner from "components/common/VerificationPromptBanner.svelte"
   import CommandPalette from "components/commandPalette/CommandPalette.svelte"
   import TourWrap from "components/portal/onboarding/TourWrap.svelte"
   import TourPopover from "components/portal/onboarding/TourPopover.svelte"
@@ -136,6 +137,7 @@
 {/if}
 
 <div class="root" class:blur={$store.showPreview}>
+  <VerificationPromptBanner />
   <div class="top-nav">
     {#if $store.initialised}
       <div class="topleftnav">

packages/builder/src/pages/builder/app/[application]/data/query/new/[datasourceId]/index.svelte

@@ -16,6 +16,9 @@
 
   const buildNewQuery = isRestQuery => {
     let query = {
+      name: "Untitled query",
+      transformer: "return data",
+      schema: {},
       datasourceId: $params.datasourceId,
       parameters: [],
       fields: {},

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUIDrawer.svelte

@@ -61,10 +61,12 @@
     key: "_css",
     type: "text",
   })
-  $: settingOptions = settings.map(setting => ({
+  $: settingOptions = settings
-    label: makeLabel(setting),
+    .filter(setting => setting.supportsConditions !== false)
-    value: setting.key,
+    .map(setting => ({
-  }))
+      label: makeLabel(setting),
+      value: setting.key,
+    }))
   $: conditions.forEach(link => {
     if (!link.id) {
       link.id = generate()

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUISection.svelte

@@ -36,10 +36,7 @@
 </script>
 
 <DetailSummary name={"Conditions"} collapsible={false}>
-  <div class="conditionCount">{conditionText}</div>
+  <ActionButton on:click={openDrawer}>{conditionText}</ActionButton>
-  <div>
-    <ActionButton on:click={openDrawer}>Configure conditions</ActionButton>
-  </div>
 </DetailSummary>
 <Drawer bind:this={drawer} title="Conditions">
   <svelte:fragment slot="description">
@@ -48,10 +45,3 @@
   <Button cta slot="buttons" on:click={() => save()}>Save</Button>
   <ConditionalUIDrawer slot="body" bind:conditions={tempValue} {bindings} />
 </Drawer>
-
-<style>
-  .conditionCount {
-    font-weight: 600;
-    margin-top: -5px;
-  }
-</style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json

@@ -69,7 +69,8 @@
       "datetimefield",
       "multifieldselect",
       "s3upload",
-      "codescanner"
+      "codescanner",
+      "bbreferencefield"
     ]
   },
   {

packages/builder/src/pages/builder/app/[application]/settings/_layout.svelte

@@ -32,9 +32,9 @@
             active={$isActive("./embed")}
           />
           <SideNavItem
-            text="Export"
+            text="Export/Import"
-            url={$url("./export")}
+            url={$url("./exportImport")}
-            active={$isActive("./export")}
+            active={$isActive("./exportImport")}
           />
           <SideNavItem
             text="Name and URL"

packages/builder/src/pages/builder/app/[application]/settings/exportImport.svelte

@@ -11,31 +11,39 @@
   import { apps } from "stores/portal"
   import { store } from "builderStore"
   import ExportAppModal from "components/start/ExportAppModal.svelte"
+  import ImportAppModal from "components/start/ImportAppModal.svelte"
 
   $: filteredApps = $apps.filter(app => app.devId == $store.appId)
   $: app = filteredApps.length ? filteredApps[0] : {}
   $: appDeployed = app?.status === AppStatus.DEPLOYED
 
-  let exportModal
+  let exportModal, importModal
   let exportPublishedVersion = false
 
   const exportApp = opts => {
     exportPublishedVersion = !!opts?.published
     exportModal.show()
   }
+
+  const importApp = () => {
+    importModal.show()
+  }
 </script>
 
 <Modal bind:this={exportModal} padding={false}>
   <ExportAppModal {app} published={exportPublishedVersion} />
 </Modal>
 
+<Modal bind:this={importModal} padding={false}>
+  <ImportAppModal {app} />
+</Modal>
+
 <Layout noPadding>
   <Layout gap="XS" noPadding>
     <Heading>Export your app</Heading>
     <Body>Export your latest edited or published app</Body>
   </Layout>
-  <Divider />
+  <div class="body">
-  <div class="export-body">
     <ActionButton secondary on:click={() => exportApp({ published: false })}>
       Export latest edited app
     </ActionButton>
@@ -47,10 +55,20 @@
       Export latest published app
     </ActionButton>
   </div>
+  <Divider />
+  <Layout gap="XS" noPadding>
+    <Heading>Import your app</Heading>
+    <Body>Import an export to update this app</Body>
+  </Layout>
+  <div class="body">
+    <ActionButton secondary on:click={() => importApp()}>
+      Import app
+    </ActionButton>
+  </div>
 </Layout>
 
 <style>
-  .export-body {
+  .body {
     display: flex;
     gap: var(--spacing-l);
   }

packages/builder/src/pages/builder/portal/_layout.svelte

@@ -8,6 +8,7 @@
   import Logo from "./_components/Logo.svelte"
   import UserDropdown from "./_components/UserDropdown.svelte"
   import HelpMenu from "components/common/HelpMenu.svelte"
+  import VerificationPromptBanner from "components/common/VerificationPromptBanner.svelte"
   import { sdk } from "@budibase/shared-core"
 
   let loaded = false
@@ -55,6 +56,7 @@
   {:else}
     <HelpMenu />
     <div class="container">
+      <VerificationPromptBanner />
       <div class="nav">
         <div class="branding">
           <Logo />
@@ -99,7 +101,7 @@
     justify-content: flex-start;
     align-items: center;
     border-bottom: var(--border-light);
-    padding: 0 24px;
+    padding: 0 var(--spacing-l);
     gap: 24px;
     position: relative;
   }

packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte

@@ -86,8 +86,16 @@
   $: userPage = $userPageInfo.page
   $: logsPage = $logsPageInfo.page
 
+  let usersObj = {}
+  $: usersObj = {
+    ...usersObj,
+    ...$users.data?.reduce((accumulator, user) => {
+      accumulator[user._id] = user
+      return accumulator
+    }, {}),
+  }
   $: sortedUsers = sort(
-    enrich($users.data || [], selectedUsers, "_id"),
+    enrich(Object.values(usersObj), selectedUsers, "_id"),
     "email"
   )
   $: sortedEvents = sort(
@@ -256,8 +264,7 @@
   <div class="controls">
     <div class="select">
       <Multiselect
-        bind:fetchTerm={userSearchTerm}
+        bind:searchTerm={userSearchTerm}
-        useFetch
         placeholder="All users"
         label="Users"
         autocomplete

packages/builder/src/pages/builder/portal/apps/[appId]/_layout.svelte

@@ -0,0 +1,15 @@
+<script>
+  import { params, redirect } from "@roxi/routify"
+  import { apps } from "stores/portal"
+
+  $: app = $apps.find(app => app.appId === $params.appId)
+  $: {
+    if (!app) {
+      $redirect("../")
+    }
+  }
+</script>
+
+{#if app}
+  <slot />
+{/if}

packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte

@@ -0,0 +1,79 @@
+<script>
+  import { params, goto } from "@roxi/routify"
+  import { apps, auth, sideBarCollapsed } from "stores/portal"
+  import { ActionButton } from "@budibase/bbui"
+  import { sdk } from "@budibase/shared-core"
+
+  $: app = $apps.find(app => app.appId === $params.appId)
+  $: iframeUrl = getIframeURL(app)
+  $: isBuilder = sdk.users.isBuilder($auth.user, app?.devId)
+
+  const getIframeURL = app => {
+    if (app.status === "published") {
+      return `/app${app.url}`
+    }
+    return `/${app.devId}`
+  }
+</script>
+
+<div class="container">
+  <div class="header">
+    {#if $sideBarCollapsed}
+      <ActionButton
+        quiet
+        icon="Rail"
+        on:click={() => sideBarCollapsed.set(false)}
+      >
+        Menu
+      </ActionButton>
+    {:else}
+      <ActionButton
+        quiet
+        icon="RailRightOpen"
+        on:click={() => sideBarCollapsed.set(true)}
+      >
+        Collapse
+      </ActionButton>
+    {/if}
+    {#if isBuilder}
+      <ActionButton
+        quiet
+        icon="Edit"
+        on:click={() => $goto(`/builder/app/${app.devId}`)}
+      >
+        Edit
+      </ActionButton>
+    {/if}
+    <ActionButton
+      quiet
+      icon="LinkOut"
+      on:click={() => window.open(iframeUrl, "_blank")}
+    >
+      Fullscreen
+    </ActionButton>
+  </div>
+  <iframe src={iframeUrl} title={app.name} />
+</div>
+
+<style>
+  .container {
+    flex: 1 1 auto;
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: stretch;
+    padding: 0 var(--spacing-l) var(--spacing-l) var(--spacing-l);
+  }
+  .header {
+    display: flex;
+    justify-content: flex-start;
+    align-items: center;
+    gap: var(--spacing-xs);
+    flex: 0 0 50px;
+  }
+  iframe {
+    flex: 1 1 auto;
+    border-radius: var(--spacing-s);
+    border: 1px solid var(--spectrum-global-color-gray-300);
+  }
+</style>

packages/builder/src/pages/builder/portal/apps/_components/PortalSideBar.svelte

@@ -0,0 +1,140 @@
+<script>
+  import { Icon, Body } from "@budibase/bbui"
+  import { apps, sideBarCollapsed } from "stores/portal"
+  import { params, goto } from "@roxi/routify"
+  import { tick } from "svelte"
+  import NavItem from "components/common/NavItem.svelte"
+
+  let searchInput
+  let searchString
+  let searching = false
+
+  $: filteredApps = $apps.filter(app => {
+    return (
+      !searchString ||
+      app.name.toLowerCase().includes(searchString.toLowerCase())
+    )
+  })
+
+  const startSearching = async () => {
+    searching = true
+    searchString = ""
+    await tick()
+    searchInput.focus()
+  }
+
+  const stopSearching = () => {
+    searching = false
+    searchString = ""
+  }
+</script>
+
+<div class="side-bar" class:collapsed={$sideBarCollapsed}>
+  <div class="side-bar-controls">
+    {#if searching}
+      <input
+        bind:this={searchInput}
+        bind:value={searchString}
+        placeholder="Search for apps"
+      />
+    {:else}
+      <Body size="S">Apps</Body>
+      <Icon name="Search" size="S" hoverable on:click={startSearching} />
+    {/if}
+    <div class="rotational" class:rotated={searching}>
+      <Icon
+        name="Add"
+        hoverable
+        on:click={searching ? stopSearching : () => $goto("./create")}
+      />
+    </div>
+  </div>
+  <div class="side-bar-nav">
+    <NavItem
+      icon="WebPages"
+      text="All apps"
+      on:click={() => $goto("./")}
+      selected={!$params.appId}
+    />
+    {#each filteredApps as app}
+      <NavItem
+        text={app.name}
+        icon={app.icon?.name || "Apps"}
+        iconColor={app.icon?.color}
+        selected={$params.appId === app.appId}
+        on:click={() => $goto(`./${app.appId}`)}
+      />
+    {/each}
+  </div>
+</div>
+
+<style>
+  .side-bar {
+    flex: 0 0 260px;
+    display: flex;
+    flex-direction: column;
+    align-items: stretch;
+    border-right: var(--border-light);
+    background: var(--spectrum-global-color-gray-100);
+    overflow: hidden;
+    transition: margin-left 300ms ease-out;
+  }
+  .side-bar.collapsed {
+    margin-left: -262px;
+  }
+  @media (max-width: 640px) {
+    .side-bar {
+      margin-left: -262px;
+    }
+  }
+
+  .side-bar-controls {
+    flex: 0 0 50px;
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    gap: var(--spacing-l);
+    padding: 0 var(--spacing-l);
+  }
+  .side-bar-controls :global(.spectrum-Body),
+  .side-bar-controls input {
+    flex: 1 1 auto;
+  }
+  .side-bar-controls :global(.spectrum-Icon) {
+    color: var(--spectrum-global-color-gray-700);
+  }
+
+  input {
+    outline: none;
+    border: none;
+    max-width: none;
+    flex: 1 1 auto;
+    color: var(--spectrum-global-color-gray-800);
+    font-size: 14px;
+    padding: 0;
+    transition: border 130ms ease-out;
+    font-family: var(--font-sans);
+    background: inherit;
+  }
+  input::placeholder {
+    color: var(--spectrum-global-color-gray-700);
+    transition: color 130ms ease-out;
+  }
+  input:hover::placeholder {
+    color: var(--spectrum-global-color-gray-800);
+  }
+
+  .side-bar-nav {
+    flex: 1 1 auto;
+    overflow: auto;
+    overflow-x: hidden;
+  }
+
+  div.rotational {
+    transition: transform 130ms ease-out;
+  }
+  div.rotational.rotated {
+    transform: rotate(45deg);
+  }
+</style>

packages/builder/src/pages/builder/portal/apps/_layout.svelte

@@ -11,6 +11,7 @@
   import { onMount } from "svelte"
   import { redirect } from "@roxi/routify"
   import { sdk } from "@budibase/shared-core"
+  import PortalSideBar from "./_components/PortalSideBar.svelte"
 
   // Don't block loading if we've already hydrated state
   let loaded = $apps.length != null
@@ -44,5 +45,20 @@
 </script>
 
 {#if loaded}
-  <slot />
+  <div class="page">
+    {#if $apps.length > 0}
+      <PortalSideBar />
+    {/if}
+    <slot />
+  </div>
 {/if}
+
+<style>
+  .page {
+    flex: 1 1 auto;
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: stretch;
+  }
+</style>

packages/builder/src/pages/builder/portal/settings/version.svelte

@@ -63,8 +63,7 @@
       )
       const githubResponse = await githubCheck.json()
 
-      //Get tag and remove the v infront of the tage name e.g. v1.0.0 is 1.0.0
+      githubVersion = githubResponse.tag_name
-      githubVersion = githubResponse.tag_name.slice(1)
 
       //Get the release date and output it in the local time format
       githubPublishedDate = new Date(githubResponse.published_at)

packages/builder/src/stores/backend/tables.js

@@ -1,7 +1,7 @@
 import { get, writable, derived } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
 import { API } from "api"
-import { SWITCHABLE_TYPES } from "constants/backend"
+import { SWITCHABLE_TYPES, FIELDS } from "constants/backend"
 
 export function createTablesStore() {
   const store = writable({
@@ -21,6 +21,23 @@ export function createTablesStore() {
     }))
   }
 
+  const singleFetch = async tableId => {
+    const table = await API.getTable(tableId)
+    store.update(state => {
+      const list = []
+      // update the list, keep order accurate
+      for (let tbl of state.list) {
+        if (table._id === tbl._id) {
+          list.push(table)
+        } else {
+          list.push(tbl)
+        }
+      }
+      state.list = list
+      return state
+    })
+  }
+
   const select = tableId => {
     store.update(state => ({
       ...state,
@@ -63,6 +80,20 @@ export function createTablesStore() {
     const savedTable = await API.saveTable(updatedTable)
     replaceTable(savedTable._id, savedTable)
     select(savedTable._id)
+    // make sure tables up to date (related)
+    let tableIdsToFetch = []
+    for (let column of Object.values(updatedTable?.schema || {})) {
+      if (column.type === FIELDS.LINK.type) {
+        tableIdsToFetch.push(column.tableId)
+      }
+    }
+    tableIdsToFetch = [...new Set(tableIdsToFetch)]
+    // too many tables to fetch, just get all
+    if (tableIdsToFetch.length > 3) {
+      await fetch()
+    } else {
+      await Promise.all(tableIdsToFetch.map(id => singleFetch(id)))
+    }
     return savedTable
   }
 

packages/builder/src/stores/portal/index.js

@@ -1,3 +1,5 @@
+import { writable } from "svelte/store"
+
 export { organisation } from "./organisation"
 export { users } from "./users"
 export { admin } from "./admin"
@@ -14,3 +16,5 @@ export { environment } from "./environment"
 export { menu } from "./menu"
 export { auditLogs } from "./auditLogs"
 export { features } from "./features"
+
+export const sideBarCollapsed = writable(false)

packages/builder/tsconfig.json

@@ -5,8 +5,5 @@
     "declaration": true,
     "sourceMap": true,
     "baseUrl": "."
-  },
-  "ts-node": {
-    "require": ["tsconfig-paths/register"]
   }
 }

packages/cli/package.json

@@ -32,7 +32,6 @@
     "@budibase/backend-core": "0.0.0",
     "@budibase/string-templates": "0.0.0",
     "@budibase/types": "0.0.0",
-    "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",
     "commander": "7.1.0",
@@ -41,7 +40,6 @@
     "download": "8.0.0",
     "find-free-port": "^2.0.0",
     "inquirer": "8.0.0",
-    "joi": "17.6.0",
     "lookpath": "1.1.0",
     "node-fetch": "2.6.7",
     "pkg": "5.8.0",
@@ -53,15 +51,11 @@
     "yaml": "^2.1.1"
   },
   "devDependencies": {
-    "@swc/core": "1.3.71",
-    "@swc/jest": "0.2.27",
     "@types/jest": "29.5.3",
     "@types/node-fetch": "2.6.4",
     "@types/pouchdb": "^6.4.0",
-    "copyfiles": "^2.4.1",
-    "eslint": "^7.20.0",
     "renamer": "^4.0.0",
     "ts-node": "^10.9.1",
-    "typescript": "4.7.3"
+    "typescript": "5.2.2"
   }
 }

packages/cli/src/plugins/index.ts

@@ -49,8 +49,8 @@ async function askAboutTopLevel(name: string) {
   }
 }
 
-async function init(opts: PluginOpts) {
+async function init(opts: PluginOpts | PluginType) {
-  const type = opts["init"] || (opts as PluginType)
+  const type = (opts as PluginOpts).init || (opts as PluginType)
   if (!type || !PLUGIN_TYPE_ARR.includes(type)) {
     console.log(
       error(

packages/client/manifest.json

@@ -2539,7 +2539,8 @@
       {
         "type": "text",
         "label": "Default value",
-        "key": "defaultValue"
+        "key": "defaultValue",
+        "supportsConditions": false
       },
       {
         "type": "event",
@@ -2629,7 +2630,8 @@
       {
         "type": "text",
         "label": "Default value",
-        "key": "defaultValue"
+        "key": "defaultValue",
+        "supportsConditions": false
       },
       {
         "type": "event",
@@ -2685,7 +2687,8 @@
       {
         "type": "text",
         "label": "Default value",
-        "key": "defaultValue"
+        "key": "defaultValue",
+        "supportsConditions": false
       },
       {
         "type": "event",
@@ -2736,7 +2739,8 @@
       {
         "type": "text",
         "label": "Default value",
-        "key": "defaultValue"
+        "key": "defaultValue",
+        "supportsConditions": false
       },
       {
         "type": "event",
@@ -2841,7 +2845,8 @@
       {
         "type": "text",
         "label": "Default value",
-        "key": "defaultValue"
+        "key": "defaultValue",
+        "supportsConditions": false
       },
       {
         "type": "boolean",
@@ -2960,7 +2965,8 @@
       {
         "type": "text",
         "label": "Default value",
-        "key": "defaultValue"
+        "key": "defaultValue",
+        "supportsConditions": false
       },
       {
         "type": "event",
@@ -3143,7 +3149,8 @@
       {
         "type": "text",
         "label": "Default value",
-        "key": "defaultValue"
+        "key": "defaultValue",
+        "supportsConditions": false
       },
       {
         "type": "event",
@@ -3200,7 +3207,8 @@
       {
         "type": "text",
         "label": "Default value",
-        "key": "defaultValue"
+        "key": "defaultValue",
+        "supportsConditions": false
       },
       {
         "type": "event",
@@ -3301,7 +3309,8 @@
       {
         "type": "text",
         "label": "Default value",
-        "key": "defaultValue"
+        "key": "defaultValue",
+        "supportsConditions": false
       },
       {
         "type": "event",
@@ -3355,7 +3364,8 @@
       {
         "type": "text",
         "label": "Default value",
-        "key": "defaultValue"
+        "key": "defaultValue",
+        "supportsConditions": false
       },
       {
         "type": "boolean",
@@ -3622,7 +3632,8 @@
       {
         "type": "text",
         "label": "Default value",
-        "key": "defaultValue"
+        "key": "defaultValue",
+        "supportsConditions": false
       },
       {
         "type": "event",
@@ -3689,7 +3700,8 @@
       {
         "type": "text",
         "label": "Default value",
-        "key": "defaultValue"
+        "key": "defaultValue",
+        "supportsConditions": false
       },
       {
         "type": "event",
@@ -5611,5 +5623,73 @@
         "defaultValue": false
       }
     ]
+  },
+  "bbreferencefield": {
+    "devComment": "As bb reference is only used for user subtype for now, we are using user for icon and labels",
+    "name": "User Field",
+    "icon": "User",
+    "styles": ["size"],
+    "requiredAncestors": ["form"],
+    "editable": true,
+    "size": {
+      "width": 400,
+      "height": 50
+    },
+    "settings": [
+      {
+        "type": "field/bb_reference",
+        "label": "Field",
+        "key": "field",
+        "required": true
+      },
+      {
+        "type": "text",
+        "label": "Label",
+        "key": "label"
+      },
+      {
+        "type": "text",
+        "label": "Placeholder",
+        "key": "placeholder"
+      },
+      {
+        "type": "text",
+        "label": "Default value",
+        "key": "defaultValue"
+      },
+      {
+        "type": "event",
+        "label": "On change",
+        "key": "onChange",
+        "context": [
+          {
+            "label": "Field Value",
+            "key": "value"
+          }
+        ]
+      },
+      {
+        "type": "validation/link",
+        "label": "Validation",
+        "key": "validation"
+      },
+      {
+        "type": "filter/relationship",
+        "label": "Filtering",
+        "key": "filter"
+      },
+      {
+        "type": "boolean",
+        "label": "Search",
+        "key": "autocomplete",
+        "defaultValue": true
+      },
+      {
+        "type": "boolean",
+        "label": "Disabled",
+        "key": "disabled",
+        "defaultValue": false
+      }
+    ]
   }
 }

packages/client/package.json

@@ -33,7 +33,7 @@
     "@spectrum-css/typography": "^3.0.2",
     "@spectrum-css/vars": "^3.0.1",
     "apexcharts": "^3.22.1",
-    "dayjs": "^1.10.5",
+    "dayjs": "^1.10.8",
     "downloadjs": "1.4.7",
     "html5-qrcode": "^2.2.1",
     "leaflet": "^1.7.1",

packages/client/src/components/app/Layout.svelte

@@ -3,16 +3,16 @@
   import { writable } from "svelte/store"
   import { Heading, Icon, clickOutside } from "@budibase/bbui"
   import { FieldTypes } from "constants"
+  import { Constants } from "@budibase/frontend-core"
   import active from "svelte-spa-router/active"
-  import { RoleUtils } from "@budibase/frontend-core"
 
   const sdk = getContext("sdk")
   const {
     routeStore,
+    roleStore,
     styleable,
     linkable,
     builderStore,
-    currentRole,
     sidePanelStore,
   } = sdk
   const component = getContext("component")
@@ -61,7 +61,7 @@
   })
   setContext("layout", store)
 
-  $: validLinks = getValidLinks(links, $currentRole)
+  $: validLinks = getValidLinks(links, $roleStore)
   $: typeClass = NavigationClasses[navigation] || NavigationClasses.None
   $: navWidthClass = WidthClasses[navWidth || width] || WidthClasses.Large
   $: pageWidthClass = WidthClasses[pageWidth || width] || WidthClasses.Large
@@ -99,14 +99,13 @@
     }
   }
 
-  const getValidLinks = (allLinks, role) => {
+  const getValidLinks = (allLinks, userRoleHierarchy) => {
     // Strip links missing required info
     let validLinks = (allLinks || []).filter(link => link.text && link.url)
-
     // Filter to only links allowed by the current role
-    const priority = RoleUtils.getRolePriority(role)
     return validLinks.filter(link => {
-      return !link.roleId || RoleUtils.getRolePriority(link.roleId) <= priority
+      const role = link.roleId || Constants.Roles.BASIC
+      return userRoleHierarchy?.find(roleId => roleId === role)
     })
   }
 

packages/client/src/components/app/blocks/form/InnerFormBlock.svelte

@@ -30,6 +30,7 @@
     link: "relationshipfield",
     json: "jsonfield",
     barcodeqr: "codescanner",
+    bb_reference: "bbreferencefield",
   }
 
   let formId

packages/client/src/components/app/forms/BBReferenceField.svelte

@@ -0,0 +1,9 @@
+<script>
+  import RelationshipField from "./RelationshipField.svelte"
+</script>
+
+<RelationshipField
+  {...$$props}
+  datasourceType={"user"}
+  primaryDisplay={"email"}
+/>

packages/client/src/components/app/forms/InnerForm.svelte

@@ -230,10 +230,16 @@
       // We want to validate every field (even if validation fails early) to
       // ensure that all fields are populated with errors if invalid
       let valid = true
+      let hasScrolled = false
       stepFields.forEach(field => {
         const fieldValid = get(field).fieldApi.validate()
         valid = valid && fieldValid
+        if (!valid && !hasScrolled) {
+          handleScrollToField({ field: get(field) })
+          hasScrolled = true
+        }
       })
+
       return valid
     },
     reset: () => {
@@ -409,10 +415,15 @@
   }
 
   const handleScrollToField = ({ field }) => {
-    const fieldId = get(getField(field)).fieldState.fieldId
+    if (!field.fieldState) {
+      field = get(getField(field))
+    }
+    const fieldId = field.fieldState.fieldId
+    const fieldElement = document.getElementById(fieldId)
+    fieldElement.focus({ preventScroll: true })
     const label = document.querySelector(`label[for="${fieldId}"]`)
-    document.getElementById(fieldId).focus({ preventScroll: true })
+    label.style.scrollMargin = "100px"
-    label.scrollIntoView({ behavior: "smooth" })
+    label.scrollIntoView({ behavior: "smooth", block: "nearest" })
   }
 
   // Action context to pass to children

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -1,11 +1,6 @@
 <script>
-  import {
+  import { CoreSelect, CoreMultiselect } from "@budibase/bbui"
-    CoreSelect,
+  import { fetchData } from "@budibase/frontend-core"
-    CoreMultiselect,
-    Input,
-    ProgressCircle,
-  } from "@budibase/bbui"
-  import { fetchData, Utils } from "@budibase/frontend-core"
   import { getContext } from "svelte"
   import Field from "./Field.svelte"
   import { FieldTypes } from "../../../constants"
@@ -21,28 +16,25 @@
   export let defaultValue
   export let onChange
   export let filter
+  export let datasourceType = "table"
+  export let primaryDisplay
 
   let fieldState
   let fieldApi
   let fieldSchema
   let tableDefinition
-  let primaryDisplay
+  let searchTerm
-  let options
+  let open
-  let selectedOptions = []
-  let isOpen = false
-  let hasFilter
 
-  let searchResults
+  $: type =
-  let searchString
+    datasourceType === "table" ? FieldTypes.LINK : FieldTypes.BB_REFERENCE
-  let searching = false
-  let lastSearchId
 
   $: multiselect = fieldSchema?.relationshipType !== "one-to-many"
   $: linkedTableId = fieldSchema?.tableId
   $: fetch = fetchData({
     API,
     datasource: {
-      type: "table",
+      type: datasourceType,
       tableId: linkedTableId,
     },
     options: {
@@ -50,54 +42,80 @@
       limit: 100,
     },
   })
-  $: hasFilter = !!filter?.filter(f => !!f.field)?.length
+
-  $: fetch.update({ filter })
-  $: {
-    options = searchResults ? searchResults : $fetch.rows
-    const nonMatchingOptions = selectedOptions.filter(
-      option => !options.map(opt => opt._id).includes(option._id)
-    )
-    // Append initially selected options if there is no filter
-    // and hasn't already been appended
-    if (!hasFilter) {
-      options = [...options, ...nonMatchingOptions]
-    }
-  }
   $: tableDefinition = $fetch.definition
-  $: primaryDisplay = tableDefinition?.primaryDisplay || "_id"
+  $: selectedValue = multiselect
-  $: singleValue = flatten(fieldState?.value)?.[0]
+    ? flatten(fieldState?.value) ?? []
-  $: multiValue = flatten(fieldState?.value) ?? []
+    : flatten(fieldState?.value)?.[0]
   $: component = multiselect ? CoreMultiselect : CoreSelect
   $: expandedDefaultValue = expand(defaultValue)
-  $: debouncedSearch(searchString)
+  $: primaryDisplay = primaryDisplay || tableDefinition?.primaryDisplay
+
+  let optionsObj = {}
+  let initialValuesProcessed
+
   $: {
-    if (searching) {
+    if (!initialValuesProcessed && primaryDisplay) {
-      isOpen = true
+      // Persist the initial values as options, allowing them to be present in the dropdown,
+      // even if they are not in the inital fetch results
+      initialValuesProcessed = true
+      optionsObj = (fieldState?.value || []).reduce((accumulator, value) => {
+        // fieldState has to be an array of strings to be valid for an update
+        // therefore we cannot guarantee value will be an object
+        // https://linear.app/budibase/issue/BUDI-7577/refactor-the-relationshipfield-component-to-have-better-support-for
+        if (!value._id) {
+          return accumulator
+        }
+        accumulator[value._id] = {
+          _id: value._id,
+          [primaryDisplay]: value.primaryDisplay,
+        }
+        return accumulator
+      }, optionsObj)
     }
   }
 
-  // Fetch the initially selected values
+  $: enrichedOptions = enrichOptions(optionsObj, $fetch.rows)
-  // as they may not be within the first 100 records
+  const enrichOptions = (optionsObj, fetchResults) => {
+    const result = (fetchResults || [])?.reduce((accumulator, row) => {
+      if (!accumulator[row._id]) {
+        accumulator[row._id] = row
+      }
+      return accumulator
+    }, optionsObj)
+
+    return Object.values(result)
+  }
   $: {
-    if (
+    // We don't want to reorder while the dropdown is open, to avoid UX jumps
-      primaryDisplay !== "_id" &&
+    if (!open) {
-      fieldState?.value?.length &&
+      enrichedOptions = enrichedOptions.sort((a, b) => {
-      !selectedOptions?.length
+        const selectedValues = flatten(fieldState?.value) || []
-    ) {
+
-      API.searchTable({
+        const aIsSelected = selectedValues.find(v => v === a._id)
-        paginate: false,
+        const bIsSelected = selectedValues.find(v => v === b._id)
-        tableId: linkedTableId,
+        if (aIsSelected && !bIsSelected) {
-        limit: 100,
+          return -1
-        query: {
+        } else if (!aIsSelected && bIsSelected) {
-          oneOf: {
+          return 1
-            [`1:${primaryDisplay}`]: fieldState?.value?.map(
+        }
-              value => value.primaryDisplay
+
-            ),
+        return a[primaryDisplay] > b[primaryDisplay]
-          },
+      })
-        },
+    }
-      }).then(response => {
+  }
-        const value = multiselect ? multiValue : singleValue
+
-        selectedOptions = response.rows.filter(row => value.includes(row._id))
+  $: fetchRows(searchTerm, primaryDisplay)
+
+  const fetchRows = (searchTerm, primaryDisplay) => {
+    const allRowsFetched =
+      $fetch.loaded &&
+      !Object.keys($fetch.query?.string || {}).length &&
+      !$fetch.hasNextPage
+    // Don't request until we have the primary display
+    if (!allRowsFetched && primaryDisplay) {
+      fetch.update({
+        query: { string: { [primaryDisplay]: searchTerm } },
       })
     }
   }
@@ -109,11 +127,16 @@
     if (!Array.isArray(values)) {
       values = [values]
     }
-    return values.map(value => (typeof value === "object" ? value._id : value))
+    values = values.map(value =>
+      typeof value === "object" ? value._id : value
+    )
+    // Make sure field state is valid
+    fieldApi.setValue(values)
+    return values
   }
 
   const getDisplayName = row => {
-    return row?.[tableDefinition?.primaryDisplay || "_id"] || "-"
+    return row?.[primaryDisplay] || "-"
   }
 
   const singleHandler = e => {
@@ -136,66 +159,16 @@
 
   const handleChange = value => {
     const changed = fieldApi.setValue(value)
-    selectedOptions = value.map(val => ({
-      _id: val,
-      [primaryDisplay]: options.find(option => option._id === val)[
-        primaryDisplay
-      ],
-    }))
     if (onChange && changed) {
       onChange({ value })
     }
   }
 
-  // Search for rows based on the search string
+  const loadMore = () => {
-  const search = async searchString => {
+    if (!$fetch.loading) {
-    // Reset state if this search is invalid
+      fetch.nextPage()
-    if (!linkedTableId || !searchString) {
-      searchResults = null
-      return
     }
-
-    // If a filter exists, then do a client side search
-    if (hasFilter) {
-      searchResults = $fetch.rows.filter(option =>
-        option[primaryDisplay].startsWith(searchString)
-      )
-      isOpen = true
-      return
-    }
-
-    // Search for results, using IDs to track invocations and ensure we're
-    // handling the latest update
-    lastSearchId = Math.random()
-    searching = true
-    const thisSearchId = lastSearchId
-    const results = await API.searchTable({
-      paginate: false,
-      tableId: linkedTableId,
-      limit: 100,
-      query: {
-        string: {
-          [`1:${primaryDisplay}`]: searchString || "",
-        },
-      },
-    })
-    searching = false
-
-    // In case searching takes longer than our debounced update, abandon these
-    // results
-    if (thisSearchId !== lastSearchId) {
-      return
-    }
-
-    // Process results
-    searchResults = results.rows?.map(row => ({
-      ...row,
-      primaryDisplay: row[primaryDisplay],
-    }))
   }
-
-  // Debounced version of searching
-  const debouncedSearch = Utils.debounce(search, 250)
 </script>
 
 <Field
@@ -204,69 +177,29 @@
   {disabled}
   {validation}
   defaultValue={expandedDefaultValue}
-  type={FieldTypes.LINK}
+  {type}
   bind:fieldState
   bind:fieldApi
   bind:fieldSchema
 >
   {#if fieldState}
-    <div class={autocomplete ? "field-with-search" : ""}>
+    <svelte:component
-      <svelte:component
+      this={component}
-        this={component}
+      options={enrichedOptions}
-        bind:open={isOpen}
+      {autocomplete}
-        {options}
+      value={selectedValue}
-        autocomplete={false}
+      on:change={multiselect ? multiHandler : singleHandler}
-        value={multiselect ? multiValue : singleValue}
+      on:loadMore={loadMore}
-        on:change={multiselect ? multiHandler : singleHandler}
+      id={fieldState.fieldId}
-        id={fieldState.fieldId}
+      disabled={fieldState.disabled}
-        disabled={fieldState.disabled}
+      error={fieldState.error}
-        error={fieldState.error}
+      getOptionLabel={getDisplayName}
-        getOptionLabel={getDisplayName}
+      getOptionValue={option => option._id}
-        getOptionValue={option => option._id}
+      {placeholder}
-        {placeholder}
+      bind:searchTerm
-        customPopoverOffsetBelow={autocomplete ? 32 : null}
+      loading={$fetch.loading}
-        customPopoverMaxHeight={autocomplete ? 240 : null}
+      bind:open
-        sort={true}
+      customPopoverMaxHeight={400}
-      />
+    />
-      {#if autocomplete}
-        <div class="search">
-          <Input
-            autofocus
-            quiet
-            type="text"
-            bind:value={searchString}
-            placeholder={primaryDisplay ? `Search by ${primaryDisplay}` : null}
-          />
-          {#if searching}
-            <div>
-              <ProgressCircle size="S" />
-            </div>
-          {/if}
-        </div>
-      {/if}
-    </div>
   {/if}
 </Field>
-
-<style>
-  .search {
-    flex: 0 0 calc(var(--default-row-height) - 1px);
-    display: flex;
-    align-items: center;
-    margin: 4px var(--cell-padding);
-    width: calc(100% - 2 * var(--cell-padding));
-  }
-  .search :global(.spectrum-Textfield) {
-    min-width: 0;
-    width: 100%;
-  }
-  .search :global(.spectrum-Textfield-input) {
-    font-size: 13px;
-  }
-  .search :global(.spectrum-Form-item) {
-    flex: 1 1 auto;
-  }
-  .field-with-search {
-    min-height: 80px;
-  }
-</style>

packages/client/src/components/app/forms/index.js

@@ -15,3 +15,4 @@ export { default as formstep } from "./FormStep.svelte"
 export { default as jsonfield } from "./JSONField.svelte"
 export { default as s3upload } from "./S3Upload.svelte"
 export { default as codescanner } from "./CodeScannerField.svelte"
+export { default as bbreferencefield } from "./BBReferenceField.svelte"

packages/client/src/components/devtools/DevToolsHeader.svelte

@@ -1,32 +1,38 @@
 <script>
   import { Heading, Select, ActionButton } from "@budibase/bbui"
-  import { devToolsStore, appStore } from "../../stores"
+  import { devToolsStore, appStore, roleStore } from "../../stores"
-  import { getContext } from "svelte"
+  import { getContext, onMount } from "svelte"
 
   const context = getContext("context")
+  const SELF_ROLE = "self"
 
-  $: previewOptions = [
+  let staticRoleList
-    {
+
+  $: previewOptions = buildRoleList(staticRoleList)
+
+  function buildRoleList(roleIds) {
+    const list = []
+    list.push({
       label: "View as yourself",
-      value: "self",
+      value: SELF_ROLE,
-    },
+    })
-    {
+    if (!roleIds) {
-      label: "View as public user",
+      return list
-      value: "PUBLIC",
+    }
-    },
+    for (let roleId of roleIds) {
-    {
+      list.push({
-      label: "View as basic user",
+        label: `View as ${roleId.toLowerCase()} user`,
-      value: "BASIC",
+        value: roleId,
-    },
+      })
-    {
+    }
-      label: "View as power user",
+    return list
-      value: "POWER",
+  }
-    },
+
-    {
+  onMount(async () => {
-      label: "View as admin user",
+    // make sure correct before starting
-      value: "ADMIN",
+    await devToolsStore.actions.changeRole(SELF_ROLE)
-    },
+    staticRoleList = await roleStore.actions.fetchAccessibleRoles()
-  ]
+  })
 </script>
 
 <div class="dev-preview-header" class:mobile={$context.device.mobile}>
@@ -34,7 +40,7 @@
   <Select
     quiet
     options={previewOptions}
-    value={$devToolsStore.role || "self"}
+    value={$devToolsStore.role || SELF_ROLE}
     placeholder={null}
     autoWidth
     on:change={e => devToolsStore.actions.changeRole(e.detail)}

packages/client/src/constants.js

@@ -1,17 +1,5 @@
-export const FieldTypes = {
+import { FieldType as FieldTypes } from "@budibase/types"
-  STRING: "string",
+export { FieldType as FieldTypes } from "@budibase/types"
-  BARCODEQR: "barcodeqr",
-  LONGFORM: "longform",
-  OPTIONS: "options",
-  NUMBER: "number",
-  BOOLEAN: "boolean",
-  ARRAY: "array",
-  DATETIME: "datetime",
-  ATTACHMENT: "attachment",
-  LINK: "link",
-  FORMULA: "formula",
-  JSON: "json",
-}
 
 export const UnsortableTypes = [
   FieldTypes.FORMULA,