Merge branch 'Budibase:develop' into allow-plugins-contribute-datasourceplus

2023-10-02 17:49:34 -04:00 · 2023-10-02 17:49:34 -04:00 · 84d30e4f4a
parent 08bbfa09c2 2ca5b0fa30
commit 84d30e4f4a
102 changed files with 1764 additions and 3901 deletions
--- a/.github/workflows/close-featurebranch.yml
+++ b/.github/workflows/close-featurebranch.yml
@ -14,7 +14,7 @@ jobs:
      - uses: passeidireto/trigger-external-workflow-action@main
        env:
          PAYLOAD_BRANCH: ${{ github.head_ref }}
-          PAYLOAD_PR_NUMBER: ${{ github.ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
        with:
          repository: budibase/budibase-deploys
          event: featurebranch-qa-close
--- a/.github/workflows/deploy-featurebranch.yml
+++ b/.github/workflows/deploy-featurebranch.yml
@ -13,7 +13,7 @@ jobs:
      - uses: passeidireto/trigger-external-workflow-action@main
        env:
          PAYLOAD_BRANCH: ${{ github.head_ref }}
-          PAYLOAD_PR_NUMBER: ${{ github.ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
        with:
          repository: budibase/budibase-deploys
          event: featurebranch-qa-deploy
--- a/lerna.json
+++ b/lerna.json
@ -1,5 +1,5 @@
 {
-  "version": "2.10.12-alpha.23",
+  "version": "2.11.5-alpha.2",
  "npmClient": "yarn",
  "packages": [
    "packages/*"
--- a/package.json
+++ b/package.json
@ -21,7 +21,7 @@
    "prettier-plugin-svelte": "^2.3.0",
    "rimraf": "^3.0.2",
    "rollup-plugin-replace": "^2.2.0",
-    "svelte": "^3.38.2",
+    "svelte": "3.49.0",
    "typescript": "5.2.2",
    "@babel/core": "^7.22.5",
    "@babel/eslint-parser": "^7.22.5",
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@ -33,17 +33,14 @@
    "bull": "4.10.1",
    "correlation-id": "4.0.0",
    "dotenv": "16.0.1",
-    "emitter-listener": "1.1.2",
    "ioredis": "5.3.2",
    "joi": "17.6.0",
    "jsonwebtoken": "9.0.0",
    "koa-passport": "4.1.4",
    "koa-pino-logger": "4.0.0",
    "lodash": "4.17.21",
-    "lodash.isarguments": "3.1.0",
    "node-fetch": "2.6.7",
    "passport-google-oauth": "2.0.0",
-    "passport-jwt": "4.0.0",
    "passport-local": "1.0.0",
    "passport-oauth2-refresh": "^2.1.0",
    "pino": "8.11.0",
@ -59,14 +56,13 @@
    "uuid": "8.3.2"
  },
  "devDependencies": {
-    "@jest/test-sequencer": "29.6.2",
    "@shopify/jest-koa-mocks": "5.1.1",
    "@swc/core": "1.3.71",
    "@swc/jest": "0.2.27",
    "@trendyol/jest-testcontainers": "^2.1.1",
    "@types/chance": "1.1.3",
+    "@types/cookies": "0.7.8",
    "@types/jest": "29.5.3",
-    "@types/koa": "2.13.4",
    "@types/lodash": "4.14.180",
    "@types/node": "18.17.0",
    "@types/node-fetch": "2.6.4",
@ -80,13 +76,9 @@
    "jest": "29.6.2",
    "jest-environment-node": "29.6.2",
    "jest-serial-runner": "1.2.1",
-    "koa": "2.13.4",
-    "nodemon": "2.0.16",
    "pino-pretty": "10.0.0",
    "pouchdb-adapter-memory": "7.2.2",
    "timekeeper": "2.2.0",
-    "ts-node": "10.8.1",
-    "tsconfig-paths": "4.0.0",
    "typescript": "5.2.2"
  },
  "nx": {
--- a/packages/backend-core/src/constants/db.ts
+++ b/packages/backend-core/src/constants/db.ts
@ -18,7 +18,7 @@ export enum ViewName {
  ROUTING = "screen_routes",
  AUTOMATION_LOGS = "automation_logs",
  ACCOUNT_BY_EMAIL = "account_by_email",
-  PLATFORM_USERS_LOWERCASE = "platform_users_lowercase",
+  PLATFORM_USERS_LOWERCASE = "platform_users_lowercase_2",
  USER_BY_GROUP = "user_by_group",
  APP_BACKUP_BY_TRIGGER = "by_trigger",
 }
--- a/packages/backend-core/src/db/views.ts
+++ b/packages/backend-core/src/db/views.ts
@ -190,6 +190,10 @@ export const createPlatformUserView = async () => {
    if (doc.tenantId) {
      emit(doc._id.toLowerCase(), doc._id)
    }
+
+    if (doc.ssoId) {
+      emit(doc.ssoId, doc._id)
+    }
  }`
  await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
 }
--- a/packages/backend-core/src/platform/users.ts
+++ b/packages/backend-core/src/platform/users.ts
@ -5,6 +5,7 @@ import {
  PlatformUser,
  PlatformUserByEmail,
  PlatformUserById,
+  PlatformUserBySsoId,
  User,
 } from "@budibase/types"

@ -45,6 +46,20 @@ function newUserEmailDoc(
  }
 }

+function newUserSsoIdDoc(
+  ssoId: string,
+  email: string,
+  userId: string,
+  tenantId: string
+): PlatformUserBySsoId {
+  return {
+    _id: ssoId,
+    userId,
+    email,
+    tenantId,
+  }
+}
+
 /**
 * Add a new user id or email doc if it doesn't exist.
 */
@ -64,11 +79,24 @@ async function addUserDoc(emailOrId: string, newDocFn: () => PlatformUser) {
  }
 }

-export async function addUser(tenantId: string, userId: string, email: string) {
-  await Promise.all([
+export async function addUser(
+  tenantId: string,
+  userId: string,
+  email: string,
+  ssoId?: string
+) {
+  const promises = [
    addUserDoc(userId, () => newUserIdDoc(userId, tenantId)),
    addUserDoc(email, () => newUserEmailDoc(userId, email, tenantId)),
-  ])
+  ]
+
+  if (ssoId) {
+    promises.push(
+      addUserDoc(ssoId, () => newUserSsoIdDoc(ssoId, email, userId, tenantId))
+    )
+  }
+
+  await Promise.all(promises)
 }

 // DELETE
--- a/packages/backend-core/src/security/permissions.ts
+++ b/packages/backend-core/src/security/permissions.ts
@ -1,8 +1,9 @@
-import { PermissionType, PermissionLevel } from "@budibase/types"
-export { PermissionType, PermissionLevel } from "@budibase/types"
+import { PermissionLevel, PermissionType } from "@budibase/types"
 import flatten from "lodash/flatten"
 import cloneDeep from "lodash/fp/cloneDeep"

+export { PermissionType, PermissionLevel } from "@budibase/types"
+
 export type RoleHierarchy = {
  permissionId: string
 }[]
@ -78,6 +79,7 @@ export const BUILTIN_PERMISSIONS = {
    permissions: [
      new Permission(PermissionType.QUERY, PermissionLevel.READ),
      new Permission(PermissionType.TABLE, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
    ],
  },
  WRITE: {
@ -88,6 +90,7 @@ export const BUILTIN_PERMISSIONS = {
      new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
      new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
      new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
    ],
  },
  POWER: {
@ -99,6 +102,7 @@ export const BUILTIN_PERMISSIONS = {
      new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
      new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
      new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
    ],
  },
  ADMIN: {
@ -111,6 +115,7 @@ export const BUILTIN_PERMISSIONS = {
      new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
      new Permission(PermissionType.QUERY, PermissionLevel.ADMIN),
      new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
    ],
  },
 }
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@ -215,21 +215,23 @@ async function getAllUserRoles(userRoleId?: string): Promise<RoleDoc[]> {
  return roles
 }

+export async function getUserRoleIdHierarchy(
+  userRoleId?: string
+): Promise<string[]> {
+  const roles = await getUserRoleHierarchy(userRoleId)
+  return roles.map(role => role._id!)
+}
+
 /**
 * Returns an ordered array of the user's inherited role IDs, this can be used
 * to determine if a user can access something that requires a specific role.
 * @param {string} userRoleId The user's role ID, this can be found in their access token.
- * @param {object} opts Various options, such as whether to only retrieve the IDs (default true).
- * @returns {Promise<string[]|object[]>} returns an ordered array of the roles, with the first being their
+ * @returns {Promise<object[]>} returns an ordered array of the roles, with the first being their
 * highest level of access and the last being the lowest level.
 */
-export async function getUserRoleHierarchy(
-  userRoleId?: string,
-  opts = { idOnly: true }
-) {
+export async function getUserRoleHierarchy(userRoleId?: string) {
  // special case, if they don't have a role then they are a public user
-  const roles = await getAllUserRoles(userRoleId)
-  return opts.idOnly ? roles.map(role => role._id) : roles
+  return getAllUserRoles(userRoleId)
 }

 // this function checks that the provided permissions are in an array format
@ -249,6 +251,11 @@ export function checkForRoleResourceArray(
  return rolePerms
 }

+export async function getAllRoleIds(appId?: string) {
+  const roles = await getAllRoles(appId)
+  return roles.map(role => role._id)
+}
+
 /**
 * Given an app ID this will retrieve all of the roles that are currently within that app.
 * @return {Promise<object[]>} An array of the role objects that were found.
@ -332,9 +339,7 @@ export class AccessController {
    }
    let roleIds = userRoleId ? this.userHierarchies[userRoleId] : null
    if (!roleIds && userRoleId) {
-      roleIds = (await getUserRoleHierarchy(userRoleId, {
-        idOnly: true,
-      })) as string[]
+      roleIds = await getUserRoleIdHierarchy(userRoleId)
      this.userHierarchies[userRoleId] = roleIds
    }

--- a/packages/backend-core/src/users/db.ts
+++ b/packages/backend-core/src/users/db.ts
@ -278,7 +278,12 @@ export class UserDB {
        builtUser._rev = response.rev

        await eventHelpers.handleSaveEvents(builtUser, dbUser)
-        await platform.users.addUser(tenantId, builtUser._id!, builtUser.email)
+        await platform.users.addUser(
+          tenantId,
+          builtUser._id!,
+          builtUser.email,
+          builtUser.ssoId
+        )
        await cache.user.invalidateUser(response.id)

        await Promise.all(groupPromises)
--- a/packages/backend-core/src/utils/utils.ts
+++ b/packages/backend-core/src/utils/utils.ts
@ -10,7 +10,7 @@ import {
  Event,
  TenantResolutionStrategy,
 } from "@budibase/types"
-import { SetOption } from "cookies"
+import type { SetOption } from "cookies"
 const jwt = require("jsonwebtoken")

 const APP_PREFIX = DocumentType.APP + SEPARATOR
--- a/packages/backend-core/tests/core/utilities/structures/accounts.ts
+++ b/packages/backend-core/tests/core/utilities/structures/accounts.ts
@ -1,4 +1,4 @@
-import { generator, uuid, quotas } from "."
+import { generator, quotas, uuid } from "."
 import { generateGlobalUserID } from "../../../../src/docIds"
 import {
  Account,
@ -6,10 +6,11 @@ import {
  AccountSSOProviderType,
  AuthType,
  CloudAccount,
-  Hosting,
-  SSOAccount,
  CreateAccount,
  CreatePassswordAccount,
+  CreateVerifiableSSOAccount,
+  Hosting,
+  SSOAccount,
 } from "@budibase/types"
 import sample from "lodash/sample"

@ -68,6 +69,23 @@ export function ssoAccount(account: Account = cloudAccount()): SSOAccount {
  }
 }

+export function verifiableSsoAccount(
+  account: Account = cloudAccount()
+): SSOAccount {
+  return {
+    ...account,
+    authType: AuthType.SSO,
+    oauth2: {
+      accessToken: generator.string(),
+      refreshToken: generator.string(),
+    },
+    pictureUrl: generator.url(),
+    provider: AccountSSOProvider.MICROSOFT,
+    providerType: AccountSSOProviderType.MICROSOFT,
+    thirdPartyProfile: { id: "abc123" },
+  }
+}
+
 export const cloudCreateAccount: CreatePassswordAccount = {
  email: "cloud@budibase.com",
  tenantId: "cloud",
@ -91,6 +109,19 @@ export const cloudSSOCreateAccount: CreateAccount = {
  profession: "Software Engineer",
 }

+export const cloudVerifiableSSOCreateAccount: CreateVerifiableSSOAccount = {
+  email: "cloud-sso@budibase.com",
+  tenantId: "cloud-sso",
+  hosting: Hosting.CLOUD,
+  authType: AuthType.SSO,
+  tenantName: "cloudsso",
+  name: "Budi Armstrong",
+  size: "10+",
+  profession: "Software Engineer",
+  provider: AccountSSOProvider.MICROSOFT,
+  thirdPartyProfile: { id: "abc123" },
+}
+
 export const selfCreateAccount: CreatePassswordAccount = {
  email: "self@budibase.com",
  tenantId: "self",
--- a/packages/bbui/package.json
+++ b/packages/bbui/package.json
@ -20,14 +20,12 @@
    "@rollup/plugin-commonjs": "^16.0.0",
    "@rollup/plugin-json": "^4.1.0",
    "@rollup/plugin-node-resolve": "^11.2.1",
-    "cross-env": "^7.0.2",
-    "nollup": "^0.14.1",
    "postcss": "^8.2.9",
    "rollup": "^2.45.2",
    "rollup-plugin-postcss": "^4.0.0",
    "rollup-plugin-svelte": "^7.1.0",
    "rollup-plugin-terser": "^7.0.2",
-    "svelte": "^3.38.2"
+    "svelte": "3.49.0"
  },
  "keywords": [
    "svelte"
--- a/packages/builder/package.json
+++ b/packages/builder/package.json
@ -65,7 +65,6 @@
    "@fortawesome/free-brands-svg-icons": "^6.2.1",
    "@fortawesome/free-solid-svg-icons": "^6.2.1",
    "@sentry/browser": "5.19.1",
-    "@spectrum-css/accordion": "^3.0.24",
    "@spectrum-css/page": "^3.0.1",
    "@spectrum-css/vars": "^3.0.1",
    "codemirror": "^5.59.0",
@ -75,18 +74,17 @@
    "lodash": "4.17.21",
    "posthog-js": "^1.36.0",
    "remixicon": "2.5.0",
+    "sanitize-html": "^2.7.0",
    "shortid": "2.2.15",
    "svelte-dnd-action": "^0.9.8",
    "svelte-loading-spinners": "^0.1.1",
    "svelte-portal": "1.0.0",
-    "uuid": "8.3.1",
    "yup": "0.29.2"
  },
  "devDependencies": {
    "@babel/core": "^7.12.14",
    "@babel/plugin-transform-runtime": "^7.13.10",
    "@babel/preset-env": "^7.13.12",
-    "@babel/runtime": "^7.13.10",
    "@rollup/plugin-replace": "^2.4.2",
    "@roxi/routify": "2.18.5",
    "@sveltejs/vite-plugin-svelte": "1.0.1",
@ -96,19 +94,10 @@
    "identity-obj-proxy": "^3.0.0",
    "jest": "29.6.2",
    "jsdom": "^21.1.1",
-    "mochawesome": "^7.1.3",
-    "mochawesome-merge": "^4.2.1",
-    "mochawesome-report-generator": "^6.2.0",
    "ncp": "^2.0.0",
-    "rimraf": "^3.0.2",
    "rollup": "^2.44.0",
-    "rollup-plugin-copy": "^3.4.0",
-    "start-server-and-test": "^1.12.1",
    "svelte": "^3.48.0",
    "svelte-jester": "^1.3.2",
-    "ts-node": "10.8.1",
-    "tsconfig-paths": "4.0.0",
-    "typescript": "5.2.2",
    "vite": "^3.0.8",
    "vite-plugin-static-copy": "^0.16.0",
    "vitest": "^0.29.2"
--- a/packages/builder/src/builderStore/store/automation/index.js
+++ b/packages/builder/src/builderStore/store/automation/index.js
@ -221,18 +221,6 @@ const automationActions = store => ({
    newAutomation.definition.steps.splice(blockIdx, 0, block)
    await store.actions.save(newAutomation)
  },
-  /**
-   * "rowControl" appears to be the name of the flag used to determine whether
-   * a certain automation block uses values or bindings as inputs
-   */
-  toggleRowControl: async (block, rowControl) => {
-    const newBlock = { ...block, rowControl }
-    const newAutomation = store.actions.getUpdatedDefinition(
-      get(selectedAutomation),
-      newBlock
-    )
-    await store.actions.save(newAutomation)
-  },
  deleteAutomationBlock: async block => {
    const automation = get(selectedAutomation)
    let newAutomation = cloneDeep(automation)
--- a/packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js
+++ b/packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js
@ -33,6 +33,8 @@ const generateTableBlock = datasource => {
      showTitleButton: true,
      titleButtonText: "Create row",
      titleButtonClickBehaviour: "new",
+      sidePanelSaveLabel: "Save",
+      sidePanelDeleteLabel: "Delete",
    })
    .instanceName(`${datasource.label} - Table block`)
  return tableBlock
--- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte
+++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte
@ -7,7 +7,6 @@
    Detail,
    Modal,
    Button,
-    Select,
    ActionButton,
    notifications,
    Label,
@ -39,9 +38,6 @@
    step => step.stepId === ActionStepID.COLLECT
  )
  $: automationId = $selectedAutomation?._id
-  $: showBindingPicker =
-    block.stepId === ActionStepID.CREATE_ROW ||
-    block.stepId === ActionStepID.UPDATE_ROW
  $: isTrigger = block.type === "TRIGGER"
  $: steps = $selectedAutomation?.definition?.steps ?? []
  $: blockIdx = steps.findIndex(step => step.id === block.id)
@ -96,15 +92,6 @@
    }
  }

-  /**
-   * "rowControl" appears to be the name of the flag used to determine whether
-   * a certain automation block uses values or bindings as inputs
-   */
-  function toggleRowControl(evt) {
-    const rowControl = evt.detail !== "Use values"
-    automationStore.actions.toggleRowControl(block, rowControl)
-  }
-
  async function addLooping() {
    const loopDefinition = $automationStore.blockDefinitions.ACTION.LOOP
    const loopBlock = automationStore.actions.constructBlock(
@ -189,16 +176,6 @@
                  Add Looping
                </ActionButton>
              {/if}
-              {#if showBindingPicker}
-                <Select
-                  on:change={toggleRowControl}
-                  defaultValue="Use values"
-                  autoWidth
-                  value={block.rowControl ? "Use bindings" : "Use values"}
-                  options={["Use values", "Use bindings"]}
-                  placeholder={null}
-                />
-              {/if}
              <ActionButton
                on:click={() => deleteStep()}
                icon="DeleteOutline"
--- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
@ -23,6 +23,7 @@
  import { environment, licensing } from "stores/portal"
  import WebhookDisplay from "../Shared/WebhookDisplay.svelte"
  import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
+  import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
  import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
  import CodeEditorModal from "./CodeEditorModal.svelte"
  import QuerySelector from "./QuerySelector.svelte"
@ -82,33 +83,6 @@
      ? [hbAutocomplete([...bindingsToCompletions(bindings, codeMode)])]
      : []

-  /**
-   * TODO - Remove after November 2023
-   * *******************************
-   * Code added to provide backwards compatibility between Values 1,2,3,4,5
-   * and the new JSON body.
-   */
-  let deprecatedSchemaProperties
-  $: {
-    if (block?.stepId === "integromat" || block?.stepId === "zapier") {
-      deprecatedSchemaProperties = schemaProperties.filter(
-        prop => !prop[0].startsWith("value")
-      )
-      if (!deprecatedSchemaProperties.map(entry => entry[0]).includes("body")) {
-        deprecatedSchemaProperties.push([
-          "body",
-          {
-            title: "Payload",
-            type: "json",
-          },
-        ])
-      }
-    } else {
-      deprecatedSchemaProperties = schemaProperties
-    }
-  }
-  /****************************************************/
-
  const getInputData = (testData, blockInputs) => {
    // Test data is not cloned for reactivity
    let newInputData = testData || cloneDeep(blockInputs)
@ -118,30 +92,6 @@
      newInputData = cloneDeep(blockInputs)
    }

-    /**
-     * TODO - Remove after November 2023
-     * *******************************
-     * Code added to provide backwards compatibility between Values 1,2,3,4,5
-     * and the new JSON body.
-     */
-    if (
-      (block?.stepId === "integromat" || block?.stepId === "zapier") &&
-      !newInputData?.body?.value
-    ) {
-      let deprecatedValues = {
-        ...newInputData,
-      }
-      delete deprecatedValues.url
-      delete deprecatedValues.body
-      newInputData = {
-        url: newInputData.url,
-        body: {
-          value: JSON.stringify(deprecatedValues),
-        },
-      }
-    }
-    /**********************************/
-
    inputData = newInputData
    setDefaultEnumValues()
  }
@ -337,7 +287,7 @@
 </script>

 <div class="fields">
-  {#each deprecatedSchemaProperties as [key, value]}
+  {#each schemaProperties as [key, value]}
    {#if canShowField(key, value)}
      <div class="block-field">
        {#if key !== "fields" && value.type !== "boolean"}
@ -362,18 +312,6 @@
            mode="json"
            value={inputData[key]?.value}
            on:change={e => {
-              /**
-               * TODO - Remove after November 2023
-               * *******************************
-               * Code added to provide backwards compatibility between Values 1,2,3,4,5
-               * and the new JSON body.
-               */
-              delete inputData.value1
-              delete inputData.value2
-              delete inputData.value3
-              delete inputData.value4
-              delete inputData.value5
-              /***********************/
              onChange(e, key)
            }}
          />
@ -386,10 +324,23 @@
            />
          </div>
        {:else if value.type === "date"}
-          <DatePicker
+          <DrawerBindableSlot
+            fillWidth
+            title={value.title}
+            panel={AutomationBindingPanel}
+            type={"date"}
            value={inputData[key]}
            on:change={e => onChange(e, key)}
-          />
+            {bindings}
+            allowJS={true}
+            updateOnChange={false}
+            drawerLeft="260px"
+          >
+            <DatePicker
+              value={inputData[key]}
+              on:change={e => onChange(e, key)}
+            />
+          </DrawerBindableSlot>
        {:else if value.customType === "column"}
          <Select
            on:change={e => onChange(e, key)}
@ -469,7 +420,6 @@
          />
        {:else if value.customType === "row"}
          <RowSelector
-            {block}
            value={inputData[key]}
            meta={inputData["meta"] || {}}
            on:change={e => {
--- a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte
@ -1,18 +1,16 @@
 <script>
  import { tables } from "stores/backend"
  import { Select, Checkbox } from "@budibase/bbui"
-  import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
-  import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
  import { createEventDispatcher } from "svelte"
  import RowSelectorTypes from "./RowSelectorTypes.svelte"
-  import ModalBindableInput from "../../common/bindings/ModalBindableInput.svelte"
+  import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
+  import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"

  const dispatch = createEventDispatcher()

  export let value
  export let meta
  export let bindings
-  export let block
  export let isTestModal
  export let isUpdateRow

@ -25,16 +23,6 @@
  let table
  let schemaFields

-  let placeholders = {
-    number: 10,
-    boolean: "true",
-    datetime: "2022-02-16T12:00:00.000Z ",
-    options: "1",
-    array: "1 2 3 4",
-    link: "ro_ta_123_456",
-    longform: "long form text",
-  }
-  $: rowControl = block.rowControl
  $: {
    table = $tables.list.find(table => table._id === value?.tableId)
    schemaFields = Object.entries(table?.schema ?? {})
@ -57,19 +45,13 @@
      return value
    }

-    if (type === "boolean") {
-      if (typeof value === "boolean") {
-        return value
-      }
-      return value === "true"
-    }
    if (type === "number") {
      if (typeof value === "number") {
        return value
      }
      return Number(value)
    }
-    if (type === "options") {
+    if (type === "options" || type === "boolean") {
      return value
    }
    if (type === "array") {
@ -127,47 +109,40 @@
 {#if schemaFields.length}
  <div class="schema-fields">
    {#each schemaFields as [field, schema]}
-      {#if !schema.autocolumn}
-        {#if schema.type !== "attachment"}
-          {#if !rowControl}
-            <RowSelectorTypes
-              {isTestModal}
-              {field}
-              {schema}
-              bindings={parsedBindings}
-              {value}
-              {onChange}
-            />
-          {:else}
-            <div>
-              <svelte:component
-                this={isTestModal ? ModalBindableInput : DrawerBindableInput}
-                placeholder={placeholders[schema.type]}
-                panel={AutomationBindingPanel}
-                value={Array.isArray(value[field])
-                  ? value[field].join(",")
-                  : value[field]}
-                on:change={e => onChange(e, field, schema.type)}
-                label={field}
-                type="string"
-                bindings={parsedBindings}
-                fillWidth={true}
-                allowJS={true}
-                updateOnChange={false}
-              />
-              {#if isUpdateRow && schema.type === "link"}
-                <div class="checkbox-field">
-                  <Checkbox
-                    value={meta.fields?.[field]?.clearRelationships}
-                    text={"Clear relationships if empty?"}
-                    size={"S"}
-                    on:change={e => onChangeSetting(e, field)}
-                  />
-                </div>
-              {/if}
-            </div>
-          {/if}
-        {/if}
+      {#if !schema.autocolumn && schema.type !== "attachment"}
+        <DrawerBindableSlot
+          fillWidth
+          title={value.title}
+          label={field}
+          panel={AutomationBindingPanel}
+          type={schema.type}
+          {schema}
+          value={value[field]}
+          on:change={e => onChange(e, field)}
+          {bindings}
+          allowJS={true}
+          updateOnChange={false}
+          drawerLeft="260px"
+        >
+          <RowSelectorTypes
+            {isTestModal}
+            {field}
+            {schema}
+            bindings={parsedBindings}
+            {value}
+            {onChange}
+          />
+        </DrawerBindableSlot>
+      {/if}
+      {#if isUpdateRow && schema.type === "link"}
+        <div class="checkbox-field">
+          <Checkbox
+            value={meta.fields?.[field]?.clearRelationships}
+            text={"Clear relationships if empty?"}
+            size={"S"}
+            on:change={e => onChangeSetting(e, field)}
+          />
+        </div>
      {/if}
    {/each}
  </div>
--- a/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte
@ -1,7 +1,6 @@
 <script>
  import {
    Select,
-    Toggle,
    DatePicker,
    Multiselect,
    TextArea,
@ -45,10 +44,14 @@
    on:change={e => onChange(e, field)}
  />
 {:else if schema.type === "boolean"}
-  <Toggle
-    text={field}
-    value={value[field]}
+  <Select
    on:change={e => onChange(e, field)}
+    label={field}
+    value={value[field]}
+    options={[
+      { label: "True", value: "true" },
+      { label: "False", value: "false" },
+    ]}
  />
 {:else if schema.type === "array"}
  <Multiselect
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@ -33,7 +33,7 @@
  import { getBindings } from "components/backend/DataTable/formula"
  import JSONSchemaModal from "./JSONSchemaModal.svelte"
  import { ValidColumnNameRegex } from "@budibase/shared-core"
-  import { FieldSubtype, FieldType } from "@budibase/types"
+  import { FieldType } from "@budibase/types"
  import RelationshipSelector from "components/common/RelationshipSelector.svelte"

  const AUTO_TYPE = "auto"
@ -43,11 +43,7 @@
  const NUMBER_TYPE = FIELDS.NUMBER.type
  const JSON_TYPE = FIELDS.JSON.type
  const DATE_TYPE = FIELDS.DATETIME.type
-  const BB_REFERENCE_TYPE = FieldType.BB_REFERENCE
-  const BB_USER_REFERENCE_TYPE = composeType(
-    BB_REFERENCE_TYPE,
-    FieldSubtype.USER
-  )
+  const USER_REFRENCE_TYPE = FIELDS.BB_REFERENCE_USER.compositeType

  const dispatch = createEventDispatcher()
  const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
@ -66,7 +62,9 @@
  let relationshipPart1 = PrettyRelationshipDefinitions.Many
  let relationshipPart2 = PrettyRelationshipDefinitions.One

+  let relationshipTableIdPrimary = null
  let relationshipTableIdSecondary = null
+
  let table = $tables.selected
  let confirmDeleteDialog
  let savingColumn
@ -79,40 +77,15 @@
    // Initial value for column name in other table for linked records
    fieldName: $tables.selected.name,
  }
-
-  const bbRefTypeMapping = {}
-
-  function composeType(fieldType, subtype) {
-    return `${fieldType}_${subtype}`
-  }
-
-  // Handling fields with subtypes
-  fieldDefinitions = Object.entries(fieldDefinitions).reduce(
-    (p, [key, field]) => {
-      if (field.type === BB_REFERENCE_TYPE) {
-        const composedType = composeType(field.type, field.subtype)
-        p[key] = {
-          ...field,
-          type: composedType,
-        }
-        bbRefTypeMapping[composedType] = {
-          type: field.type,
-          subtype: field.subtype,
-        }
-      } else {
-        p[key] = field
-      }
-      return p
-    },
-    {}
-  )
+  let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions)
+  let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions)

  $: if (primaryDisplay) {
    editableColumn.constraints.presence = { allowEmpty: false }
  }

  let relationshipMap = {
-    [RelationshipType.MANY_TO_ONE]: {
+    [RelationshipType.ONE_TO_MANY]: {
      part1: PrettyRelationshipDefinitions.MANY,
      part2: PrettyRelationshipDefinitions.ONE,
    },
@ -120,14 +93,32 @@
      part1: PrettyRelationshipDefinitions.MANY,
      part2: PrettyRelationshipDefinitions.MANY,
    },
-    [RelationshipType.ONE_TO_MANY]: {
+    [RelationshipType.MANY_TO_ONE]: {
      part1: PrettyRelationshipDefinitions.ONE,
      part2: PrettyRelationshipDefinitions.MANY,
    },
  }

  $: {
+    // this parses any changes the user has made when creating a new internal relationship
+    // into what we expect the schema to look like
    if (editableColumn.type === LINK_TYPE) {
+      relationshipTableIdPrimary = table._id
+      if (relationshipPart1 === PrettyRelationshipDefinitions.ONE) {
+        relationshipOpts2 = relationshipOpts2.filter(
+          opt => opt !== PrettyRelationshipDefinitions.ONE
+        )
+      } else {
+        relationshipOpts2 = Object.values(PrettyRelationshipDefinitions)
+      }
+
+      if (relationshipPart2 === PrettyRelationshipDefinitions.ONE) {
+        relationshipOpts1 = relationshipOpts1.filter(
+          opt => opt !== PrettyRelationshipDefinitions.ONE
+        )
+      } else {
+        relationshipOpts1 = Object.values(PrettyRelationshipDefinitions)
+      }
      // Determine the relationship type based on the selected values of both parts
      editableColumn.relationshipType = Object.entries(relationshipMap).find(
        ([_, parts]) =>
@ -137,7 +128,6 @@
      editableColumn.tableId = relationshipTableIdSecondary
    }
  }
-
  const initialiseField = (field, savingColumn) => {
    isCreating = !field

@ -149,12 +139,21 @@
        $tables.selected.primaryDisplay == null ||
        $tables.selected.primaryDisplay === editableColumn.name

-      const mapped = Object.entries(bbRefTypeMapping).find(
-        ([_, v]) => v.type === field.type && v.subtype === field.subtype
-      )
-      if (mapped) {
-        editableColumn.type = mapped[0]
-        delete editableColumn.subtype
+      if (editableColumn.type === FieldType.BB_REFERENCE) {
+        editableColumn.type = `${editableColumn.type}_${editableColumn.subtype}`
+      }
+      // Here we are setting the relationship values based on the editableColumn
+      // This part of the code is used when viewing an existing field hence the check
+      // for the tableId
+      if (editableColumn.type === LINK_TYPE && editableColumn.tableId) {
+        relationshipTableIdPrimary = table._id
+        relationshipTableIdSecondary = editableColumn.tableId
+        if (editableColumn.relationshipType in relationshipMap) {
+          const { part1, part2 } =
+            relationshipMap[editableColumn.relationshipType]
+          relationshipPart1 = part1
+          relationshipPart2 = part2
+        }
      }
    } else if (!savingColumn) {
      let highestNumber = 0
@ -174,22 +173,10 @@
    }

    allowedTypes = getAllowedTypes()
-
-    if (editableColumn.type === LINK_TYPE && editableColumn.tableId) {
-      relationshipTableIdSecondary = editableColumn.tableId
-      if (editableColumn.relationshipType in relationshipMap) {
-        const { part1, part2 } =
-          relationshipMap[editableColumn.relationshipType]
-        relationshipPart1 = part1
-        relationshipPart2 = part2
-      }
-    }
  }

  $: initialiseField(field, savingColumn)

-  $: isBBReference = !!bbRefTypeMapping[editableColumn.type]
-
  $: checkConstraints(editableColumn)
  $: required = !!editableColumn?.constraints?.presence || primaryDisplay
  $: uneditable =
@ -246,10 +233,7 @@
  $: external = table.type === "external"
  // in the case of internal tables the sourceId will just be undefined
  $: tableOptions = $tables.list.filter(
-    opt =>
-      opt._id !== $tables.selected._id &&
-      opt.type === table.type &&
-      table.sourceId === opt.sourceId
+    opt => opt.type === table.type && table.sourceId === opt.sourceId
  )
  $: typeEnabled =
    !originalName ||
@ -265,11 +249,12 @@

    let saveColumn = cloneDeep(editableColumn)

-    if (bbRefTypeMapping[saveColumn.type]) {
-      saveColumn = {
-        ...saveColumn,
-        ...bbRefTypeMapping[saveColumn.type],
-      }
+    // Handle types on composite types
+    const definition = fieldDefinitions[saveColumn.type.toUpperCase()]
+    if (definition && saveColumn.type === definition.compositeType) {
+      saveColumn.type = definition.type
+      saveColumn.subtype = definition.subtype
+      delete saveColumn.compositeType
    }

    if (saveColumn.type === AUTO_TYPE) {
@ -292,10 +277,7 @@
      dispatch("updatecolumns")
      gridDispatch("close-edit-column")

-      if (
-        saveColumn.type === LINK_TYPE &&
-        saveColumn.relationshipType === RelationshipType.MANY_TO_MANY
-      ) {
+      if (saveColumn.type === LINK_TYPE) {
        // Fetching the new tables
        tables.fetch()
        // Fetching the new relationships
@ -327,6 +309,11 @@
        confirmDeleteDialog.hide()
        dispatch("updatecolumns")
        gridDispatch("close-edit-column")
+
+        if (editableColumn.type === LINK_TYPE) {
+          // Updating the relationships
+          datasources.fetch()
+        }
      }
    } catch (error) {
      notifications.error(`Error deleting column: ${error.message}`)
@ -352,7 +339,7 @@
      editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
    } else if (editableColumn.type === FORMULA_TYPE) {
      editableColumn.formulaType = "dynamic"
-    } else if (editableColumn.type === BB_USER_REFERENCE_TYPE) {
+    } else if (editableColumn.type === USER_REFRENCE_TYPE) {
      editableColumn.relationshipType = RelationshipType.ONE_TO_MANY
    }
  }
@ -410,14 +397,12 @@
        FIELDS.BOOLEAN,
        FIELDS.FORMULA,
        FIELDS.BIGINT,
+        FIELDS.BB_REFERENCE_USER,
      ]
      // no-sql or a spreadsheet
      if (!external || table.sql) {
        fields = [...fields, FIELDS.LINK, FIELDS.ARRAY]
      }
-      if (fieldDefinitions.USER) {
-        fields.push(fieldDefinitions.USER)
-      }
      return fields
    }
  }
@ -426,8 +411,9 @@
    if (!fieldToCheck) {
      return
    }
+
    // most types need this, just make sure its always present
-    if (fieldToCheck && !fieldToCheck.constraints) {
+    if (!fieldToCheck.constraints) {
      fieldToCheck.constraints = {}
    }
    // some string types may have been built by server, may not always have constraints
@ -507,7 +493,7 @@
    on:change={handleTypeChange}
    options={allowedTypes}
    getOptionLabel={field => field.name}
-    getOptionValue={field => field.type}
+    getOptionValue={field => field.compositeType || field.type}
    getOptionIcon={field => field.icon}
    isOptionEnabled={option => {
      if (option.type == AUTO_TYPE) {
@ -619,9 +605,11 @@
    <RelationshipSelector
      bind:relationshipPart1
      bind:relationshipPart2
-      bind:relationshipTableIdPrimary={table.name}
+      bind:relationshipTableIdPrimary
      bind:relationshipTableIdSecondary
      bind:editableColumn
+      {relationshipOpts1}
+      {relationshipOpts2}
      {linkEditDisabled}
      {tableOptions}
      {errors}
@ -671,7 +659,7 @@
    <Button primary text on:click={openJsonSchemaEditor}
      >Open schema editor</Button
    >
-  {:else if isBBReference}
+  {:else if editableColumn.type === USER_REFRENCE_TYPE}
    <Toggle
      value={editableColumn.relationshipType === RelationshipType.MANY_TO_MANY}
      on:change={e =>
--- a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte
+++ b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte
@ -57,7 +57,8 @@
    label: table.name,
    value: table._id,
  }))
-  $: valid = getErrorCount(errors) === 0 && allRequiredAttributesSet()
+  $: valid =
+    getErrorCount(errors) === 0 && allRequiredAttributesSet(relationshipType)
  $: isManyToMany = relationshipType === RelationshipType.MANY_TO_MANY
  $: isManyToOne = relationshipType === RelationshipType.MANY_TO_ONE

@ -114,7 +115,7 @@
    return Object.entries(errors).filter(entry => !!entry[1]).length
  }

-  function allRequiredAttributesSet() {
+  function allRequiredAttributesSet(relationshipType) {
    const base = getTable(fromId) && getTable(toId) && fromColumn && toColumn
    if (relationshipType === RelationshipType.MANY_TO_ONE) {
      return base && fromPrimary && fromForeign
@ -124,9 +125,10 @@
  }

  function validate() {
-    if (!allRequiredAttributesSet() && !hasValidated) {
+    if (!allRequiredAttributesSet(relationshipType) && !hasValidated) {
      return
    }
+
    hasValidated = true
    errorChecker.setType(relationshipType)
    const fromTable = getTable(fromId),
--- a/packages/builder/src/components/common/LinkedRowSelector.svelte
+++ b/packages/builder/src/components/common/LinkedRowSelector.svelte
@ -11,11 +11,11 @@
  const dispatch = createEventDispatcher()

  let rows = []
-  let linkedIds = (Array.isArray(linkedRows) ? linkedRows : [])?.map(
+  let linkedIds = []
+
+  $: linkedIds = (Array.isArray(linkedRows) ? linkedRows : [])?.map(
    row => row?._id || row
  )
-
-  $: linkedRows = linkedIds
  $: label = capitalise(schema.name)
  $: linkedTableId = schema.tableId
  $: linkedTable = $tables.list.find(table => table._id === linkedTableId)
--- a/packages/builder/src/components/common/RelationshipSelector.svelte
+++ b/packages/builder/src/components/common/RelationshipSelector.svelte
@ -1,6 +1,5 @@
 <script>
  import { Select, Input } from "@budibase/bbui"
-  import { PrettyRelationshipDefinitions } from "constants/backend"

  export let relationshipPart1
  export let relationshipPart2
@ -10,6 +9,8 @@
  export let linkEditDisabled
  export let tableOptions
  export let errors
+  export let relationshipOpts1
+  export let relationshipOpts2
 </script>

 <div class="relationship-container">
@ -17,15 +18,17 @@
    <Select
      disabled={linkEditDisabled}
      bind:value={relationshipPart1}
-      options={Object.values(PrettyRelationshipDefinitions)}
+      options={relationshipOpts1}
    />
  </div>
  <div class="relationship-label">in</div>
  <div class="relationship-part">
    <Select
      disabled
-      options={[relationshipTableIdPrimary]}
-      value={relationshipTableIdPrimary}
+      options={tableOptions}
+      getOptionLabel={table => table.name}
+      getOptionValue={table => table._id}
+      bind:value={relationshipTableIdPrimary}
    />
  </div>
 </div>
@ -34,7 +37,7 @@
    <Select
      disabled={linkEditDisabled}
      bind:value={relationshipPart2}
-      options={Object.values(PrettyRelationshipDefinitions)}
+      options={relationshipOpts2}
      getOptionLabel={option => "To " + option.toLowerCase()}
    />
  </div>
@ -43,7 +46,9 @@
    <Select
      disabled={linkEditDisabled}
      bind:value={relationshipTableIdSecondary}
-      options={tableOptions}
+      options={tableOptions.filter(
+        table => table._id !== relationshipTableIdPrimary
+      )}
      getOptionLabel={table => table.name}
      getOptionValue={table => table._id}
    />
--- a/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte
+++ b/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte
@ -0,0 +1,250 @@
+<script>
+  import { Icon, Input, Drawer, Button } from "@budibase/bbui"
+  import {
+    readableToRuntimeBinding,
+    runtimeToReadableBinding,
+  } from "builderStore/dataBinding"
+
+  import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
+  import { createEventDispatcher, setContext } from "svelte"
+  import { isJSBinding } from "@budibase/string-templates"
+
+  export let panel = ClientBindingPanel
+  export let value = ""
+  export let bindings = []
+  export let title = "Bindings"
+  export let placeholder
+  export let label
+  export let disabled = false
+  export let fillWidth
+  export let allowJS = true
+  export let allowHelpers = true
+  export let updateOnChange = true
+  export let drawerLeft
+  export let type
+  export let schema
+
+  const dispatch = createEventDispatcher()
+  let bindingDrawer
+  let valid = true
+  let currentVal = value
+
+  $: readableValue = runtimeToReadableBinding(bindings, value)
+  $: tempValue = readableValue
+  $: isJS = isJSBinding(value)
+
+  const saveBinding = () => {
+    onChange(tempValue)
+    onBlur()
+    bindingDrawer.hide()
+  }
+
+  setContext("binding-drawer-actions", {
+    save: saveBinding,
+  })
+
+  const onChange = value => {
+    if (type === "link" && value && hasValidLinks(value)) {
+      currentVal = value.split(",")
+    } else if (type === "array" && value && hasValidOptions(value)) {
+      currentVal = value.split(",")
+    } else {
+      currentVal = readableToRuntimeBinding(bindings, value)
+    }
+    dispatch("change", currentVal)
+  }
+
+  const onBlur = () => {
+    dispatch("blur", currentVal)
+  }
+
+  const isValidDate = value => {
+    return !value || !isNaN(new Date(value).valueOf())
+  }
+
+  const hasValidLinks = value => {
+    let links = []
+    if (Array.isArray(value)) {
+      links = value
+    } else if (value && typeof value === "string") {
+      links = value.split(",")
+    } else {
+      return !value
+    }
+
+    return links.every(link => link.startsWith("ro_"))
+  }
+
+  const hasValidOptions = value => {
+    let links = []
+    if (Array.isArray(value)) {
+      links = value
+    } else if (value && typeof value === "string") {
+      links = value.split(",")
+    } else {
+      return !value
+    }
+    return links.every(link => schema?.constraints?.inclusion?.includes(link))
+  }
+
+  const isValidBoolean = value => {
+    return value === "false" || value === "true" || value == ""
+  }
+
+  const validationMap = {
+    date: isValidDate,
+    datetime: isValidDate,
+    link: hasValidLinks,
+    array: hasValidOptions,
+    longform: value => !isJSBinding(value),
+    json: value => !isJSBinding(value),
+    boolean: isValidBoolean,
+  }
+
+  const isValid = value => {
+    const validate = validationMap[type]
+    return validate ? validate(value) : true
+  }
+
+  const getIconClass = (value, type) => {
+    if (type === "longform" && !isJSBinding(value)) {
+      return "text-area-slot-icon"
+    }
+    if (type === "json" && !isJSBinding(value)) {
+      return "json-slot-icon"
+    }
+    if (type !== "string" && type !== "number") {
+      return "slot-icon"
+    }
+    return ""
+  }
+</script>
+
+<div class="control" class:disabled>
+  {#if !isValid(value)}
+    <Input
+      {label}
+      {disabled}
+      readonly={isJS}
+      value={isJS ? "(JavaScript function)" : readableValue}
+      on:change={event => onChange(event.detail)}
+      on:blur={onBlur}
+      {placeholder}
+      {updateOnChange}
+    />
+    <div
+      class="icon"
+      on:click={() => {
+        if (!isJS) {
+          dispatch("change", "")
+        }
+      }}
+    >
+      <Icon disabled={isJS} size="S" name="Close" />
+    </div>
+  {:else}
+    <slot
+      {label}
+      {disabled}
+      readonly={isJS}
+      value={isJS ? "(JavaScript function)" : readableValue}
+      {placeholder}
+      {updateOnChange}
+    />
+  {/if}
+  {#if !disabled && type !== "formula"}
+    <div
+      class={`icon ${getIconClass(value, type)}`}
+      on:click={() => {
+        bindingDrawer.show()
+      }}
+    >
+      <Icon size="S" name="FlashOn" />
+    </div>
+  {/if}
+</div>
+<Drawer
+  on:drawerHide
+  on:drawerShow
+  {fillWidth}
+  bind:this={bindingDrawer}
+  {title}
+  left={drawerLeft}
+  headless
+>
+  <svelte:fragment slot="description">
+    Add the objects on the left to enrich your text.
+  </svelte:fragment>
+  <Button cta slot="buttons" disabled={!valid} on:click={saveBinding}>
+    Save
+  </Button>
+  <svelte:component
+    this={panel}
+    slot="body"
+    bind:valid
+    value={readableValue}
+    on:change={event => (tempValue = event.detail)}
+    {bindings}
+    {allowJS}
+    {allowHelpers}
+  />
+</Drawer>
+
+<style>
+  .control {
+    flex: 1;
+    position: relative;
+  }
+
+  .slot-icon {
+    right: 31px !important;
+    border-right: 1px solid var(--spectrum-alias-border-color);
+    border-top-right-radius: 0px !important;
+    border-bottom-right-radius: 0px !important;
+  }
+
+  .text-area-slot-icon {
+    border-bottom: 1px solid var(--spectrum-alias-border-color);
+    border-bottom-right-radius: 0px !important;
+    top: 26px !important;
+  }
+  .json-slot-icon {
+    border-bottom: 1px solid var(--spectrum-alias-border-color);
+    border-bottom-right-radius: 0px !important;
+    top: 23px !important;
+    right: 0px !important;
+  }
+
+  .icon {
+    right: 1px;
+    bottom: 1px;
+    position: absolute;
+    justify-content: center;
+    align-items: center;
+    display: flex;
+    flex-direction: row;
+    box-sizing: border-box;
+    border-left: 1px solid var(--spectrum-alias-border-color);
+    border-top-right-radius: var(--spectrum-alias-border-radius-regular);
+    border-bottom-right-radius: var(--spectrum-alias-border-radius-regular);
+    width: 31px;
+    color: var(--spectrum-alias-text-color);
+    background-color: var(--spectrum-global-color-gray-75);
+    transition: background-color
+        var(--spectrum-global-animation-duration-100, 130ms),
+      box-shadow var(--spectrum-global-animation-duration-100, 130ms),
+      border-color var(--spectrum-global-animation-duration-100, 130ms);
+    height: calc(var(--spectrum-alias-item-height-m) - 2px);
+  }
+
+  .icon:hover {
+    cursor: pointer;
+    color: var(--spectrum-alias-text-color-hover);
+    background-color: var(--spectrum-global-color-gray-50);
+    border-color: var(--spectrum-alias-border-color-hover);
+  }
+
+  .control:not(.disabled) :global(.spectrum-Textfield-input) {
+    padding-right: 40px;
+  }
+</style>
--- a/packages/builder/src/components/integration/QueryViewer.svelte
+++ b/packages/builder/src/components/integration/QueryViewer.svelte
@ -337,11 +337,12 @@
    padding: 8px 10px 8px 16px;
    display: flex;
    border-bottom: 2px solid transparent;
-    transition: border-bottom 130ms ease-out;
+    transition: border-bottom 130ms ease-out, background 130ms ease-out;
  }

  .header.scrolling {
    border-bottom: var(--border-light);
+    background: var(--background);
  }

  .body {
--- a/packages/builder/src/constants/backend/index.js
+++ b/packages/builder/src/constants/backend/index.js
@ -120,10 +120,11 @@ export const FIELDS = {
      presence: false,
    },
  },
-  USER: {
+  BB_REFERENCE_USER: {
    name: "User",
    type: "bb_reference",
    subtype: "user",
+    compositeType: "bb_reference_user", // Used for working with the subtype on CreateEditColumn as is it was a primary type
    icon: "User",
  },
 }
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Relationships.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Relationships.svelte
@ -21,15 +21,22 @@
  function getRelationships(tables) {
    const relatedColumns = {}

-    tables.forEach(({ name: tableName, schema }) => {
+    tables.forEach(({ name: tableName, schema, _id: tableId }) => {
      Object.values(schema).forEach(column => {
        if (column.type !== "link") return

-        relatedColumns[column._id] ??= {}
-        relatedColumns[column._id].through =
-          relatedColumns[column._id].through || column.through
+        const columnId =
+          column.through ||
+          column._id ||
+          (column.main
+            ? `${tableId}_${column.fieldName}__${column.tableId}_${column.foreignKey}`
+            : `${column.tableId}_${column.foreignKey}__${tableId}_${column.fieldName}`)

-        relatedColumns[column._id][column.main ? "from" : "to"] = {
+        relatedColumns[columnId] ??= {}
+        relatedColumns[columnId].through =
+          relatedColumns[columnId].through || column.through
+
+        relatedColumns[columnId][column.main ? "from" : "to"] = {
          ...column,
          tableName,
        }
--- a/packages/builder/src/pages/builder/portal/apps/_layout.svelte
+++ b/packages/builder/src/pages/builder/portal/apps/_layout.svelte
@ -46,7 +46,9 @@

 {#if loaded}
  <div class="page">
-    <PortalSideBar />
+    {#if $apps.length > 0}
+      <PortalSideBar />
+    {/if}
    <slot />
  </div>
 {/if}
--- a/packages/builder/src/stores/backend/tables.js
+++ b/packages/builder/src/stores/backend/tables.js
@ -1,7 +1,7 @@
 import { get, writable, derived } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
 import { API } from "api"
-import { SWITCHABLE_TYPES } from "constants/backend"
+import { SWITCHABLE_TYPES, FIELDS } from "constants/backend"

 export function createTablesStore() {
  const store = writable({
@ -21,6 +21,23 @@ export function createTablesStore() {
    }))
  }

+  const singleFetch = async tableId => {
+    const table = await API.getTable(tableId)
+    store.update(state => {
+      const list = []
+      // update the list, keep order accurate
+      for (let tbl of state.list) {
+        if (table._id === tbl._id) {
+          list.push(table)
+        } else {
+          list.push(tbl)
+        }
+      }
+      state.list = list
+      return state
+    })
+  }
+
  const select = tableId => {
    store.update(state => ({
      ...state,
@ -63,6 +80,20 @@ export function createTablesStore() {
    const savedTable = await API.saveTable(updatedTable)
    replaceTable(savedTable._id, savedTable)
    select(savedTable._id)
+    // make sure tables up to date (related)
+    let tableIdsToFetch = []
+    for (let column of Object.values(updatedTable?.schema || {})) {
+      if (column.type === FIELDS.LINK.type) {
+        tableIdsToFetch.push(column.tableId)
+      }
+    }
+    tableIdsToFetch = [...new Set(tableIdsToFetch)]
+    // too many tables to fetch, just get all
+    if (tableIdsToFetch.length > 3) {
+      await fetch()
+    } else {
+      await Promise.all(tableIdsToFetch.map(id => singleFetch(id)))
+    }
    return savedTable
  }

--- a/packages/builder/tsconfig.json
+++ b/packages/builder/tsconfig.json
@ -5,8 +5,5 @@
    "declaration": true,
    "sourceMap": true,
    "baseUrl": "."
-  },
-  "ts-node": {
-    "require": ["tsconfig-paths/register"]
  }
 }
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@ -32,7 +32,6 @@
    "@budibase/backend-core": "0.0.0",
    "@budibase/string-templates": "0.0.0",
    "@budibase/types": "0.0.0",
-    "axios": "0.21.2",
    "chalk": "4.1.0",
    "cli-progress": "3.11.2",
    "commander": "7.1.0",
@ -41,7 +40,6 @@
    "download": "8.0.0",
    "find-free-port": "^2.0.0",
    "inquirer": "8.0.0",
-    "joi": "17.6.0",
    "lookpath": "1.1.0",
    "node-fetch": "2.6.7",
    "pkg": "5.8.0",
@ -53,13 +51,9 @@
    "yaml": "^2.1.1"
  },
  "devDependencies": {
-    "@swc/core": "1.3.71",
-    "@swc/jest": "0.2.27",
    "@types/jest": "29.5.3",
    "@types/node-fetch": "2.6.4",
    "@types/pouchdb": "^6.4.0",
-    "copyfiles": "^2.4.1",
-    "eslint": "^7.20.0",
    "renamer": "^4.0.0",
    "ts-node": "^10.9.1",
    "typescript": "5.2.2"
--- a/packages/client/src/components/app/Layout.svelte
+++ b/packages/client/src/components/app/Layout.svelte
@ -3,16 +3,16 @@
  import { writable } from "svelte/store"
  import { Heading, Icon, clickOutside } from "@budibase/bbui"
  import { FieldTypes } from "constants"
+  import { Constants } from "@budibase/frontend-core"
  import active from "svelte-spa-router/active"
-  import { RoleUtils } from "@budibase/frontend-core"

  const sdk = getContext("sdk")
  const {
    routeStore,
+    roleStore,
    styleable,
    linkable,
    builderStore,
-    currentRole,
    sidePanelStore,
  } = sdk
  const component = getContext("component")
@ -61,7 +61,7 @@
  })
  setContext("layout", store)

-  $: validLinks = getValidLinks(links, $currentRole)
+  $: validLinks = getValidLinks(links, $roleStore)
  $: typeClass = NavigationClasses[navigation] || NavigationClasses.None
  $: navWidthClass = WidthClasses[navWidth || width] || WidthClasses.Large
  $: pageWidthClass = WidthClasses[pageWidth || width] || WidthClasses.Large
@ -99,14 +99,13 @@
    }
  }

-  const getValidLinks = (allLinks, role) => {
+  const getValidLinks = (allLinks, userRoleHierarchy) => {
    // Strip links missing required info
    let validLinks = (allLinks || []).filter(link => link.text && link.url)
-
    // Filter to only links allowed by the current role
-    const priority = RoleUtils.getRolePriority(role)
    return validLinks.filter(link => {
-      return !link.roleId || RoleUtils.getRolePriority(link.roleId) <= priority
+      const role = link.roleId || Constants.Roles.BASIC
+      return userRoleHierarchy?.find(roleId => roleId === role)
    })
  }

--- a/packages/client/src/components/app/Section.svelte
+++ b/packages/client/src/components/app/Section.svelte
@ -47,28 +47,29 @@
 <style>
  div {
    display: grid;
-    grid-gap: 16px;
+    --gap: 16px;
+    grid-gap: var(--gap);
  }
  .mainSidebar {
-    grid-template-columns: 3fr 1fr;
+    grid-template-columns:
+      calc((100% - var(--gap)) / 4 * 3) /* 75% */
+      calc((100% - var(--gap)) / 4); /* 25% */
  }
  .sidebarMain {
-    grid-template-columns: 1fr 3fr;
-  }
-  .oneColumn {
-    grid-template-columns: 1fr;
-  }
-  .twoColumns {
-    grid-template-columns: 1fr 1fr;
-  }
-  .threeColumns {
-    grid-template-columns: 1fr 1fr 1fr;
+    grid-template-columns:
+      calc((100% - var(--gap)) / 4) /* 25% */
+      calc((100% - var(--gap)) / 4 * 3); /* 75% */
  }
+  .oneColumn,
  .columns-1 {
    grid-template-columns: 1fr;
  }
+  .twoColumns,
  .columns-2 {
-    grid-template-columns: 1fr 1fr;
+    grid-template-columns: repeat(2, calc((100% - var(--gap)) / 2));
+  }
+  .threeColumns {
+    grid-template-columns: repeat(3, calc((100% - var(--gap)) / 3));
  }
  .placeholder {
    border: 2px dashed var(--spectrum-global-color-gray-600);
--- a/packages/client/src/components/app/blocks/TableBlock.svelte
+++ b/packages/client/src/components/app/blocks/TableBlock.svelte
@ -45,8 +45,21 @@
  let enrichedSearchColumns
  let schemaLoaded = false

-  // Accommodate old config to ensure delete button does not reappear
-  $: deleteLabel = sidePanelShowDelete === false ? "" : sidePanelDeleteLabel
+  $: deleteLabel = setDeleteLabel(sidePanelDeleteLabel, sidePanelShowDelete)
+
+  const setDeleteLabel = sidePanelDeleteLabel => {
+    // Accommodate old config to ensure delete button does not reappear
+    let labelText = sidePanelShowDelete === false ? "" : sidePanelDeleteLabel
+
+    // Empty text is considered hidden.
+    if (labelText?.trim() === "") {
+      return ""
+    }
+
+    // Default to "Delete" if the value is unset
+    return labelText || "Delete"
+  }
+
  $: isDSPlus = dataSource?.type === "table" || dataSource?.type === "viewV2"
  $: fetchSchema(dataSource)
  $: enrichSearchColumns(searchColumns, schema).then(
@ -249,7 +262,7 @@
            props={{
              dataSource,
              saveButtonLabel: sidePanelSaveLabel || "Save", //always show
-              deleteButtonLabel: deleteLabel, //respect config
+              deleteButtonLabel: deleteLabel,
              actionType: "Update",
              rowId: `{{ ${safe("state")}.${safe(stateKey)} }}`,
              fields: sidePanelFields || normalFields,
--- a/packages/client/src/components/app/forms/RelationshipField.svelte
+++ b/packages/client/src/components/app/forms/RelationshipField.svelte
@ -60,6 +60,12 @@
      // even if they are not in the inital fetch results
      initialValuesProcessed = true
      optionsObj = (fieldState?.value || []).reduce((accumulator, value) => {
+        // fieldState has to be an array of strings to be valid for an update
+        // therefore we cannot guarantee value will be an object
+        // https://linear.app/budibase/issue/BUDI-7577/refactor-the-relationshipfield-component-to-have-better-support-for
+        if (!value._id) {
+          return accumulator
+        }
        accumulator[value._id] = {
          _id: value._id,
          [primaryDisplay]: value.primaryDisplay,
@ -121,7 +127,12 @@
    if (!Array.isArray(values)) {
      values = [values]
    }
-    return values.map(value => (typeof value === "object" ? value._id : value))
+    values = values.map(value =>
+      typeof value === "object" ? value._id : value
+    )
+    // Make sure field state is valid
+    fieldApi.setValue(values)
+    return values
  }

  const getDisplayName = row => {
--- a/packages/client/src/components/devtools/DevToolsHeader.svelte
+++ b/packages/client/src/components/devtools/DevToolsHeader.svelte
@ -1,32 +1,38 @@
 <script>
  import { Heading, Select, ActionButton } from "@budibase/bbui"
-  import { devToolsStore, appStore } from "../../stores"
-  import { getContext } from "svelte"
+  import { devToolsStore, appStore, roleStore } from "../../stores"
+  import { getContext, onMount } from "svelte"

  const context = getContext("context")
+  const SELF_ROLE = "self"

-  $: previewOptions = [
-    {
+  let staticRoleList
+
+  $: previewOptions = buildRoleList(staticRoleList)
+
+  function buildRoleList(roleIds) {
+    const list = []
+    list.push({
      label: "View as yourself",
-      value: "self",
-    },
-    {
-      label: "View as public user",
-      value: "PUBLIC",
-    },
-    {
-      label: "View as basic user",
-      value: "BASIC",
-    },
-    {
-      label: "View as power user",
-      value: "POWER",
-    },
-    {
-      label: "View as admin user",
-      value: "ADMIN",
-    },
-  ]
+      value: SELF_ROLE,
+    })
+    if (!roleIds) {
+      return list
+    }
+    for (let roleId of roleIds) {
+      list.push({
+        label: `View as ${roleId.toLowerCase()} user`,
+        value: roleId,
+      })
+    }
+    return list
+  }
+
+  onMount(async () => {
+    // make sure correct before starting
+    await devToolsStore.actions.changeRole(SELF_ROLE)
+    staticRoleList = await roleStore.actions.fetchAccessibleRoles()
+  })
 </script>

 <div class="dev-preview-header" class:mobile={$context.device.mobile}>
@ -34,7 +40,7 @@
  <Select
    quiet
    options={previewOptions}
-    value={$devToolsStore.role || "self"}
+    value={$devToolsStore.role || SELF_ROLE}
    placeholder={null}
    autoWidth
    on:change={e => devToolsStore.actions.changeRole(e.detail)}
--- a/packages/client/src/sdk.js
+++ b/packages/client/src/sdk.js
@ -13,6 +13,7 @@ import {
  sidePanelStore,
  dndIsDragging,
  confirmationStore,
+  roleStore,
 } from "stores"
 import { styleable } from "utils/styleable"
 import { linkable } from "utils/linkable"
@ -39,6 +40,7 @@ export default {
  dndIsDragging,
  currentRole,
  confirmationStore,
+  roleStore,
  styleable,
  linkable,
  getAction,
--- a/packages/client/src/stores/devTools.js
+++ b/packages/client/src/stores/devTools.js
@ -2,6 +2,7 @@ import { createLocalStorageStore } from "@budibase/frontend-core"
 import { initialise } from "./initialise"
 import { authStore } from "./auth"
 import { API } from "../api"
+import { get } from "svelte/store"

 const initialState = {
  visible: false,
@ -27,9 +28,15 @@ const createDevToolStore = () => {
  }

  const changeRole = async role => {
+    if (role === "self") {
+      role = null
+    }
+    if (role === get(store).role) {
+      return
+    }
    store.update(state => ({
      ...state,
-      role: role === "self" ? null : role,
+      role,
    }))
    API.invalidateCache()
    await authStore.actions.fetchUser()
--- a/packages/client/src/stores/index.js
+++ b/packages/client/src/stores/index.js
@ -11,12 +11,13 @@ export { stateStore } from "./state"
 export { themeStore } from "./theme"
 export { devToolsStore } from "./devTools"
 export { componentStore } from "./components"
-export { uploadStore } from "./uploads.js"
-export { rowSelectionStore } from "./rowSelection.js"
-export { blockStore } from "./blocks.js"
+export { uploadStore } from "./uploads"
+export { rowSelectionStore } from "./rowSelection"
+export { blockStore } from "./blocks"
 export { environmentStore } from "./environment"
-export { eventStore } from "./events.js"
-export { orgStore } from "./org.js"
+export { eventStore } from "./events"
+export { orgStore } from "./org"
+export { roleStore } from "./roles"
 export {
  dndStore,
  dndIndex,
@ -25,7 +26,7 @@ export {
  dndIsNewComponent,
  dndIsDragging,
 } from "./dnd"
-export { sidePanelStore } from "./sidePanel.js"
+export { sidePanelStore } from "./sidePanel"

 // Context stores are layered and duplicated, so it is not a singleton
 export { createContextStore } from "./context"
--- a/packages/client/src/stores/roles.js
+++ b/packages/client/src/stores/roles.js
@ -0,0 +1,24 @@
+import { API } from "api"
+import { writable } from "svelte/store"
+import { currentRole } from "./derived"
+
+const createRoleStore = () => {
+  const store = writable([])
+
+  // Fetches the user object if someone is logged in and has reloaded the page
+  const fetchAccessibleRoles = async () => {
+    const accessible = await API.getAccessibleRoles()
+    // Use the app self if present, otherwise fallback to the global self
+    store.set(accessible || [])
+    return accessible
+  }
+
+  return {
+    subscribe: store.subscribe,
+    actions: { fetchAccessibleRoles },
+  }
+}
+
+export const roleStore = createRoleStore()
+
+currentRole.subscribe(roleStore.actions.fetchAccessibleRoles)
--- a/packages/frontend-core/src/api/roles.js
+++ b/packages/frontend-core/src/api/roles.js
@ -38,4 +38,13 @@ export const buildRoleEndpoints = API => ({
      url: `/api/global/roles/${appId}`,
    })
  },
+
+  /**
+   * For the logged in user and current app - retrieves accessible roles.
+   */
+  getAccessibleRoles: async () => {
+    return await API.get({
+      url: `/api/roles/accessible`,
+    })
+  },
 })
--- a/packages/frontend-core/src/api/tables.js
+++ b/packages/frontend-core/src/api/tables.js
@ -82,8 +82,9 @@ export const buildTableEndpoints = API => ({
      },
    })
  },
+
  /**
-   * Gets a list o tables.
+   * Gets a list of tables.
   */
  getTables: async () => {
    return await API.get({
@ -91,6 +92,15 @@ export const buildTableEndpoints = API => ({
    })
  },

+  /**
+   * Get a single table based on table ID.
+   */
+  getTable: async tableId => {
+    return await API.get({
+      url: `/api/tables/${tableId}`,
+    })
+  },
+
  /**
   * Saves a table.
   * @param table the table to save
--- a/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte
@ -3,6 +3,8 @@
  import RelationshipCell from "./RelationshipCell.svelte"
  import { FieldSubtype } from "@budibase/types"

+  export let api
+
  const { API } = getContext("grid")
  const { subtype } = $$props.schema

@ -17,8 +19,11 @@
      throw `Search for '${subtype}' not implemented`
    }

+    // As we are overriding the search function from RelationshipCell, we want to map one shape to the expected one for the specific API
+    const email = Object.values(searchParams.query.string)[0]
+
    const results = await API.searchUsers({
-      ...searchParams,
+      email,
    })

    // Mapping to the expected data within RelationshipCell
@ -31,6 +36,7 @@
 </script>

 <RelationshipCell
+  bind:api
  {...$$props}
  {schema}
  {searchFunction}
--- a/packages/sdk/package.json
+++ b/packages/sdk/package.json
@ -10,14 +10,10 @@
    "generate": "cd scripts && bash generate-sdk.sh",
    "build:sdk": "yarn run generate && rollup -c"
  },
-  "dependencies": {
-    "superagent": "^5.3.0"
-  },
  "devDependencies": {
    "@rollup/plugin-commonjs": "^18.0.0",
    "@rollup/plugin-node-resolve": "^11.2.1",
    "rollup": "^2.44.0",
-    "rollup-plugin-polyfill-node": "^0.8.0",
    "rollup-plugin-terser": "^7.0.2"
  }
 }
--- a/packages/sdk/scripts/generate-sdk.sh
+++ b/packages/sdk/scripts/generate-sdk.sh
@ -19,7 +19,7 @@ docker run --rm \
  -v ${PWD}/generated:/generated \
  -v ${PWD}/config.json:/config.json \
  -u $(id -u):$(id -g) \
-  swaggerapi/swagger-codegen-cli-v3 generate \
+  swaggerapi/swagger-codegen-cli-v3:3.0.46 generate \
  -i /openapi.yml \
  -l javascript \
  -o /generated \
@ -34,4 +34,4 @@ if [[ -f "openapi.yaml" ]]; then
 fi
 if [[ -d "generated" ]]; then
    rm -r generated
-fi
+fi
--- a/packages/server/package.json
+++ b/packages/server/package.json
@ -57,7 +57,6 @@
    "@elastic/elasticsearch": "7.10.0",
    "@google-cloud/firestore": "5.0.2",
    "@koa/router": "8.0.8",
-    "@sendgrid/mail": "7.1.1",
    "@sentry/node": "6.17.7",
    "@socket.io/redis-adapter": "^8.2.1",
    "airtable": "0.10.1",
@ -66,18 +65,14 @@
    "bcrypt": "5.1.0",
    "bcryptjs": "2.4.3",
    "bull": "4.10.1",
-    "chmodr": "1.2.0",
    "chokidar": "3.5.3",
    "cookies": "0.8.0",
    "csvtojson": "2.0.10",
    "curlconverter": "3.21.0",
    "dd-trace": "3.13.2",
    "dotenv": "8.2.0",
-    "download": "8.0.0",
-    "elastic-apm-node": "3.38.0",
    "fix-path": "3.0.0",
    "form-data": "4.0.0",
-    "fs-extra": "8.1.0",
    "global-agent": "3.0.0",
    "google-auth-library": "7.12.0",
    "google-spreadsheet": "3.2.0",
@ -90,10 +85,7 @@
    "koa": "2.13.4",
    "koa-body": "4.2.0",
    "koa-compress": "4.0.1",
-    "koa-connect": "2.1.0",
    "koa-send": "5.0.0",
-    "koa-session": "5.12.0",
-    "koa-static": "5.0.0",
    "koa-useragent": "^4.1.0",
    "koa2-ratelimit": "1.1.1",
    "lodash": "4.17.21",
@ -108,7 +100,6 @@
    "pg": "8.10.0",
    "posthog-node": "1.3.0",
    "pouchdb": "7.3.0",
-    "pouchdb-adapter-memory": "7.2.2",
    "pouchdb-all-dbs": "1.0.2",
    "pouchdb-find": "7.2.2",
    "pouchdb-replication-stream": "1.2.9",
@ -116,8 +107,7 @@
    "server-destroy": "1.0.1",
    "snowflake-promise": "^4.5.0",
    "socket.io": "4.6.1",
-    "svelte": "3.49.0",
-    "swagger-parser": "10.0.3",
+    "svelte": "^3.49.0",
    "tar": "6.1.15",
    "to-json-schema": "0.2.5",
    "uuid": "3.3.2",
@ -130,13 +120,9 @@
  "devDependencies": {
    "@babel/core": "7.17.4",
    "@babel/preset-env": "7.16.11",
-    "@budibase/standard-components": "^0.9.139",
-    "@jest/test-sequencer": "29.6.2",
    "@swc/core": "1.3.71",
    "@swc/jest": "0.2.27",
    "@trendyol/jest-testcontainers": "2.1.1",
-    "@types/apidoc": "0.50.0",
-    "@types/bson": "4.2.0",
    "@types/global-agent": "2.1.1",
    "@types/google-spreadsheet": "3.1.5",
    "@types/jest": "29.5.3",
@ -148,17 +134,12 @@
    "@types/node-fetch": "2.6.4",
    "@types/oracledb": "5.2.2",
    "@types/pg": "8.6.6",
-    "@types/pouchdb": "6.4.0",
-    "@types/redis": "4.0.11",
    "@types/server-destroy": "1.0.1",
    "@types/supertest": "2.0.12",
    "@types/tar": "6.1.5",
    "apidoc": "0.50.4",
-    "babel-jest": "29.6.2",
    "copyfiles": "2.4.1",
    "docker-compose": "0.23.17",
-    "eslint": "6.8.0",
-    "is-wsl": "2.2.0",
    "jest": "29.6.2",
    "jest-openapi": "0.14.2",
    "jest-runner": "29.6.2",
--- a/packages/server/specs/openapi.json
+++ b/packages/server/specs/openapi.json
@ -1567,8 +1567,7 @@
          }
        },
        "required": [
-          "email",
-          "roles"
+          "email"
        ]
      },
      "userOutput": {
@ -1639,7 +1638,6 @@
            },
            "required": [
              "email",
-              "roles",
              "_id"
            ]
          }
@ -1718,7 +1716,6 @@
              },
              "required": [
                "email",
-                "roles",
                "_id"
              ]
            }
--- a/packages/server/specs/openapi.yaml
+++ b/packages/server/specs/openapi.yaml
@ -1337,7 +1337,6 @@ components:
              role ID, e.g. ADMIN.
      required:
        - email
-        - roles
    userOutput:
      type: object
      properties:
@ -1398,7 +1397,6 @@ components:
              type: string
          required:
            - email
-            - roles
            - _id
      required:
        - data
@ -1464,7 +1462,6 @@ components:
                type: string
            required:
              - email
-              - roles
              - _id
      required:
        - data
--- a/packages/server/specs/resources/user.ts
+++ b/packages/server/specs/resources/user.ts
@ -92,7 +92,7 @@ const userSchema = object(
      },
    },
  },
-  { required: ["email", "roles"] }
+  { required: ["email"] }
 )

 const userOutputSchema = {
--- a/packages/server/src/api/controllers/public/mapping/users.ts
+++ b/packages/server/src/api/controllers/public/mapping/users.ts
@ -15,10 +15,15 @@ function user(body: any): User {
  }
 }

-function mapUser(ctx: any): { data: User } {
-  return {
+function mapUser(ctx: any) {
+  const body: { data: User; message?: string } = {
    data: user(ctx.body),
  }
+  if (ctx.extra?.message) {
+    body.message = ctx.extra.message
+    delete ctx.extra
+  }
+  return body
 }

 function mapUsers(ctx: any): { data: User[] } {
--- a/packages/server/src/api/controllers/public/users.ts
+++ b/packages/server/src/api/controllers/public/users.ts
@ -10,6 +10,32 @@ import { search as stringSearch } from "./utils"
 import { UserCtx, User } from "@budibase/types"
 import { Next } from "koa"
 import { sdk } from "@budibase/pro"
+import { isEqual, cloneDeep } from "lodash"
+
+function rolesRemoved(base: User, ctx: UserCtx) {
+  return (
+    !isEqual(base.builder, ctx.request.body.builder) ||
+    !isEqual(base.admin, ctx.request.body.admin) ||
+    !isEqual(base.roles, ctx.request.body.roles)
+  )
+}
+
+const NO_ROLES_MSG =
+  "Roles/admin/builder can only be set on business/enterprise licenses - input ignored."
+
+async function createUpdateResponse(ctx: UserCtx, user?: User) {
+  const base = cloneDeep(ctx.request.body)
+  ctx = await sdk.publicApi.users.roleCheck(ctx, user)
+  // check the ctx before any updates to it
+  const removed = rolesRemoved(base, ctx)
+  ctx = publicApiUserFix(ctx)
+  const response = await saveGlobalUser(ctx)
+  ctx.body = await getUser(ctx, response._id)
+  if (removed) {
+    ctx.extra = { message: NO_ROLES_MSG }
+  }
+  return ctx
+}

 function isLoggedInUser(ctx: UserCtx, user: User) {
  const loggedInId = ctx.user?._id
@ -35,9 +61,7 @@ export async function search(ctx: UserCtx, next: Next) {
 }

 export async function create(ctx: UserCtx, next: Next) {
-  ctx = publicApiUserFix(await sdk.publicApi.users.roleCheck(ctx))
-  const response = await saveGlobalUser(ctx)
-  ctx.body = await getUser(ctx, response._id)
+  await createUpdateResponse(ctx)
  await next()
 }

@ -52,9 +76,7 @@ export async function update(ctx: UserCtx, next: Next) {
    ...ctx.request.body,
    _rev: user._rev,
  }
-  ctx = publicApiUserFix(await sdk.publicApi.users.roleCheck(ctx, user))
-  const response = await saveGlobalUser(ctx)
-  ctx.body = await getUser(ctx, response._id)
+  await createUpdateResponse(ctx, user)
  await next()
 }

--- a/packages/server/src/api/controllers/query/index.ts
+++ b/packages/server/src/api/controllers/query/index.ts
@ -9,7 +9,7 @@ import { quotas } from "@budibase/pro"
 import { events, context, utils, constants } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import { QueryEvent } from "../../../threads/definitions"
-import { Query } from "@budibase/types"
+import { ConfigType, Query, UserCtx } from "@budibase/types"
 import { ValidQueryNameRegex } from "@budibase/shared-core"

 const Runner = new Thread(ThreadType.QUERY, {
@ -28,11 +28,11 @@ function enrichQueries(input: any) {
  return wasArray ? queries : queries[0]
 }

-export async function fetch(ctx: any) {
+export async function fetch(ctx: UserCtx) {
  ctx.body = await sdk.queries.fetch()
 }

-const _import = async (ctx: any) => {
+const _import = async (ctx: UserCtx) => {
  const body = ctx.request.body
  const data = body.data

@ -73,7 +73,7 @@ const _import = async (ctx: any) => {
 }
 export { _import as import }

-export async function save(ctx: any) {
+export async function save(ctx: UserCtx) {
  const db = context.getAppDB()
  const query = ctx.request.body

@ -100,19 +100,19 @@ export async function save(ctx: any) {
  ctx.message = `Query ${query.name} saved successfully.`
 }

-export async function find(ctx: any) {
+export async function find(ctx: UserCtx) {
  const queryId = ctx.params.queryId
  ctx.body = await sdk.queries.find(queryId)
 }

 //Required to discern between OIDC OAuth config entries
-function getOAuthConfigCookieId(ctx: any) {
-  if (ctx.user.providerType === constants.Config.OIDC) {
+function getOAuthConfigCookieId(ctx: UserCtx) {
+  if (ctx.user.providerType === ConfigType.OIDC) {
    return utils.getCookie(ctx, constants.Cookie.OIDC_CONFIG)
  }
 }

-function getAuthConfig(ctx: any) {
+function getAuthConfig(ctx: UserCtx) {
  const authCookie = utils.getCookie(ctx, constants.Cookie.Auth)
  let authConfigCtx: any = {}
  authConfigCtx["configId"] = getOAuthConfigCookieId(ctx)
@ -120,7 +120,7 @@ function getAuthConfig(ctx: any) {
  return authConfigCtx
 }

-export async function preview(ctx: any) {
+export async function preview(ctx: UserCtx) {
  const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
    ctx.request.body.datasourceId
  )
@ -129,6 +129,19 @@ export async function preview(ctx: any) {
  // this stops dynamic variables from calling the same query
  const { fields, parameters, queryVerb, transformer, queryId, schema } = query

+  let existingSchema = schema
+  if (queryId && !existingSchema) {
+    try {
+      const db = context.getAppDB()
+      const existing = (await db.get(queryId)) as Query
+      existingSchema = existing.schema
+    } catch (err: any) {
+      if (err.status !== 404) {
+        ctx.throw(500, "Unable to retrieve existing query")
+      }
+    }
+  }
+
  const authConfigCtx: any = getAuthConfig(ctx)

  try {
@ -180,6 +193,14 @@ export async function preview(ctx: any) {
        schemaFields[key] = fieldType
      }
    }
+    // if existing schema, update to include any previous schema keys
+    if (existingSchema) {
+      for (let key of Object.keys(schemaFields)) {
+        if (existingSchema[key]?.type) {
+          schemaFields[key] = existingSchema[key].type
+        }
+      }
+    }
    // remove configuration before sending event
    delete datasource.config
    await events.query.previewed(datasource, query)
@ -189,13 +210,13 @@ export async function preview(ctx: any) {
      info,
      extra,
    }
-  } catch (err) {
+  } catch (err: any) {
    ctx.throw(400, err)
  }
 }

 async function execute(
-  ctx: any,
+  ctx: UserCtx,
  opts: any = { rowsOnly: false, isAutomation: false }
 ) {
  const db = context.getAppDB()
@ -255,17 +276,17 @@ async function execute(
    } else {
      ctx.body = { data: rows, pagination, ...extra, ...info }
    }
-  } catch (err) {
+  } catch (err: any) {
    ctx.throw(400, err)
  }
 }

-export async function executeV1(ctx: any) {
+export async function executeV1(ctx: UserCtx) {
  return execute(ctx, { rowsOnly: true, isAutomation: false })
 }

 export async function executeV2(
-  ctx: any,
+  ctx: UserCtx,
  { isAutomation }: { isAutomation?: boolean } = {}
 ) {
  return execute(ctx, { rowsOnly: false, isAutomation })
@ -292,7 +313,7 @@ const removeDynamicVariables = async (queryId: any) => {
  }
 }

-export async function destroy(ctx: any) {
+export async function destroy(ctx: UserCtx) {
  const db = context.getAppDB()
  const queryId = ctx.params.queryId
  await removeDynamicVariables(queryId)
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@ -1,6 +1,7 @@
-import { roles, context, events, db as dbCore } from "@budibase/backend-core"
+import { context, db as dbCore, events, roles } from "@budibase/backend-core"
 import { getUserMetadataParams, InternalTables } from "../../db/utils"
-import { UserCtx, Database, UserRoles, Role } from "@budibase/types"
+import { Database, Role, UserCtx, UserRoles } from "@budibase/types"
+import { sdk as sharedSdk } from "@budibase/shared-core"
 import sdk from "../../sdk"

 const UpdateRolesOptions = {
@ -94,7 +95,6 @@ export async function save(ctx: UserCtx) {
  )
  role._rev = result.rev
  ctx.body = role
-  ctx.message = `Role '${role.name}' created successfully.`
 }

 export async function destroy(ctx: UserCtx) {
@ -131,3 +131,16 @@ export async function destroy(ctx: UserCtx) {
  ctx.message = `Role ${ctx.params.roleId} deleted successfully`
  ctx.status = 200
 }
+
+export async function accessible(ctx: UserCtx) {
+  let roleId = ctx.user?.roleId
+  if (!roleId) {
+    roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
+  }
+  if (ctx.user && sharedSdk.users.isAdminOrBuilder(ctx.user)) {
+    const appId = context.getAppId()
+    ctx.body = await roles.getAllRoleIds(appId)
+  } else {
+    ctx.body = await roles.getUserRoleIdHierarchy(roleId!)
+  }
+}
--- a/packages/server/src/api/controllers/routing.ts
+++ b/packages/server/src/api/controllers/routing.ts
@ -63,9 +63,7 @@ export async function fetch(ctx: UserCtx) {
 export async function clientFetch(ctx: UserCtx) {
  const routing = await getRoutingStructure()
  let roleId = ctx.user?.role?._id
-  const roleIds = (await roles.getUserRoleHierarchy(roleId, {
-    idOnly: true,
-  })) as string[]
+  const roleIds = await roles.getUserRoleIdHierarchy(roleId)
  for (let topLevel of Object.values(routing.routes) as any) {
    for (let subpathKey of Object.keys(topLevel.subpaths)) {
      let found = false
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@ -269,13 +269,25 @@ function isEditableColumn(column: FieldSchema) {
  return !(isExternalAutoColumn || isFormula)
 }

-export class ExternalRequest {
-  private operation: Operation
-  private tableId: string
+export type ExternalRequestReturnType<T> = T extends Operation.READ
+  ?
+      | Row[]
+      | {
+          row: Row
+          table: Table
+        }
+  : {
+      row: Row
+      table: Table
+    }
+
+export class ExternalRequest<T extends Operation> {
+  private readonly operation: T
+  private readonly tableId: string
  private datasource?: Datasource
  private tables: { [key: string]: Table } = {}

-  constructor(operation: Operation, tableId: string, datasource?: Datasource) {
+  constructor(operation: T, tableId: string, datasource?: Datasource) {
    this.operation = operation
    this.tableId = tableId
    this.datasource = datasource
@ -328,10 +340,16 @@ export class ExternalRequest {
      // one to many
      if (isOneSide(field)) {
        let id = row[key][0]
-        if (typeof row[key] === "string") {
-          id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1]
+        if (id) {
+          if (typeof row[key] === "string") {
+            id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1]
+          }
+          newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(id)[0]
+        } else {
+          // Removing from both new and row, as we don't know if it has already been processed
+          row[field.foreignKey || linkTablePrimary] = null
+          newRow[field.foreignKey || linkTablePrimary] = null
        }
-        newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(id)[0]
      }
      // many to many
      else if (field.through) {
@ -739,7 +757,7 @@ export class ExternalRequest {
    return fields
  }

-  async run(config: RunConfig) {
+  async run(config: RunConfig): Promise<ExternalRequestReturnType<T>> {
    const { operation, tableId } = this
    let { datasourceId, tableName } = breakExternalTableId(tableId)
    if (!tableName) {
@ -818,7 +836,7 @@ export class ExternalRequest {
    // can't really use response right now
    const response = await getDatasourceAndQuery(json)
    // handle many to many relationships now if we know the ID (could be auto increment)
-    if (operation !== Operation.READ && processed.manyRelationships) {
+    if (operation !== Operation.READ) {
      await this.handleManyRelationships(
        table._id || "",
        response[0],
@ -827,8 +845,11 @@ export class ExternalRequest {
    }
    const output = this.outputProcessing(response, table, relationships)
    // if reading it'll just be an array of rows, return whole thing
-    return operation === Operation.READ && Array.isArray(response)
-      ? output
-      : { row: output[0], table }
+    const result = (
+      operation === Operation.READ && Array.isArray(response)
+        ? output
+        : { row: output[0], table }
+    ) as ExternalRequestReturnType<T>
+    return result
  }
 }
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@ -1,9 +1,13 @@
-import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
+import { FieldTypes } from "../../../constants"
 import {
  breakExternalTableId,
  breakRowIdField,
 } from "../../../integrations/utils"
-import { ExternalRequest, RunConfig } from "./ExternalRequest"
+import {
+  ExternalRequest,
+  ExternalRequestReturnType,
+  RunConfig,
+} from "./ExternalRequest"
 import {
  Datasource,
  IncludeRelationship,
@ -18,14 +22,17 @@ import {
 import sdk from "../../../sdk"
 import * as utils from "./utils"
 import { dataFilters } from "@budibase/shared-core"
-import { inputProcessing } from "../../../utilities/rowProcessor"
+import {
+  inputProcessing,
+  outputProcessing,
+} from "../../../utilities/rowProcessor"
 import { cloneDeep, isEqual } from "lodash"

-export async function handleRequest(
-  operation: Operation,
+export async function handleRequest<T extends Operation>(
+  operation: T,
  tableId: string,
  opts?: RunConfig
-) {
+): Promise<ExternalRequestReturnType<T>> {
  // make sure the filters are cleaned up, no empty strings for equals, fuzzy or string
  if (opts && opts.filters) {
    opts.filters = sdk.rows.removeEmptyFilters(opts.filters)
@ -34,7 +41,7 @@ export async function handleRequest(
    !dataFilters.hasFilters(opts?.filters) &&
    opts?.filters?.onEmptyFilter === EmptyFilterOption.RETURN_NONE
  ) {
-    return []
+    return [] as any
  }

  return new ExternalRequest(operation, tableId, opts?.datasource).run(
@ -46,24 +53,34 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
  const tableId = utils.getTableId(ctx)
  const { _id, ...rowData } = ctx.request.body

+  const table = await sdk.tables.getTable(tableId)
+  const { row: dataToUpdate } = await inputProcessing(
+    ctx.user?._id,
+    cloneDeep(table),
+    rowData
+  )
+
  const validateResult = await sdk.rows.utils.validate({
-    row: rowData,
+    row: dataToUpdate,
    tableId,
  })
  if (!validateResult.valid) {
    throw { validation: validateResult.errors }
  }
+
  const response = await handleRequest(Operation.UPDATE, tableId, {
    id: breakRowIdField(_id),
-    row: rowData,
+    row: dataToUpdate,
  })
  const row = await sdk.rows.external.getRow(tableId, _id, {
    relationships: true,
  })
-  const table = await sdk.tables.getTable(tableId)
+  const enrichedRow = await outputProcessing(table, row, {
+    preserveLinks: true,
+  })
  return {
    ...response,
-    row,
+    row: enrichedRow,
    table,
  }
 }
@ -71,13 +88,6 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
 export async function save(ctx: UserCtx) {
  const inputs = ctx.request.body
  const tableId = utils.getTableId(ctx)
-  const validateResult = await sdk.rows.utils.validate({
-    row: inputs,
-    tableId,
-  })
-  if (!validateResult.valid) {
-    throw { validation: validateResult.errors }
-  }

  const table = await sdk.tables.getTable(tableId)
  const { table: updatedTable, row } = await inputProcessing(
@ -86,24 +96,30 @@ export async function save(ctx: UserCtx) {
    inputs
  )

+  const validateResult = await sdk.rows.utils.validate({
+    row,
+    tableId,
+  })
+  if (!validateResult.valid) {
+    throw { validation: validateResult.errors }
+  }
+
  const response = await handleRequest(Operation.CREATE, tableId, {
    row,
  })

-  const responseRow = response as { row: Row }
-
  if (!isEqual(table, updatedTable)) {
    await sdk.tables.saveTable(updatedTable)
  }

-  const rowId = responseRow.row._id
+  const rowId = response.row._id
  if (rowId) {
    const row = await sdk.rows.external.getRow(tableId, rowId, {
      relationships: true,
    })
    return {
      ...response,
-      row,
+      row: await outputProcessing(table, row, { preserveLinks: true }),
    }
  } else {
    return response
@ -121,7 +137,12 @@ export async function find(ctx: UserCtx): Promise<Row> {
    ctx.throw(404)
  }

-  return row
+  const table = await sdk.tables.getTable(tableId)
+  // Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case
+  return await outputProcessing(table, row, {
+    squash: false,
+    preserveLinks: true,
+  })
 }

 export async function destroy(ctx: UserCtx) {
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@ -107,6 +107,11 @@ export const serveApp = async function (ctx: any) {
  //Public Settings
  const { config } = await configs.getSettingsConfigDoc()
  const branding = await pro.branding.getBrandingConfig(config)
+  // incase running direct from TS
+  let appHbsPath = join(__dirname, "app.hbs")
+  if (!fs.existsSync(appHbsPath)) {
+    appHbsPath = join(__dirname, "templates", "app.hbs")
+  }

  let db
  try {
@ -138,7 +143,7 @@ export const serveApp = async function (ctx: any) {
            ? objectStore.getGlobalFileUrl("settings", "logoUrl")
            : "",
      })
-      const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
+      const appHbs = loadHandlebarsFile(appHbsPath)
      ctx.body = await processString(appHbs, {
        head,
        body: html,
@ -166,7 +171,7 @@ export const serveApp = async function (ctx: any) {
            : "",
      })

-      const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
+      const appHbs = loadHandlebarsFile(appHbsPath)
      ctx.body = await processString(appHbs, {
        head,
        body: html,
@ -193,8 +198,13 @@ export const serveBuilderPreview = async function (ctx: any) {
 }

 export const serveClientLibrary = async function (ctx: any) {
+  let rootPath = join(NODE_MODULES_PATH, "@budibase", "client", "dist")
+  // incase running from TS directly
+  if (env.isDev() && !fs.existsSync(rootPath)) {
+    rootPath = join(require.resolve("@budibase/client"), "..")
+  }
  return send(ctx, "budibase-client.js", {
-    root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
+    root: rootPath,
  })
 }

--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@ -18,6 +18,7 @@ import {
 import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"
 import { builderSocket } from "../../../websockets"
+import { cloneDeep } from "lodash"

 function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
  if (table && !tableId) {
@ -35,16 +36,16 @@ function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
 export async function fetch(ctx: UserCtx<void, FetchTablesResponse>) {
  const internal = await sdk.tables.getAllInternalTables()

-  const externalTables = await sdk.datasources.getExternalDatasources()
+  const datasources = await sdk.datasources.getExternalDatasources()

-  const external = externalTables.flatMap(table => {
-    let entities = table.entities
+  const external = datasources.flatMap(datasource => {
+    let entities = datasource.entities
    if (entities) {
      return Object.values(entities).map<Table>((entity: Table) => ({
        ...entity,
        type: "external",
-        sourceId: table._id,
-        sql: isSQL(table),
+        sourceId: datasource._id,
+        sql: isSQL(datasource),
      }))
    } else {
      return []
@ -80,7 +81,7 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
  ctx.eventEmitter &&
    ctx.eventEmitter.emitTable(`table:save`, appId, { ...savedTable })
  ctx.body = savedTable
-  builderSocket?.emitTableUpdate(ctx, { ...savedTable })
+  builderSocket?.emitTableUpdate(ctx, cloneDeep(savedTable))
 }

 export async function destroy(ctx: UserCtx) {
--- a/packages/server/src/api/routes/public/tests/users.spec.ts
+++ b/packages/server/src/api/routes/public/tests/users.spec.ts
@ -68,6 +68,7 @@ describe("no user role update in free", () => {
    })
    expect(res.status).toBe(200)
    expect(res.body.data.roles["app_a"]).toBeUndefined()
+    expect(res.body.message).toBeDefined()
  })

  it("should not allow 'admin' to be updated", async () => {
@ -77,6 +78,7 @@ describe("no user role update in free", () => {
    })
    expect(res.status).toBe(200)
    expect(res.body.data.admin).toBeUndefined()
+    expect(res.body.message).toBeDefined()
  })

  it("should not allow 'builder' to be updated", async () => {
@ -86,6 +88,7 @@ describe("no user role update in free", () => {
    })
    expect(res.status).toBe(200)
    expect(res.body.data.builder).toBeUndefined()
+    expect(res.body.message).toBeDefined()
  })
 })

@ -102,6 +105,7 @@ describe("no user role update in business", () => {
    })
    expect(res.status).toBe(200)
    expect(res.body.data.roles["app_a"]).toBe("BASIC")
+    expect(res.body.message).toBeUndefined()
  })

  it("should allow 'admin' to be updated", async () => {
@ -112,6 +116,7 @@ describe("no user role update in business", () => {
    })
    expect(res.status).toBe(200)
    expect(res.body.data.admin.global).toBe(true)
+    expect(res.body.message).toBeUndefined()
  })

  it("should allow 'builder' to be updated", async () => {
@ -122,5 +127,6 @@ describe("no user role update in business", () => {
    })
    expect(res.status).toBe(200)
    expect(res.body.data.builder.global).toBe(true)
+    expect(res.body.message).toBeUndefined()
  })
 })
--- a/packages/server/src/api/routes/role.ts
+++ b/packages/server/src/api/routes/role.ts
@ -7,6 +7,9 @@ import { roleValidator } from "./utils/validators"
 const router: Router = new Router()

 router
+  // retrieve a list of the roles a user can access
+  // needs to be public for public screens
+  .get("/api/roles/accessible", controller.accessible)
  .post(
    "/api/roles",
    authorized(permissions.BUILDER),
--- a/packages/server/src/api/routes/tests/role.spec.js
+++ b/packages/server/src/api/routes/tests/role.spec.js
@ -15,7 +15,7 @@ describe("/roles", () => {
    await config.init()
  })

-  const createRole = async (role) => {
+  const createRole = async role => {
    if (!role) {
      role = basicRole()
    }
@ -33,9 +33,6 @@ describe("/roles", () => {
      const role = basicRole()
      const res = await createRole(role)

-      expect(res.res.statusMessage).toEqual(
-        `Role '${role.name}' created successfully.`
-      )
      expect(res.body._id).toBeDefined()
      expect(res.body._rev).toBeDefined()
      expect(events.role.updated).not.toBeCalled()
@ -51,9 +48,6 @@ describe("/roles", () => {
      jest.clearAllMocks()
      res = await createRole(res.body)

-      expect(res.res.statusMessage).toEqual(
-        `Role '${role.name}' created successfully.`
-      )
      expect(res.body._id).toBeDefined()
      expect(res.body._rev).toBeDefined()
      expect(events.role.created).not.toBeCalled()
@ -99,7 +93,11 @@ describe("/roles", () => {

    it("should be able to get the role with a permission added", async () => {
      const table = await config.createTable()
-      await config.api.permission.set({ roleId: BUILTIN_ROLE_IDS.POWER, resourceId: table._id, level: PermissionLevel.READ })
+      await config.api.permission.set({
+        roleId: BUILTIN_ROLE_IDS.POWER,
+        resourceId: table._id,
+        level: PermissionLevel.READ,
+      })
      const res = await request
        .get(`/api/roles`)
        .set(config.defaultHeaders())
@ -131,4 +129,34 @@ describe("/roles", () => {
      expect(events.role.deleted).toBeCalledWith(customRole)
    })
  })
+
+  describe("accessible", () => {
+    it("should be able to fetch accessible roles (with builder)", async () => {
+      const res = await request
+        .get("/api/roles/accessible")
+        .set(config.defaultHeaders())
+        .expect(200)
+      expect(res.body.length).toBe(5)
+      expect(typeof res.body[0]).toBe("string")
+    })
+
+    it("should be able to fetch accessible roles (basic user)", async () => {
+      const res = await request
+        .get("/api/roles/accessible")
+        .set(await config.basicRoleHeaders())
+        .expect(200)
+      expect(res.body.length).toBe(2)
+      expect(res.body[0]).toBe("BASIC")
+      expect(res.body[1]).toBe("PUBLIC")
+    })
+
+    it("should be able to fetch accessible roles (no user)", async () => {
+      const res = await request
+        .get("/api/roles/accessible")
+        .set(config.publicHeaders())
+        .expect(200)
+      expect(res.body.length).toBe(1)
+      expect(res.body[0]).toBe("PUBLIC")
+    })
+  })
 })
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@ -7,6 +7,7 @@ import { context, InternalTable, roles, tenancy } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
  FieldType,
+  FieldTypeSubtypes,
  MonthlyQuotaName,
  PermissionLevel,
  QuotaUsageType,
@ -25,6 +26,7 @@ import {
  mocks,
  structures,
 } from "@budibase/backend-core/tests"
+import _ from "lodash"

 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
@ -34,7 +36,7 @@ const { basicRow } = setup.structures
 describe.each([
  ["internal", undefined],
  ["postgres", databaseTestProviders.postgres],
-])("/rows (%s)", (_, dsProvider) => {
+])("/rows (%s)", (__, dsProvider) => {
  const isInternal = !dsProvider

  const request = setup.getRequest()
@ -1511,4 +1513,413 @@ describe.each([
      })
    })
  })
+
+  let o2mTable: Table
+  let m2mTable: Table
+  beforeAll(async () => {
+    o2mTable = await config.createTable(
+      { ...generateTableConfig(), name: "o2m" },
+      {
+        skipReassigning: true,
+      }
+    )
+    m2mTable = await config.createTable(
+      { ...generateTableConfig(), name: "m2m" },
+      {
+        skipReassigning: true,
+      }
+    )
+  })
+
+  describe.each([
+    [
+      "relationship fields",
+      () => ({
+        user: {
+          name: "user",
+          relationshipType: RelationshipType.ONE_TO_MANY,
+          type: FieldType.LINK,
+          tableId: o2mTable._id!,
+          fieldName: "fk_o2m",
+        },
+        users: {
+          name: "users",
+          relationshipType: RelationshipType.MANY_TO_MANY,
+          type: FieldType.LINK,
+          tableId: m2mTable._id!,
+          fieldName: "fk_m2m",
+        },
+      }),
+      (tableId: string) =>
+        config.api.row.save(tableId, {
+          name: generator.word(),
+          description: generator.paragraph(),
+          tableId,
+        }),
+      (row: Row) => ({
+        _id: row._id,
+        primaryDisplay: row.name,
+      }),
+    ],
+    [
+      "bb reference fields",
+      () => ({
+        user: {
+          name: "user",
+          relationshipType: RelationshipType.ONE_TO_MANY,
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+        },
+        users: {
+          name: "users",
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+          relationshipType: RelationshipType.MANY_TO_MANY,
+        },
+      }),
+      () => config.createUser(),
+      (row: Row) => ({
+        _id: row._id,
+        email: row.email,
+        firstName: row.firstName,
+        lastName: row.lastName,
+        primaryDisplay: row.email,
+      }),
+    ],
+  ])("links - %s", (__, relSchema, dataGenerator, resultMapper) => {
+    let tableId: string
+    let o2mData: Row[]
+    let m2mData: Row[]
+
+    beforeAll(async () => {
+      const tableConfig = generateTableConfig()
+
+      if (config.datasource) {
+        tableConfig.sourceId = config.datasource._id
+        if (config.datasource.plus) {
+          tableConfig.type = "external"
+        }
+      }
+      const table = await config.api.table.create({
+        ...tableConfig,
+        schema: {
+          ...tableConfig.schema,
+          ...relSchema(),
+        },
+      })
+      tableId = table._id!
+
+      o2mData = [
+        await dataGenerator(o2mTable._id!),
+        await dataGenerator(o2mTable._id!),
+        await dataGenerator(o2mTable._id!),
+        await dataGenerator(o2mTable._id!),
+      ]
+
+      m2mData = [
+        await dataGenerator(m2mTable._id!),
+        await dataGenerator(m2mTable._id!),
+        await dataGenerator(m2mTable._id!),
+        await dataGenerator(m2mTable._id!),
+      ]
+    })
+
+    it("can save a row when relationship fields are empty", async () => {
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      expect(row).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        _id: expect.any(String),
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+      })
+    })
+
+    it("can save a row with a single relationship field", async () => {
+      const user = _.sample(o2mData)!
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        user: [user],
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      expect(row).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: [user].map(u => resultMapper(u)),
+        _id: expect.any(String),
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
+      })
+    })
+
+    it("can save a row with a multiple relationship field", async () => {
+      const selectedUsers = _.sampleSize(m2mData, 2)
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        users: selectedUsers,
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      expect(row).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        users: expect.arrayContaining(selectedUsers.map(u => resultMapper(u))),
+        _id: expect.any(String),
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+      })
+    })
+
+    it("can retrieve rows with no populated relationships", async () => {
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      const { body: retrieved } = await config.api.row.get(tableId, row._id!)
+      expect(retrieved).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: undefined,
+        users: undefined,
+        _id: row._id,
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        ...defaultRowFields,
+      })
+    })
+
+    it("can retrieve rows with populated relationships", async () => {
+      const user1 = _.sample(o2mData)!
+      const [user2, user3] = _.sampleSize(m2mData, 2)
+
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        users: [user2, user3],
+        user: [user1],
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      const { body: retrieved } = await config.api.row.get(tableId, row._id!)
+      expect(retrieved).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: expect.arrayContaining([user1].map(u => resultMapper(u))),
+        users: expect.arrayContaining([user2, user3].map(u => resultMapper(u))),
+        _id: row._id,
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user1.id,
+        ...defaultRowFields,
+      })
+    })
+
+    it("can update an existing populated row", async () => {
+      const user = _.sample(o2mData)!
+      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
+
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        users: [users1, users2],
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      const updatedRow = await config.api.row.save(tableId, {
+        ...row,
+        user: [user],
+        users: [users3, users1],
+      })
+      expect(updatedRow).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: expect.arrayContaining([user].map(u => resultMapper(u))),
+        users: expect.arrayContaining(
+          [users3, users1].map(u => resultMapper(u))
+        ),
+        _id: row._id,
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
+      })
+    })
+
+    it("can wipe an existing populated relationships in row", async () => {
+      const [user1, user2] = _.sampleSize(m2mData, 2)
+
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        users: [user1, user2],
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      const updatedRow = await config.api.row.save(tableId, {
+        ...row,
+        user: null,
+        users: null,
+      })
+      expect(updatedRow).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        _id: row._id,
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+      })
+    })
+
+    it("fetch all will populate the relationships", async () => {
+      const [user1] = _.sampleSize(o2mData, 1)
+      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
+
+      const rows: {
+        name: string
+        description: string
+        user?: Row[]
+        users?: Row[]
+        tableId: string
+      }[] = [
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [users1, users2],
+        },
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          user: [user1],
+          users: [users1, users3],
+        },
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [users3],
+        },
+      ]
+
+      await config.api.row.save(tableId, rows[0])
+      await config.api.row.save(tableId, rows[1])
+      await config.api.row.save(tableId, rows[2])
+
+      const res = await config.api.row.fetch(tableId)
+
+      expect(res).toEqual(
+        expect.arrayContaining(
+          rows.map(r => ({
+            name: r.name,
+            description: r.description,
+            tableId,
+            user: r.user?.map(u => resultMapper(u)),
+            users: r.users?.length
+              ? expect.arrayContaining(r.users?.map(u => resultMapper(u)))
+              : undefined,
+            _id: expect.any(String),
+            _rev: expect.any(String),
+            id: isInternal ? undefined : expect.any(Number),
+            [`fk_${o2mTable.name}_fk_o2m`]:
+              isInternal || !r.user?.length ? undefined : r.user[0].id,
+            ...defaultRowFields,
+          }))
+        )
+      )
+    })
+
+    it("search all will populate the relationships", async () => {
+      const [user1] = _.sampleSize(o2mData, 1)
+      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
+
+      const rows: {
+        name: string
+        description: string
+        user?: Row[]
+        users?: Row[]
+        tableId: string
+      }[] = [
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [users1, users2],
+        },
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          user: [user1],
+          users: [users1, users3],
+        },
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [users3],
+        },
+      ]
+
+      await config.api.row.save(tableId, rows[0])
+      await config.api.row.save(tableId, rows[1])
+      await config.api.row.save(tableId, rows[2])
+
+      const res = await config.api.row.search(tableId)
+
+      expect(res).toEqual({
+        rows: expect.arrayContaining(
+          rows.map(r => ({
+            name: r.name,
+            description: r.description,
+            tableId,
+            user: r.user?.map(u => resultMapper(u)),
+            users: r.users?.length
+              ? expect.arrayContaining(r.users?.map(u => resultMapper(u)))
+              : undefined,
+            _id: expect.any(String),
+            _rev: expect.any(String),
+            id: isInternal ? undefined : expect.any(Number),
+            [`fk_${o2mTable.name}_fk_o2m`]:
+              isInternal || !r.user?.length ? undefined : r.user[0].id,
+            ...defaultRowFields,
+          }))
+        ),
+        ...(isInternal
+          ? {}
+          : {
+              hasNextPage: false,
+              bookmark: null,
+            }),
+      })
+    })
+  })
 })
--- a/packages/server/src/automations/steps/make.ts
+++ b/packages/server/src/automations/steps/make.ts
@ -78,8 +78,7 @@ export const definition: AutomationStepSchema = {
 }

 export async function run({ inputs }: AutomationStepInput) {
-  //TODO - Remove deprecated values 1,2,3,4,5 after November 2023
-  const { url, value1, value2, value3, value4, value5, body } = inputs
+  const { url, body } = inputs

  let payload = {}
  try {
@ -104,11 +103,6 @@ export async function run({ inputs }: AutomationStepInput) {
    response = await fetch(url, {
      method: "post",
      body: JSON.stringify({
-        value1,
-        value2,
-        value3,
-        value4,
-        value5,
        ...payload,
      }),
      headers: {
--- a/packages/server/src/automations/steps/zapier.ts
+++ b/packages/server/src/automations/steps/zapier.ts
@ -71,8 +71,7 @@ export const definition: AutomationStepSchema = {
 }

 export async function run({ inputs }: AutomationStepInput) {
-  //TODO - Remove deprecated values 1,2,3,4,5 after November 2023
-  const { url, value1, value2, value3, value4, value5, body } = inputs
+  const { url, body } = inputs

  let payload = {}
  try {
@ -100,11 +99,6 @@ export async function run({ inputs }: AutomationStepInput) {
      method: "post",
      body: JSON.stringify({
        platform: "budibase",
-        value1,
-        value2,
-        value3,
-        value4,
-        value5,
        ...payload,
      }),
      headers: {
--- a/packages/server/src/definitions/openapi.ts
+++ b/packages/server/src/definitions/openapi.ts
@ -613,7 +613,7 @@ export interface components {
        global?: boolean;
      };
      /** @description Contains the roles of the user per app (assuming they are not a builder user). This field can only be set on a business or enterprise license. */
-      roles: { [key: string]: string };
+      roles?: { [key: string]: string };
    };
    userOutput: {
      data: {
@ -643,7 +643,7 @@ export interface components {
          global?: boolean;
        };
        /** @description Contains the roles of the user per app (assuming they are not a builder user). This field can only be set on a business or enterprise license. */
-        roles: { [key: string]: string };
+        roles?: { [key: string]: string };
        /** @description The ID of the user. */
        _id: string;
      };
@ -676,7 +676,7 @@ export interface components {
          global?: boolean;
        };
        /** @description Contains the roles of the user per app (assuming they are not a builder user). This field can only be set on a business or enterprise license. */
-        roles: { [key: string]: string };
+        roles?: { [key: string]: string };
        /** @description The ID of the user. */
        _id: string;
      }[];
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@ -1,6 +1,5 @@
 import {
  ConnectionInfo,
-  Datasource,
  DatasourceFeature,
  DatasourceFieldType,
  DatasourcePlus,
@ -23,7 +22,6 @@ import fetch from "node-fetch"
 import { cache, configs, context, HTTPError } from "@budibase/backend-core"
 import { dataFilters, utils } from "@budibase/shared-core"
 import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
-import sdk from "../sdk"

 interface GoogleSheetsConfig {
  spreadsheetId: string
@ -56,6 +54,7 @@ const ALLOWED_TYPES = [
  FieldType.OPTIONS,
  FieldType.BOOLEAN,
  FieldType.BARCODEQR,
+  FieldType.BB_REFERENCE,
 ]

 const SCHEMA: Integration = {
@ -213,7 +212,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
      await setupCreationAuth(this.config)

      // Initialise oAuth client
-      let googleConfig = await configs.getGoogleDatasourceConfig()
+      const googleConfig = await configs.getGoogleDatasourceConfig()
      if (!googleConfig) {
        throw new HTTPError("Google config not found", 400)
      }
@ -323,14 +322,14 @@ class GoogleSheetsIntegration implements DatasourcePlus {
      case Operation.UPDATE:
        return this.update({
          // exclude the header row and zero index
-          rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
+          rowIndex: json.extra?.idFilter?.equal?.rowNumber,
          sheet,
          row: json.body,
        })
      case Operation.DELETE:
        return this.delete({
          // exclude the header row and zero index
-          rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
+          rowIndex: json.extra?.idFilter?.equal?.rowNumber,
          sheet,
        })
      case Operation.CREATE_TABLE:
@ -541,17 +540,30 @@ class GoogleSheetsIntegration implements DatasourcePlus {
    }
  }

+  private async getRowByIndex(sheetTitle: string, rowIndex: number) {
+    const sheet = this.client.sheetsByTitle[sheetTitle]
+    const rows = await sheet.getRows()
+    // We substract 2, as the SDK is skipping the header automatically and Google Spreadsheets is base 1
+    const row = rows[rowIndex - 2]
+    return { sheet, row }
+  }
+
  async update(query: { sheet: string; rowIndex: number; row: any }) {
    try {
      await this.connect()
-      const sheet = this.client.sheetsByTitle[query.sheet]
-      const rows = await sheet.getRows()
-      const row = rows[query.rowIndex]
+      const { sheet, row } = await this.getRowByIndex(
+        query.sheet,
+        query.rowIndex
+      )
      if (row) {
        const updateValues =
          typeof query.row === "string" ? JSON.parse(query.row) : query.row
        for (let key in updateValues) {
          row[key] = updateValues[key]
+
+          if (row[key] === null) {
+            row[key] = ""
+          }
        }
        await row.save()
        return [
@ -568,9 +580,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {

  async delete(query: { sheet: string; rowIndex: number }) {
    await this.connect()
-    const sheet = this.client.sheetsByTitle[query.sheet]
-    const rows = await sheet.getRows()
-    const row = rows[query.rowIndex]
+    const { row } = await this.getRowByIndex(query.sheet, query.rowIndex)
    if (row) {
      await row.delete()
      return [
--- a/packages/server/src/integrations/utils.ts
+++ b/packages/server/src/integrations/utils.ts
@ -1,4 +1,4 @@
-import { SqlQuery, Table, SearchFilters } from "@budibase/types"
+import { SqlQuery, Table, SearchFilters, Datasource } from "@budibase/types"
 import { DocumentType, SEPARATOR } from "../db/utils"
 import {
  FieldTypes,
@ -184,7 +184,9 @@ export function getSqlQuery(query: SqlQuery | string): SqlQuery {
  }
 }

-export const isSQL = helpers.isSQL
+export function isSQL(datasource: Datasource) {
+  return helpers.isSQL(datasource)
+}

 export function isIsoDateString(str: string) {
  const trimmedValue = str.trim()
--- a/packages/server/src/middleware/authorized.ts
+++ b/packages/server/src/middleware/authorized.ts
@ -55,9 +55,7 @@ const checkAuthorizedResource = async (
 ) => {
  // get the user's roles
  const roleId = ctx.roleId || roles.BUILTIN_ROLE_IDS.PUBLIC
-  const userRoles = (await roles.getUserRoleHierarchy(roleId, {
-    idOnly: false,
-  })) as Role[]
+  const userRoles = await roles.getUserRoleHierarchy(roleId)
  const permError = "User does not have permission"
  // check if the user has the required role
  if (resourceRoles.length > 0) {
--- a/packages/server/src/sdk/app/rows/search/external.ts
+++ b/packages/server/src/sdk/app/rows/search/external.ts
@ -17,6 +17,7 @@ import { utils } from "@budibase/shared-core"
 import { ExportRowsParams, ExportRowsResult } from "../search"
 import { HTTPError, db } from "@budibase/backend-core"
 import pick from "lodash/pick"
+import { outputProcessing } from "../../../../utilities/rowProcessor"

 export async function search(options: SearchParams) {
  const { tableId } = options
@ -75,6 +76,9 @@ export async function search(options: SearchParams) {
      rows = rows.map((r: any) => pick(r, fields))
    }

+    const table = await sdk.tables.getTable(tableId)
+    rows = await outputProcessing(table, rows, { preserveLinks: true })
+
    // need wrapper object for bookmarks etc when paginating
    return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 }
  } catch (err: any) {
@ -166,9 +170,11 @@ export async function exportRows(
 }

 export async function fetch(tableId: string) {
-  return handleRequest(Operation.READ, tableId, {
+  const response = await handleRequest(Operation.READ, tableId, {
    includeSqlRelationships: IncludeRelationship.INCLUDE,
  })
+  const table = await sdk.tables.getTable(tableId)
+  return await outputProcessing(table, response, { preserveLinks: true })
 }

 export async function fetchView(viewName: string) {
--- a/packages/server/src/sdk/tests/rows/row.spec.ts
+++ b/packages/server/src/sdk/tests/rows/row.spec.ts
@ -7,9 +7,14 @@ import { HTTPError } from "@budibase/backend-core"
 import { Operation } from "@budibase/types"

 const mockDatasourcesGet = jest.fn()
+const mockTableGet = jest.fn()
 sdk.datasources.get = mockDatasourcesGet
+sdk.tables.getTable = mockTableGet

 jest.mock("../../../api/controllers/row/ExternalRequest")
+jest.mock("../../../utilities/rowProcessor", () => ({
+  outputProcessing: jest.fn((_, rows) => rows),
+}))

 jest.mock("../../../api/controllers/view/exporters", () => ({
  ...jest.requireActual("../../../api/controllers/view/exporters"),
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@ -425,6 +425,15 @@ class TestConfiguration {
    return headers
  }

+  async basicRoleHeaders() {
+    return await this.roleHeaders({
+      email: this.defaultUserValues.email,
+      builder: false,
+      prodApp: true,
+      roleId: roles.BUILTIN_ROLE_IDS.BASIC,
+    })
+  }
+
  async roleHeaders({
    email = this.defaultUserValues.email,
    roleId = roles.BUILTIN_ROLE_IDS.ADMIN,
--- a/packages/server/src/tests/utilities/api/row.ts
+++ b/packages/server/src/tests/utilities/api/row.ts
@ -44,12 +44,12 @@ export class RowAPI extends TestAPI {
  }

  save = async (
-    sourceId: string,
+    tableId: string,
    row: SaveRowRequest,
    { expectStatus } = { expectStatus: 200 }
  ): Promise<Row> => {
    const resp = await this.request
-      .post(`/api/${sourceId}/rows`)
+      .post(`/api/${tableId}/rows`)
      .send(row)
      .set(this.config.defaultHeaders())
      .expect("Content-Type", /json/)
@ -122,4 +122,16 @@ export class RowAPI extends TestAPI {
      .expect(expectStatus)
    return request
  }
+
+  search = async (
+    sourceId: string,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Row[]> => {
+    const request = this.request
+      .post(`/api/${sourceId}/search`)
+      .set(this.config.defaultHeaders())
+      .expect(expectStatus)
+
+    return (await request).body
+  }
 }
--- a/packages/server/src/threads/definitions.ts
+++ b/packages/server/src/threads/definitions.ts
@ -11,12 +11,7 @@ export interface QueryEvent {
  queryId: string
  environmentVariables?: Record<string, string>
  ctx?: any
-  schema?: {
-    [key: string]: {
-      name: string
-      type: string
-    }
-  }
+  schema?: Record<string, { name?: string; type: string }>
 }

 export interface QueryVariable {
--- a/packages/server/src/utilities/centralPath.ts
+++ b/packages/server/src/utilities/centralPath.ts
@ -8,7 +8,7 @@ import path from "path"
 * @param args Any number of string arguments to add to a path
 * @returns {string} The final path ready to use
 */
-export function join(...args: any) {
+export function join(...args: string[]) {
  return path.join(...args)
 }

@ -17,6 +17,6 @@ export function join(...args: any) {
 * @param args Any number of string arguments to add to a path
 * @returns {string} The final path ready to use
 */
-export function resolve(...args: any) {
+export function resolve(...args: string[]) {
  return path.resolve(...args)
 }
--- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
+++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
@ -6,7 +6,7 @@ import { InvalidBBRefError } from "./errors"
 export async function processInputBBReferences(
  value: string | string[] | { _id: string } | { _id: string }[],
  subtype: FieldSubtype
-): Promise<string | undefined> {
+): Promise<string | null> {
  const referenceIds: string[] = []

  if (Array.isArray(value)) {
@ -39,7 +39,7 @@ export async function processInputBBReferences(
      throw utils.unreachable(subtype)
  }

-  return referenceIds.join(",") || undefined
+  return referenceIds.join(",") || null
 }

 export async function processOutputBBReferences(
@ -48,7 +48,7 @@ export async function processOutputBBReferences(
 ) {
  if (typeof value !== "string") {
    // Already processed or nothing to process
-    return value
+    return value || undefined
  }

  const ids = value.split(",").filter(id => !!id)
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@ -11,6 +11,7 @@ import {
  processInputBBReferences,
  processOutputBBReferences,
 } from "./bbReferenceProcessor"
+import { isExternalTable } from "../../integrations/utils"
 export * from "./utils"

 type AutoColumnProcessingOpts = {
@ -200,7 +201,10 @@ export async function inputProcessing(
 export async function outputProcessing<T extends Row[] | Row>(
  table: Table,
  rows: T,
-  opts = { squash: true }
+  opts: { squash?: boolean; preserveLinks?: boolean } = {
+    squash: true,
+    preserveLinks: false,
+  }
 ): Promise<T> {
  let safeRows: Row[]
  let wasArray = true
@ -211,7 +215,9 @@ export async function outputProcessing<T extends Row[] | Row>(
    safeRows = rows
  }
  // attach any linked row information
-  let enriched = await linkRows.attachFullLinkedDocs(table, safeRows)
+  let enriched = !opts.preserveLinks
+    ? await linkRows.attachFullLinkedDocs(table, safeRows)
+    : safeRows

  // process formulas
  enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]
@ -229,9 +235,6 @@ export async function outputProcessing<T extends Row[] | Row>(
      }
    } else if (column.type == FieldTypes.BB_REFERENCE) {
      for (let row of enriched) {
-        if (row[property] == null) {
-          continue
-        }
        row[property] = await processOutputBBReferences(
          row[property],
          column.subtype as FieldSubtype
@ -245,6 +248,16 @@ export async function outputProcessing<T extends Row[] | Row>(
      enriched
    )) as Row[]
  }
+  // remove null properties to match internal API
+  if (isExternalTable(table._id!)) {
+    for (let row of enriched) {
+      for (let key of Object.keys(row)) {
+        if (row[key] === null) {
+          delete row[key]
+        }
+      }
+    }
+  }
  return (wasArray ? enriched : enriched[0]) as T
 }

--- a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
@ -139,20 +139,20 @@ describe("bbReferenceProcessor", () => {
        expect(cacheGetUsersSpy).toBeCalledWith(userIds)
      })

-      it("empty strings will return undefined", async () => {
+      it("empty strings will return null", async () => {
        const result = await config.doInTenant(() =>
          processInputBBReferences("", FieldSubtype.USER)
        )

-        expect(result).toEqual(undefined)
+        expect(result).toEqual(null)
      })

-      it("empty arrays will return undefined", async () => {
+      it("empty arrays will return null", async () => {
        const result = await config.doInTenant(() =>
          processInputBBReferences([], FieldSubtype.USER)
        )

-        expect(result).toEqual(undefined)
+        expect(result).toEqual(null)
      })
    })
  })
--- a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts
@ -66,7 +66,7 @@ describe("rowProcessor - outputProcessing", () => {
    )
  })

-  it("does not fetch bb references when fields are empty", async () => {
+  it("process output even when the field is not empty", async () => {
    const table: Table = {
      _id: generator.guid(),
      name: "TestTable",
@ -100,7 +100,7 @@ describe("rowProcessor - outputProcessing", () => {

    expect(result).toEqual({ name: "Jack" })

-    expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled()
+    expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledTimes(1)
  })

  it("does not fetch bb references when not in the schema", async () => {
--- a/packages/server/src/utilities/users.ts
+++ b/packages/server/src/utilities/users.ts
@ -36,5 +36,8 @@ export function publicApiUserFix(ctx: UserCtx) {
  if (!ctx.request.body._id && ctx.params.userId) {
    ctx.request.body._id = ctx.params.userId
  }
+  if (!ctx.request.body.roles) {
+    ctx.request.body.roles = {}
+  }
  return ctx
 }
--- a/packages/shared-core/package.json
+++ b/packages/shared-core/package.json
@ -17,7 +17,6 @@
    "@budibase/types": "0.0.0"
  },
  "devDependencies": {
-    "concurrently": "^7.6.0",
    "rimraf": "3.0.2",
    "typescript": "5.2.2"
  },
--- a/packages/string-templates/package.json
+++ b/packages/string-templates/package.json
@ -28,7 +28,6 @@
    "@budibase/handlebars-helpers": "^0.11.9",
    "dayjs": "^1.10.8",
    "handlebars": "^4.7.6",
-    "handlebars-utils": "^1.0.6",
    "lodash": "^4.17.20",
    "vm2": "^3.9.15"
  },
@ -37,7 +36,6 @@
    "@rollup/plugin-json": "^4.1.0",
    "doctrine": "^3.0.0",
    "jest": "29.6.2",
-    "jest-environment-node": "29.6.2",
    "marked": "^4.0.10",
    "rollup": "^2.36.2",
    "rollup-plugin-inject-process-env": "^1.3.1",
--- a/packages/types/package.json
+++ b/packages/types/package.json
@ -16,13 +16,10 @@
  "jest": {},
  "devDependencies": {
    "@budibase/nano": "10.1.2",
-    "@types/json5": "2.2.0",
    "@types/koa": "2.13.4",
    "@types/node": "18.17.0",
    "@types/pouchdb": "6.4.0",
    "@types/redlock": "4.0.3",
-    "concurrently": "^7.6.0",
-    "koa-body": "4.2.0",
    "rimraf": "3.0.2",
    "typescript": "5.2.2"
  },
--- a/packages/types/src/api/account/accounts.ts
+++ b/packages/types/src/api/account/accounts.ts
@ -1,4 +1,4 @@
-import { Account } from "../../documents"
+import { Account, AccountSSOProvider } from "../../documents"
 import { Hosting } from "../../sdk"

 export interface CreateAccountRequest {
@ -11,6 +11,8 @@ export interface CreateAccountRequest {
  tenantName?: string
  name?: string
  password: string
+  provider?: AccountSSOProvider
+  thirdPartyProfile: object
 }

 export interface SearchAccountsRequest {
--- a/packages/types/src/api/web/user.ts
+++ b/packages/types/src/api/web/user.ts
@ -61,6 +61,7 @@ export interface CreateAdminUserRequest {
  email: string
  password: string
  tenantId: string
+  ssoId?: string
 }

 export interface CreateAdminUserResponse {
--- a/packages/types/src/documents/account/account.ts
+++ b/packages/types/src/documents/account/account.ts
@ -20,6 +20,11 @@ export interface CreatePassswordAccount extends CreateAccount {
  password: string
 }

+export interface CreateVerifiableSSOAccount extends CreateAccount {
+  provider?: AccountSSOProvider
+  thirdPartyProfile?: any
+}
+
 export const isCreatePasswordAccount = (
  account: CreateAccount
 ): account is CreatePassswordAccount => account.authType === AuthType.PASSWORD
@ -50,6 +55,8 @@ export interface Account extends CreateAccount {
  licenseKeyActivatedAt?: number
  licenseRequestedAt?: number
  licenseOverrides?: LicenseOverrides
+  provider?: AccountSSOProvider
+  providerType?: AccountSSOProviderType
  quotaUsage?: QuotaUsage
  offlineLicenseToken?: string
 }
@ -87,6 +94,13 @@ export enum AccountSSOProvider {
  MICROSOFT = "microsoft",
 }

+const verifiableSSOProviders: AccountSSOProvider[] = [
+  AccountSSOProvider.MICROSOFT,
+]
+export function isVerifiableSSOProvider(provider: AccountSSOProvider): boolean {
+  return verifiableSSOProviders.includes(provider)
+}
+
 export interface AccountSSO {
  provider: AccountSSOProvider
  providerType: AccountSSOProviderType
--- a/packages/types/src/documents/app/query.ts
+++ b/packages/types/src/documents/app/query.ts
@ -6,7 +6,7 @@ export interface Query extends Document {
  parameters: QueryParameter[]
  fields: RestQueryFields | any
  transformer: string | null
-  schema: any
+  schema: Record<string, { name?: string; type: string }>
  readable: boolean
  queryVerb: string
 }
--- a/packages/types/src/documents/global/user.ts
+++ b/packages/types/src/documents/global/user.ts
@ -55,6 +55,7 @@ export interface User extends Document {
  userGroups?: string[]
  onboardedAt?: string
  scimInfo?: { isSync: true } & Record<string, any>
+  ssoId?: string
 }

 export enum UserStatus {
--- a/packages/types/src/documents/platform/users.ts
+++ b/packages/types/src/documents/platform/users.ts
@ -15,4 +15,16 @@ export interface PlatformUserById extends Document {
  tenantId: string
 }

-export type PlatformUser = PlatformUserByEmail | PlatformUserById
+/**
+ * doc id is a unique SSO provider ID for the user
+ */
+export interface PlatformUserBySsoId extends Document {
+  tenantId: string
+  userId: string
+  email: string
+}
+
+export type PlatformUser =
+  | PlatformUserByEmail
+  | PlatformUserById
+  | PlatformUserBySsoId
--- a/packages/types/src/sdk/db.ts
+++ b/packages/types/src/sdk/db.ts
@ -1,7 +1,6 @@
 import Nano from "@budibase/nano"
 import { AllDocsResponse, AnyDocument, Document } from "../"
 import { Writable } from "stream"
-import PouchDB from "pouchdb"

 export enum SearchIndex {
  ROWS = "rows",
--- a/packages/types/src/sdk/koa.ts
+++ b/packages/types/src/sdk/koa.ts
@ -1,5 +1,5 @@
 import { Context, Request } from "koa"
-import { User, Role, UserRoles, Account } from "../documents"
+import { User, Role, UserRoles, Account, ConfigType } from "../documents"
 import { FeatureFlag, License } from "../sdk"
 import { Files } from "formidable"

@ -13,6 +13,7 @@ export interface ContextUser extends Omit<User, "roles"> {
  csrfToken?: string
  featureFlags?: FeatureFlag[]
  accountPortalAccess?: boolean
+  providerType?: ConfigType
  account?: Account
 }

--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@ -67,7 +67,6 @@
    "node-fetch": "2.6.7",
    "nodemailer": "6.7.2",
    "passport-google-oauth": "2.0.0",
-    "passport-jwt": "4.0.0",
    "passport-local": "1.0.0",
    "pouchdb": "7.3.0",
    "pouchdb-all-dbs": "1.1.1",
@ -84,21 +83,15 @@
    "@types/lodash": "^4.14.191",
    "@types/node": "18.17.0",
    "@types/node-fetch": "2.6.4",
-    "@types/pouchdb": "6.4.0",
    "@types/server-destroy": "1.0.1",
    "@types/supertest": "2.0.12",
    "@types/uuid": "8.3.4",
-    "copyfiles": "2.4.1",
-    "eslint": "6.8.0",
    "jest": "29.6.2",
    "lodash": "4.17.21",
    "nodemon": "2.0.15",
-    "pouchdb-adapter-memory": "7.2.2",
    "rimraf": "3.0.2",
    "supertest": "6.2.2",
    "timekeeper": "2.2.0",
-    "ts-node": "10.8.1",
-    "tsconfig-paths": "4.0.0",
    "typescript": "5.2.2",
    "update-dotenv": "1.1.1"
  },
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@ -95,7 +95,7 @@ const parseBooleanParam = (param: any) => {
 export const adminUser = async (
  ctx: Ctx<CreateAdminUserRequest, CreateAdminUserResponse>
 ) => {
-  const { email, password, tenantId } = ctx.request.body
+  const { email, password, tenantId, ssoId } = ctx.request.body

  if (await platform.tenants.exists(tenantId)) {
    ctx.throw(403, "Organisation already exists.")
@ -136,6 +136,7 @@ export const adminUser = async (
        global: true,
      },
      tenantId,
+      ssoId,
    }
    try {
      // always bust checklist beforehand, if an error occurs but can proceed, don't get
--- a/packages/worker/src/api/routes/global/tests/roles.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/roles.spec.ts
@ -1,6 +1,5 @@
 import { structures, TestConfiguration } from "../../../../tests"
 import { context, db, permissions, roles } from "@budibase/backend-core"
-import { Mock } from "jest-mock"
 import { Database } from "@budibase/types"

 jest.mock("@budibase/backend-core", () => {
@ -47,7 +46,7 @@ describe("/api/global/roles", () => {
  beforeEach(async () => {
    appId = db.generateAppID(config.tenantId)
    appDb = db.getDB(appId)
-    const mockAppDB = context.getAppDB as Mock
+    const mockAppDB = context.getAppDB as jest.Mock
    mockAppDB.mockReturnValue(appDb)

    await addAppMetadata()
--- a/packages/worker/src/api/routes/global/tests/scim.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/scim.spec.ts
@ -11,7 +11,7 @@ import { TestConfiguration } from "../../../../tests"
 import { events } from "@budibase/backend-core"

 // this test can 409 - retries reduce issues with this
-jest.retryTimes(2)
+jest.retryTimes(2, { logErrorsBeforeRetry: true })
 jest.setTimeout(30000)

 mocks.licenses.useScimIntegration()
--- a/packages/worker/src/api/routes/global/users.ts
+++ b/packages/worker/src/api/routes/global/users.ts
@ -14,6 +14,7 @@ function buildAdminInitValidation() {
      email: Joi.string().required(),
      password: Joi.string(),
      tenantId: Joi.string().required(),
+      ssoId: Joi.string(),
    })
      .required()
      .unknown(false)
--- a/packages/worker/tsconfig.json
+++ b/packages/worker/tsconfig.json
@ -4,10 +4,6 @@
    "composite": true,
    "baseUrl": "."
  },
-  "ts-node": {
-    "require": ["tsconfig-paths/register"],
-    "swc": true
-  },
  "include": ["src/**/*", "__mocks__/**/*"],
  "exclude": ["node_modules", "dist"]
 }
--- a/scripts/syncProPackage.js
+++ b/scripts/syncProPackage.js
@ -33,7 +33,7 @@ Object.keys(data).forEach(workspace => {

  let hasChanges = false

-  if (packageJson.dependencies["@budibase/pro"]) {
+  if (packageJson.dependencies && packageJson.dependencies["@budibase/pro"]) {
    packageJson.dependencies["@budibase/pro"] = version
    hasChanges = true
  }
--- a/specs/openapi.json
+++ b/specs/openapi.json
@ -1 +0,0 @@
-../packages/server/specs/openapi.json
--- a/Show More
+++ b/Show More