Merge branch 'Budibase:develop' into allow-plugins-contribute-datasourceplus

.github/workflows/close-featurebranch.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: passeidireto/trigger-external-workflow-action@main
         env:
           PAYLOAD_BRANCH: ${{ github.head_ref }}
-          PAYLOAD_PR_NUMBER: ${{ github.ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-close

.github/workflows/deploy-featurebranch.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: passeidireto/trigger-external-workflow-action@main
         env:
           PAYLOAD_BRANCH: ${{ github.head_ref }}
-          PAYLOAD_PR_NUMBER: ${{ github.ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-deploy

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.10.12-alpha.23",
+  "version": "2.11.5-alpha.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -21,7 +21,7 @@
     "prettier-plugin-svelte": "^2.3.0",
     "rimraf": "^3.0.2",
     "rollup-plugin-replace": "^2.2.0",
-    "svelte": "^3.38.2",
+    "svelte": "3.49.0",
     "typescript": "5.2.2",
     "@babel/core": "^7.22.5",
     "@babel/eslint-parser": "^7.22.5",

packages/backend-core/package.json

@@ -33,17 +33,14 @@
     "bull": "4.10.1",
     "correlation-id": "4.0.0",
     "dotenv": "16.0.1",
-    "emitter-listener": "1.1.2",
     "ioredis": "5.3.2",
     "joi": "17.6.0",
     "jsonwebtoken": "9.0.0",
     "koa-passport": "4.1.4",
     "koa-pino-logger": "4.0.0",
     "lodash": "4.17.21",
-    "lodash.isarguments": "3.1.0",
     "node-fetch": "2.6.7",
     "passport-google-oauth": "2.0.0",
-    "passport-jwt": "4.0.0",
     "passport-local": "1.0.0",
     "passport-oauth2-refresh": "^2.1.0",
     "pino": "8.11.0",
@@ -59,14 +56,13 @@
     "uuid": "8.3.2"
   },
   "devDependencies": {
-    "@jest/test-sequencer": "29.6.2",
     "@shopify/jest-koa-mocks": "5.1.1",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
     "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/chance": "1.1.3",
+    "@types/cookies": "0.7.8",
     "@types/jest": "29.5.3",
-    "@types/koa": "2.13.4",
     "@types/lodash": "4.14.180",
     "@types/node": "18.17.0",
     "@types/node-fetch": "2.6.4",
@@ -80,13 +76,9 @@
     "jest": "29.6.2",
     "jest-environment-node": "29.6.2",
     "jest-serial-runner": "1.2.1",
-    "koa": "2.13.4",
-    "nodemon": "2.0.16",
     "pino-pretty": "10.0.0",
     "pouchdb-adapter-memory": "7.2.2",
     "timekeeper": "2.2.0",
-    "ts-node": "10.8.1",
-    "tsconfig-paths": "4.0.0",
     "typescript": "5.2.2"
   },
   "nx": {

packages/backend-core/src/constants/db.ts

@@ -18,7 +18,7 @@ export enum ViewName {
   ROUTING = "screen_routes",
   AUTOMATION_LOGS = "automation_logs",
   ACCOUNT_BY_EMAIL = "account_by_email",
-  PLATFORM_USERS_LOWERCASE = "platform_users_lowercase",
+  PLATFORM_USERS_LOWERCASE = "platform_users_lowercase_2",
   USER_BY_GROUP = "user_by_group",
   APP_BACKUP_BY_TRIGGER = "by_trigger",
 }

packages/backend-core/src/db/views.ts

@@ -190,6 +190,10 @@ export const createPlatformUserView = async () => {
     if (doc.tenantId) {
       emit(doc._id.toLowerCase(), doc._id)
     }
+
+    if (doc.ssoId) {
+      emit(doc.ssoId, doc._id)
+    }
   }`
   await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
 }

packages/backend-core/src/platform/users.ts

@@ -5,6 +5,7 @@ import {
   PlatformUser,
   PlatformUserByEmail,
   PlatformUserById,
+  PlatformUserBySsoId,
   User,
 } from "@budibase/types"
 
@@ -45,6 +46,20 @@ function newUserEmailDoc(
   }
 }
 
+function newUserSsoIdDoc(
+  ssoId: string,
+  email: string,
+  userId: string,
+  tenantId: string
+): PlatformUserBySsoId {
+  return {
+    _id: ssoId,
+    userId,
+    email,
+    tenantId,
+  }
+}
+
 /**
  * Add a new user id or email doc if it doesn't exist.
  */
@@ -64,11 +79,24 @@ async function addUserDoc(emailOrId: string, newDocFn: () => PlatformUser) {
   }
 }
 
-export async function addUser(tenantId: string, userId: string, email: string) {
+export async function addUser(
-  await Promise.all([
+  tenantId: string,
+  userId: string,
+  email: string,
+  ssoId?: string
+) {
+  const promises = [
     addUserDoc(userId, () => newUserIdDoc(userId, tenantId)),
     addUserDoc(email, () => newUserEmailDoc(userId, email, tenantId)),
-  ])
+  ]
+
+  if (ssoId) {
+    promises.push(
+      addUserDoc(ssoId, () => newUserSsoIdDoc(ssoId, email, userId, tenantId))
+    )
+  }
+
+  await Promise.all(promises)
 }
 
 // DELETE

packages/backend-core/src/security/permissions.ts

@@ -1,8 +1,9 @@
-import { PermissionType, PermissionLevel } from "@budibase/types"
+import { PermissionLevel, PermissionType } from "@budibase/types"
-export { PermissionType, PermissionLevel } from "@budibase/types"
 import flatten from "lodash/flatten"
 import cloneDeep from "lodash/fp/cloneDeep"
 
+export { PermissionType, PermissionLevel } from "@budibase/types"
+
 export type RoleHierarchy = {
   permissionId: string
 }[]
@@ -78,6 +79,7 @@ export const BUILTIN_PERMISSIONS = {
     permissions: [
       new Permission(PermissionType.QUERY, PermissionLevel.READ),
       new Permission(PermissionType.TABLE, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
     ],
   },
   WRITE: {
@@ -88,6 +90,7 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
       new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
       new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
     ],
   },
   POWER: {
@@ -99,6 +102,7 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
       new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
       new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
     ],
   },
   ADMIN: {
@@ -111,6 +115,7 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
       new Permission(PermissionType.QUERY, PermissionLevel.ADMIN),
       new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
     ],
   },
 }

packages/backend-core/src/security/roles.ts

@@ -215,21 +215,23 @@ async function getAllUserRoles(userRoleId?: string): Promise<RoleDoc[]> {
   return roles
 }
 
+export async function getUserRoleIdHierarchy(
+  userRoleId?: string
+): Promise<string[]> {
+  const roles = await getUserRoleHierarchy(userRoleId)
+  return roles.map(role => role._id!)
+}
+
 /**
  * Returns an ordered array of the user's inherited role IDs, this can be used
  * to determine if a user can access something that requires a specific role.
  * @param {string} userRoleId The user's role ID, this can be found in their access token.
- * @param {object} opts Various options, such as whether to only retrieve the IDs (default true).
+ * @returns {Promise<object[]>} returns an ordered array of the roles, with the first being their
- * @returns {Promise<string[]|object[]>} returns an ordered array of the roles, with the first being their
  * highest level of access and the last being the lowest level.
  */
-export async function getUserRoleHierarchy(
+export async function getUserRoleHierarchy(userRoleId?: string) {
-  userRoleId?: string,
-  opts = { idOnly: true }
-) {
   // special case, if they don't have a role then they are a public user
-  const roles = await getAllUserRoles(userRoleId)
+  return getAllUserRoles(userRoleId)
-  return opts.idOnly ? roles.map(role => role._id) : roles
 }
 
 // this function checks that the provided permissions are in an array format
@@ -249,6 +251,11 @@ export function checkForRoleResourceArray(
   return rolePerms
 }
 
+export async function getAllRoleIds(appId?: string) {
+  const roles = await getAllRoles(appId)
+  return roles.map(role => role._id)
+}
+
 /**
  * Given an app ID this will retrieve all of the roles that are currently within that app.
  * @return {Promise<object[]>} An array of the role objects that were found.
@@ -332,9 +339,7 @@ export class AccessController {
     }
     let roleIds = userRoleId ? this.userHierarchies[userRoleId] : null
     if (!roleIds && userRoleId) {
-      roleIds = (await getUserRoleHierarchy(userRoleId, {
+      roleIds = await getUserRoleIdHierarchy(userRoleId)
-        idOnly: true,
-      })) as string[]
       this.userHierarchies[userRoleId] = roleIds
     }
 

packages/backend-core/src/users/db.ts

@@ -278,7 +278,12 @@ export class UserDB {
         builtUser._rev = response.rev
 
         await eventHelpers.handleSaveEvents(builtUser, dbUser)
-        await platform.users.addUser(tenantId, builtUser._id!, builtUser.email)
+        await platform.users.addUser(
+          tenantId,
+          builtUser._id!,
+          builtUser.email,
+          builtUser.ssoId
+        )
         await cache.user.invalidateUser(response.id)
 
         await Promise.all(groupPromises)

packages/backend-core/src/utils/utils.ts

@@ -10,7 +10,7 @@ import {
   Event,
   TenantResolutionStrategy,
 } from "@budibase/types"
-import { SetOption } from "cookies"
+import type { SetOption } from "cookies"
 const jwt = require("jsonwebtoken")
 
 const APP_PREFIX = DocumentType.APP + SEPARATOR

packages/backend-core/tests/core/utilities/structures/accounts.ts

@@ -1,4 +1,4 @@
-import { generator, uuid, quotas } from "."
+import { generator, quotas, uuid } from "."
 import { generateGlobalUserID } from "../../../../src/docIds"
 import {
   Account,
@@ -6,10 +6,11 @@ import {
   AccountSSOProviderType,
   AuthType,
   CloudAccount,
-  Hosting,
-  SSOAccount,
   CreateAccount,
   CreatePassswordAccount,
+  CreateVerifiableSSOAccount,
+  Hosting,
+  SSOAccount,
 } from "@budibase/types"
 import sample from "lodash/sample"
 
@@ -68,6 +69,23 @@ export function ssoAccount(account: Account = cloudAccount()): SSOAccount {
   }
 }
 
+export function verifiableSsoAccount(
+  account: Account = cloudAccount()
+): SSOAccount {
+  return {
+    ...account,
+    authType: AuthType.SSO,
+    oauth2: {
+      accessToken: generator.string(),
+      refreshToken: generator.string(),
+    },
+    pictureUrl: generator.url(),
+    provider: AccountSSOProvider.MICROSOFT,
+    providerType: AccountSSOProviderType.MICROSOFT,
+    thirdPartyProfile: { id: "abc123" },
+  }
+}
+
 export const cloudCreateAccount: CreatePassswordAccount = {
   email: "cloud@budibase.com",
   tenantId: "cloud",
@@ -91,6 +109,19 @@ export const cloudSSOCreateAccount: CreateAccount = {
   profession: "Software Engineer",
 }
 
+export const cloudVerifiableSSOCreateAccount: CreateVerifiableSSOAccount = {
+  email: "cloud-sso@budibase.com",
+  tenantId: "cloud-sso",
+  hosting: Hosting.CLOUD,
+  authType: AuthType.SSO,
+  tenantName: "cloudsso",
+  name: "Budi Armstrong",
+  size: "10+",
+  profession: "Software Engineer",
+  provider: AccountSSOProvider.MICROSOFT,
+  thirdPartyProfile: { id: "abc123" },
+}
+
 export const selfCreateAccount: CreatePassswordAccount = {
   email: "self@budibase.com",
   tenantId: "self",

packages/bbui/package.json

@@ -20,14 +20,12 @@
     "@rollup/plugin-commonjs": "^16.0.0",
     "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-node-resolve": "^11.2.1",
-    "cross-env": "^7.0.2",
-    "nollup": "^0.14.1",
     "postcss": "^8.2.9",
     "rollup": "^2.45.2",
     "rollup-plugin-postcss": "^4.0.0",
     "rollup-plugin-svelte": "^7.1.0",
     "rollup-plugin-terser": "^7.0.2",
-    "svelte": "^3.38.2"
+    "svelte": "3.49.0"
   },
   "keywords": [
     "svelte"

packages/builder/package.json

@@ -65,7 +65,6 @@
     "@fortawesome/free-brands-svg-icons": "^6.2.1",
     "@fortawesome/free-solid-svg-icons": "^6.2.1",
     "@sentry/browser": "5.19.1",
-    "@spectrum-css/accordion": "^3.0.24",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
     "codemirror": "^5.59.0",
@@ -75,18 +74,17 @@
     "lodash": "4.17.21",
     "posthog-js": "^1.36.0",
     "remixicon": "2.5.0",
+    "sanitize-html": "^2.7.0",
     "shortid": "2.2.15",
     "svelte-dnd-action": "^0.9.8",
     "svelte-loading-spinners": "^0.1.1",
     "svelte-portal": "1.0.0",
-    "uuid": "8.3.1",
     "yup": "0.29.2"
   },
   "devDependencies": {
     "@babel/core": "^7.12.14",
     "@babel/plugin-transform-runtime": "^7.13.10",
     "@babel/preset-env": "^7.13.12",
-    "@babel/runtime": "^7.13.10",
     "@rollup/plugin-replace": "^2.4.2",
     "@roxi/routify": "2.18.5",
     "@sveltejs/vite-plugin-svelte": "1.0.1",
@@ -96,19 +94,10 @@
     "identity-obj-proxy": "^3.0.0",
     "jest": "29.6.2",
     "jsdom": "^21.1.1",
-    "mochawesome": "^7.1.3",
-    "mochawesome-merge": "^4.2.1",
-    "mochawesome-report-generator": "^6.2.0",
     "ncp": "^2.0.0",
-    "rimraf": "^3.0.2",
     "rollup": "^2.44.0",
-    "rollup-plugin-copy": "^3.4.0",
-    "start-server-and-test": "^1.12.1",
     "svelte": "^3.48.0",
     "svelte-jester": "^1.3.2",
-    "ts-node": "10.8.1",
-    "tsconfig-paths": "4.0.0",
-    "typescript": "5.2.2",
     "vite": "^3.0.8",
     "vite-plugin-static-copy": "^0.16.0",
     "vitest": "^0.29.2"

packages/builder/src/builderStore/store/automation/index.js

@@ -221,18 +221,6 @@ const automationActions = store => ({
     newAutomation.definition.steps.splice(blockIdx, 0, block)
     await store.actions.save(newAutomation)
   },
-  /**
-   * "rowControl" appears to be the name of the flag used to determine whether
-   * a certain automation block uses values or bindings as inputs
-   */
-  toggleRowControl: async (block, rowControl) => {
-    const newBlock = { ...block, rowControl }
-    const newAutomation = store.actions.getUpdatedDefinition(
-      get(selectedAutomation),
-      newBlock
-    )
-    await store.actions.save(newAutomation)
-  },
   deleteAutomationBlock: async block => {
     const automation = get(selectedAutomation)
     let newAutomation = cloneDeep(automation)

packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js

@@ -33,6 +33,8 @@ const generateTableBlock = datasource => {
       showTitleButton: true,
       titleButtonText: "Create row",
       titleButtonClickBehaviour: "new",
+      sidePanelSaveLabel: "Save",
+      sidePanelDeleteLabel: "Delete",
     })
     .instanceName(`${datasource.label} - Table block`)
   return tableBlock

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -7,7 +7,6 @@
     Detail,
     Modal,
     Button,
-    Select,
     ActionButton,
     notifications,
     Label,
@@ -39,9 +38,6 @@
     step => step.stepId === ActionStepID.COLLECT
   )
   $: automationId = $selectedAutomation?._id
-  $: showBindingPicker =
-    block.stepId === ActionStepID.CREATE_ROW ||
-    block.stepId === ActionStepID.UPDATE_ROW
   $: isTrigger = block.type === "TRIGGER"
   $: steps = $selectedAutomation?.definition?.steps ?? []
   $: blockIdx = steps.findIndex(step => step.id === block.id)
@@ -96,15 +92,6 @@
     }
   }
 
-  /**
-   * "rowControl" appears to be the name of the flag used to determine whether
-   * a certain automation block uses values or bindings as inputs
-   */
-  function toggleRowControl(evt) {
-    const rowControl = evt.detail !== "Use values"
-    automationStore.actions.toggleRowControl(block, rowControl)
-  }
-
   async function addLooping() {
     const loopDefinition = $automationStore.blockDefinitions.ACTION.LOOP
     const loopBlock = automationStore.actions.constructBlock(
@@ -189,16 +176,6 @@
                   Add Looping
                 </ActionButton>
               {/if}
-              {#if showBindingPicker}
-                <Select
-                  on:change={toggleRowControl}
-                  defaultValue="Use values"
-                  autoWidth
-                  value={block.rowControl ? "Use bindings" : "Use values"}
-                  options={["Use values", "Use bindings"]}
-                  placeholder={null}
-                />
-              {/if}
               <ActionButton
                 on:click={() => deleteStep()}
                 icon="DeleteOutline"

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -23,6 +23,7 @@
   import { environment, licensing } from "stores/portal"
   import WebhookDisplay from "../Shared/WebhookDisplay.svelte"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
+  import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import CodeEditorModal from "./CodeEditorModal.svelte"
   import QuerySelector from "./QuerySelector.svelte"
@@ -82,33 +83,6 @@
       ? [hbAutocomplete([...bindingsToCompletions(bindings, codeMode)])]
       : []
 
-  /**
-   * TODO - Remove after November 2023
-   * *******************************
-   * Code added to provide backwards compatibility between Values 1,2,3,4,5
-   * and the new JSON body.
-   */
-  let deprecatedSchemaProperties
-  $: {
-    if (block?.stepId === "integromat" || block?.stepId === "zapier") {
-      deprecatedSchemaProperties = schemaProperties.filter(
-        prop => !prop[0].startsWith("value")
-      )
-      if (!deprecatedSchemaProperties.map(entry => entry[0]).includes("body")) {
-        deprecatedSchemaProperties.push([
-          "body",
-          {
-            title: "Payload",
-            type: "json",
-          },
-        ])
-      }
-    } else {
-      deprecatedSchemaProperties = schemaProperties
-    }
-  }
-  /****************************************************/
-
   const getInputData = (testData, blockInputs) => {
     // Test data is not cloned for reactivity
     let newInputData = testData || cloneDeep(blockInputs)
@@ -118,30 +92,6 @@
       newInputData = cloneDeep(blockInputs)
     }
 
-    /**
-     * TODO - Remove after November 2023
-     * *******************************
-     * Code added to provide backwards compatibility between Values 1,2,3,4,5
-     * and the new JSON body.
-     */
-    if (
-      (block?.stepId === "integromat" || block?.stepId === "zapier") &&
-      !newInputData?.body?.value
-    ) {
-      let deprecatedValues = {
-        ...newInputData,
-      }
-      delete deprecatedValues.url
-      delete deprecatedValues.body
-      newInputData = {
-        url: newInputData.url,
-        body: {
-          value: JSON.stringify(deprecatedValues),
-        },
-      }
-    }
-    /**********************************/
-
     inputData = newInputData
     setDefaultEnumValues()
   }
@@ -337,7 +287,7 @@
 </script>
 
 <div class="fields">
-  {#each deprecatedSchemaProperties as [key, value]}
+  {#each schemaProperties as [key, value]}
     {#if canShowField(key, value)}
       <div class="block-field">
         {#if key !== "fields" && value.type !== "boolean"}
@@ -362,18 +312,6 @@
             mode="json"
             value={inputData[key]?.value}
             on:change={e => {
-              /**
-               * TODO - Remove after November 2023
-               * *******************************
-               * Code added to provide backwards compatibility between Values 1,2,3,4,5
-               * and the new JSON body.
-               */
-              delete inputData.value1
-              delete inputData.value2
-              delete inputData.value3
-              delete inputData.value4
-              delete inputData.value5
-              /***********************/
               onChange(e, key)
             }}
           />
@@ -386,10 +324,23 @@
             />
           </div>
         {:else if value.type === "date"}
-          <DatePicker
+          <DrawerBindableSlot
+            fillWidth
+            title={value.title}
+            panel={AutomationBindingPanel}
+            type={"date"}
             value={inputData[key]}
             on:change={e => onChange(e, key)}
-          />
+            {bindings}
+            allowJS={true}
+            updateOnChange={false}
+            drawerLeft="260px"
+          >
+            <DatePicker
+              value={inputData[key]}
+              on:change={e => onChange(e, key)}
+            />
+          </DrawerBindableSlot>
         {:else if value.customType === "column"}
           <Select
             on:change={e => onChange(e, key)}
@@ -469,7 +420,6 @@
           />
         {:else if value.customType === "row"}
           <RowSelector
-            {block}
             value={inputData[key]}
             meta={inputData["meta"] || {}}
             on:change={e => {

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -1,18 +1,16 @@
 <script>
   import { tables } from "stores/backend"
   import { Select, Checkbox } from "@budibase/bbui"
-  import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
-  import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
   import RowSelectorTypes from "./RowSelectorTypes.svelte"
-  import ModalBindableInput from "../../common/bindings/ModalBindableInput.svelte"
+  import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
+  import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
 
   const dispatch = createEventDispatcher()
 
   export let value
   export let meta
   export let bindings
-  export let block
   export let isTestModal
   export let isUpdateRow
 
@@ -25,16 +23,6 @@
   let table
   let schemaFields
 
-  let placeholders = {
-    number: 10,
-    boolean: "true",
-    datetime: "2022-02-16T12:00:00.000Z ",
-    options: "1",
-    array: "1 2 3 4",
-    link: "ro_ta_123_456",
-    longform: "long form text",
-  }
-  $: rowControl = block.rowControl
   $: {
     table = $tables.list.find(table => table._id === value?.tableId)
     schemaFields = Object.entries(table?.schema ?? {})
@@ -57,19 +45,13 @@
       return value
     }
 
-    if (type === "boolean") {
-      if (typeof value === "boolean") {
-        return value
-      }
-      return value === "true"
-    }
     if (type === "number") {
       if (typeof value === "number") {
         return value
       }
       return Number(value)
     }
-    if (type === "options") {
+    if (type === "options" || type === "boolean") {
       return value
     }
     if (type === "array") {
@@ -127,47 +109,40 @@
 {#if schemaFields.length}
   <div class="schema-fields">
     {#each schemaFields as [field, schema]}
-      {#if !schema.autocolumn}
+      {#if !schema.autocolumn && schema.type !== "attachment"}
-        {#if schema.type !== "attachment"}
+        <DrawerBindableSlot
-          {#if !rowControl}
+          fillWidth
-            <RowSelectorTypes
+          title={value.title}
-              {isTestModal}
+          label={field}
-              {field}
+          panel={AutomationBindingPanel}
-              {schema}
+          type={schema.type}
-              bindings={parsedBindings}
+          {schema}
-              {value}
+          value={value[field]}
-              {onChange}
+          on:change={e => onChange(e, field)}
-            />
+          {bindings}
-          {:else}
+          allowJS={true}
-            <div>
+          updateOnChange={false}
-              <svelte:component
+          drawerLeft="260px"
-                this={isTestModal ? ModalBindableInput : DrawerBindableInput}
+        >
-                placeholder={placeholders[schema.type]}
+          <RowSelectorTypes
-                panel={AutomationBindingPanel}
+            {isTestModal}
-                value={Array.isArray(value[field])
+            {field}
-                  ? value[field].join(",")
+            {schema}
-                  : value[field]}
+            bindings={parsedBindings}
-                on:change={e => onChange(e, field, schema.type)}
+            {value}
-                label={field}
+            {onChange}
-                type="string"
+          />
-                bindings={parsedBindings}
+        </DrawerBindableSlot>
-                fillWidth={true}
+      {/if}
-                allowJS={true}
+      {#if isUpdateRow && schema.type === "link"}
-                updateOnChange={false}
+        <div class="checkbox-field">
-              />
+          <Checkbox
-              {#if isUpdateRow && schema.type === "link"}
+            value={meta.fields?.[field]?.clearRelationships}
-                <div class="checkbox-field">
+            text={"Clear relationships if empty?"}
-                  <Checkbox
+            size={"S"}
-                    value={meta.fields?.[field]?.clearRelationships}
+            on:change={e => onChangeSetting(e, field)}
-                    text={"Clear relationships if empty?"}
+          />
-                    size={"S"}
+        </div>
-                    on:change={e => onChangeSetting(e, field)}
-                  />
-                </div>
-              {/if}
-            </div>
-          {/if}
-        {/if}
       {/if}
     {/each}
   </div>

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -1,7 +1,6 @@
 <script>
   import {
     Select,
-    Toggle,
     DatePicker,
     Multiselect,
     TextArea,
@@ -45,10 +44,14 @@
     on:change={e => onChange(e, field)}
   />
 {:else if schema.type === "boolean"}
-  <Toggle
+  <Select
-    text={field}
-    value={value[field]}
     on:change={e => onChange(e, field)}
+    label={field}
+    value={value[field]}
+    options={[
+      { label: "True", value: "true" },
+      { label: "False", value: "false" },
+    ]}
   />
 {:else if schema.type === "array"}
   <Multiselect

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -33,7 +33,7 @@
   import { getBindings } from "components/backend/DataTable/formula"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
   import { ValidColumnNameRegex } from "@budibase/shared-core"
-  import { FieldSubtype, FieldType } from "@budibase/types"
+  import { FieldType } from "@budibase/types"
   import RelationshipSelector from "components/common/RelationshipSelector.svelte"
 
   const AUTO_TYPE = "auto"
@@ -43,11 +43,7 @@
   const NUMBER_TYPE = FIELDS.NUMBER.type
   const JSON_TYPE = FIELDS.JSON.type
   const DATE_TYPE = FIELDS.DATETIME.type
-  const BB_REFERENCE_TYPE = FieldType.BB_REFERENCE
+  const USER_REFRENCE_TYPE = FIELDS.BB_REFERENCE_USER.compositeType
-  const BB_USER_REFERENCE_TYPE = composeType(
-    BB_REFERENCE_TYPE,
-    FieldSubtype.USER
-  )
 
   const dispatch = createEventDispatcher()
   const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
@@ -66,7 +62,9 @@
   let relationshipPart1 = PrettyRelationshipDefinitions.Many
   let relationshipPart2 = PrettyRelationshipDefinitions.One
 
+  let relationshipTableIdPrimary = null
   let relationshipTableIdSecondary = null
+
   let table = $tables.selected
   let confirmDeleteDialog
   let savingColumn
@@ -79,40 +77,15 @@
     // Initial value for column name in other table for linked records
     fieldName: $tables.selected.name,
   }
-
+  let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions)
-  const bbRefTypeMapping = {}
+  let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions)
-
-  function composeType(fieldType, subtype) {
-    return `${fieldType}_${subtype}`
-  }
-
-  // Handling fields with subtypes
-  fieldDefinitions = Object.entries(fieldDefinitions).reduce(
-    (p, [key, field]) => {
-      if (field.type === BB_REFERENCE_TYPE) {
-        const composedType = composeType(field.type, field.subtype)
-        p[key] = {
-          ...field,
-          type: composedType,
-        }
-        bbRefTypeMapping[composedType] = {
-          type: field.type,
-          subtype: field.subtype,
-        }
-      } else {
-        p[key] = field
-      }
-      return p
-    },
-    {}
-  )
 
   $: if (primaryDisplay) {
     editableColumn.constraints.presence = { allowEmpty: false }
   }
 
   let relationshipMap = {
-    [RelationshipType.MANY_TO_ONE]: {
+    [RelationshipType.ONE_TO_MANY]: {
       part1: PrettyRelationshipDefinitions.MANY,
       part2: PrettyRelationshipDefinitions.ONE,
     },
@@ -120,14 +93,32 @@
       part1: PrettyRelationshipDefinitions.MANY,
       part2: PrettyRelationshipDefinitions.MANY,
     },
-    [RelationshipType.ONE_TO_MANY]: {
+    [RelationshipType.MANY_TO_ONE]: {
       part1: PrettyRelationshipDefinitions.ONE,
       part2: PrettyRelationshipDefinitions.MANY,
     },
   }
 
   $: {
+    // this parses any changes the user has made when creating a new internal relationship
+    // into what we expect the schema to look like
     if (editableColumn.type === LINK_TYPE) {
+      relationshipTableIdPrimary = table._id
+      if (relationshipPart1 === PrettyRelationshipDefinitions.ONE) {
+        relationshipOpts2 = relationshipOpts2.filter(
+          opt => opt !== PrettyRelationshipDefinitions.ONE
+        )
+      } else {
+        relationshipOpts2 = Object.values(PrettyRelationshipDefinitions)
+      }
+
+      if (relationshipPart2 === PrettyRelationshipDefinitions.ONE) {
+        relationshipOpts1 = relationshipOpts1.filter(
+          opt => opt !== PrettyRelationshipDefinitions.ONE
+        )
+      } else {
+        relationshipOpts1 = Object.values(PrettyRelationshipDefinitions)
+      }
       // Determine the relationship type based on the selected values of both parts
       editableColumn.relationshipType = Object.entries(relationshipMap).find(
         ([_, parts]) =>
@@ -137,7 +128,6 @@
       editableColumn.tableId = relationshipTableIdSecondary
     }
   }
-
   const initialiseField = (field, savingColumn) => {
     isCreating = !field
 
@@ -149,12 +139,21 @@
         $tables.selected.primaryDisplay == null ||
         $tables.selected.primaryDisplay === editableColumn.name
 
-      const mapped = Object.entries(bbRefTypeMapping).find(
+      if (editableColumn.type === FieldType.BB_REFERENCE) {
-        ([_, v]) => v.type === field.type && v.subtype === field.subtype
+        editableColumn.type = `${editableColumn.type}_${editableColumn.subtype}`
-      )
+      }
-      if (mapped) {
+      // Here we are setting the relationship values based on the editableColumn
-        editableColumn.type = mapped[0]
+      // This part of the code is used when viewing an existing field hence the check
-        delete editableColumn.subtype
+      // for the tableId
+      if (editableColumn.type === LINK_TYPE && editableColumn.tableId) {
+        relationshipTableIdPrimary = table._id
+        relationshipTableIdSecondary = editableColumn.tableId
+        if (editableColumn.relationshipType in relationshipMap) {
+          const { part1, part2 } =
+            relationshipMap[editableColumn.relationshipType]
+          relationshipPart1 = part1
+          relationshipPart2 = part2
+        }
       }
     } else if (!savingColumn) {
       let highestNumber = 0
@@ -174,22 +173,10 @@
     }
 
     allowedTypes = getAllowedTypes()
-
-    if (editableColumn.type === LINK_TYPE && editableColumn.tableId) {
-      relationshipTableIdSecondary = editableColumn.tableId
-      if (editableColumn.relationshipType in relationshipMap) {
-        const { part1, part2 } =
-          relationshipMap[editableColumn.relationshipType]
-        relationshipPart1 = part1
-        relationshipPart2 = part2
-      }
-    }
   }
 
   $: initialiseField(field, savingColumn)
 
-  $: isBBReference = !!bbRefTypeMapping[editableColumn.type]
-
   $: checkConstraints(editableColumn)
   $: required = !!editableColumn?.constraints?.presence || primaryDisplay
   $: uneditable =
@@ -246,10 +233,7 @@
   $: external = table.type === "external"
   // in the case of internal tables the sourceId will just be undefined
   $: tableOptions = $tables.list.filter(
-    opt =>
+    opt => opt.type === table.type && table.sourceId === opt.sourceId
-      opt._id !== $tables.selected._id &&
-      opt.type === table.type &&
-      table.sourceId === opt.sourceId
   )
   $: typeEnabled =
     !originalName ||
@@ -265,11 +249,12 @@
 
     let saveColumn = cloneDeep(editableColumn)
 
-    if (bbRefTypeMapping[saveColumn.type]) {
+    // Handle types on composite types
-      saveColumn = {
+    const definition = fieldDefinitions[saveColumn.type.toUpperCase()]
-        ...saveColumn,
+    if (definition && saveColumn.type === definition.compositeType) {
-        ...bbRefTypeMapping[saveColumn.type],
+      saveColumn.type = definition.type
-      }
+      saveColumn.subtype = definition.subtype
+      delete saveColumn.compositeType
     }
 
     if (saveColumn.type === AUTO_TYPE) {
@@ -292,10 +277,7 @@
       dispatch("updatecolumns")
       gridDispatch("close-edit-column")
 
-      if (
+      if (saveColumn.type === LINK_TYPE) {
-        saveColumn.type === LINK_TYPE &&
-        saveColumn.relationshipType === RelationshipType.MANY_TO_MANY
-      ) {
         // Fetching the new tables
         tables.fetch()
         // Fetching the new relationships
@@ -327,6 +309,11 @@
         confirmDeleteDialog.hide()
         dispatch("updatecolumns")
         gridDispatch("close-edit-column")
+
+        if (editableColumn.type === LINK_TYPE) {
+          // Updating the relationships
+          datasources.fetch()
+        }
       }
     } catch (error) {
       notifications.error(`Error deleting column: ${error.message}`)
@@ -352,7 +339,7 @@
       editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
     } else if (editableColumn.type === FORMULA_TYPE) {
       editableColumn.formulaType = "dynamic"
-    } else if (editableColumn.type === BB_USER_REFERENCE_TYPE) {
+    } else if (editableColumn.type === USER_REFRENCE_TYPE) {
       editableColumn.relationshipType = RelationshipType.ONE_TO_MANY
     }
   }
@@ -410,14 +397,12 @@
         FIELDS.BOOLEAN,
         FIELDS.FORMULA,
         FIELDS.BIGINT,
+        FIELDS.BB_REFERENCE_USER,
       ]
       // no-sql or a spreadsheet
       if (!external || table.sql) {
         fields = [...fields, FIELDS.LINK, FIELDS.ARRAY]
       }
-      if (fieldDefinitions.USER) {
-        fields.push(fieldDefinitions.USER)
-      }
       return fields
     }
   }
@@ -426,8 +411,9 @@
     if (!fieldToCheck) {
       return
     }
+
     // most types need this, just make sure its always present
-    if (fieldToCheck && !fieldToCheck.constraints) {
+    if (!fieldToCheck.constraints) {
       fieldToCheck.constraints = {}
     }
     // some string types may have been built by server, may not always have constraints
@@ -507,7 +493,7 @@
     on:change={handleTypeChange}
     options={allowedTypes}
     getOptionLabel={field => field.name}
-    getOptionValue={field => field.type}
+    getOptionValue={field => field.compositeType || field.type}
     getOptionIcon={field => field.icon}
     isOptionEnabled={option => {
       if (option.type == AUTO_TYPE) {
@@ -619,9 +605,11 @@
     <RelationshipSelector
       bind:relationshipPart1
       bind:relationshipPart2
-      bind:relationshipTableIdPrimary={table.name}
+      bind:relationshipTableIdPrimary
       bind:relationshipTableIdSecondary
       bind:editableColumn
+      {relationshipOpts1}
+      {relationshipOpts2}
       {linkEditDisabled}
       {tableOptions}
       {errors}
@@ -671,7 +659,7 @@
     <Button primary text on:click={openJsonSchemaEditor}
       >Open schema editor</Button
     >
-  {:else if isBBReference}
+  {:else if editableColumn.type === USER_REFRENCE_TYPE}
     <Toggle
       value={editableColumn.relationshipType === RelationshipType.MANY_TO_MANY}
       on:change={e =>

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -57,7 +57,8 @@
     label: table.name,
     value: table._id,
   }))
-  $: valid = getErrorCount(errors) === 0 && allRequiredAttributesSet()
+  $: valid =
+    getErrorCount(errors) === 0 && allRequiredAttributesSet(relationshipType)
   $: isManyToMany = relationshipType === RelationshipType.MANY_TO_MANY
   $: isManyToOne = relationshipType === RelationshipType.MANY_TO_ONE
 
@@ -114,7 +115,7 @@
     return Object.entries(errors).filter(entry => !!entry[1]).length
   }
 
-  function allRequiredAttributesSet() {
+  function allRequiredAttributesSet(relationshipType) {
     const base = getTable(fromId) && getTable(toId) && fromColumn && toColumn
     if (relationshipType === RelationshipType.MANY_TO_ONE) {
       return base && fromPrimary && fromForeign
@@ -124,9 +125,10 @@
   }
 
   function validate() {
-    if (!allRequiredAttributesSet() && !hasValidated) {
+    if (!allRequiredAttributesSet(relationshipType) && !hasValidated) {
       return
     }
+
     hasValidated = true
     errorChecker.setType(relationshipType)
     const fromTable = getTable(fromId),

packages/builder/src/components/common/LinkedRowSelector.svelte

@@ -11,11 +11,11 @@
   const dispatch = createEventDispatcher()
 
   let rows = []
-  let linkedIds = (Array.isArray(linkedRows) ? linkedRows : [])?.map(
+  let linkedIds = []
+
+  $: linkedIds = (Array.isArray(linkedRows) ? linkedRows : [])?.map(
     row => row?._id || row
   )
-
-  $: linkedRows = linkedIds
   $: label = capitalise(schema.name)
   $: linkedTableId = schema.tableId
   $: linkedTable = $tables.list.find(table => table._id === linkedTableId)

packages/builder/src/components/common/RelationshipSelector.svelte

@@ -1,6 +1,5 @@
 <script>
   import { Select, Input } from "@budibase/bbui"
-  import { PrettyRelationshipDefinitions } from "constants/backend"
 
   export let relationshipPart1
   export let relationshipPart2
@@ -10,6 +9,8 @@
   export let linkEditDisabled
   export let tableOptions
   export let errors
+  export let relationshipOpts1
+  export let relationshipOpts2
 </script>
 
 <div class="relationship-container">
@@ -17,15 +18,17 @@
     <Select
       disabled={linkEditDisabled}
       bind:value={relationshipPart1}
-      options={Object.values(PrettyRelationshipDefinitions)}
+      options={relationshipOpts1}
     />
   </div>
   <div class="relationship-label">in</div>
   <div class="relationship-part">
     <Select
       disabled
-      options={[relationshipTableIdPrimary]}
+      options={tableOptions}
-      value={relationshipTableIdPrimary}
+      getOptionLabel={table => table.name}
+      getOptionValue={table => table._id}
+      bind:value={relationshipTableIdPrimary}
     />
   </div>
 </div>
@@ -34,7 +37,7 @@
     <Select
       disabled={linkEditDisabled}
       bind:value={relationshipPart2}
-      options={Object.values(PrettyRelationshipDefinitions)}
+      options={relationshipOpts2}
       getOptionLabel={option => "To " + option.toLowerCase()}
     />
   </div>
@@ -43,7 +46,9 @@
     <Select
       disabled={linkEditDisabled}
       bind:value={relationshipTableIdSecondary}
-      options={tableOptions}
+      options={tableOptions.filter(
+        table => table._id !== relationshipTableIdPrimary
+      )}
       getOptionLabel={table => table.name}
       getOptionValue={table => table._id}
     />

packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte

@@ -0,0 +1,250 @@
+<script>
+  import { Icon, Input, Drawer, Button } from "@budibase/bbui"
+  import {
+    readableToRuntimeBinding,
+    runtimeToReadableBinding,
+  } from "builderStore/dataBinding"
+
+  import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
+  import { createEventDispatcher, setContext } from "svelte"
+  import { isJSBinding } from "@budibase/string-templates"
+
+  export let panel = ClientBindingPanel
+  export let value = ""
+  export let bindings = []
+  export let title = "Bindings"
+  export let placeholder
+  export let label
+  export let disabled = false
+  export let fillWidth
+  export let allowJS = true
+  export let allowHelpers = true
+  export let updateOnChange = true
+  export let drawerLeft
+  export let type
+  export let schema
+
+  const dispatch = createEventDispatcher()
+  let bindingDrawer
+  let valid = true
+  let currentVal = value
+
+  $: readableValue = runtimeToReadableBinding(bindings, value)
+  $: tempValue = readableValue
+  $: isJS = isJSBinding(value)
+
+  const saveBinding = () => {
+    onChange(tempValue)
+    onBlur()
+    bindingDrawer.hide()
+  }
+
+  setContext("binding-drawer-actions", {
+    save: saveBinding,
+  })
+
+  const onChange = value => {
+    if (type === "link" && value && hasValidLinks(value)) {
+      currentVal = value.split(",")
+    } else if (type === "array" && value && hasValidOptions(value)) {
+      currentVal = value.split(",")
+    } else {
+      currentVal = readableToRuntimeBinding(bindings, value)
+    }
+    dispatch("change", currentVal)
+  }
+
+  const onBlur = () => {
+    dispatch("blur", currentVal)
+  }
+
+  const isValidDate = value => {
+    return !value || !isNaN(new Date(value).valueOf())
+  }
+
+  const hasValidLinks = value => {
+    let links = []
+    if (Array.isArray(value)) {
+      links = value
+    } else if (value && typeof value === "string") {
+      links = value.split(",")
+    } else {
+      return !value
+    }
+
+    return links.every(link => link.startsWith("ro_"))
+  }
+
+  const hasValidOptions = value => {
+    let links = []
+    if (Array.isArray(value)) {
+      links = value
+    } else if (value && typeof value === "string") {
+      links = value.split(",")
+    } else {
+      return !value
+    }
+    return links.every(link => schema?.constraints?.inclusion?.includes(link))
+  }
+
+  const isValidBoolean = value => {
+    return value === "false" || value === "true" || value == ""
+  }
+
+  const validationMap = {
+    date: isValidDate,
+    datetime: isValidDate,
+    link: hasValidLinks,
+    array: hasValidOptions,
+    longform: value => !isJSBinding(value),
+    json: value => !isJSBinding(value),
+    boolean: isValidBoolean,
+  }
+
+  const isValid = value => {
+    const validate = validationMap[type]
+    return validate ? validate(value) : true
+  }
+
+  const getIconClass = (value, type) => {
+    if (type === "longform" && !isJSBinding(value)) {
+      return "text-area-slot-icon"
+    }
+    if (type === "json" && !isJSBinding(value)) {
+      return "json-slot-icon"
+    }
+    if (type !== "string" && type !== "number") {
+      return "slot-icon"
+    }
+    return ""
+  }
+</script>
+
+<div class="control" class:disabled>
+  {#if !isValid(value)}
+    <Input
+      {label}
+      {disabled}
+      readonly={isJS}
+      value={isJS ? "(JavaScript function)" : readableValue}
+      on:change={event => onChange(event.detail)}
+      on:blur={onBlur}
+      {placeholder}
+      {updateOnChange}
+    />
+    <div
+      class="icon"
+      on:click={() => {
+        if (!isJS) {
+          dispatch("change", "")
+        }
+      }}
+    >
+      <Icon disabled={isJS} size="S" name="Close" />
+    </div>
+  {:else}
+    <slot
+      {label}
+      {disabled}
+      readonly={isJS}
+      value={isJS ? "(JavaScript function)" : readableValue}
+      {placeholder}
+      {updateOnChange}
+    />
+  {/if}
+  {#if !disabled && type !== "formula"}
+    <div
+      class={`icon ${getIconClass(value, type)}`}
+      on:click={() => {
+        bindingDrawer.show()
+      }}
+    >
+      <Icon size="S" name="FlashOn" />
+    </div>
+  {/if}
+</div>
+<Drawer
+  on:drawerHide
+  on:drawerShow
+  {fillWidth}
+  bind:this={bindingDrawer}
+  {title}
+  left={drawerLeft}
+  headless
+>
+  <svelte:fragment slot="description">
+    Add the objects on the left to enrich your text.
+  </svelte:fragment>
+  <Button cta slot="buttons" disabled={!valid} on:click={saveBinding}>
+    Save
+  </Button>
+  <svelte:component
+    this={panel}
+    slot="body"
+    bind:valid
+    value={readableValue}
+    on:change={event => (tempValue = event.detail)}
+    {bindings}
+    {allowJS}
+    {allowHelpers}
+  />
+</Drawer>
+
+<style>
+  .control {
+    flex: 1;
+    position: relative;
+  }
+
+  .slot-icon {
+    right: 31px !important;
+    border-right: 1px solid var(--spectrum-alias-border-color);
+    border-top-right-radius: 0px !important;
+    border-bottom-right-radius: 0px !important;
+  }
+
+  .text-area-slot-icon {
+    border-bottom: 1px solid var(--spectrum-alias-border-color);
+    border-bottom-right-radius: 0px !important;
+    top: 26px !important;
+  }
+  .json-slot-icon {
+    border-bottom: 1px solid var(--spectrum-alias-border-color);
+    border-bottom-right-radius: 0px !important;
+    top: 23px !important;
+    right: 0px !important;
+  }
+
+  .icon {
+    right: 1px;
+    bottom: 1px;
+    position: absolute;
+    justify-content: center;
+    align-items: center;
+    display: flex;
+    flex-direction: row;
+    box-sizing: border-box;
+    border-left: 1px solid var(--spectrum-alias-border-color);
+    border-top-right-radius: var(--spectrum-alias-border-radius-regular);
+    border-bottom-right-radius: var(--spectrum-alias-border-radius-regular);
+    width: 31px;
+    color: var(--spectrum-alias-text-color);
+    background-color: var(--spectrum-global-color-gray-75);
+    transition: background-color
+        var(--spectrum-global-animation-duration-100, 130ms),
+      box-shadow var(--spectrum-global-animation-duration-100, 130ms),
+      border-color var(--spectrum-global-animation-duration-100, 130ms);
+    height: calc(var(--spectrum-alias-item-height-m) - 2px);
+  }
+
+  .icon:hover {
+    cursor: pointer;
+    color: var(--spectrum-alias-text-color-hover);
+    background-color: var(--spectrum-global-color-gray-50);
+    border-color: var(--spectrum-alias-border-color-hover);
+  }
+
+  .control:not(.disabled) :global(.spectrum-Textfield-input) {
+    padding-right: 40px;
+  }
+</style>

packages/builder/src/components/integration/QueryViewer.svelte

@@ -337,11 +337,12 @@
     padding: 8px 10px 8px 16px;
     display: flex;
     border-bottom: 2px solid transparent;
-    transition: border-bottom 130ms ease-out;
+    transition: border-bottom 130ms ease-out, background 130ms ease-out;
   }
 
   .header.scrolling {
     border-bottom: var(--border-light);
+    background: var(--background);
   }
 
   .body {

packages/builder/src/constants/backend/index.js

@@ -120,10 +120,11 @@ export const FIELDS = {
       presence: false,
     },
   },
-  USER: {
+  BB_REFERENCE_USER: {
     name: "User",
     type: "bb_reference",
     subtype: "user",
+    compositeType: "bb_reference_user", // Used for working with the subtype on CreateEditColumn as is it was a primary type
     icon: "User",
   },
 }

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Relationships.svelte

@@ -21,15 +21,22 @@
   function getRelationships(tables) {
     const relatedColumns = {}
 
-    tables.forEach(({ name: tableName, schema }) => {
+    tables.forEach(({ name: tableName, schema, _id: tableId }) => {
       Object.values(schema).forEach(column => {
         if (column.type !== "link") return
 
-        relatedColumns[column._id] ??= {}
+        const columnId =
-        relatedColumns[column._id].through =
+          column.through ||
-          relatedColumns[column._id].through || column.through
+          column._id ||
+          (column.main
+            ? `${tableId}_${column.fieldName}__${column.tableId}_${column.foreignKey}`
+            : `${column.tableId}_${column.foreignKey}__${tableId}_${column.fieldName}`)
 
-        relatedColumns[column._id][column.main ? "from" : "to"] = {
+        relatedColumns[columnId] ??= {}
+        relatedColumns[columnId].through =
+          relatedColumns[columnId].through || column.through
+
+        relatedColumns[columnId][column.main ? "from" : "to"] = {
           ...column,
           tableName,
         }

packages/builder/src/pages/builder/portal/apps/_layout.svelte

@@ -46,7 +46,9 @@
 
 {#if loaded}
   <div class="page">
-    <PortalSideBar />
+    {#if $apps.length > 0}
+      <PortalSideBar />
+    {/if}
     <slot />
   </div>
 {/if}

packages/builder/src/stores/backend/tables.js

@@ -1,7 +1,7 @@
 import { get, writable, derived } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
 import { API } from "api"
-import { SWITCHABLE_TYPES } from "constants/backend"
+import { SWITCHABLE_TYPES, FIELDS } from "constants/backend"
 
 export function createTablesStore() {
   const store = writable({
@@ -21,6 +21,23 @@ export function createTablesStore() {
     }))
   }
 
+  const singleFetch = async tableId => {
+    const table = await API.getTable(tableId)
+    store.update(state => {
+      const list = []
+      // update the list, keep order accurate
+      for (let tbl of state.list) {
+        if (table._id === tbl._id) {
+          list.push(table)
+        } else {
+          list.push(tbl)
+        }
+      }
+      state.list = list
+      return state
+    })
+  }
+
   const select = tableId => {
     store.update(state => ({
       ...state,
@@ -63,6 +80,20 @@ export function createTablesStore() {
     const savedTable = await API.saveTable(updatedTable)
     replaceTable(savedTable._id, savedTable)
     select(savedTable._id)
+    // make sure tables up to date (related)
+    let tableIdsToFetch = []
+    for (let column of Object.values(updatedTable?.schema || {})) {
+      if (column.type === FIELDS.LINK.type) {
+        tableIdsToFetch.push(column.tableId)
+      }
+    }
+    tableIdsToFetch = [...new Set(tableIdsToFetch)]
+    // too many tables to fetch, just get all
+    if (tableIdsToFetch.length > 3) {
+      await fetch()
+    } else {
+      await Promise.all(tableIdsToFetch.map(id => singleFetch(id)))
+    }
     return savedTable
   }
 

packages/builder/tsconfig.json

@@ -5,8 +5,5 @@
     "declaration": true,
     "sourceMap": true,
     "baseUrl": "."
-  },
-  "ts-node": {
-    "require": ["tsconfig-paths/register"]
   }
 }

packages/cli/package.json

@@ -32,7 +32,6 @@
     "@budibase/backend-core": "0.0.0",
     "@budibase/string-templates": "0.0.0",
     "@budibase/types": "0.0.0",
-    "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",
     "commander": "7.1.0",
@@ -41,7 +40,6 @@
     "download": "8.0.0",
     "find-free-port": "^2.0.0",
     "inquirer": "8.0.0",
-    "joi": "17.6.0",
     "lookpath": "1.1.0",
     "node-fetch": "2.6.7",
     "pkg": "5.8.0",
@@ -53,13 +51,9 @@
     "yaml": "^2.1.1"
   },
   "devDependencies": {
-    "@swc/core": "1.3.71",
-    "@swc/jest": "0.2.27",
     "@types/jest": "29.5.3",
     "@types/node-fetch": "2.6.4",
     "@types/pouchdb": "^6.4.0",
-    "copyfiles": "^2.4.1",
-    "eslint": "^7.20.0",
     "renamer": "^4.0.0",
     "ts-node": "^10.9.1",
     "typescript": "5.2.2"

packages/client/src/components/app/Layout.svelte

@@ -3,16 +3,16 @@
   import { writable } from "svelte/store"
   import { Heading, Icon, clickOutside } from "@budibase/bbui"
   import { FieldTypes } from "constants"
+  import { Constants } from "@budibase/frontend-core"
   import active from "svelte-spa-router/active"
-  import { RoleUtils } from "@budibase/frontend-core"
 
   const sdk = getContext("sdk")
   const {
     routeStore,
+    roleStore,
     styleable,
     linkable,
     builderStore,
-    currentRole,
     sidePanelStore,
   } = sdk
   const component = getContext("component")
@@ -61,7 +61,7 @@
   })
   setContext("layout", store)
 
-  $: validLinks = getValidLinks(links, $currentRole)
+  $: validLinks = getValidLinks(links, $roleStore)
   $: typeClass = NavigationClasses[navigation] || NavigationClasses.None
   $: navWidthClass = WidthClasses[navWidth || width] || WidthClasses.Large
   $: pageWidthClass = WidthClasses[pageWidth || width] || WidthClasses.Large
@@ -99,14 +99,13 @@
     }
   }
 
-  const getValidLinks = (allLinks, role) => {
+  const getValidLinks = (allLinks, userRoleHierarchy) => {
     // Strip links missing required info
     let validLinks = (allLinks || []).filter(link => link.text && link.url)
-
     // Filter to only links allowed by the current role
-    const priority = RoleUtils.getRolePriority(role)
     return validLinks.filter(link => {
-      return !link.roleId || RoleUtils.getRolePriority(link.roleId) <= priority
+      const role = link.roleId || Constants.Roles.BASIC
+      return userRoleHierarchy?.find(roleId => roleId === role)
     })
   }
 

packages/client/src/components/app/Section.svelte

@@ -47,28 +47,29 @@
 <style>
   div {
     display: grid;
-    grid-gap: 16px;
+    --gap: 16px;
+    grid-gap: var(--gap);
   }
   .mainSidebar {
-    grid-template-columns: 3fr 1fr;
+    grid-template-columns:
+      calc((100% - var(--gap)) / 4 * 3) /* 75% */
+      calc((100% - var(--gap)) / 4); /* 25% */
   }
   .sidebarMain {
-    grid-template-columns: 1fr 3fr;
+    grid-template-columns:
-  }
+      calc((100% - var(--gap)) / 4) /* 25% */
-  .oneColumn {
+      calc((100% - var(--gap)) / 4 * 3); /* 75% */
-    grid-template-columns: 1fr;
-  }
-  .twoColumns {
-    grid-template-columns: 1fr 1fr;
-  }
-  .threeColumns {
-    grid-template-columns: 1fr 1fr 1fr;
   }
+  .oneColumn,
   .columns-1 {
     grid-template-columns: 1fr;
   }
+  .twoColumns,
   .columns-2 {
-    grid-template-columns: 1fr 1fr;
+    grid-template-columns: repeat(2, calc((100% - var(--gap)) / 2));
+  }
+  .threeColumns {
+    grid-template-columns: repeat(3, calc((100% - var(--gap)) / 3));
   }
   .placeholder {
     border: 2px dashed var(--spectrum-global-color-gray-600);

packages/client/src/components/app/blocks/TableBlock.svelte

@@ -45,8 +45,21 @@
   let enrichedSearchColumns
   let schemaLoaded = false
 
-  // Accommodate old config to ensure delete button does not reappear
+  $: deleteLabel = setDeleteLabel(sidePanelDeleteLabel, sidePanelShowDelete)
-  $: deleteLabel = sidePanelShowDelete === false ? "" : sidePanelDeleteLabel
+
+  const setDeleteLabel = sidePanelDeleteLabel => {
+    // Accommodate old config to ensure delete button does not reappear
+    let labelText = sidePanelShowDelete === false ? "" : sidePanelDeleteLabel
+
+    // Empty text is considered hidden.
+    if (labelText?.trim() === "") {
+      return ""
+    }
+
+    // Default to "Delete" if the value is unset
+    return labelText || "Delete"
+  }
+
   $: isDSPlus = dataSource?.type === "table" || dataSource?.type === "viewV2"
   $: fetchSchema(dataSource)
   $: enrichSearchColumns(searchColumns, schema).then(
@@ -249,7 +262,7 @@
             props={{
               dataSource,
               saveButtonLabel: sidePanelSaveLabel || "Save", //always show
-              deleteButtonLabel: deleteLabel, //respect config
+              deleteButtonLabel: deleteLabel,
               actionType: "Update",
               rowId: `{{ ${safe("state")}.${safe(stateKey)} }}`,
               fields: sidePanelFields || normalFields,

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -60,6 +60,12 @@
       // even if they are not in the inital fetch results
       initialValuesProcessed = true
       optionsObj = (fieldState?.value || []).reduce((accumulator, value) => {
+        // fieldState has to be an array of strings to be valid for an update
+        // therefore we cannot guarantee value will be an object
+        // https://linear.app/budibase/issue/BUDI-7577/refactor-the-relationshipfield-component-to-have-better-support-for
+        if (!value._id) {
+          return accumulator
+        }
         accumulator[value._id] = {
           _id: value._id,
           [primaryDisplay]: value.primaryDisplay,
@@ -121,7 +127,12 @@
     if (!Array.isArray(values)) {
       values = [values]
     }
-    return values.map(value => (typeof value === "object" ? value._id : value))
+    values = values.map(value =>
+      typeof value === "object" ? value._id : value
+    )
+    // Make sure field state is valid
+    fieldApi.setValue(values)
+    return values
   }
 
   const getDisplayName = row => {

packages/client/src/components/devtools/DevToolsHeader.svelte

@@ -1,32 +1,38 @@
 <script>
   import { Heading, Select, ActionButton } from "@budibase/bbui"
-  import { devToolsStore, appStore } from "../../stores"
+  import { devToolsStore, appStore, roleStore } from "../../stores"
-  import { getContext } from "svelte"
+  import { getContext, onMount } from "svelte"
 
   const context = getContext("context")
+  const SELF_ROLE = "self"
 
-  $: previewOptions = [
+  let staticRoleList
-    {
+
+  $: previewOptions = buildRoleList(staticRoleList)
+
+  function buildRoleList(roleIds) {
+    const list = []
+    list.push({
       label: "View as yourself",
-      value: "self",
+      value: SELF_ROLE,
-    },
+    })
-    {
+    if (!roleIds) {
-      label: "View as public user",
+      return list
-      value: "PUBLIC",
+    }
-    },
+    for (let roleId of roleIds) {
-    {
+      list.push({
-      label: "View as basic user",
+        label: `View as ${roleId.toLowerCase()} user`,
-      value: "BASIC",
+        value: roleId,
-    },
+      })
-    {
+    }
-      label: "View as power user",
+    return list
-      value: "POWER",
+  }
-    },
+
-    {
+  onMount(async () => {
-      label: "View as admin user",
+    // make sure correct before starting
-      value: "ADMIN",
+    await devToolsStore.actions.changeRole(SELF_ROLE)
-    },
+    staticRoleList = await roleStore.actions.fetchAccessibleRoles()
-  ]
+  })
 </script>
 
 <div class="dev-preview-header" class:mobile={$context.device.mobile}>
@@ -34,7 +40,7 @@
   <Select
     quiet
     options={previewOptions}
-    value={$devToolsStore.role || "self"}
+    value={$devToolsStore.role || SELF_ROLE}
     placeholder={null}
     autoWidth
     on:change={e => devToolsStore.actions.changeRole(e.detail)}

packages/client/src/sdk.js

@@ -13,6 +13,7 @@ import {
   sidePanelStore,
   dndIsDragging,
   confirmationStore,
+  roleStore,
 } from "stores"
 import { styleable } from "utils/styleable"
 import { linkable } from "utils/linkable"
@@ -39,6 +40,7 @@ export default {
   dndIsDragging,
   currentRole,
   confirmationStore,
+  roleStore,
   styleable,
   linkable,
   getAction,

packages/client/src/stores/devTools.js

@@ -2,6 +2,7 @@ import { createLocalStorageStore } from "@budibase/frontend-core"
 import { initialise } from "./initialise"
 import { authStore } from "./auth"
 import { API } from "../api"
+import { get } from "svelte/store"
 
 const initialState = {
   visible: false,
@@ -27,9 +28,15 @@ const createDevToolStore = () => {
   }
 
   const changeRole = async role => {
+    if (role === "self") {
+      role = null
+    }
+    if (role === get(store).role) {
+      return
+    }
     store.update(state => ({
       ...state,
-      role: role === "self" ? null : role,
+      role,
     }))
     API.invalidateCache()
     await authStore.actions.fetchUser()

packages/client/src/stores/index.js

@@ -11,12 +11,13 @@ export { stateStore } from "./state"
 export { themeStore } from "./theme"
 export { devToolsStore } from "./devTools"
 export { componentStore } from "./components"
-export { uploadStore } from "./uploads.js"
+export { uploadStore } from "./uploads"
-export { rowSelectionStore } from "./rowSelection.js"
+export { rowSelectionStore } from "./rowSelection"
-export { blockStore } from "./blocks.js"
+export { blockStore } from "./blocks"
 export { environmentStore } from "./environment"
-export { eventStore } from "./events.js"
+export { eventStore } from "./events"
-export { orgStore } from "./org.js"
+export { orgStore } from "./org"
+export { roleStore } from "./roles"
 export {
   dndStore,
   dndIndex,
@@ -25,7 +26,7 @@ export {
   dndIsNewComponent,
   dndIsDragging,
 } from "./dnd"
-export { sidePanelStore } from "./sidePanel.js"
+export { sidePanelStore } from "./sidePanel"
 
 // Context stores are layered and duplicated, so it is not a singleton
 export { createContextStore } from "./context"

packages/client/src/stores/roles.js

@@ -0,0 +1,24 @@
+import { API } from "api"
+import { writable } from "svelte/store"
+import { currentRole } from "./derived"
+
+const createRoleStore = () => {
+  const store = writable([])
+
+  // Fetches the user object if someone is logged in and has reloaded the page
+  const fetchAccessibleRoles = async () => {
+    const accessible = await API.getAccessibleRoles()
+    // Use the app self if present, otherwise fallback to the global self
+    store.set(accessible || [])
+    return accessible
+  }
+
+  return {
+    subscribe: store.subscribe,
+    actions: { fetchAccessibleRoles },
+  }
+}
+
+export const roleStore = createRoleStore()
+
+currentRole.subscribe(roleStore.actions.fetchAccessibleRoles)

packages/frontend-core/src/api/roles.js

@@ -38,4 +38,13 @@ export const buildRoleEndpoints = API => ({
       url: `/api/global/roles/${appId}`,
     })
   },
+
+  /**
+   * For the logged in user and current app - retrieves accessible roles.
+   */
+  getAccessibleRoles: async () => {
+    return await API.get({
+      url: `/api/roles/accessible`,
+    })
+  },
 })

packages/frontend-core/src/api/tables.js

@@ -82,8 +82,9 @@ export const buildTableEndpoints = API => ({
       },
     })
   },
+
   /**
-   * Gets a list o tables.
+   * Gets a list of tables.
    */
   getTables: async () => {
     return await API.get({
@@ -91,6 +92,15 @@ export const buildTableEndpoints = API => ({
     })
   },
 
+  /**
+   * Get a single table based on table ID.
+   */
+  getTable: async tableId => {
+    return await API.get({
+      url: `/api/tables/${tableId}`,
+    })
+  },
+
   /**
    * Saves a table.
    * @param table the table to save

packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte

@@ -3,6 +3,8 @@
   import RelationshipCell from "./RelationshipCell.svelte"
   import { FieldSubtype } from "@budibase/types"
 
+  export let api
+
   const { API } = getContext("grid")
   const { subtype } = $$props.schema
 
@@ -17,8 +19,11 @@
       throw `Search for '${subtype}' not implemented`
     }
 
+    // As we are overriding the search function from RelationshipCell, we want to map one shape to the expected one for the specific API
+    const email = Object.values(searchParams.query.string)[0]
+
     const results = await API.searchUsers({
-      ...searchParams,
+      email,
     })
 
     // Mapping to the expected data within RelationshipCell
@@ -31,6 +36,7 @@
 </script>
 
 <RelationshipCell
+  bind:api
   {...$$props}
   {schema}
   {searchFunction}

packages/sdk/package.json

@@ -10,14 +10,10 @@
     "generate": "cd scripts && bash generate-sdk.sh",
     "build:sdk": "yarn run generate && rollup -c"
   },
-  "dependencies": {
-    "superagent": "^5.3.0"
-  },
   "devDependencies": {
     "@rollup/plugin-commonjs": "^18.0.0",
     "@rollup/plugin-node-resolve": "^11.2.1",
     "rollup": "^2.44.0",
-    "rollup-plugin-polyfill-node": "^0.8.0",
     "rollup-plugin-terser": "^7.0.2"
   }
 }

packages/sdk/scripts/generate-sdk.sh

@@ -19,7 +19,7 @@ docker run --rm \
   -v ${PWD}/generated:/generated \
   -v ${PWD}/config.json:/config.json \
   -u $(id -u):$(id -g) \
-  swaggerapi/swagger-codegen-cli-v3 generate \
+  swaggerapi/swagger-codegen-cli-v3:3.0.46 generate \
   -i /openapi.yml \
   -l javascript \
   -o /generated \
@@ -34,4 +34,4 @@ if [[ -f "openapi.yaml" ]]; then
 fi
 if [[ -d "generated" ]]; then
     rm -r generated
-fi
+fi

packages/server/package.json

@@ -57,7 +57,6 @@
     "@elastic/elasticsearch": "7.10.0",
     "@google-cloud/firestore": "5.0.2",
     "@koa/router": "8.0.8",
-    "@sendgrid/mail": "7.1.1",
     "@sentry/node": "6.17.7",
     "@socket.io/redis-adapter": "^8.2.1",
     "airtable": "0.10.1",
@@ -66,18 +65,14 @@
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
-    "chmodr": "1.2.0",
     "chokidar": "3.5.3",
     "cookies": "0.8.0",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
     "dd-trace": "3.13.2",
     "dotenv": "8.2.0",
-    "download": "8.0.0",
-    "elastic-apm-node": "3.38.0",
     "fix-path": "3.0.0",
     "form-data": "4.0.0",
-    "fs-extra": "8.1.0",
     "global-agent": "3.0.0",
     "google-auth-library": "7.12.0",
     "google-spreadsheet": "3.2.0",
@@ -90,10 +85,7 @@
     "koa": "2.13.4",
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
-    "koa-connect": "2.1.0",
     "koa-send": "5.0.0",
-    "koa-session": "5.12.0",
-    "koa-static": "5.0.0",
     "koa-useragent": "^4.1.0",
     "koa2-ratelimit": "1.1.1",
     "lodash": "4.17.21",
@@ -108,7 +100,6 @@
     "pg": "8.10.0",
     "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",
-    "pouchdb-adapter-memory": "7.2.2",
     "pouchdb-all-dbs": "1.0.2",
     "pouchdb-find": "7.2.2",
     "pouchdb-replication-stream": "1.2.9",
@@ -116,8 +107,7 @@
     "server-destroy": "1.0.1",
     "snowflake-promise": "^4.5.0",
     "socket.io": "4.6.1",
-    "svelte": "3.49.0",
+    "svelte": "^3.49.0",
-    "swagger-parser": "10.0.3",
     "tar": "6.1.15",
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
@@ -130,13 +120,9 @@
   "devDependencies": {
     "@babel/core": "7.17.4",
     "@babel/preset-env": "7.16.11",
-    "@budibase/standard-components": "^0.9.139",
-    "@jest/test-sequencer": "29.6.2",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
     "@trendyol/jest-testcontainers": "2.1.1",
-    "@types/apidoc": "0.50.0",
-    "@types/bson": "4.2.0",
     "@types/global-agent": "2.1.1",
     "@types/google-spreadsheet": "3.1.5",
     "@types/jest": "29.5.3",
@@ -148,17 +134,12 @@
     "@types/node-fetch": "2.6.4",
     "@types/oracledb": "5.2.2",
     "@types/pg": "8.6.6",
-    "@types/pouchdb": "6.4.0",
-    "@types/redis": "4.0.11",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.12",
     "@types/tar": "6.1.5",
     "apidoc": "0.50.4",
-    "babel-jest": "29.6.2",
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",
-    "eslint": "6.8.0",
-    "is-wsl": "2.2.0",
     "jest": "29.6.2",
     "jest-openapi": "0.14.2",
     "jest-runner": "29.6.2",

packages/server/specs/openapi.json

@@ -1567,8 +1567,7 @@
           }
         },
         "required": [
-          "email",
+          "email"
-          "roles"
         ]
       },
       "userOutput": {
@@ -1639,7 +1638,6 @@
             },
             "required": [
               "email",
-              "roles",
               "_id"
             ]
           }
@@ -1718,7 +1716,6 @@
               },
               "required": [
                 "email",
-                "roles",
                 "_id"
               ]
             }

packages/server/specs/openapi.yaml

@@ -1337,7 +1337,6 @@ components:
               role ID, e.g. ADMIN.
       required:
         - email
-        - roles
     userOutput:
       type: object
       properties:
@@ -1398,7 +1397,6 @@ components:
               type: string
           required:
             - email
-            - roles
             - _id
       required:
         - data
@@ -1464,7 +1462,6 @@ components:
                 type: string
             required:
               - email
-              - roles
               - _id
       required:
         - data

packages/server/specs/resources/user.ts

@@ -92,7 +92,7 @@ const userSchema = object(
       },
     },
   },
-  { required: ["email", "roles"] }
+  { required: ["email"] }
 )
 
 const userOutputSchema = {

packages/server/src/api/controllers/public/mapping/users.ts

@@ -15,10 +15,15 @@ function user(body: any): User {
   }
 }
 
-function mapUser(ctx: any): { data: User } {
+function mapUser(ctx: any) {
-  return {
+  const body: { data: User; message?: string } = {
     data: user(ctx.body),
   }
+  if (ctx.extra?.message) {
+    body.message = ctx.extra.message
+    delete ctx.extra
+  }
+  return body
 }
 
 function mapUsers(ctx: any): { data: User[] } {

packages/server/src/api/controllers/public/users.ts

@@ -10,6 +10,32 @@ import { search as stringSearch } from "./utils"
 import { UserCtx, User } from "@budibase/types"
 import { Next } from "koa"
 import { sdk } from "@budibase/pro"
+import { isEqual, cloneDeep } from "lodash"
+
+function rolesRemoved(base: User, ctx: UserCtx) {
+  return (
+    !isEqual(base.builder, ctx.request.body.builder) ||
+    !isEqual(base.admin, ctx.request.body.admin) ||
+    !isEqual(base.roles, ctx.request.body.roles)
+  )
+}
+
+const NO_ROLES_MSG =
+  "Roles/admin/builder can only be set on business/enterprise licenses - input ignored."
+
+async function createUpdateResponse(ctx: UserCtx, user?: User) {
+  const base = cloneDeep(ctx.request.body)
+  ctx = await sdk.publicApi.users.roleCheck(ctx, user)
+  // check the ctx before any updates to it
+  const removed = rolesRemoved(base, ctx)
+  ctx = publicApiUserFix(ctx)
+  const response = await saveGlobalUser(ctx)
+  ctx.body = await getUser(ctx, response._id)
+  if (removed) {
+    ctx.extra = { message: NO_ROLES_MSG }
+  }
+  return ctx
+}
 
 function isLoggedInUser(ctx: UserCtx, user: User) {
   const loggedInId = ctx.user?._id
@@ -35,9 +61,7 @@ export async function search(ctx: UserCtx, next: Next) {
 }
 
 export async function create(ctx: UserCtx, next: Next) {
-  ctx = publicApiUserFix(await sdk.publicApi.users.roleCheck(ctx))
+  await createUpdateResponse(ctx)
-  const response = await saveGlobalUser(ctx)
-  ctx.body = await getUser(ctx, response._id)
   await next()
 }
 
@@ -52,9 +76,7 @@ export async function update(ctx: UserCtx, next: Next) {
     ...ctx.request.body,
     _rev: user._rev,
   }
-  ctx = publicApiUserFix(await sdk.publicApi.users.roleCheck(ctx, user))
+  await createUpdateResponse(ctx, user)
-  const response = await saveGlobalUser(ctx)
-  ctx.body = await getUser(ctx, response._id)
   await next()
 }
 

packages/server/src/api/controllers/query/index.ts

@@ -9,7 +9,7 @@ import { quotas } from "@budibase/pro"
 import { events, context, utils, constants } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import { QueryEvent } from "../../../threads/definitions"
-import { Query } from "@budibase/types"
+import { ConfigType, Query, UserCtx } from "@budibase/types"
 import { ValidQueryNameRegex } from "@budibase/shared-core"
 
 const Runner = new Thread(ThreadType.QUERY, {
@@ -28,11 +28,11 @@ function enrichQueries(input: any) {
   return wasArray ? queries : queries[0]
 }
 
-export async function fetch(ctx: any) {
+export async function fetch(ctx: UserCtx) {
   ctx.body = await sdk.queries.fetch()
 }
 
-const _import = async (ctx: any) => {
+const _import = async (ctx: UserCtx) => {
   const body = ctx.request.body
   const data = body.data
 
@@ -73,7 +73,7 @@ const _import = async (ctx: any) => {
 }
 export { _import as import }
 
-export async function save(ctx: any) {
+export async function save(ctx: UserCtx) {
   const db = context.getAppDB()
   const query = ctx.request.body
 
@@ -100,19 +100,19 @@ export async function save(ctx: any) {
   ctx.message = `Query ${query.name} saved successfully.`
 }
 
-export async function find(ctx: any) {
+export async function find(ctx: UserCtx) {
   const queryId = ctx.params.queryId
   ctx.body = await sdk.queries.find(queryId)
 }
 
 //Required to discern between OIDC OAuth config entries
-function getOAuthConfigCookieId(ctx: any) {
+function getOAuthConfigCookieId(ctx: UserCtx) {
-  if (ctx.user.providerType === constants.Config.OIDC) {
+  if (ctx.user.providerType === ConfigType.OIDC) {
     return utils.getCookie(ctx, constants.Cookie.OIDC_CONFIG)
   }
 }
 
-function getAuthConfig(ctx: any) {
+function getAuthConfig(ctx: UserCtx) {
   const authCookie = utils.getCookie(ctx, constants.Cookie.Auth)
   let authConfigCtx: any = {}
   authConfigCtx["configId"] = getOAuthConfigCookieId(ctx)
@@ -120,7 +120,7 @@ function getAuthConfig(ctx: any) {
   return authConfigCtx
 }
 
-export async function preview(ctx: any) {
+export async function preview(ctx: UserCtx) {
   const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
     ctx.request.body.datasourceId
   )
@@ -129,6 +129,19 @@ export async function preview(ctx: any) {
   // this stops dynamic variables from calling the same query
   const { fields, parameters, queryVerb, transformer, queryId, schema } = query
 
+  let existingSchema = schema
+  if (queryId && !existingSchema) {
+    try {
+      const db = context.getAppDB()
+      const existing = (await db.get(queryId)) as Query
+      existingSchema = existing.schema
+    } catch (err: any) {
+      if (err.status !== 404) {
+        ctx.throw(500, "Unable to retrieve existing query")
+      }
+    }
+  }
+
   const authConfigCtx: any = getAuthConfig(ctx)
 
   try {
@@ -180,6 +193,14 @@ export async function preview(ctx: any) {
         schemaFields[key] = fieldType
       }
     }
+    // if existing schema, update to include any previous schema keys
+    if (existingSchema) {
+      for (let key of Object.keys(schemaFields)) {
+        if (existingSchema[key]?.type) {
+          schemaFields[key] = existingSchema[key].type
+        }
+      }
+    }
     // remove configuration before sending event
     delete datasource.config
     await events.query.previewed(datasource, query)
@@ -189,13 +210,13 @@ export async function preview(ctx: any) {
       info,
       extra,
     }
-  } catch (err) {
+  } catch (err: any) {
     ctx.throw(400, err)
   }
 }
 
 async function execute(
-  ctx: any,
+  ctx: UserCtx,
   opts: any = { rowsOnly: false, isAutomation: false }
 ) {
   const db = context.getAppDB()
@@ -255,17 +276,17 @@ async function execute(
     } else {
       ctx.body = { data: rows, pagination, ...extra, ...info }
     }
-  } catch (err) {
+  } catch (err: any) {
     ctx.throw(400, err)
   }
 }
 
-export async function executeV1(ctx: any) {
+export async function executeV1(ctx: UserCtx) {
   return execute(ctx, { rowsOnly: true, isAutomation: false })
 }
 
 export async function executeV2(
-  ctx: any,
+  ctx: UserCtx,
   { isAutomation }: { isAutomation?: boolean } = {}
 ) {
   return execute(ctx, { rowsOnly: false, isAutomation })
@@ -292,7 +313,7 @@ const removeDynamicVariables = async (queryId: any) => {
   }
 }
 
-export async function destroy(ctx: any) {
+export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const queryId = ctx.params.queryId
   await removeDynamicVariables(queryId)

packages/server/src/api/controllers/role.ts

@@ -1,6 +1,7 @@
-import { roles, context, events, db as dbCore } from "@budibase/backend-core"
+import { context, db as dbCore, events, roles } from "@budibase/backend-core"
 import { getUserMetadataParams, InternalTables } from "../../db/utils"
-import { UserCtx, Database, UserRoles, Role } from "@budibase/types"
+import { Database, Role, UserCtx, UserRoles } from "@budibase/types"
+import { sdk as sharedSdk } from "@budibase/shared-core"
 import sdk from "../../sdk"
 
 const UpdateRolesOptions = {
@@ -94,7 +95,6 @@ export async function save(ctx: UserCtx) {
   )
   role._rev = result.rev
   ctx.body = role
-  ctx.message = `Role '${role.name}' created successfully.`
 }
 
 export async function destroy(ctx: UserCtx) {
@@ -131,3 +131,16 @@ export async function destroy(ctx: UserCtx) {
   ctx.message = `Role ${ctx.params.roleId} deleted successfully`
   ctx.status = 200
 }
+
+export async function accessible(ctx: UserCtx) {
+  let roleId = ctx.user?.roleId
+  if (!roleId) {
+    roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
+  }
+  if (ctx.user && sharedSdk.users.isAdminOrBuilder(ctx.user)) {
+    const appId = context.getAppId()
+    ctx.body = await roles.getAllRoleIds(appId)
+  } else {
+    ctx.body = await roles.getUserRoleIdHierarchy(roleId!)
+  }
+}

packages/server/src/api/controllers/routing.ts

@@ -63,9 +63,7 @@ export async function fetch(ctx: UserCtx) {
 export async function clientFetch(ctx: UserCtx) {
   const routing = await getRoutingStructure()
   let roleId = ctx.user?.role?._id
-  const roleIds = (await roles.getUserRoleHierarchy(roleId, {
+  const roleIds = await roles.getUserRoleIdHierarchy(roleId)
-    idOnly: true,
-  })) as string[]
   for (let topLevel of Object.values(routing.routes) as any) {
     for (let subpathKey of Object.keys(topLevel.subpaths)) {
       let found = false

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -269,13 +269,25 @@ function isEditableColumn(column: FieldSchema) {
   return !(isExternalAutoColumn || isFormula)
 }
 
-export class ExternalRequest {
+export type ExternalRequestReturnType<T> = T extends Operation.READ
-  private operation: Operation
+  ?
-  private tableId: string
+      | Row[]
+      | {
+          row: Row
+          table: Table
+        }
+  : {
+      row: Row
+      table: Table
+    }
+
+export class ExternalRequest<T extends Operation> {
+  private readonly operation: T
+  private readonly tableId: string
   private datasource?: Datasource
   private tables: { [key: string]: Table } = {}
 
-  constructor(operation: Operation, tableId: string, datasource?: Datasource) {
+  constructor(operation: T, tableId: string, datasource?: Datasource) {
     this.operation = operation
     this.tableId = tableId
     this.datasource = datasource
@@ -328,10 +340,16 @@ export class ExternalRequest {
       // one to many
       if (isOneSide(field)) {
         let id = row[key][0]
-        if (typeof row[key] === "string") {
+        if (id) {
-          id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1]
+          if (typeof row[key] === "string") {
+            id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1]
+          }
+          newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(id)[0]
+        } else {
+          // Removing from both new and row, as we don't know if it has already been processed
+          row[field.foreignKey || linkTablePrimary] = null
+          newRow[field.foreignKey || linkTablePrimary] = null
         }
-        newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(id)[0]
       }
       // many to many
       else if (field.through) {
@@ -739,7 +757,7 @@ export class ExternalRequest {
     return fields
   }
 
-  async run(config: RunConfig) {
+  async run(config: RunConfig): Promise<ExternalRequestReturnType<T>> {
     const { operation, tableId } = this
     let { datasourceId, tableName } = breakExternalTableId(tableId)
     if (!tableName) {
@@ -818,7 +836,7 @@ export class ExternalRequest {
     // can't really use response right now
     const response = await getDatasourceAndQuery(json)
     // handle many to many relationships now if we know the ID (could be auto increment)
-    if (operation !== Operation.READ && processed.manyRelationships) {
+    if (operation !== Operation.READ) {
       await this.handleManyRelationships(
         table._id || "",
         response[0],
@@ -827,8 +845,11 @@ export class ExternalRequest {
     }
     const output = this.outputProcessing(response, table, relationships)
     // if reading it'll just be an array of rows, return whole thing
-    return operation === Operation.READ && Array.isArray(response)
+    const result = (
-      ? output
+      operation === Operation.READ && Array.isArray(response)
-      : { row: output[0], table }
+        ? output
+        : { row: output[0], table }
+    ) as ExternalRequestReturnType<T>
+    return result
   }
 }

packages/server/src/api/controllers/row/external.ts

@@ -1,9 +1,13 @@
-import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
+import { FieldTypes } from "../../../constants"
 import {
   breakExternalTableId,
   breakRowIdField,
 } from "../../../integrations/utils"
-import { ExternalRequest, RunConfig } from "./ExternalRequest"
+import {
+  ExternalRequest,
+  ExternalRequestReturnType,
+  RunConfig,
+} from "./ExternalRequest"
 import {
   Datasource,
   IncludeRelationship,
@@ -18,14 +22,17 @@ import {
 import sdk from "../../../sdk"
 import * as utils from "./utils"
 import { dataFilters } from "@budibase/shared-core"
-import { inputProcessing } from "../../../utilities/rowProcessor"
+import {
+  inputProcessing,
+  outputProcessing,
+} from "../../../utilities/rowProcessor"
 import { cloneDeep, isEqual } from "lodash"
 
-export async function handleRequest(
+export async function handleRequest<T extends Operation>(
-  operation: Operation,
+  operation: T,
   tableId: string,
   opts?: RunConfig
-) {
+): Promise<ExternalRequestReturnType<T>> {
   // make sure the filters are cleaned up, no empty strings for equals, fuzzy or string
   if (opts && opts.filters) {
     opts.filters = sdk.rows.removeEmptyFilters(opts.filters)
@@ -34,7 +41,7 @@ export async function handleRequest(
     !dataFilters.hasFilters(opts?.filters) &&
     opts?.filters?.onEmptyFilter === EmptyFilterOption.RETURN_NONE
   ) {
-    return []
+    return [] as any
   }
 
   return new ExternalRequest(operation, tableId, opts?.datasource).run(
@@ -46,24 +53,34 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const tableId = utils.getTableId(ctx)
   const { _id, ...rowData } = ctx.request.body
 
+  const table = await sdk.tables.getTable(tableId)
+  const { row: dataToUpdate } = await inputProcessing(
+    ctx.user?._id,
+    cloneDeep(table),
+    rowData
+  )
+
   const validateResult = await sdk.rows.utils.validate({
-    row: rowData,
+    row: dataToUpdate,
     tableId,
   })
   if (!validateResult.valid) {
     throw { validation: validateResult.errors }
   }
+
   const response = await handleRequest(Operation.UPDATE, tableId, {
     id: breakRowIdField(_id),
-    row: rowData,
+    row: dataToUpdate,
   })
   const row = await sdk.rows.external.getRow(tableId, _id, {
     relationships: true,
   })
-  const table = await sdk.tables.getTable(tableId)
+  const enrichedRow = await outputProcessing(table, row, {
+    preserveLinks: true,
+  })
   return {
     ...response,
-    row,
+    row: enrichedRow,
     table,
   }
 }
@@ -71,13 +88,6 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
 export async function save(ctx: UserCtx) {
   const inputs = ctx.request.body
   const tableId = utils.getTableId(ctx)
-  const validateResult = await sdk.rows.utils.validate({
-    row: inputs,
-    tableId,
-  })
-  if (!validateResult.valid) {
-    throw { validation: validateResult.errors }
-  }
 
   const table = await sdk.tables.getTable(tableId)
   const { table: updatedTable, row } = await inputProcessing(
@@ -86,24 +96,30 @@ export async function save(ctx: UserCtx) {
     inputs
   )
 
+  const validateResult = await sdk.rows.utils.validate({
+    row,
+    tableId,
+  })
+  if (!validateResult.valid) {
+    throw { validation: validateResult.errors }
+  }
+
   const response = await handleRequest(Operation.CREATE, tableId, {
     row,
   })
 
-  const responseRow = response as { row: Row }
-
   if (!isEqual(table, updatedTable)) {
     await sdk.tables.saveTable(updatedTable)
   }
 
-  const rowId = responseRow.row._id
+  const rowId = response.row._id
   if (rowId) {
     const row = await sdk.rows.external.getRow(tableId, rowId, {
       relationships: true,
     })
     return {
       ...response,
-      row,
+      row: await outputProcessing(table, row, { preserveLinks: true }),
     }
   } else {
     return response
@@ -121,7 +137,12 @@ export async function find(ctx: UserCtx): Promise<Row> {
     ctx.throw(404)
   }
 
-  return row
+  const table = await sdk.tables.getTable(tableId)
+  // Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case
+  return await outputProcessing(table, row, {
+    squash: false,
+    preserveLinks: true,
+  })
 }
 
 export async function destroy(ctx: UserCtx) {

packages/server/src/api/controllers/static/index.ts

@@ -107,6 +107,11 @@ export const serveApp = async function (ctx: any) {
   //Public Settings
   const { config } = await configs.getSettingsConfigDoc()
   const branding = await pro.branding.getBrandingConfig(config)
+  // incase running direct from TS
+  let appHbsPath = join(__dirname, "app.hbs")
+  if (!fs.existsSync(appHbsPath)) {
+    appHbsPath = join(__dirname, "templates", "app.hbs")
+  }
 
   let db
   try {
@@ -138,7 +143,7 @@ export const serveApp = async function (ctx: any) {
             ? objectStore.getGlobalFileUrl("settings", "logoUrl")
             : "",
       })
-      const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
+      const appHbs = loadHandlebarsFile(appHbsPath)
       ctx.body = await processString(appHbs, {
         head,
         body: html,
@@ -166,7 +171,7 @@ export const serveApp = async function (ctx: any) {
             : "",
       })
 
-      const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
+      const appHbs = loadHandlebarsFile(appHbsPath)
       ctx.body = await processString(appHbs, {
         head,
         body: html,
@@ -193,8 +198,13 @@ export const serveBuilderPreview = async function (ctx: any) {
 }
 
 export const serveClientLibrary = async function (ctx: any) {
+  let rootPath = join(NODE_MODULES_PATH, "@budibase", "client", "dist")
+  // incase running from TS directly
+  if (env.isDev() && !fs.existsSync(rootPath)) {
+    rootPath = join(require.resolve("@budibase/client"), "..")
+  }
   return send(ctx, "budibase-client.js", {
-    root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
+    root: rootPath,
   })
 }
 

packages/server/src/api/controllers/table/index.ts

@@ -18,6 +18,7 @@ import {
 import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"
 import { builderSocket } from "../../../websockets"
+import { cloneDeep } from "lodash"
 
 function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
   if (table && !tableId) {
@@ -35,16 +36,16 @@ function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
 export async function fetch(ctx: UserCtx<void, FetchTablesResponse>) {
   const internal = await sdk.tables.getAllInternalTables()
 
-  const externalTables = await sdk.datasources.getExternalDatasources()
+  const datasources = await sdk.datasources.getExternalDatasources()
 
-  const external = externalTables.flatMap(table => {
+  const external = datasources.flatMap(datasource => {
-    let entities = table.entities
+    let entities = datasource.entities
     if (entities) {
       return Object.values(entities).map<Table>((entity: Table) => ({
         ...entity,
         type: "external",
-        sourceId: table._id,
+        sourceId: datasource._id,
-        sql: isSQL(table),
+        sql: isSQL(datasource),
       }))
     } else {
       return []
@@ -80,7 +81,7 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   ctx.eventEmitter &&
     ctx.eventEmitter.emitTable(`table:save`, appId, { ...savedTable })
   ctx.body = savedTable
-  builderSocket?.emitTableUpdate(ctx, { ...savedTable })
+  builderSocket?.emitTableUpdate(ctx, cloneDeep(savedTable))
 }
 
 export async function destroy(ctx: UserCtx) {

packages/server/src/api/routes/public/tests/users.spec.ts

@@ -68,6 +68,7 @@ describe("no user role update in free", () => {
     })
     expect(res.status).toBe(200)
     expect(res.body.data.roles["app_a"]).toBeUndefined()
+    expect(res.body.message).toBeDefined()
   })
 
   it("should not allow 'admin' to be updated", async () => {
@@ -77,6 +78,7 @@ describe("no user role update in free", () => {
     })
     expect(res.status).toBe(200)
     expect(res.body.data.admin).toBeUndefined()
+    expect(res.body.message).toBeDefined()
   })
 
   it("should not allow 'builder' to be updated", async () => {
@@ -86,6 +88,7 @@ describe("no user role update in free", () => {
     })
     expect(res.status).toBe(200)
     expect(res.body.data.builder).toBeUndefined()
+    expect(res.body.message).toBeDefined()
   })
 })
 
@@ -102,6 +105,7 @@ describe("no user role update in business", () => {
     })
     expect(res.status).toBe(200)
     expect(res.body.data.roles["app_a"]).toBe("BASIC")
+    expect(res.body.message).toBeUndefined()
   })
 
   it("should allow 'admin' to be updated", async () => {
@@ -112,6 +116,7 @@ describe("no user role update in business", () => {
     })
     expect(res.status).toBe(200)
     expect(res.body.data.admin.global).toBe(true)
+    expect(res.body.message).toBeUndefined()
   })
 
   it("should allow 'builder' to be updated", async () => {
@@ -122,5 +127,6 @@ describe("no user role update in business", () => {
     })
     expect(res.status).toBe(200)
     expect(res.body.data.builder.global).toBe(true)
+    expect(res.body.message).toBeUndefined()
   })
 })

packages/server/src/api/routes/role.ts

@@ -7,6 +7,9 @@ import { roleValidator } from "./utils/validators"
 const router: Router = new Router()
 
 router
+  // retrieve a list of the roles a user can access
+  // needs to be public for public screens
+  .get("/api/roles/accessible", controller.accessible)
   .post(
     "/api/roles",
     authorized(permissions.BUILDER),

packages/server/src/api/routes/tests/role.spec.js

@@ -15,7 +15,7 @@ describe("/roles", () => {
     await config.init()
   })
 
-  const createRole = async (role) => {
+  const createRole = async role => {
     if (!role) {
       role = basicRole()
     }
@@ -33,9 +33,6 @@ describe("/roles", () => {
       const role = basicRole()
       const res = await createRole(role)
 
-      expect(res.res.statusMessage).toEqual(
-        `Role '${role.name}' created successfully.`
-      )
       expect(res.body._id).toBeDefined()
       expect(res.body._rev).toBeDefined()
       expect(events.role.updated).not.toBeCalled()
@@ -51,9 +48,6 @@ describe("/roles", () => {
       jest.clearAllMocks()
       res = await createRole(res.body)
 
-      expect(res.res.statusMessage).toEqual(
-        `Role '${role.name}' created successfully.`
-      )
       expect(res.body._id).toBeDefined()
       expect(res.body._rev).toBeDefined()
       expect(events.role.created).not.toBeCalled()
@@ -99,7 +93,11 @@ describe("/roles", () => {
 
     it("should be able to get the role with a permission added", async () => {
       const table = await config.createTable()
-      await config.api.permission.set({ roleId: BUILTIN_ROLE_IDS.POWER, resourceId: table._id, level: PermissionLevel.READ })
+      await config.api.permission.set({
+        roleId: BUILTIN_ROLE_IDS.POWER,
+        resourceId: table._id,
+        level: PermissionLevel.READ,
+      })
       const res = await request
         .get(`/api/roles`)
         .set(config.defaultHeaders())
@@ -131,4 +129,34 @@ describe("/roles", () => {
       expect(events.role.deleted).toBeCalledWith(customRole)
     })
   })
+
+  describe("accessible", () => {
+    it("should be able to fetch accessible roles (with builder)", async () => {
+      const res = await request
+        .get("/api/roles/accessible")
+        .set(config.defaultHeaders())
+        .expect(200)
+      expect(res.body.length).toBe(5)
+      expect(typeof res.body[0]).toBe("string")
+    })
+
+    it("should be able to fetch accessible roles (basic user)", async () => {
+      const res = await request
+        .get("/api/roles/accessible")
+        .set(await config.basicRoleHeaders())
+        .expect(200)
+      expect(res.body.length).toBe(2)
+      expect(res.body[0]).toBe("BASIC")
+      expect(res.body[1]).toBe("PUBLIC")
+    })
+
+    it("should be able to fetch accessible roles (no user)", async () => {
+      const res = await request
+        .get("/api/roles/accessible")
+        .set(config.publicHeaders())
+        .expect(200)
+      expect(res.body.length).toBe(1)
+      expect(res.body[0]).toBe("PUBLIC")
+    })
+  })
 })

packages/server/src/api/routes/tests/row.spec.ts

@@ -7,6 +7,7 @@ import { context, InternalTable, roles, tenancy } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
   FieldType,
+  FieldTypeSubtypes,
   MonthlyQuotaName,
   PermissionLevel,
   QuotaUsageType,
@@ -25,6 +26,7 @@ import {
   mocks,
   structures,
 } from "@budibase/backend-core/tests"
+import _ from "lodash"
 
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
@@ -34,7 +36,7 @@ const { basicRow } = setup.structures
 describe.each([
   ["internal", undefined],
   ["postgres", databaseTestProviders.postgres],
-])("/rows (%s)", (_, dsProvider) => {
+])("/rows (%s)", (__, dsProvider) => {
   const isInternal = !dsProvider
 
   const request = setup.getRequest()
@@ -1511,4 +1513,413 @@ describe.each([
       })
     })
   })
+
+  let o2mTable: Table
+  let m2mTable: Table
+  beforeAll(async () => {
+    o2mTable = await config.createTable(
+      { ...generateTableConfig(), name: "o2m" },
+      {
+        skipReassigning: true,
+      }
+    )
+    m2mTable = await config.createTable(
+      { ...generateTableConfig(), name: "m2m" },
+      {
+        skipReassigning: true,
+      }
+    )
+  })
+
+  describe.each([
+    [
+      "relationship fields",
+      () => ({
+        user: {
+          name: "user",
+          relationshipType: RelationshipType.ONE_TO_MANY,
+          type: FieldType.LINK,
+          tableId: o2mTable._id!,
+          fieldName: "fk_o2m",
+        },
+        users: {
+          name: "users",
+          relationshipType: RelationshipType.MANY_TO_MANY,
+          type: FieldType.LINK,
+          tableId: m2mTable._id!,
+          fieldName: "fk_m2m",
+        },
+      }),
+      (tableId: string) =>
+        config.api.row.save(tableId, {
+          name: generator.word(),
+          description: generator.paragraph(),
+          tableId,
+        }),
+      (row: Row) => ({
+        _id: row._id,
+        primaryDisplay: row.name,
+      }),
+    ],
+    [
+      "bb reference fields",
+      () => ({
+        user: {
+          name: "user",
+          relationshipType: RelationshipType.ONE_TO_MANY,
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+        },
+        users: {
+          name: "users",
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+          relationshipType: RelationshipType.MANY_TO_MANY,
+        },
+      }),
+      () => config.createUser(),
+      (row: Row) => ({
+        _id: row._id,
+        email: row.email,
+        firstName: row.firstName,
+        lastName: row.lastName,
+        primaryDisplay: row.email,
+      }),
+    ],
+  ])("links - %s", (__, relSchema, dataGenerator, resultMapper) => {
+    let tableId: string
+    let o2mData: Row[]
+    let m2mData: Row[]
+
+    beforeAll(async () => {
+      const tableConfig = generateTableConfig()
+
+      if (config.datasource) {
+        tableConfig.sourceId = config.datasource._id
+        if (config.datasource.plus) {
+          tableConfig.type = "external"
+        }
+      }
+      const table = await config.api.table.create({
+        ...tableConfig,
+        schema: {
+          ...tableConfig.schema,
+          ...relSchema(),
+        },
+      })
+      tableId = table._id!
+
+      o2mData = [
+        await dataGenerator(o2mTable._id!),
+        await dataGenerator(o2mTable._id!),
+        await dataGenerator(o2mTable._id!),
+        await dataGenerator(o2mTable._id!),
+      ]
+
+      m2mData = [
+        await dataGenerator(m2mTable._id!),
+        await dataGenerator(m2mTable._id!),
+        await dataGenerator(m2mTable._id!),
+        await dataGenerator(m2mTable._id!),
+      ]
+    })
+
+    it("can save a row when relationship fields are empty", async () => {
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      expect(row).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        _id: expect.any(String),
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+      })
+    })
+
+    it("can save a row with a single relationship field", async () => {
+      const user = _.sample(o2mData)!
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        user: [user],
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      expect(row).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: [user].map(u => resultMapper(u)),
+        _id: expect.any(String),
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
+      })
+    })
+
+    it("can save a row with a multiple relationship field", async () => {
+      const selectedUsers = _.sampleSize(m2mData, 2)
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        users: selectedUsers,
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      expect(row).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        users: expect.arrayContaining(selectedUsers.map(u => resultMapper(u))),
+        _id: expect.any(String),
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+      })
+    })
+
+    it("can retrieve rows with no populated relationships", async () => {
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      const { body: retrieved } = await config.api.row.get(tableId, row._id!)
+      expect(retrieved).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: undefined,
+        users: undefined,
+        _id: row._id,
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        ...defaultRowFields,
+      })
+    })
+
+    it("can retrieve rows with populated relationships", async () => {
+      const user1 = _.sample(o2mData)!
+      const [user2, user3] = _.sampleSize(m2mData, 2)
+
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        users: [user2, user3],
+        user: [user1],
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      const { body: retrieved } = await config.api.row.get(tableId, row._id!)
+      expect(retrieved).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: expect.arrayContaining([user1].map(u => resultMapper(u))),
+        users: expect.arrayContaining([user2, user3].map(u => resultMapper(u))),
+        _id: row._id,
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user1.id,
+        ...defaultRowFields,
+      })
+    })
+
+    it("can update an existing populated row", async () => {
+      const user = _.sample(o2mData)!
+      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
+
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        users: [users1, users2],
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      const updatedRow = await config.api.row.save(tableId, {
+        ...row,
+        user: [user],
+        users: [users3, users1],
+      })
+      expect(updatedRow).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: expect.arrayContaining([user].map(u => resultMapper(u))),
+        users: expect.arrayContaining(
+          [users3, users1].map(u => resultMapper(u))
+        ),
+        _id: row._id,
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
+      })
+    })
+
+    it("can wipe an existing populated relationships in row", async () => {
+      const [user1, user2] = _.sampleSize(m2mData, 2)
+
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        users: [user1, user2],
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      const updatedRow = await config.api.row.save(tableId, {
+        ...row,
+        user: null,
+        users: null,
+      })
+      expect(updatedRow).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        _id: row._id,
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+      })
+    })
+
+    it("fetch all will populate the relationships", async () => {
+      const [user1] = _.sampleSize(o2mData, 1)
+      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
+
+      const rows: {
+        name: string
+        description: string
+        user?: Row[]
+        users?: Row[]
+        tableId: string
+      }[] = [
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [users1, users2],
+        },
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          user: [user1],
+          users: [users1, users3],
+        },
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [users3],
+        },
+      ]
+
+      await config.api.row.save(tableId, rows[0])
+      await config.api.row.save(tableId, rows[1])
+      await config.api.row.save(tableId, rows[2])
+
+      const res = await config.api.row.fetch(tableId)
+
+      expect(res).toEqual(
+        expect.arrayContaining(
+          rows.map(r => ({
+            name: r.name,
+            description: r.description,
+            tableId,
+            user: r.user?.map(u => resultMapper(u)),
+            users: r.users?.length
+              ? expect.arrayContaining(r.users?.map(u => resultMapper(u)))
+              : undefined,
+            _id: expect.any(String),
+            _rev: expect.any(String),
+            id: isInternal ? undefined : expect.any(Number),
+            [`fk_${o2mTable.name}_fk_o2m`]:
+              isInternal || !r.user?.length ? undefined : r.user[0].id,
+            ...defaultRowFields,
+          }))
+        )
+      )
+    })
+
+    it("search all will populate the relationships", async () => {
+      const [user1] = _.sampleSize(o2mData, 1)
+      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
+
+      const rows: {
+        name: string
+        description: string
+        user?: Row[]
+        users?: Row[]
+        tableId: string
+      }[] = [
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [users1, users2],
+        },
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          user: [user1],
+          users: [users1, users3],
+        },
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [users3],
+        },
+      ]
+
+      await config.api.row.save(tableId, rows[0])
+      await config.api.row.save(tableId, rows[1])
+      await config.api.row.save(tableId, rows[2])
+
+      const res = await config.api.row.search(tableId)
+
+      expect(res).toEqual({
+        rows: expect.arrayContaining(
+          rows.map(r => ({
+            name: r.name,
+            description: r.description,
+            tableId,
+            user: r.user?.map(u => resultMapper(u)),
+            users: r.users?.length
+              ? expect.arrayContaining(r.users?.map(u => resultMapper(u)))
+              : undefined,
+            _id: expect.any(String),
+            _rev: expect.any(String),
+            id: isInternal ? undefined : expect.any(Number),
+            [`fk_${o2mTable.name}_fk_o2m`]:
+              isInternal || !r.user?.length ? undefined : r.user[0].id,
+            ...defaultRowFields,
+          }))
+        ),
+        ...(isInternal
+          ? {}
+          : {
+              hasNextPage: false,
+              bookmark: null,
+            }),
+      })
+    })
+  })
 })

packages/server/src/automations/steps/make.ts

@@ -78,8 +78,7 @@ export const definition: AutomationStepSchema = {
 }
 
 export async function run({ inputs }: AutomationStepInput) {
-  //TODO - Remove deprecated values 1,2,3,4,5 after November 2023
+  const { url, body } = inputs
-  const { url, value1, value2, value3, value4, value5, body } = inputs
 
   let payload = {}
   try {
@@ -104,11 +103,6 @@ export async function run({ inputs }: AutomationStepInput) {
     response = await fetch(url, {
       method: "post",
       body: JSON.stringify({
-        value1,
-        value2,
-        value3,
-        value4,
-        value5,
         ...payload,
       }),
       headers: {

packages/server/src/automations/steps/zapier.ts

@@ -71,8 +71,7 @@ export const definition: AutomationStepSchema = {
 }
 
 export async function run({ inputs }: AutomationStepInput) {
-  //TODO - Remove deprecated values 1,2,3,4,5 after November 2023
+  const { url, body } = inputs
-  const { url, value1, value2, value3, value4, value5, body } = inputs
 
   let payload = {}
   try {
@@ -100,11 +99,6 @@ export async function run({ inputs }: AutomationStepInput) {
       method: "post",
       body: JSON.stringify({
         platform: "budibase",
-        value1,
-        value2,
-        value3,
-        value4,
-        value5,
         ...payload,
       }),
       headers: {

packages/server/src/definitions/openapi.ts

@@ -613,7 +613,7 @@ export interface components {
         global?: boolean;
       };
       /** @description Contains the roles of the user per app (assuming they are not a builder user). This field can only be set on a business or enterprise license. */
-      roles: { [key: string]: string };
+      roles?: { [key: string]: string };
     };
     userOutput: {
       data: {
@@ -643,7 +643,7 @@ export interface components {
           global?: boolean;
         };
         /** @description Contains the roles of the user per app (assuming they are not a builder user). This field can only be set on a business or enterprise license. */
-        roles: { [key: string]: string };
+        roles?: { [key: string]: string };
         /** @description The ID of the user. */
         _id: string;
       };
@@ -676,7 +676,7 @@ export interface components {
           global?: boolean;
         };
         /** @description Contains the roles of the user per app (assuming they are not a builder user). This field can only be set on a business or enterprise license. */
-        roles: { [key: string]: string };
+        roles?: { [key: string]: string };
         /** @description The ID of the user. */
         _id: string;
       }[];

packages/server/src/integrations/googlesheets.ts

@@ -1,6 +1,5 @@
 import {
   ConnectionInfo,
-  Datasource,
   DatasourceFeature,
   DatasourceFieldType,
   DatasourcePlus,
@@ -23,7 +22,6 @@ import fetch from "node-fetch"
 import { cache, configs, context, HTTPError } from "@budibase/backend-core"
 import { dataFilters, utils } from "@budibase/shared-core"
 import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
-import sdk from "../sdk"
 
 interface GoogleSheetsConfig {
   spreadsheetId: string
@@ -56,6 +54,7 @@ const ALLOWED_TYPES = [
   FieldType.OPTIONS,
   FieldType.BOOLEAN,
   FieldType.BARCODEQR,
+  FieldType.BB_REFERENCE,
 ]
 
 const SCHEMA: Integration = {
@@ -213,7 +212,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
       await setupCreationAuth(this.config)
 
       // Initialise oAuth client
-      let googleConfig = await configs.getGoogleDatasourceConfig()
+      const googleConfig = await configs.getGoogleDatasourceConfig()
       if (!googleConfig) {
         throw new HTTPError("Google config not found", 400)
       }
@@ -323,14 +322,14 @@ class GoogleSheetsIntegration implements DatasourcePlus {
       case Operation.UPDATE:
         return this.update({
           // exclude the header row and zero index
-          rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
+          rowIndex: json.extra?.idFilter?.equal?.rowNumber,
           sheet,
           row: json.body,
         })
       case Operation.DELETE:
         return this.delete({
           // exclude the header row and zero index
-          rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
+          rowIndex: json.extra?.idFilter?.equal?.rowNumber,
           sheet,
         })
       case Operation.CREATE_TABLE:
@@ -541,17 +540,30 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     }
   }
 
+  private async getRowByIndex(sheetTitle: string, rowIndex: number) {
+    const sheet = this.client.sheetsByTitle[sheetTitle]
+    const rows = await sheet.getRows()
+    // We substract 2, as the SDK is skipping the header automatically and Google Spreadsheets is base 1
+    const row = rows[rowIndex - 2]
+    return { sheet, row }
+  }
+
   async update(query: { sheet: string; rowIndex: number; row: any }) {
     try {
       await this.connect()
-      const sheet = this.client.sheetsByTitle[query.sheet]
+      const { sheet, row } = await this.getRowByIndex(
-      const rows = await sheet.getRows()
+        query.sheet,
-      const row = rows[query.rowIndex]
+        query.rowIndex
+      )
       if (row) {
         const updateValues =
           typeof query.row === "string" ? JSON.parse(query.row) : query.row
         for (let key in updateValues) {
           row[key] = updateValues[key]
+
+          if (row[key] === null) {
+            row[key] = ""
+          }
         }
         await row.save()
         return [
@@ -568,9 +580,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async delete(query: { sheet: string; rowIndex: number }) {
     await this.connect()
-    const sheet = this.client.sheetsByTitle[query.sheet]
+    const { row } = await this.getRowByIndex(query.sheet, query.rowIndex)
-    const rows = await sheet.getRows()
-    const row = rows[query.rowIndex]
     if (row) {
       await row.delete()
       return [

packages/server/src/integrations/utils.ts

@@ -1,4 +1,4 @@
-import { SqlQuery, Table, SearchFilters } from "@budibase/types"
+import { SqlQuery, Table, SearchFilters, Datasource } from "@budibase/types"
 import { DocumentType, SEPARATOR } from "../db/utils"
 import {
   FieldTypes,
@@ -184,7 +184,9 @@ export function getSqlQuery(query: SqlQuery | string): SqlQuery {
   }
 }
 
-export const isSQL = helpers.isSQL
+export function isSQL(datasource: Datasource) {
+  return helpers.isSQL(datasource)
+}
 
 export function isIsoDateString(str: string) {
   const trimmedValue = str.trim()

packages/server/src/middleware/authorized.ts

@@ -55,9 +55,7 @@ const checkAuthorizedResource = async (
 ) => {
   // get the user's roles
   const roleId = ctx.roleId || roles.BUILTIN_ROLE_IDS.PUBLIC
-  const userRoles = (await roles.getUserRoleHierarchy(roleId, {
+  const userRoles = await roles.getUserRoleHierarchy(roleId)
-    idOnly: false,
-  })) as Role[]
   const permError = "User does not have permission"
   // check if the user has the required role
   if (resourceRoles.length > 0) {

packages/server/src/sdk/app/rows/search/external.ts

@@ -17,6 +17,7 @@ import { utils } from "@budibase/shared-core"
 import { ExportRowsParams, ExportRowsResult } from "../search"
 import { HTTPError, db } from "@budibase/backend-core"
 import pick from "lodash/pick"
+import { outputProcessing } from "../../../../utilities/rowProcessor"
 
 export async function search(options: SearchParams) {
   const { tableId } = options
@@ -75,6 +76,9 @@ export async function search(options: SearchParams) {
       rows = rows.map((r: any) => pick(r, fields))
     }
 
+    const table = await sdk.tables.getTable(tableId)
+    rows = await outputProcessing(table, rows, { preserveLinks: true })
+
     // need wrapper object for bookmarks etc when paginating
     return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 }
   } catch (err: any) {
@@ -166,9 +170,11 @@ export async function exportRows(
 }
 
 export async function fetch(tableId: string) {
-  return handleRequest(Operation.READ, tableId, {
+  const response = await handleRequest(Operation.READ, tableId, {
     includeSqlRelationships: IncludeRelationship.INCLUDE,
   })
+  const table = await sdk.tables.getTable(tableId)
+  return await outputProcessing(table, response, { preserveLinks: true })
 }
 
 export async function fetchView(viewName: string) {

packages/server/src/sdk/tests/rows/row.spec.ts

@@ -7,9 +7,14 @@ import { HTTPError } from "@budibase/backend-core"
 import { Operation } from "@budibase/types"
 
 const mockDatasourcesGet = jest.fn()
+const mockTableGet = jest.fn()
 sdk.datasources.get = mockDatasourcesGet
+sdk.tables.getTable = mockTableGet
 
 jest.mock("../../../api/controllers/row/ExternalRequest")
+jest.mock("../../../utilities/rowProcessor", () => ({
+  outputProcessing: jest.fn((_, rows) => rows),
+}))
 
 jest.mock("../../../api/controllers/view/exporters", () => ({
   ...jest.requireActual("../../../api/controllers/view/exporters"),

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -425,6 +425,15 @@ class TestConfiguration {
     return headers
   }
 
+  async basicRoleHeaders() {
+    return await this.roleHeaders({
+      email: this.defaultUserValues.email,
+      builder: false,
+      prodApp: true,
+      roleId: roles.BUILTIN_ROLE_IDS.BASIC,
+    })
+  }
+
   async roleHeaders({
     email = this.defaultUserValues.email,
     roleId = roles.BUILTIN_ROLE_IDS.ADMIN,

packages/server/src/tests/utilities/api/row.ts

@@ -44,12 +44,12 @@ export class RowAPI extends TestAPI {
   }
 
   save = async (
-    sourceId: string,
+    tableId: string,
     row: SaveRowRequest,
     { expectStatus } = { expectStatus: 200 }
   ): Promise<Row> => {
     const resp = await this.request
-      .post(`/api/${sourceId}/rows`)
+      .post(`/api/${tableId}/rows`)
       .send(row)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
@@ -122,4 +122,16 @@ export class RowAPI extends TestAPI {
       .expect(expectStatus)
     return request
   }
+
+  search = async (
+    sourceId: string,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Row[]> => {
+    const request = this.request
+      .post(`/api/${sourceId}/search`)
+      .set(this.config.defaultHeaders())
+      .expect(expectStatus)
+
+    return (await request).body
+  }
 }

packages/server/src/threads/definitions.ts

@@ -11,12 +11,7 @@ export interface QueryEvent {
   queryId: string
   environmentVariables?: Record<string, string>
   ctx?: any
-  schema?: {
+  schema?: Record<string, { name?: string; type: string }>
-    [key: string]: {
-      name: string
-      type: string
-    }
-  }
 }
 
 export interface QueryVariable {

packages/server/src/utilities/centralPath.ts

@@ -8,7 +8,7 @@ import path from "path"
  * @param args Any number of string arguments to add to a path
  * @returns {string} The final path ready to use
  */
-export function join(...args: any) {
+export function join(...args: string[]) {
   return path.join(...args)
 }
 
@@ -17,6 +17,6 @@ export function join(...args: any) {
  * @param args Any number of string arguments to add to a path
  * @returns {string} The final path ready to use
  */
-export function resolve(...args: any) {
+export function resolve(...args: string[]) {
   return path.resolve(...args)
 }

packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts

@@ -6,7 +6,7 @@ import { InvalidBBRefError } from "./errors"
 export async function processInputBBReferences(
   value: string | string[] | { _id: string } | { _id: string }[],
   subtype: FieldSubtype
-): Promise<string | undefined> {
+): Promise<string | null> {
   const referenceIds: string[] = []
 
   if (Array.isArray(value)) {
@@ -39,7 +39,7 @@ export async function processInputBBReferences(
       throw utils.unreachable(subtype)
   }
 
-  return referenceIds.join(",") || undefined
+  return referenceIds.join(",") || null
 }
 
 export async function processOutputBBReferences(
@@ -48,7 +48,7 @@ export async function processOutputBBReferences(
 ) {
   if (typeof value !== "string") {
     // Already processed or nothing to process
-    return value
+    return value || undefined
   }
 
   const ids = value.split(",").filter(id => !!id)

packages/server/src/utilities/rowProcessor/index.ts

@@ -11,6 +11,7 @@ import {
   processInputBBReferences,
   processOutputBBReferences,
 } from "./bbReferenceProcessor"
+import { isExternalTable } from "../../integrations/utils"
 export * from "./utils"
 
 type AutoColumnProcessingOpts = {
@@ -200,7 +201,10 @@ export async function inputProcessing(
 export async function outputProcessing<T extends Row[] | Row>(
   table: Table,
   rows: T,
-  opts = { squash: true }
+  opts: { squash?: boolean; preserveLinks?: boolean } = {
+    squash: true,
+    preserveLinks: false,
+  }
 ): Promise<T> {
   let safeRows: Row[]
   let wasArray = true
@@ -211,7 +215,9 @@ export async function outputProcessing<T extends Row[] | Row>(
     safeRows = rows
   }
   // attach any linked row information
-  let enriched = await linkRows.attachFullLinkedDocs(table, safeRows)
+  let enriched = !opts.preserveLinks
+    ? await linkRows.attachFullLinkedDocs(table, safeRows)
+    : safeRows
 
   // process formulas
   enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]
@@ -229,9 +235,6 @@ export async function outputProcessing<T extends Row[] | Row>(
       }
     } else if (column.type == FieldTypes.BB_REFERENCE) {
       for (let row of enriched) {
-        if (row[property] == null) {
-          continue
-        }
         row[property] = await processOutputBBReferences(
           row[property],
           column.subtype as FieldSubtype
@@ -245,6 +248,16 @@ export async function outputProcessing<T extends Row[] | Row>(
       enriched
     )) as Row[]
   }
+  // remove null properties to match internal API
+  if (isExternalTable(table._id!)) {
+    for (let row of enriched) {
+      for (let key of Object.keys(row)) {
+        if (row[key] === null) {
+          delete row[key]
+        }
+      }
+    }
+  }
   return (wasArray ? enriched : enriched[0]) as T
 }
 

packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts

@@ -139,20 +139,20 @@ describe("bbReferenceProcessor", () => {
         expect(cacheGetUsersSpy).toBeCalledWith(userIds)
       })
 
-      it("empty strings will return undefined", async () => {
+      it("empty strings will return null", async () => {
         const result = await config.doInTenant(() =>
           processInputBBReferences("", FieldSubtype.USER)
         )
 
-        expect(result).toEqual(undefined)
+        expect(result).toEqual(null)
       })
 
-      it("empty arrays will return undefined", async () => {
+      it("empty arrays will return null", async () => {
         const result = await config.doInTenant(() =>
           processInputBBReferences([], FieldSubtype.USER)
         )
 
-        expect(result).toEqual(undefined)
+        expect(result).toEqual(null)
       })
     })
   })

packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts

@@ -66,7 +66,7 @@ describe("rowProcessor - outputProcessing", () => {
     )
   })
 
-  it("does not fetch bb references when fields are empty", async () => {
+  it("process output even when the field is not empty", async () => {
     const table: Table = {
       _id: generator.guid(),
       name: "TestTable",
@@ -100,7 +100,7 @@ describe("rowProcessor - outputProcessing", () => {
 
     expect(result).toEqual({ name: "Jack" })
 
-    expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled()
+    expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledTimes(1)
   })
 
   it("does not fetch bb references when not in the schema", async () => {

packages/server/src/utilities/users.ts

@@ -36,5 +36,8 @@ export function publicApiUserFix(ctx: UserCtx) {
   if (!ctx.request.body._id && ctx.params.userId) {
     ctx.request.body._id = ctx.params.userId
   }
+  if (!ctx.request.body.roles) {
+    ctx.request.body.roles = {}
+  }
   return ctx
 }

packages/shared-core/package.json

@@ -17,7 +17,6 @@
     "@budibase/types": "0.0.0"
   },
   "devDependencies": {
-    "concurrently": "^7.6.0",
     "rimraf": "3.0.2",
     "typescript": "5.2.2"
   },

packages/string-templates/package.json

@@ -28,7 +28,6 @@
     "@budibase/handlebars-helpers": "^0.11.9",
     "dayjs": "^1.10.8",
     "handlebars": "^4.7.6",
-    "handlebars-utils": "^1.0.6",
     "lodash": "^4.17.20",
     "vm2": "^3.9.15"
   },
@@ -37,7 +36,6 @@
     "@rollup/plugin-json": "^4.1.0",
     "doctrine": "^3.0.0",
     "jest": "29.6.2",
-    "jest-environment-node": "29.6.2",
     "marked": "^4.0.10",
     "rollup": "^2.36.2",
     "rollup-plugin-inject-process-env": "^1.3.1",

packages/types/package.json

@@ -16,13 +16,10 @@
   "jest": {},
   "devDependencies": {
     "@budibase/nano": "10.1.2",
-    "@types/json5": "2.2.0",
     "@types/koa": "2.13.4",
     "@types/node": "18.17.0",
     "@types/pouchdb": "6.4.0",
     "@types/redlock": "4.0.3",
-    "concurrently": "^7.6.0",
-    "koa-body": "4.2.0",
     "rimraf": "3.0.2",
     "typescript": "5.2.2"
   },

packages/types/src/api/account/accounts.ts

@@ -1,4 +1,4 @@
-import { Account } from "../../documents"
+import { Account, AccountSSOProvider } from "../../documents"
 import { Hosting } from "../../sdk"
 
 export interface CreateAccountRequest {
@@ -11,6 +11,8 @@ export interface CreateAccountRequest {
   tenantName?: string
   name?: string
   password: string
+  provider?: AccountSSOProvider
+  thirdPartyProfile: object
 }
 
 export interface SearchAccountsRequest {

packages/types/src/api/web/user.ts

@@ -61,6 +61,7 @@ export interface CreateAdminUserRequest {
   email: string
   password: string
   tenantId: string
+  ssoId?: string
 }
 
 export interface CreateAdminUserResponse {

packages/types/src/documents/account/account.ts

@@ -20,6 +20,11 @@ export interface CreatePassswordAccount extends CreateAccount {
   password: string
 }
 
+export interface CreateVerifiableSSOAccount extends CreateAccount {
+  provider?: AccountSSOProvider
+  thirdPartyProfile?: any
+}
+
 export const isCreatePasswordAccount = (
   account: CreateAccount
 ): account is CreatePassswordAccount => account.authType === AuthType.PASSWORD
@@ -50,6 +55,8 @@ export interface Account extends CreateAccount {
   licenseKeyActivatedAt?: number
   licenseRequestedAt?: number
   licenseOverrides?: LicenseOverrides
+  provider?: AccountSSOProvider
+  providerType?: AccountSSOProviderType
   quotaUsage?: QuotaUsage
   offlineLicenseToken?: string
 }
@@ -87,6 +94,13 @@ export enum AccountSSOProvider {
   MICROSOFT = "microsoft",
 }
 
+const verifiableSSOProviders: AccountSSOProvider[] = [
+  AccountSSOProvider.MICROSOFT,
+]
+export function isVerifiableSSOProvider(provider: AccountSSOProvider): boolean {
+  return verifiableSSOProviders.includes(provider)
+}
+
 export interface AccountSSO {
   provider: AccountSSOProvider
   providerType: AccountSSOProviderType

packages/types/src/documents/app/query.ts

@@ -6,7 +6,7 @@ export interface Query extends Document {
   parameters: QueryParameter[]
   fields: RestQueryFields | any
   transformer: string | null
-  schema: any
+  schema: Record<string, { name?: string; type: string }>
   readable: boolean
   queryVerb: string
 }

packages/types/src/documents/global/user.ts

@@ -55,6 +55,7 @@ export interface User extends Document {
   userGroups?: string[]
   onboardedAt?: string
   scimInfo?: { isSync: true } & Record<string, any>
+  ssoId?: string
 }
 
 export enum UserStatus {

packages/types/src/documents/platform/users.ts

@@ -15,4 +15,16 @@ export interface PlatformUserById extends Document {
   tenantId: string
 }
 
-export type PlatformUser = PlatformUserByEmail | PlatformUserById
+/**
+ * doc id is a unique SSO provider ID for the user
+ */
+export interface PlatformUserBySsoId extends Document {
+  tenantId: string
+  userId: string
+  email: string
+}
+
+export type PlatformUser =
+  | PlatformUserByEmail
+  | PlatformUserById
+  | PlatformUserBySsoId

packages/types/src/sdk/db.ts

@@ -1,7 +1,6 @@
 import Nano from "@budibase/nano"
 import { AllDocsResponse, AnyDocument, Document } from "../"
 import { Writable } from "stream"
-import PouchDB from "pouchdb"
 
 export enum SearchIndex {
   ROWS = "rows",

packages/types/src/sdk/koa.ts

@@ -1,5 +1,5 @@
 import { Context, Request } from "koa"
-import { User, Role, UserRoles, Account } from "../documents"
+import { User, Role, UserRoles, Account, ConfigType } from "../documents"
 import { FeatureFlag, License } from "../sdk"
 import { Files } from "formidable"
 
@@ -13,6 +13,7 @@ export interface ContextUser extends Omit<User, "roles"> {
   csrfToken?: string
   featureFlags?: FeatureFlag[]
   accountPortalAccess?: boolean
+  providerType?: ConfigType
   account?: Account
 }
 

packages/worker/package.json

@@ -67,7 +67,6 @@
     "node-fetch": "2.6.7",
     "nodemailer": "6.7.2",
     "passport-google-oauth": "2.0.0",
-    "passport-jwt": "4.0.0",
     "passport-local": "1.0.0",
     "pouchdb": "7.3.0",
     "pouchdb-all-dbs": "1.1.1",
@@ -84,21 +83,15 @@
     "@types/lodash": "^4.14.191",
     "@types/node": "18.17.0",
     "@types/node-fetch": "2.6.4",
-    "@types/pouchdb": "6.4.0",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.12",
     "@types/uuid": "8.3.4",
-    "copyfiles": "2.4.1",
-    "eslint": "6.8.0",
     "jest": "29.6.2",
     "lodash": "4.17.21",
     "nodemon": "2.0.15",
-    "pouchdb-adapter-memory": "7.2.2",
     "rimraf": "3.0.2",
     "supertest": "6.2.2",
     "timekeeper": "2.2.0",
-    "ts-node": "10.8.1",
-    "tsconfig-paths": "4.0.0",
     "typescript": "5.2.2",
     "update-dotenv": "1.1.1"
   },

packages/worker/src/api/controllers/global/users.ts

@@ -95,7 +95,7 @@ const parseBooleanParam = (param: any) => {
 export const adminUser = async (
   ctx: Ctx<CreateAdminUserRequest, CreateAdminUserResponse>
 ) => {
-  const { email, password, tenantId } = ctx.request.body
+  const { email, password, tenantId, ssoId } = ctx.request.body
 
   if (await platform.tenants.exists(tenantId)) {
     ctx.throw(403, "Organisation already exists.")
@@ -136,6 +136,7 @@ export const adminUser = async (
         global: true,
       },
       tenantId,
+      ssoId,
     }
     try {
       // always bust checklist beforehand, if an error occurs but can proceed, don't get

packages/worker/src/api/routes/global/tests/roles.spec.ts

@@ -1,6 +1,5 @@
 import { structures, TestConfiguration } from "../../../../tests"
 import { context, db, permissions, roles } from "@budibase/backend-core"
-import { Mock } from "jest-mock"
 import { Database } from "@budibase/types"
 
 jest.mock("@budibase/backend-core", () => {
@@ -47,7 +46,7 @@ describe("/api/global/roles", () => {
   beforeEach(async () => {
     appId = db.generateAppID(config.tenantId)
     appDb = db.getDB(appId)
-    const mockAppDB = context.getAppDB as Mock
+    const mockAppDB = context.getAppDB as jest.Mock
     mockAppDB.mockReturnValue(appDb)
 
     await addAppMetadata()

packages/worker/src/api/routes/global/tests/scim.spec.ts

@@ -11,7 +11,7 @@ import { TestConfiguration } from "../../../../tests"
 import { events } from "@budibase/backend-core"
 
 // this test can 409 - retries reduce issues with this
-jest.retryTimes(2)
+jest.retryTimes(2, { logErrorsBeforeRetry: true })
 jest.setTimeout(30000)
 
 mocks.licenses.useScimIntegration()

packages/worker/src/api/routes/global/users.ts

@@ -14,6 +14,7 @@ function buildAdminInitValidation() {
       email: Joi.string().required(),
       password: Joi.string(),
       tenantId: Joi.string().required(),
+      ssoId: Joi.string(),
     })
       .required()
       .unknown(false)

packages/worker/tsconfig.json

@@ -4,10 +4,6 @@
     "composite": true,
     "baseUrl": "."
   },
-  "ts-node": {
-    "require": ["tsconfig-paths/register"],
-    "swc": true
-  },
   "include": ["src/**/*", "__mocks__/**/*"],
   "exclude": ["node_modules", "dist"]
 }

scripts/syncProPackage.js

@@ -33,7 +33,7 @@ Object.keys(data).forEach(workspace => {
 
   let hasChanges = false
 
-  if (packageJson.dependencies["@budibase/pro"]) {
+  if (packageJson.dependencies && packageJson.dependencies["@budibase/pro"]) {
     packageJson.dependencies["@budibase/pro"] = version
     hasChanges = true
   }

specs/openapi.json

@@ -1 +0,0 @@
-../packages/server/specs/openapi.json