Merge branch 'Budibase:develop' into allow-plugins-contribute-datasourceplus

.github/workflows/budibase_ci.yml

@@ -30,8 +30,8 @@ jobs:
         with:
           root-reserve-mb: 35000
           swap-size-mb: 1024
-          remove-android: 'true'
+          remove-android: "true"
-          remove-dotnet: 'true'
+          remove-dotnet: "true"
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
         if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
@@ -121,7 +121,7 @@ jobs:
           name: codecov-umbrella
           verbose: true
 
-  test-services:
+  test-worker:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo and submodules
@@ -143,12 +143,48 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
-      - name: Test worker and server
+      - name: Test worker
         run: |
           if ${{ env.USE_NX_AFFECTED }}; then
-            yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
+            yarn test --scope=@budibase/worker --since=${{ env.NX_BASE_BRANCH }}
           else
-            yarn test --scope=@budibase/worker --scope=@budibase/server
+            yarn test --scope=@budibase/worker
+          fi
+
+      - uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
+          name: codecov-umbrella
+          verbose: true
+
+  test-server:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo and submodules
+        uses: actions/checkout@v3
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
+      - name: Checkout repo only
+        uses: actions/checkout@v3
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
+        with:
+          fetch-depth: 0
+
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18.x
+          cache: "yarn"
+      - run: yarn --frozen-lockfile
+      - name: Test server
+        run: |
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn test --scope=@budibase/server
           fi
 
       - uses: codecov/codecov-action@v3
@@ -259,4 +295,4 @@ jobs:
               process.exit(1);
             } else {
               console.log('All good, the submodule had been merged and setup correctly!')
-            }
+            }

.github/workflows/close-featurebranch.yml

@@ -0,0 +1,21 @@
+name: close-featurebranch
+
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - develop
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: passeidireto/trigger-external-workflow-action@main
+        env:
+          PAYLOAD_BRANCH: ${{ github.head_ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.ref }}
+        with:
+          repository: budibase/budibase-deploys
+          event: featurebranch-qa-close
+          github_pat: ${{ secrets.GH_ACCESS_TOKEN }}

.github/workflows/deploy-featurebranch.yml

@@ -12,7 +12,8 @@ jobs:
       - uses: actions/checkout@v3
       - uses: passeidireto/trigger-external-workflow-action@main
         env:
-            BRANCH: ${{ github.head_ref }}
+          PAYLOAD_BRANCH: ${{ github.head_ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.ref }}
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-deploy

hosting/proxy/nginx.prod.conf

@@ -55,7 +55,7 @@ http {
     set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
     set $csp_object "object-src 'none'";
     set $csp_base_uri "base-uri 'self'";
-    set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibase.qa https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";    
+    set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibaseqa.app https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";    
     set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com  https://js.intercomcdn.com  https://fonts.intercomcdn.com";
     set $csp_frame "frame-src 'self' https:";
     set $csp_img "img-src http: https: data: blob:";

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.10.12-alpha.5",
+  "version": "2.10.12-alpha.23",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -5,7 +5,7 @@
     "@esbuild-plugins/tsconfig-paths": "^0.1.2",
     "@nx/js": "16.4.3",
     "@rollup/plugin-json": "^4.0.2",
-    "@typescript-eslint/parser": "5.45.0",
+    "@typescript-eslint/parser": "6.7.2",
     "esbuild": "^0.18.17",
     "esbuild-node-externals": "^1.8.0",
     "eslint": "^8.44.0",
@@ -22,7 +22,7 @@
     "rimraf": "^3.0.2",
     "rollup-plugin-replace": "^2.2.0",
     "svelte": "^3.38.2",
-    "typescript": "4.7.3",
+    "typescript": "5.2.2",
     "@babel/core": "^7.22.5",
     "@babel/eslint-parser": "^7.22.5",
     "@babel/preset-env": "^7.22.5",

packages/backend-core/package.json

@@ -87,7 +87,7 @@
     "timekeeper": "2.2.0",
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
-    "typescript": "4.7.3"
+    "typescript": "5.2.2"
   },
   "nx": {
     "targets": {

packages/backend-core/src/cache/user.ts

@@ -120,7 +120,7 @@ export async function getUsers(
 ): Promise<{ users: User[]; notFoundIds?: string[] }> {
   const client = await redis.getUserClient()
   // try cache
-  let usersFromCache = await client.bulkGet(userIds)
+  let usersFromCache = await client.bulkGet<User>(userIds)
   const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid])
   const users = Object.values(usersFromCache)
   let notFoundIds

packages/backend-core/src/redis/redis.ts

@@ -242,7 +242,7 @@ class RedisWrapper {
     }
   }
 
-  async bulkGet(keys: string[]) {
+  async bulkGet<T>(keys: string[]) {
     const db = this._db
     if (keys.length === 0) {
       return {}
@@ -250,7 +250,7 @@ class RedisWrapper {
     const prefixedKeys = keys.map(key => addDbPrefix(db, key))
     let response = await this.getClient().mget(prefixedKeys)
     if (Array.isArray(response)) {
-      let final: Record<string, any> = {}
+      let final: Record<string, T> = {}
       let count = 0
       for (let result of response) {
         if (result) {

packages/backend-core/tests/core/utilities/mocks/licenses.ts

@@ -86,8 +86,8 @@ export const useAuditLogs = () => {
   return useFeature(Feature.AUDIT_LOGS)
 }
 
-export const usePublicApiUserRoles = () => {
+export const useExpandedPublicApi = () => {
-  return useFeature(Feature.USER_ROLE_PUBLIC_API)
+  return useFeature(Feature.EXPANDED_PUBLIC_API)
 }
 
 export const useScimIntegration = () => {

packages/backend-core/tests/core/utilities/structures/users.ts

@@ -10,14 +10,13 @@ import { authDetails } from "./sso"
 import { uuid } from "./common"
 import { generator } from "./generator"
 import { tenant } from "."
-import { generateGlobalUserID } from "../../../../src/docIds"
 
 export const newEmail = () => {
   return `${uuid()}@test.com`
 }
 
 export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
-  const userId = userProps?._id || generateGlobalUserID()
+  const userId = userProps?._id
   return {
     _id: userId,
     userId,
@@ -53,7 +52,7 @@ export const adminOnlyUser = (userProps?: any): AdminOnlyUser => {
   }
 }
 
-export const builderUser = (userProps?: any): BuilderUser => {
+export const builderUser = (userProps?: Partial<User>): BuilderUser => {
   return {
     ...user(userProps),
     builder: {

packages/backend-core/tests/extra/DBTestConfiguration.ts

@@ -18,7 +18,7 @@ class DBTestConfiguration {
 
   // TENANCY
 
-  doInTenant(task: any) {
+  doInTenant<T>(task: () => Promise<T>) {
     return context.doInTenant(this.tenantId, () => {
       return task()
     })

packages/backend-core/tests/index.ts

@@ -1 +1,2 @@
 export * from "./core/utilities"
+export * from "./extra"

packages/bbui/src/Form/Core/Multiselect.svelte

@@ -14,12 +14,12 @@
   export let autocomplete = false
   export let sort = false
   export let autoWidth = false
-  export let fetchTerm = null
+  export let searchTerm = null
-  export let useFetch = false
   export let customPopoverHeight
   export let customPopoverOffsetBelow
   export let customPopoverMaxHeight
   export let open = false
+  export let loading
 
   const dispatch = createEventDispatcher()
 
@@ -82,6 +82,7 @@
 </script>
 
 <Picker
+  on:loadMore
   {id}
   {error}
   {disabled}
@@ -90,9 +91,8 @@
   {options}
   isPlaceholder={!arrayValue.length}
   {autocomplete}
-  bind:fetchTerm
+  bind:searchTerm
   bind:open
-  {useFetch}
   {isOptionSelected}
   {getOptionLabel}
   {getOptionValue}
@@ -102,4 +102,5 @@
   {customPopoverHeight}
   {customPopoverOffsetBelow}
   {customPopoverMaxHeight}
+  {loading}
 />

packages/bbui/src/Form/Core/Picker.svelte

@@ -2,7 +2,7 @@
   import "@spectrum-css/picker/dist/index-vars.css"
   import "@spectrum-css/popover/dist/index-vars.css"
   import "@spectrum-css/menu/dist/index-vars.css"
-  import { createEventDispatcher } from "svelte"
+  import { createEventDispatcher, onDestroy } from "svelte"
   import clickOutside from "../../Actions/click_outside"
   import Search from "./Search.svelte"
   import Icon from "../../Icon/Icon.svelte"
@@ -10,6 +10,7 @@
   import Popover from "../../Popover/Popover.svelte"
   import Tags from "../../Tags/Tags.svelte"
   import Tag from "../../Tags/Tag.svelte"
+  import ProgressCircle from "../../ProgressCircle/ProgressCircle.svelte"
 
   export let id = null
   export let disabled = false
@@ -35,19 +36,20 @@
   export let autoWidth = false
   export let autocomplete = false
   export let sort = false
-  export let fetchTerm = null
+  export let searchTerm = null
-  export let useFetch = false
   export let customPopoverHeight
   export let customPopoverOffsetBelow
   export let customPopoverMaxHeight
   export let align = "left"
   export let footer = null
   export let customAnchor = null
+  export let loading
+
   const dispatch = createEventDispatcher()
 
-  let searchTerm = null
   let button
   let popover
+  let component
 
   $: sortedOptions = getSortedOptions(options, getOptionLabel, sort)
   $: filteredOptions = getFilteredOptions(
@@ -82,7 +84,7 @@
   }
 
   const getFilteredOptions = (options, term, getLabel) => {
-    if (autocomplete && term && !fetchTerm) {
+    if (autocomplete && term) {
       const lowerCaseTerm = term.toLowerCase()
       return options.filter(option => {
         return `${getLabel(option)}`.toLowerCase().includes(lowerCaseTerm)
@@ -90,6 +92,20 @@
     }
     return options
   }
+
+  const onScroll = e => {
+    const scrollPxThreshold = 100
+    const scrollPositionFromBottom =
+      e.target.scrollHeight - e.target.clientHeight - e.target.scrollTop
+    if (scrollPositionFromBottom < scrollPxThreshold) {
+      dispatch("loadMore")
+    }
+  }
+
+  $: component?.addEventListener("scroll", onScroll)
+  onDestroy(() => {
+    component?.removeEventListener("scroll", null)
+  })
 </script>
 
 <button
@@ -163,14 +179,13 @@
   >
     {#if autocomplete}
       <Search
-        value={useFetch ? fetchTerm : searchTerm}
+        value={searchTerm}
-        on:change={event =>
+        on:change={event => (searchTerm = event.detail)}
-          useFetch ? (fetchTerm = event.detail) : (searchTerm = event.detail)}
         {disabled}
         placeholder="Search"
       />
     {/if}
-    <ul class="spectrum-Menu" role="listbox">
+    <ul class="spectrum-Menu" role="listbox" bind:this={component}>
       {#if placeholderOption}
         <li
           class="spectrum-Menu-item placeholder"
@@ -248,6 +263,12 @@
       {/if}
     </ul>
 
+    {#if loading}
+      <div class="loading" class:loading--withAutocomplete={autocomplete}>
+        <ProgressCircle size="S" />
+      </div>
+    {/if}
+
     {#if footer}
       <div class="footer">
         {footer}
@@ -323,18 +344,19 @@
   /* Search styles inside popover */
   .popover-content :global(.spectrum-Search) {
     margin-top: -1px;
-    margin-left: -1px;
+    width: 100%;
-    width: calc(100% + 2px);
   }
   .popover-content :global(.spectrum-Search input) {
     height: auto;
     border-bottom-left-radius: 0;
     border-bottom-right-radius: 0;
+    border-left: 0;
+    border-right: 0;
     padding-top: var(--spectrum-global-dimension-size-100);
     padding-bottom: var(--spectrum-global-dimension-size-100);
   }
   .popover-content :global(.spectrum-Search .spectrum-ClearButton) {
-    right: 1px;
+    right: 2px;
     top: 2px;
   }
   .popover-content :global(.spectrum-Search .spectrum-Textfield-icon) {
@@ -359,4 +381,14 @@
   .option-tag :global(.spectrum-Tags-item > .spectrum-Icon) {
     margin-top: 2px;
   }
+
+  .loading {
+    position: fixed;
+    justify-content: center;
+    right: var(--spacing-s);
+    top: var(--spacing-s);
+  }
+  .loading--withAutocomplete {
+    top: calc(34px + var(--spacing-m));
+  }
 </style>

packages/bbui/src/Form/Core/Select.svelte

@@ -25,6 +25,8 @@
   export let tag = null
   export let customPopoverOffsetBelow
   export let customPopoverMaxHeight
+  export let searchTerm = null
+  export let loading
 
   const dispatch = createEventDispatcher()
 
@@ -65,6 +67,8 @@
 <Picker
   on:click
   bind:open
+  bind:searchTerm
+  on:loadMore
   {quiet}
   {id}
   {error}
@@ -92,4 +96,5 @@
   placeholderOption={placeholder === false ? null : placeholder}
   isOptionSelected={option => option === value}
   onSelectOption={selectOption}
+  {loading}
 />

packages/bbui/src/Form/Core/TextField.svelte

@@ -96,8 +96,8 @@
     {disabled}
     {readonly}
     {id}
-    value={value || ""}
+    value={value ?? ""}
-    placeholder={placeholder || ""}
+    placeholder={placeholder ?? ""}
     on:click
     on:blur
     on:focus

packages/bbui/src/Form/Multiselect.svelte

@@ -16,8 +16,7 @@
   export let sort = false
   export let autoWidth = false
   export let autocomplete = false
-  export let fetchTerm = null
+  export let searchTerm = null
-  export let useFetch = false
   export let customPopoverHeight
 
   const dispatch = createEventDispatcher()
@@ -41,8 +40,7 @@
     {autoWidth}
     {autocomplete}
     {customPopoverHeight}
-    bind:fetchTerm
+    bind:searchTerm
-    {useFetch}
     on:change={onChange}
     on:click
   />

packages/bbui/src/Table/CellRenderer.svelte

@@ -25,6 +25,7 @@
     longform: StringRenderer,
     array: ArrayRenderer,
     internal: InternalRenderer,
+    bb_reference: RelationshipRenderer,
   }
   $: type = getType(schema)
   $: customRenderer = customRenderers?.find(x => x.column === schema?.name)

packages/builder/package.json

@@ -108,7 +108,7 @@
     "svelte-jester": "^1.3.2",
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
-    "typescript": "4.7.3",
+    "typescript": "5.2.2",
     "vite": "^3.0.8",
     "vite-plugin-static-copy": "^0.16.0",
     "vitest": "^0.29.2"

packages/builder/src/analytics/constants.js

@@ -2,6 +2,7 @@ export const Events = {
   COMPONENT_CREATED: "component:created",
   COMPONENT_UPDATED: "component:updated",
   APP_VIEW_PUBLISHED: "app:view_published",
+  BLOCK_EJECTED: "block:ejected",
 }
 
 export const EventSource = {

packages/builder/src/builderStore/store/frontend.js

@@ -1287,6 +1287,11 @@ export const getFrontendStore = () => {
             return false
           }
 
+          // Log event
+          analytics.captureEvent(Events.BLOCK_EJECTED, {
+            block: block._component,
+          })
+
           // Attach block children back into ejected definition, using the
           // _containsSlot flag to know where to insert them
           const slotContainer = findAllMatchingComponents(

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -83,8 +83,7 @@
       if (Array.isArray(value)) {
         return value
       }
-
+      return value.split(",").map(x => x.trim())
-      return [value]
     }
 
     if (type === "json") {
@@ -146,7 +145,7 @@
                 placeholder={placeholders[schema.type]}
                 panel={AutomationBindingPanel}
                 value={Array.isArray(value[field])
-                  ? value[field].join(" ")
+                  ? value[field].join(",")
                   : value[field]}
                 on:change={e => onChange(e, field, schema.type)}
                 label={field}

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -55,9 +55,14 @@
     bind:value={value[field]}
     label={field}
     options={schema.constraints.inclusion}
+    on:change={e => onChange(e, field)}
   />
 {:else if schema.type === "longform"}
-  <TextArea label={field} bind:value={value[field]} />
+  <TextArea
+    label={field}
+    bind:value={value[field]}
+    on:change={e => onChange(e, field)}
+  />
 {:else if schema.type === "json"}
   <span>
     <Label>{field}</Label>
@@ -73,7 +78,11 @@
     />
   </span>
 {:else if schema.type === "link"}
-  <LinkedRowSelector bind:linkedRows={value[field]} {schema} />
+  <LinkedRowSelector
+    bind:linkedRows={value[field]}
+    {schema}
+    on:change={e => onChange(e, field)}
+  />
 {:else if schema.type === "string" || schema.type === "number"}
   <svelte:component
     this={isTestModal ? ModalBindableInput : DrawerBindableInput}

packages/builder/src/components/backend/DataTable/ExternalDataSourceTable.svelte

@@ -1,33 +0,0 @@
-<script>
-  import Table from "./Table.svelte"
-
-  export let query = {}
-  export let data = []
-  export let editRows = false
-
-  let loading = false
-  let error = false
-  let type = "external"
-</script>
-
-{#if error}
-  <div class="errors">{error}</div>
-{/if}
-<Table
-  schema={query.schema}
-  {data}
-  {loading}
-  {type}
-  rowCount={5}
-  allowEditing={editRows}
-/>
-
-<style>
-  .errors {
-    color: var(--red);
-    background: var(--red-light);
-    padding: var(--spacing-m);
-    border-radius: var(--border-radius-m);
-    margin-bottom: var(--spacing-m);
-  }
-</style>

packages/builder/src/components/backend/DataTable/TableDataTable.svelte

@@ -14,13 +14,14 @@
   import GridManageAccessButton from "components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte"
   import GridRelationshipButton from "components/backend/DataTable/buttons/grid/GridRelationshipButton.svelte"
   import GridEditColumnModal from "components/backend/DataTable/modals/grid/GridEditColumnModal.svelte"
+  import GridUsersTableButton from "components/backend/DataTable/modals/grid/GridUsersTableButton.svelte"
 
   const userSchemaOverrides = {
-    firstName: { displayName: "First name", disabled: true },
+    firstName: { displayName: "First name" },
-    lastName: { displayName: "Last name", disabled: true },
+    lastName: { displayName: "Last name" },
-    email: { displayName: "Email", disabled: true },
+    email: { displayName: "Email" },
-    roleId: { displayName: "Role", disabled: true },
+    roleId: { displayName: "Role" },
-    status: { displayName: "Status", disabled: true },
+    status: { displayName: "Status" },
   }
 
   $: id = $tables.selected?._id
@@ -59,22 +60,22 @@
     datasource={gridDatasource}
     canAddRows={!isUsersTable}
     canDeleteRows={!isUsersTable}
+    canEditRows={!isUsersTable}
+    canEditColumns={!isUsersTable}
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
     on:updatedatasource={handleGridTableUpdate}
   >
     <svelte:fragment slot="filter">
+      {#if isUsersTable}
+        <GridUsersTableButton />
+      {/if}
       <GridFilterButton />
     </svelte:fragment>
-    <svelte:fragment slot="edit-column">
-      <GridEditColumnModal />
-    </svelte:fragment>
-    <svelte:fragment slot="add-column">
-      <GridAddColumnModal />
-    </svelte:fragment>
-
     <svelte:fragment slot="controls">
-      <GridCreateViewButton />
+      {#if !isUsersTable}
+        <GridCreateViewButton />
+      {/if}
       <GridManageAccessButton />
       {#if relationshipsEnabled}
         <GridRelationshipButton />
@@ -84,7 +85,6 @@
       {:else}
         <GridImportButton />
       {/if}
-
       <GridExportButton />
       {#if isUsersTable}
         <GridEditUserModal />
@@ -92,6 +92,12 @@
         <GridCreateEditRowModal />
       {/if}
     </svelte:fragment>
+    <svelte:fragment slot="edit-column">
+      <GridEditColumnModal />
+    </svelte:fragment>
+    <svelte:fragment slot="add-column">
+      <GridAddColumnModal />
+    </svelte:fragment>
   </Grid>
 </div>
 

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -5,7 +5,6 @@
     Label,
     Select,
     Toggle,
-    RadioGroup,
     Icon,
     DatePicker,
     Modal,
@@ -26,14 +25,16 @@
     ALLOWABLE_STRING_TYPES,
     ALLOWABLE_NUMBER_TYPES,
     SWITCHABLE_TYPES,
+    PrettyRelationshipDefinitions,
   } from "constants/backend"
   import { getAutoColumnInformation, buildAutoColumn } from "builderStore/utils"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
-  import { truncate } from "lodash"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import { getBindings } from "components/backend/DataTable/formula"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
   import { ValidColumnNameRegex } from "@budibase/shared-core"
+  import { FieldSubtype, FieldType } from "@budibase/types"
+  import RelationshipSelector from "components/common/RelationshipSelector.svelte"
 
   const AUTO_TYPE = "auto"
   const FORMULA_TYPE = FIELDS.FORMULA.type
@@ -42,6 +43,11 @@
   const NUMBER_TYPE = FIELDS.NUMBER.type
   const JSON_TYPE = FIELDS.JSON.type
   const DATE_TYPE = FIELDS.DATETIME.type
+  const BB_REFERENCE_TYPE = FieldType.BB_REFERENCE
+  const BB_USER_REFERENCE_TYPE = composeType(
+    BB_REFERENCE_TYPE,
+    FieldSubtype.USER
+  )
 
   const dispatch = createEventDispatcher()
   const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
@@ -57,6 +63,10 @@
   let indexes = [...($tables.selected.indexes || [])]
   let isCreating = undefined
 
+  let relationshipPart1 = PrettyRelationshipDefinitions.Many
+  let relationshipPart2 = PrettyRelationshipDefinitions.One
+
+  let relationshipTableIdSecondary = null
   let table = $tables.selected
   let confirmDeleteDialog
   let savingColumn
@@ -64,18 +74,73 @@
   let jsonSchemaModal
   let allowedTypes = []
   let editableColumn = {
-    type: "string",
+    type: fieldDefinitions.STRING.type,
     constraints: fieldDefinitions.STRING.constraints,
     // Initial value for column name in other table for linked records
     fieldName: $tables.selected.name,
   }
 
+  const bbRefTypeMapping = {}
+
+  function composeType(fieldType, subtype) {
+    return `${fieldType}_${subtype}`
+  }
+
+  // Handling fields with subtypes
+  fieldDefinitions = Object.entries(fieldDefinitions).reduce(
+    (p, [key, field]) => {
+      if (field.type === BB_REFERENCE_TYPE) {
+        const composedType = composeType(field.type, field.subtype)
+        p[key] = {
+          ...field,
+          type: composedType,
+        }
+        bbRefTypeMapping[composedType] = {
+          type: field.type,
+          subtype: field.subtype,
+        }
+      } else {
+        p[key] = field
+      }
+      return p
+    },
+    {}
+  )
+
   $: if (primaryDisplay) {
     editableColumn.constraints.presence = { allowEmpty: false }
   }
 
+  let relationshipMap = {
+    [RelationshipType.MANY_TO_ONE]: {
+      part1: PrettyRelationshipDefinitions.MANY,
+      part2: PrettyRelationshipDefinitions.ONE,
+    },
+    [RelationshipType.MANY_TO_MANY]: {
+      part1: PrettyRelationshipDefinitions.MANY,
+      part2: PrettyRelationshipDefinitions.MANY,
+    },
+    [RelationshipType.ONE_TO_MANY]: {
+      part1: PrettyRelationshipDefinitions.ONE,
+      part2: PrettyRelationshipDefinitions.MANY,
+    },
+  }
+
+  $: {
+    if (editableColumn.type === LINK_TYPE) {
+      // Determine the relationship type based on the selected values of both parts
+      editableColumn.relationshipType = Object.entries(relationshipMap).find(
+        ([_, parts]) =>
+          parts.part1 === relationshipPart1 && parts.part2 === relationshipPart2
+      )?.[0]
+      // Set the tableId based on the selected table
+      editableColumn.tableId = relationshipTableIdSecondary
+    }
+  }
+
   const initialiseField = (field, savingColumn) => {
     isCreating = !field
+
     if (field && !savingColumn) {
       editableColumn = cloneDeep(field)
       originalName = editableColumn.name ? editableColumn.name + "" : null
@@ -83,6 +148,14 @@
       primaryDisplay =
         $tables.selected.primaryDisplay == null ||
         $tables.selected.primaryDisplay === editableColumn.name
+
+      const mapped = Object.entries(bbRefTypeMapping).find(
+        ([_, v]) => v.type === field.type && v.subtype === field.subtype
+      )
+      if (mapped) {
+        editableColumn.type = mapped[0]
+        delete editableColumn.subtype
+      }
     } else if (!savingColumn) {
       let highestNumber = 0
       Object.keys(table.schema).forEach(columnName => {
@@ -99,11 +172,24 @@
         editableColumn.name = "Column 01"
       }
     }
+
     allowedTypes = getAllowedTypes()
+
+    if (editableColumn.type === LINK_TYPE && editableColumn.tableId) {
+      relationshipTableIdSecondary = editableColumn.tableId
+      if (editableColumn.relationshipType in relationshipMap) {
+        const { part1, part2 } =
+          relationshipMap[editableColumn.relationshipType]
+        relationshipPart1 = part1
+        relationshipPart2 = part2
+      }
+    }
   }
 
   $: initialiseField(field, savingColumn)
 
+  $: isBBReference = !!bbRefTypeMapping[editableColumn.type]
+
   $: checkConstraints(editableColumn)
   $: required = !!editableColumn?.constraints?.presence || primaryDisplay
   $: uneditable =
@@ -157,7 +243,6 @@
     !uneditable &&
     editableColumn?.type !== AUTO_TYPE &&
     !editableColumn.autocolumn
-  $: relationshipOptions = getRelationshipOptions(editableColumn)
   $: external = table.type === "external"
   // in the case of internal tables the sourceId will just be undefined
   $: tableOptions = $tables.list.filter(
@@ -180,6 +265,13 @@
 
     let saveColumn = cloneDeep(editableColumn)
 
+    if (bbRefTypeMapping[saveColumn.type]) {
+      saveColumn = {
+        ...saveColumn,
+        ...bbRefTypeMapping[saveColumn.type],
+      }
+    }
+
     if (saveColumn.type === AUTO_TYPE) {
       saveColumn = buildAutoColumn(
         $tables.selected.name,
@@ -258,9 +350,10 @@
     // Default relationships many to many
     if (editableColumn.type === LINK_TYPE) {
       editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
-    }
+    } else if (editableColumn.type === FORMULA_TYPE) {
-    if (editableColumn.type === FORMULA_TYPE) {
       editableColumn.formulaType = "dynamic"
+    } else if (editableColumn.type === BB_USER_REFERENCE_TYPE) {
+      editableColumn.relationshipType = RelationshipType.ONE_TO_MANY
     }
   }
 
@@ -288,35 +381,6 @@
     return match ? parseInt(match[1]) : 0
   }
 
-  function getRelationshipOptions(field) {
-    if (!field || !field.tableId) {
-      return null
-    }
-    const linkTable = tableOptions?.find(table => table._id === field.tableId)
-    if (!linkTable) {
-      return null
-    }
-    const thisName = truncate(table.name, { length: 14 }),
-      linkName = truncate(linkTable.name, { length: 14 })
-    return [
-      {
-        name: `Many ${thisName} rows → many ${linkName} rows`,
-        alt: `Many ${table.name} rows → many ${linkTable.name} rows`,
-        value: RelationshipType.MANY_TO_MANY,
-      },
-      {
-        name: `One ${linkName} row → many ${thisName} rows`,
-        alt: `One ${linkTable.name} rows → many ${table.name} rows`,
-        value: RelationshipType.ONE_TO_MANY,
-      },
-      {
-        name: `One ${thisName} row → many ${linkName} rows`,
-        alt: `One ${table.name} rows → many ${linkTable.name} rows`,
-        value: RelationshipType.MANY_TO_ONE,
-      },
-    ]
-  }
-
   function getAllowedTypes() {
     if (
       originalName &&
@@ -328,7 +392,9 @@
       ALLOWABLE_NUMBER_TYPES.indexOf(editableColumn.type) !== -1
     ) {
       return ALLOWABLE_NUMBER_OPTIONS
-    } else if (!external) {
+    }
+
+    if (!external) {
       return [
         ...Object.values(fieldDefinitions),
         { name: "Auto Column", type: AUTO_TYPE },
@@ -349,6 +415,9 @@
       if (!external || table.sql) {
         fields = [...fields, FIELDS.LINK, FIELDS.ARRAY]
       }
+      if (fieldDefinitions.USER) {
+        fields.push(fieldDefinitions.USER)
+      }
       return fields
     }
   }
@@ -500,7 +569,7 @@
         <DatePicker bind:value={editableColumn.constraints.datetime.latest} />
       </div>
     </div>
-    {#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER"}
+    {#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER" && !editableColumn.dateOnly}
       <div>
         <div class="row">
           <Label>Time zones</Label>
@@ -520,6 +589,7 @@
         />
       </div>
     {/if}
+    <Toggle bind:value={editableColumn.dateOnly} text="Date only" />
   {:else if editableColumn.type === "number" && !editableColumn.autocolumn}
     <div class="split-label">
       <div class="label-length">
@@ -546,30 +616,15 @@
       </div>
     </div>
   {:else if editableColumn.type === "link"}
-    <Select
+    <RelationshipSelector
-      label="Table"
+      bind:relationshipPart1
-      disabled={linkEditDisabled}
+      bind:relationshipPart2
-      bind:value={editableColumn.tableId}
+      bind:relationshipTableIdPrimary={table.name}
-      options={tableOptions}
+      bind:relationshipTableIdSecondary
-      getOptionLabel={table => table.name}
+      bind:editableColumn
-      getOptionValue={table => table._id}
+      {linkEditDisabled}
-    />
+      {tableOptions}
-    {#if relationshipOptions && relationshipOptions.length > 0}
+      {errors}
-      <RadioGroup
-        disabled={linkEditDisabled}
-        label="Define the relationship"
-        bind:value={editableColumn.relationshipType}
-        options={relationshipOptions}
-        getOptionLabel={option => option.name}
-        getOptionValue={option => option.value}
-        getOptionTitle={option => option.alt}
-      />
-    {/if}
-    <Input
-      disabled={linkEditDisabled}
-      label={`Column name in other table`}
-      bind:value={editableColumn.fieldName}
-      error={errors.relatedName}
     />
   {:else if editableColumn.type === FORMULA_TYPE}
     {#if !table.sql}
@@ -616,6 +671,17 @@
     <Button primary text on:click={openJsonSchemaEditor}
       >Open schema editor</Button
     >
+  {:else if isBBReference}
+    <Toggle
+      value={editableColumn.relationshipType === RelationshipType.MANY_TO_MANY}
+      on:change={e =>
+        (editableColumn.relationshipType = e.detail
+          ? RelationshipType.MANY_TO_MANY
+          : RelationshipType.ONE_TO_MANY)}
+      disabled={!isCreating}
+      thin
+      text="Allow multiple users"
+    />
   {/if}
   {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
     <Select

packages/builder/src/components/backend/DataTable/modals/grid/GridUsersTableButton.svelte

@@ -0,0 +1,54 @@
+<script>
+  import { ActionButton, Popover, Heading, Body, Button } from "@budibase/bbui"
+  import { store } from "builderStore"
+
+  let anchor
+  let open = false
+
+  const openSidePanel = () => {
+    store.update(state => ({
+      ...state,
+      builderSidePanel: true,
+    }))
+    open = false
+  }
+</script>
+
+<div bind:this={anchor}>
+  <ActionButton on:click={() => (open = true)} icon="Help" quiet>
+    Why can't I edit this table?
+  </ActionButton>
+</div>
+
+<Popover bind:open {anchor} align="left">
+  <div class="content">
+    <Heading size="XS">The app users table is read only</Heading>
+    <Body size="S">
+      You can continue to view the users that have access to your application.
+    </Body>
+    <Body size="S">
+      Manage and invite more application users using the user side panel in the
+      top right of your screen.
+    </Body>
+    <div class="button">
+      <Button cta on:click={openSidePanel}>Open users panel</Button>
+    </div>
+  </div>
+</Popover>
+
+<style>
+  .content {
+    width: 300px;
+    padding: var(--spacing-l);
+    display: flex;
+    flex-direction: column;
+    gap: var(--spacing-l);
+  }
+  .content :global(.spectrum-Heading) {
+    font-weight: 400;
+  }
+  .button {
+    display: flex;
+    justify-content: flex-end;
+  }
+</style>

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -180,7 +180,7 @@
   <div class="hierarchy-items-container">
     <NavItem
       icon="UserGroup"
-      text="Users"
+      text="App users"
       selected={$isActive("./table/:tableId") &&
         $tables.selected?._id === TableNames.USERS}
       on:click={() => selectTable(TableNames.USERS)}

packages/builder/src/components/common/LinkedRowSelector.svelte

@@ -3,10 +3,13 @@
   import { API } from "api"
   import { Select, Label, Multiselect } from "@budibase/bbui"
   import { capitalise } from "../../helpers"
+  import { createEventDispatcher } from "svelte"
 
   export let schema
   export let linkedRows = []
 
+  const dispatch = createEventDispatcher()
+
   let rows = []
   let linkedIds = (Array.isArray(linkedRows) ? linkedRows : [])?.map(
     row => row?._id || row
@@ -44,7 +47,10 @@
     options={rows}
     getOptionLabel={getPrettyName}
     getOptionValue={row => row._id}
-    on:change={e => (linkedIds = e.detail ? [e.detail] : [])}
+    on:change={e => {
+      linkedIds = e.detail ? [e.detail] : []
+      dispatch("change", linkedIds)
+    }}
     {label}
     sort
   />
@@ -56,5 +62,6 @@
     getOptionLabel={getPrettyName}
     getOptionValue={row => row._id}
     sort
+    on:change={() => dispatch("change", linkedIds)}
   />
 {/if}

packages/builder/src/components/common/RelationshipSelector.svelte

@@ -0,0 +1,69 @@
+<script>
+  import { Select, Input } from "@budibase/bbui"
+  import { PrettyRelationshipDefinitions } from "constants/backend"
+
+  export let relationshipPart1
+  export let relationshipPart2
+  export let relationshipTableIdPrimary
+  export let relationshipTableIdSecondary
+  export let editableColumn
+  export let linkEditDisabled
+  export let tableOptions
+  export let errors
+</script>
+
+<div class="relationship-container">
+  <div class="relationship-part">
+    <Select
+      disabled={linkEditDisabled}
+      bind:value={relationshipPart1}
+      options={Object.values(PrettyRelationshipDefinitions)}
+    />
+  </div>
+  <div class="relationship-label">in</div>
+  <div class="relationship-part">
+    <Select
+      disabled
+      options={[relationshipTableIdPrimary]}
+      value={relationshipTableIdPrimary}
+    />
+  </div>
+</div>
+<div class="relationship-container">
+  <div class="relationship-part">
+    <Select
+      disabled={linkEditDisabled}
+      bind:value={relationshipPart2}
+      options={Object.values(PrettyRelationshipDefinitions)}
+      getOptionLabel={option => "To " + option.toLowerCase()}
+    />
+  </div>
+  <div class="relationship-label">in</div>
+  <div class="relationship-part">
+    <Select
+      disabled={linkEditDisabled}
+      bind:value={relationshipTableIdSecondary}
+      options={tableOptions}
+      getOptionLabel={table => table.name}
+      getOptionValue={table => table._id}
+    />
+  </div>
+</div>
+<Input
+  disabled={linkEditDisabled}
+  label={`Column name in other table`}
+  bind:value={editableColumn.fieldName}
+  error={errors.relatedName}
+/>
+
+<style>
+  .relationship-container {
+    display: flex;
+    align-items: center;
+    gap: 20px;
+  }
+
+  .relationship-part {
+    flex-basis: 60%;
+  }
+</style>

packages/builder/src/components/common/UndoRedoControl.svelte

@@ -38,14 +38,12 @@
     hoverable
     on:click={store.undo}
     disabled={!$store.canUndo}
-    tooltip="Undo latest change"
   />
   <Icon
     name="Redo"
     hoverable
     on:click={store.redo}
     disabled={!$store.canRedo}
-    tooltip="Redo latest undo"
   />
 </div>
 

packages/builder/src/components/design/Panel.svelte

@@ -12,11 +12,19 @@
   export let borderLeft = false
   export let borderRight = false
   export let wide = false
+  export let extraWide = false
+  export let closeButtonIcon = "Close"
 
   $: customHeaderContent = $$slots["panel-header-content"]
 </script>
 
-<div class="panel" class:wide class:borderLeft class:borderRight>
+<div
+  class="panel"
+  class:wide
+  class:extraWide
+  class:borderLeft
+  class:borderRight
+>
   <div class="header" class:custom={customHeaderContent}>
     {#if showBackButton}
       <Icon name="ArrowLeft" hoverable on:click={onClickBackButton} />
@@ -33,7 +41,7 @@
       </div>
     {/if}
     {#if showCloseButton}
-      <Icon name="Close" hoverable on:click={onClickCloseButton} />
+      <Icon name={closeButtonIcon} hoverable on:click={onClickCloseButton} />
     {/if}
   </div>
 
@@ -70,6 +78,10 @@
     width: 310px;
     flex: 0 0 310px;
   }
+  .panel.extraWide {
+    width: 450px;
+    flex: 0 0 450px;
+  }
   .header {
     flex: 0 0 48px;
     display: flex;

packages/builder/src/components/design/settings/componentSettings.js

@@ -65,6 +65,7 @@ const componentMap = {
   "field/array": FormFieldSelect,
   "field/json": FormFieldSelect,
   "field/barcodeqr": FormFieldSelect,
+  "field/bb_reference": FormFieldSelect,
   // Some validation types are the same as others, so not all types are
   // explicitly listed here. e.g. options uses string validation
   "validation/string": ValidationEditor,
@@ -74,6 +75,7 @@ const componentMap = {
   "validation/datetime": ValidationEditor,
   "validation/attachment": ValidationEditor,
   "validation/link": ValidationEditor,
+  "validation/bb_reference": ValidationEditor,
 }
 
 export const getComponentForSetting = setting => {

packages/builder/src/components/design/settings/controls/FieldConfiguration/utils.js

@@ -43,4 +43,5 @@ export const FieldTypeToComponentMap = {
   link: "relationshipfield",
   json: "jsonfield",
   barcodeqr: "codescanner",
+  bb_reference: "bbreferencefield",
 }

packages/builder/src/components/integration/ExtraQueryConfig.svelte

@@ -18,31 +18,20 @@
 </script>
 
 {#each extraFields as { key, displayName, type }}
-  <div class="config-field">
+  <Label>{displayName}</Label>
-    <Label>{displayName}</Label>
+  {#if type === "string"}
-    {#if type === "string"}
+    <Input
-      <Input
+      on:change={() => populateExtraQuery(extraQueryFields)}
-        on:change={() => populateExtraQuery(extraQueryFields)}
+      bind:value={extraQueryFields[key]}
-        bind:value={extraQueryFields[key]}
+    />
-      />
+  {/if}
-    {/if}
 
-    {#if type === "list"}
+  {#if type === "list"}
-      <Select
+    <Select
-        on:change={() => populateExtraQuery(extraQueryFields)}
+      on:change={() => populateExtraQuery(extraQueryFields)}
-        bind:value={extraQueryFields[key]}
+      bind:value={extraQueryFields[key]}
-        options={config[key].data[query.queryVerb]}
+      options={config[key].data[query.queryVerb]}
-        getOptionLabel={current => current}
+      getOptionLabel={current => current}
-      />
+    />
-    {/if}
+  {/if}
-  </div>
 {/each}
-
-<style>
-  .config-field {
-    display: grid;
-    grid-template-columns: 20% 1fr;
-    grid-gap: var(--spacing-l);
-    align-items: center;
-  }
-</style>

packages/builder/src/components/integration/KeyValueBuilder.svelte

@@ -34,6 +34,7 @@
   export let bindings = []
   export let bindingDrawerLeft
   export let allowHelpers = true
+  export let customButtonText = null
 
   let fields = Object.entries(object || {}).map(([name, value]) => ({
     name,
@@ -158,9 +159,13 @@
 {/if}
 {#if !readOnly && !noAddButton}
   <div>
-    <ActionButton icon="Add" secondary thin outline on:click={addEntry}
+    <ActionButton icon="Add" secondary thin outline on:click={addEntry}>
-      >Add{name ? ` ${lowercase(name)}` : ""}</ActionButton
+      {#if customButtonText}
-    >
+        {customButtonText}
+      {:else}
+        {`Add${name ? ` ${lowercase(name)}` : ""}`}
+      {/if}
+    </ActionButton>
   </div>
 {/if}
 

packages/builder/src/components/integration/QueryViewer.svelte

@@ -1,364 +1,443 @@
 <script>
-  import { goto, beforeUrlChange } from "@roxi/routify"
+  import { goto } from "@roxi/routify"
+  import { datasources, integrations, queries } from "stores/backend"
   import {
     Icon,
     Select,
-    Button,
-    ButtonGroup,
-    Body,
-    Label,
-    Layout,
     Input,
-    Heading,
+    Label,
-    Tabs,
-    Tab,
-    Modal,
-    ModalContent,
     notifications,
+    Heading,
+    Body,
     Divider,
+    Button,
   } from "@budibase/bbui"
-  import ExtraQueryConfig from "./ExtraQueryConfig.svelte"
+  import { capitalise } from "helpers"
-  import IntegrationQueryEditor from "components/integration/index.svelte"
-  import ExternalDataSourceTable from "components/backend/DataTable/ExternalDataSourceTable.svelte"
-  import BindingBuilder from "components/integration/QueryViewerBindingBuilder.svelte"
-  import { datasources, integrations, queries } from "stores/backend"
-  import { capitalise } from "../../helpers"
-  import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
-  import JSONPreview from "./JSONPreview.svelte"
-  import { SchemaTypeOptions } from "constants/backend"
-  import KeyValueBuilder from "./KeyValueBuilder.svelte"
-  import { fieldsToSchema, schemaToFields } from "helpers/data/utils"
   import AccessLevelSelect from "./AccessLevelSelect.svelte"
+  import IntegrationQueryEditor from "components/integration/index.svelte"
+  import QueryViewerSidePanel from "./QueryViewerSidePanel/index.svelte"
+  import { cloneDeep } from "lodash/fp"
+  import BindingBuilder from "components/integration/QueryViewerBindingBuilder.svelte"
+  import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
   import { ValidQueryNameRegex } from "@budibase/shared-core"
+  import ExtraQueryConfig from "./ExtraQueryConfig.svelte"
+  import QueryViewerSavePromptModal from "./QueryViewerSavePromptModal.svelte"
+  import { Utils } from "@budibase/frontend-core"
 
   export let query
+  let queryHash
 
-  const resumeNavigation = () => {
+  let loading = false
-    if (typeof navigateTo == "string") {
+  let modified = false
-      $goto(typeof navigateTo == "string" ? `${navigateTo}` : navigateTo)
+  let scrolling = false
-    }
+  let showSidePanel = false
+  let nameError
+
+  let newQuery
+
+  let datasource
+  let integration
+  let schemaType
+
+  let autoSchema = {}
+  let rows = []
+
+  const parseQuery = query => {
+    modified = false
+
+    datasource = $datasources.list.find(ds => ds._id === query.datasourceId)
+    integration = $integrations[datasource.source]
+    schemaType = integration.query[query.queryVerb].type
+
+    newQuery = cloneDeep(query)
+    // Set the location where the query code will be written to an empty string so that it doesn't
+    // get changed from undefined -> "" by the input, breaking our unsaved changes checks
+    newQuery.fields[schemaType] ??= ""
+
+    queryHash = JSON.stringify(newQuery)
   }
 
-  const transformerDocs = "https://docs.budibase.com/docs/transformers"
+  $: parseQuery(query)
 
-  let fields = query?.schema ? schemaToFields(query.schema) : []
+  const checkIsModified = newQuery => {
-  let parameters
+    const newQueryHash = JSON.stringify(newQuery)
-  let data = []
+    modified = newQueryHash !== queryHash
-  let saveId
-  let currentTab = "JSON"
-  let saveModal
-  let override = false
-  let navigateTo = null
-  let nameError = null
 
-  // seed the transformer
+    return modified
-  if (query && !query.transformer) {
-    query.transformer = "return data"
   }
 
-  // initialise a new empty schema
+  const debouncedCheckIsModified = Utils.debounce(checkIsModified, 1000)
-  if (query && !query.schema) {
-    query.schema = {}
-  }
 
-  let queryStr = JSON.stringify(query)
+  $: debouncedCheckIsModified(newQuery)
 
-  $beforeUrlChange(event => {
+  async function runQuery({ suppressErrors = true }) {
-    const updated = JSON.stringify(query)
-
-    if (updated !== queryStr && !override) {
-      navigateTo = event.type == "pushstate" ? event.url : null
-      saveModal.show()
-      return false
-    } else return true
-  })
-
-  $: datasource = $datasources.list.find(ds => ds._id === query.datasourceId)
-  $: query.schema = fieldsToSchema(fields)
-  $: datasourceType = datasource?.source
-  $: integrationInfo = datasourceType ? $integrations[datasourceType] : null
-  $: queryConfig = integrationInfo?.query
-  $: shouldShowQueryConfig = queryConfig && query.queryVerb
-  $: readQuery = query.queryVerb === "read" || query.readable
-  $: queryInvalid = !query.name || nameError || (readQuery && data.length === 0)
-
-  //Cast field in query preview response to number if specified by schema
-  $: {
-    for (let i = 0; i < data.length; i++) {
-      let row = data[i]
-      for (let fieldName of Object.keys(fields)) {
-        if (fields[fieldName] === "number" && !isNaN(Number(row[fieldName]))) {
-          row[fieldName] = Number(row[fieldName])
-        } else {
-          row[fieldName] = row[fieldName]?.toString()
-        }
-      }
-    }
-  }
-
-  function resetDependentFields() {
-    if (query.fields.extra) {
-      query.fields.extra = {}
-    }
-  }
-
-  function populateExtraQuery(extraQueryFields) {
-    query.fields.extra = extraQueryFields
-  }
-
-  async function previewQuery() {
     try {
-      const response = await queries.preview(query)
+      showSidePanel = true
+      loading = true
+      const response = await queries.preview(newQuery)
       if (response.rows.length === 0) {
         notifications.info(
           "Query results empty. Please execute a query with results to create your schema."
         )
         return
       }
-      data = response.rows
+
-      // need to merge fields that already exist/might have changed
+      if (Object.keys(newQuery.schema).length === 0) {
-      if (fields) {
+        // Assign this to a variable instead of directly to the newQuery.schema so that a user
-        for (let key of Object.keys(response.schema)) {
+        // can change the table they're querying and have the schema update until they first
-          if (fields[key]) {
+        // edit it
-            response.schema[key] = fields[key]
+        autoSchema = response.schema
-          }
-        }
       }
-      fields = response.schema
+
-      currentTab = "JSON"
+      rows = response.rows
+
       notifications.success("Query executed successfully")
     } catch (error) {
       notifications.error(`Query Error: ${error.message}`)
+
+      if (!suppressErrors) {
+        throw error
+      }
+    } finally {
+      loading = false
     }
   }
 
-  // return the query.
   async function saveQuery() {
     try {
-      const response = await queries.save(query.datasourceId, query)
+      showSidePanel = true
-      saveId = response._id
+      loading = true
-
+      const response = await queries.save(newQuery.datasourceId, {
-      if (response?._rev) {
+        ...newQuery,
-        queryStr = JSON.stringify(query)
+        schema:
-      }
+          Object.keys(newQuery.schema).length === 0
+            ? autoSchema
+            : newQuery.schema,
+      })
 
       notifications.success("Query saved successfully")
       return response
     } catch (error) {
       notifications.error(error.message || "Error saving query")
+    } finally {
+      loading = false
     }
   }
+
+  function resetDependentFields() {
+    if (newQuery.fields.extra) {
+      newQuery.fields.extra = {}
+    }
+  }
+
+  function populateExtraQuery(extraQueryFields) {
+    newQuery.fields.extra = extraQueryFields
+  }
+
+  const handleScroll = e => {
+    scrolling = e.target.scrollTop !== 0
+  }
 </script>
 
-<Modal
+<QueryViewerSavePromptModal
-  bind:this={saveModal}
+  checkIsModified={() => checkIsModified(newQuery)}
-  on:hide={() => {
+  attemptSave={() => runQuery({ suppressErrors: false }).then(saveQuery)}
-    navigateTo = null
+/>
-  }}
+<div class="queryViewer">
->
+  <div class="main">
-  <ModalContent
+    <div class="header" class:scrolling>
-    title="You have unsaved changes"
+      <div class="title">
-    confirmText="Save and Continue"
-    cancelText="Discard Changes"
-    size="L"
-    onConfirm={async () => {
-      await saveQuery()
-      override = true
-      resumeNavigation()
-    }}
-    onCancel={async () => {
-      override = true
-      resumeNavigation()
-    }}
-  >
-    <Body>Leaving this section will mean losing and changes to your query</Body>
-  </ModalContent>
-</Modal>
-
-<div class="wrapper">
-  <Layout gap="S" noPadding>
-    <Heading size="M">Query {integrationInfo?.friendlyName}</Heading>
-    <Divider />
-    <Heading size="S">Config</Heading>
-    <div class="config">
-      <div class="config-field">
-        <Label>Query Name</Label>
-        <Input
-          value={query.name}
-          on:input={e => {
-            let newValue = e.target.value || ""
-            if (newValue.match(ValidQueryNameRegex)) {
-              query.name = newValue.trim()
-              nameError = null
-            } else {
-              nameError = "Invalid query name"
-            }
-          }}
-          error={nameError}
-        />
-      </div>
-      {#if queryConfig}
-        <div class="config-field">
-          <Label>Function</Label>
-          <Select
-            bind:value={query.queryVerb}
-            on:change={resetDependentFields}
-            options={Object.keys(queryConfig)}
-            getOptionLabel={verb =>
-              queryConfig[verb]?.displayName || capitalise(verb)}
-          />
-        </div>
-        <div class="config-field">
-          <AccessLevelSelect {saveId} {query} label="Access Level" />
-        </div>
-        {#if integrationInfo?.extra && query.queryVerb}
-          <ExtraQueryConfig
-            {query}
-            {populateExtraQuery}
-            config={integrationInfo.extra}
-          />
-        {/if}
-        {#key query.parameters}
-          <div class="binding-wrap">
-            <BindingBuilder
-              queryBindings={query.parameters}
-              bindable={false}
-              on:change={e => {
-                query.parameters = e.detail.map(binding => {
-                  return {
-                    name: binding.name,
-                    default: binding.value,
-                  }
-                })
-              }}
-            />
-          </div>
-        {/key}
-      {/if}
-    </div>
-    {#if shouldShowQueryConfig}
-      <Divider />
-      <div class="config">
-        <Heading size="S">Fields</Heading>
-        <Body size="S">Fill in the fields specific to this query.</Body>
-        <IntegrationQueryEditor
-          {datasource}
-          {query}
-          height={200}
-          schema={queryConfig[query.queryVerb]}
-          bind:parameters
-        />
-        <Divider />
-      </div>
-      <div class="config">
-        <div class="help-heading">
-          <Heading size="S">Transformer</Heading>
-          <Icon
-            on:click={() => window.open(transformerDocs)}
-            hoverable
-            name="Help"
-            size="L"
-          />
-        </div>
         <Body size="S">
-          Add a JavaScript function to transform the query result.
+          {newQuery.name || "Untitled query"}<span class="unsaved"
+            >{modified ? "*" : ""}</span
+          >
         </Body>
-        <CodeMirrorEditor
-          height={200}
-          label="Transformer"
-          value={query.transformer}
-          resize="vertical"
-          on:change={e => (query.transformer = e.detail)}
-        />
-        <Divider />
       </div>
-      <div class="viewer-controls">
+      <div class="controls">
-        <Heading size="S">Results</Heading>
+        <Button disabled={loading} on:click={runQuery} overBackground>
-        <ButtonGroup gap="S">
+          <Icon size="S" name="Play" />
+          Run query</Button
+        >
+        <div class="tooltip" title="Run your query to enable saving">
           <Button
-            cta
-            disabled={queryInvalid}
             on:click={async () => {
-              await saveQuery()
+              const response = await saveQuery()
-              // Go to the correct URL if we just created a new query
+
-              if (!query._rev) {
+              // When creating a new query the initally passed in query object will have no id.
-                $goto(`../../${query._id}`)
+              if (response._id && !newQuery._id) {
+                // Set the comparison query hash to match the new query so that the user doesn't
+                // get nagged when navigating to the edit view
+                queryHash = JSON.stringify(newQuery)
+                $goto(`../../${response._id}`)
               }
             }}
+            disabled={loading ||
+              !newQuery.name ||
+              nameError ||
+              rows.length === 0}
+            overBackground
           >
-            Save Query
+            <Icon size="S" name="SaveFloppy" />
+            Save
           </Button>
-          <Button secondary on:click={previewQuery}>Run Query</Button>
+        </div>
-        </ButtonGroup>
       </div>
-      <Body size="S">
+    </div>
-        Below, you can preview the results from your query and change the
+
-        schema.
+    <div class="body" on:scroll={handleScroll}>
-      </Body>
+      <div class="bodyInner">
-      <section class="viewer">
+        <div class="configField">
-        {#if data}
+          <Label>Name</Label>
-          <Tabs bind:selected={currentTab}>
+          <Input
-            <Tab title="JSON">
+            value={newQuery.name}
-              <JSONPreview data={data[0]} minHeight="120" />
+            on:input={e => {
-            </Tab>
+              let newValue = e.target.value || ""
-            <Tab title="Schema">
+              if (newValue.match(ValidQueryNameRegex)) {
-              <KeyValueBuilder
+                newQuery.name = newValue.trim()
-                bind:object={fields}
+                nameError = null
-                name="field"
+              } else {
-                headings
+                nameError = "Invalid query name"
-                options={SchemaTypeOptions}
+              }
+            }}
+            error={nameError}
+          />
+          {#if integration.query}
+            <Label>Function</Label>
+            <Select
+              bind:value={newQuery.queryVerb}
+              on:change={resetDependentFields}
+              options={Object.keys(integration.query)}
+              getOptionLabel={verb =>
+                integration.query[verb]?.displayName || capitalise(verb)}
+            />
+            <Label>Access</Label>
+            <AccessLevelSelect query={newQuery} />
+            {#if integration?.extra && newQuery.queryVerb}
+              <ExtraQueryConfig
+                query={newQuery}
+                {populateExtraQuery}
+                config={integration.extra}
               />
-            </Tab>
+            {/if}
-            <Tab title="Preview">
+          {/if}
-              <ExternalDataSourceTable {query} {data} />
+        </div>
-            </Tab>
+
-          </Tabs>
+        <Divider />
-        {/if}
+
-      </section>
+        <div class="heading">
-    {/if}
+          <Heading weight="L" size="XS">Query</Heading>
-  </Layout>
+        </div>
+        <div class="copy">
+          <Body size="S">
+            {#if schemaType === "sql"}
+              Add some SQL to query your data
+            {:else if schemaType === "json"}
+              Add some JSON to query your data
+            {:else if schemaType === "fields"}
+              Add some fields to query your data
+            {:else}
+              Enter your query below
+            {/if}
+          </Body>
+        </div>
+        <IntegrationQueryEditor
+          noLabel
+          {datasource}
+          bind:query={newQuery}
+          height={200}
+          schema={integration.query[newQuery.queryVerb]}
+        />
+
+        <Divider />
+
+        <div class="heading">
+          <Heading weight="L" size="XS">Bindings</Heading>
+        </div>
+        <div class="copy">
+          <Body size="S">
+            Bindings come in two parts: the binding name, and a default/fallback
+            value. These bindings can be used as Handlebars expressions
+            throughout the query.
+          </Body>
+        </div>
+        {#key newQuery.parameters}
+          <BindingBuilder
+            hideHeading
+            queryBindings={newQuery.parameters}
+            on:change={e => {
+              newQuery.parameters = e.detail.map(binding => {
+                return {
+                  name: binding.name,
+                  default: binding.value,
+                }
+              })
+            }}
+          />
+        {/key}
+
+        <Divider />
+        <div class="heading">
+          <Heading weight="L" size="XS">Transformer</Heading>
+        </div>
+        <div class="copy">
+          <Body size="S">
+            Add a JavaScript function to transform the query result.
+          </Body>
+        </div>
+        <CodeMirrorEditor
+          height={200}
+          value={newQuery.transformer}
+          resize="vertical"
+          on:change={e => (newQuery.transformer = e.detail)}
+        />
+      </div>
+    </div>
+  </div>
+
+  <div class:showSidePanel class="sidePanel">
+    <QueryViewerSidePanel
+      onClose={() => (showSidePanel = false)}
+      onSchemaChange={newSchema => {
+        newQuery.schema = newSchema
+      }}
+      {rows}
+      schema={Object.keys(newQuery.schema).length === 0
+        ? autoSchema
+        : newQuery.schema}
+    />
+  </div>
 </div>
 
 <style>
-  .wrapper {
+  .unsaved {
-    width: 640px;
+    color: var(--grey-5);
+    font-style: italic;
+  }
+
+  .queryViewer {
+    height: 100%;
+    margin: -28px -40px -40px -40px;
+    display: flex;
+    flex: 1;
+  }
+
+  .queryViewer :global(.spectrum-Divider) {
+    margin: 35px 0;
+  }
+
+  .main {
+    flex-grow: 1;
+    height: 100%;
+    display: flex;
+    flex-direction: column;
+  }
+
+  .header {
+    align-items: center;
+    padding: 8px 10px 8px 16px;
+    display: flex;
+    border-bottom: 2px solid transparent;
+    transition: border-bottom 130ms ease-out;
+  }
+
+  .header.scrolling {
+    border-bottom: var(--border-light);
+  }
+
+  .body {
+    flex-grow: 1;
+    overflow-y: scroll;
+    padding: 23px 23px 80px;
+    box-sizing: border-box;
+  }
+
+  .bodyInner {
+    max-width: 520px;
     margin: auto;
   }
 
-  .config {
+  .title {
-    display: grid;
+    /* width 0 paired with flex-grow necessary here for the truncation to work properly*/
-    grid-gap: var(--spacing-s);
+    width: 0;
-    z-index: 1;
+    flex-grow: 1;
   }
 
-  .config-field {
+  .title :global(p) {
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
+
+  .controls {
+    flex-shrink: 0;
+  }
+
+  .tooltip {
+    display: inline-block;
+  }
+
+  .controls :global(button) {
+    border: none;
+    color: var(--grey-7);
+    font-weight: 300;
+  }
+
+  .controls :global(button):hover {
+    background-color: transparent;
+    color: var(--ink);
+  }
+
+  .controls :global(.is-disabled) {
+    pointer-events: none;
+    background-color: transparent;
+    color: var(--grey-3);
+  }
+
+  .controls :global(span) {
+    display: flex;
+    align-items: center;
+  }
+
+  .controls :global(.icon) {
+    margin-right: 8px;
+  }
+
+  .configField {
     display: grid;
     grid-template-columns: 20% 1fr;
     grid-gap: var(--spacing-l);
     align-items: center;
   }
 
-  .help-heading {
+  .configField :global(label) {
-    display: flex;
+    color: var(--grey-6);
-    justify-content: space-between;
   }
 
-  .viewer {
+  .heading {
-    min-height: 200px;
+    margin-bottom: 8px;
-    width: 640px;
   }
 
-  .viewer-controls {
+  .copy {
-    display: flex;
+    margin-bottom: 14px;
-    flex-direction: row;
-    justify-content: space-between;
-    gap: var(--spacing-m);
-    min-width: 150px;
-    align-items: center;
   }
 
-  .binding-wrap :global(div.container) {
+  .copy :global(p) {
-    padding-left: 0px;
+    color: var(--grey-7);
-    padding-right: 0px;
+  }
+
+  .sidePanel {
+    flex-shrink: 0;
+    height: 100%;
+    width: 0;
+    overflow: hidden;
+    transition: width 150ms;
+  }
+
+  .sidePanel :global(.panel) {
+    height: 100%;
+  }
+
+  .showSidePanel {
+    width: 450px;
   }
 </style>

packages/builder/src/components/integration/QueryViewerBindingBuilder.svelte

@@ -1,9 +1,9 @@
 <script>
-  import { Body, Heading, Layout } from "@budibase/bbui"
   import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
   import { getUserBindings } from "builderStore/dataBinding"
   export let bindable = true
   export let queryBindings = []
+  export let hideHeading = false
 
   const userBindings = getUserBindings()
 
@@ -13,44 +13,16 @@
   }, {})
 </script>
 
-<Layout noPadding={bindable} gap="S">
+<KeyValueBuilder
-  <div class="controls" class:height={!bindable}>
+  bind:object={internalBindings}
-    <Heading size="XS">Bindings</Heading>
+  tooltip="Set the name of the binding which can be used in Handlebars statements throughout your query"
-  </div>
+  name="binding"
-  <Body size="S">
+  customButtonText="Bindings"
-    {#if !bindable}
+  headings
-      Bindings come in two parts: the binding name, and a default/fallback
+  keyPlaceholder="Binding name"
-      value. These bindings can be used as Handlebars expressions throughout the
+  valuePlaceholder="Default"
-      query.
+  bindings={[...userBindings]}
-    {:else}
+  bindingDrawerLeft="260px"
-      Enter a value for each binding. The default values will be used for any
+  allowHelpers={false}
-      values left blank.
+  on:change
-    {/if}
+/>
-  </Body>
-  <div class="bindings" class:bindable>
-    <KeyValueBuilder
-      bind:object={internalBindings}
-      tooltip="Set the name of the binding which can be used in Handlebars statements throughout your query"
-      name="binding"
-      headings
-      keyPlaceholder="Binding name"
-      valuePlaceholder="Default"
-      bindings={[...userBindings]}
-      bindingDrawerLeft="260px"
-      allowHelpers={false}
-      on:change
-    />
-  </div>
-</Layout>
-
-<style>
-  .controls {
-    display: flex;
-    align-items: center;
-    justify-content: space-between;
-  }
-
-  .height {
-    height: 40px;
-  }
-</style>

packages/builder/src/components/integration/QueryViewerSavePromptModal.svelte

@@ -0,0 +1,53 @@
+<script>
+  import { goto, beforeUrlChange } from "@roxi/routify"
+  import { Body, Modal, ModalContent } from "@budibase/bbui"
+
+  export let checkIsModified = () => {}
+  export let attemptSave = () => {}
+  let modal
+  let navigateTo
+  let override = false
+
+  $beforeUrlChange(event => {
+    if (checkIsModified() && !override) {
+      navigateTo = event.type == "pushstate" ? event.url : null
+      modal.show()
+      return false
+    } else return true
+  })
+
+  const resumeNavigation = () => {
+    if (typeof navigateTo == "string") {
+      $goto(typeof navigateTo == "string" ? `${navigateTo}` : navigateTo)
+    }
+  }
+</script>
+
+<Modal
+  bind:this={modal}
+  on:hide={() => {
+    navigateTo = null
+  }}
+>
+  <ModalContent
+    title="You have unsaved changes"
+    confirmText="Save and Continue"
+    cancelText="Discard Changes"
+    size="L"
+    onConfirm={async () => {
+      try {
+        await attemptSave()
+        override = true
+        resumeNavigation()
+      } catch (e) {
+        navigateTo = false
+      }
+    }}
+    onCancel={async () => {
+      override = true
+      resumeNavigation()
+    }}
+  >
+    <Body>Leaving this section will mean losing any changes to your query</Body>
+  </ModalContent>
+</Modal>

packages/builder/src/components/integration/QueryViewerSidePanel/JSONPanel.svelte

@@ -0,0 +1,22 @@
+<script>
+  export let data
+
+  $: string = JSON.stringify(data || {}, null, 2)
+</script>
+
+<textarea class="json" disabled value={string} />
+
+<style>
+  .json {
+    width: 100%;
+    height: 100%;
+    color: var(--ink);
+    padding: 12px;
+    box-sizing: border-box;
+    resize: none;
+    background-color: var(
+      --spectrum-textfield-m-background-color,
+      var(--spectrum-global-color-gray-50)
+    );
+  }
+</style>

packages/builder/src/components/integration/QueryViewerSidePanel/PreviewPanel.svelte

@@ -0,0 +1,33 @@
+<script>
+  import Table from "components/backend/DataTable/Table.svelte"
+  import { cloneDeep } from "lodash/fp"
+
+  export let schema = {}
+  export let rows = []
+
+  $: rowsCopy = cloneDeep(rows)
+
+  // Cast field in query preview response to number if specified by schema
+  $: {
+    for (let i = 0; i < rowsCopy.length; i++) {
+      let row = rowsCopy[i]
+      for (let fieldName of Object.keys(schema)) {
+        if (schema[fieldName] === "number" && !isNaN(Number(row[fieldName]))) {
+          row[fieldName] = Number(row[fieldName])
+        } else {
+          row[fieldName] = row[fieldName]?.toString()
+        }
+      }
+    }
+  }
+</script>
+
+<div class="table">
+  <Table {schema} data={rowsCopy} type="external" allowEditing={false} />
+</div>
+
+<style>
+  .table :global(.spectrum-Table-cell) {
+    min-width: 100px;
+  }
+</style>

packages/builder/src/components/integration/QueryViewerSidePanel/SchemaPanel.svelte

@@ -0,0 +1,29 @@
+<script>
+  import KeyValueBuilder from "../KeyValueBuilder.svelte"
+  import { SchemaTypeOptions } from "constants/backend"
+
+  export let schema
+  export let onSchemaChange = () => {}
+
+  const handleChange = e => {
+    let newSchema = {}
+
+    // KeyValueBuilder on change event returns an array of objects with each object
+    // containing a field
+    e.detail.forEach(({ name, value }) => {
+      newSchema[name] = value
+    })
+
+    onSchemaChange(newSchema)
+  }
+</script>
+
+{#key schema}
+  <KeyValueBuilder
+    on:change={handleChange}
+    object={schema}
+    name="field"
+    headings
+    options={SchemaTypeOptions}
+  />
+{/key}

packages/builder/src/components/integration/QueryViewerSidePanel/index.svelte

@@ -0,0 +1,66 @@
+<script>
+  import Panel from "components/design/Panel.svelte"
+  import { ActionButton } from "@budibase/bbui"
+  import JSONPanel from "./JSONPanel.svelte"
+  import SchemaPanel from "./SchemaPanel.svelte"
+  import PreviewPanel from "./PreviewPanel.svelte"
+
+  export let rows
+  export let schema
+  export let onSchemaChange = () => {}
+  export let onClose = () => {}
+
+  const tabs = ["JSON", "Schema", "Preview"]
+  let activeTab = "JSON"
+</script>
+
+<Panel
+  showCloseButton
+  closeButtonIcon="RailRightClose"
+  onClickCloseButton={onClose}
+  title="Query results"
+  icon={"SQLQuery"}
+  borderLeft
+  extraWide
+>
+  <div slot="panel-header-content">
+    <div class="settings-tabs">
+      {#each tabs as tab}
+        <ActionButton
+          size="M"
+          quiet
+          selected={activeTab === tab}
+          on:click={() => {
+            activeTab = tab
+          }}
+        >
+          {tab}
+        </ActionButton>
+      {/each}
+    </div>
+  </div>
+  <div class="content">
+    {#if activeTab === "JSON"}
+      <JSONPanel data={rows[0] || {}} />
+    {:else if activeTab === "Schema"}
+      <SchemaPanel {onSchemaChange} {schema} />
+    {:else}
+      <PreviewPanel {schema} {rows} />
+    {/if}
+  </div>
+</Panel>
+
+<style>
+  .settings-tabs {
+    display: flex;
+    gap: var(--spacing-s);
+    padding: 0 var(--spacing-l);
+    padding-bottom: var(--spacing-l);
+  }
+
+  .content {
+    padding: 14px;
+    height: 100%;
+    overflow: scroll;
+  }
+</style>

packages/builder/src/components/integration/index.svelte

@@ -23,6 +23,7 @@
   export let schema
   export let editable = true
   export let height = 500
+  export let noLabel = false
 
   let stepEditors = []
 
@@ -75,7 +76,7 @@
     {#if schema.type === QueryTypes.SQL}
       <Editor
         editorHeight={height}
-        label="Query"
+        label={noLabel ? null : "Query"}
         mode="sql"
         on:change={updateQuery}
         readOnly={!editable}
@@ -85,7 +86,7 @@
     {:else if shouldDisplayJsonBox}
       <Editor
         editorHeight={height}
-        label="Query"
+        label={noLabel ? null : "Query"}
         mode="json"
         on:change={updateQuery}
         readOnly={!editable}

packages/builder/src/components/start/ImportAppModal.svelte

@@ -0,0 +1,69 @@
+<script>
+  import {
+    ModalContent,
+    Toggle,
+    Input,
+    Layout,
+    Dropzone,
+    notifications,
+    Body,
+  } from "@budibase/bbui"
+  import { API } from "api"
+  import { automationStore, store } from "../../builderStore"
+
+  export let app
+
+  $: disabled = (encrypted && !password) || !file
+  let encrypted = false,
+    password
+  let file
+
+  async function updateApp() {
+    try {
+      let data = new FormData()
+      data.append("appExport", file)
+      if (encrypted) {
+        data.append("encryptionPassword", password.trim())
+      }
+      const appId = app.devId
+      await API.updateAppFromExport(appId, data)
+      const pkg = await API.fetchAppPackage(appId)
+      await store.actions.initialise(pkg)
+      await automationStore.actions.fetch()
+      notifications.success("App updated successfully")
+    } catch (err) {
+      notifications.error(`Failed to update app - ${err.message || err}`)
+    }
+  }
+</script>
+
+<ModalContent
+  title={`Update ${app.name}`}
+  confirmText="Update"
+  onConfirm={updateApp}
+  bind:disabled
+>
+  <Body size="S"
+    >Updating an app using an app export will replace all tables, datasources,
+    queries, screens and automations. It is recommended to perform a backup
+    before running this operation.</Body
+  >
+  <Layout noPadding gap="XS">
+    <Dropzone
+      gallery={false}
+      label="App export"
+      on:change={e => {
+        file = e.detail?.[0]
+      }}
+    />
+    <Toggle text="Encrypted" bind:value={encrypted} />
+    {#if encrypted}
+      <Input
+        type="password"
+        label="Password"
+        placeholder="Type here..."
+        bind:value={password}
+      />
+    {/if}
+  </Layout>
+</ModalContent>

packages/builder/src/constants/backend/index.js

@@ -120,6 +120,12 @@ export const FIELDS = {
       presence: false,
     },
   },
+  USER: {
+    name: "User",
+    type: "bb_reference",
+    subtype: "user",
+    icon: "User",
+  },
 }
 
 export const AUTO_COLUMN_SUB_TYPES = {
@@ -170,6 +176,11 @@ export const RelationshipType = {
   MANY_TO_ONE: "many-to-one",
 }
 
+export const PrettyRelationshipDefinitions = {
+  MANY: "Many rows",
+  ONE: "One row",
+}
+
 export const ALLOWABLE_STRING_OPTIONS = [
   FIELDS.STRING,
   FIELDS.OPTIONS,

packages/builder/src/pages/builder/app/[application]/data/query/new/[datasourceId]/index.svelte

@@ -16,6 +16,9 @@
 
   const buildNewQuery = isRestQuery => {
     let query = {
+      name: "Untitled query",
+      transformer: "return data",
+      schema: {},
       datasourceId: $params.datasourceId,
       parameters: [],
       fields: {},

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json

@@ -69,7 +69,8 @@
       "datetimefield",
       "multifieldselect",
       "s3upload",
-      "codescanner"
+      "codescanner",
+      "bbreferencefield"
     ]
   },
   {

packages/builder/src/pages/builder/app/[application]/settings/_layout.svelte

@@ -32,9 +32,9 @@
             active={$isActive("./embed")}
           />
           <SideNavItem
-            text="Export"
+            text="Export/Import"
-            url={$url("./export")}
+            url={$url("./exportImport")}
-            active={$isActive("./export")}
+            active={$isActive("./exportImport")}
           />
           <SideNavItem
             text="Name and URL"

packages/builder/src/pages/builder/app/[application]/settings/exportImport.svelte

@@ -11,31 +11,39 @@
   import { apps } from "stores/portal"
   import { store } from "builderStore"
   import ExportAppModal from "components/start/ExportAppModal.svelte"
+  import ImportAppModal from "components/start/ImportAppModal.svelte"
 
   $: filteredApps = $apps.filter(app => app.devId == $store.appId)
   $: app = filteredApps.length ? filteredApps[0] : {}
   $: appDeployed = app?.status === AppStatus.DEPLOYED
 
-  let exportModal
+  let exportModal, importModal
   let exportPublishedVersion = false
 
   const exportApp = opts => {
     exportPublishedVersion = !!opts?.published
     exportModal.show()
   }
+
+  const importApp = () => {
+    importModal.show()
+  }
 </script>
 
 <Modal bind:this={exportModal} padding={false}>
   <ExportAppModal {app} published={exportPublishedVersion} />
 </Modal>
 
+<Modal bind:this={importModal} padding={false}>
+  <ImportAppModal {app} />
+</Modal>
+
 <Layout noPadding>
   <Layout gap="XS" noPadding>
     <Heading>Export your app</Heading>
     <Body>Export your latest edited or published app</Body>
   </Layout>
-  <Divider />
+  <div class="body">
-  <div class="export-body">
     <ActionButton secondary on:click={() => exportApp({ published: false })}>
       Export latest edited app
     </ActionButton>
@@ -47,10 +55,20 @@
       Export latest published app
     </ActionButton>
   </div>
+  <Divider />
+  <Layout gap="XS" noPadding>
+    <Heading>Import your app</Heading>
+    <Body>Import an export to update this app</Body>
+  </Layout>
+  <div class="body">
+    <ActionButton secondary on:click={() => importApp()}>
+      Import app
+    </ActionButton>
+  </div>
 </Layout>
 
 <style>
-  .export-body {
+  .body {
     display: flex;
     gap: var(--spacing-l);
   }

packages/builder/src/pages/builder/portal/_layout.svelte

@@ -101,7 +101,7 @@
     justify-content: flex-start;
     align-items: center;
     border-bottom: var(--border-light);
-    padding: 0 24px;
+    padding: 0 var(--spacing-l);
     gap: 24px;
     position: relative;
   }

packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte

@@ -86,8 +86,16 @@
   $: userPage = $userPageInfo.page
   $: logsPage = $logsPageInfo.page
 
+  let usersObj = {}
+  $: usersObj = {
+    ...usersObj,
+    ...$users.data?.reduce((accumulator, user) => {
+      accumulator[user._id] = user
+      return accumulator
+    }, {}),
+  }
   $: sortedUsers = sort(
-    enrich($users.data || [], selectedUsers, "_id"),
+    enrich(Object.values(usersObj), selectedUsers, "_id"),
     "email"
   )
   $: sortedEvents = sort(
@@ -256,8 +264,7 @@
   <div class="controls">
     <div class="select">
       <Multiselect
-        bind:fetchTerm={userSearchTerm}
+        bind:searchTerm={userSearchTerm}
-        useFetch
         placeholder="All users"
         label="Users"
         autocomplete

packages/builder/src/pages/builder/portal/apps/[appId]/_layout.svelte

@@ -0,0 +1,15 @@
+<script>
+  import { params, redirect } from "@roxi/routify"
+  import { apps } from "stores/portal"
+
+  $: app = $apps.find(app => app.appId === $params.appId)
+  $: {
+    if (!app) {
+      $redirect("../")
+    }
+  }
+</script>
+
+{#if app}
+  <slot />
+{/if}

packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte

@@ -0,0 +1,79 @@
+<script>
+  import { params, goto } from "@roxi/routify"
+  import { apps, auth, sideBarCollapsed } from "stores/portal"
+  import { ActionButton } from "@budibase/bbui"
+  import { sdk } from "@budibase/shared-core"
+
+  $: app = $apps.find(app => app.appId === $params.appId)
+  $: iframeUrl = getIframeURL(app)
+  $: isBuilder = sdk.users.isBuilder($auth.user, app?.devId)
+
+  const getIframeURL = app => {
+    if (app.status === "published") {
+      return `/app${app.url}`
+    }
+    return `/${app.devId}`
+  }
+</script>
+
+<div class="container">
+  <div class="header">
+    {#if $sideBarCollapsed}
+      <ActionButton
+        quiet
+        icon="Rail"
+        on:click={() => sideBarCollapsed.set(false)}
+      >
+        Menu
+      </ActionButton>
+    {:else}
+      <ActionButton
+        quiet
+        icon="RailRightOpen"
+        on:click={() => sideBarCollapsed.set(true)}
+      >
+        Collapse
+      </ActionButton>
+    {/if}
+    {#if isBuilder}
+      <ActionButton
+        quiet
+        icon="Edit"
+        on:click={() => $goto(`/builder/app/${app.devId}`)}
+      >
+        Edit
+      </ActionButton>
+    {/if}
+    <ActionButton
+      quiet
+      icon="LinkOut"
+      on:click={() => window.open(iframeUrl, "_blank")}
+    >
+      Fullscreen
+    </ActionButton>
+  </div>
+  <iframe src={iframeUrl} title={app.name} />
+</div>
+
+<style>
+  .container {
+    flex: 1 1 auto;
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: stretch;
+    padding: 0 var(--spacing-l) var(--spacing-l) var(--spacing-l);
+  }
+  .header {
+    display: flex;
+    justify-content: flex-start;
+    align-items: center;
+    gap: var(--spacing-xs);
+    flex: 0 0 50px;
+  }
+  iframe {
+    flex: 1 1 auto;
+    border-radius: var(--spacing-s);
+    border: 1px solid var(--spectrum-global-color-gray-300);
+  }
+</style>

packages/builder/src/pages/builder/portal/apps/_components/PortalSideBar.svelte

@@ -0,0 +1,140 @@
+<script>
+  import { Icon, Body } from "@budibase/bbui"
+  import { apps, sideBarCollapsed } from "stores/portal"
+  import { params, goto } from "@roxi/routify"
+  import { tick } from "svelte"
+  import NavItem from "components/common/NavItem.svelte"
+
+  let searchInput
+  let searchString
+  let searching = false
+
+  $: filteredApps = $apps.filter(app => {
+    return (
+      !searchString ||
+      app.name.toLowerCase().includes(searchString.toLowerCase())
+    )
+  })
+
+  const startSearching = async () => {
+    searching = true
+    searchString = ""
+    await tick()
+    searchInput.focus()
+  }
+
+  const stopSearching = () => {
+    searching = false
+    searchString = ""
+  }
+</script>
+
+<div class="side-bar" class:collapsed={$sideBarCollapsed}>
+  <div class="side-bar-controls">
+    {#if searching}
+      <input
+        bind:this={searchInput}
+        bind:value={searchString}
+        placeholder="Search for apps"
+      />
+    {:else}
+      <Body size="S">Apps</Body>
+      <Icon name="Search" size="S" hoverable on:click={startSearching} />
+    {/if}
+    <div class="rotational" class:rotated={searching}>
+      <Icon
+        name="Add"
+        hoverable
+        on:click={searching ? stopSearching : () => $goto("./create")}
+      />
+    </div>
+  </div>
+  <div class="side-bar-nav">
+    <NavItem
+      icon="WebPages"
+      text="All apps"
+      on:click={() => $goto("./")}
+      selected={!$params.appId}
+    />
+    {#each filteredApps as app}
+      <NavItem
+        text={app.name}
+        icon={app.icon?.name || "Apps"}
+        iconColor={app.icon?.color}
+        selected={$params.appId === app.appId}
+        on:click={() => $goto(`./${app.appId}`)}
+      />
+    {/each}
+  </div>
+</div>
+
+<style>
+  .side-bar {
+    flex: 0 0 260px;
+    display: flex;
+    flex-direction: column;
+    align-items: stretch;
+    border-right: var(--border-light);
+    background: var(--spectrum-global-color-gray-100);
+    overflow: hidden;
+    transition: margin-left 300ms ease-out;
+  }
+  .side-bar.collapsed {
+    margin-left: -262px;
+  }
+  @media (max-width: 640px) {
+    .side-bar {
+      margin-left: -262px;
+    }
+  }
+
+  .side-bar-controls {
+    flex: 0 0 50px;
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    gap: var(--spacing-l);
+    padding: 0 var(--spacing-l);
+  }
+  .side-bar-controls :global(.spectrum-Body),
+  .side-bar-controls input {
+    flex: 1 1 auto;
+  }
+  .side-bar-controls :global(.spectrum-Icon) {
+    color: var(--spectrum-global-color-gray-700);
+  }
+
+  input {
+    outline: none;
+    border: none;
+    max-width: none;
+    flex: 1 1 auto;
+    color: var(--spectrum-global-color-gray-800);
+    font-size: 14px;
+    padding: 0;
+    transition: border 130ms ease-out;
+    font-family: var(--font-sans);
+    background: inherit;
+  }
+  input::placeholder {
+    color: var(--spectrum-global-color-gray-700);
+    transition: color 130ms ease-out;
+  }
+  input:hover::placeholder {
+    color: var(--spectrum-global-color-gray-800);
+  }
+
+  .side-bar-nav {
+    flex: 1 1 auto;
+    overflow: auto;
+    overflow-x: hidden;
+  }
+
+  div.rotational {
+    transition: transform 130ms ease-out;
+  }
+  div.rotational.rotated {
+    transform: rotate(45deg);
+  }
+</style>

packages/builder/src/pages/builder/portal/apps/_layout.svelte

@@ -11,6 +11,7 @@
   import { onMount } from "svelte"
   import { redirect } from "@roxi/routify"
   import { sdk } from "@budibase/shared-core"
+  import PortalSideBar from "./_components/PortalSideBar.svelte"
 
   // Don't block loading if we've already hydrated state
   let loaded = $apps.length != null
@@ -44,5 +45,18 @@
 </script>
 
 {#if loaded}
-  <slot />
+  <div class="page">
+    <PortalSideBar />
+    <slot />
+  </div>
 {/if}
+
+<style>
+  .page {
+    flex: 1 1 auto;
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: stretch;
+  }
+</style>

packages/builder/src/stores/portal/index.js

@@ -1,3 +1,5 @@
+import { writable } from "svelte/store"
+
 export { organisation } from "./organisation"
 export { users } from "./users"
 export { admin } from "./admin"
@@ -14,3 +16,5 @@ export { environment } from "./environment"
 export { menu } from "./menu"
 export { auditLogs } from "./auditLogs"
 export { features } from "./features"
+
+export const sideBarCollapsed = writable(false)

packages/cli/package.json

@@ -62,6 +62,6 @@
     "eslint": "^7.20.0",
     "renamer": "^4.0.0",
     "ts-node": "^10.9.1",
-    "typescript": "4.7.3"
+    "typescript": "5.2.2"
   }
 }

packages/cli/src/plugins/index.ts

@@ -49,8 +49,8 @@ async function askAboutTopLevel(name: string) {
   }
 }
 
-async function init(opts: PluginOpts) {
+async function init(opts: PluginOpts | PluginType) {
-  const type = opts["init"] || (opts as PluginType)
+  const type = (opts as PluginOpts).init || (opts as PluginType)
   if (!type || !PLUGIN_TYPE_ARR.includes(type)) {
     console.log(
       error(

packages/client/manifest.json

@@ -5623,5 +5623,73 @@
         "defaultValue": false
       }
     ]
+  },
+  "bbreferencefield": {
+    "devComment": "As bb reference is only used for user subtype for now, we are using user for icon and labels",
+    "name": "User Field",
+    "icon": "User",
+    "styles": ["size"],
+    "requiredAncestors": ["form"],
+    "editable": true,
+    "size": {
+      "width": 400,
+      "height": 50
+    },
+    "settings": [
+      {
+        "type": "field/bb_reference",
+        "label": "Field",
+        "key": "field",
+        "required": true
+      },
+      {
+        "type": "text",
+        "label": "Label",
+        "key": "label"
+      },
+      {
+        "type": "text",
+        "label": "Placeholder",
+        "key": "placeholder"
+      },
+      {
+        "type": "text",
+        "label": "Default value",
+        "key": "defaultValue"
+      },
+      {
+        "type": "event",
+        "label": "On change",
+        "key": "onChange",
+        "context": [
+          {
+            "label": "Field Value",
+            "key": "value"
+          }
+        ]
+      },
+      {
+        "type": "validation/link",
+        "label": "Validation",
+        "key": "validation"
+      },
+      {
+        "type": "filter/relationship",
+        "label": "Filtering",
+        "key": "filter"
+      },
+      {
+        "type": "boolean",
+        "label": "Search",
+        "key": "autocomplete",
+        "defaultValue": true
+      },
+      {
+        "type": "boolean",
+        "label": "Disabled",
+        "key": "disabled",
+        "defaultValue": false
+      }
+    ]
   }
 }

packages/client/src/components/app/blocks/form/InnerFormBlock.svelte

@@ -30,6 +30,7 @@
     link: "relationshipfield",
     json: "jsonfield",
     barcodeqr: "codescanner",
+    bb_reference: "bbreferencefield",
   }
 
   let formId

packages/client/src/components/app/forms/BBReferenceField.svelte

@@ -0,0 +1,9 @@
+<script>
+  import RelationshipField from "./RelationshipField.svelte"
+</script>
+
+<RelationshipField
+  {...$$props}
+  datasourceType={"user"}
+  primaryDisplay={"email"}
+/>

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -1,11 +1,6 @@
 <script>
-  import {
+  import { CoreSelect, CoreMultiselect } from "@budibase/bbui"
-    CoreSelect,
+  import { fetchData } from "@budibase/frontend-core"
-    CoreMultiselect,
-    Input,
-    ProgressCircle,
-  } from "@budibase/bbui"
-  import { fetchData, Utils } from "@budibase/frontend-core"
   import { getContext } from "svelte"
   import Field from "./Field.svelte"
   import { FieldTypes } from "../../../constants"
@@ -21,28 +16,25 @@
   export let defaultValue
   export let onChange
   export let filter
+  export let datasourceType = "table"
+  export let primaryDisplay
 
   let fieldState
   let fieldApi
   let fieldSchema
   let tableDefinition
-  let primaryDisplay
+  let searchTerm
-  let options
+  let open
-  let selectedOptions = []
-  let isOpen = false
-  let hasFilter
 
-  let searchResults
+  $: type =
-  let searchString
+    datasourceType === "table" ? FieldTypes.LINK : FieldTypes.BB_REFERENCE
-  let searching = false
-  let lastSearchId
 
   $: multiselect = fieldSchema?.relationshipType !== "one-to-many"
   $: linkedTableId = fieldSchema?.tableId
   $: fetch = fetchData({
     API,
     datasource: {
-      type: "table",
+      type: datasourceType,
       tableId: linkedTableId,
     },
     options: {
@@ -50,54 +42,74 @@
       limit: 100,
     },
   })
-  $: hasFilter = !!filter?.filter(f => !!f.field)?.length
+
-  $: fetch.update({ filter })
-  $: {
-    options = searchResults ? searchResults : $fetch.rows
-    const nonMatchingOptions = selectedOptions.filter(
-      option => !options.map(opt => opt._id).includes(option._id)
-    )
-    // Append initially selected options if there is no filter
-    // and hasn't already been appended
-    if (!hasFilter) {
-      options = [...options, ...nonMatchingOptions]
-    }
-  }
   $: tableDefinition = $fetch.definition
-  $: primaryDisplay = tableDefinition?.primaryDisplay || "_id"
+  $: selectedValue = multiselect
-  $: singleValue = flatten(fieldState?.value)?.[0]
+    ? flatten(fieldState?.value) ?? []
-  $: multiValue = flatten(fieldState?.value) ?? []
+    : flatten(fieldState?.value)?.[0]
   $: component = multiselect ? CoreMultiselect : CoreSelect
   $: expandedDefaultValue = expand(defaultValue)
-  $: debouncedSearch(searchString)
+  $: primaryDisplay = primaryDisplay || tableDefinition?.primaryDisplay
+
+  let optionsObj = {}
+  let initialValuesProcessed
+
   $: {
-    if (searching) {
+    if (!initialValuesProcessed && primaryDisplay) {
-      isOpen = true
+      // Persist the initial values as options, allowing them to be present in the dropdown,
+      // even if they are not in the inital fetch results
+      initialValuesProcessed = true
+      optionsObj = (fieldState?.value || []).reduce((accumulator, value) => {
+        accumulator[value._id] = {
+          _id: value._id,
+          [primaryDisplay]: value.primaryDisplay,
+        }
+        return accumulator
+      }, optionsObj)
     }
   }
 
-  // Fetch the initially selected values
+  $: enrichedOptions = enrichOptions(optionsObj, $fetch.rows)
-  // as they may not be within the first 100 records
+  const enrichOptions = (optionsObj, fetchResults) => {
+    const result = (fetchResults || [])?.reduce((accumulator, row) => {
+      if (!accumulator[row._id]) {
+        accumulator[row._id] = row
+      }
+      return accumulator
+    }, optionsObj)
+
+    return Object.values(result)
+  }
   $: {
-    if (
+    // We don't want to reorder while the dropdown is open, to avoid UX jumps
-      primaryDisplay !== "_id" &&
+    if (!open) {
-      fieldState?.value?.length &&
+      enrichedOptions = enrichedOptions.sort((a, b) => {
-      !selectedOptions?.length
+        const selectedValues = flatten(fieldState?.value) || []
-    ) {
+
-      API.searchTable({
+        const aIsSelected = selectedValues.find(v => v === a._id)
-        paginate: false,
+        const bIsSelected = selectedValues.find(v => v === b._id)
-        tableId: linkedTableId,
+        if (aIsSelected && !bIsSelected) {
-        limit: 100,
+          return -1
-        query: {
+        } else if (!aIsSelected && bIsSelected) {
-          oneOf: {
+          return 1
-            [`1:${primaryDisplay}`]: fieldState?.value?.map(
+        }
-              value => value.primaryDisplay
+
-            ),
+        return a[primaryDisplay] > b[primaryDisplay]
-          },
+      })
-        },
+    }
-      }).then(response => {
+  }
-        const value = multiselect ? multiValue : singleValue
+
-        selectedOptions = response.rows.filter(row => value.includes(row._id))
+  $: fetchRows(searchTerm, primaryDisplay)
+
+  const fetchRows = (searchTerm, primaryDisplay) => {
+    const allRowsFetched =
+      $fetch.loaded &&
+      !Object.keys($fetch.query?.string || {}).length &&
+      !$fetch.hasNextPage
+    // Don't request until we have the primary display
+    if (!allRowsFetched && primaryDisplay) {
+      fetch.update({
+        query: { string: { [primaryDisplay]: searchTerm } },
       })
     }
   }
@@ -113,7 +125,7 @@
   }
 
   const getDisplayName = row => {
-    return row?.[tableDefinition?.primaryDisplay || "_id"] || "-"
+    return row?.[primaryDisplay] || "-"
   }
 
   const singleHandler = e => {
@@ -136,66 +148,16 @@
 
   const handleChange = value => {
     const changed = fieldApi.setValue(value)
-    selectedOptions = value.map(val => ({
-      _id: val,
-      [primaryDisplay]: options.find(option => option._id === val)[
-        primaryDisplay
-      ],
-    }))
     if (onChange && changed) {
       onChange({ value })
     }
   }
 
-  // Search for rows based on the search string
+  const loadMore = () => {
-  const search = async searchString => {
+    if (!$fetch.loading) {
-    // Reset state if this search is invalid
+      fetch.nextPage()
-    if (!linkedTableId || !searchString) {
-      searchResults = null
-      return
     }
-
-    // If a filter exists, then do a client side search
-    if (hasFilter) {
-      searchResults = $fetch.rows.filter(option =>
-        option[primaryDisplay].startsWith(searchString)
-      )
-      isOpen = true
-      return
-    }
-
-    // Search for results, using IDs to track invocations and ensure we're
-    // handling the latest update
-    lastSearchId = Math.random()
-    searching = true
-    const thisSearchId = lastSearchId
-    const results = await API.searchTable({
-      paginate: false,
-      tableId: linkedTableId,
-      limit: 100,
-      query: {
-        string: {
-          [`1:${primaryDisplay}`]: searchString || "",
-        },
-      },
-    })
-    searching = false
-
-    // In case searching takes longer than our debounced update, abandon these
-    // results
-    if (thisSearchId !== lastSearchId) {
-      return
-    }
-
-    // Process results
-    searchResults = results.rows?.map(row => ({
-      ...row,
-      primaryDisplay: row[primaryDisplay],
-    }))
   }
-
-  // Debounced version of searching
-  const debouncedSearch = Utils.debounce(search, 250)
 </script>
 
 <Field
@@ -204,69 +166,29 @@
   {disabled}
   {validation}
   defaultValue={expandedDefaultValue}
-  type={FieldTypes.LINK}
+  {type}
   bind:fieldState
   bind:fieldApi
   bind:fieldSchema
 >
   {#if fieldState}
-    <div class={autocomplete ? "field-with-search" : ""}>
+    <svelte:component
-      <svelte:component
+      this={component}
-        this={component}
+      options={enrichedOptions}
-        bind:open={isOpen}
+      {autocomplete}
-        {options}
+      value={selectedValue}
-        autocomplete={false}
+      on:change={multiselect ? multiHandler : singleHandler}
-        value={multiselect ? multiValue : singleValue}
+      on:loadMore={loadMore}
-        on:change={multiselect ? multiHandler : singleHandler}
+      id={fieldState.fieldId}
-        id={fieldState.fieldId}
+      disabled={fieldState.disabled}
-        disabled={fieldState.disabled}
+      error={fieldState.error}
-        error={fieldState.error}
+      getOptionLabel={getDisplayName}
-        getOptionLabel={getDisplayName}
+      getOptionValue={option => option._id}
-        getOptionValue={option => option._id}
+      {placeholder}
-        {placeholder}
+      bind:searchTerm
-        customPopoverOffsetBelow={autocomplete ? 32 : null}
+      loading={$fetch.loading}
-        customPopoverMaxHeight={autocomplete ? 240 : null}
+      bind:open
-        sort={true}
+      customPopoverMaxHeight={400}
-      />
+    />
-      {#if autocomplete}
-        <div class="search">
-          <Input
-            autofocus
-            quiet
-            type="text"
-            bind:value={searchString}
-            placeholder={primaryDisplay ? `Search by ${primaryDisplay}` : null}
-          />
-          {#if searching}
-            <div>
-              <ProgressCircle size="S" />
-            </div>
-          {/if}
-        </div>
-      {/if}
-    </div>
   {/if}
 </Field>
-
-<style>
-  .search {
-    flex: 0 0 calc(var(--default-row-height) - 1px);
-    display: flex;
-    align-items: center;
-    margin: 4px var(--cell-padding);
-    width: calc(100% - 2 * var(--cell-padding));
-  }
-  .search :global(.spectrum-Textfield) {
-    min-width: 0;
-    width: 100%;
-  }
-  .search :global(.spectrum-Textfield-input) {
-    font-size: 13px;
-  }
-  .search :global(.spectrum-Form-item) {
-    flex: 1 1 auto;
-  }
-  .field-with-search {
-    min-height: 80px;
-  }
-</style>

packages/client/src/components/app/forms/index.js

@@ -15,3 +15,4 @@ export { default as formstep } from "./FormStep.svelte"
 export { default as jsonfield } from "./JSONField.svelte"
 export { default as s3upload } from "./S3Upload.svelte"
 export { default as codescanner } from "./CodeScannerField.svelte"
+export { default as bbreferencefield } from "./BBReferenceField.svelte"

packages/client/src/constants.js

@@ -1,17 +1,5 @@
-export const FieldTypes = {
+import { FieldType as FieldTypes } from "@budibase/types"
-  STRING: "string",
+export { FieldType as FieldTypes } from "@budibase/types"
-  BARCODEQR: "barcodeqr",
-  LONGFORM: "longform",
-  OPTIONS: "options",
-  NUMBER: "number",
-  BOOLEAN: "boolean",
-  ARRAY: "array",
-  DATETIME: "datetime",
-  ATTACHMENT: "attachment",
-  LINK: "link",
-  FORMULA: "formula",
-  JSON: "json",
-}
 
 export const UnsortableTypes = [
   FieldTypes.FORMULA,

packages/frontend-core/src/api/app.js

@@ -1,3 +1,5 @@
+import { sdk } from "@budibase/shared-core"
+
 export const buildAppEndpoints = API => ({
   /**
    * Fetches screen definition for an app.
@@ -81,6 +83,22 @@ export const buildAppEndpoints = API => ({
     })
   },
 
+  /**
+   * Update an application using an export - the body
+   * should be of type FormData, with a "file" and a "password" if encrypted.
+   * @param appId The ID of the app to update - this will always be
+   * converted to development ID.
+   * @param body a FormData body with a file and password.
+   */
+  updateAppFromExport: async (appId, body) => {
+    const devId = sdk.applications.getDevAppID(appId)
+    return await API.post({
+      url: `/api/applications/${devId}/import`,
+      body,
+      json: false,
+    })
+  },
+
   /**
    * Imports an export of all apps.
    * @param apps the FormData containing the apps to import

packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte

@@ -0,0 +1,38 @@
+<script>
+  import { getContext } from "svelte"
+  import RelationshipCell from "./RelationshipCell.svelte"
+  import { FieldSubtype } from "@budibase/types"
+
+  const { API } = getContext("grid")
+  const { subtype } = $$props.schema
+
+  const schema = {
+    ...$$props.schema,
+    // This is not really used, just adding some content to be able to render the relationship cell
+    tableId: "external",
+  }
+
+  async function searchFunction(searchParams) {
+    if (subtype !== FieldSubtype.USER) {
+      throw `Search for '${subtype}' not implemented`
+    }
+
+    const results = await API.searchUsers({
+      ...searchParams,
+    })
+
+    // Mapping to the expected data within RelationshipCell
+    return {
+      ...results,
+      data: undefined,
+      rows: results.data,
+    }
+  }
+</script>
+
+<RelationshipCell
+  {...$$props}
+  {schema}
+  {searchFunction}
+  primaryDisplay={"email"}
+/>

packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte

@@ -21,6 +21,8 @@
   import { Icon, Input, ProgressCircle, clickOutside } from "@budibase/bbui"
   import { debounce } from "../../../utils/utils"
 
+  const { API, dispatch } = getContext("grid")
+
   export let value
   export let api
   export let readonly
@@ -30,15 +32,15 @@
   export let invertX = false
   export let invertY = false
   export let contentLines = 1
+  export let searchFunction = API.searchTable
+  export let primaryDisplay
 
-  const { API, dispatch } = getContext("grid")
   const color = getColor(0)
 
   let isOpen = false
   let searchResults
   let searchString
   let lastSearchString
-  let primaryDisplay
   let candidateIndex
   let lastSearchId
   let searching = false
@@ -96,7 +98,7 @@
     lastSearchId = Math.random()
     searching = true
     const thisSearchId = lastSearchId
-    const results = await API.searchTable({
+    const results = await searchFunction({
       paginate: false,
       tableId: schema.tableId,
       limit: 20,
@@ -259,14 +261,16 @@
       on:wheel={e => (focused ? e.stopPropagation() : null)}
     >
       {#each value || [] as relationship}
-        {#if relationship.primaryDisplay}
+        {#if relationship[primaryDisplay] || relationship.primaryDisplay}
           <div class="badge">
             <span
               on:click={editable
                 ? () => showRelationship(relationship._id)
                 : null}
             >
-              {readable(relationship.primaryDisplay)}
+              {readable(
+                relationship[primaryDisplay] || relationship.primaryDisplay
+              )}
             </span>
             {#if editable}
               <Icon

packages/frontend-core/src/components/grid/cells/TextCell.svelte

@@ -45,7 +45,7 @@
     on:focus={() => (active = true)}
     on:blur={() => (active = false)}
     {type}
-    value={value || ""}
+    value={value ?? ""}
     on:change={handleChange}
     spellcheck="false"
   />

packages/frontend-core/src/components/grid/lib/renderers.js

@@ -9,6 +9,7 @@ import BooleanCell from "../cells/BooleanCell.svelte"
 import FormulaCell from "../cells/FormulaCell.svelte"
 import JSONCell from "../cells/JSONCell.svelte"
 import AttachmentCell from "../cells/AttachmentCell.svelte"
+import BBReferenceCell from "../cells/BBReferenceCell.svelte"
 
 const TypeComponentMap = {
   text: TextCell,
@@ -23,6 +24,7 @@ const TypeComponentMap = {
   link: RelationshipCell,
   formula: FormulaCell,
   json: JSONCell,
+  bb_reference: BBReferenceCell,
 }
 export const getCellRenderer = column => {
   return TypeComponentMap[column?.schema?.type] || TextCell

packages/frontend-core/src/components/grid/lib/utils.js

@@ -19,12 +19,21 @@ const TypeIconMap = {
   formula: "Calculator",
   json: "Brackets",
   bigint: "TagBold",
+  bb_reference: {
+    user: "User",
+  },
 }
 
 export const getColumnIcon = column => {
   if (column.schema.autocolumn) {
     return "MagicWand"
   }
-  const type = column.schema.type
+  const { type, subtype } = column.schema
-  return TypeIconMap[type] || "Text"
+
+  const result =
+    typeof TypeIconMap[type] === "object" && subtype
+      ? TypeIconMap[type][subtype]
+      : TypeIconMap[type]
+
+  return result || "Text"
 }

packages/pro

@@ -1 +1 @@
-Subproject commit 4638ae916e55ce89166095578cbd01745d0ee9ee
+Subproject commit 30385682141e5ba9d98de7d71d5be1672109cd15

packages/server/package.json

@@ -153,7 +153,6 @@
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.12",
     "@types/tar": "6.1.5",
-    "@typescript-eslint/parser": "5.45.0",
     "apidoc": "0.50.4",
     "babel-jest": "29.6.2",
     "copyfiles": "2.4.1",
@@ -174,7 +173,7 @@
     "timekeeper": "2.2.0",
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
-    "typescript": "4.7.3",
+    "typescript": "5.2.2",
     "update-dotenv": "1.1.1"
   },
   "optionalDependencies": {

packages/server/scripts/integrations/postgres/init.sql

@@ -9,6 +9,7 @@ CREATE TABLE Persons (
     Address varchar(255),
     City varchar(255) DEFAULT 'Belfast',
     Age INTEGER DEFAULT 20 NOT NULL,
+    Year INTEGER,
     Type person_job
 );
 CREATE TABLE Tasks (
@@ -49,9 +50,10 @@ CREATE TABLE CompositeTable (
   Name varchar(255),
   PRIMARY KEY (KeyPartOne, KeyPartTwo)
 );
-INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('Mike', 'Hughes', '123 Fake Street', 'Belfast', 'qa');
+INSERT INTO Persons (FirstName, LastName, Address, City, Type, Year) VALUES ('Mike', 'Hughes', '123 Fake Street', 'Belfast', 'qa', 1999);
-INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('John', 'Smith', '64 Updown Road', 'Dublin', 'programmer');
+INSERT INTO Persons (FirstName, LastName, Address, City, Type, Year) VALUES ('John', 'Smith', '64 Updown Road', 'Dublin', 'programmer', 1996);
-INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age) VALUES ('Foo', 'Bar', 'Foo Street', 'Bartown', 'support', 0);
+INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age, Year) VALUES ('Foo', 'Bar', 'Foo Street', 'Bartown', 'support', 0, 1993);
+INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('Jonny', 'Muffin', 'Muffin Street', 'Cork', 'support');
 INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (1, 2, 'assembling', TRUE);
 INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (2, 1, 'processing', FALSE);
 INSERT INTO Products (ProductName) VALUES ('Computers');

packages/server/scripts/test.sh

@@ -3,9 +3,8 @@ set -e
 
 if [[ -n $CI ]]
 then
-  # --runInBand performs better in ci where resources are limited
+  # Running in ci, where resources are limited
   export NODE_OPTIONS="--max-old-space-size=4096"
-  node ../../node_modules/jest/bin/jest.js --version
   echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail"
   jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail
 else

packages/server/specs/openapi.json

@@ -613,6 +613,23 @@
           "data"
         ]
       },
+      "appExport": {
+        "type": "object",
+        "properties": {
+          "encryptPassword": {
+            "description": "An optional password used to encrypt the export.",
+            "type": "string"
+          },
+          "excludeRows": {
+            "description": "Set whether the internal table rows should be excluded from the export.",
+            "type": "boolean"
+          }
+        },
+        "required": [
+          "encryptPassword",
+          "excludeRows"
+        ]
+      },
       "row": {
         "description": "The row to be created/updated, based on the table schema.",
         "type": "object",
@@ -2163,6 +2180,87 @@
         }
       }
     },
+    "/applications/{appId}/import": {
+      "post": {
+        "operationId": "appImport",
+        "summary": "Import an app to an existing app 🔒",
+        "description": "This endpoint is only available on a business or enterprise license.",
+        "tags": [
+          "applications"
+        ],
+        "parameters": [
+          {
+            "$ref": "#/components/parameters/appIdUrl"
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "multipart/form-data": {
+              "schema": {
+                "type": "object",
+                "properties": {
+                  "encryptedPassword": {
+                    "description": "Password for the export if it is encrypted.",
+                    "type": "string"
+                  },
+                  "appExport": {
+                    "description": "The app export to import.",
+                    "type": "string",
+                    "format": "binary"
+                  }
+                },
+                "required": [
+                  "appExport"
+                ]
+              }
+            }
+          }
+        },
+        "responses": {
+          "204": {
+            "description": "Application has been updated."
+          }
+        }
+      }
+    },
+    "/applications/{appId}/export": {
+      "post": {
+        "operationId": "appExport",
+        "summary": "Export an app 🔒",
+        "description": "This endpoint is only available on a business or enterprise license.",
+        "tags": [
+          "applications"
+        ],
+        "parameters": [
+          {
+            "$ref": "#/components/parameters/appIdUrl"
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/appExport"
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "A gzip tarball containing the app export, encrypted if password provided.",
+            "content": {
+              "application/gzip": {
+                "schema": {
+                  "type": "string",
+                  "format": "binary",
+                  "example": "Tarball containing database and object store contents..."
+                }
+              }
+            }
+          }
+        }
+      }
+    },
     "/applications/search": {
       "post": {
         "operationId": "appSearch",

packages/server/specs/openapi.yaml

@@ -587,6 +587,19 @@ components:
             - appUrl
       required:
         - data
+    appExport:
+      type: object
+      properties:
+        encryptPassword:
+          description: An optional password used to encrypt the export.
+          type: string
+        excludeRows:
+          description: Set whether the internal table rows should be excluded from the
+            export.
+          type: boolean
+      required:
+        - encryptPassword
+        - excludeRows
     row:
       description: The row to be created/updated, based on the table schema.
       type: object
@@ -1763,6 +1776,57 @@ paths:
               examples:
                 deployment:
                   $ref: "#/components/examples/deploymentOutput"
+  "/applications/{appId}/import":
+    post:
+      operationId: appImport
+      summary: Import an app to an existing app 🔒
+      description: This endpoint is only available on a business or enterprise license.
+      tags:
+        - applications
+      parameters:
+        - $ref: "#/components/parameters/appIdUrl"
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              type: object
+              properties:
+                encryptedPassword:
+                  description: Password for the export if it is encrypted.
+                  type: string
+                appExport:
+                  description: The app export to import.
+                  type: string
+                  format: binary
+              required:
+                - appExport
+      responses:
+        "204":
+          description: Application has been updated.
+  "/applications/{appId}/export":
+    post:
+      operationId: appExport
+      summary: Export an app 🔒
+      description: This endpoint is only available on a business or enterprise license.
+      tags:
+        - applications
+      parameters:
+        - $ref: "#/components/parameters/appIdUrl"
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/appExport"
+      responses:
+        "200":
+          description: A gzip tarball containing the app export, encrypted if password
+            provided.
+          content:
+            application/gzip:
+              schema:
+                type: string
+                format: binary
+                example: Tarball containing database and object store contents...
   /applications/search:
     post:
       operationId: appSearch

packages/server/specs/resources/application.ts

@@ -134,4 +134,15 @@ export default new Resource()
     deploymentOutput: object({
       data: deploymentOutputSchema,
     }),
+    appExport: object({
+      encryptPassword: {
+        description: "An optional password used to encrypt the export.",
+        type: "string",
+      },
+      excludeRows: {
+        description:
+          "Set whether the internal table rows should be excluded from the export.",
+        type: "boolean",
+      },
+    }),
   })

packages/server/src/api/controllers/application.ts

@@ -39,9 +39,8 @@ import {
 } from "../../db/defaultData/datasource_bb_default"
 import { removeAppFromUserRoles } from "../../utilities/workerRequests"
 import { stringToReadStream } from "../../utilities"
-import { doesUserHaveLock, getLocksById } from "../../utilities/redis"
+import { doesUserHaveLock } from "../../utilities/redis"
 import { cleanupAutomations } from "../../automations/utils"
-import { checkAppMetadata } from "../../automations/logging"
 import { getUniqueRows } from "../../utilities/usageQuota/rows"
 import { groups, licensing, quotas } from "@budibase/pro"
 import {
@@ -51,7 +50,6 @@ import {
   PlanType,
   Screen,
   UserCtx,
-  ContextUser,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import sdk from "../../sdk"
@@ -168,7 +166,7 @@ export const addSampleData = async (ctx: UserCtx) => {
     // Check if default datasource exists before creating it
     await sdk.datasources.get(DEFAULT_BB_DATASOURCE_ID)
   } catch (err: any) {
-    const defaultDbDocs = buildDefaultDocs()
+    const defaultDbDocs = await buildDefaultDocs()
 
     // add in the default db data docs - tables, datasource, rows and links
     await db.bulkDocs([...defaultDbDocs])
@@ -575,6 +573,28 @@ export async function sync(ctx: UserCtx) {
   }
 }
 
+export async function importToApp(ctx: UserCtx) {
+  const { appId } = ctx.params
+  const appExport = ctx.request.files?.appExport
+  const password = ctx.request.body.encryptionPassword as string
+  if (!appExport) {
+    ctx.throw(400, "Must supply app export to import")
+  }
+  if (Array.isArray(appExport)) {
+    ctx.throw(400, "Must only supply one app export")
+  }
+  const fileAttributes = { type: appExport.type!, path: appExport.path! }
+  try {
+    await sdk.applications.updateWithExport(appId, fileAttributes, password)
+  } catch (err: any) {
+    ctx.throw(
+      500,
+      `Unable to perform update, please retry - ${err?.message || err}`
+    )
+  }
+  ctx.body = { message: "app updated" }
+}
+
 export async function updateAppPackage(appPackage: any, appId: any) {
   return context.doInAppContext(appId, async () => {
     const db = context.getAppDB()

packages/server/src/api/controllers/public/applications.ts

@@ -2,9 +2,11 @@ import { db as dbCore, context } from "@budibase/backend-core"
 import { search as stringSearch, addRev } from "./utils"
 import * as controller from "../application"
 import * as deployController from "../deploy"
+import * as backupController from "../backup"
 import { Application } from "../../../definitions/common"
 import { UserCtx } from "@budibase/types"
 import { Next } from "koa"
+import { sdk as proSdk } from "@budibase/pro"
 
 function fixAppID(app: Application, params: any) {
   if (!params) {
@@ -80,6 +82,8 @@ export async function destroy(ctx: UserCtx, next: Next) {
 export async function unpublish(ctx: UserCtx, next: Next) {
   await context.doInAppContext(ctx.params.appId, async () => {
     await controller.unpublish(ctx)
+    ctx.body = undefined
+    ctx.status = 204
     await next()
   })
 }
@@ -91,12 +95,22 @@ export async function publish(ctx: UserCtx, next: Next) {
   })
 }
 
+// get licensed endpoints from pro
+export const importToApp = proSdk.publicApi.applications.buildImportFn(
+  controller.importToApp
+)
+export const exportApp = proSdk.publicApi.applications.buildExportFn(
+  backupController.exportAppDump
+)
+
 export default {
   create,
   update,
   read,
   destroy,
   search,
-  publish,
   unpublish,
+  publish,
+  importToApp,
+  exportApp,
 }

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -305,12 +305,7 @@ export class ExternalRequest {
       manyRelationships: ManyRelationship[] = []
     for (let [key, field] of Object.entries(table.schema)) {
       // if set already, or not set just skip it
-      if (row[key] == null || newRow[key] || !isEditableColumn(field)) {
+      if (row[key] === undefined || newRow[key] || !isEditableColumn(field)) {
-        continue
-      }
-      // if its an empty string then it means return the column to null (if possible)
-      if (row[key] === "") {
-        newRow[key] = null
         continue
       }
       // parse floats/numbers

packages/server/src/api/controllers/row/external.ts

@@ -80,8 +80,8 @@ export async function save(ctx: UserCtx) {
   }
 
   const table = await sdk.tables.getTable(tableId)
-  const { table: updatedTable, row } = inputProcessing(
+  const { table: updatedTable, row } = await inputProcessing(
-    ctx.user,
+    ctx.user?._id,
     cloneDeep(table),
     inputs
   )

packages/server/src/api/controllers/row/internal.ts

@@ -59,7 +59,11 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const tableClone = cloneDeep(dbTable)
 
   // this returns the table and row incase they have been updated
-  let { table, row } = inputProcessing(ctx.user, tableClone, combinedRow)
+  let { table, row } = await inputProcessing(
+    ctx.user?._id,
+    tableClone,
+    combinedRow
+  )
   const validateResult = await sdk.rows.utils.validate({
     row,
     table,
@@ -106,7 +110,7 @@ export async function save(ctx: UserCtx) {
   // need to copy the table so it can be differenced on way out
   const tableClone = cloneDeep(dbTable)
 
-  let { table, row } = inputProcessing(ctx.user, tableClone, inputs)
+  let { table, row } = await inputProcessing(ctx.user?._id, tableClone, inputs)
 
   const validateResult = await sdk.rows.utils.validate({
     row,

packages/server/src/api/controllers/table/tests/utils.spec.ts

@@ -42,7 +42,7 @@ describe("utils", () => {
 
         const data = [{ name: "Alice" }, { name: "Bob" }, { name: "Claire" }]
 
-        const result = importToRows(data, table, config.user)
+        const result = await importToRows(data, table, config.user)
         expect(result).toEqual([
           expect.objectContaining({
             autoId: 1,
@@ -89,7 +89,7 @@ describe("utils", () => {
 
         const data = [{ name: "Alice" }, { name: "Bob" }, { name: "Claire" }]
 
-        const result = importToRows(data, table)
+        const result = await importToRows(data, table)
         expect(result).toHaveLength(3)
       })
     })

packages/server/src/api/controllers/table/utils.ts

@@ -99,7 +99,7 @@ export function makeSureTableUpToDate(table: any, tableToSave: any) {
   return tableToSave
 }
 
-export function importToRows(
+export async function importToRows(
   data: any[],
   table: Table,
   user: ContextUser | null = null
@@ -113,7 +113,7 @@ export function importToRows(
 
     // We use a reference to table here and update it after input processing,
     // so that we can auto increment auto IDs in imported data properly
-    const processed = inputProcessing(user, table, row, {
+    const processed = await inputProcessing(user?._id, table, row, {
       noAutoRelationships: true,
     })
     row = processed.row
@@ -158,7 +158,7 @@ export async function handleDataImport(
   const db = context.getAppDB()
   const data = parse(rows, schema)
 
-  let finalData: any = importToRows(data, table, user)
+  let finalData: any = await importToRows(data, table, user)
 
   //Set IDs of finalData to match existing row if an update is expected
   if (identifierFields.length > 0) {
@@ -422,13 +422,11 @@ export function hasTypeChanged(table: Table, oldTable: Table | undefined) {
   if (!oldTable) {
     return false
   }
-  let key: any
+  for (let [key, field] of Object.entries(oldTable.schema)) {
-  let field: any
-  for ([key, field] of Object.entries(oldTable.schema)) {
-    const oldType = field.type
     if (!table.schema[key]) {
       continue
     }
+    const oldType = field.type
     const newType = table.schema[key].type
     if (oldType !== newType && !areSwitchableTypes(oldType, newType)) {
       return true

packages/server/src/api/routes/application.ts

@@ -4,6 +4,7 @@ import * as deploymentController from "../controllers/deploy"
 import authorized from "../../middleware/authorized"
 import { permissions } from "@budibase/backend-core"
 import { applicationValidator } from "./utils/validators"
+import { importToApp } from "../controllers/application"
 
 const router: Router = new Router()
 
@@ -58,5 +59,10 @@ router
     authorized(permissions.GLOBAL_BUILDER),
     controller.destroy
   )
+  .post(
+    "/api/applications/:appId/import",
+    authorized(permissions.BUILDER),
+    controller.importToApp
+  )
 
 export default router

packages/server/src/api/routes/public/applications.ts

@@ -137,6 +137,70 @@ write.push(
   new Endpoint("post", "/applications/:appId/publish", controller.publish)
 )
 
+/**
+ * @openapi
+ * /applications/{appId}/import:
+ *   post:
+ *     operationId: appImport
+ *     summary: Import an app to an existing app 🔒
+ *     description: This endpoint is only available on a business or enterprise license.
+ *     tags:
+ *       - applications
+ *     parameters:
+ *       - $ref: '#/components/parameters/appIdUrl'
+ *     requestBody:
+ *       content:
+ *         multipart/form-data:
+ *           schema:
+ *             type: object
+ *             properties:
+ *               encryptedPassword:
+ *                 description: Password for the export if it is encrypted.
+ *                 type: string
+ *               appExport:
+ *                 description: The app export to import.
+ *                 type: string
+ *                 format: binary
+ *             required:
+ *               - appExport
+ *     responses:
+ *       204:
+ *         description: Application has been updated.
+ */
+write.push(
+  new Endpoint("post", "/applications/:appId/import", controller.importToApp)
+)
+
+/**
+ * @openapi
+ * /applications/{appId}/export:
+ *   post:
+ *     operationId: appExport
+ *     summary: Export an app 🔒
+ *     description: This endpoint is only available on a business or enterprise license.
+ *     tags:
+ *       - applications
+ *     parameters:
+ *       - $ref: '#/components/parameters/appIdUrl'
+ *     requestBody:
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/appExport'
+ *     responses:
+ *       200:
+ *         description: A gzip tarball containing the app export, encrypted if password provided.
+ *         content:
+ *           application/gzip:
+ *             schema:
+ *               type: string
+ *               format: binary
+ *               example: Tarball containing database and object store contents...
+ */
+read.push(
+  new Endpoint("post", "/applications/:appId/export", controller.exportApp)
+)
+
 /**
  * @openapi
  * /applications/{appId}:

packages/server/src/api/routes/public/middleware/mapper.ts

@@ -1,3 +1,4 @@
+import { Ctx } from "@budibase/types"
 import mapping from "../../../controllers/public/mapping"
 
 enum Resources {
@@ -9,11 +10,19 @@ enum Resources {
   SEARCH = "search",
 }
 
-function isArrayResponse(ctx: any) {
+function isAttachment(ctx: Ctx) {
+  return ctx.body?.path && ctx.body?.flags && ctx.body?.mode
+}
+
+function isArrayResponse(ctx: Ctx) {
   return ctx.url.endsWith(Resources.SEARCH) || Array.isArray(ctx.body)
 }
 
-function processApplications(ctx: any) {
+function noResponse(ctx: Ctx) {
+  return !Array.isArray(ctx.body) && Object.keys(ctx.body).length === 0
+}
+
+function processApplications(ctx: Ctx) {
   if (isArrayResponse(ctx)) {
     return mapping.mapApplications(ctx)
   } else {
@@ -21,7 +30,7 @@ function processApplications(ctx: any) {
   }
 }
 
-function processTables(ctx: any) {
+function processTables(ctx: Ctx) {
   if (isArrayResponse(ctx)) {
     return mapping.mapTables(ctx)
   } else {
@@ -29,7 +38,7 @@ function processTables(ctx: any) {
   }
 }
 
-function processRows(ctx: any) {
+function processRows(ctx: Ctx) {
   if (isArrayResponse(ctx)) {
     return mapping.mapRowSearch(ctx)
   } else {
@@ -37,7 +46,7 @@ function processRows(ctx: any) {
   }
 }
 
-function processUsers(ctx: any) {
+function processUsers(ctx: Ctx) {
   if (isArrayResponse(ctx)) {
     return mapping.mapUsers(ctx)
   } else {
@@ -45,7 +54,7 @@ function processUsers(ctx: any) {
   }
 }
 
-function processQueries(ctx: any) {
+function processQueries(ctx: Ctx) {
   if (isArrayResponse(ctx)) {
     return mapping.mapQueries(ctx)
   } else {
@@ -53,8 +62,8 @@ function processQueries(ctx: any) {
   }
 }
 
-export default async (ctx: any, next: any) => {
+export default async (ctx: Ctx, next: any) => {
-  if (!ctx.body) {
+  if (!ctx.body || noResponse(ctx) || isAttachment(ctx)) {
     return await next()
   }
   let urlParts = ctx.url.split("/")

packages/server/src/api/routes/public/tests/applications.spec.ts

@@ -0,0 +1,91 @@
+import * as setup from "../../tests/utilities"
+import {
+  generateMakeRequest,
+  generateMakeRequestWithFormData,
+  MakeRequestResponse,
+  MakeRequestWithFormDataResponse,
+} from "./utils"
+import { User } from "@budibase/types"
+import { join } from "path"
+import { mocks } from "@budibase/backend-core/tests"
+
+const PASSWORD = "testtest"
+const NO_LICENSE_MSG = "Endpoint unavailable, license required."
+
+let config = setup.getConfig()
+let apiKey: string,
+  globalUser: User,
+  makeRequest: MakeRequestResponse,
+  makeRequestFormData: MakeRequestWithFormDataResponse
+
+beforeAll(async () => {
+  await config.init()
+  globalUser = await config.globalUser()
+  apiKey = await config.generateApiKey(globalUser._id)
+  makeRequest = generateMakeRequest(apiKey)
+  makeRequestFormData = generateMakeRequestWithFormData(apiKey)
+})
+
+afterAll(setup.afterAll)
+
+describe("check export/import", () => {
+  async function runExport() {
+    return await makeRequest("post", `/applications/${config.appId}/export`, {
+      encryptionPassword: PASSWORD,
+      excludeRows: true,
+    })
+  }
+
+  async function runImport() {
+    const pathToExport = join(
+      __dirname,
+      "..",
+      "..",
+      "tests",
+      "assets",
+      "export.tar.gz"
+    )
+    return await makeRequestFormData(
+      "post",
+      `/applications/${config.appId}/import`,
+      {
+        encryptionPassword: PASSWORD,
+        appExport: { path: pathToExport },
+      }
+    )
+  }
+
+  it("check licensing for export", async () => {
+    const res = await runExport()
+    expect(res.status).toBe(403)
+    expect(res.body.message).toBe(NO_LICENSE_MSG)
+  })
+
+  it("check licensing for import", async () => {
+    const res = await runImport()
+    expect(res.status).toBe(403)
+    expect(res.body.message).toBe(NO_LICENSE_MSG)
+  })
+
+  it("should be able to export app", async () => {
+    mocks.licenses.useExpandedPublicApi()
+    const res = await runExport()
+    expect(res.headers["content-disposition"]).toMatch(
+      /attachment; filename=".*-export-.*\.tar.gz"/g
+    )
+    expect(res.body instanceof Buffer).toBe(true)
+    expect(res.status).toBe(200)
+  })
+
+  it("should be able to import app", async () => {
+    mocks.licenses.useExpandedPublicApi()
+    const res = await runImport()
+    expect(Object.keys(res.body).length).toBe(0)
+    // check screens imported correctly
+    const screens = await config.api.screen.list()
+    expect(screens.length).toBe(2)
+    expect(screens[0].routing.route).toBe("/derp")
+    expect(screens[1].routing.route).toBe("/blank")
+    expect(res.status).toBe(204)
+  })
+})

packages/server/src/api/routes/public/tests/users.spec.ts

@@ -92,7 +92,7 @@ describe("no user role update in free", () => {
 describe("no user role update in business", () => {
   beforeAll(() => {
     updateMock()
-    mocks.licenses.usePublicApiUserRoles()
+    mocks.licenses.useExpandedPublicApi()
   })
 
   it("should allow 'roles' to be updated", async () => {
@@ -105,7 +105,7 @@ describe("no user role update in business", () => {
   })
 
   it("should allow 'admin' to be updated", async () => {
-    mocks.licenses.usePublicApiUserRoles()
+    mocks.licenses.useExpandedPublicApi()
     const res = await makeRequest("post", "/users", {
       ...base(),
       admin: { global: true },
@@ -115,7 +115,7 @@ describe("no user role update in business", () => {
   })
 
   it("should allow 'builder' to be updated", async () => {
-    mocks.licenses.usePublicApiUserRoles()
+    mocks.licenses.useExpandedPublicApi()
     const res = await makeRequest("post", "/users", {
       ...base(),
       builder: { global: true },

packages/server/src/api/routes/public/tests/utils.ts

@@ -11,6 +11,32 @@ export type MakeRequestResponse = (
   intAppId?: string
 ) => Promise<supertest.Response>
 
+export type MakeRequestWithFormDataResponse = (
+  method: HttpMethod,
+  endpoint: string,
+  fields: Record<string, string | { path: string }>,
+  intAppId?: string
+) => Promise<supertest.Response>
+
+function base(
+  apiKey: string,
+  endpoint: string,
+  intAppId: string | null,
+  isInternal: boolean
+) {
+  const extraHeaders: any = {
+    "x-budibase-api-key": apiKey,
+  }
+  if (intAppId) {
+    extraHeaders["x-budibase-app-id"] = intAppId
+  }
+
+  const url = isInternal
+    ? endpoint
+    : checkSlashesInUrl(`/api/public/v1/${endpoint}`)
+  return { headers: extraHeaders, url }
+}
+
 export function generateMakeRequest(
   apiKey: string,
   isInternal = false
@@ -23,18 +49,8 @@ export function generateMakeRequest(
     body?: any,
     intAppId: string | null = config.getAppId()
   ) => {
-    const extraHeaders: any = {
+    const { headers, url } = base(apiKey, endpoint, intAppId, isInternal)
-      "x-budibase-api-key": apiKey,
+    const req = request[method](url).set(config.defaultHeaders(headers))
-    }
-    if (intAppId) {
-      extraHeaders["x-budibase-app-id"] = intAppId
-    }
-
-    const url = isInternal
-      ? endpoint
-      : checkSlashesInUrl(`/api/public/v1/${endpoint}`)
-
-    const req = request[method](url).set(config.defaultHeaders(extraHeaders))
     if (body) {
       req.send(body)
     }
@@ -43,3 +59,30 @@ export function generateMakeRequest(
     return res
   }
 }
+
+export function generateMakeRequestWithFormData(
+  apiKey: string,
+  isInternal = false
+): MakeRequestWithFormDataResponse {
+  const request = setup.getRequest()!
+  const config = setup.getConfig()!
+  return async (
+    method: HttpMethod,
+    endpoint: string,
+    fields: Record<string, string | { path: string }>,
+    intAppId: string | null = config.getAppId()
+  ) => {
+    const { headers, url } = base(apiKey, endpoint, intAppId, isInternal)
+    const req = request[method](url).set(config.defaultHeaders(headers))
+    for (let [field, value] of Object.entries(fields)) {
+      if (typeof value === "string") {
+        req.field(field, value)
+      } else {
+        req.attach(field, value.path)
+      }
+    }
+    const res = await req
+    expect(res.body).toBeDefined()
+    return res
+  }
+}

packages/server/src/api/routes/tests/appImport.spec.ts

@@ -0,0 +1,32 @@
+import * as setup from "./utilities"
+import path from "path"
+
+jest.setTimeout(15000)
+const PASSWORD = "testtest"
+
+describe("/applications/:appId/import", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+
+  afterAll(setup.afterAll)
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  it("should be able to perform import", async () => {
+    const appId = config.getAppId()
+    const res = await request
+      .post(`/api/applications/${appId}/import`)
+      .field("encryptionPassword", PASSWORD)
+      .attach("appExport", path.join(__dirname, "assets", "export.tar.gz"))
+      .set(config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+    expect(res.body.message).toBe("app updated")
+    const screens = await config.api.screen.list()
+    expect(screens.length).toBe(2)
+    expect(screens[0].routing.route).toBe("/derp")
+    expect(screens[1].routing.route).toBe("/blank")
+  })
+})

packages/server/src/db/defaultData/datasource_bb_default.ts

@@ -34,9 +34,9 @@ function syncLastIds(table: Table, rowCount: number) {
   })
 }
 
-function tableImport(table: Table, data: Row[]) {
+async function tableImport(table: Table, data: Row[]) {
   const cloneTable = cloneDeep(table)
-  const rowDocs = importToRows(data, cloneTable)
+  const rowDocs = await importToRows(data, cloneTable)
   syncLastIds(cloneTable, rowDocs.length)
   return { rows: rowDocs, table: cloneTable }
 }
@@ -601,20 +601,20 @@ export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = {
   },
 }
 
-export function buildDefaultDocs() {
+export async function buildDefaultDocs() {
-  const inventoryData = tableImport(
+  const inventoryData = await tableImport(
     DEFAULT_INVENTORY_TABLE_SCHEMA,
     inventoryImport
   )
 
-  const employeeData = tableImport(
+  const employeeData = await tableImport(
     DEFAULT_EMPLOYEE_TABLE_SCHEMA,
     employeeImport
   )
 
-  const jobData = tableImport(DEFAULT_JOBS_TABLE_SCHEMA, jobsImport)
+  const jobData = await tableImport(DEFAULT_JOBS_TABLE_SCHEMA, jobsImport)
 
-  const expensesData = tableImport(
+  const expensesData = await tableImport(
     DEFAULT_EXPENSES_TABLE_SCHEMA,
     expensesImport
   )

packages/server/src/definitions/openapi.ts

@@ -18,6 +18,14 @@ export interface paths {
   "/applications/{appId}/publish": {
     post: operations["appPublish"];
   };
+  "/applications/{appId}/import": {
+    /** This endpoint is only available on a business or enterprise license. */
+    post: operations["appImport"];
+  };
+  "/applications/{appId}/export": {
+    /** This endpoint is only available on a business or enterprise license. */
+    post: operations["appExport"];
+  };
   "/applications/search": {
     /** Based on application properties (currently only name) search for applications. */
     post: operations["appSearch"];
@@ -158,6 +166,12 @@ export interface components {
         appUrl: string;
       };
     };
+    appExport: {
+      /** @description An optional password used to encrypt the export. */
+      encryptPassword: string;
+      /** @description Set whether the internal table rows should be excluded from the export. */
+      excludeRows: boolean;
+    };
     /** @description The row to be created/updated, based on the table schema. */
     row: { [key: string]: unknown };
     searchOutput: {
@@ -889,6 +903,54 @@ export interface operations {
       };
     };
   };
+  /** This endpoint is only available on a business or enterprise license. */
+  appImport: {
+    parameters: {
+      path: {
+        /** The ID of the app which this request is targeting. */
+        appId: components["parameters"]["appIdUrl"];
+      };
+    };
+    responses: {
+      /** Application has been updated. */
+      204: never;
+    };
+    requestBody: {
+      content: {
+        "multipart/form-data": {
+          /** @description Password for the export if it is encrypted. */
+          encryptedPassword?: string;
+          /**
+           * Format: binary
+           * @description The app export to import.
+           */
+          appExport: string;
+        };
+      };
+    };
+  };
+  /** This endpoint is only available on a business or enterprise license. */
+  appExport: {
+    parameters: {
+      path: {
+        /** The ID of the app which this request is targeting. */
+        appId: components["parameters"]["appIdUrl"];
+      };
+    };
+    responses: {
+      /** A gzip tarball containing the app export, encrypted if password provided. */
+      200: {
+        content: {
+          "application/gzip": string;
+        };
+      };
+    };
+    requestBody: {
+      content: {
+        "application/json": components["schemas"]["appExport"];
+      };
+    };
+  };
   /** Based on application properties (currently only name) search for applications. */
   appSearch: {
     responses: {

packages/server/src/integrations/base/sqlTable.ts

@@ -41,6 +41,7 @@ function generateSchema(
       case FieldTypes.OPTIONS:
       case FieldTypes.LONGFORM:
       case FieldTypes.BARCODEQR:
+      case FieldTypes.BB_REFERENCE:
         schema.text(key)
         break
       case FieldTypes.NUMBER:

packages/server/src/sdk/app/applications/applications.ts

@@ -41,7 +41,7 @@ export async function fetch(status: AppStatus, user: ContextUser) {
     for (let app of apps) {
       const lock = locks[app.appId]
       if (lock) {
-        app.lockedBy = lock
+        app.lockedBy = lock as any
       } else {
         // make sure its definitely not present
         delete app.lockedBy

packages/server/src/sdk/app/applications/import.ts

@@ -0,0 +1,102 @@
+import { db as dbCore } from "@budibase/backend-core"
+import {
+  DocumentTypesToImport,
+  Document,
+  Database,
+  RowValue,
+} from "@budibase/types"
+import backups from "../backups"
+
+export type FileAttributes = {
+  type: string
+  path: string
+}
+
+function mergeUpdateAndDeleteDocuments(
+  updateDocs: Document[],
+  deleteDocs: Document[]
+) {
+  // compress the documents to create and to delete (if same ID, then just update the rev)
+  const finalToDelete = []
+  for (let deleteDoc of deleteDocs) {
+    const found = updateDocs.find(doc => doc._id === deleteDoc._id)
+    if (found) {
+      found._rev = deleteDoc._rev
+    } else {
+      finalToDelete.push(deleteDoc)
+    }
+  }
+  return [...updateDocs, ...finalToDelete]
+}
+
+async function removeImportableDocuments(db: Database) {
+  // get the references to the documents, not the whole document
+  const docPromises = []
+  for (let docType of DocumentTypesToImport) {
+    docPromises.push(db.allDocs(dbCore.getDocParams(docType)))
+  }
+  let documentRefs: { _id: string; _rev: string }[] = []
+  for (let response of await Promise.all(docPromises)) {
+    documentRefs = documentRefs.concat(
+      response.rows.map(row => ({
+        _id: row.id,
+        _rev: (row.value as RowValue).rev,
+      }))
+    )
+  }
+  // add deletion key
+  return documentRefs.map(ref => ({ _deleted: true, ...ref }))
+}
+
+async function getImportableDocuments(db: Database) {
+  // get the whole document
+  const docPromises = []
+  for (let docType of DocumentTypesToImport) {
+    docPromises.push(
+      db.allDocs(dbCore.getDocParams(docType, null, { include_docs: true }))
+    )
+  }
+  // map the responses to the document itself
+  let documents: Document[] = []
+  for (let response of await Promise.all(docPromises)) {
+    documents = documents.concat(response.rows.map(row => row.doc))
+  }
+  // remove the _rev, stops it being written
+  documents.forEach(doc => {
+    delete doc._rev
+  })
+  return documents
+}
+
+export async function updateWithExport(
+  appId: string,
+  file: FileAttributes,
+  password?: string
+) {
+  const devId = dbCore.getDevAppID(appId)
+  const tempAppName = `temp_${devId}`
+  const tempDb = dbCore.getDB(tempAppName)
+  const appDb = dbCore.getDB(devId)
+  try {
+    const template = {
+      file: {
+        type: file.type!,
+        path: file.path!,
+        password,
+      },
+    }
+    // get a temporary version of the import
+    // don't need obj store, the existing app already has everything we need
+    await backups.importApp(devId, tempDb, template, {
+      importObjStoreContents: false,
+    })
+    // get the documents to copy
+    const toUpdate = await getImportableDocuments(tempDb)
+    // clear out the old documents
+    const toDelete = await removeImportableDocuments(appDb)
+    // now bulk update documents - add new ones, delete old ones and update common ones
+    await appDb.bulkDocs(mergeUpdateAndDeleteDocuments(toUpdate, toDelete))
+  } finally {
+    await tempDb.destroy()
+  }
+}

packages/server/src/sdk/app/applications/index.ts

@@ -1,9 +1,11 @@
 import * as sync from "./sync"
 import * as utils from "./utils"
 import * as applications from "./applications"
+import * as imports from "./import"
 
 export default {
   ...sync,
   ...utils,
   ...applications,
+  ...imports,
 }