From b4a8f22b2ee54914294eadb0259fde5f1006879a Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Wed, 11 Oct 2023 10:09:14 +0100
Subject: [PATCH 001/134] Initial commit

---
 .../design/settings/componentSettings.js      |   2 +
 .../ButtonConfiguration.svelte                | 104 ++++++++++
 .../ButtonConfiguration/ButtonSetting.svelte  |  61 ++++++
 .../controls/FieldConfiguration.svelte        |  91 ---------
 .../EditFieldPopover.svelte                   |  38 ++--
 .../FieldConfiguration.svelte                 |   4 +-
 .../new/_components/componentStructure.json   |   1 +
 packages/client/manifest.json                 | 183 ++++++++++++++++++
 .../components/app/forms/ButtonGroup.svelte   |  38 ++++
 .../client/src/components/app/forms/index.js  |   1 +
 10 files changed, 412 insertions(+), 111 deletions(-)
 create mode 100644 packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
 create mode 100644 packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
 delete mode 100644 packages/builder/src/components/design/settings/controls/FieldConfiguration.svelte
 create mode 100644 packages/client/src/components/app/forms/ButtonGroup.svelte

diff --git a/packages/builder/src/components/design/settings/componentSettings.js b/packages/builder/src/components/design/settings/componentSettings.js
index 4c49587372..232b4bef31 100644
--- a/packages/builder/src/components/design/settings/componentSettings.js
+++ b/packages/builder/src/components/design/settings/componentSettings.js
@@ -23,6 +23,7 @@ import BasicColumnEditor from "./controls/ColumnEditor/BasicColumnEditor.svelte"
 import GridColumnEditor from "./controls/ColumnEditor/GridColumnEditor.svelte"
 import BarButtonList from "./controls/BarButtonList.svelte"
 import FieldConfiguration from "./controls/FieldConfiguration/FieldConfiguration.svelte"
+import ButtonConfiguration from "./controls/ButtonConfiguration/ButtonConfiguration.svelte"
 import RelationshipFilterEditor from "./controls/RelationshipFilterEditor.svelte"
 
 const componentMap = {
@@ -48,6 +49,7 @@ const componentMap = {
   "filter/relationship": RelationshipFilterEditor,
   url: URLSelect,
   fieldConfiguration: FieldConfiguration,
+  buttonConfiguration: ButtonConfiguration,
   columns: ColumnEditor,
   "columns/basic": BasicColumnEditor,
   "columns/grid": GridColumnEditor,
diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
new file mode 100644
index 0000000000..baa8ad733c
--- /dev/null
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
@@ -0,0 +1,104 @@
+<script>
+  import DraggableList from "../DraggableList.svelte"
+  import ButtonSetting from "./ButtonSetting.svelte"
+  import {
+    getBindableProperties,
+    getComponentBindableProperties,
+  } from "builderStore/dataBinding"
+  import { createEventDispatcher, onMount } from "svelte"
+  import { selectedScreen, store } from "builderStore"
+  import { isEqual, cloneDeep } from "lodash"
+
+  export let componentInstance
+  export let value
+
+  $: bindings = getBindableProperties($selectedScreen, componentInstance._id)
+
+  const dispatch = createEventDispatcher()
+  let buttonList
+  let mounted
+  let cachedValue
+  let componentBindings = []
+
+  $: componentBindings = getComponentBindableProperties(
+    $selectedScreen,
+    componentInstance._id
+  )
+
+  $: if (!isEqual(value, cachedValue) && mounted) {
+    cachedValue = value
+      ? cloneDeep(value)
+      : [
+          buildPseudoInstance({ name: `Button 1`, type: "cta" }),
+          buildPseudoInstance({ name: `Button 2` }),
+        ]
+  }
+
+  const updateState = value => {
+    buttonList = value
+  }
+
+  $: updateState(cachedValue)
+
+  const processItemUpdate = e => {
+    const updatedField = e.detail
+    const parentButtonsUpdated = buttonList ? cloneDeep(buttonList) : []
+
+    let parentFieldIdx = parentButtonsUpdated.findIndex(pSetting => {
+      return pSetting._id === updatedField?._id
+    })
+
+    if (parentFieldIdx == -1) {
+      parentButtonsUpdated.push(updatedField)
+    } else {
+      parentButtonsUpdated[parentFieldIdx] = updatedField
+    }
+    console.log("On value update ", parentButtonsUpdated)
+    dispatch("change", parentButtonsUpdated)
+  }
+
+  const listUpdated = e => {
+    dispatch("change", [...e.detail])
+  }
+
+  // May not be necessary without
+  const buildPseudoInstance = cfg => {
+    const pseudoComponentInstance = store.actions.components.createInstance(
+      `@budibase/standard-components/button`,
+      {
+        _instanceName: cfg.name,
+        text: cfg.name,
+        type: cfg.type || "primary",
+      },
+      {}
+    )
+
+    return pseudoComponentInstance
+  }
+
+  onMount(() => {
+    mounted = true
+  })
+</script>
+
+<div class="button-configuration">
+  {#if buttonList?.length}
+    <DraggableList
+      on:change={listUpdated}
+      on:itemChange={processItemUpdate}
+      items={buttonList}
+      listItemKey={"_id"}
+      listType={ButtonSetting}
+      listTypeProps={{
+        componentBindings,
+        bindings,
+      }}
+    />
+  {/if}
+</div>
+
+<style>
+  .button-configuration :global(.spectrum-ActionButton) {
+    width: 100%;
+  }
+</style>
diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
new file mode 100644
index 0000000000..851cbe289c
--- /dev/null
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
@@ -0,0 +1,61 @@
+<script>
+  import EditFieldPopover from "../FieldConfiguration/EditFieldPopover.svelte"
+  import { Icon } from "@budibase/bbui"
+  import { createEventDispatcher } from "svelte"
+  // import { cloneDeep } from "lodash/fp"
+
+  export let item
+  export let componentBindings
+  export let bindings
+  export let anchor
+
+  const dispatch = createEventDispatcher()
+  // const onToggle = item => {
+  //   return e => {
+  //     item.active = e.detail
+  //     dispatch("change", { ...cloneDeep(item), active: e.detail })
+  //   }
+  // }
+</script>
+
+<div class="list-item-body">
+  <div class="list-item-left">
+    <EditFieldPopover
+      {anchor}
+      field={item}
+      {componentBindings}
+      {bindings}
+      on:change
+    />
+    <div class="field-label">{item.text}</div>
+  </div>
+  <div class="list-item-right">
+    <Icon size="S" name="Close" hoverable on:click={() => {console.log("REMOVE ME")}}
+  />
+  </div>
+</div>
+
+<style>
+  .field-label {
+    text-overflow: ellipsis;
+    white-space: nowrap;
+    overflow: hidden;
+  }
+  .list-item-body,
+  .list-item-left {
+    display: flex;
+    align-items: center;
+    gap: var(--spacing-m);
+    min-width: 0;
+  }
+  .list-item-body {
+    margin-top: 8px;
+    margin-bottom: 8px;
+  }
+  .list-item-right :global(div.spectrum-Switch) {
+    margin: 0px;
+  }
+  .list-item-body {
+    justify-content: space-between;
+  }
+</style>
diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration.svelte
deleted file mode 100644
index 80f4829d71..0000000000
--- a/packages/builder/src/components/design/settings/controls/FieldConfiguration.svelte
+++ /dev/null
@@ -1,91 +0,0 @@
-<script>
-  import { Button, ActionButton, Drawer } from "@budibase/bbui"
-  import { createEventDispatcher } from "svelte"
-  import ColumnDrawer from "./ColumnEditor/ColumnDrawer.svelte"
-  import { cloneDeep } from "lodash/fp"
-  import {
-    getDatasourceForProvider,
-    getSchemaForDatasource,
-  } from "builderStore/dataBinding"
-  import { currentAsset } from "builderStore"
-  import { getFields } from "helpers/searchFields"
-
-  export let componentInstance
-  export let value = []
-  export let allowCellEditing = true
-  export let subject = "Table"
-
-  const dispatch = createEventDispatcher()
-
-  let drawer
-  let boundValue
-
-  $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchema($currentAsset, datasource)
-  $: options = allowCellEditing
-    ? Object.keys(schema || {})
-    : enrichedSchemaFields?.map(field => field.name)
-  $: sanitisedValue = getValidColumns(value, options)
-  $: updateBoundValue(sanitisedValue)
-  $: enrichedSchemaFields = getFields(Object.values(schema || {}), {
-    allowLinks: true,
-  })
-
-  const getSchema = (asset, datasource) => {
-    const schema = getSchemaForDatasource(asset, datasource).schema
-
-    // Don't show ID and rev in tables
-    if (schema) {
-      delete schema._id
-      delete schema._rev
-    }
-
-    return schema
-  }
-
-  const updateBoundValue = value => {
-    boundValue = cloneDeep(value)
-  }
-
-  const getValidColumns = (columns, options) => {
-    if (!Array.isArray(columns) || !columns.length) {
-      return []
-    }
-    // We need to account for legacy configs which would just be an array
-    // of strings
-    if (typeof columns[0] === "string") {
-      columns = columns.map(col => ({
-        name: col,
-        displayName: col,
-      }))
-    }
-    return columns.filter(column => {
-      return options.includes(column.name)
-    })
-  }
-
-  const open = () => {
-    updateBoundValue(sanitisedValue)
-    drawer.show()
-  }
-
-  const save = () => {
-    dispatch("change", getValidColumns(boundValue, options))
-    drawer.hide()
-  }
-</script>
-
-<ActionButton on:click={open}>Configure columns</ActionButton>
-<Drawer bind:this={drawer} title="{subject} Columns">
-  <svelte:fragment slot="description">
-    Configure the columns in your {subject.toLowerCase()}.
-  </svelte:fragment>
-  <Button cta slot="buttons" on:click={save}>Save</Button>
-  <ColumnDrawer
-    slot="body"
-    bind:columns={boundValue}
-    {options}
-    {schema}
-    {allowCellEditing}
-  />
-</Drawer>
diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/EditFieldPopover.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/EditFieldPopover.svelte
index 7d2eaae478..72e7784727 100644
--- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/EditFieldPopover.svelte
+++ b/packages/builder/src/components/design/settings/controls/FieldConfiguration/EditFieldPopover.svelte
@@ -61,6 +61,25 @@
 
     dispatch("change", update)
   }
+
+  const customPositionHandler = (anchorBounds, eleBounds, cfg) => {
+    let { left, top } = cfg
+    let percentageOffset = 30
+    // left-outside
+    left = anchorBounds.left - eleBounds.width - 18
+
+    // shift up from the anchor, if space allows
+    let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset
+    let defaultTop = anchorBounds.top - offsetPos
+
+    if (window.innerHeight - defaultTop < eleBounds.height) {
+      top = window.innerHeight - eleBounds.height - 5
+    } else {
+      top = anchorBounds.top - offsetPos
+    }
+
+    return { ...cfg, left, top }
+  }
 </script>
 
 <Icon
@@ -92,24 +111,7 @@
   showPopover={drawers.length == 0}
   clickOutsideOverride={drawers.length > 0}
   maxHeight={600}
-  handlePostionUpdate={(anchorBounds, eleBounds, cfg) => {
-    let { left, top } = cfg
-    let percentageOffset = 30
-    // left-outside
-    left = anchorBounds.left - eleBounds.width - 18
-
-    // shift up from the anchor, if space allows
-    let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset
-    let defaultTop = anchorBounds.top - offsetPos
-
-    if (window.innerHeight - defaultTop < eleBounds.height) {
-      top = window.innerHeight - eleBounds.height - 5
-    } else {
-      top = anchorBounds.top - offsetPos
-    }
-
-    return { ...cfg, left, top }
-  }}
+  handlePostionUpdate={customPositionHandler}
 >
   <span class="popover-wrap">
     <Layout noPadding noGap>
diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
index 4169cb7d3d..42651a4d84 100644
--- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
@@ -50,7 +50,7 @@
     updateSanitsedFields(sanitisedValue)
     unconfigured = buildUnconfiguredOptions(schema, sanitisedFields)
     fieldList = [...sanitisedFields, ...unconfigured]
-      .map(buildSudoInstance)
+      .map(buildPseudoInstance)
       .filter(x => x != null)
   }
 
@@ -104,7 +104,7 @@
     })
   }
 
-  const buildSudoInstance = instance => {
+  const buildPseudoInstance = instance => {
     if (instance._component) {
       return instance
     }
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json
index 11a130490a..dd129be11e 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json
@@ -36,6 +36,7 @@
       "heading",
       "text",
       "button",
+      "buttongroup",
       "tag",
       "spectrumcard",
       "cardstat",
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 4e56ca758d..27e56d94db 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -258,6 +258,189 @@
     "description": "Contains your app screens",
     "static": true
   },
+
+  "buttongroup": {
+    "name": "Button group",
+    "icon": "Button",
+    "hasChildren": false,
+    "settings": [
+      {
+        "section": true,
+        "name": "Buttons",
+        "settings": [
+          {
+            "type": "buttonConfiguration",
+            "key": "buttons",
+            "nested": true,
+            "defaultValue" : [{
+              "component" : "button",
+              "props" : {
+                "type": "cta"
+              }
+            },{
+              "component" : "button",
+              "props" : {
+                "type" : "primary"
+              }
+            }]
+          }
+        ]
+      },
+      {
+        "section": true,
+        "name": "Layout",
+        "settings": [
+          {
+            "type": "select",
+            "label": "Direction",
+            "key": "direction",
+            "showInBar": true,
+            "barStyle": "buttons",
+            "options": [
+              {
+                "label": "Column",
+                "value": "column",
+                "barIcon": "ViewColumn",
+                "barTitle": "Column layout"
+              },
+              {
+                "label": "Row",
+                "value": "row",
+                "barIcon": "ViewRow",
+                "barTitle": "Row layout"
+              }
+            ],
+            "defaultValue": "column"
+          },
+          {
+            "type": "select",
+            "label": "Horiz. align",
+            "key": "hAlign",
+            "showInBar": true,
+            "barStyle": "buttons",
+            "options": [
+              {
+                "label": "Left",
+                "value": "left",
+                "barIcon": "AlignLeft",
+                "barTitle": "Align left"
+              },
+              {
+                "label": "Center",
+                "value": "center",
+                "barIcon": "AlignCenter",
+                "barTitle": "Align center"
+              },
+              {
+                "label": "Right",
+                "value": "right",
+                "barIcon": "AlignRight",
+                "barTitle": "Align right"
+              },
+              {
+                "label": "Stretch",
+                "value": "stretch",
+                "barIcon": "MoveLeftRight",
+                "barTitle": "Align stretched horizontally"
+              }
+            ],
+            "defaultValue": "left"
+          },
+          {
+            "type": "select",
+            "label": "Vert. align",
+            "key": "vAlign",
+            "showInBar": true,
+            "barStyle": "buttons",
+            "options": [
+              {
+                "label": "Top",
+                "value": "top",
+                "barIcon": "AlignTop",
+                "barTitle": "Align top"
+              },
+              {
+                "label": "Middle",
+                "value": "middle",
+                "barIcon": "AlignMiddle",
+                "barTitle": "Align middle"
+              },
+              {
+                "label": "Bottom",
+                "value": "bottom",
+                "barIcon": "AlignBottom",
+                "barTitle": "Align bottom"
+              },
+              {
+                "label": "Stretch",
+                "value": "stretch",
+                "barIcon": "MoveUpDown",
+                "barTitle": "Align stretched vertically"
+              }
+            ],
+            "defaultValue": "top"
+          },
+          {
+            "type": "select",
+            "label": "Size",
+            "key": "size",
+            "showInBar": true,
+            "barStyle": "buttons",
+            "options": [
+              {
+                "label": "Shrink",
+                "value": "shrink",
+                "barIcon": "Minimize",
+                "barTitle": "Shrink container"
+              },
+              {
+                "label": "Grow",
+                "value": "grow",
+                "barIcon": "Maximize",
+                "barTitle": "Grow container"
+              }
+            ],
+            "defaultValue": "shrink"
+          },
+          {
+            "type": "select",
+            "label": "Gap",
+            "key": "gap",
+            "showInBar": true,
+            "barStyle": "picker",
+            "options": [
+              {
+                "label": "None",
+                "value": "N"
+              },
+              {
+                "label": "Small",
+                "value": "S"
+              },
+              {
+                "label": "Medium",
+                "value": "M"
+              },
+              {
+                "label": "Large",
+                "value": "L"
+              }
+            ],
+            "defaultValue": "M"
+          },
+          {
+            "type": "boolean",
+            "label": "Wrap",
+            "key": "wrap",
+            "showInBar": true,
+            "barIcon": "ModernGridView",
+            "barTitle": "Wrap"
+          }
+        ]
+      }
+    ]
+  },
+
   "button": {
     "name": "Button",
     "description": "A basic html button that is ready for styling",
diff --git a/packages/client/src/components/app/forms/ButtonGroup.svelte b/packages/client/src/components/app/forms/ButtonGroup.svelte
new file mode 100644
index 0000000000..222e91a55f
--- /dev/null
+++ b/packages/client/src/components/app/forms/ButtonGroup.svelte
@@ -0,0 +1,38 @@
+<script>
+  import { getContext } from "svelte"
+  import Container from "components/app/Container.svelte"
+  import BlockComponent from "../../BlockComponent.svelte"
+  import Block from "../../Block.svelte"
+
+  const { styleable } = getContext("sdk")
+  const component = getContext("component")
+
+  export let buttons = []
+  export let direction
+  export let hAlign
+  export let vAlign
+  export let gap = "S"
+  export let wrap
+
+  $: console.log("client buttons/wrap", buttons, wrap)
+</script>
+
+<!-- <div use:styleable={$component.styles}> -->
+  <Block>
+    <Container {direction} {hAlign} {vAlign} {gap} {wrap}>
+      {#each buttons as { text, type, quiet, disabled, onClick, size }}
+        <BlockComponent
+          type="button"
+          props={{
+            text,
+            onClick,
+            type,
+            quiet,
+            disabled,
+            size
+          }}
+        />
+      {/each}
+    </Container>
+  </Block>
+<!-- </div> -->
diff --git a/packages/client/src/components/app/forms/index.js b/packages/client/src/components/app/forms/index.js
index 5804d3a79d..24d7f11c0c 100644
--- a/packages/client/src/components/app/forms/index.js
+++ b/packages/client/src/components/app/forms/index.js
@@ -16,3 +16,4 @@ export { default as jsonfield } from "./JSONField.svelte"
 export { default as s3upload } from "./S3Upload.svelte"
 export { default as codescanner } from "./CodeScannerField.svelte"
 export { default as bbreferencefield } from "./BBReferenceField.svelte"
+export { default as buttongroup } from "./ButtonGroup.svelte"
\ No newline at end of file

From b8c87224f76b0ec77e65072fd78df917bff7a7bf Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Fri, 13 Oct 2023 14:53:39 +0100
Subject: [PATCH 002/134] Add/Remove button behaviour, binding label support
 and default text behaviour. Refactoring and renaming of EditComponentPopoever
 component

---
 .../ButtonConfiguration.svelte                | 112 ++++++++++++++----
 .../ButtonConfiguration/ButtonSetting.svelte  |  33 +++---
 .../{ => DraggableList}/DraggableList.svelte  |  32 ++++-
 .../controls/DraggableList/drag-handle.svelte |  31 +++++
 ...ver.svelte => EditComponentPopover.svelte} |  65 ++++------
 .../FieldConfiguration.svelte                 |   2 +-
 .../FieldConfiguration/FieldSetting.svelte    |  56 ++++++++-
 packages/client/manifest.json                 |  22 ++--
 .../src/components/app/ButtonGroup.svelte     |  37 ++++++
 .../components/app/forms/ButtonGroup.svelte   |  38 ------
 .../client/src/components/app/forms/index.js  |   3 +-
 packages/client/src/components/app/index.js   |   1 +
 12 files changed, 281 insertions(+), 151 deletions(-)
 rename packages/builder/src/components/design/settings/controls/{ => DraggableList}/DraggableList.svelte (82%)
 create mode 100644 packages/builder/src/components/design/settings/controls/DraggableList/drag-handle.svelte
 rename packages/builder/src/components/design/settings/controls/{FieldConfiguration/EditFieldPopover.svelte => EditComponentPopover.svelte} (63%)
 create mode 100644 packages/client/src/components/app/ButtonGroup.svelte
 delete mode 100644 packages/client/src/components/app/forms/ButtonGroup.svelte

diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
index baa8ad733c..fde888d17b 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
@@ -1,5 +1,5 @@
 <script>
-  import DraggableList from "../DraggableList.svelte"
+  import DraggableList from "../DraggableList/DraggableList.svelte"
   import ButtonSetting from "./ButtonSetting.svelte"
   import {
     getBindableProperties,
@@ -8,37 +8,47 @@
   import { createEventDispatcher, onMount } from "svelte"
   import { selectedScreen, store } from "builderStore"
   import { isEqual, cloneDeep } from "lodash"
+  import { Helpers } from "@budibase/bbui"
 
   export let componentInstance
   export let value
 
-  $: bindings = getBindableProperties($selectedScreen, componentInstance._id)
-
   const dispatch = createEventDispatcher()
-  let buttonList
+
   let mounted
   let cachedValue
   let componentBindings = []
+  let focusItem
 
+  const updateState = value => {
+    return (value || []).map((button, idx) => {
+      if (!button._component) {
+        return buildPseudoInstance({ ...button, name: `Button ${idx + 1}` })
+      }
+      return button
+    })
+  }
+
+  const buildItemProps = buttonList => {
+    return {
+      componentBindings,
+      bindings,
+      removeButton,
+      canRemove: buttonList?.length > 1,
+    }
+  }
+
+  $: if (!isEqual(value, cachedValue) && mounted) {
+    cachedValue = value ? cloneDeep(value) : []
+  }
+  $: buttonList = updateState(cachedValue)
+  $: buttonCount = buttonList?.length
+  $: bindings = getBindableProperties($selectedScreen, componentInstance._id)
   $: componentBindings = getComponentBindableProperties(
     $selectedScreen,
     componentInstance._id
   )
-
-  $: if (!isEqual(value, cachedValue) && mounted) {
-    cachedValue = value
-      ? cloneDeep(value)
-      : [
-          buildPseudoInstance({ name: `Button 1`, type: "cta" }),
-          buildPseudoInstance({ name: `Button 2` }),
-        ]
-  }
-
-  const updateState = value => {
-    buttonList = value
-  }
-
-  $: updateState(cachedValue)
+  $: itemProps = buildItemProps(buttonList)
 
   const processItemUpdate = e => {
     const updatedField = e.detail
@@ -53,7 +63,6 @@
     } else {
       parentButtonsUpdated[parentFieldIdx] = updatedField
     }
-    console.log("On value update ", parentButtonsUpdated)
     dispatch("change", parentButtonsUpdated)
   }
 
@@ -61,12 +70,11 @@
     dispatch("change", [...e.detail])
   }
 
-  // May not be necessary without
   const buildPseudoInstance = cfg => {
     const pseudoComponentInstance = store.actions.components.createInstance(
       `@budibase/standard-components/button`,
       {
-        _instanceName: cfg.name,
+        _instanceName: Helpers.uuid(),
         text: cfg.name,
         type: cfg.type || "primary",
       },
@@ -76,6 +84,21 @@
     return pseudoComponentInstance
   }
 
+  const addButton = () => {
+    const newButton = buildPseudoInstance({
+      name: `Button ${buttonList.length + 1}`,
+    })
+    dispatch("change", [...buttonList, newButton])
+    focusItem = newButton._id
+  }
+
+  const removeButton = id => {
+    dispatch(
+      "change",
+      buttonList.filter(button => button._id !== id)
+    )
+  }
+
   onMount(() => {
     mounted = true
   })
@@ -89,11 +112,14 @@
       items={buttonList}
       listItemKey={"_id"}
       listType={ButtonSetting}
-      listTypeProps={{
-        componentBindings,
-        bindings,
-      }}
+      listTypeProps={itemProps}
+      focus={focusItem}
+      draggable={buttonCount > 1}
     />
+
+    <div class="list-footer" on:click={addButton}>
+      <div class="add-button">Add button</div>
+    </div>
   {/if}
 </div>
 
@@ -101,4 +127,38 @@
   .button-configuration :global(.spectrum-ActionButton) {
     width: 100%;
   }
+
+  .button-configuration :global(.list-wrap > li:last-child),
+  .button-configuration :global(.list-wrap) {
+    border-bottom-left-radius: unset;
+    border-bottom-right-radius: unset;
+    border-bottom: 0px;
+  }
+
+  .list-footer {
+    width: 100%;
+    border-bottom-left-radius: 4px;
+    border-bottom-right-radius: 4px;
+    background-color: var(
+      --spectrum-table-background-color,
+      var(--spectrum-global-color-gray-50)
+    );
+    transition: background-color ease-in-out 130ms;
+    display: flex;
+    justify-content: center;
+    border: 1px solid
+      var(--spectrum-table-border-color, var(--spectrum-alias-border-color-mid));
+    cursor: pointer;
+  }
+
+  .add-button {
+    margin: var(--spacing-s);
+  }
+
+  .list-footer:hover {
+    background-color: var(
+      --spectrum-table-row-background-color-hover,
+      var(--spectrum-alias-highlight-hover)
+    );
+  }
 </style>
diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
index 851cbe289c..a05fd9a39b 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
@@ -1,37 +1,40 @@
 <script>
-  import EditFieldPopover from "../FieldConfiguration/EditFieldPopover.svelte"
+  import EditComponentPopover from "../EditComponentPopover.svelte"
   import { Icon } from "@budibase/bbui"
-  import { createEventDispatcher } from "svelte"
-  // import { cloneDeep } from "lodash/fp"
+  import { runtimeToReadableBinding } from "builderStore/dataBinding"
+  import { isJSBinding } from "@budibase/string-templates"
 
   export let item
   export let componentBindings
   export let bindings
   export let anchor
+  export let removeButton
+  export let canRemove
 
-  const dispatch = createEventDispatcher()
-  // const onToggle = item => {
-  //   return e => {
-  //     item.active = e.detail
-  //     dispatch("change", { ...cloneDeep(item), active: e.detail })
-  //   }
-  // }
+  $: readableText = isJSBinding(item.text)
+    ? "(JavaScript function)"
+    : runtimeToReadableBinding([...bindings, componentBindings], item.text)
 </script>
 
 <div class="list-item-body">
   <div class="list-item-left">
-    <EditFieldPopover
+    <EditComponentPopover
       {anchor}
-      field={item}
+      componentInstance={item}
       {componentBindings}
       {bindings}
       on:change
     />
-    <div class="field-label">{item.text}</div>
+    <div class="field-label">{readableText || "Button"}</div>
   </div>
   <div class="list-item-right">
-    <Icon size="S" name="Close" hoverable on:click={() => {console.log("REMOVE ME")}}
-  />
+    <Icon
+      disabled={!canRemove}
+      size="S"
+      name="Close"
+      hoverable
+      on:click={() => removeButton(item._id)}
+    />
   </div>
 </div>
 
diff --git a/packages/builder/src/components/design/settings/controls/DraggableList.svelte b/packages/builder/src/components/design/settings/controls/DraggableList/DraggableList.svelte
similarity index 82%
rename from packages/builder/src/components/design/settings/controls/DraggableList.svelte
rename to packages/builder/src/components/design/settings/controls/DraggableList/DraggableList.svelte
index c8395b2a1f..1992299e90 100644
--- a/packages/builder/src/components/design/settings/controls/DraggableList.svelte
+++ b/packages/builder/src/components/design/settings/controls/DraggableList/DraggableList.svelte
@@ -1,10 +1,10 @@
 <script>
-  import { Icon } from "@budibase/bbui"
   import { dndzone } from "svelte-dnd-action"
   import { createEventDispatcher } from "svelte"
   import { generate } from "shortid"
   import { setContext } from "svelte"
-  import { writable } from "svelte/store"
+  import { writable, get } from "svelte/store"
+  import DragHandle from "./drag-handle.svelte"
 
   export let items = []
   export let showHandle = true
@@ -12,6 +12,7 @@
   export let listTypeProps = {}
   export let listItemKey
   export let draggable = true
+  export let focus
 
   let store = writable({
     selected: null,
@@ -27,6 +28,10 @@
 
   setContext("draggable", store)
 
+  $: if (focus && store) {
+    get(store).actions.select(focus)
+  }
+
   const dispatch = createEventDispatcher()
   const flipDurationMs = 150
 
@@ -82,13 +87,16 @@
 >
   {#each draggableItems as draggable (draggable.id)}
     <li
+      on:mousedown={() => {
+        get(store).actions.select()
+      }}
       bind:this={anchors[draggable.id]}
       class:highlighted={draggable.id === $store.selected}
     >
       <div class="left-content">
         {#if showHandle}
-          <div class="handle" aria-label="drag-handle">
-            <Icon name="DragHandle" size="XL" />
+          <div class="handle">
+            <DragHandle />
           </div>
         {/if}
       </div>
@@ -142,8 +150,9 @@
     border-top-right-radius: 4px;
   }
   .list-wrap > li:last-child {
-    border-top-left-radius: var(--spectrum-table-regular-border-radius);
-    border-top-right-radius: var(--spectrum-table-regular-border-radius);
+    border-bottom-left-radius: 4px;
+    border-bottom-right-radius: 4px;
+    border-bottom: 0px;
   }
   .right-content {
     flex: 1;
@@ -153,4 +162,15 @@
     padding-left: var(--spacing-s);
     padding-right: var(--spacing-s);
   }
+  .handle {
+    display: flex;
+    height: var(--spectrum-global-dimension-size-150);
+  }
+  .handle :global(svg) {
+    fill: var(--spectrum-global-color-gray-500);
+    margin-right: var(--spacing-m);
+    margin-left: 2px;
+    width: var(--spectrum-global-dimension-size-65);
+    height: 100%;
+  }
 </style>
diff --git a/packages/builder/src/components/design/settings/controls/DraggableList/drag-handle.svelte b/packages/builder/src/components/design/settings/controls/DraggableList/drag-handle.svelte
new file mode 100644
index 0000000000..5cfefdbe54
--- /dev/null
+++ b/packages/builder/src/components/design/settings/controls/DraggableList/drag-handle.svelte
@@ -0,0 +1,31 @@
+<svg
+  class="drag-handle spectrum-Icon spectrum-Icon--sizeS"
+  focusable="false"
+  aria-hidden="true"
+  xmlns="http://www.w3.org/2000/svg"
+>
+  <path
+    d="m1,11c0.55228,0 1,-0.4477 1,-1c0,-0.5523 -0.44772,-1 -1,-1c-0.55228,0 -1,0.4477 -1,1c0,0.5523 0.44772,1 1,1z"
+  />
+  <path
+    d="m1,8c0.55228,0 1,-0.4477 1,-1c0,-0.55228 -0.44772,-1 -1,-1c-0.55228,0 -1,0.44772 -1,1c0,0.5523 0.44772,1 1,1z"
+  />
+  <path
+    d="m1,5c0.55228,0 1,-0.44772 1,-1c0,-0.55228 -0.44772,-1 -1,-1c-0.55228,0 -1,0.44772 -1,1c0,0.55228 0.44772,1 1,1z"
+  />
+  <path
+    d="m1,2c0.55228,0 1,-0.44772 1,-1c0,-0.55228 -0.44772,-1 -1,-1c-0.55228,0 -1,0.44772 -1,1c0,0.55228 0.44772,1 1,1z"
+  />
+  <path
+    d="m4,11c0.5523,0 1,-0.4477 1,-1c0,-0.5523 -0.4477,-1 -1,-1c-0.55228,0 -1,0.4477 -1,1c0,0.5523 0.44772,1 1,1z"
+  />
+  <path
+    d="m4,8c0.5523,0 1,-0.4477 1,-1c0,-0.55228 -0.4477,-1 -1,-1c-0.55228,0 -1,0.44772 -1,1c0,0.5523 0.44772,1 1,1z"
+  />
+  <path
+    d="m4,5c0.5523,0 1,-0.44772 1,-1c0,-0.55228 -0.4477,-1 -1,-1c-0.55228,0 -1,0.44772 -1,1c0,0.55228 0.44772,1 1,1z"
+  />
+  <path
+    d="m4,2c0.5523,0 1,-0.44772 1,-1c0,-0.55228 -0.4477,-1 -1,-1c-0.55228,0 -1,0.44772 -1,1c0,0.55228 0.44772,1 1,1z"
+  />
+</svg>
diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/EditFieldPopover.svelte b/packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte
similarity index 63%
rename from packages/builder/src/components/design/settings/controls/FieldConfiguration/EditFieldPopover.svelte
rename to packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte
index 72e7784727..1533c0d1d5 100644
--- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/EditFieldPopover.svelte
+++ b/packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte
@@ -3,31 +3,35 @@
   import { store } from "builderStore"
   import { cloneDeep } from "lodash/fp"
   import { createEventDispatcher } from "svelte"
-  import ComponentSettingsSection from "../../../../../pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte"
+  import ComponentSettingsSection from "../../../../pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte"
   import { getContext } from "svelte"
 
   export let anchor
-  export let field
+  export let componentInstance
   export let componentBindings
   export let bindings
+  export let parseSettings
 
   const draggable = getContext("draggable")
   const dispatch = createEventDispatcher()
 
   let popover
   let drawers = []
-  let pseudoComponentInstance
   let open = false
 
-  $: if (open && $draggable.selected && $draggable.selected != field._id) {
+  // Auto hide the component when another item is selected
+  $: if (open && $draggable.selected != componentInstance._id) {
     popover.hide()
   }
 
-  $: if (field) {
-    pseudoComponentInstance = field
+  // Open automatically if the component is marked as selected
+  $: if (!open && $draggable.selected === componentInstance._id && popover) {
+    popover.show()
+    open = true
   }
+
   $: componentDef = store.actions.components.getDefinition(
-    pseudoComponentInstance._component
+    componentInstance._component
   )
   $: parsedComponentDef = processComponentDefinitionSettings(componentDef)
 
@@ -36,17 +40,16 @@
       return {}
     }
     const clone = cloneDeep(componentDef)
-    const updatedSettings = clone.settings
-      .filter(setting => setting.key !== "field")
-      .map(setting => {
-        return { ...setting, nested: true }
-      })
-    clone.settings = updatedSettings
+
+    if (typeof parseSettings === "function") {
+      clone.settings = parseSettings(clone.settings)
+    }
+
     return clone
   }
 
   const updateSetting = async (setting, value) => {
-    const nestedComponentInstance = cloneDeep(pseudoComponentInstance)
+    const nestedComponentInstance = cloneDeep(componentInstance)
 
     const patchFn = store.actions.components.updateComponentSetting(
       setting.key,
@@ -54,12 +57,7 @@
     )
     patchFn(nestedComponentInstance)
 
-    const update = {
-      ...nestedComponentInstance,
-      active: pseudoComponentInstance.active,
-    }
-
-    dispatch("change", update)
+    dispatch("change", nestedComponentInstance)
   }
 
   const customPositionHandler = (anchorBounds, eleBounds, cfg) => {
@@ -98,11 +96,11 @@
   bind:this={popover}
   on:open={() => {
     drawers = []
-    $draggable.actions.select(field._id)
+    $draggable.actions.select(componentInstance._id)
   }}
   on:close={() => {
     open = false
-    if ($draggable.selected == field._id) {
+    if ($draggable.selected == componentInstance._id) {
       $draggable.actions.select()
     }
   }}
@@ -115,12 +113,9 @@
 >
   <span class="popover-wrap">
     <Layout noPadding noGap>
-      <div class="type-icon">
-        <Icon name={parsedComponentDef.icon} />
-        <span>{field.field}</span>
-      </div>
+      <slot name="header" />
       <ComponentSettingsSection
-        componentInstance={pseudoComponentInstance}
+        {componentInstance}
         componentDefinition={parsedComponentDef}
         isScreen={false}
         onUpdateSetting={updateSetting}
@@ -143,20 +138,4 @@
   .popover-wrap {
     background-color: var(--spectrum-alias-background-color-primary);
   }
-  .type-icon {
-    display: flex;
-    gap: var(--spacing-m);
-    margin: var(--spacing-xl);
-    margin-bottom: 0px;
-    height: var(--spectrum-alias-item-height-m);
-    padding: 0px var(--spectrum-alias-item-padding-m);
-    border-width: var(--spectrum-actionbutton-border-size);
-    border-radius: var(--spectrum-alias-border-radius-regular);
-    border: 1px solid
-      var(
-        --spectrum-actionbutton-m-border-color,
-        var(--spectrum-alias-border-color)
-      );
-    align-items: center;
-  }
 </style>
diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
index 42651a4d84..6c74705ab0 100644
--- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
@@ -7,7 +7,7 @@
     getComponentBindableProperties,
   } from "builderStore/dataBinding"
   import { currentAsset } from "builderStore"
-  import DraggableList from "../DraggableList.svelte"
+  import DraggableList from "../DraggableList/DraggableList.svelte"
   import { createEventDispatcher } from "svelte"
   import { store, selectedScreen } from "builderStore"
   import FieldSetting from "./FieldSetting.svelte"
diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte
index b5cfcb12d9..1d9ce733b8 100644
--- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte
+++ b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte
@@ -1,8 +1,11 @@
 <script>
-  import EditFieldPopover from "./EditFieldPopover.svelte"
-  import { Toggle } from "@budibase/bbui"
+  import EditComponentPopover from "../EditComponentPopover.svelte"
+  import { Toggle, Icon } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { cloneDeep } from "lodash/fp"
+  import { store } from "builderStore"
+  import { runtimeToReadableBinding } from "builderStore/dataBinding"
+  import { isJSBinding } from "@budibase/string-templates"
 
   export let item
   export let componentBindings
@@ -16,18 +19,43 @@
       dispatch("change", { ...cloneDeep(item), active: e.detail })
     }
   }
+  const getReadableText = () => {
+    if (item.label) {
+      return isJSBinding(item.label)
+        ? "(JavaScript function)"
+        : runtimeToReadableBinding([...bindings, componentBindings], item.label)
+    }
+    return item.field
+  }
+
+  const parseSettings = settings => {
+    return settings
+      .filter(setting => setting.key !== "field")
+      .map(setting => {
+        return { ...setting, nested: true }
+      })
+  }
+
+  $: readableText = getReadableText(item)
+  $: componentDef = store.actions.components.getDefinition(item._component)
 </script>
 
 <div class="list-item-body">
   <div class="list-item-left">
-    <EditFieldPopover
+    <EditComponentPopover
       {anchor}
-      field={item}
+      componentInstance={item}
       {componentBindings}
       {bindings}
+      {parseSettings}
       on:change
-    />
-    <div class="field-label">{item.label || item.field}</div>
+    >
+      <div slot="header" class="type-icon">
+        <Icon name={componentDef.icon} />
+        <span>{item.field}</span>
+      </div>
+    </EditComponentPopover>
+    <div class="field-label">{readableText}</div>
   </div>
   <div class="list-item-right">
     <Toggle on:change={onToggle(item)} text="" value={item.active} thin />
@@ -53,4 +81,20 @@
   .list-item-body {
     justify-content: space-between;
   }
+  .type-icon {
+    display: flex;
+    gap: var(--spacing-m);
+    margin: var(--spacing-xl);
+    margin-bottom: 0px;
+    height: var(--spectrum-alias-item-height-m);
+    padding: 0px var(--spectrum-alias-item-padding-m);
+    border-width: var(--spectrum-actionbutton-border-size);
+    border-radius: var(--spectrum-alias-border-radius-regular);
+    border: 1px solid
+      var(
+        --spectrum-actionbutton-m-border-color,
+        var(--spectrum-alias-border-color)
+      );
+    align-items: center;
+  }
 </style>
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 27e56d94db..a8559d4f79 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -258,7 +258,6 @@
     "description": "Contains your app screens",
     "static": true
   },
-
   "buttongroup": {
     "name": "Button group",
     "icon": "Button",
@@ -272,17 +271,14 @@
             "type": "buttonConfiguration",
             "key": "buttons",
             "nested": true,
-            "defaultValue" : [{
-              "component" : "button",
-              "props" : {
+            "defaultValue": [
+              {
                 "type": "cta"
+              },
+              {
+                "type": "primary"
               }
-            },{
-              "component" : "button",
-              "props" : {
-                "type" : "primary"
-              }
-            }]
+            ]
           }
         ]
       },
@@ -310,7 +306,7 @@
                 "barTitle": "Row layout"
               }
             ],
-            "defaultValue": "column"
+            "defaultValue": "row"
           },
           {
             "type": "select",
@@ -440,7 +436,6 @@
       }
     ]
   },
-
   "button": {
     "name": "Button",
     "description": "A basic html button that is ready for styling",
@@ -2592,7 +2587,6 @@
         "key": "disabled",
         "defaultValue": false
       },
-
       {
         "type": "text",
         "label": "Initial form step",
@@ -5875,4 +5869,4 @@
       }
     ]
   }
-}
+}
\ No newline at end of file
diff --git a/packages/client/src/components/app/ButtonGroup.svelte b/packages/client/src/components/app/ButtonGroup.svelte
new file mode 100644
index 0000000000..8d6cacd7a5
--- /dev/null
+++ b/packages/client/src/components/app/ButtonGroup.svelte
@@ -0,0 +1,37 @@
+<script>
+  import BlockComponent from "../../BlockComponent.svelte"
+  import Block from "../../Block.svelte"
+
+  export let buttons = []
+  export let direction
+  export let hAlign
+  export let vAlign
+  export let gap = "S"
+</script>
+
+<Block>
+  <BlockComponent
+    type="container"
+    props={{
+      direction,
+      hAlign,
+      vAlign,
+      gap,
+      wrap: true,
+    }}
+  >
+    {#each buttons as { text, type, quiet, disabled, onClick, size }, idx}
+      <BlockComponent
+        type="button"
+        props={{
+          text: text || `Button ${idx + 1}`,
+          onClick,
+          type,
+          quiet,
+          disabled,
+          size,
+        }}
+      />
+    {/each}
+  </BlockComponent>
+</Block>
diff --git a/packages/client/src/components/app/forms/ButtonGroup.svelte b/packages/client/src/components/app/forms/ButtonGroup.svelte
deleted file mode 100644
index 222e91a55f..0000000000
--- a/packages/client/src/components/app/forms/ButtonGroup.svelte
+++ /dev/null
@@ -1,38 +0,0 @@
-<script>
-  import { getContext } from "svelte"
-  import Container from "components/app/Container.svelte"
-  import BlockComponent from "../../BlockComponent.svelte"
-  import Block from "../../Block.svelte"
-
-  const { styleable } = getContext("sdk")
-  const component = getContext("component")
-
-  export let buttons = []
-  export let direction
-  export let hAlign
-  export let vAlign
-  export let gap = "S"
-  export let wrap
-
-  $: console.log("client buttons/wrap", buttons, wrap)
-</script>
-
-<!-- <div use:styleable={$component.styles}> -->
-  <Block>
-    <Container {direction} {hAlign} {vAlign} {gap} {wrap}>
-      {#each buttons as { text, type, quiet, disabled, onClick, size }}
-        <BlockComponent
-          type="button"
-          props={{
-            text,
-            onClick,
-            type,
-            quiet,
-            disabled,
-            size
-          }}
-        />
-      {/each}
-    </Container>
-  </Block>
-<!-- </div> -->
diff --git a/packages/client/src/components/app/forms/index.js b/packages/client/src/components/app/forms/index.js
index 24d7f11c0c..38372f32d4 100644
--- a/packages/client/src/components/app/forms/index.js
+++ b/packages/client/src/components/app/forms/index.js
@@ -15,5 +15,4 @@ export { default as formstep } from "./FormStep.svelte"
 export { default as jsonfield } from "./JSONField.svelte"
 export { default as s3upload } from "./S3Upload.svelte"
 export { default as codescanner } from "./CodeScannerField.svelte"
-export { default as bbreferencefield } from "./BBReferenceField.svelte"
-export { default as buttongroup } from "./ButtonGroup.svelte"
\ No newline at end of file
+export { default as bbreferencefield } from "./BBReferenceField.svelte"
\ No newline at end of file
diff --git a/packages/client/src/components/app/index.js b/packages/client/src/components/app/index.js
index 060c15a857..97df3741e1 100644
--- a/packages/client/src/components/app/index.js
+++ b/packages/client/src/components/app/index.js
@@ -19,6 +19,7 @@ export { default as dataprovider } from "./DataProvider.svelte"
 export { default as divider } from "./Divider.svelte"
 export { default as screenslot } from "./ScreenSlot.svelte"
 export { default as button } from "./Button.svelte"
+export { default as buttongroup } from "./ButtonGroup.svelte"
 export { default as repeater } from "./Repeater.svelte"
 export { default as text } from "./Text.svelte"
 export { default as layout } from "./Layout.svelte"

From 166c517ff82d4644f37f5e6fe3b4ff2fef2b0d94 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Fri, 13 Oct 2023 15:02:18 +0100
Subject: [PATCH 003/134] Fix Block import locations

---
 packages/client/src/components/app/ButtonGroup.svelte | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/client/src/components/app/ButtonGroup.svelte b/packages/client/src/components/app/ButtonGroup.svelte
index 8d6cacd7a5..0af16e6950 100644
--- a/packages/client/src/components/app/ButtonGroup.svelte
+++ b/packages/client/src/components/app/ButtonGroup.svelte
@@ -1,6 +1,6 @@
 <script>
-  import BlockComponent from "../../BlockComponent.svelte"
-  import Block from "../../Block.svelte"
+  import BlockComponent from "../BlockComponent.svelte"
+  import Block from "../Block.svelte"
 
   export let buttons = []
   export let direction

From e75624f51996d55775bad769cd46e3cc0df830ba Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Fri, 13 Oct 2023 16:55:50 +0100
Subject: [PATCH 004/134] Lint

---
 packages/client/src/components/app/forms/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/components/app/forms/index.js b/packages/client/src/components/app/forms/index.js
index 38372f32d4..5804d3a79d 100644
--- a/packages/client/src/components/app/forms/index.js
+++ b/packages/client/src/components/app/forms/index.js
@@ -15,4 +15,4 @@ export { default as formstep } from "./FormStep.svelte"
 export { default as jsonfield } from "./JSONField.svelte"
 export { default as s3upload } from "./S3Upload.svelte"
 export { default as codescanner } from "./CodeScannerField.svelte"
-export { default as bbreferencefield } from "./BBReferenceField.svelte"
\ No newline at end of file
+export { default as bbreferencefield } from "./BBReferenceField.svelte"

From ddda57fa2a030a3e15a09836047efa4b9c84d88a Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 16 Oct 2023 12:01:07 +0100
Subject: [PATCH 005/134] Review comments and simplification

---
 .../ButtonConfiguration.svelte                | 82 ++++++-------------
 packages/client/manifest.json                 |  6 +-
 .../src/components/app/ButtonGroup.svelte     |  4 +-
 3 files changed, 32 insertions(+), 60 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
index fde888d17b..324418511b 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
@@ -1,69 +1,45 @@
 <script>
   import DraggableList from "../DraggableList/DraggableList.svelte"
   import ButtonSetting from "./ButtonSetting.svelte"
-  import {
-    getBindableProperties,
-    getComponentBindableProperties,
-  } from "builderStore/dataBinding"
-  import { createEventDispatcher, onMount } from "svelte"
-  import { selectedScreen, store } from "builderStore"
-  import { isEqual, cloneDeep } from "lodash"
+  import { createEventDispatcher } from "svelte"
+  import { store } from "builderStore"
   import { Helpers } from "@budibase/bbui"
 
-  export let componentInstance
+  export let componentBindings
+  export let bindings
   export let value
 
   const dispatch = createEventDispatcher()
 
-  let mounted
-  let cachedValue
-  let componentBindings = []
   let focusItem
 
-  const updateState = value => {
-    return (value || []).map((button, idx) => {
-      if (!button._component) {
-        return buildPseudoInstance({ ...button, name: `Button ${idx + 1}` })
-      }
-      return button
+  $: buttonList = sanitizeValue(value) || []
+  $: buttonCount = buttonList.length
+  $: itemProps = {
+    componentBindings: componentBindings || [],
+    bindings,
+    removeButton,
+    canRemove: buttonCount > 1,
+  }
+
+  const sanitizeValue = val => {
+    return val?.map(button => {
+      return button._component ? button : buildPseudoInstance(button)
     })
   }
 
-  const buildItemProps = buttonList => {
-    return {
-      componentBindings,
-      bindings,
-      removeButton,
-      canRemove: buttonList?.length > 1,
-    }
-  }
-
-  $: if (!isEqual(value, cachedValue) && mounted) {
-    cachedValue = value ? cloneDeep(value) : []
-  }
-  $: buttonList = updateState(cachedValue)
-  $: buttonCount = buttonList?.length
-  $: bindings = getBindableProperties($selectedScreen, componentInstance._id)
-  $: componentBindings = getComponentBindableProperties(
-    $selectedScreen,
-    componentInstance._id
-  )
-  $: itemProps = buildItemProps(buttonList)
-
   const processItemUpdate = e => {
     const updatedField = e.detail
-    const parentButtonsUpdated = buttonList ? cloneDeep(buttonList) : []
-
-    let parentFieldIdx = parentButtonsUpdated.findIndex(pSetting => {
+    const newButtonList = [...buttonList]
+    const fieldIdx = newButtonList.findIndex(pSetting => {
       return pSetting._id === updatedField?._id
     })
-
-    if (parentFieldIdx == -1) {
-      parentButtonsUpdated.push(updatedField)
+    if (fieldIdx === -1) {
+      newButtonList.push(updatedField)
     } else {
-      parentButtonsUpdated[parentFieldIdx] = updatedField
+      newButtonList[fieldIdx] = updatedField
     }
-    dispatch("change", parentButtonsUpdated)
+    dispatch("change", newButtonList)
   }
 
   const listUpdated = e => {
@@ -71,22 +47,20 @@
   }
 
   const buildPseudoInstance = cfg => {
-    const pseudoComponentInstance = store.actions.components.createInstance(
+    return store.actions.components.createInstance(
       `@budibase/standard-components/button`,
       {
         _instanceName: Helpers.uuid(),
-        text: cfg.name,
+        text: cfg.text,
         type: cfg.type || "primary",
       },
       {}
     )
-
-    return pseudoComponentInstance
   }
 
   const addButton = () => {
     const newButton = buildPseudoInstance({
-      name: `Button ${buttonList.length + 1}`,
+      text: `Button ${buttonCount + 1}`,
     })
     dispatch("change", [...buttonList, newButton])
     focusItem = newButton._id
@@ -98,14 +72,10 @@
       buttonList.filter(button => button._id !== id)
     )
   }
-
-  onMount(() => {
-    mounted = true
-  })
 </script>
 
 <div class="button-configuration">
-  {#if buttonList?.length}
+  {#if buttonCount}
     <DraggableList
       on:change={listUpdated}
       on:itemChange={processItemUpdate}
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 7cc7814462..8f78a123ea 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -273,10 +273,12 @@
             "nested": true,
             "defaultValue": [
               {
-                "type": "cta"
+                "type": "cta",
+                "text": "Button 1"
               },
               {
-                "type": "primary"
+                "type": "primary",
+                "text": "Button 2"
               }
             ]
           }
diff --git a/packages/client/src/components/app/ButtonGroup.svelte b/packages/client/src/components/app/ButtonGroup.svelte
index 0af16e6950..87b0990701 100644
--- a/packages/client/src/components/app/ButtonGroup.svelte
+++ b/packages/client/src/components/app/ButtonGroup.svelte
@@ -20,11 +20,11 @@
       wrap: true,
     }}
   >
-    {#each buttons as { text, type, quiet, disabled, onClick, size }, idx}
+    {#each buttons as { text, type, quiet, disabled, onClick, size }}
       <BlockComponent
         type="button"
         props={{
-          text: text || `Button ${idx + 1}`,
+          text: text || "Button",
           onClick,
           type,
           quiet,

From 36346874428a999206f79d5a396dbeaefa15743c Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 18 Oct 2023 10:31:36 +0100
Subject: [PATCH 006/134] Create endpoint and controller function for user
 column migration.

---
 packages/server/src/api/controllers/table/index.ts | 2 ++
 packages/server/src/api/routes/table.ts            | 7 +++++++
 2 files changed, 9 insertions(+)

diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index 44f673f284..2780185ed7 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -158,3 +158,5 @@ export async function validateExistingTableImport(ctx: UserCtx) {
     ctx.status = 422
   }
 }
+
+export async function migrate(ctx: UserCtx) {}
diff --git a/packages/server/src/api/routes/table.ts b/packages/server/src/api/routes/table.ts
index 7ffa5acb3e..1aa23257fb 100644
--- a/packages/server/src/api/routes/table.ts
+++ b/packages/server/src/api/routes/table.ts
@@ -167,4 +167,11 @@ router
     tableController.bulkImport
   )
 
+  .post(
+    "/api/tables/:tableId/migrate",
+    paramResource("tableId"),
+    authorized(BUILDER),
+    tableController.migrate
+  )
+
 export default router

From 5747f30b5f5cccd7ea5b78a6565e56edfc919400 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 18 Oct 2023 12:04:50 +0100
Subject: [PATCH 007/134] Precondition checks to make sure the migration is
 from the right column to the right column.

---
 .../server/src/api/controllers/table/index.ts | 56 ++++++++++++++++++-
 packages/types/src/api/web/app/table.ts       | 11 ++++
 .../types/src/documents/app/table/schema.ts   | 30 ++++++++++
 3 files changed, 96 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index 2780185ed7..36dee4811e 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -11,11 +11,16 @@ import {
   BulkImportRequest,
   BulkImportResponse,
   FetchTablesResponse,
+  InternalTable,
+  MigrateRequest,
+  MigrateResponse,
   SaveTableRequest,
   SaveTableResponse,
   Table,
   TableResponse,
   UserCtx,
+  isBBReferenceField,
+  isRelationshipField,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"
@@ -159,4 +164,53 @@ export async function validateExistingTableImport(ctx: UserCtx) {
   }
 }
 
-export async function migrate(ctx: UserCtx) {}
+function error(ctx: UserCtx, message: string, status = 400) {
+  ctx.status = status
+  ctx.body = { message }
+}
+
+export async function migrate(ctx: UserCtx<MigrateRequest, MigrateResponse>) {
+  const { tableId, oldColumn, newColumn } = ctx.request.body
+
+  // For now we're only supporting migrations of user relationships to user
+  // columns in internal tables. In future we may want to support other
+  // migrations but for now return an error if we aren't migrating a user
+  // relationship.
+  if (isExternalTable(tableId)) {
+    return error(ctx, "External tables cannot be migrated")
+  }
+
+  const table = await sdk.tables.getTable(tableId)
+
+  if (!(oldColumn.name in table.schema)) {
+    return error(
+      ctx,
+      `Column "${oldColumn.name}" does not exist on table "${table.name}"`
+    )
+  }
+
+  if (newColumn.name in table.schema) {
+    return error(
+      ctx,
+      `Column "${newColumn.name}" already exists on table "${table.name}"`
+    )
+  }
+
+  if (!isBBReferenceField(newColumn)) {
+    return error(ctx, `Column "${newColumn.name}" is not a user column`)
+  }
+
+  if (newColumn.subtype !== "user" && newColumn.subtype !== "users") {
+    return error(ctx, `Column "${newColumn.name}" is not a user column`)
+  }
+
+  if (!isRelationshipField(oldColumn)) {
+    return error(ctx, `Column "${oldColumn.name}" is not a user relationship`)
+  }
+
+  if (oldColumn.tableId !== InternalTable.USER_METADATA) {
+    return error(ctx, `Column "${oldColumn.name}" is not a user relationship`)
+  }
+
+  let rows = await sdk.rows.fetch(tableId)
+}
diff --git a/packages/types/src/api/web/app/table.ts b/packages/types/src/api/web/app/table.ts
index cb5faaa9ea..61da12d041 100644
--- a/packages/types/src/api/web/app/table.ts
+++ b/packages/types/src/api/web/app/table.ts
@@ -1,4 +1,5 @@
 import {
+  FieldSchema,
   Row,
   Table,
   TableRequest,
@@ -33,3 +34,13 @@ export interface BulkImportRequest {
 export interface BulkImportResponse {
   message: string
 }
+
+export interface MigrateRequest {
+  tableId: string
+  oldColumn: FieldSchema
+  newColumn: FieldSchema
+}
+
+export interface MigrateResponse {
+  message: string
+}
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index e529a8e8b7..755ccf61e7 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -164,3 +164,33 @@ export type FieldSchema =
 export interface TableSchema {
   [key: string]: FieldSchema
 }
+
+export function isRelationshipField(
+  field: FieldSchema
+): field is RelationshipFieldMetadata {
+  return field.type === FieldType.LINK
+}
+
+export function isManyToMany(
+  field: RelationshipFieldMetadata
+): field is ManyToManyRelationshipFieldMetadata {
+  return field.relationshipType === RelationshipType.MANY_TO_MANY
+}
+
+export function isOneToMany(
+  field: RelationshipFieldMetadata
+): field is OneToManyRelationshipFieldMetadata {
+  return field.relationshipType === RelationshipType.ONE_TO_MANY
+}
+
+export function isManyToOne(
+  field: RelationshipFieldMetadata
+): field is ManyToOneRelationshipFieldMetadata {
+  return field.relationshipType === RelationshipType.MANY_TO_ONE
+}
+
+export function isBBReferenceField(
+  field: FieldSchema
+): field is BBReferenceFieldMetadata {
+  return field.type === FieldType.BB_REFERENCE
+}

From 0d67f000f03e1c27692e92dcf866cd36b55435cd Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Wed, 18 Oct 2023 12:19:24 +0100
Subject: [PATCH 008/134] Initial reflow of the block settings

---
 packages/client/manifest.json | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 8d0a4e456f..d5fca15fd7 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -5288,17 +5288,17 @@
     },
     "settings": [
       {
-        "type": "select",
+        "type": "table",
+        "label": "Data",
+        "key": "dataSource"
+      },
+      {
+        "type": "radio",
         "label": "Type",
         "key": "actionType",
         "options": ["Create", "Update", "View"],
         "defaultValue": "Create"
       },
-      {
-        "type": "table",
-        "label": "Data",
-        "key": "dataSource"
-      },
       {
         "type": "text",
         "label": "Title",
@@ -5323,7 +5323,7 @@
           },
           {
             "type": "text",
-            "label": "Empty text",
+            "label": "No rows found",
             "key": "noRowsMessage",
             "defaultValue": "We couldn't find a row to display",
             "nested": true

From c25de74e174c9e2fa895f5aa8e2a5cc51f9968bf Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 18 Oct 2023 15:14:34 +0100
Subject: [PATCH 009/134] Action Michael's feedback about the structure of this
 feature.

---
 .../server/src/api/controllers/table/index.ts | 50 +----------
 packages/server/src/sdk/app/tables/index.ts   | 12 +++
 .../server/src/sdk/app/tables/migration.ts    | 84 +++++++++++++++++++
 3 files changed, 97 insertions(+), 49 deletions(-)
 create mode 100644 packages/server/src/sdk/app/tables/migration.ts

diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index 36dee4811e..132bae3d81 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -11,7 +11,6 @@ import {
   BulkImportRequest,
   BulkImportResponse,
   FetchTablesResponse,
-  InternalTable,
   MigrateRequest,
   MigrateResponse,
   SaveTableRequest,
@@ -19,8 +18,6 @@ import {
   Table,
   TableResponse,
   UserCtx,
-  isBBReferenceField,
-  isRelationshipField,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"
@@ -164,53 +161,8 @@ export async function validateExistingTableImport(ctx: UserCtx) {
   }
 }
 
-function error(ctx: UserCtx, message: string, status = 400) {
-  ctx.status = status
-  ctx.body = { message }
-}
-
 export async function migrate(ctx: UserCtx<MigrateRequest, MigrateResponse>) {
   const { tableId, oldColumn, newColumn } = ctx.request.body
-
-  // For now we're only supporting migrations of user relationships to user
-  // columns in internal tables. In future we may want to support other
-  // migrations but for now return an error if we aren't migrating a user
-  // relationship.
-  if (isExternalTable(tableId)) {
-    return error(ctx, "External tables cannot be migrated")
-  }
-
   const table = await sdk.tables.getTable(tableId)
-
-  if (!(oldColumn.name in table.schema)) {
-    return error(
-      ctx,
-      `Column "${oldColumn.name}" does not exist on table "${table.name}"`
-    )
-  }
-
-  if (newColumn.name in table.schema) {
-    return error(
-      ctx,
-      `Column "${newColumn.name}" already exists on table "${table.name}"`
-    )
-  }
-
-  if (!isBBReferenceField(newColumn)) {
-    return error(ctx, `Column "${newColumn.name}" is not a user column`)
-  }
-
-  if (newColumn.subtype !== "user" && newColumn.subtype !== "users") {
-    return error(ctx, `Column "${newColumn.name}" is not a user column`)
-  }
-
-  if (!isRelationshipField(oldColumn)) {
-    return error(ctx, `Column "${oldColumn.name}" is not a user relationship`)
-  }
-
-  if (oldColumn.tableId !== InternalTable.USER_METADATA) {
-    return error(ctx, `Column "${oldColumn.name}" is not a user relationship`)
-  }
-
-  let rows = await sdk.rows.fetch(tableId)
+  await sdk.tables.migrate(table, oldColumn, newColumn)
 }
diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts
index 64fcde4bff..e4884b2ec4 100644
--- a/packages/server/src/sdk/app/tables/index.ts
+++ b/packages/server/src/sdk/app/tables/index.ts
@@ -7,6 +7,7 @@ import {
 } from "../../../integrations/utils"
 import {
   Database,
+  FieldSchema,
   Table,
   TableResponse,
   TableViewsResponse,
@@ -14,6 +15,7 @@ import {
 import datasources from "../datasources"
 import { populateExternalTableSchemas } from "./validation"
 import sdk from "../../../sdk"
+import { migrate } from "./migration"
 
 async function getAllInternalTables(db?: Database): Promise<Table[]> {
   if (!db) {
@@ -84,6 +86,14 @@ async function saveTable(table: Table) {
   }
 }
 
+async function addColumn(table: Table, newColumn: FieldSchema) {
+  if (newColumn.name in table.schema) {
+    throw `Column "${newColumn.name}" already exists on table "${table.name}"`
+  }
+  table.schema[newColumn.name] = newColumn
+  await saveTable(table)
+}
+
 export default {
   getAllInternalTables,
   getAllExternalTables,
@@ -92,4 +102,6 @@ export default {
   populateExternalTableSchemas,
   enrichViewSchemas,
   saveTable,
+  addColumn,
+  migrate,
 }
diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts
new file mode 100644
index 0000000000..f727b5ea9d
--- /dev/null
+++ b/packages/server/src/sdk/app/tables/migration.ts
@@ -0,0 +1,84 @@
+import { BadRequestError } from "@budibase/backend-core"
+import {
+  BBReferenceFieldMetadata,
+  FieldSchema,
+  InternalTable,
+  RelationshipFieldMetadata,
+  Table,
+  isBBReferenceField,
+  isRelationshipField,
+} from "@budibase/types"
+import { isExternalTable } from "src/integrations/utils"
+import sdk from "../../../sdk"
+
+export async function migrate(
+  table: Table,
+  oldColumn: FieldSchema,
+  newColumn: FieldSchema
+) {
+  let migrator = getColumnMigrator(table, oldColumn, newColumn)
+
+  await sdk.tables.addColumn(table, newColumn)
+
+  migrator.doMigration()
+}
+
+interface ColumnMigrator {
+  doMigration(): Promise<void>
+}
+
+function getColumnMigrator(
+  table: Table,
+  oldColumn: FieldSchema,
+  newColumn: FieldSchema
+): ColumnMigrator {
+  // For now we're only supporting migrations of user relationships to user
+  // columns in internal tables. In future we may want to support other
+  // migrations but for now return an error if we aren't migrating a user
+  // relationship.
+  if (isExternalTable(table._id!)) {
+    throw new BadRequestError("External tables cannot be migrated")
+  }
+
+  if (!(oldColumn.name in table.schema)) {
+    throw new BadRequestError(`Column "${oldColumn.name}" does not exist`)
+  }
+
+  if (newColumn.name in table.schema) {
+    throw new BadRequestError(`Column "${newColumn.name}" already exists`)
+  }
+
+  if (!isBBReferenceField(newColumn)) {
+    throw new BadRequestError(`Column "${newColumn.name}" is not a user column`)
+  }
+
+  if (newColumn.subtype !== "user" && newColumn.subtype !== "users") {
+    throw new BadRequestError(`Column "${newColumn.name}" is not a user column`)
+  }
+
+  if (!isRelationshipField(oldColumn)) {
+    throw new BadRequestError(
+      `Column "${oldColumn.name}" is not a user relationship`
+    )
+  }
+
+  if (oldColumn.tableId !== InternalTable.USER_METADATA) {
+    throw new BadRequestError(
+      `Column "${oldColumn.name}" is not a user relationship`
+    )
+  }
+
+  return new UserColumnMigrator(table, oldColumn, newColumn)
+}
+
+class UserColumnMigrator implements ColumnMigrator {
+  constructor(
+    private table: Table,
+    private oldColumn: RelationshipFieldMetadata,
+    private newColumn: BBReferenceFieldMetadata
+  ) {}
+
+  async doMigration() {
+    let rows = await sdk.rows.fetch(this.table._id!)
+  }
+}

From 6ae5451fdf46db2dce172047a310b64ebbf596a7 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 18 Oct 2023 16:56:55 +0100
Subject: [PATCH 010/134] Create failing test.

---
 .../server/src/api/controllers/table/index.ts |  6 ++-
 .../server/src/api/routes/tests/table.spec.ts | 44 +++++++++++++++++++
 packages/server/src/sdk/app/links/index.ts    |  5 +++
 packages/server/src/sdk/app/links/links.ts    | 32 ++++++++++++++
 packages/server/src/sdk/app/rows/search.ts    |  6 +--
 .../server/src/sdk/app/tables/migration.ts    |  3 +-
 packages/server/src/sdk/index.ts              |  2 +
 .../server/src/tests/utilities/api/table.ts   | 22 +++++++++-
 packages/types/src/api/web/app/table.ts       |  1 -
 9 files changed, 114 insertions(+), 7 deletions(-)
 create mode 100644 packages/server/src/sdk/app/links/index.ts
 create mode 100644 packages/server/src/sdk/app/links/links.ts

diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index 132bae3d81..75532dbfdf 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -162,7 +162,11 @@ export async function validateExistingTableImport(ctx: UserCtx) {
 }
 
 export async function migrate(ctx: UserCtx<MigrateRequest, MigrateResponse>) {
-  const { tableId, oldColumn, newColumn } = ctx.request.body
+  const { oldColumn, newColumn } = ctx.request.body
+  let tableId = ctx.params.tableId as string
   const table = await sdk.tables.getTable(tableId)
   await sdk.tables.migrate(table, oldColumn, newColumn)
+
+  ctx.status = 200
+  ctx.body = { message: `Column ${oldColumn.name} migrated.` }
 }
diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts
index ded54729b9..eaf3757ea8 100644
--- a/packages/server/src/api/routes/tests/table.spec.ts
+++ b/packages/server/src/api/routes/tests/table.spec.ts
@@ -6,6 +6,8 @@ import {
   Table,
   ViewCalculation,
   AutoFieldSubTypes,
+  InternalTable,
+  FieldSubtype,
 } from "@budibase/types"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
@@ -417,4 +419,46 @@ describe("/tables", () => {
       })
     })
   })
+
+  describe("migrate", () => {
+    it("should successfully migrate a user relationship to a user column", async () => {
+      const users = await config.api.row.fetch(InternalTable.USER_METADATA)
+      const table = await config.api.table.create({
+        name: "table",
+        type: "table",
+        schema: {
+          "user relationship": {
+            type: FieldType.LINK,
+            fieldName: "test",
+            name: "user relationship",
+            constraints: {
+              type: "array",
+              presence: false,
+            },
+            relationshipType: RelationshipType.ONE_TO_MANY,
+            tableId: InternalTable.USER_METADATA,
+          },
+        },
+      })
+
+      await config.api.row.save(table._id!, {
+        "user relationship": users,
+      })
+
+      await config.api.table.migrate(table._id!, {
+        oldColumn: table.schema["user relationship"],
+        newColumn: {
+          name: "user column",
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldSubtype.USER,
+        },
+      })
+
+      const migratedTable = await config.api.table.get(table._id!)
+      expect(migratedTable.schema["user column"]).toBeDefined()
+
+      const rows = await config.api.row.fetch(table._id!)
+      expect(rows[0]["user column"]).toBeDefined()
+    })
+  })
 })
diff --git a/packages/server/src/sdk/app/links/index.ts b/packages/server/src/sdk/app/links/index.ts
new file mode 100644
index 0000000000..6655a76656
--- /dev/null
+++ b/packages/server/src/sdk/app/links/index.ts
@@ -0,0 +1,5 @@
+import * as links from "./links"
+
+export default {
+  ...links,
+}
diff --git a/packages/server/src/sdk/app/links/links.ts b/packages/server/src/sdk/app/links/links.ts
new file mode 100644
index 0000000000..7f3dcbd4ac
--- /dev/null
+++ b/packages/server/src/sdk/app/links/links.ts
@@ -0,0 +1,32 @@
+import { context } from "@budibase/backend-core"
+import { isTableId } from "@budibase/backend-core/src/docIds"
+import { LinkDocument, LinkDocumentValue } from "@budibase/types"
+import { ViewName, getQueryIndex } from "../../../../src/db/utils"
+
+export async function fetch(tableId: string): Promise<LinkDocumentValue[]> {
+  if (!isTableId(tableId)) {
+    throw new Error(`Invalid tableId: ${tableId}`)
+  }
+
+  const db = context.getAppDB()
+  const params: any = { startkey: [tableId], endkey: [tableId, {}] }
+  const linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows
+  return linkRows.map(row => row.value as LinkDocumentValue)
+}
+
+export async function fetchWithDocument(
+  tableId: string
+): Promise<LinkDocument[]> {
+  if (!isTableId(tableId)) {
+    throw new Error(`Invalid tableId: ${tableId}`)
+  }
+
+  const db = context.getAppDB()
+  const params: any = {
+    startkey: [tableId],
+    endkey: [tableId, {}],
+    include_docs: true,
+  }
+  const linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows
+  return linkRows.map(row => row.doc as LinkDocument)
+}
diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index ced35db9be..f75bd07437 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -1,4 +1,4 @@
-import { SearchFilters, SearchParams, Row } from "@budibase/types"
+import { SearchFilters, SearchParams } from "@budibase/types"
 import { isExternalTable } from "../../../integrations/utils"
 import * as internal from "./search/internal"
 import * as external from "./search/external"
@@ -45,7 +45,7 @@ export async function exportRows(
   return pickApi(options.tableId).exportRows(options)
 }
 
-export async function fetch(tableId: string): Promise<Row[]> {
+export async function fetch(tableId: string) {
   return pickApi(tableId).fetch(tableId)
 }
 
@@ -53,6 +53,6 @@ export async function fetchView(
   tableId: string,
   viewName: string,
   params: ViewParams
-): Promise<Row[]> {
+) {
   return pickApi(tableId).fetchView(viewName, params)
 }
diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts
index f727b5ea9d..ac7f3a3765 100644
--- a/packages/server/src/sdk/app/tables/migration.ts
+++ b/packages/server/src/sdk/app/tables/migration.ts
@@ -8,8 +8,8 @@ import {
   isBBReferenceField,
   isRelationshipField,
 } from "@budibase/types"
-import { isExternalTable } from "src/integrations/utils"
 import sdk from "../../../sdk"
+import { isExternalTable } from "../../../../src/integrations/utils"
 
 export async function migrate(
   table: Table,
@@ -80,5 +80,6 @@ class UserColumnMigrator implements ColumnMigrator {
 
   async doMigration() {
     let rows = await sdk.rows.fetch(this.table._id!)
+    let links = await sdk.links.fetchWithDocument(this.table._id!)
   }
 }
diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts
index 24eb1ebf3c..c3057e3d4f 100644
--- a/packages/server/src/sdk/index.ts
+++ b/packages/server/src/sdk/index.ts
@@ -5,6 +5,7 @@ import { default as applications } from "./app/applications"
 import { default as datasources } from "./app/datasources"
 import { default as queries } from "./app/queries"
 import { default as rows } from "./app/rows"
+import { default as links } from "./app/links"
 import { default as users } from "./users"
 import { default as plugins } from "./plugins"
 import * as views from "./app/views"
@@ -22,6 +23,7 @@ const sdk = {
   plugins,
   views,
   permissions,
+  links,
 }
 
 // default export for TS
diff --git a/packages/server/src/tests/utilities/api/table.ts b/packages/server/src/tests/utilities/api/table.ts
index 04432a788a..501841f6e7 100644
--- a/packages/server/src/tests/utilities/api/table.ts
+++ b/packages/server/src/tests/utilities/api/table.ts
@@ -1,4 +1,10 @@
-import { SaveTableRequest, SaveTableResponse, Table } from "@budibase/types"
+import {
+  MigrateRequest,
+  MigrateResponse,
+  SaveTableRequest,
+  SaveTableResponse,
+  Table,
+} from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
 
@@ -42,4 +48,18 @@ export class TableAPI extends TestAPI {
       .expect(expectStatus)
     return res.body
   }
+
+  migrate = async (
+    tableId: string,
+    data: MigrateRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<MigrateResponse> => {
+    const res = await this.request
+      .post(`/api/tables/${tableId}/migrate`)
+      .send(data)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return res.body
+  }
 }
diff --git a/packages/types/src/api/web/app/table.ts b/packages/types/src/api/web/app/table.ts
index 61da12d041..f4d6720516 100644
--- a/packages/types/src/api/web/app/table.ts
+++ b/packages/types/src/api/web/app/table.ts
@@ -36,7 +36,6 @@ export interface BulkImportResponse {
 }
 
 export interface MigrateRequest {
-  tableId: string
   oldColumn: FieldSchema
   newColumn: FieldSchema
 }

From 1771b5905a37e544261a2fb8b2deca58786aeabc Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 18 Oct 2023 18:02:10 +0100
Subject: [PATCH 011/134] Most of the way to getting my first test passing.

---
 .../server/src/api/routes/tests/table.spec.ts |  2 +
 packages/server/src/sdk/app/tables/index.ts   | 17 ++++++---
 .../server/src/sdk/app/tables/migration.ts    | 38 +++++++++++++++++--
 3 files changed, 49 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts
index eaf3757ea8..dcf1704ed0 100644
--- a/packages/server/src/api/routes/tests/table.spec.ts
+++ b/packages/server/src/api/routes/tests/table.spec.ts
@@ -456,9 +456,11 @@ describe("/tables", () => {
 
       const migratedTable = await config.api.table.get(table._id!)
       expect(migratedTable.schema["user column"]).toBeDefined()
+      expect(migratedTable.schema["user relationship"]).not.toBeDefined()
 
       const rows = await config.api.row.fetch(table._id!)
       expect(rows[0]["user column"]).toBeDefined()
+      expect(rows[0]["user relationship"]).not.toBeDefined()
     })
   })
 })
diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts
index e4884b2ec4..fab8c7d198 100644
--- a/packages/server/src/sdk/app/tables/index.ts
+++ b/packages/server/src/sdk/app/tables/index.ts
@@ -16,6 +16,8 @@ import datasources from "../datasources"
 import { populateExternalTableSchemas } from "./validation"
 import sdk from "../../../sdk"
 import { migrate } from "./migration"
+import { DocumentInsertResponse } from "@budibase/nano"
+import { cloneDeep } from "lodash"
 
 async function getAllInternalTables(db?: Database): Promise<Table[]> {
   if (!db) {
@@ -75,23 +77,28 @@ function enrichViewSchemas(table: Table): TableResponse {
   }
 }
 
-async function saveTable(table: Table) {
+async function saveTable(table: Table): Promise<Table> {
   const db = context.getAppDB()
+  let resp: DocumentInsertResponse
   if (isExternalTable(table._id!)) {
     const datasource = await sdk.datasources.get(table.sourceId!)
     datasource.entities![table.name] = table
-    await db.put(datasource)
+    resp = await db.put(datasource)
   } else {
-    await db.put(table)
+    resp = await db.put(table)
   }
+
+  let tableClone = cloneDeep(table)
+  tableClone._rev = resp.rev
+  return tableClone
 }
 
-async function addColumn(table: Table, newColumn: FieldSchema) {
+async function addColumn(table: Table, newColumn: FieldSchema): Promise<Table> {
   if (newColumn.name in table.schema) {
     throw `Column "${newColumn.name}" already exists on table "${table.name}"`
   }
   table.schema[newColumn.name] = newColumn
-  await saveTable(table)
+  return await saveTable(table)
 }
 
 export default {
diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts
index ac7f3a3765..a08b56826d 100644
--- a/packages/server/src/sdk/app/tables/migration.ts
+++ b/packages/server/src/sdk/app/tables/migration.ts
@@ -1,15 +1,19 @@
-import { BadRequestError } from "@budibase/backend-core"
+import { BadRequestError, context } from "@budibase/backend-core"
 import {
   BBReferenceFieldMetadata,
   FieldSchema,
   InternalTable,
   RelationshipFieldMetadata,
+  Row,
   Table,
   isBBReferenceField,
   isRelationshipField,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { isExternalTable } from "../../../../src/integrations/utils"
+import { db as dbCore } from "@budibase/backend-core"
+import { EventType, updateLinks } from "../../../../src/db/linkedRows"
+import { cloneDeep } from "lodash"
 
 export async function migrate(
   table: Table,
@@ -17,10 +21,15 @@ export async function migrate(
   newColumn: FieldSchema
 ) {
   let migrator = getColumnMigrator(table, oldColumn, newColumn)
+  let oldTable = cloneDeep(table)
 
-  await sdk.tables.addColumn(table, newColumn)
+  table = await sdk.tables.addColumn(table, newColumn)
 
-  migrator.doMigration()
+  await migrator.doMigration()
+
+  delete table.schema[oldColumn.name]
+  await sdk.tables.saveTable(table)
+  await updateLinks({ eventType: EventType.TABLE_UPDATED, table, oldTable })
 }
 
 interface ColumnMigrator {
@@ -80,6 +89,29 @@ class UserColumnMigrator implements ColumnMigrator {
 
   async doMigration() {
     let rows = await sdk.rows.fetch(this.table._id!)
+    let rowsById = rows.reduce((acc, row) => {
+      acc[row._id!] = row
+      return acc
+    }, {} as Record<string, Row>)
+
     let links = await sdk.links.fetchWithDocument(this.table._id!)
+    for (let link of links) {
+      if (link.doc1.tableId !== this.table._id) {
+        continue
+      }
+      if (link.doc1.fieldName !== this.oldColumn.name) {
+        continue
+      }
+      if (link.doc2.tableId !== InternalTable.USER_METADATA) {
+        continue
+      }
+
+      let userId = dbCore.getGlobalIDFromUserMetadataID(link.doc2.rowId)
+      let row = rowsById[link.doc1.rowId]
+      row[this.newColumn.name] = userId
+    }
+
+    let db = context.getAppDB()
+    await db.bulkDocs(rows)
   }
 }

From 77729737bce3cceabcef7290bbf46321939bccd4 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Oct 2023 09:47:50 +0100
Subject: [PATCH 012/134] First test passes!

---
 packages/server/src/sdk/app/rows/external.ts        | 2 +-
 packages/server/src/sdk/app/rows/search.ts          | 8 ++++++--
 packages/server/src/sdk/app/rows/search/external.ts | 6 ++++++
 packages/server/src/sdk/app/rows/search/internal.ts | 9 ++++-----
 packages/server/src/sdk/app/tables/migration.ts     | 4 ++--
 5 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/external.ts b/packages/server/src/sdk/app/rows/external.ts
index 8bcf89a3f5..beae02e134 100644
--- a/packages/server/src/sdk/app/rows/external.ts
+++ b/packages/server/src/sdk/app/rows/external.ts
@@ -1,4 +1,4 @@
-import { IncludeRelationship, Operation, Row } from "@budibase/types"
+import { IncludeRelationship, Operation } from "@budibase/types"
 import { handleRequest } from "../../../api/controllers/row/external"
 import { breakRowIdField } from "../../../integrations/utils"
 
diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index f75bd07437..221aa6486c 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -1,4 +1,4 @@
-import { SearchFilters, SearchParams } from "@budibase/types"
+import { Row, SearchFilters, SearchParams } from "@budibase/types"
 import { isExternalTable } from "../../../integrations/utils"
 import * as internal from "./search/internal"
 import * as external from "./search/external"
@@ -45,10 +45,14 @@ export async function exportRows(
   return pickApi(options.tableId).exportRows(options)
 }
 
-export async function fetch(tableId: string) {
+export async function fetch(tableId: string): Promise<Row[]> {
   return pickApi(tableId).fetch(tableId)
 }
 
+export async function fetchRaw(tableId: string): Promise<Row[]> {
+  return pickApi(tableId).fetchRaw(tableId)
+}
+
 export async function fetchView(
   tableId: string,
   viewName: string,
diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts
index c41efad171..981ae1bf8d 100644
--- a/packages/server/src/sdk/app/rows/search/external.ts
+++ b/packages/server/src/sdk/app/rows/search/external.ts
@@ -186,6 +186,12 @@ export async function fetch(tableId: string): Promise<Row[]> {
   })
 }
 
+export async function fetchRaw(tableId: string): Promise<Row[]> {
+  return await handleRequest<Operation.READ>(Operation.READ, tableId, {
+    includeSqlRelationships: IncludeRelationship.INCLUDE,
+  })
+}
+
 export async function fetchView(viewName: string) {
   // there are no views in external datasources, shouldn't ever be called
   // for now just fetch
diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts
index 779ff5f777..58611c8849 100644
--- a/packages/server/src/sdk/app/rows/search/internal.ts
+++ b/packages/server/src/sdk/app/rows/search/internal.ts
@@ -140,14 +140,13 @@ export async function exportRows(
 }
 
 export async function fetch(tableId: string): Promise<Row[]> {
-  const db = context.getAppDB()
-
   const table = await sdk.tables.getTable(tableId)
-  const rows = await getRawTableData(db, tableId)
+  const rows = await fetchRaw(tableId)
   return await outputProcessing(table, rows)
 }
 
-async function getRawTableData(db: Database, tableId: string) {
+export async function fetchRaw(tableId: string): Promise<Row[]> {
+  const db = context.getAppDB()
   let rows
   if (tableId === InternalTables.USER_METADATA) {
     rows = await sdk.users.fetchMetadata()
@@ -182,7 +181,7 @@ export async function fetchView(
     })
   } else {
     const tableId = viewInfo.meta.tableId
-    const data = await getRawTableData(db, tableId)
+    const data = await fetchRaw(tableId)
     response = await inMemoryViews.runView(
       viewInfo,
       calculation as string,
diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts
index a08b56826d..cea15114a1 100644
--- a/packages/server/src/sdk/app/tables/migration.ts
+++ b/packages/server/src/sdk/app/tables/migration.ts
@@ -28,7 +28,7 @@ export async function migrate(
   await migrator.doMigration()
 
   delete table.schema[oldColumn.name]
-  await sdk.tables.saveTable(table)
+  table = await sdk.tables.saveTable(table)
   await updateLinks({ eventType: EventType.TABLE_UPDATED, table, oldTable })
 }
 
@@ -88,7 +88,7 @@ class UserColumnMigrator implements ColumnMigrator {
   ) {}
 
   async doMigration() {
-    let rows = await sdk.rows.fetch(this.table._id!)
+    let rows = await sdk.rows.fetchRaw(this.table._id!)
     let rowsById = rows.reduce((acc, row) => {
       acc[row._id!] = row
       return acc

From ad54b31667cffa18c9ccf7971fc1af8085a9e233 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Thu, 19 Oct 2023 12:00:47 +0100
Subject: [PATCH 013/134] Moved label align and button size into styles section
 of form block. Added tagged component  settings section to the style section.
 General settings and sections now have a tag property for filtering.

---
 .../Component/ComponentSettingsPanel.svelte   |  7 +-
 .../Component/ComponentSettingsSection.svelte | 20 ++++--
 .../Component/DesignSection.svelte            | 15 +++++
 packages/client/manifest.json                 | 66 ++++++++++---------
 4 files changed, 70 insertions(+), 38 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte
index afcada4138..8b9ff0ae2c 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte
@@ -65,7 +65,12 @@
         />
       {/if}
       {#if section == "styles"}
-        <DesignSection {componentInstance} {componentDefinition} {bindings} />
+        <DesignSection
+          {componentInstance}
+          {componentBindings}
+          {componentDefinition}
+          {bindings}
+        />
         <CustomStylesSection
           {componentInstance}
           {componentDefinition}
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
index b4ed8995a0..f2d6c9cb52 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
@@ -17,13 +17,23 @@
   export let onUpdateSetting
   export let showSectionTitle = true
   export let showInstanceName = true
+  export let tag
 
-  $: sections = getSections(componentInstance, componentDefinition, isScreen)
+  $: sections = getSections(
+    componentInstance,
+    componentDefinition,
+    isScreen,
+    tag
+  )
 
   const getSections = (instance, definition, isScreen) => {
     const settings = definition?.settings ?? []
-    const generalSettings = settings.filter(setting => !setting.section)
-    const customSections = settings.filter(setting => setting.section)
+    const generalSettings = settings.filter(
+      setting => !setting.section && setting.tag === tag
+    )
+    const customSections = settings.filter(
+      setting => setting.section && setting.tag === tag
+    )
     let sections = [
       {
         name: "General",
@@ -131,7 +141,7 @@
         <div class="section-info">
           <InfoDisplay body={section.info} />
         </div>
-      {:else if idx === 0 && section.name === "General" && componentDefinition.info}
+      {:else if idx === 0 && section.name === "General" && componentDefinition.info && !tag}
         <InfoDisplay
           title={componentDefinition.name}
           body={componentDefinition.info}
@@ -189,7 +199,7 @@
     </DetailSummary>
   {/if}
 {/each}
-{#if componentDefinition?.block}
+{#if componentDefinition?.block && !tag}
   <DetailSummary name="Eject" collapsible={false}>
     <EjectBlockButton />
   </DetailSummary>
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/DesignSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/DesignSection.svelte
index 444ded7e1f..def1fcf24b 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/DesignSection.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/DesignSection.svelte
@@ -1,10 +1,12 @@
 <script>
   import StyleSection from "./StyleSection.svelte"
   import * as ComponentStyles from "./componentStyles"
+  import ComponentSettingsSection from "./ComponentSettingsSection.svelte"
 
   export let componentDefinition
   export let componentInstance
   export let bindings
+  export let componentBindings
 
   const getStyles = def => {
     if (!def?.styles?.length) {
@@ -22,6 +24,19 @@
   $: styles = getStyles(componentDefinition)
 </script>
 
+<!--
+  Load any general settings or sections tagged as "style"
+-->
+<ComponentSettingsSection
+  {componentInstance}
+  {componentDefinition}
+  isScreen={false}
+  showInstanceName={false}
+  {bindings}
+  {componentBindings}
+  tag="style"
+/>
+
 {#if styles?.length > 0}
   {#each styles as style}
     <StyleSection
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 8d0a4e456f..33c11dc297 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -5385,38 +5385,6 @@
         "section": true,
         "name": "Fields",
         "settings": [
-          {
-            "type": "select",
-            "label": "Align labels",
-            "key": "labelPosition",
-            "defaultValue": "left",
-            "options": [
-              {
-                "label": "Left",
-                "value": "left"
-              },
-              {
-                "label": "Above",
-                "value": "above"
-              }
-            ]
-          },
-          {
-            "type": "select",
-            "label": "Size",
-            "key": "size",
-            "options": [
-              {
-                "label": "Medium",
-                "value": "spectrum--medium"
-              },
-              {
-                "label": "Large",
-                "value": "spectrum--large"
-              }
-            ],
-            "defaultValue": "spectrum--medium"
-          },
           {
             "type": "fieldConfiguration",
             "key": "fields",
@@ -5436,6 +5404,40 @@
             }
           }
         ]
+      },
+      {
+        "tag": "style",
+        "type": "select",
+        "label": "Align labels",
+        "key": "labelPosition",
+        "defaultValue": "left",
+        "options": [
+          {
+            "label": "Left",
+            "value": "left"
+          },
+          {
+            "label": "Above",
+            "value": "above"
+          }
+        ]
+      },
+      {
+        "tag": "style",
+        "type": "select",
+        "label": "Size",
+        "key": "size",
+        "options": [
+          {
+            "label": "Medium",
+            "value": "spectrum--medium"
+          },
+          {
+            "label": "Large",
+            "value": "spectrum--large"
+          }
+        ],
+        "defaultValue": "spectrum--medium"
       }
     ],
     "context": [

From f59d6222912443484cb98fb666dd92442199abc7 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Thu, 19 Oct 2023 12:28:28 +0100
Subject: [PATCH 014/134] Moved buttons section below fields

---
 packages/client/manifest.json | 102 +++++++++++++++++-----------------
 1 file changed, 51 insertions(+), 51 deletions(-)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index d5fca15fd7..1919f4ccd2 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -5330,57 +5330,6 @@
           }
         ]
       },
-      {
-        "section": true,
-        "name": "Buttons",
-        "dependsOn": {
-          "setting": "actionType",
-          "value": "View",
-          "invert": true
-        },
-        "settings": [
-          {
-            "type": "text",
-            "key": "saveButtonLabel",
-            "label": "Save button",
-            "nested": true,
-            "defaultValue": "Save"
-          },
-          {
-            "type": "text",
-            "key": "deleteButtonLabel",
-            "label": "Delete button",
-            "nested": true,
-            "defaultValue": "Delete",
-            "dependsOn": {
-              "setting": "actionType",
-              "value": "Update"
-            }
-          },
-          {
-            "type": "url",
-            "label": "Navigate after button press",
-            "key": "actionUrl",
-            "placeholder": "Choose a screen",
-            "dependsOn": {
-              "setting": "actionType",
-              "value": "View",
-              "invert": true
-            }
-          },
-          {
-            "type": "boolean",
-            "label": "Hide notifications",
-            "key": "notificationOverride",
-            "defaultValue": false,
-            "dependsOn": {
-              "setting": "actionType",
-              "value": "View",
-              "invert": true
-            }
-          }
-        ]
-      },
       {
         "section": true,
         "name": "Fields",
@@ -5436,6 +5385,57 @@
             }
           }
         ]
+      },
+      {
+        "section": true,
+        "name": "Buttons",
+        "dependsOn": {
+          "setting": "actionType",
+          "value": "View",
+          "invert": true
+        },
+        "settings": [
+          {
+            "type": "text",
+            "key": "saveButtonLabel",
+            "label": "Save button",
+            "nested": true,
+            "defaultValue": "Save"
+          },
+          {
+            "type": "text",
+            "key": "deleteButtonLabel",
+            "label": "Delete button",
+            "nested": true,
+            "defaultValue": "Delete",
+            "dependsOn": {
+              "setting": "actionType",
+              "value": "Update"
+            }
+          },
+          {
+            "type": "url",
+            "label": "Navigate after button press",
+            "key": "actionUrl",
+            "placeholder": "Choose a screen",
+            "dependsOn": {
+              "setting": "actionType",
+              "value": "View",
+              "invert": true
+            }
+          },
+          {
+            "type": "boolean",
+            "label": "Hide notifications",
+            "key": "notificationOverride",
+            "defaultValue": false,
+            "dependsOn": {
+              "setting": "actionType",
+              "value": "View",
+              "invert": true
+            }
+          }
+        ]
       }
     ],
     "context": [

From a3ad8780deb6089bc96d1a66d21b0ed9ebf30c2e Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Oct 2023 17:28:55 +0100
Subject: [PATCH 015/134] Implement many-to-many user column migrations.

---
 .../server/src/api/routes/tests/table.spec.ts | 95 +++++++++++++++++--
 packages/server/src/sdk/app/rows/search.ts    |  2 +-
 .../server/src/sdk/app/tables/migration.ts    | 66 ++++++++++++-
 .../src/tests/utilities/TestConfiguration.ts  |  3 +-
 .../server/src/tests/utilities/api/index.ts   |  1 +
 5 files changed, 154 insertions(+), 13 deletions(-)

diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts
index dcf1704ed0..907b020246 100644
--- a/packages/server/src/api/routes/tests/table.spec.ts
+++ b/packages/server/src/api/routes/tests/table.spec.ts
@@ -8,6 +8,7 @@ import {
   AutoFieldSubTypes,
   InternalTable,
   FieldSubtype,
+  Row,
 } from "@budibase/types"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
@@ -421,8 +422,13 @@ describe("/tables", () => {
   })
 
   describe("migrate", () => {
-    it("should successfully migrate a user relationship to a user column", async () => {
-      const users = await config.api.row.fetch(InternalTable.USER_METADATA)
+    it("should successfully migrate a one-to-many user relationship to a user column", async () => {
+      const users = await Promise.all([
+        config.createUser({ email: "1@example.com" }),
+        config.createUser({ email: "2@example.com" }),
+        config.createUser({ email: "3@example.com" }),
+      ])
+
       const table = await config.api.table.create({
         name: "table",
         type: "table",
@@ -441,9 +447,11 @@ describe("/tables", () => {
         },
       })
 
-      await config.api.row.save(table._id!, {
-        "user relationship": users,
-      })
+      const rows = await Promise.all(
+        users.map(u =>
+          config.api.row.save(table._id!, { "user relationship": [u] })
+        )
+      )
 
       await config.api.table.migrate(table._id!, {
         oldColumn: table.schema["user relationship"],
@@ -458,9 +466,80 @@ describe("/tables", () => {
       expect(migratedTable.schema["user column"]).toBeDefined()
       expect(migratedTable.schema["user relationship"]).not.toBeDefined()
 
-      const rows = await config.api.row.fetch(table._id!)
-      expect(rows[0]["user column"]).toBeDefined()
-      expect(rows[0]["user relationship"]).not.toBeDefined()
+      const migratedRows = await config.api.row.fetch(table._id!)
+
+      rows.sort((a, b) => a._id!.localeCompare(b._id!))
+      migratedRows.sort((a, b) => a._id!.localeCompare(b._id!))
+
+      for (const [i, row] of rows.entries()) {
+        const migratedRow = migratedRows[i]
+        expect(migratedRow["user column"]).toBeDefined()
+        expect(migratedRow["user relationship"]).not.toBeDefined()
+        expect(row["user relationship"][0]._id).toEqual(
+          migratedRow["user column"][0]._id
+        )
+      }
+    })
+
+    it("should successfully migrate a many-to-many user relationship to a users column", async () => {
+      const users = await Promise.all([
+        config.createUser({ email: "1@example.com" }),
+        config.createUser({ email: "2@example.com" }),
+        config.createUser({ email: "3@example.com" }),
+      ])
+
+      const table = await config.api.table.create({
+        name: "table",
+        type: "table",
+        schema: {
+          "user relationship": {
+            type: FieldType.LINK,
+            fieldName: "test",
+            name: "user relationship",
+            constraints: {
+              type: "array",
+              presence: false,
+            },
+            relationshipType: RelationshipType.MANY_TO_MANY,
+            tableId: InternalTable.USER_METADATA,
+          },
+        },
+      })
+
+      const row1 = await config.api.row.save(table._id!, {
+        "user relationship": [users[0], users[1]],
+      })
+
+      const row2 = await config.api.row.save(table._id!, {
+        "user relationship": [users[2]],
+      })
+
+      await config.api.table.migrate(table._id!, {
+        oldColumn: table.schema["user relationship"],
+        newColumn: {
+          name: "user column",
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldSubtype.USERS,
+        },
+      })
+
+      const migratedTable = await config.api.table.get(table._id!)
+      expect(migratedTable.schema["user column"]).toBeDefined()
+      expect(migratedTable.schema["user relationship"]).not.toBeDefined()
+
+      const row1Migrated = (await config.api.row.get(table._id!, row1._id!))
+        .body as Row
+      expect(row1Migrated["user relationship"]).not.toBeDefined()
+      expect(row1Migrated["user column"].map((r: Row) => r._id)).toEqual(
+        expect.arrayContaining([users[0]._id, users[1]._id])
+      )
+
+      const row2Migrated = (await config.api.row.get(table._id!, row2._id!))
+        .body as Row
+      expect(row2Migrated["user relationship"]).not.toBeDefined()
+      expect(row2Migrated["user column"].map((r: Row) => r._id)).toEqual([
+        users[2]._id,
+      ])
     })
   })
 })
diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index 221aa6486c..caf9ffea02 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -57,6 +57,6 @@ export async function fetchView(
   tableId: string,
   viewName: string,
   params: ViewParams
-) {
+): Promise<Row[]> {
   return pickApi(tableId).fetchView(viewName, params)
 }
diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts
index cea15114a1..e3f817f893 100644
--- a/packages/server/src/sdk/app/tables/migration.ts
+++ b/packages/server/src/sdk/app/tables/migration.ts
@@ -2,8 +2,12 @@ import { BadRequestError, context } from "@budibase/backend-core"
 import {
   BBReferenceFieldMetadata,
   FieldSchema,
+  FieldSubtype,
   InternalTable,
+  ManyToManyRelationshipFieldMetadata,
+  OneToManyRelationshipFieldMetadata,
   RelationshipFieldMetadata,
+  RelationshipType,
   Row,
   Table,
   isBBReferenceField,
@@ -77,13 +81,30 @@ function getColumnMigrator(
     )
   }
 
-  return new UserColumnMigrator(table, oldColumn, newColumn)
+  if (oldColumn.relationshipType === RelationshipType.ONE_TO_MANY) {
+    if (newColumn.subtype !== FieldSubtype.USER) {
+      throw new BadRequestError(
+        `Column "${oldColumn.name}" is a one-to-many column but "${newColumn.name}" is not a single user column`
+      )
+    }
+    return new OneToManyUserColumnMigrator(table, oldColumn, newColumn)
+  }
+  if (oldColumn.relationshipType === RelationshipType.MANY_TO_MANY) {
+    if (newColumn.subtype !== FieldSubtype.USERS) {
+      throw new BadRequestError(
+        `Column "${oldColumn.name}" is a many-to-many column but "${newColumn.name}" is not a multi user column`
+      )
+    }
+    return new ManyToManyUserColumnMigrator(table, oldColumn, newColumn)
+  }
+
+  throw new BadRequestError(`Unknown migration type`)
 }
 
-class UserColumnMigrator implements ColumnMigrator {
+class OneToManyUserColumnMigrator implements ColumnMigrator {
   constructor(
     private table: Table,
-    private oldColumn: RelationshipFieldMetadata,
+    private oldColumn: OneToManyRelationshipFieldMetadata,
     private newColumn: BBReferenceFieldMetadata
   ) {}
 
@@ -115,3 +136,42 @@ class UserColumnMigrator implements ColumnMigrator {
     await db.bulkDocs(rows)
   }
 }
+
+class ManyToManyUserColumnMigrator implements ColumnMigrator {
+  constructor(
+    private table: Table,
+    private oldColumn: ManyToManyRelationshipFieldMetadata,
+    private newColumn: BBReferenceFieldMetadata
+  ) {}
+
+  async doMigration() {
+    let rows = await sdk.rows.fetchRaw(this.table._id!)
+    let rowsById = rows.reduce((acc, row) => {
+      acc[row._id!] = row
+      return acc
+    }, {} as Record<string, Row>)
+
+    let links = await sdk.links.fetchWithDocument(this.table._id!)
+    for (let link of links) {
+      if (link.doc1.tableId !== this.table._id) {
+        continue
+      }
+      if (link.doc1.fieldName !== this.oldColumn.name) {
+        continue
+      }
+      if (link.doc2.tableId !== InternalTable.USER_METADATA) {
+        continue
+      }
+
+      let userId = dbCore.getGlobalIDFromUserMetadataID(link.doc2.rowId)
+      let row = rowsById[link.doc1.rowId]
+      if (!row[this.newColumn.name]) {
+        row[this.newColumn.name] = []
+      }
+      row[this.newColumn.name].push(userId)
+    }
+
+    let db = context.getAppDB()
+    await db.bulkDocs(rows)
+  }
+}
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index cec8c8aa12..c00772ef63 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -55,6 +55,7 @@ import {
   RelationshipType,
   CreateViewRequest,
   RelationshipFieldMetadata,
+  User,
 } from "@budibase/types"
 
 import API from "./api"
@@ -254,7 +255,7 @@ class TestConfiguration {
     } catch (err) {
       existing = { email }
     }
-    const user = {
+    const user: User = {
       _id: id,
       ...existing,
       roles: roles || {},
diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts
index fce8237760..1352874914 100644
--- a/packages/server/src/tests/utilities/api/index.ts
+++ b/packages/server/src/tests/utilities/api/index.ts
@@ -27,5 +27,6 @@ export default class API {
     this.datasource = new DatasourceAPI(config)
     this.screen = new ScreenAPI(config)
     this.application = new ApplicationAPI(config)
+    this.global = new GlobalAPI(config)
   }
 }

From 3e51557281a82383450e28e875ecb518852e8727 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Fri, 20 Oct 2023 10:38:34 +0100
Subject: [PATCH 016/134] Feedback update, add tag to getsections fn definition

---
 .../_components/Component/ComponentSettingsSection.svelte       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
index f2d6c9cb52..6eb4b4c440 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
@@ -26,7 +26,7 @@
     tag
   )
 
-  const getSections = (instance, definition, isScreen) => {
+  const getSections = (instance, definition, isScreen, tag) => {
     const settings = definition?.settings ?? []
     const generalSettings = settings.filter(
       setting => !setting.section && setting.tag === tag

From 871891c2a2f2e25cb1e821a9dfe8b7c19c199aaf Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Fri, 20 Oct 2023 15:35:16 +0100
Subject: [PATCH 017/134] Parse view v1 schema to fix builder crashed caused by
 'groupedby' views

---
 .../components/backend/DataTable/Table.svelte | 44 ++++++++++---------
 .../backend/DataTable/ViewDataTable.svelte    | 16 ++++++-
 2 files changed, 39 insertions(+), 21 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/Table.svelte b/packages/builder/src/components/backend/DataTable/Table.svelte
index f8087d8a39..45e440eca5 100644
--- a/packages/builder/src/components/backend/DataTable/Table.svelte
+++ b/packages/builder/src/components/backend/DataTable/Table.svelte
@@ -24,17 +24,21 @@
 
   let selectedRows = []
   let customRenderers = []
+  let parsedSchema = {}
+
+  $: if (schema) {
+    parsedSchema = Object.keys(schema).reduce((acc, key) => {
+      acc[key] = { ...schema[key] }
+      if (!canBeSortColumn(acc[key].type)) {
+        acc[key].sortable = false
+      }
+      return acc
+    }, {})
+  }
 
   $: selectedRows, dispatch("selectionUpdated", selectedRows)
   $: isUsersTable = tableId === TableNames.USERS
   $: data && resetSelectedRows()
-  $: {
-    Object.values(schema || {}).forEach(col => {
-      if (!canBeSortColumn(col.type)) {
-        col.sortable = false
-      }
-    })
-  }
   $: {
     if (isUsersTable) {
       customRenderers = [
@@ -44,24 +48,24 @@
         },
       ]
       UNEDITABLE_USER_FIELDS.forEach(field => {
-        if (schema[field]) {
-          schema[field].editable = false
+        if (parsedSchema[field]) {
+          parsedSchema[field].editable = false
         }
       })
-      if (schema.email) {
-        schema.email.displayName = "Email"
+      if (parsedSchema.email) {
+        parsedSchema.email.displayName = "Email"
       }
-      if (schema.roleId) {
-        schema.roleId.displayName = "Role"
+      if (parsedSchema.roleId) {
+        parsedSchema.roleId.displayName = "Role"
       }
-      if (schema.firstName) {
-        schema.firstName.displayName = "First Name"
+      if (parsedSchema.firstName) {
+        parsedSchema.firstName.displayName = "First Name"
       }
-      if (schema.lastName) {
-        schema.lastName.displayName = "Last Name"
+      if (parsedSchema.lastName) {
+        parsedSchema.lastName.displayName = "Last Name"
       }
-      if (schema.status) {
-        schema.status.displayName = "Status"
+      if (parsedSchema.status) {
+        parsedSchema.status.displayName = "Status"
       }
     }
   }
@@ -97,7 +101,7 @@
     <div class="table-wrapper">
       <Table
         {data}
-        {schema}
+        schema={parsedSchema}
         {loading}
         {customRenderers}
         {rowCount}
diff --git a/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte
index f6160e3caa..12e4e3c270 100644
--- a/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte
@@ -19,7 +19,21 @@
   let loading = false
   let type = "internal"
 
+  // Grouped views have a fixed schema layout
+  const parseSchema = view => {
+    if (!view.groupBy) {
+      return view.schema
+    }
+    const { group, field, value } = view.schema
+    return {
+      group,
+      field: { type: field },
+      value: { type: value },
+    }
+  }
+
   $: name = view.name
+  $: schema = parseSchema(view)
   $: calculation = view.calculation
 
   $: supportedFormats = Object.values(ROW_EXPORT_FORMATS).filter(key => {
@@ -61,7 +75,7 @@
 
 <Table
   title={decodeURI(name)}
-  schema={view.schema}
+  {schema}
   tableId={view.tableId}
   {data}
   {loading}

From 2d26597d07d8c4042de04b6a68c8178402e7164e Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 23 Oct 2023 09:49:57 +0100
Subject: [PATCH 018/134] Fix tests after merge.

---
 packages/server/src/sdk/app/tables/index.ts     |  2 ++
 packages/server/src/sdk/app/tables/migration.ts |  3 ++-
 packages/server/src/sdk/app/tables/update.ts    | 13 ++++++++++---
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts
index 8542250517..ed71498d44 100644
--- a/packages/server/src/sdk/app/tables/index.ts
+++ b/packages/server/src/sdk/app/tables/index.ts
@@ -2,10 +2,12 @@ import { populateExternalTableSchemas } from "./validation"
 import * as getters from "./getters"
 import * as updates from "./update"
 import * as utils from "./utils"
+import { migrate } from "./migration"
 
 export default {
   populateExternalTableSchemas,
   ...updates,
   ...getters,
   ...utils,
+  migrate,
 }
diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts
index cea15114a1..76b824959c 100644
--- a/packages/server/src/sdk/app/tables/migration.ts
+++ b/packages/server/src/sdk/app/tables/migration.ts
@@ -23,7 +23,8 @@ export async function migrate(
   let migrator = getColumnMigrator(table, oldColumn, newColumn)
   let oldTable = cloneDeep(table)
 
-  table = await sdk.tables.addColumn(table, newColumn)
+  table.schema[newColumn.name] = newColumn
+  table = await sdk.tables.saveTable(table)
 
   await migrator.doMigration()
 
diff --git a/packages/server/src/sdk/app/tables/update.ts b/packages/server/src/sdk/app/tables/update.ts
index 9bba4a967e..c59f8a8a42 100644
--- a/packages/server/src/sdk/app/tables/update.ts
+++ b/packages/server/src/sdk/app/tables/update.ts
@@ -3,21 +3,28 @@ import { isExternalTable } from "../../../integrations/utils"
 import sdk from "../../index"
 import { context } from "@budibase/backend-core"
 import { isExternal } from "./utils"
+import { DocumentInsertResponse } from "@budibase/nano"
 
 import * as external from "./external"
 import * as internal from "./internal"
+import { cloneDeep } from "lodash"
 export * as external from "./external"
 export * as internal from "./internal"
 
-export async function saveTable(table: Table) {
+export async function saveTable(table: Table): Promise<Table> {
   const db = context.getAppDB()
+  let resp: DocumentInsertResponse
   if (isExternalTable(table._id!)) {
     const datasource = await sdk.datasources.get(table.sourceId!)
     datasource.entities![table.name] = table
-    await db.put(datasource)
+    resp = await db.put(datasource)
   } else {
-    await db.put(table)
+    resp = await db.put(table)
   }
+
+  let tableClone = cloneDeep(table)
+  tableClone._rev = resp.rev
+  return tableClone
 }
 
 export async function update(table: Table, renaming?: RenameColumn) {

From 9dd16381a7e1c83eb3339f1790289af14b7b8774 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 23 Oct 2023 09:52:17 +0100
Subject: [PATCH 019/134] Merge base branch.

---
 packages/server/src/tests/utilities/api/index.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts
index 1352874914..fce8237760 100644
--- a/packages/server/src/tests/utilities/api/index.ts
+++ b/packages/server/src/tests/utilities/api/index.ts
@@ -27,6 +27,5 @@ export default class API {
     this.datasource = new DatasourceAPI(config)
     this.screen = new ScreenAPI(config)
     this.application = new ApplicationAPI(config)
-    this.global = new GlobalAPI(config)
   }
 }

From 06f0a8da1ae5ee15f4e583a526e81f8c93eb6fef Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 23 Oct 2023 09:59:11 +0100
Subject: [PATCH 020/134] Update pro submodule.

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 570d14aa44..56d968bfe6 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76
+Subproject commit 56d968bfe6998e1077d3fce4eb1c9e483d1d6fc9

From 2e0b528331cbc08f9cac96da2dbb19d0548283f5 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 23 Oct 2023 09:59:27 +0100
Subject: [PATCH 021/134] Update pro submodule.

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 570d14aa44..56d968bfe6 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76
+Subproject commit 56d968bfe6998e1077d3fce4eb1c9e483d1d6fc9

From 9e279be4c334c5e613066a59fe0ab8b42a842e0c Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 23 Oct 2023 10:35:41 +0100
Subject: [PATCH 022/134] Put pro back to where it was.

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 56d968bfe6..570d14aa44 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 56d968bfe6998e1077d3fce4eb1c9e483d1d6fc9
+Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76

From 3fc2ff2c9b1bf30eea4a2cf96c217935a283c0eb Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 23 Oct 2023 10:36:05 +0100
Subject: [PATCH 023/134] Put pro back to where it was.

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 56d968bfe6..570d14aa44 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 56d968bfe6998e1077d3fce4eb1c9e483d1d6fc9
+Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76

From febfab092717f106ef13f9827221e67d74c244b8 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 23 Oct 2023 10:48:10 +0100
Subject: [PATCH 024/134] Fix tests/types.

---
 packages/server/src/sdk/users/tests/utils.spec.ts  | 14 +++++++-------
 .../src/tests/utilities/TestConfiguration.ts       |  7 ++-----
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/packages/server/src/sdk/users/tests/utils.spec.ts b/packages/server/src/sdk/users/tests/utils.spec.ts
index 5c6777df59..f7c9413ebd 100644
--- a/packages/server/src/sdk/users/tests/utils.spec.ts
+++ b/packages/server/src/sdk/users/tests/utils.spec.ts
@@ -39,12 +39,12 @@ describe("syncGlobalUsers", () => {
       expect(metadata).toHaveLength(3)
       expect(metadata).toContainEqual(
         expect.objectContaining({
-          _id: db.generateUserMetadataID(user1._id),
+          _id: db.generateUserMetadataID(user1._id!),
         })
       )
       expect(metadata).toContainEqual(
         expect.objectContaining({
-          _id: db.generateUserMetadataID(user2._id),
+          _id: db.generateUserMetadataID(user2._id!),
         })
       )
     })
@@ -59,7 +59,7 @@ describe("syncGlobalUsers", () => {
       expect(metadata).toHaveLength(1)
       expect(metadata).not.toContainEqual(
         expect.objectContaining({
-          _id: db.generateUserMetadataID(user._id),
+          _id: db.generateUserMetadataID(user._id!),
         })
       )
     })
@@ -70,7 +70,7 @@ describe("syncGlobalUsers", () => {
       const group = await proSdk.groups.save(structures.userGroups.userGroup())
       const user1 = await config.createUser({ admin: false, builder: false })
       const user2 = await config.createUser({ admin: false, builder: false })
-      await proSdk.groups.addUsers(group.id, [user1._id, user2._id])
+      await proSdk.groups.addUsers(group.id, [user1._id!, user2._id!])
 
       await config.doInContext(config.appId, async () => {
         await syncGlobalUsers()
@@ -87,12 +87,12 @@ describe("syncGlobalUsers", () => {
         expect(metadata).toHaveLength(3)
         expect(metadata).toContainEqual(
           expect.objectContaining({
-            _id: db.generateUserMetadataID(user1._id),
+            _id: db.generateUserMetadataID(user1._id!),
           })
         )
         expect(metadata).toContainEqual(
           expect.objectContaining({
-            _id: db.generateUserMetadataID(user2._id),
+            _id: db.generateUserMetadataID(user2._id!),
           })
         )
       })
@@ -109,7 +109,7 @@ describe("syncGlobalUsers", () => {
           { appId: config.prodAppId!, roleId: roles.BUILTIN_ROLE_IDS.BASIC },
         ],
       })
-      await proSdk.groups.addUsers(group.id, [user1._id, user2._id])
+      await proSdk.groups.addUsers(group.id, [user1._id!, user2._id!])
 
       await config.doInContext(config.appId, async () => {
         await syncGlobalUsers()
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index c00772ef63..5f5b211975 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -295,7 +295,7 @@ class TestConfiguration {
       admin?: boolean
       roles?: UserRoles
     } = {}
-  ) {
+  ): Promise<User> {
     let { id, firstName, lastName, email, builder, admin, roles } = user
     firstName = firstName || this.defaultUserValues.firstName
     lastName = lastName || this.defaultUserValues.lastName
@@ -315,10 +315,7 @@ class TestConfiguration {
       roles,
     })
     await cache.user.invalidateUser(globalId)
-    return {
-      ...resp,
-      globalId,
-    }
+    return resp
   }
 
   async createGroup(roleId: string = roles.BUILTIN_ROLE_IDS.BASIC) {

From 5e6ed0fd670f762fb5b4ec4f084697baa1264135 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 23 Oct 2023 11:54:27 +0100
Subject: [PATCH 025/134] Implement many-to-one user column migration.

---
 .../server/src/api/routes/tests/table.spec.ts | 61 +++++++++++++++++++
 .../server/src/sdk/app/tables/migration.ts    | 21 ++++---
 2 files changed, 74 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts
index 907b020246..420717f7f0 100644
--- a/packages/server/src/api/routes/tests/table.spec.ts
+++ b/packages/server/src/api/routes/tests/table.spec.ts
@@ -510,6 +510,67 @@ describe("/tables", () => {
         "user relationship": [users[0], users[1]],
       })
 
+      const row2 = await config.api.row.save(table._id!, {
+        "user relationship": [users[1], users[2]],
+      })
+
+      await config.api.table.migrate(table._id!, {
+        oldColumn: table.schema["user relationship"],
+        newColumn: {
+          name: "user column",
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldSubtype.USERS,
+        },
+      })
+
+      const migratedTable = await config.api.table.get(table._id!)
+      expect(migratedTable.schema["user column"]).toBeDefined()
+      expect(migratedTable.schema["user relationship"]).not.toBeDefined()
+
+      const row1Migrated = (await config.api.row.get(table._id!, row1._id!))
+        .body as Row
+      expect(row1Migrated["user relationship"]).not.toBeDefined()
+      expect(row1Migrated["user column"].map((r: Row) => r._id)).toEqual(
+        expect.arrayContaining([users[0]._id, users[1]._id])
+      )
+
+      const row2Migrated = (await config.api.row.get(table._id!, row2._id!))
+        .body as Row
+      expect(row2Migrated["user relationship"]).not.toBeDefined()
+      expect(row2Migrated["user column"].map((r: Row) => r._id)).toEqual(
+        expect.arrayContaining([users[1]._id, users[2]._id])
+      )
+    })
+
+    it("should successfully migrate a many-to-one user relationship to a users column", async () => {
+      const users = await Promise.all([
+        config.createUser({ email: "1@example.com" }),
+        config.createUser({ email: "2@example.com" }),
+        config.createUser({ email: "3@example.com" }),
+      ])
+
+      const table = await config.api.table.create({
+        name: "table",
+        type: "table",
+        schema: {
+          "user relationship": {
+            type: FieldType.LINK,
+            fieldName: "test",
+            name: "user relationship",
+            constraints: {
+              type: "array",
+              presence: false,
+            },
+            relationshipType: RelationshipType.MANY_TO_ONE,
+            tableId: InternalTable.USER_METADATA,
+          },
+        },
+      })
+
+      const row1 = await config.api.row.save(table._id!, {
+        "user relationship": [users[0], users[1]],
+      })
+
       const row2 = await config.api.row.save(table._id!, {
         "user relationship": [users[2]],
       })
diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts
index 94b8841136..08c06b28b9 100644
--- a/packages/server/src/sdk/app/tables/migration.ts
+++ b/packages/server/src/sdk/app/tables/migration.ts
@@ -5,8 +5,8 @@ import {
   FieldSubtype,
   InternalTable,
   ManyToManyRelationshipFieldMetadata,
+  ManyToOneRelationshipFieldMetadata,
   OneToManyRelationshipFieldMetadata,
-  RelationshipFieldMetadata,
   RelationshipType,
   Row,
   Table,
@@ -88,21 +88,24 @@ function getColumnMigrator(
         `Column "${oldColumn.name}" is a one-to-many column but "${newColumn.name}" is not a single user column`
       )
     }
-    return new OneToManyUserColumnMigrator(table, oldColumn, newColumn)
+    return new SingleUserColumnMigrator(table, oldColumn, newColumn)
   }
-  if (oldColumn.relationshipType === RelationshipType.MANY_TO_MANY) {
+  if (
+    oldColumn.relationshipType === RelationshipType.MANY_TO_MANY ||
+    oldColumn.relationshipType === RelationshipType.MANY_TO_ONE
+  ) {
     if (newColumn.subtype !== FieldSubtype.USERS) {
       throw new BadRequestError(
-        `Column "${oldColumn.name}" is a many-to-many column but "${newColumn.name}" is not a multi user column`
+        `Column "${oldColumn.name}" is a ${oldColumn.relationshipType} column but "${newColumn.name}" is not a multi user column`
       )
     }
-    return new ManyToManyUserColumnMigrator(table, oldColumn, newColumn)
+    return new MultiUserColumnMigrator(table, oldColumn, newColumn)
   }
 
   throw new BadRequestError(`Unknown migration type`)
 }
 
-class OneToManyUserColumnMigrator implements ColumnMigrator {
+class SingleUserColumnMigrator implements ColumnMigrator {
   constructor(
     private table: Table,
     private oldColumn: OneToManyRelationshipFieldMetadata,
@@ -138,10 +141,12 @@ class OneToManyUserColumnMigrator implements ColumnMigrator {
   }
 }
 
-class ManyToManyUserColumnMigrator implements ColumnMigrator {
+class MultiUserColumnMigrator implements ColumnMigrator {
   constructor(
     private table: Table,
-    private oldColumn: ManyToManyRelationshipFieldMetadata,
+    private oldColumn:
+      | ManyToManyRelationshipFieldMetadata
+      | ManyToOneRelationshipFieldMetadata,
     private newColumn: BBReferenceFieldMetadata
   ) {}
 

From 08a22f1525af1fc7225f4792a0183b7e07d3365c Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Mon, 23 Oct 2023 16:47:05 +0100
Subject: [PATCH 026/134] Show 'Creator' instead of 'Admin' for the global user
 role picker

---
 .../src/components/common/RoleSelect.svelte       | 15 ++++++++++++---
 .../_components/BuilderSidePanel.svelte           | 11 +++++++++--
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/packages/builder/src/components/common/RoleSelect.svelte b/packages/builder/src/components/common/RoleSelect.svelte
index 82752554d5..2df61926e1 100644
--- a/packages/builder/src/components/common/RoleSelect.svelte
+++ b/packages/builder/src/components/common/RoleSelect.svelte
@@ -39,7 +39,15 @@
     allowCreator
   ) => {
     if (allowedRoles?.length) {
-      return roles.filter(role => allowedRoles.includes(role._id))
+      const filteredRoles = roles.filter(role =>
+        allowedRoles.includes(role._id)
+      )
+      return [
+        ...filteredRoles,
+        ...(allowedRoles.includes(Constants.Roles.CREATOR)
+          ? [{ _id: Constants.Roles.CREATOR, name: "Creator", enabled: false }]
+          : []),
+      ]
     }
     let newRoles = [...roles]
 
@@ -129,8 +137,9 @@
     getOptionColour={getColor}
     getOptionIcon={getIcon}
     isOptionEnabled={option =>
-      option._id !== Constants.Roles.CREATOR ||
-      $licensing.perAppBuildersEnabled}
+      (option._id !== Constants.Roles.CREATOR ||
+        $licensing.perAppBuildersEnabled) &&
+      option.enabled !== false}
     {placeholder}
     {error}
   />
diff --git a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
index a7d9584330..f9a40b09a6 100644
--- a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
@@ -516,6 +516,13 @@
     }
     return null
   }
+
+  const parseRole = user => {
+    if (user.isAdminOrGlobalBuilder) {
+      return Constants.Roles.CREATOR
+    }
+    return user.role
+  }
 </script>
 
 <svelte:window on:keydown={handleKeyDown} />
@@ -725,7 +732,7 @@
                     <RoleSelect
                       footer={getRoleFooter(user)}
                       placeholder={false}
-                      value={user.role}
+                      value={parseRole(user)}
                       allowRemove={user.role && !user.group}
                       allowPublic={false}
                       allowCreator={true}
@@ -744,7 +751,7 @@
                       autoWidth
                       align="right"
                       allowedRoles={user.isAdminOrGlobalBuilder
-                        ? [Constants.Roles.ADMIN]
+                        ? [Constants.Roles.CREATOR]
                         : null}
                     />
                   </div>

From a701933f48ad5f4f141cd85d18f9f86938cbe410 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 23 Oct 2023 17:57:25 +0100
Subject: [PATCH 027/134] Frontend changes for the user column migration work.

---
 packages/frontend-core/src/api/tables.js      |  9 +++++++
 .../components/grid/cells/HeaderCell.svelte   | 24 +++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/packages/frontend-core/src/api/tables.js b/packages/frontend-core/src/api/tables.js
index a08e35d3d8..34d2371e1a 100644
--- a/packages/frontend-core/src/api/tables.js
+++ b/packages/frontend-core/src/api/tables.js
@@ -140,4 +140,13 @@ export const buildTableEndpoints = API => ({
       },
     })
   },
+  migrateColumn: async ({ tableId, oldColumn, newColumn }) => {
+    return await API.post({
+      url: `/api/tables/${tableId}/migrate`,
+      body: {
+        oldColumn,
+        newColumn,
+      },
+    })
+  },
 })
diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index d6cbcb582d..01c759a15c 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -10,6 +10,7 @@
   export let orderable = true
 
   const {
+    API,
     reorder,
     isReordering,
     isResizing,
@@ -114,6 +115,24 @@
     open = false
   }
 
+  const migrateUserColumn = async () => {
+    let subtype = "users"
+    if (column.schema.relationshipType === "one-to-many") {
+      subtype = "user"
+    }
+
+    await API.migrateColumn({
+      tableId: $definition._id,
+      oldColumn: column.schema,
+      newColumn: {
+        name: `${column.schema.name} migrated`,
+        type: "bb_reference",
+        subtype,
+      },
+    })
+    open = false
+  }
+
   const duplicateColumn = async () => {
     open = false
 
@@ -262,6 +281,11 @@
       >
         Hide column
       </MenuItem>
+      {#if column.schema.type === "link" && column.schema.tableId === "ta_users"}
+        <MenuItem icon="User" on:click={migrateUserColumn}>
+          Migrate to user column
+        </MenuItem>
+      {/if}
     </Menu>
   {/if}
 </Popover>

From 2f0a40e9bb8c8e8893517d6dfcca1433ff088173 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 24 Oct 2023 15:18:46 +0100
Subject: [PATCH 028/134] Introduce modal to show warning to users, and toast
 to show success.

---
 .../backend/DataTable/TableDataTable.svelte   |  6 +++
 .../components/grid/cells/HeaderCell.svelte   | 42 +++++++++----------
 .../grid/controls/MigrationModal.svelte       | 42 +++++++++++++++++++
 3 files changed, 69 insertions(+), 21 deletions(-)
 create mode 100644 packages/frontend-core/src/components/grid/controls/MigrationModal.svelte

diff --git a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
index 5fee849afb..22deacbe03 100644
--- a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
@@ -53,6 +53,11 @@
       await datasources.fetch()
     }
   }
+
+  const refreshDefinitions = async () => {
+    await tables.fetch()
+    await datasources.fetch()
+  }
 </script>
 
 <div class="wrapper">
@@ -66,6 +71,7 @@
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
     on:updatedatasource={handleGridTableUpdate}
+    on:refreshdefinitions={refreshDefinitions}
   >
     <svelte:fragment slot="filter">
       {#if isUsersTable && $store.features.disableUserMetadata}
diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index 01c759a15c..9b05f9ef79 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -1,9 +1,17 @@
 <script>
   import { getContext, onMount, tick } from "svelte"
   import { canBeDisplayColumn, canBeSortColumn } from "@budibase/shared-core"
-  import { Icon, Popover, Menu, MenuItem, clickOutside } from "@budibase/bbui"
+  import {
+    Icon,
+    Popover,
+    Menu,
+    MenuItem,
+    clickOutside,
+    Modal,
+  } from "@budibase/bbui"
   import GridCell from "./GridCell.svelte"
   import { getColumnIcon } from "../lib/utils"
+  import MigrationModal from "../controls/MigrationModal.svelte"
 
   export let column
   export let idx
@@ -32,6 +40,7 @@
   let editIsOpen = false
   let timeout
   let popover
+  let migrationModal
 
   $: sortedBy = column.name === $sort.column
   $: canMoveLeft = orderable && idx > 0
@@ -115,24 +124,6 @@
     open = false
   }
 
-  const migrateUserColumn = async () => {
-    let subtype = "users"
-    if (column.schema.relationshipType === "one-to-many") {
-      subtype = "user"
-    }
-
-    await API.migrateColumn({
-      tableId: $definition._id,
-      oldColumn: column.schema,
-      newColumn: {
-        name: `${column.schema.name} migrated`,
-        type: "bb_reference",
-        subtype,
-      },
-    })
-    open = false
-  }
-
   const duplicateColumn = async () => {
     open = false
 
@@ -160,9 +151,18 @@
     })
   }
 
+  const openMigrationModal = () => {
+    migrationModal.show()
+    open = false
+  }
+
   onMount(() => subscribe("close-edit-column", cancelEdit))
 </script>
 
+<Modal bind:this={migrationModal}>
+  <MigrationModal {column} />
+</Modal>
+
 <div
   class="header-cell"
   class:open
@@ -281,8 +281,8 @@
       >
         Hide column
       </MenuItem>
-      {#if column.schema.type === "link" && column.schema.tableId === "ta_users"}
-        <MenuItem icon="User" on:click={migrateUserColumn}>
+      {#if $config.canEditColumns && column.schema.type === "link" && column.schema.tableId === "ta_users"}
+        <MenuItem icon="User" on:click={openMigrationModal}>
           Migrate to user column
         </MenuItem>
       {/if}
diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
new file mode 100644
index 0000000000..b262f93797
--- /dev/null
+++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
@@ -0,0 +1,42 @@
+<script>
+  import { ModalContent, notifications } from "@budibase/bbui"
+  import { getContext } from "svelte"
+
+  const { API, dispatch, definition, rows } = getContext("grid")
+
+  export let column
+
+  const migrateUserColumn = async () => {
+    let subtype = "users"
+    if (column.schema.relationshipType === "one-to-many") {
+      subtype = "user"
+    }
+
+    try {
+      await API.migrateColumn({
+        tableId: $definition._id,
+        oldColumn: column.schema,
+        newColumn: {
+          name: `${column.schema.name} migrated`,
+          type: "bb_reference",
+          subtype,
+        },
+      })
+      notifications.success("Column migrated")
+    } catch (e) {
+      notifications.error(`Failed to migrate: ${e.message}`)
+    }
+    await rows.actions.refreshData()
+    dispatch("refreshdefintions")
+  }
+</script>
+
+<ModalContent
+  title="Migrate column"
+  confirmText="Continue"
+  cancelText="Cancel"
+  onConfirm={migrateUserColumn}
+  size="M"
+>
+  TODO: copy here
+</ModalContent>

From cea1c04b73c508d96f1a1a828b964ec07b2858b5 Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Tue, 24 Oct 2023 17:16:44 +0200
Subject: [PATCH 029/134] Creators count functionality

---
 .../backend-core/src/cache/writethrough.ts    |   4 +-
 packages/backend-core/src/users/db.ts         | 110 ++++++++++--------
 packages/backend-core/src/users/users.ts      |   2 +-
 .../tests/core/users/users.spec.js            |  54 +++++++++
 .../core/utilities/structures/licenses.ts     |   8 ++
 packages/types/src/sdk/featureFlag.ts         |   3 +
 packages/types/src/sdk/licensing/billing.ts   |   7 ++
 packages/types/src/sdk/licensing/plan.ts      |   4 +
 8 files changed, 142 insertions(+), 50 deletions(-)
 create mode 100644 packages/backend-core/tests/core/users/users.spec.js

diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts
index e64c116663..c331d791a6 100644
--- a/packages/backend-core/src/cache/writethrough.ts
+++ b/packages/backend-core/src/cache/writethrough.ts
@@ -119,8 +119,8 @@ export class Writethrough {
     this.writeRateMs = writeRateMs
   }
 
-  async put(doc: any) {
-    return put(this.db, doc, this.writeRateMs)
+  async put(doc: any, writeRateMs: number = this.writeRateMs) {
+    return put(this.db, doc, writeRateMs)
   }
 
   async get(id: string) {
diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts
index a2539e836e..daa09bee6f 100644
--- a/packages/backend-core/src/users/db.ts
+++ b/packages/backend-core/src/users/db.ts
@@ -25,12 +25,17 @@ import {
 import {
   getAccountHolderFromUserIds,
   isAdmin,
+  isCreator,
   validateUniqueUser,
 } from "./utils"
 import { searchExistingEmails } from "./lookup"
 import { hash } from "../utils"
 
-type QuotaUpdateFn = (change: number, cb?: () => Promise<any>) => Promise<any>
+type QuotaUpdateFn = (
+  change: number,
+  creatorsChange: number,
+  cb?: () => Promise<any>
+) => Promise<any>
 type GroupUpdateFn = (groupId: string, userIds: string[]) => Promise<any>
 type FeatureFn = () => Promise<Boolean>
 type GroupGetFn = (ids: string[]) => Promise<UserGroup[]>
@@ -245,7 +250,8 @@ export class UserDB {
     }
 
     const change = dbUser ? 0 : 1 // no change if there is existing user
-    return UserDB.quotas.addUsers(change, async () => {
+    const creatorsChange = isCreator(dbUser) !== isCreator(user) ? 1 : 0
+    return UserDB.quotas.addUsers(change, creatorsChange, async () => {
       await validateUniqueUser(email, tenantId)
 
       let builtUser = await UserDB.buildUser(user, opts, tenantId, dbUser)
@@ -307,6 +313,7 @@ export class UserDB {
 
     let usersToSave: any[] = []
     let newUsers: any[] = []
+    let newCreators: any[] = []
 
     const emails = newUsersRequested.map((user: User) => user.email)
     const existingEmails = await searchExistingEmails(emails)
@@ -327,59 +334,66 @@ export class UserDB {
       }
       newUser.userGroups = groups
       newUsers.push(newUser)
+      if (isCreator(newUser)) {
+        newCreators.push(newUser)
+      }
     }
 
     const account = await accountSdk.getAccountByTenantId(tenantId)
-    return UserDB.quotas.addUsers(newUsers.length, async () => {
-      // create the promises array that will be called by bulkDocs
-      newUsers.forEach((user: any) => {
-        usersToSave.push(
-          UserDB.buildUser(
-            user,
-            {
-              hashPassword: true,
-              requirePassword: user.requirePassword,
-            },
-            tenantId,
-            undefined, // no dbUser
-            account
+    return UserDB.quotas.addUsers(
+      newUsers.length,
+      newCreators.length,
+      async () => {
+        // create the promises array that will be called by bulkDocs
+        newUsers.forEach((user: any) => {
+          usersToSave.push(
+            UserDB.buildUser(
+              user,
+              {
+                hashPassword: true,
+                requirePassword: user.requirePassword,
+              },
+              tenantId,
+              undefined, // no dbUser
+              account
+            )
           )
-        )
-      })
+        })
 
-      const usersToBulkSave = await Promise.all(usersToSave)
-      await usersCore.bulkUpdateGlobalUsers(usersToBulkSave)
+        const usersToBulkSave = await Promise.all(usersToSave)
+        await usersCore.bulkUpdateGlobalUsers(usersToBulkSave)
 
-      // Post-processing of bulk added users, e.g. events and cache operations
-      for (const user of usersToBulkSave) {
-        // TODO: Refactor to bulk insert users into the info db
-        // instead of relying on looping tenant creation
-        await platform.users.addUser(tenantId, user._id, user.email)
-        await eventHelpers.handleSaveEvents(user, undefined)
-      }
+        // Post-processing of bulk added users, e.g. events and cache operations
+        for (const user of usersToBulkSave) {
+          // TODO: Refactor to bulk insert users into the info db
+          // instead of relying on looping tenant creation
+          await platform.users.addUser(tenantId, user._id, user.email)
+          await eventHelpers.handleSaveEvents(user, undefined)
+        }
+
+        const saved = usersToBulkSave.map(user => {
+          return {
+            _id: user._id,
+            email: user.email,
+          }
+        })
+
+        // now update the groups
+        if (Array.isArray(saved) && groups) {
+          const groupPromises = []
+          const createdUserIds = saved.map(user => user._id)
+          for (let groupId of groups) {
+            groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds))
+          }
+          await Promise.all(groupPromises)
+        }
 
-      const saved = usersToBulkSave.map(user => {
         return {
-          _id: user._id,
-          email: user.email,
+          successful: saved,
+          unsuccessful,
         }
-      })
-
-      // now update the groups
-      if (Array.isArray(saved) && groups) {
-        const groupPromises = []
-        const createdUserIds = saved.map(user => user._id)
-        for (let groupId of groups) {
-          groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds))
-        }
-        await Promise.all(groupPromises)
       }
-
-      return {
-        successful: saved,
-        unsuccessful,
-      }
-    })
+    )
   }
 
   static async bulkDelete(userIds: string[]): Promise<BulkUserDeleted> {
@@ -419,11 +433,12 @@ export class UserDB {
       _deleted: true,
     }))
     const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete)
+    const creatorsToDelete = usersToDelete.filter(isCreator)
 
-    await UserDB.quotas.removeUsers(toDelete.length)
     for (let user of usersToDelete) {
       await bulkDeleteProcessing(user)
     }
+    await UserDB.quotas.removeUsers(toDelete.length, creatorsToDelete.length)
 
     // Build Response
     // index users by id
@@ -472,7 +487,8 @@ export class UserDB {
 
     await db.remove(userId, dbUser._rev)
 
-    await UserDB.quotas.removeUsers(1)
+    const creatorsToDelete = isCreator(dbUser) ? 1 : 0
+    await UserDB.quotas.removeUsers(1, creatorsToDelete)
     await eventHelpers.handleDeleteEvents(dbUser)
     await cache.user.invalidateUser(userId)
     await sessions.invalidateSessions(userId, { reason: "deletion" })
diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts
index 6237c23972..bad108ab84 100644
--- a/packages/backend-core/src/users/users.ts
+++ b/packages/backend-core/src/users/users.ts
@@ -14,11 +14,11 @@ import {
 } from "../db"
 import {
   BulkDocsResponse,
-  ContextUser,
   SearchQuery,
   SearchQueryOperators,
   SearchUsersRequest,
   User,
+  ContextUser,
   DatabaseQueryOpts,
 } from "@budibase/types"
 import { getGlobalDB } from "../context"
diff --git a/packages/backend-core/tests/core/users/users.spec.js b/packages/backend-core/tests/core/users/users.spec.js
new file mode 100644
index 0000000000..ae7109344a
--- /dev/null
+++ b/packages/backend-core/tests/core/users/users.spec.js
@@ -0,0 +1,54 @@
+const _ = require('lodash/fp')
+const {structures} = require("../../../tests")
+
+jest.mock("../../../src/context")
+jest.mock("../../../src/db")
+
+const context = require("../../../src/context")
+const db = require("../../../src/db")
+
+const {getCreatorCount} = require('../../../src/users/users')
+
+describe("Users", () => {
+
+  let getGlobalDBMock
+  let getGlobalUserParamsMock
+  let paginationMock
+
+  beforeEach(() => {
+    jest.resetAllMocks()
+
+    getGlobalDBMock = jest.spyOn(context, "getGlobalDB")
+    getGlobalUserParamsMock = jest.spyOn(db, "getGlobalUserParams")
+    paginationMock = jest.spyOn(db, "pagination")
+  })
+
+  it("Retrieves the number of creators", async () => {
+    const getUsers = (offset, limit, creators = false) => {
+      const range = _.range(offset, limit)
+      const opts = creators ? {builder: {global: true}} : undefined
+      return range.map(() => structures.users.user(opts))
+    }
+    const page1Data = getUsers(0, 8)
+    const page2Data = getUsers(8, 12, true)
+    getGlobalDBMock.mockImplementation(() => ({
+      name   : "fake-db",
+      allDocs: () => ({
+        rows: [...page1Data, ...page2Data]
+      })
+    }))
+    paginationMock.mockImplementationOnce(() => ({
+      data: page1Data,
+      hasNextPage: true,
+      nextPage: "1"
+    }))
+    paginationMock.mockImplementation(() => ({
+      data: page2Data,
+      hasNextPage: false,
+      nextPage: undefined
+    }))
+    const creatorsCount = await getCreatorCount()
+    expect(creatorsCount).toBe(4)
+    expect(paginationMock).toHaveBeenCalledTimes(2)
+  })
+})
diff --git a/packages/backend-core/tests/core/utilities/structures/licenses.ts b/packages/backend-core/tests/core/utilities/structures/licenses.ts
index 0e34f2e9bb..bb452f9ad5 100644
--- a/packages/backend-core/tests/core/utilities/structures/licenses.ts
+++ b/packages/backend-core/tests/core/utilities/structures/licenses.ts
@@ -123,6 +123,10 @@ export function customer(): Customer {
 export function subscription(): Subscription {
   return {
     amount: 10000,
+    amounts: {
+      user: 10000,
+      creator: 0,
+    },
     cancelAt: undefined,
     currency: "usd",
     currentPeriodEnd: 0,
@@ -131,6 +135,10 @@ export function subscription(): Subscription {
     duration: PriceDuration.MONTHLY,
     pastDueAt: undefined,
     quantity: 0,
+    quantities: {
+      user: 0,
+      creator: 0,
+    },
     status: "active",
   }
 }
diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts
index 53aa4842c4..e3935bc7ee 100644
--- a/packages/types/src/sdk/featureFlag.ts
+++ b/packages/types/src/sdk/featureFlag.ts
@@ -1,5 +1,8 @@
 export enum FeatureFlag {
   LICENSING = "LICENSING",
+  // Feature IDs in Posthog
+  PER_CREATOR_PER_USER_PRICE = "18873",
+  PER_CREATOR_PER_USER_PRICE_ALERT = "18530",
 }
 
 export interface TenantFeatureFlags {
diff --git a/packages/types/src/sdk/licensing/billing.ts b/packages/types/src/sdk/licensing/billing.ts
index 35f366c811..bcbc7abd18 100644
--- a/packages/types/src/sdk/licensing/billing.ts
+++ b/packages/types/src/sdk/licensing/billing.ts
@@ -5,10 +5,17 @@ export interface Customer {
   currency: string | null | undefined
 }
 
+export interface SubscriptionItems {
+  user: number | undefined
+  creator: number | undefined
+}
+
 export interface Subscription {
   amount: number
+  amounts: SubscriptionItems | undefined
   currency: string
   quantity: number
+  quantities: SubscriptionItems | undefined
   duration: PriceDuration
   cancelAt: number | null | undefined
   currentPeriodStart: number
diff --git a/packages/types/src/sdk/licensing/plan.ts b/packages/types/src/sdk/licensing/plan.ts
index 3e214a01ff..1604dfb8af 100644
--- a/packages/types/src/sdk/licensing/plan.ts
+++ b/packages/types/src/sdk/licensing/plan.ts
@@ -4,7 +4,9 @@ export enum PlanType {
   PRO = "pro",
   /** @deprecated */
   TEAM = "team",
+  /** @deprecated */
   PREMIUM = "premium",
+  PREMIUM_PLUS = "premium_plus",
   BUSINESS = "business",
   ENTERPRISE = "enterprise",
 }
@@ -26,10 +28,12 @@ export interface AvailablePrice {
   currency: string
   duration: PriceDuration
   priceId: string
+  type?: string
 }
 
 export enum PlanModel {
   PER_USER = "perUser",
+  PER_CREATOR_PER_USER = "per_creator_per_user",
   DAY_PASS = "dayPass",
 }
 

From 102a0824844f58a51ba784290271ed7e1fcdd10c Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Tue, 24 Oct 2023 17:18:13 +0200
Subject: [PATCH 030/134] Update pro submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index d24c0dc3a3..39bff12817 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit d24c0dc3a30014cbe61860252aa48104cad36376
+Subproject commit 39bff1281715c647f5d0c1db9bbf1d53c9fd4fc6

From d3670ddf21a43552aee52602c3a40acd7eea7f38 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 24 Oct 2023 17:22:49 +0100
Subject: [PATCH 031/134] Add an input to allow the user to choose the new
 column name.

---
 .../backend/DataTable/TableDataTable.svelte   |  1 +
 .../grid/controls/MigrationModal.svelte       | 22 ++++++++++++++++---
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
index 22deacbe03..40bcf2b74e 100644
--- a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
@@ -55,6 +55,7 @@
   }
 
   const refreshDefinitions = async () => {
+    console.log("woot")
     await tables.fetch()
     await datasources.fetch()
   }
diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
index b262f93797..e329038217 100644
--- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
+++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
@@ -1,11 +1,18 @@
 <script>
-  import { ModalContent, notifications } from "@budibase/bbui"
+  import {
+    ModalContent,
+    notifications,
+    Input,
+    InlineAlert,
+  } from "@budibase/bbui"
   import { getContext } from "svelte"
 
   const { API, dispatch, definition, rows } = getContext("grid")
 
   export let column
 
+  let newColumnName = `${column.schema.name} (migrated)`
+
   const migrateUserColumn = async () => {
     let subtype = "users"
     if (column.schema.relationshipType === "one-to-many") {
@@ -17,7 +24,7 @@
         tableId: $definition._id,
         oldColumn: column.schema,
         newColumn: {
-          name: `${column.schema.name} migrated`,
+          name: newColumnName,
           type: "bb_reference",
           subtype,
         },
@@ -38,5 +45,14 @@
   onConfirm={migrateUserColumn}
   size="M"
 >
-  TODO: copy here
+  This operation will kick off a migration of the column "{column.schema.name}"
+  to a new column, with the name provided - this operation may take a moment to
+  complete.
+
+  <InlineAlert
+    type="error"
+    header="Are you sure?"
+    message="This will leave bindings which utilised the user relationship column in a state where they will need to be updated to use the new column instead."
+  />
+  <Input bind:value={newColumnName} label="New column name" />
 </ModalContent>

From f4c6ab7f48960b780d398cd56858512b08221d70 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 24 Oct 2023 17:33:49 +0100
Subject: [PATCH 032/134] Make sure the user cannot enter the same column name
 as the old column for the migrated column.

---
 .../src/components/grid/cells/HeaderCell.svelte        |  1 -
 .../src/components/grid/controls/MigrationModal.svelte | 10 +++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index 9b05f9ef79..420be480e9 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -18,7 +18,6 @@
   export let orderable = true
 
   const {
-    API,
     reorder,
     isReordering,
     isResizing,
diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
index e329038217..f4070e7a98 100644
--- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
+++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
@@ -12,6 +12,7 @@
   export let column
 
   let newColumnName = `${column.schema.name} (migrated)`
+  $: newAndOldNameMatch = column.schema.name === newColumnName
 
   const migrateUserColumn = async () => {
     let subtype = "users"
@@ -43,6 +44,7 @@
   confirmText="Continue"
   cancelText="Cancel"
   onConfirm={migrateUserColumn}
+  disabled={newAndOldNameMatch}
   size="M"
 >
   This operation will kick off a migration of the column "{column.schema.name}"
@@ -54,5 +56,11 @@
     header="Are you sure?"
     message="This will leave bindings which utilised the user relationship column in a state where they will need to be updated to use the new column instead."
   />
-  <Input bind:value={newColumnName} label="New column name" />
+  <Input
+    bind:value={newColumnName}
+    label="New column name"
+    error={newAndOldNameMatch
+      ? "New column name can't be the same as the existing column name"
+      : undefined}
+  />
 </ModalContent>

From 0f5c2cff00305f4f6e2a87bd9ab8a2f60c1a8b67 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 24 Oct 2023 18:06:25 +0100
Subject: [PATCH 033/134] Handle a couple more edge cases more gracefully.

---
 .../server/src/sdk/app/tables/migration.ts    | 24 ++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts
index 08c06b28b9..407302cf1a 100644
--- a/packages/server/src/sdk/app/tables/migration.ts
+++ b/packages/server/src/sdk/app/tables/migration.ts
@@ -30,7 +30,15 @@ export async function migrate(
   table.schema[newColumn.name] = newColumn
   table = await sdk.tables.saveTable(table)
 
-  await migrator.doMigration()
+  try {
+    await migrator.doMigration()
+  } catch (e) {
+    // If the migration fails then we need to roll back the table schema
+    // change.
+    delete table.schema[newColumn.name]
+    await sdk.tables.saveTable(table)
+    throw e
+  }
 
   delete table.schema[oldColumn.name]
   table = await sdk.tables.saveTable(table)
@@ -133,6 +141,13 @@ class SingleUserColumnMigrator implements ColumnMigrator {
 
       let userId = dbCore.getGlobalIDFromUserMetadataID(link.doc2.rowId)
       let row = rowsById[link.doc1.rowId]
+      if (!row) {
+        // This can happen if the row has been deleted but the link hasn't,
+        // which was a state that was found during the initial testing of this
+        // feature. Not sure exactly what can cause it, but best to be safe.
+        continue
+      }
+
       row[this.newColumn.name] = userId
     }
 
@@ -171,6 +186,13 @@ class MultiUserColumnMigrator implements ColumnMigrator {
 
       let userId = dbCore.getGlobalIDFromUserMetadataID(link.doc2.rowId)
       let row = rowsById[link.doc1.rowId]
+      if (!row) {
+        // This can happen if the row has been deleted but the link hasn't,
+        // which was a state that was found during the initial testing of this
+        // feature. Not sure exactly what can cause it, but best to be safe.
+        continue
+      }
+
       if (!row[this.newColumn.name]) {
         row[this.newColumn.name] = []
       }

From 78afba63dec506e823589858af382b65804f61d1 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 25 Oct 2023 11:03:35 +0100
Subject: [PATCH 034/134] Emit table updates to the builder socket, and some
 minor refactoring.

---
 .../server/src/api/controllers/table/index.ts |   6 +-
 .../server/src/sdk/app/tables/migration.ts    | 148 ++++++++----------
 .../server/src/tests/utilities/api/table.ts   |   9 +-
 3 files changed, 78 insertions(+), 85 deletions(-)

diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index fb7f8d2a5a..1ad8542afd 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -165,7 +165,11 @@ export async function migrate(ctx: UserCtx<MigrateRequest, MigrateResponse>) {
   const { oldColumn, newColumn } = ctx.request.body
   let tableId = ctx.params.tableId as string
   const table = await sdk.tables.getTable(tableId)
-  await sdk.tables.migrate(table, oldColumn, newColumn)
+  let result = await sdk.tables.migrate(table, oldColumn, newColumn)
+
+  for (let table of result.tablesUpdated) {
+    builderSocket?.emitTableUpdate(ctx, table)
+  }
 
   ctx.status = 200
   ctx.body = { message: `Column ${oldColumn.name} migrated.` }
diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts
index 407302cf1a..b678b64318 100644
--- a/packages/server/src/sdk/app/tables/migration.ts
+++ b/packages/server/src/sdk/app/tables/migration.ts
@@ -1,37 +1,43 @@
-import { BadRequestError, context } from "@budibase/backend-core"
+import { BadRequestError, context, db as dbCore } from "@budibase/backend-core"
 import {
   BBReferenceFieldMetadata,
   FieldSchema,
   FieldSubtype,
   InternalTable,
-  ManyToManyRelationshipFieldMetadata,
-  ManyToOneRelationshipFieldMetadata,
-  OneToManyRelationshipFieldMetadata,
+  isBBReferenceField,
+  isRelationshipField,
+  LinkDocument,
+  RelationshipFieldMetadata,
   RelationshipType,
   Row,
   Table,
-  isBBReferenceField,
-  isRelationshipField,
 } from "@budibase/types"
 import sdk from "../../../sdk"
-import { isExternalTable } from "../../../../src/integrations/utils"
-import { db as dbCore } from "@budibase/backend-core"
-import { EventType, updateLinks } from "../../../../src/db/linkedRows"
+import { isExternalTable } from "../../../integrations/utils"
+import { EventType, updateLinks } from "../../../db/linkedRows"
 import { cloneDeep } from "lodash"
 
+export interface MigrationResult {
+  tablesUpdated: Table[]
+}
+
 export async function migrate(
   table: Table,
   oldColumn: FieldSchema,
   newColumn: FieldSchema
-) {
-  let migrator = getColumnMigrator(table, oldColumn, newColumn)
-  let oldTable = cloneDeep(table)
+): Promise<MigrationResult> {
+  if (newColumn.name in table.schema) {
+    throw new BadRequestError(`Column "${newColumn.name}" already exists`)
+  }
 
   table.schema[newColumn.name] = newColumn
   table = await sdk.tables.saveTable(table)
 
+  let migrator = getColumnMigrator(table, oldColumn, newColumn)
+  let result: MigrationResult
+
   try {
-    await migrator.doMigration()
+    result = await migrator.doMigration()
   } catch (e) {
     // If the migration fails then we need to roll back the table schema
     // change.
@@ -40,13 +46,11 @@ export async function migrate(
     throw e
   }
 
-  delete table.schema[oldColumn.name]
-  table = await sdk.tables.saveTable(table)
-  await updateLinks({ eventType: EventType.TABLE_UPDATED, table, oldTable })
+  return result
 }
 
 interface ColumnMigrator {
-  doMigration(): Promise<void>
+  doMigration(): Promise<MigrationResult>
 }
 
 function getColumnMigrator(
@@ -54,8 +58,8 @@ function getColumnMigrator(
   oldColumn: FieldSchema,
   newColumn: FieldSchema
 ): ColumnMigrator {
-  // For now we're only supporting migrations of user relationships to user
-  // columns in internal tables. In future we may want to support other
+  // For now, we're only supporting migrations of user relationships to user
+  // columns in internal tables. In the future, we may want to support other
   // migrations but for now return an error if we aren't migrating a user
   // relationship.
   if (isExternalTable(table._id!)) {
@@ -66,10 +70,6 @@ function getColumnMigrator(
     throw new BadRequestError(`Column "${oldColumn.name}" does not exist`)
   }
 
-  if (newColumn.name in table.schema) {
-    throw new BadRequestError(`Column "${newColumn.name}" already exists`)
-  }
-
   if (!isBBReferenceField(newColumn)) {
     throw new BadRequestError(`Column "${newColumn.name}" is not a user column`)
   }
@@ -113,14 +113,17 @@ function getColumnMigrator(
   throw new BadRequestError(`Unknown migration type`)
 }
 
-class SingleUserColumnMigrator implements ColumnMigrator {
+abstract class UserColumnMigrator implements ColumnMigrator {
   constructor(
-    private table: Table,
-    private oldColumn: OneToManyRelationshipFieldMetadata,
-    private newColumn: BBReferenceFieldMetadata
+    protected table: Table,
+    protected oldColumn: RelationshipFieldMetadata,
+    protected newColumn: BBReferenceFieldMetadata
   ) {}
 
-  async doMigration() {
+  abstract updateRow(row: Row, link: LinkDocument): void
+
+  async doMigration(): Promise<MigrationResult> {
+    let oldTable = cloneDeep(this.table)
     let rows = await sdk.rows.fetchRaw(this.table._id!)
     let rowsById = rows.reduce((acc, row) => {
       acc[row._id!] = row
@@ -129,17 +132,14 @@ class SingleUserColumnMigrator implements ColumnMigrator {
 
     let links = await sdk.links.fetchWithDocument(this.table._id!)
     for (let link of links) {
-      if (link.doc1.tableId !== this.table._id) {
-        continue
-      }
-      if (link.doc1.fieldName !== this.oldColumn.name) {
-        continue
-      }
-      if (link.doc2.tableId !== InternalTable.USER_METADATA) {
+      if (
+        link.doc1.tableId !== this.table._id ||
+        link.doc1.fieldName !== this.oldColumn.name ||
+        link.doc2.tableId !== InternalTable.USER_METADATA
+      ) {
         continue
       }
 
-      let userId = dbCore.getGlobalIDFromUserMetadataID(link.doc2.rowId)
       let row = rowsById[link.doc1.rowId]
       if (!row) {
         // This can happen if the row has been deleted but the link hasn't,
@@ -148,58 +148,42 @@ class SingleUserColumnMigrator implements ColumnMigrator {
         continue
       }
 
-      row[this.newColumn.name] = userId
+      this.updateRow(row, link)
     }
 
     let db = context.getAppDB()
     await db.bulkDocs(rows)
-  }
-}
 
-class MultiUserColumnMigrator implements ColumnMigrator {
-  constructor(
-    private table: Table,
-    private oldColumn:
-      | ManyToManyRelationshipFieldMetadata
-      | ManyToOneRelationshipFieldMetadata,
-    private newColumn: BBReferenceFieldMetadata
-  ) {}
+    delete this.table.schema[this.oldColumn.name]
+    this.table = await sdk.tables.saveTable(this.table)
+    await updateLinks({
+      eventType: EventType.TABLE_UPDATED,
+      table: this.table,
+      oldTable,
+    })
 
-  async doMigration() {
-    let rows = await sdk.rows.fetchRaw(this.table._id!)
-    let rowsById = rows.reduce((acc, row) => {
-      acc[row._id!] = row
-      return acc
-    }, {} as Record<string, Row>)
-
-    let links = await sdk.links.fetchWithDocument(this.table._id!)
-    for (let link of links) {
-      if (link.doc1.tableId !== this.table._id) {
-        continue
-      }
-      if (link.doc1.fieldName !== this.oldColumn.name) {
-        continue
-      }
-      if (link.doc2.tableId !== InternalTable.USER_METADATA) {
-        continue
-      }
-
-      let userId = dbCore.getGlobalIDFromUserMetadataID(link.doc2.rowId)
-      let row = rowsById[link.doc1.rowId]
-      if (!row) {
-        // This can happen if the row has been deleted but the link hasn't,
-        // which was a state that was found during the initial testing of this
-        // feature. Not sure exactly what can cause it, but best to be safe.
-        continue
-      }
-
-      if (!row[this.newColumn.name]) {
-        row[this.newColumn.name] = []
-      }
-      row[this.newColumn.name].push(userId)
+    let otherTable = await sdk.tables.getTable(this.oldColumn.tableId)
+    return {
+      tablesUpdated: [this.table, otherTable],
     }
-
-    let db = context.getAppDB()
-    await db.bulkDocs(rows)
+  }
+}
+
+class SingleUserColumnMigrator extends UserColumnMigrator {
+  updateRow(row: Row, link: LinkDocument): void {
+    row[this.newColumn.name] = dbCore.getGlobalIDFromUserMetadataID(
+      link.doc2.rowId
+    )
+  }
+}
+
+class MultiUserColumnMigrator extends UserColumnMigrator {
+  updateRow(row: Row, link: LinkDocument): void {
+    if (!row[this.newColumn.name]) {
+      row[this.newColumn.name] = []
+    }
+    row[this.newColumn.name].push(
+      dbCore.getGlobalIDFromUserMetadataID(link.doc2.rowId)
+    )
   }
 }
diff --git a/packages/server/src/tests/utilities/api/table.ts b/packages/server/src/tests/utilities/api/table.ts
index 501841f6e7..b80c940697 100644
--- a/packages/server/src/tests/utilities/api/table.ts
+++ b/packages/server/src/tests/utilities/api/table.ts
@@ -58,8 +58,13 @@ export class TableAPI extends TestAPI {
       .post(`/api/tables/${tableId}/migrate`)
       .send(data)
       .set(this.config.defaultHeaders())
-      .expect("Content-Type", /json/)
-      .expect(expectStatus)
+    if (res.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          res.status
+        } with body ${JSON.stringify(res.body)}`
+      )
+    }
     return res.body
   }
 }

From 52f97fbd1fdbb9b017f5f3c001f992dac7ae2f94 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 25 Oct 2023 11:38:31 +0100
Subject: [PATCH 035/134] Attempt to emit events that include the originator.

---
 .../server/src/api/controllers/table/index.ts |  2 +-
 packages/server/src/websockets/builder.ts     | 18 ++++++++++-----
 packages/server/src/websockets/websocket.ts   | 23 +++++++++++++++----
 3 files changed, 31 insertions(+), 12 deletions(-)

diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index 1ad8542afd..af3c2c0c15 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -168,7 +168,7 @@ export async function migrate(ctx: UserCtx<MigrateRequest, MigrateResponse>) {
   let result = await sdk.tables.migrate(table, oldColumn, newColumn)
 
   for (let table of result.tablesUpdated) {
-    builderSocket?.emitTableUpdate(ctx, table)
+    builderSocket?.emitTableUpdate(ctx, table, { includeSelf: true })
   }
 
   ctx.status = 200
diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts
index d2fdbca20c..b1ef473d6c 100644
--- a/packages/server/src/websockets/builder.ts
+++ b/packages/server/src/websockets/builder.ts
@@ -1,5 +1,5 @@
 import authorized from "../middleware/authorized"
-import { BaseSocket } from "./websocket"
+import { BaseSocket, EmitOptions } from "./websocket"
 import { permissions, events, context } from "@budibase/backend-core"
 import http from "http"
 import Koa from "koa"
@@ -100,11 +100,17 @@ export default class BuilderSocket extends BaseSocket {
     })
   }
 
-  emitTableUpdate(ctx: any, table: Table) {
-    this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.TableChange, {
-      id: table._id,
-      table,
-    })
+  emitTableUpdate(ctx: any, table: Table, options?: EmitOptions) {
+    this.emitToRoom(
+      ctx,
+      ctx.appId,
+      BuilderSocketEvent.TableChange,
+      {
+        id: table._id,
+        table,
+      },
+      options
+    )
     gridSocket?.emitTableUpdate(ctx, table)
   }
 
diff --git a/packages/server/src/websockets/websocket.ts b/packages/server/src/websockets/websocket.ts
index ffaf9e2763..a0dc388d4f 100644
--- a/packages/server/src/websockets/websocket.ts
+++ b/packages/server/src/websockets/websocket.ts
@@ -11,6 +11,12 @@ import { SocketSession } from "@budibase/types"
 import { v4 as uuid } from "uuid"
 import { createContext, runMiddlewares } from "./middleware"
 
+export interface EmitOptions {
+  // Whether to include the originator of the request from the broadcast,
+  // defaults to false.
+  includeSelf?: boolean
+}
+
 const anonUser = () => ({
   _id: uuid(),
   email: "user@mail.com",
@@ -270,10 +276,17 @@ export class BaseSocket {
 
   // Emit an event to everyone in a room, including metadata of whom
   // the originator of the request was
-  emitToRoom(ctx: any, room: string | string[], event: string, payload: any) {
-    this.io.in(room).emit(event, {
-      ...payload,
-      apiSessionId: ctx.headers?.[Header.SESSION_ID],
-    })
+  emitToRoom(
+    ctx: any,
+    room: string | string[],
+    event: string,
+    payload: any,
+    options?: EmitOptions
+  ) {
+    let emitPayload = { ...payload }
+    if (!options?.includeSelf) {
+      emitPayload.apiSessionId = ctx.headers?.[Header.SESSION_ID]
+    }
+    this.io.in(room).emit(event, emitPayload)
   }
 }

From b952892c42587c04d1bcc1608c156de215178d23 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Wed, 25 Oct 2023 12:08:56 +0100
Subject: [PATCH 036/134] Generalised the fix as not all calculations will have
 a groupby per PR feedback

---
 .../src/components/backend/DataTable/Table.svelte |  4 +++-
 .../backend/DataTable/ViewDataTable.svelte        | 15 +--------------
 2 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/Table.svelte b/packages/builder/src/components/backend/DataTable/Table.svelte
index 45e440eca5..f7eccd5242 100644
--- a/packages/builder/src/components/backend/DataTable/Table.svelte
+++ b/packages/builder/src/components/backend/DataTable/Table.svelte
@@ -28,7 +28,9 @@
 
   $: if (schema) {
     parsedSchema = Object.keys(schema).reduce((acc, key) => {
-      acc[key] = { ...schema[key] }
+      acc[key] =
+        typeof schema[key] === "string" ? { type: schema[key] } : schema[key]
+
       if (!canBeSortColumn(acc[key].type)) {
         acc[key].sortable = false
       }
diff --git a/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte
index 12e4e3c270..6068439dbd 100644
--- a/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte
@@ -19,21 +19,8 @@
   let loading = false
   let type = "internal"
 
-  // Grouped views have a fixed schema layout
-  const parseSchema = view => {
-    if (!view.groupBy) {
-      return view.schema
-    }
-    const { group, field, value } = view.schema
-    return {
-      group,
-      field: { type: field },
-      value: { type: value },
-    }
-  }
-
   $: name = view.name
-  $: schema = parseSchema(view)
+  $: schema = view.schema
   $: calculation = view.calculation
 
   $: supportedFormats = Object.values(ROW_EXPORT_FORMATS).filter(key => {

From 70f39b6b6bc86ad34ef7f80826b1d6e4707ef39f Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 25 Oct 2023 14:41:23 +0100
Subject: [PATCH 037/134] Fix missing sourceId on Table sent to the websocket.

---
 packages/builder/src/builderStore/websocket.js     |  1 +
 packages/server/src/api/controllers/table/index.ts |  5 ++++-
 packages/server/src/sdk/app/tables/getters.ts      | 12 ++++++++----
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/packages/builder/src/builderStore/websocket.js b/packages/builder/src/builderStore/websocket.js
index 6121831c38..2aed5a3f8f 100644
--- a/packages/builder/src/builderStore/websocket.js
+++ b/packages/builder/src/builderStore/websocket.js
@@ -47,6 +47,7 @@ export const createBuilderWebsocket = appId => {
 
   // Data section events
   socket.onOther(BuilderSocketEvent.TableChange, ({ id, table }) => {
+    console.log("Table change", id, table)
     tables.replaceTable(id, table)
   })
   socket.onOther(BuilderSocketEvent.DatasourceChange, ({ id, datasource }) => {
diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index af3c2c0c15..f139313086 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -24,6 +24,7 @@ import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"
 import { builderSocket } from "../../../websockets"
 import { cloneDeep, isEqual } from "lodash"
+import { processInternalTable } from "../../../sdk/app/tables/getters"
 
 function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
   if (table && !tableId) {
@@ -168,7 +169,9 @@ export async function migrate(ctx: UserCtx<MigrateRequest, MigrateResponse>) {
   let result = await sdk.tables.migrate(table, oldColumn, newColumn)
 
   for (let table of result.tablesUpdated) {
-    builderSocket?.emitTableUpdate(ctx, table, { includeSelf: true })
+    builderSocket?.emitTableUpdate(ctx, processInternalTable(table), {
+      includeSelf: true,
+    })
   }
 
   ctx.status = 200
diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts
index 02cef748c5..34cddc8dc7 100644
--- a/packages/server/src/sdk/app/tables/getters.ts
+++ b/packages/server/src/sdk/app/tables/getters.ts
@@ -20,11 +20,15 @@ import datasources from "../datasources"
 import sdk from "../../../sdk"
 
 function processInternalTables(docs: AllDocsResponse<Table[]>): Table[] {
-  return docs.rows.map((tableDoc: any) => ({
-    ...tableDoc.doc,
+  return docs.rows.map(tableDoc => processInternalTable(tableDoc.doc))
+}
+
+export function processInternalTable(table: Table): Table {
+  return {
+    ...table,
     type: "internal",
-    sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id,
-  }))
+    sourceId: table.sourceId || BudibaseInternalDB._id,
+  }
 }
 
 export async function getAllInternalTables(db?: Database): Promise<Table[]> {

From 007e919d0edf97bd138c101981d02a5c06fc8153 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 25 Oct 2023 14:53:51 +0100
Subject: [PATCH 038/134] Spring cleaning before review.

---
 packages/server/src/api/controllers/table/index.ts | 4 ++--
 packages/server/src/integrations/utils.ts          | 4 ++++
 packages/server/src/websockets/builder.ts          | 9 +++++++++
 packages/server/src/websockets/websocket.ts        | 8 +++++---
 4 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index f139313086..d2ad63c13e 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -169,8 +169,8 @@ export async function migrate(ctx: UserCtx<MigrateRequest, MigrateResponse>) {
   let result = await sdk.tables.migrate(table, oldColumn, newColumn)
 
   for (let table of result.tablesUpdated) {
-    builderSocket?.emitTableUpdate(ctx, processInternalTable(table), {
-      includeSelf: true,
+    builderSocket?.emitTableUpdate(ctx, table, {
+      includeOriginator: true,
     })
   }
 
diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts
index 79b18e767c..b37fe9f0ed 100644
--- a/packages/server/src/integrations/utils.ts
+++ b/packages/server/src/integrations/utils.ts
@@ -87,6 +87,10 @@ export function isExternalTable(tableId: string) {
   return tableId.includes(DocumentType.DATASOURCE)
 }
 
+export function isInternalTable(tableId: string) {
+  return !isExternalTable(tableId)
+}
+
 export function buildExternalTableId(datasourceId: string, tableName: string) {
   // encode spaces
   if (tableName.includes(" ")) {
diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts
index b1ef473d6c..2cff24e635 100644
--- a/packages/server/src/websockets/builder.ts
+++ b/packages/server/src/websockets/builder.ts
@@ -16,6 +16,8 @@ import { gridSocket } from "./index"
 import { clearLock, updateLock } from "../utilities/redis"
 import { Socket } from "socket.io"
 import { BuilderSocketEvent } from "@budibase/shared-core"
+import { processInternalTable } from "../sdk/app/tables/getters"
+import { isExternalTable, isInternalTable } from "../integrations/utils"
 
 export default class BuilderSocket extends BaseSocket {
   constructor(app: Koa, server: http.Server) {
@@ -101,6 +103,13 @@ export default class BuilderSocket extends BaseSocket {
   }
 
   emitTableUpdate(ctx: any, table: Table, options?: EmitOptions) {
+    // This was added to make sure that sourceId is always present when
+    // sending this message to clients. Without this, tables without a
+    // sourceId (e.g. ta_users) won't get correctly updated client-side.
+    if (isInternalTable(table._id!)) {
+      table = processInternalTable(table)
+    }
+
     this.emitToRoom(
       ctx,
       ctx.appId,
diff --git a/packages/server/src/websockets/websocket.ts b/packages/server/src/websockets/websocket.ts
index a0dc388d4f..1dba108d24 100644
--- a/packages/server/src/websockets/websocket.ts
+++ b/packages/server/src/websockets/websocket.ts
@@ -13,8 +13,10 @@ import { createContext, runMiddlewares } from "./middleware"
 
 export interface EmitOptions {
   // Whether to include the originator of the request from the broadcast,
-  // defaults to false.
-  includeSelf?: boolean
+  // defaults to false because it is assumed that the user who triggered
+  // an action will already have the changes of that action reflected in their
+  // own UI, so there is no need to send them again.
+  includeOriginator?: boolean
 }
 
 const anonUser = () => ({
@@ -284,7 +286,7 @@ export class BaseSocket {
     options?: EmitOptions
   ) {
     let emitPayload = { ...payload }
-    if (!options?.includeSelf) {
+    if (!options?.includeOriginator) {
       emitPayload.apiSessionId = ctx.headers?.[Header.SESSION_ID]
     }
     this.io.in(room).emit(event, emitPayload)

From 5779e97e6eaf50687a0829592246d3081fc73a03 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 25 Oct 2023 15:29:56 +0100
Subject: [PATCH 039/134] Add column name input validation.

---
 .../builder/src/builderStore/websocket.js     |  1 -
 .../grid/controls/MigrationModal.svelte       | 24 +++++++++++--------
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/packages/builder/src/builderStore/websocket.js b/packages/builder/src/builderStore/websocket.js
index 2aed5a3f8f..6121831c38 100644
--- a/packages/builder/src/builderStore/websocket.js
+++ b/packages/builder/src/builderStore/websocket.js
@@ -47,7 +47,6 @@ export const createBuilderWebsocket = appId => {
 
   // Data section events
   socket.onOther(BuilderSocketEvent.TableChange, ({ id, table }) => {
-    console.log("Table change", id, table)
     tables.replaceTable(id, table)
   })
   socket.onOther(BuilderSocketEvent.DatasourceChange, ({ id, datasource }) => {
diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
index f4070e7a98..ee7fd7bfbe 100644
--- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
+++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
@@ -6,13 +6,23 @@
     InlineAlert,
   } from "@budibase/bbui"
   import { getContext } from "svelte"
+  import { ValidColumnNameRegex } from "@budibase/shared-core"
 
   const { API, dispatch, definition, rows } = getContext("grid")
 
   export let column
 
-  let newColumnName = `${column.schema.name} (migrated)`
-  $: newAndOldNameMatch = column.schema.name === newColumnName
+  let newColumnName = `${column.schema.name} migrated`
+  $: error = checkNewColumnName(newColumnName)
+
+  const checkNewColumnName = newColumnName => {
+    if (column.schema.name === newColumnName) {
+      return "New column name can't be the same as the existing column name."
+    }
+    if (newColumnName.match(ValidColumnNameRegex) === null) {
+      return "Illegal character; must be alpha-numeric."
+    }
+  }
 
   const migrateUserColumn = async () => {
     let subtype = "users"
@@ -44,7 +54,7 @@
   confirmText="Continue"
   cancelText="Cancel"
   onConfirm={migrateUserColumn}
-  disabled={newAndOldNameMatch}
+  disabled={error !== undefined}
   size="M"
 >
   This operation will kick off a migration of the column "{column.schema.name}"
@@ -56,11 +66,5 @@
     header="Are you sure?"
     message="This will leave bindings which utilised the user relationship column in a state where they will need to be updated to use the new column instead."
   />
-  <Input
-    bind:value={newColumnName}
-    label="New column name"
-    error={newAndOldNameMatch
-      ? "New column name can't be the same as the existing column name"
-      : undefined}
-  />
+  <Input bind:value={newColumnName} label="New column name" {error} />
 </ModalContent>

From 402a53cacb3e413f64a0473247e63e78355132c9 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 25 Oct 2023 15:33:12 +0100
Subject: [PATCH 040/134] User constant for user table ID.

---
 .../frontend-core/src/components/grid/cells/HeaderCell.svelte  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index dee73d17e9..02cf751e30 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -14,6 +14,7 @@
   import MigrationModal from "../controls/MigrationModal.svelte"
   import { debounce } from "../../../utils/utils"
   import { FieldType, FormulaTypes } from "@budibase/types"
+  import { TableNames } from "../../../constants"
 
   export let column
   export let idx
@@ -381,7 +382,7 @@
       >
         Hide column
       </MenuItem>
-      {#if $config.canEditColumns && column.schema.type === "link" && column.schema.tableId === "ta_users"}
+      {#if $config.canEditColumns && column.schema.type === "link" && column.schema.tableId === TableNames.USERS}
         <MenuItem icon="User" on:click={openMigrationModal}>
           Migrate to user column
         </MenuItem>

From 0ac6b17ff09441c06ae7d19fe7c51843bf99c069 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 25 Oct 2023 15:33:53 +0100
Subject: [PATCH 041/134] Remove stray console.log

---
 .../src/components/backend/DataTable/TableDataTable.svelte       | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
index 40bcf2b74e..22deacbe03 100644
--- a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
@@ -55,7 +55,6 @@
   }
 
   const refreshDefinitions = async () => {
-    console.log("woot")
     await tables.fetch()
     await datasources.fetch()
   }

From c9e3b7a348ce41bd3e9e4bc1bf406a83463f6d35 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 25 Oct 2023 16:00:14 +0100
Subject: [PATCH 042/134] Remove the previous attempt at syncing grid state, it
 is no longer needed.

---
 .../src/components/backend/DataTable/TableDataTable.svelte  | 6 ------
 .../src/components/grid/controls/MigrationModal.svelte      | 1 -
 2 files changed, 7 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
index 22deacbe03..5fee849afb 100644
--- a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
@@ -53,11 +53,6 @@
       await datasources.fetch()
     }
   }
-
-  const refreshDefinitions = async () => {
-    await tables.fetch()
-    await datasources.fetch()
-  }
 </script>
 
 <div class="wrapper">
@@ -71,7 +66,6 @@
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
     on:updatedatasource={handleGridTableUpdate}
-    on:refreshdefinitions={refreshDefinitions}
   >
     <svelte:fragment slot="filter">
       {#if isUsersTable && $store.features.disableUserMetadata}
diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
index ee7fd7bfbe..111ff86170 100644
--- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
+++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
@@ -45,7 +45,6 @@
       notifications.error(`Failed to migrate: ${e.message}`)
     }
     await rows.actions.refreshData()
-    dispatch("refreshdefintions")
   }
 </script>
 

From 7acce7b7c0b0a8ca4363263a8d728007c0eecfb8 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 25 Oct 2023 16:01:08 +0100
Subject: [PATCH 043/134] Remove unused dispatch import.

---
 .../src/components/grid/controls/MigrationModal.svelte          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
index 111ff86170..ec2972a03e 100644
--- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
+++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
@@ -8,7 +8,7 @@
   import { getContext } from "svelte"
   import { ValidColumnNameRegex } from "@budibase/shared-core"
 
-  const { API, dispatch, definition, rows } = getContext("grid")
+  const { API, definition, rows } = getContext("grid")
 
   export let column
 

From 2c5dd99da202a45dfb97a5cb87351862ecc3627e Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 25 Oct 2023 16:37:15 +0100
Subject: [PATCH 044/134] Use FieldSubtype enum instead of raw strings.

---
 .../src/components/grid/controls/MigrationModal.svelte       | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
index ec2972a03e..0f173da8b9 100644
--- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
+++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
@@ -7,6 +7,7 @@
   } from "@budibase/bbui"
   import { getContext } from "svelte"
   import { ValidColumnNameRegex } from "@budibase/shared-core"
+  import { FieldSubtype } from "@budibase/types"
 
   const { API, definition, rows } = getContext("grid")
 
@@ -25,9 +26,9 @@
   }
 
   const migrateUserColumn = async () => {
-    let subtype = "users"
+    let subtype = FieldSubtype.USERS
     if (column.schema.relationshipType === "one-to-many") {
-      subtype = "user"
+      subtype = FieldSubtype.USER
     }
 
     try {

From ef84e96f98c7a38a6145fb47beee4bdbf5237a73 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 25 Oct 2023 16:38:14 +0100
Subject: [PATCH 045/134] Use RelationshipType enum instead of raw string.

---
 .../src/components/grid/controls/MigrationModal.svelte        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
index 0f173da8b9..c6a49c6a48 100644
--- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
+++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
@@ -7,7 +7,7 @@
   } from "@budibase/bbui"
   import { getContext } from "svelte"
   import { ValidColumnNameRegex } from "@budibase/shared-core"
-  import { FieldSubtype } from "@budibase/types"
+  import { FieldSubtype, RelationshipType } from "@budibase/types"
 
   const { API, definition, rows } = getContext("grid")
 
@@ -27,7 +27,7 @@
 
   const migrateUserColumn = async () => {
     let subtype = FieldSubtype.USERS
-    if (column.schema.relationshipType === "one-to-many") {
+    if (column.schema.relationshipType === RelationshipType.ONE_TO_MANY) {
       subtype = FieldSubtype.USER
     }
 

From c5097487e22438a0e4fbc961f2cb422c07329a00 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 25 Oct 2023 16:38:55 +0100
Subject: [PATCH 046/134] Use FieldType constant instead of raw string.

---
 .../src/components/grid/controls/MigrationModal.svelte        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
index c6a49c6a48..73ea32408b 100644
--- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
+++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
@@ -7,7 +7,7 @@
   } from "@budibase/bbui"
   import { getContext } from "svelte"
   import { ValidColumnNameRegex } from "@budibase/shared-core"
-  import { FieldSubtype, RelationshipType } from "@budibase/types"
+  import { FieldSubtype, FieldType, RelationshipType } from "@budibase/types"
 
   const { API, definition, rows } = getContext("grid")
 
@@ -37,7 +37,7 @@
         oldColumn: column.schema,
         newColumn: {
           name: newColumnName,
-          type: "bb_reference",
+          type: FieldType.BB_REFERENCE,
           subtype,
         },
       })

From e03b1be9d1b2c7c3e8c3ded8f0b3e304a23446bc Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 25 Oct 2023 16:41:37 +0100
Subject: [PATCH 047/134] Make sure new column name cannot be the same as an
 existing column name.

---
 .../src/components/grid/controls/MigrationModal.svelte        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
index 73ea32408b..1957c3259f 100644
--- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
+++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
@@ -17,8 +17,8 @@
   $: error = checkNewColumnName(newColumnName)
 
   const checkNewColumnName = newColumnName => {
-    if (column.schema.name === newColumnName) {
-      return "New column name can't be the same as the existing column name."
+    if (newColumnName in $definition.schema) {
+      return "New column name can't be the same as an existing column name."
     }
     if (newColumnName.match(ValidColumnNameRegex) === null) {
       return "Illegal character; must be alpha-numeric."

From 4a00649f7f7c01d1280a08ff35cedd4963779021 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 25 Oct 2023 16:46:14 +0100
Subject: [PATCH 048/134] Simplify the function signature of
 processInternalTables

---
 packages/server/src/sdk/app/tables/getters.ts | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts
index 34cddc8dc7..47da0beb40 100644
--- a/packages/server/src/sdk/app/tables/getters.ts
+++ b/packages/server/src/sdk/app/tables/getters.ts
@@ -19,8 +19,8 @@ import {
 import datasources from "../datasources"
 import sdk from "../../../sdk"
 
-function processInternalTables(docs: AllDocsResponse<Table[]>): Table[] {
-  return docs.rows.map(tableDoc => processInternalTable(tableDoc.doc))
+function processInternalTables(tables: Table[]): Table[] {
+  return tables.map(processInternalTable)
 }
 
 export function processInternalTable(table: Table): Table {
@@ -40,7 +40,7 @@ export async function getAllInternalTables(db?: Database): Promise<Table[]> {
       include_docs: true,
     })
   )
-  return processInternalTables(internalTables)
+  return processInternalTables(internalTables.rows.map(row => row.doc!))
 }
 
 async function getAllExternalTables(): Promise<Table[]> {
@@ -110,7 +110,9 @@ export async function getTables(tableIds: string[]): Promise<Table[]> {
     const internalTableDocs = await db.allDocs<Table[]>(
       getMultiIDParams(internalTableIds)
     )
-    tables = tables.concat(processInternalTables(internalTableDocs))
+    tables = tables.concat(
+      processInternalTables(internalTableDocs.rows.map(row => row.doc!))
+    )
   }
   return tables
 }

From 6c3b535863878c521bb312606218b89fa2f0f787 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 25 Oct 2023 16:49:29 +0100
Subject: [PATCH 049/134] Simplify try-catch in the migrate function.

---
 packages/server/src/sdk/app/tables/migration.ts | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts
index b678b64318..293f9184d6 100644
--- a/packages/server/src/sdk/app/tables/migration.ts
+++ b/packages/server/src/sdk/app/tables/migration.ts
@@ -34,10 +34,8 @@ export async function migrate(
   table = await sdk.tables.saveTable(table)
 
   let migrator = getColumnMigrator(table, oldColumn, newColumn)
-  let result: MigrationResult
-
   try {
-    result = await migrator.doMigration()
+    return await migrator.doMigration()
   } catch (e) {
     // If the migration fails then we need to roll back the table schema
     // change.
@@ -45,8 +43,6 @@ export async function migrate(
     await sdk.tables.saveTable(table)
     throw e
   }
-
-  return result
 }
 
 interface ColumnMigrator {

From 455b26bac95917be3b5d15176248498fd55635e9 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 25 Oct 2023 19:00:25 +0100
Subject: [PATCH 050/134] Making sure the source ID is always set when creating
 a table - the frontend expects this to be set for every table so making the
 type represent this correctly.

---
 .../server/src/api/controllers/datasource.ts  |  3 +-
 .../server/src/api/controllers/table/index.ts |  3 +-
 .../server/src/api/routes/tests/row.spec.ts   |  6 +-
 .../server/src/api/routes/tests/table.spec.ts |  4 ++
 .../src/api/routes/tests/viewV2.spec.ts       |  2 +
 packages/server/src/constants/index.ts        | 14 ++---
 packages/server/src/db/utils.ts               |  3 +-
 .../server/src/integrations/googlesheets.ts   |  8 +--
 .../src/integrations/microsoftSqlServer.ts    |  6 +-
 packages/server/src/integrations/mysql.ts     |  6 +-
 packages/server/src/integrations/oracle.ts    |  6 +-
 packages/server/src/integrations/postgres.ts  |  6 +-
 .../integrations/tests/googlesheets.spec.ts   |  2 +
 packages/server/src/integrations/utils.ts     |  9 ++-
 .../middleware/tests/trimViewRowInfo.spec.ts  |  9 ++-
 .../src/sdk/app/rows/search/internal.ts       |  6 +-
 .../app/rows/search/tests/external.spec.ts    | 62 ++++++++++---------
 .../app/rows/search/tests/internal.spec.ts    |  9 ++-
 .../sdk/app/rows/search/tests/utils.spec.ts   |  3 +
 .../src/sdk/app/tables/external/index.ts      |  2 +-
 .../src/sdk/app/tables/external/utils.ts      |  4 +-
 packages/server/src/sdk/app/tables/getters.ts | 10 +--
 .../src/sdk/app/tables/tests/tables.spec.ts   |  8 ++-
 .../sdk/app/tables/tests/validation.spec.ts   | 41 ++++++++----
 .../src/sdk/app/views/tests/views.spec.ts     |  2 +
 .../src/tests/utilities/TestConfiguration.ts  | 27 ++++++--
 .../server/src/tests/utilities/structures.ts  |  2 +
 .../tests/inputProcessing.spec.ts             | 11 +++-
 .../tests/outputProcessing.spec.ts            |  4 ++
 packages/server/src/websockets/builder.ts     |  2 +-
 .../types/src/documents/app/table/table.ts    |  8 +--
 packages/types/src/sdk/datasources.ts         |  6 +-
 32 files changed, 185 insertions(+), 109 deletions(-)

diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts
index b50c2464f0..5d024d51b6 100644
--- a/packages/server/src/api/controllers/datasource.ts
+++ b/packages/server/src/api/controllers/datasource.ts
@@ -12,7 +12,6 @@ import {
   CreateDatasourceResponse,
   Datasource,
   DatasourcePlus,
-  ExternalTable,
   FetchDatasourceInfoRequest,
   FetchDatasourceInfoResponse,
   IntegrationBase,
@@ -59,7 +58,7 @@ async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
   const connector = (await getConnector(datasource)) as DatasourcePlus
   return await connector.buildSchema(
     datasource._id!,
-    datasource.entities! as Record<string, ExternalTable>
+    datasource.entities! as Record<string, Table>
   )
 }
 
diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index d2ad63c13e..c814a37ad9 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -24,7 +24,6 @@ import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"
 import { builderSocket } from "../../../websockets"
 import { cloneDeep, isEqual } from "lodash"
-import { processInternalTable } from "../../../sdk/app/tables/getters"
 
 function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
   if (table && !tableId) {
@@ -50,7 +49,7 @@ export async function fetch(ctx: UserCtx<void, FetchTablesResponse>) {
       return Object.values(entities).map<Table>((entity: Table) => ({
         ...entity,
         type: "external",
-        sourceId: datasource._id,
+        sourceId: datasource._id!,
         sql: isSQL(datasource),
       }))
     } else {
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 4c2e7a7494..e5a24c27e0 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -21,6 +21,7 @@ import {
   SortType,
   StaticQuotaName,
   Table,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 import {
   expectAnyExternalColsAttributes,
@@ -65,6 +66,7 @@ describe.each([
       type: "table",
       primary: ["id"],
       primaryDisplay: "name",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         id: {
           type: FieldType.AUTO,
@@ -880,6 +882,7 @@ describe.each([
     async function userTable(): Promise<Table> {
       return {
         name: `users_${generator.word()}`,
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
         type: "table",
         primary: ["id"],
         schema: {
@@ -1062,6 +1065,7 @@ describe.each([
       async function userTable(): Promise<Table> {
         return {
           name: `users_${generator.word()}`,
+          sourceId: INTERNAL_TABLE_SOURCE_ID,
           type: "table",
           primary: ["id"],
           schema: {
@@ -1597,7 +1601,7 @@ describe.each([
       const tableConfig = generateTableConfig()
 
       if (config.datasource) {
-        tableConfig.sourceId = config.datasource._id
+        tableConfig.sourceId = config.datasource._id!
         if (config.datasource.plus) {
           tableConfig.type = "external"
         }
diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts
index 420717f7f0..f988b44e0f 100644
--- a/packages/server/src/api/routes/tests/table.spec.ts
+++ b/packages/server/src/api/routes/tests/table.spec.ts
@@ -9,6 +9,7 @@ import {
   InternalTable,
   FieldSubtype,
   Row,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
@@ -432,6 +433,7 @@ describe("/tables", () => {
       const table = await config.api.table.create({
         name: "table",
         type: "table",
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
         schema: {
           "user relationship": {
             type: FieldType.LINK,
@@ -491,6 +493,7 @@ describe("/tables", () => {
       const table = await config.api.table.create({
         name: "table",
         type: "table",
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
         schema: {
           "user relationship": {
             type: FieldType.LINK,
@@ -552,6 +555,7 @@ describe("/tables", () => {
       const table = await config.api.table.create({
         name: "table",
         type: "table",
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
         schema: {
           "user relationship": {
             type: FieldType.LINK,
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 40060aef48..a1ce5ec10e 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -10,6 +10,7 @@ import {
   UIFieldMetadata,
   UpdateViewRequest,
   ViewV2,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 import { generator } from "@budibase/backend-core/tests"
 import { generateDatasourceID } from "../../../db/utils"
@@ -18,6 +19,7 @@ function priceTable(): Table {
   return {
     name: "table",
     type: "table",
+    sourceId: INTERNAL_TABLE_SOURCE_ID,
     schema: {
       Price: {
         type: FieldType.NUMBER,
diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts
index b37a4b36c1..fe69b3c9c8 100644
--- a/packages/server/src/constants/index.ts
+++ b/packages/server/src/constants/index.ts
@@ -1,5 +1,9 @@
 import { objectStore, roles, constants } from "@budibase/backend-core"
-import { FieldType as FieldTypes } from "@budibase/types"
+import {
+  FieldType as FieldTypes,
+  Table,
+  INTERNAL_TABLE_SOURCE_ID,
+} from "@budibase/types"
 export {
   FieldType as FieldTypes,
   RelationshipType,
@@ -70,9 +74,10 @@ export enum SortDirection {
   DESCENDING = "DESCENDING",
 }
 
-export const USERS_TABLE_SCHEMA = {
+export const USERS_TABLE_SCHEMA: Table = {
   _id: "ta_users",
   type: "table",
+  sourceId: INTERNAL_TABLE_SOURCE_ID,
   views: {},
   name: "Users",
   // TODO: ADMIN PANEL - when implemented this doesn't need to be carried out
@@ -87,12 +92,10 @@ export const USERS_TABLE_SCHEMA = {
         },
         presence: true,
       },
-      fieldName: "email",
       name: "email",
     },
     firstName: {
       name: "firstName",
-      fieldName: "firstName",
       type: FieldTypes.STRING,
       constraints: {
         type: FieldTypes.STRING,
@@ -101,7 +104,6 @@ export const USERS_TABLE_SCHEMA = {
     },
     lastName: {
       name: "lastName",
-      fieldName: "lastName",
       type: FieldTypes.STRING,
       constraints: {
         type: FieldTypes.STRING,
@@ -109,7 +111,6 @@ export const USERS_TABLE_SCHEMA = {
       },
     },
     roleId: {
-      fieldName: "roleId",
       name: "roleId",
       type: FieldTypes.OPTIONS,
       constraints: {
@@ -119,7 +120,6 @@ export const USERS_TABLE_SCHEMA = {
       },
     },
     status: {
-      fieldName: "status",
       name: "status",
       type: FieldTypes.OPTIONS,
       constraints: {
diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts
index 2c07bd8d22..d532d8a8b2 100644
--- a/packages/server/src/db/utils.ts
+++ b/packages/server/src/db/utils.ts
@@ -5,6 +5,7 @@ import {
   FieldSchema,
   RelationshipFieldMetadata,
   VirtualDocumentType,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 import { FieldTypes } from "../constants"
 export { DocumentType, VirtualDocumentType } from "@budibase/types"
@@ -18,7 +19,7 @@ export const enum AppStatus {
 }
 
 export const BudibaseInternalDB = {
-  _id: "bb_internal",
+  _id: INTERNAL_TABLE_SOURCE_ID,
   type: dbCore.BUDIBASE_DATASOURCE_TYPE,
   name: "Budibase DB",
   source: "BUDIBASE",
diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index 57b6682cc8..4433f45863 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -12,7 +12,7 @@ import {
   Row,
   SearchFilters,
   SortJson,
-  ExternalTable,
+  Table,
   TableRequest,
   Schema,
 } from "@budibase/types"
@@ -262,7 +262,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     id?: string
   ) {
     // base table
-    const table: ExternalTable = {
+    const table: Table = {
       name: title,
       primary: [GOOGLE_SHEETS_PRIMARY_KEY],
       schema: {},
@@ -283,7 +283,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema> {
     // not fully configured yet
     if (!this.config.auth) {
@@ -291,7 +291,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     }
     await this.connect()
     const sheets = this.client.sheetsByIndex
-    const tables: Record<string, ExternalTable> = {}
+    const tables: Record<string, Table> = {}
     let errors: Record<string, string> = {}
     await utils.parallelForeach(
       sheets,
diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts
index ff68026369..b86286756c 100644
--- a/packages/server/src/integrations/microsoftSqlServer.ts
+++ b/packages/server/src/integrations/microsoftSqlServer.ts
@@ -2,7 +2,7 @@ import {
   DatasourceFieldType,
   Integration,
   Operation,
-  ExternalTable,
+  Table,
   TableSchema,
   QueryJson,
   QueryType,
@@ -380,7 +380,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
    */
   async buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema> {
     await this.connect()
     let tableInfo: MSSQLTablesResponse[] = await this.runSQL(this.TABLES_SQL)
@@ -394,7 +394,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
       .map((record: any) => record.TABLE_NAME)
       .filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1)
 
-    const tables: Record<string, ExternalTable> = {}
+    const tables: Record<string, Table> = {}
     for (let tableName of tableNames) {
       // get the column definition (type)
       const definition = await this.runSQL(
diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts
index 3a954da9bd..fe7eae51be 100644
--- a/packages/server/src/integrations/mysql.ts
+++ b/packages/server/src/integrations/mysql.ts
@@ -4,7 +4,7 @@ import {
   QueryType,
   QueryJson,
   SqlQuery,
-  ExternalTable,
+  Table,
   TableSchema,
   DatasourcePlus,
   DatasourceFeature,
@@ -278,9 +278,9 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
 
   async buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema> {
-    const tables: { [key: string]: ExternalTable } = {}
+    const tables: { [key: string]: Table } = {}
     await this.connect()
 
     try {
diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index b3936320ac..5fde565180 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -5,7 +5,7 @@ import {
   QueryJson,
   QueryType,
   SqlQuery,
-  ExternalTable,
+  Table,
   DatasourcePlus,
   DatasourceFeature,
   ConnectionInfo,
@@ -263,14 +263,14 @@ class OracleIntegration extends Sql implements DatasourcePlus {
    */
   async buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema> {
     const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
       sql: this.COLUMNS_SQL,
     })
     const oracleTables = this.mapColumns(columnsResponse)
 
-    const tables: { [key: string]: ExternalTable } = {}
+    const tables: { [key: string]: Table } = {}
 
     // iterate each table
     Object.values(oracleTables).forEach(oracleTable => {
diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts
index 8479cd05d8..38339499b2 100644
--- a/packages/server/src/integrations/postgres.ts
+++ b/packages/server/src/integrations/postgres.ts
@@ -5,7 +5,7 @@ import {
   QueryType,
   QueryJson,
   SqlQuery,
-  ExternalTable,
+  Table,
   DatasourcePlus,
   DatasourceFeature,
   ConnectionInfo,
@@ -273,7 +273,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
    */
   async buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema> {
     let tableKeys: { [key: string]: string[] } = {}
     await this.openConnection()
@@ -300,7 +300,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
       const columnsResponse: { rows: PostgresColumn[] } =
         await this.client.query(this.COLUMNS_SQL)
 
-      const tables: { [key: string]: ExternalTable } = {}
+      const tables: { [key: string]: Table } = {}
 
       for (let column of columnsResponse.rows) {
         const tableName: string = column.table_name
diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index 748baddc39..842b681867 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -31,6 +31,7 @@ import { structures } from "@budibase/backend-core/tests"
 import TestConfiguration from "../../tests/utilities/TestConfiguration"
 import GoogleSheetsIntegration from "../googlesheets"
 import { FieldType, Table, TableSchema } from "@budibase/types"
+import { generateDatasourceID } from "../../db/utils"
 
 describe("Google Sheets Integration", () => {
   let integration: any,
@@ -61,6 +62,7 @@ describe("Google Sheets Integration", () => {
   function createBasicTable(name: string, columns: string[]): Table {
     return {
       name,
+      sourceId: generateDatasourceID(),
       schema: {
         ...columns.reduce((p, c) => {
           p[c] = {
diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts
index b37fe9f0ed..9e6c5ca3af 100644
--- a/packages/server/src/integrations/utils.ts
+++ b/packages/server/src/integrations/utils.ts
@@ -4,7 +4,6 @@ import {
   SearchFilters,
   Datasource,
   FieldType,
-  ExternalTable,
 } from "@budibase/types"
 import { DocumentType, SEPARATOR } from "../db/utils"
 import { InvalidColumns, NoEmptyFilterStrings } from "../constants"
@@ -301,9 +300,9 @@ function copyExistingPropsOver(
  * @param entities The old list of tables, if there was any to look for definitions in.
  */
 export function finaliseExternalTables(
-  tables: Record<string, ExternalTable>,
-  entities: Record<string, ExternalTable>
-): Record<string, ExternalTable> {
+  tables: Record<string, Table>,
+  entities: Record<string, Table>
+): Record<string, Table> {
   let finalTables: Record<string, Table> = {}
   const tableIds = Object.values(tables).map(table => table._id!)
   for (let [name, table] of Object.entries(tables)) {
@@ -316,7 +315,7 @@ export function finaliseExternalTables(
 }
 
 export function checkExternalTables(
-  tables: Record<string, ExternalTable>
+  tables: Record<string, Table>
 ): Record<string, string> {
   const invalidColumns = Object.values(InvalidColumns) as string[]
   const errors: Record<string, string> = {}
diff --git a/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts b/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts
index bf717d5828..106129f6c9 100644
--- a/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts
+++ b/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts
@@ -1,5 +1,11 @@
 import { generator } from "@budibase/backend-core/tests"
-import { BBRequest, FieldType, Row, Table } from "@budibase/types"
+import {
+  BBRequest,
+  FieldType,
+  Row,
+  Table,
+  INTERNAL_TABLE_SOURCE_ID,
+} from "@budibase/types"
 import * as utils from "../../db/utils"
 import trimViewRowInfoMiddleware from "../trimViewRowInfo"
 
@@ -73,6 +79,7 @@ describe("trimViewRowInfo middleware", () => {
   const table: Table = {
     _id: tableId,
     name: generator.word(),
+    sourceId: INTERNAL_TABLE_SOURCE_ID,
     type: "table",
     schema: {
       name: {
diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts
index 58611c8849..1aec8a321e 100644
--- a/packages/server/src/sdk/app/rows/search/internal.ts
+++ b/packages/server/src/sdk/app/rows/search/internal.ts
@@ -197,11 +197,7 @@ export async function fetchView(
     try {
       table = await sdk.tables.getTable(viewInfo.meta.tableId)
     } catch (err) {
-      /* istanbul ignore next */
-      table = {
-        name: "",
-        schema: {},
-      }
+      throw new Error("Unable to retrieve view table.")
     }
     rows = await outputProcessing(table, response.rows)
   }
diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts
index b3bddfbc97..1afdca35fa 100644
--- a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts
+++ b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts
@@ -15,6 +15,7 @@ import {
   expectAnyExternalColsAttributes,
   generator,
 } from "@budibase/backend-core/tests"
+import datasource from "../../../../../api/routes/datasource"
 
 jest.unmock("mysql2/promise")
 
@@ -23,36 +24,7 @@ jest.setTimeout(30000)
 describe.skip("external", () => {
   const config = new TestConfiguration()
 
-  let externalDatasource: Datasource
-
-  const tableData: Table = {
-    name: generator.word(),
-    type: "external",
-    primary: ["id"],
-    schema: {
-      id: {
-        name: "id",
-        type: FieldType.AUTO,
-        autocolumn: true,
-      },
-      name: {
-        name: "name",
-        type: FieldType.STRING,
-      },
-      surname: {
-        name: "surname",
-        type: FieldType.STRING,
-      },
-      age: {
-        name: "age",
-        type: FieldType.NUMBER,
-      },
-      address: {
-        name: "address",
-        type: FieldType.STRING,
-      },
-    },
-  }
+  let externalDatasource: Datasource, tableData: Table
 
   beforeAll(async () => {
     const container = await new GenericContainer("mysql")
@@ -84,6 +56,36 @@ describe.skip("external", () => {
         },
       },
     })
+
+    tableData = {
+      name: generator.word(),
+      type: "external",
+      primary: ["id"],
+      sourceId: externalDatasource._id!,
+      schema: {
+        id: {
+          name: "id",
+          type: FieldType.AUTO,
+          autocolumn: true,
+        },
+        name: {
+          name: "name",
+          type: FieldType.STRING,
+        },
+        surname: {
+          name: "surname",
+          type: FieldType.STRING,
+        },
+        age: {
+          name: "age",
+          type: FieldType.NUMBER,
+        },
+        address: {
+          name: "address",
+          type: FieldType.STRING,
+        },
+      },
+    }
   })
 
   describe("search", () => {
diff --git a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts
index b3e98a1149..b3cac2321e 100644
--- a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts
+++ b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts
@@ -1,4 +1,10 @@
-import { FieldType, Row, Table, SearchParams } from "@budibase/types"
+import {
+  FieldType,
+  Row,
+  Table,
+  SearchParams,
+  INTERNAL_TABLE_SOURCE_ID,
+} from "@budibase/types"
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
 import { search } from "../internal"
 import {
@@ -12,6 +18,7 @@ describe("internal", () => {
   const tableData: Table = {
     name: generator.word(),
     type: "table",
+    sourceId: INTERNAL_TABLE_SOURCE_ID,
     schema: {
       name: {
         name: "name",
diff --git a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts
index d946eea432..428c57be64 100644
--- a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts
+++ b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts
@@ -5,12 +5,14 @@ import {
   FieldTypeSubtypes,
   Table,
   SearchParams,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 
 const tableId = "ta_a"
 const tableWithUserCol: Table = {
   _id: tableId,
   name: "table",
+  sourceId: INTERNAL_TABLE_SOURCE_ID,
   schema: {
     user: {
       name: "user",
@@ -23,6 +25,7 @@ const tableWithUserCol: Table = {
 const tableWithUsersCol: Table = {
   _id: tableId,
   name: "table",
+  sourceId: INTERNAL_TABLE_SOURCE_ID,
   schema: {
     user: {
       name: "user",
diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts
index 402baada78..f445fcaf08 100644
--- a/packages/server/src/sdk/app/tables/external/index.ts
+++ b/packages/server/src/sdk/app/tables/external/index.ts
@@ -35,10 +35,10 @@ export async function save(
   opts?: { tableId?: string; renaming?: RenameColumn }
 ) {
   let tableToSave: TableRequest = {
+    ...update,
     type: "table",
     _id: buildExternalTableId(datasourceId, update.name),
     sourceId: datasourceId,
-    ...update,
   }
 
   const tableId = opts?.tableId || update._id
diff --git a/packages/server/src/sdk/app/tables/external/utils.ts b/packages/server/src/sdk/app/tables/external/utils.ts
index 10c755a7d6..a60667f44f 100644
--- a/packages/server/src/sdk/app/tables/external/utils.ts
+++ b/packages/server/src/sdk/app/tables/external/utils.ts
@@ -76,12 +76,14 @@ export function generateManyLinkSchema(
   const primary = table.name + table.primary[0]
   const relatedPrimary = relatedTable.name + relatedTable.primary[0]
   const jcTblName = generateJunctionTableName(column, table, relatedTable)
+  const datasourceId = datasource._id!
   // first create the new table
   const junctionTable = {
-    _id: buildExternalTableId(datasource._id!, jcTblName),
+    _id: buildExternalTableId(datasourceId, jcTblName),
     name: jcTblName,
     primary: [primary, relatedPrimary],
     constrained: [primary, relatedPrimary],
+    sourceId: datasourceId,
     schema: {
       [primary]: foreignKeyStructure(primary, {
         toTable: table.name,
diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts
index 47da0beb40..af779bcc2b 100644
--- a/packages/server/src/sdk/app/tables/getters.ts
+++ b/packages/server/src/sdk/app/tables/getters.ts
@@ -1,20 +1,16 @@
 import { context } from "@budibase/backend-core"
-import {
-  BudibaseInternalDB,
-  getMultiIDParams,
-  getTableParams,
-} from "../../../db/utils"
+import { getMultiIDParams, getTableParams } from "../../../db/utils"
 import {
   breakExternalTableId,
   isExternalTable,
   isSQL,
 } from "../../../integrations/utils"
 import {
-  AllDocsResponse,
   Database,
   Table,
   TableResponse,
   TableViewsResponse,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 import datasources from "../datasources"
 import sdk from "../../../sdk"
@@ -27,7 +23,7 @@ export function processInternalTable(table: Table): Table {
   return {
     ...table,
     type: "internal",
-    sourceId: table.sourceId || BudibaseInternalDB._id,
+    sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID,
   }
 }
 
diff --git a/packages/server/src/sdk/app/tables/tests/tables.spec.ts b/packages/server/src/sdk/app/tables/tests/tables.spec.ts
index 78ebe59f01..ab45eeb8bb 100644
--- a/packages/server/src/sdk/app/tables/tests/tables.spec.ts
+++ b/packages/server/src/sdk/app/tables/tests/tables.spec.ts
@@ -1,4 +1,9 @@
-import { FieldType, Table, ViewV2 } from "@budibase/types"
+import {
+  FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
+  Table,
+  ViewV2,
+} from "@budibase/types"
 import { generator } from "@budibase/backend-core/tests"
 import sdk from "../../.."
 
@@ -13,6 +18,7 @@ describe("table sdk", () => {
       _id: generator.guid(),
       name: "TestTable",
       type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         name: {
           type: FieldType.STRING,
diff --git a/packages/server/src/sdk/app/tables/tests/validation.spec.ts b/packages/server/src/sdk/app/tables/tests/validation.spec.ts
index 5347eede90..62a9c61dbb 100644
--- a/packages/server/src/sdk/app/tables/tests/validation.spec.ts
+++ b/packages/server/src/sdk/app/tables/tests/validation.spec.ts
@@ -1,36 +1,50 @@
 import { populateExternalTableSchemas } from "../validation"
 import { cloneDeep } from "lodash/fp"
-import { AutoReason, Datasource, Table } from "@budibase/types"
+import {
+  AutoReason,
+  Datasource,
+  FieldType,
+  RelationshipType,
+  SourceName,
+  Table,
+} from "@budibase/types"
 import { isEqual } from "lodash"
+import { generateDatasourceID } from "../../../../db/utils"
 
-const SCHEMA = {
+const datasourceId = generateDatasourceID()
+
+const SCHEMA: Datasource = {
+  source: SourceName.POSTGRES,
+  type: "datasource",
+  _id: datasourceId,
   entities: {
     client: {
       _id: "tableA",
       name: "client",
       primary: ["idC"],
       primaryDisplay: "Name",
+      sourceId: datasourceId,
       schema: {
         idC: {
           autocolumn: true,
           externalType: "int unsigned",
           name: "idC",
-          type: "number",
+          type: FieldType.NUMBER,
         },
         Name: {
           autocolumn: false,
           externalType: "varchar(255)",
           name: "Name",
-          type: "string",
+          type: FieldType.STRING,
         },
         project: {
           fieldName: "idC",
           foreignKey: "idC",
           main: true,
           name: "project",
-          relationshipType: "many-to-one",
+          relationshipType: RelationshipType.MANY_TO_ONE,
           tableId: "tableB",
-          type: "link",
+          type: FieldType.LINK,
         },
       },
     },
@@ -39,31 +53,32 @@ const SCHEMA = {
       name: "project",
       primary: ["idP"],
       primaryDisplay: "Name",
+      sourceId: datasourceId,
       schema: {
         idC: {
           externalType: "int unsigned",
           name: "idC",
-          type: "number",
+          type: FieldType.NUMBER,
         },
         idP: {
           autocolumn: true,
           externalType: "int unsigned",
           name: "idProject",
-          type: "number",
+          type: FieldType.NUMBER,
         },
         Name: {
           autocolumn: false,
           externalType: "varchar(255)",
           name: "Name",
-          type: "string",
+          type: FieldType.STRING,
         },
         client: {
           fieldName: "idC",
           foreignKey: "idC",
           name: "client",
-          relationshipType: "one-to-many",
+          relationshipType: RelationshipType.ONE_TO_MANY,
           tableId: "tableA",
-          type: "link",
+          type: FieldType.LINK,
         },
       },
       sql: true,
@@ -95,12 +110,12 @@ describe("validation and update of external table schemas", () => {
   function noOtherTableChanges(response: any) {
     checkOtherColumns(
       response.entities!.client!,
-      SCHEMA.entities.client as Table,
+      SCHEMA.entities!.client,
       OTHER_CLIENT_COLS
     )
     checkOtherColumns(
       response.entities!.project!,
-      SCHEMA.entities.project as Table,
+      SCHEMA.entities!.project,
       OTHER_PROJECT_COLS
     )
   }
diff --git a/packages/server/src/sdk/app/views/tests/views.spec.ts b/packages/server/src/sdk/app/views/tests/views.spec.ts
index 8fcc6405ef..2068eda4c3 100644
--- a/packages/server/src/sdk/app/views/tests/views.spec.ts
+++ b/packages/server/src/sdk/app/views/tests/views.spec.ts
@@ -2,6 +2,7 @@ import _ from "lodash"
 import {
   FieldSchema,
   FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
   Table,
   TableSchema,
   ViewV2,
@@ -14,6 +15,7 @@ describe("table sdk", () => {
     _id: generator.guid(),
     name: "TestTable",
     type: "table",
+    sourceId: INTERNAL_TABLE_SOURCE_ID,
     schema: {
       name: {
         type: FieldType.STRING,
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 5f5b211975..3d4046b91b 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -56,6 +56,7 @@ import {
   CreateViewRequest,
   RelationshipFieldMetadata,
   User,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 
 import API from "./api"
@@ -68,6 +69,10 @@ type DefaultUserValues = {
   csrfToken: string
 }
 
+interface TableToBuild extends Omit<Table, "sourceId"> {
+  sourceId?: string
+}
+
 class TestConfiguration {
   server: any
   request: supertest.SuperTest<supertest.Test> | undefined
@@ -538,10 +543,13 @@ class TestConfiguration {
   // TABLE
 
   async updateTable(
-    config?: Table,
+    config?: TableToBuild,
     { skipReassigning } = { skipReassigning: false }
   ): Promise<Table> {
     config = config || basicTable()
+    if (!config.sourceId) {
+      config.sourceId = INTERNAL_TABLE_SOURCE_ID
+    }
     const response = await this._req(config, null, controllers.table.save)
     if (!skipReassigning) {
       this.table = response
@@ -549,13 +557,19 @@ class TestConfiguration {
     return response
   }
 
-  async createTable(config?: Table, options = { skipReassigning: false }) {
+  async createTable(
+    config?: TableToBuild,
+    options = { skipReassigning: false }
+  ) {
     if (config != null && config._id) {
       delete config._id
     }
     config = config || basicTable()
+    if (!config.sourceId) {
+      config.sourceId = INTERNAL_TABLE_SOURCE_ID
+    }
     if (this.datasource && !config.sourceId) {
-      config.sourceId = this.datasource._id
+      config.sourceId = this.datasource._id || INTERNAL_TABLE_SOURCE_ID
       if (this.datasource.plus) {
         config.type = "external"
       }
@@ -572,12 +586,15 @@ class TestConfiguration {
   async createLinkedTable(
     relationshipType = RelationshipType.ONE_TO_MANY,
     links: any = ["link"],
-    config?: Table
+    config?: TableToBuild
   ) {
     if (!this.table) {
       throw "Must have created a table first."
     }
     const tableConfig = config || basicTable()
+    if (!tableConfig.sourceId) {
+      tableConfig.sourceId = INTERNAL_TABLE_SOURCE_ID
+    }
     tableConfig.primaryDisplay = "name"
     for (let link of links) {
       tableConfig.schema[link] = {
@@ -590,7 +607,7 @@ class TestConfiguration {
     }
 
     if (this.datasource && !tableConfig.sourceId) {
-      tableConfig.sourceId = this.datasource._id
+      tableConfig.sourceId = this.datasource._id || INTERNAL_TABLE_SOURCE_ID
       if (this.datasource.plus) {
         tableConfig.type = "external"
       }
diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts
index d3e92ea34d..52f3450454 100644
--- a/packages/server/src/tests/utilities/structures.ts
+++ b/packages/server/src/tests/utilities/structures.ts
@@ -19,12 +19,14 @@ import {
   FieldType,
   SourceName,
   Table,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 
 export function basicTable(): Table {
   return {
     name: "TestTable",
     type: "table",
+    sourceId: INTERNAL_TABLE_SOURCE_ID,
     schema: {
       name: {
         type: FieldType.STRING,
diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
index 18d5128986..10274ed4d9 100644
--- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
@@ -1,6 +1,11 @@
 import { inputProcessing } from ".."
 import { generator, structures } from "@budibase/backend-core/tests"
-import { FieldType, FieldTypeSubtypes, Table } from "@budibase/types"
+import {
+  FieldType,
+  FieldTypeSubtypes,
+  INTERNAL_TABLE_SOURCE_ID,
+  Table,
+} from "@budibase/types"
 import * as bbReferenceProcessor from "../bbReferenceProcessor"
 
 jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
@@ -20,6 +25,7 @@ describe("rowProcessor - inputProcessing", () => {
       _id: generator.guid(),
       name: "TestTable",
       type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         name: {
           type: FieldType.STRING,
@@ -70,6 +76,7 @@ describe("rowProcessor - inputProcessing", () => {
       _id: generator.guid(),
       name: "TestTable",
       type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         name: {
           type: FieldType.STRING,
@@ -110,6 +117,7 @@ describe("rowProcessor - inputProcessing", () => {
         _id: generator.guid(),
         name: "TestTable",
         type: "table",
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
         schema: {
           name: {
             type: FieldType.STRING,
@@ -150,6 +158,7 @@ describe("rowProcessor - inputProcessing", () => {
       _id: generator.guid(),
       name: "TestTable",
       type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         name: {
           type: FieldType.STRING,
diff --git a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts
index ecb8856c88..56db285f8b 100644
--- a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts
@@ -2,6 +2,7 @@ import {
   FieldSubtype,
   FieldType,
   FieldTypeSubtypes,
+  INTERNAL_TABLE_SOURCE_ID,
   Table,
 } from "@budibase/types"
 import { outputProcessing } from ".."
@@ -26,6 +27,7 @@ describe("rowProcessor - outputProcessing", () => {
       _id: generator.guid(),
       name: "TestTable",
       type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         name: {
           type: FieldType.STRING,
@@ -71,6 +73,7 @@ describe("rowProcessor - outputProcessing", () => {
       _id: generator.guid(),
       name: "TestTable",
       type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         name: {
           type: FieldType.STRING,
@@ -108,6 +111,7 @@ describe("rowProcessor - outputProcessing", () => {
       _id: generator.guid(),
       name: "TestTable",
       type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         name: {
           type: FieldType.STRING,
diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts
index 2cff24e635..0428561bfe 100644
--- a/packages/server/src/websockets/builder.ts
+++ b/packages/server/src/websockets/builder.ts
@@ -17,7 +17,7 @@ import { clearLock, updateLock } from "../utilities/redis"
 import { Socket } from "socket.io"
 import { BuilderSocketEvent } from "@budibase/shared-core"
 import { processInternalTable } from "../sdk/app/tables/getters"
-import { isExternalTable, isInternalTable } from "../integrations/utils"
+import { isInternalTable } from "../integrations/utils"
 
 export default class BuilderSocket extends BaseSocket {
   constructor(app: Koa, server: http.Server) {
diff --git a/packages/types/src/documents/app/table/table.ts b/packages/types/src/documents/app/table/table.ts
index 5174ec608f..df17351c12 100644
--- a/packages/types/src/documents/app/table/table.ts
+++ b/packages/types/src/documents/app/table/table.ts
@@ -3,14 +3,16 @@ import { View, ViewV2 } from "../view"
 import { RenameColumn } from "../../../sdk"
 import { TableSchema } from "./schema"
 
+export const INTERNAL_TABLE_SOURCE_ID = "bb_internal"
+
 export interface Table extends Document {
   type?: string
   views?: { [key: string]: View | ViewV2 }
   name: string
+  sourceId: string
   primary?: string[]
   schema: TableSchema
   primaryDisplay?: string
-  sourceId?: string
   relatedFormula?: string[]
   constrained?: string[]
   sql?: boolean
@@ -19,10 +21,6 @@ export interface Table extends Document {
   rowHeight?: number
 }
 
-export interface ExternalTable extends Table {
-  sourceId: string
-}
-
 export interface TableRequest extends Table {
   _rename?: RenameColumn
   created?: boolean
diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts
index 39a10961de..7a335eb3b9 100644
--- a/packages/types/src/sdk/datasources.ts
+++ b/packages/types/src/sdk/datasources.ts
@@ -1,4 +1,4 @@
-import { ExternalTable, Table } from "../documents"
+import { Table } from "../documents"
 
 export const PASSWORD_REPLACEMENT = "--secret-value--"
 
@@ -176,7 +176,7 @@ export interface IntegrationBase {
 }
 
 export interface Schema {
-  tables: Record<string, ExternalTable>
+  tables: Record<string, Table>
   errors: Record<string, string>
 }
 
@@ -187,7 +187,7 @@ export interface DatasourcePlus extends IntegrationBase {
   getStringConcat(parts: string[]): string
   buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema>
   getTableNames(): Promise<string[]>
 }

From fd0d8f17f2a3ac4c4413c5b64ed0c730d3f75bcd Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 25 Oct 2023 19:07:51 +0100
Subject: [PATCH 051/134] Making sure single table get also includes sourceId.

---
 packages/server/src/sdk/app/tables/getters.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts
index af779bcc2b..b9c67c573b 100644
--- a/packages/server/src/sdk/app/tables/getters.ts
+++ b/packages/server/src/sdk/app/tables/getters.ts
@@ -67,7 +67,7 @@ export async function getTable(tableId: string): Promise<Table> {
     const table = await getExternalTable(datasourceId!, tableName!)
     return { ...table, sql: isSQL(datasource) }
   } else {
-    return db.get(tableId)
+    return processInternalTable(await db.get<Table>(tableId))
   }
 }
 

From 7ddaf6479f934af59832269f06bc2f4c144aad17 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 26 Oct 2023 09:54:38 +0200
Subject: [PATCH 052/134] Clean unused pipelines

---
 .../workflows/release-singleimage-test.yml    | 72 -------------------
 package.json                                  |  1 -
 2 files changed, 73 deletions(-)
 delete mode 100644 .github/workflows/release-singleimage-test.yml

diff --git a/.github/workflows/release-singleimage-test.yml b/.github/workflows/release-singleimage-test.yml
deleted file mode 100644
index c3a14226ce..0000000000
--- a/.github/workflows/release-singleimage-test.yml
+++ /dev/null
@@ -1,72 +0,0 @@
-name: Test
-
-on:
-  workflow_dispatch:
-
-env:
-  CI: true
-  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-  REGISTRY_URL: registry.hub.docker.com
-  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-jobs:
-  build:
-    name: "build"
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [18.x]
-    steps:
-      - name: "Checkout"
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ matrix.node-version }}
-          cache: "yarn"
-      - name: Setup QEMU
-        uses: docker/setup-qemu-action@v3
-      - name: Setup Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Run Yarn
-        run: yarn
-      - name: Run Yarn Build
-        run: yarn build --scope @budibase/server --scope @budibase/worker
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_API_KEY }}
-      - name: Get the latest release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo $release_version
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-      - name: Tag and release Budibase service docker image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          pull: true
-          platforms: linux/amd64,linux/arm64
-          build-args: BUDIBASE_VERSION=0.0.0+test
-          tags: budibase/budibase-test:test
-          file: ./hosting/single/Dockerfile.v2
-          cache-from: type=registry,ref=budibase/budibase-test:test
-          cache-to: type=inline
-      - name: Tag and release Budibase Azure App Service docker image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          platforms: linux/amd64
-          build-args: |
-            TARGETBUILD=aas
-            BUDIBASE_VERSION=0.0.0+test
-          tags: budibase/budibase-test:aas
-          file: ./hosting/single/Dockerfile.v2
diff --git a/package.json b/package.json
index 100a306a35..d3f4903e6c 100644
--- a/package.json
+++ b/package.json
@@ -33,7 +33,6 @@
     "build:sdk": "lerna run --stream build:sdk",
     "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
     "release": "lerna publish from-package --yes --force-publish --no-git-tag-version --no-push --no-git-reset",
-    "release:develop": "yarn release --dist-tag develop",
     "restore": "yarn run clean && yarn && yarn run build",
     "nuke": "yarn run nuke:packages && yarn run nuke:docker",
     "nuke:packages": "yarn run restore",

From 389848efbefe4e37d3bf397d16fda5ad04e9abdc Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 26 Oct 2023 11:05:07 +0200
Subject: [PATCH 053/134] Add update versions v2

---
 scripts/updateWorkepaceVersions.V2.sh | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100755 scripts/updateWorkepaceVersions.V2.sh

diff --git a/scripts/updateWorkepaceVersions.V2.sh b/scripts/updateWorkepaceVersions.V2.sh
new file mode 100755
index 0000000000..634bcbcfb0
--- /dev/null
+++ b/scripts/updateWorkepaceVersions.V2.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+version=$1
+echo "Setting version $version"
+yarn lerna exec "yarn version --no-git-tag-version --new-version=$version"
+echo "Updating dependencies"
+node scripts/syncLocalDependencies.js $version
+echo "Syncing yarn workspace"
+yarn

From 6032216e2f4b9f21fb640f554fe1a68c731cce9e Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Thu, 26 Oct 2023 10:13:22 +0100
Subject: [PATCH 054/134] Added fix for scenario where componentdefinition was
 queried befor it existed. Fix to ensure an empty general settings block is
 not shown

---
 .../Component/ComponentSettingsSection.svelte      | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
index cd91e974ce..6dc9078f2c 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
@@ -34,10 +34,14 @@
       setting => setting.section && setting.tag === tag
     )
     let sections = [
-      {
-        name: "General",
-        settings: generalSettings,
-      },
+      ...(generalSettings?.length
+        ? [
+            {
+              name: "General",
+              settings: generalSettings,
+            },
+          ]
+        : []),
       ...(customSections || []),
     ]
 
@@ -142,7 +146,7 @@
         <div class="section-info">
           <InfoDisplay body={section.info} />
         </div>
-      {:else if idx === 0 && section.name === "General" && componentDefinition.info && !tag}
+      {:else if idx === 0 && section.name === "General" && componentDefinition?.info && !tag}
         <InfoDisplay
           title={componentDefinition.name}
           body={componentDefinition.info}

From 980e223aef9e9aab8545b5339a9776cce75b7a82 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 26 Oct 2023 11:24:02 +0200
Subject: [PATCH 055/134] Fix typo

---
 ...pdateWorkepaceVersions.V2.sh => updateWorkspaceVersions.V2.sh} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename scripts/{updateWorkepaceVersions.V2.sh => updateWorkspaceVersions.V2.sh} (100%)

diff --git a/scripts/updateWorkepaceVersions.V2.sh b/scripts/updateWorkspaceVersions.V2.sh
similarity index 100%
rename from scripts/updateWorkepaceVersions.V2.sh
rename to scripts/updateWorkspaceVersions.V2.sh

From 5909b4a7c723c8e13fc735e7844fcf00331f8b0b Mon Sep 17 00:00:00 2001
From: Gerard Burns <Ghrehh@users.noreply.github.com>
Date: Thu, 26 Oct 2023 11:24:43 +0100
Subject: [PATCH 056/134] Add template for grid blocks (#12086)

* wip

* linting

---------

Co-authored-by: Mihail Hadzhiev <102024164+MihailHadzhiev2022@users.noreply.github.com>
---
 .../store/screenTemplates/rowListScreen.js    |  22 ++++++++++++++----
 .../NewScreen/CreateScreenModal.svelte        |   8 +++++--
 .../NewScreen/DatasourceModal.svelte          |   6 ++++-
 .../design/_components/NewScreen/grid.png     | Bin 0 -> 24819 bytes
 .../design/_components/NewScreen/index.svelte |  11 +++++++++
 5 files changed, 40 insertions(+), 7 deletions(-)
 create mode 100644 packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/grid.png

diff --git a/packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js b/packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js
index b17bd99e10..59bcd0d5e8 100644
--- a/packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js
+++ b/packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js
@@ -2,14 +2,14 @@ import sanitizeUrl from "./utils/sanitizeUrl"
 import { Screen } from "./utils/Screen"
 import { Component } from "./utils/Component"
 
-export default function (datasources) {
+export default function (datasources, mode = "table") {
   if (!Array.isArray(datasources)) {
     return []
   }
   return datasources.map(datasource => {
     return {
       name: `${datasource.label} - List`,
-      create: () => createScreen(datasource),
+      create: () => createScreen(datasource, mode),
       id: ROW_LIST_TEMPLATE,
       resourceId: datasource.resourceId,
     }
@@ -40,10 +40,24 @@ const generateTableBlock = datasource => {
   return tableBlock
 }
 
-const createScreen = datasource => {
+const generateGridBlock = datasource => {
+  const gridBlock = new Component("@budibase/standard-components/gridblock")
+  gridBlock
+    .customProps({
+      table: datasource,
+    })
+    .instanceName(`${datasource.label} - Grid block`)
+  return gridBlock
+}
+
+const createScreen = (datasource, mode) => {
   return new Screen()
     .route(rowListUrl(datasource))
     .instanceName(`${datasource.label} - List`)
-    .addChild(generateTableBlock(datasource))
+    .addChild(
+      mode === "table"
+        ? generateTableBlock(datasource)
+        : generateGridBlock(datasource)
+    )
     .json()
 }
diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte
index 9a96242b30..92ed3dcfc7 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte
@@ -12,6 +12,7 @@
   import { capitalise } from "helpers"
   import { goto } from "@roxi/routify"
 
+  let mode
   let pendingScreen
 
   // Modal refs
@@ -100,14 +101,15 @@
   }
 
   // Handler for NewScreenModal
-  export const show = mode => {
+  export const show = newMode => {
+    mode = newMode
     selectedTemplates = null
     blankScreenUrl = null
     screenMode = mode
     pendingScreen = null
     screenAccessRole = Roles.BASIC
 
-    if (mode === "table") {
+    if (mode === "table" || mode === "grid") {
       datasourceModal.show()
     } else if (mode === "blank") {
       let templates = getTemplates($tables.list)
@@ -123,6 +125,7 @@
 
   // Handler for DatasourceModal confirmation, move to screen access select
   const confirmScreenDatasources = async ({ templates }) => {
+    console.log(templates)
     selectedTemplates = templates
     screenAccessRoleModal.show()
   }
@@ -177,6 +180,7 @@
 
 <Modal bind:this={datasourceModal} autoFocus={false}>
   <DatasourceModal
+    {mode}
     onConfirm={confirmScreenDatasources}
     initialScreens={!selectedTemplates ? [] : [...selectedTemplates]}
   />
diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte
index a866cd23d4..731c60a406 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte
@@ -7,6 +7,7 @@
   import rowListScreen from "builderStore/store/screenTemplates/rowListScreen"
   import DatasourceTemplateRow from "./DatasourceTemplateRow.svelte"
 
+  export let mode
   export let onCancel
   export let onConfirm
   export let initialScreens = []
@@ -24,7 +25,10 @@
         screen => screen.resourceId !== resourceId
       )
     } else {
-      selectedScreens = [...selectedScreens, rowListScreen([datasource])[0]]
+      selectedScreens = [
+        ...selectedScreens,
+        rowListScreen([datasource], mode)[0],
+      ]
     }
   }
 
diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/grid.png b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/grid.png
new file mode 100644
index 0000000000000000000000000000000000000000..c3efa30a67d733434fade938263e4aba9fd7cdee
GIT binary patch
literal 24819
zcmeF2^K&Loxc0NLZQHhujjfGsdxK|VZfx7e6Wbfxwr#!nzID!jaekPZs+sDUuIZ`n
z>;7C<hbt*aA;99of`EV^NdFX90Rj23_<cY^gMMGplq4hlKK*b}krD-|p2R-^0U-vF
z78g<T%)IE7uW^@k#=zE2F-Jr@76M^S?OVp?a#d7P{3T!s0|Nst3ZKfhb9A2z#Xv&x
z00L81L<m%OZ!IsqmBP_^N9clVnS_CynjFVKp))?Z%Y5oAksF-tyjtP&@Ee%SYIQa<
zz5Iv*d@_ATvg7`h++(}Wn&Qlzv1Ipinn7Q;WXH`3Dc+VWn0aPrhv35fJ5-WK)AD^+
z5JA58uVlfEl_jkGcRNKfRmXxEOCQ|y0KD&q%uWAa)BpKQfWroaXRltNw-SKHLeMgs
zljfXola=~1iF5@mt1WrDhQyEp9^5S^%_z03n**>L3KO$}rD9phyHMe_&zC(uBi@p;
zpX62_{m(=0o8h$7HfmC975vH_%!#LESVN^s5=$b5ter^`V8h(6r2d>}Ae-oOjVn3d
zXqxs})V6x>pi*NsicRA!yYA<(=RWI*`X&iWj$6Qojz=vUQxPZquRyPVKPB}N!|T8{
zI2NQOldnj>+Nf^B8VV3R7VfSbQFMVGQ76uOJD!34`l*=t6K(VsRd!3iQG3mtHwwzb
zO0DIL2%&VNn;1YBIMxjkc98qJ|GG57ay~15c|GQ;L>uKQgN(>{Ax4)h&YS`xdD05a
z4u+m-X?abL|8bdHie;XLvv}mm0d{<Gvash66}QB5HiHdO=$%Vj_nlH<T|+3+YJ{hN
z=O=TI+YeW@HwlM3WPh5<m!Uc2TO$4ub~ZT{ty?7kgJSf>KX(m0wR_(C^3<6{il9Y<
z7K(}dbyWQKN5vB=N34;o-hZ-_=CkH<M|s`Uke?2kwH`}ev8TwNwK+ur8$0Mlj_k=}
z_b#48$8EyJmby{EvZ2%4kTw`7-tlgeIH(+v@`!1)Us15-t3>QQlNUJu+$@D1Yu)?~
zIVGY1Rx`LrIhV{8SHXuc3~j~<Q*a^9pr4Dp0nnk#yZWOSjCVlIidYz#22PbDvV!As
z&w4Zpv=%qfVFcDFkms7@LQIgT&H~z?9lK3JV(ly6gcMyklks*)S!quDpvI_OT%i|(
zL<JCB`dMU|{7>|zev^vT7rd^3XvnGjUB<73UN9>xU$)k7KsF)M=<CQb9=|K5rM^_E
zxM_5XbQf0(Lt_VjB@(w)#4luWiZ`~XP8y3RiIZkwg<P#{@H0}w5a%eO)35u;X$>?%
zd7<6w3dTY{oFvDZ+)#cIWOa3?91sZUsE9dd%U~qXt4ArmGq%~pV4o(+_l`sGU0hZp
zPQzvb$1(iaE7fVKq4t=%uzg0g#PAVO=H?gGLK?7Yxs|k6VL%AJGjLlk>0&`zip;h?
zOOyXm0e#enG4{#6V1}g{mYP9nt00N~3-fNr^mB^^vV8m4QL8MoUqH)aaJ)m&K7u&!
z;GA1lX{~KTF^46cHwO|I28+-yL@Oae3TRm;UuM7Kcbp`^y#3*PrMibL-%CSy8E{`d
zQslx`9x*%Aq<os6<!jq=?a667a(-8yE1#y|LXg1NksBP1D4F~i*SDF7BMT~zerAP(
zDW!D5>D$Y^ngllLtTK0$NptMWyqz7=dOkc~N0%@Rt$2<)gS~`^emv*7-cTL>=M^3b
zoOWGrwNKr2U+imsQZkYKt1HM|96R+RTV!dzU1TYE1ckb6{=Lo^x{XDKP@lSB$(M}P
zwbWYTfbWPJ@+hvgnlid3woo!^%kcmWnq9U1rjd9H!++Q4x;-9caOx%@e_<1myngf}
z_(HHcQ&YqbcEA7%+eRa}4%)>2%X!3Gi-m5-|MkYR$T((-WRe6I=NANCVLnW=Fa*J=
z)`Ho=LG88R<4CW#IieZ8O5L3vsm)5h;3UBT&Oorka9$Mb@U=&Y>O>j_RdTK5Mv-Di
zk5*lf=}hW+)bZEfxY6~Yo2HNv`!Iq#Aqx_@m|PM42kU%`z-c+WtmopL;5u==pX05$
zx02z-hM-3QuJ3$efpR~wUE~1Oj57Ix;1IhedSP5A^7pUH&{05y<oGrxfwlZN+*i}m
zd;;!-fgKTHcZ`hpT^h()XDfEpmZ_ue!1YAR_OnOo;=^b{fF}wkkG1k(&(2)=n9>uQ
z`mob)1QD&fC2XrZ1MKmB=5zRQ!DfNSj`qc|RS*ORQ%hLEzT0-YoL~0`Wh2C$9&Sl=
zRc8B1x^+9{%VT-D!s%j?Z780pg)#PJV{ruDDA?Vv`K^1~6jZe87j*=)5sWfS2VD^A
z>bk)NuXJt32u%|vx_lIzsR90lT1j{N(Jf}c&^B%k0Nr1h)-;l7PHZawEZL<fw4OKC
zVTCAsg3AklC#HNa;XDNWaS6dV+Y`4pmcNGOF*h|X6tDk-zF~^BfjSyCdFmQvp7VuI
z-$N-y5zt%h$*x_U+qL054$_-GF<4{EUAZkv!-mdPyyYZv*RWv>`Ygz!+-^>Z-8R)T
zvdU4eX!PbJmgEPLNEH&BnnCL?v}((1DqY9{GmDh9qVyH=m~2isp<w;!wXoBuC!cxr
zepenWuci@?3TiI1)yz=&%SFl9SG}nm!R1I1fxBOUH36G{jU%)$c?VqsNb)t-twtiu
ze-3vKH0Q?XS$boEkfmUBL68boDrp!+wPNu*F4!<-KyeU#XqmhetXHBpDNZ+w@m&vC
zAc)ZyhpC@xTsU?NaZ2le5;np;Lo~t)imJKy8>FJ5QZ~fwJ@sr;8uB?H$TIFuFZ81G
zV!J+2JhXHnl6(JU@Bf7w3S~l^`AZRQGDU!EVn2;EGGbw@ZnqP=xG6vPj%?SIqItcj
zxt%Zf7V4P@yuxQN?J55`cN@c<3*>6n++>*pvH`YD7IjSyy+3g)3s1z*)cJ8hx4DSK
z75)|+uh|k-Vds_P2*q+h(uHRw;QS;KCiBJb;Qd(02C>E2x$J2-;;n+-Wo^y}pdIOH
zSi9O7U4m}R>rlp5`Jj9Vpw>k;C#y_<#<sL|TKl#A`261AdM+VH`lv_f!DP>V`@|x>
za%jWt^e1CPK+qz=*uGkqv)|>@AY})4y5T6fu6C+JIiDugqzu~$-uS-ogsLmN{dil6
zlcg3)^durb8F)DH86hxDA&u!Bt1WNO0$7D>UUF!YTIBbn8Q2J|>XXNZduLyRIOohS
zs>t=%9=Q+RNavCKJtzk?^Bm-Bo3S$AohcgaBPMDRtnXmN&C$7A*(-K92dEjO0roTJ
zGQLJHs5t(%*>PcqFfjah=gn>Ci(+M;<f?(Y#=>nwvkby4y`G8u4MksyaLhwObB`?X
zy2%*rYjc;**q^WNx%9Xgk?!>acy#u<H0XruzScmd_thdY({MnZ!Xnpq!PJKW=K-e$
zs!}Sc0&iMmOf<~zELI5i3K&0^YPJInbmHA^4G{f>;Paa`fOGPDr8MHbJEgV8J?pYf
zaNL4mPE3d6dv`(*h~lf-o6o|#o+)(Db*c}}_?uG)sNnj?`g3MDA8s{HQy3knY*fXa
zo5l0>+XZNo>8U3_{$$l%<Zy-B`&*~Ab(npQ5l@Bbv@!fJA`OZ7<G|jZ>`D@uFSmMo
zB(HiTs}K0@Ho<XqKIe3w>l9hM{kJp1*eol|jjXit^P+gU#W0}K6r1)Ri|h}5PNVl_
zCyPw~C^i#2R=Gy(>TYG=>~xl#Mh|7_SaK8lMfjfx#=+s&j5p*M*-uZ*mhfxVcm$J`
zpSiG%hMcJ{v1woR?gF|;Z3l=@X4+$!s5{vsqs1pJ{A;u!^I(LX6pciCZSuWm$s3I0
zRhM{Bmav<Sx5Lvoz`>rtlkKa<*?1~z1$kK8t_p|1KsWtf4+-JWN#Q~eQT1fRaR|CO
z(rQM50-QFdCp>i3*Uf=(5!o8EP6&B?h;4VnN_E+k^^RMxVQgSF`Vq&G5%!zcg-{@Q
z?@%!<s{I>o%*tDur7mwkL>b!AWVLeX_7L2$Z!g~QSi2y!4bR}oKsk~aYk~HmAk;(J
zWotu|RegRQ>h_3NDQ8lSo^>u5Ot7;5VZP^OUC|aywU&u?n9Glt^I__|$H(Q`aa1Sq
z2HF9n=46B#%jzLR+X7i5L52Z?CAnLXcoT2zu|k#(ILbu;_2dFpnKe5$sIN=gSg8I5
zM5|aM?BjH6LQ3PIxSBqe6JEdVMuuZqhfKY*0Gr)isTha8e?n<$v6tXuz6UEY^4;V0
zpY~(ym(5#;8^2m_I2^K?IlZI2Ax=D|8aSO^wgDNh5gwmDv|nOz0}*b?;#%!XvJ0sd
zFg?D*)~8=L&&C)$Hf1pNtKB#dUSW!e4@#(RCI#x(5W7EWKc;nIt(y79VKdc~jxVEA
zlZWaB!NsPc;grVHASjNm-9UKEUuiC!Pf_cB)l1*oOW`8GprA+lt~Pm^olZ0SlQ+>m
zWFvaZWjknrQ*}P6r&2A{Zbwz!#aHg5nlT;5UnJc3(8P#YA0PIePF)Z^&dwYN6wh(k
z(`JY8hF?lqgec6;=ZUcTl>3=xJkJ0ky0ttR)cY32^2>*c{iX_4Ty~y{*j)81Fs8wt
z4yOpwiF-=o{HZ>|iIfOwfWnx|2C6HIL!0MM7K32_13Ss`jO7TNAk27rXaivT4>~T!
zq~<lF9HP;Lys^ZJr1zb%QGqxgDSz#nXH5Au42Sg9?{of#EW5+jnrPvzw)hHjfvL`a
zJHJJ90~cQ6=n4I=D&Oq>_6N1?ZqC+%-`nO?KA;1nJrK=5jaX2{6Bxe&Ri|IBt!^bi
zonO%t=0v#d-v%>Q8TI<RQ*1md`Pbzmn27>fXc3qOcXp7CnTz&f_;(@mX_o(R_CnR0
zAAT4F-k=|X`kKQ?DlE3^fRhDKyC^KP*ovGPIR2&$npfgTwkiE95^dh&Pr8XO0}0|*
zrhD|H5B-`M5_3L=u_8@lb!m&br#bGLD3^}RoK7U@4>03_MW@fh|3wTxzN>q*!;uLF
z6~|*AGtgiO%NZu`O0e-pbQpJ)psjtTr><%G%s%i58*@h@+T0&q=o$cjWG!Z!G9?rZ
zCU|BPsk`oUw$Jda32LxMi=Ra-RLR1{7gbgPjUgnkr3vPdAr&K5sW2H4M>g13h9|jz
z@ar*eJjIf|IO#<MxuaO?YZ4rPL95}Oq4&0|{{#K{Pdy&u1spJSBJsDv4IO!AiJB>Z
z+czZ7HVskS$5)u-Lf>TO;;yqx0_KHwpAty6LuLM`E`#G-M%e9I_z9w(-{3CM^LjIi
zCO*SnfyIri=IH7a{jAZ9x14+X612MJI%wc=zHuhD-`U1jNyNLo@)F+MW7__hFK?r(
z!5S4@_vs7G!NF71$^-XAR(fZ>Z;2si2!6F2CJtKqEO=!)qtBn=^lZvdT`SNvd~3O*
zyc2e^Q-NnYnozhL_`{F0@{;W)tF!Lv@`h?*vf}u<ju3xV&T2=3do(3{JS<;x+)K0g
zcz<%c%Q0v!l5tQF6Q+A0g3vVhQ51VF$tHCf_hcyl#QYWkijXKv(s0}^gR<4=UP&(z
zAhG}P;g>&0Jx#j|O0f@?C&1(6<P>K!sjL$Lv!Wkn)K>4}0WK}T!0MY-`Gz%=`-%HF
zzfruL%8whF<At}*P8w5rwv{k0VQVLJX){ZO6RN9T`5KBZ?wNU}`otJa02i)IY!cdJ
z9rNHloa&o;(<?E)vd|{d4bJ7#Vm;JF?#${j>dA@Nd4cE$TNUubnlCR&8P4X$LW11t
zsJkt_W%8=e&^|8*nG;K~C7M{R#^Ys(9vbIIf^B)T6MEP9iM<OBkj<2U9!*}mQL5V)
zy41p(egj<uc6L;zND_~eU!3daF~!&2eM!BE<%aa|IhVhrAnRf`8=YAYShvuaKg2Z$
zw;}nmg8iNiqG3hpNmIWAbPChxMQeZ>GUNKqTkF9+gZ_XeXNiy{!2=`+vnFS*FMpuK
z1z2YCh-Kfqi~HuzbqDBqq(=OH4}VDT!e$Prt!5;>*Gc?Drx{o8YD%zX57y@S9i%-N
z+jvMzErUp5v5x{j<H3f?o6HsCw?fwOsj5SK$AR5K<#wx`Q-a;9T%cYH_2>w*zOO@t
zUn*43a)hhkdU^@U)u8n`L2*rr+hNMCR;WhTOFdJF=kJso|Mr<fvlx{d^Ncm_*t}Q4
z;$u03FPuKk(;cd1UF*GQE$aR$cXyfG5K_lt%x;h=i&xNsc@l*p^t&_H2x-A-4_Uq!
zi#<DNs}DC&o}@$}aa1NURX3$i4_OUgAe!@wu}x&iNqQQFTh6p8&ck_xp?c>${Ng<(
znnq!wvxC^=&1JBAWm{^OjwGe1<s<e_KmK}si;N46Wsx|L0El>z+bsHrs|rLZ%-*E>
zm5P!0j!FyGU^I+uYSd-CLl_sR$E7fh-mwEQ!M9~rL;|!r$W3(*^|FvhbGd5`8@U&P
zTq94thgOqJk%|68e6BWcS?Ujv-PLOFUIaT~4{(994WfR}$OCC<K(WTKU6#n0U+Tuj
z)>PT}OT2a&HIIvS5!QPgku4Uxqgyz?gAl3n3ZxQaLULmby!A+|W3c)<hrw+4EBW_Z
zr3kn*M~nKK3@XaEF%CoD&yLKT@I2xoKC+-l>-8vwofTmRR>Y2UKX#G768#DG?Rxoa
zXwxi!<0e;IhgMc<t#IUhb{ZCCm^Q{J3;VshsIKH$W2>x>U#OYMA;9Wd!HDOu6mfnf
zfWDKdpr8cTc=4Q^%U2YGts|SvFKLN>e9!`uwU%m3l-fOrywL)wkT2{}c%PSQDfK5O
z5DwFRZShRM4lO&)V;@}2r54^#u`M$sxP{t(#X1Hfdc}sx5M?pd>Bk8LhIHS^(jm$V
z>0%SghKz|&<X~Uq6Nl+3w)T;L3^gx1q{k0d^DFt51yL^S?fyy#+lg$d$%y2hBTEea
zI5u>nAnJhq5CDfb9?49{q*~@#@{T$*6QpKd2Dy7<A|R!!zdfVR3P+B@dZ@#b<8192
zQ?e*A0l9*@H6@UeeJb-*k<W)QKa{tt`oap#*S+WYyK>W8+!NK^H&_{+0CNlOi5w|6
zwZvh>)t5QMZ{Bm|0|-SIZ-?hXJSMH+>p?O5YCMIr`tlMVNjUZw|BV(uPvFZ*3xC5z
zN^hbzcmLfn2<`~SH*>n6DqyA{Y|?jKGKPT`fA)$+CjN5XxL@xwLz=~pJfs|V!nT2g
z4w;=Ls}(bGthk+*8%NoebOf7xkrK;c<mfTAdlu?w883e;=^5sW89J$%o{}S~dOc(<
z+vv;mA{`GR#g}&S(f#NMyI(8%J%o{BBH%`!>u$M2*^wESYI(vUZ}20f!o^-DhR0um
z`iVW%IUaW7uxW9R-Gf=QlmiYVB7`x<m<er5#GP;YdNMkhV@s?2$9|nObz+Q<>`{&p
zU@(#zIXf5EIKx7(3S_|<@Ei)$UE6<^*=6qM5Vo>J1L%?vm|>VJIzNk5``dE(j0Fvh
ze<a9`p*C&6Tt!cG+u2KZJL5R>0I8#HYTlH&m;5*UaZvxr>Fs+)+|!Eczj8KmcsZ&f
z=|0?LE6MKDEX&Ok4vK$Tt!<e`Fc={)kVCB{vodxetc#M@Hi&&_D3Il=Ux!f6HM(KE
z9l7CR;EcJ0J;C_j8EsqhKHhNva=h}!91db$Cy}XXTTEA}6$n@JX97{WPq$h56Mu3l
zMOFw-xmA7=qYCI<_2CSalVb2epCs|$UX`mAk=?ldNF#sf+3u1UAZJG8f5;ZfdxCR~
z+NR}i{Ni9LqiKJ*8v90DJEwYbDy{k`68TF?+LtMZ4?HalmYX2!g`CVu7S?I63T9}f
zlQ{v=VB?Ht=d&Lhz1~F8UJhHP42K=}N3mAFBqCfN#ECSDp%|=;L;97iZxQ5-FscvJ
zwBSg>ZS-OiNE%3w8-VYVbo4N~M7t{{m^kmz3Jp+hXVaD**CgoyJnJdon4!O)O5+t;
z=Fujhh~|%1%VBcMVc8eHUOm3xi{Hzi0bkuY;s3rqyDv};Xr?vgg8?57pW^DDn_o9V
z9ZJz1hS~qRUjd)u;-9@+fI4w&G2iyDr|x$&lozTm>cr=T?8C1Yz{e8yNan2|VjBVL
zk|T=fovl(MSwN3VseK<ub&qf<QYu!Q-FO^@DY)Ha!(L6$e&_y?<9PS2KjB+`*1Y_U
z(LByKd=psM;w=qvi>CZnNOp+|Lk)D}BK-y7QW_6aoKu()$(KhC<J3AeHL&Lp-;ceG
z_rW&*h$6mpM&oDio%?z+Mzwc_7v}|Iy)>|E(6)Yx3G!2esO}jHzA6C0-Nt2tn!ezL
zd)dd#-ZTCTnjm1!Au?-9x#Pj%xeC{y+uw%UlQGQ)@BT63Q%{l9$6+UO`?5Ij&-=fE
z6PpaJ`o`wXFR3uv#;+lQUl88sUO2$o+yIOeCu7kt80`!_WNTIqh;W<UJu{Ve<m{V+
zh%h|%u6X6bM%87b^D*4gydd)Ukl32w)aE+|CC#h8*L&2?bURf#83(0C*iBD%;wPSe
z4_BK*L76t1<AwHK$X@gormixRu@h~)Q{5ZxAG~pXk-Q%~++L3q=RUMC4o5!-lXoAB
zjNX_=xZpY-P`IeSIi9T>?tCdIh13^VTu;20rwFJj5}4l%M*Zs*o+86GzVu{RO~FLY
zzXkePKaMDY7Kim$WPP~d3LBC_iff)<5C$#eJ-kFBf2*^A*iw(qX*&>Wq$~wSZVn<<
zIYl&!(D$xVuNuEM&w|MXb7J<N&^M@EBzzFo5tZ%xnuWW{|BW<HcgMZraeVg#+v`m5
zEg)OjH7#7dy#k@WMm;?2w5e$V?kMdm-?smdsIx)!{LR}v;G^okgXtXmrFLQp_UuM9
zo#{tgGkqKG=2nH8cCL2<@|xWqW>o-}?%WMyK;cp8uck<zAY_!;3w66abv(OiaRay-
ztoxFxu?uSeNmW?KK#_qC#IU!sT05xpDx0E2vJA_yqK?V4_&2~<W<@M2{cDtoile!*
zz^|;VPG%?a9<6=7tM4{Y1zfL*!h(B6MQvexayQp2l8g7<M*wXYyVmyZ!d*^`4C20N
z|BTGR?wpt~iY*vISVgJxw+DaK4|&4gw@z^l*<NS?p{TNRj1YZZLq%tzYiHtUV7&G0
zlsW?XUL#Z=sQ@AwXOEJhmK*}`&tDD#lUL06izF+yO&VddYd1mkP%&E2$`;Ra;dw|P
zVfUoBoXpl`x21^RyJnV#JiSUTQjQ&(xt=+AADWxDUy`I$<)eKYx8L2ogHW2c;*cE=
z$5(dJmpBw=(D`@pp&KXU5!LbjAPUxmh3X1HQ56&2k^|}23vU=oBQ!gZ4eZUz9OJw6
z`!*Oli5HK5dgX-R`$_v;^Gq{z21BjPc|cpE_(Z#l%T4>Cc4srDgm9?gnO!F)dSlhD
z9(%pH<6y4eL!c}?xxjR_KoHSiR4gn<M&rU%moh;KWb~RQ{-z5x$$iUGvXI-O`Sy5n
z<;3S}(|j&oTHu_BK!MfKbUAMPL%O*cXZ~PzukI!8-NZFK0%KS;a_C)|HI!nWVr;3g
zK0Kd}y$sf-L=0Ug>)ev6%wmiACpBXha~2C15bk(m>_Kw8kBAX9SDFP+uP(D0`gt7|
ztY4glDLUQs`Zp1B1695839KcDyk>jw^1YZkiS>>fjH?&-5m$qGUppwpoOGT-lUz<q
zi&Chct<Ste1G)z1i$X$fW-zyrYi=q+E0(95eX-*7C51i&Y+r*@iXUHA!m72wceKme
z<k#0BFR<XcJ%5plan`){u*nVfEEO_I48W(%-Z&(z65p6G-2@%D8Mh@2K%4f*)VD|H
zazCNk;27>W2jtO_qL8f*9VhyREn$JR3Obn*m`tiOhVPgM_Q|-S0sXy=$4>KeT@)Sl
zb{8-{{!tG+a8n-)iZSZa^-Nk`V4ni=&W}ag7-=%wBXu&-ety3$ruc@0zOcRy8tk51
zdxSni0UBzb{Q!99WUC)F#HJp84VP8sdu<aQvXJzsA9%XxYpONFp+vOfAInV53s+S`
zYf&iMS|?L8C02XhjE@>@&hywV7u;b@0tHpAY`%MtHF?ex`khX0mxO_<*d(mFhC;1}
z453)=%>CmgN@kW|XQ>O(jUVTrPm`;kL|?le;`>j4S4nzHt?$9VCi6L`suzl&RA4dy
zi14&DGmDC(5P89FX3cG?#{GmRC&LRH3iF1-#V%YfQWOLK$-|2vQMhdoRfBDjoIffl
zAoms>EGoCRL?bO_YxWue0q6G1%o+F&C#L5!Wu9|s$8foNZQDXYMIe4bFqjf9Pm#ib
zLpF~`7+*K1KK?KAdy<-d3qZ)%)HGV?i^HNAe%8T05;ZrV*iiVqHF%BA^)edDisPs&
z{7FYj3b-G>D!<hPobU-UjIBe}LU##s^hbrg_kR10`|W^>4Eir>Lv3Yp&<DBFe~Nxr
z^5T9Enq6fsBQmY`OulZE(r*QTw_;MjnS&o&0#$JRlw4>&T_yC^S8oPf>z2__J?UK7
zJ{lgb#tZKEB&j%l$iA)EHqO_ah>$QOrA6};(hvlIFe65u!{8Phzc#ev+c7YDA8S*_
zCd}N*rM6hJ8*`gs4Nx{n!$s`FPU+jC+vl)u#mtT~LJj!eEI>-P{k?-H>RjN}?B%V^
znak(H?=*>0AL#K`SHuE<37YoIFKD5_@8V#?&isnxua&zEmY1)ijQdCwh|V9Lt=3o^
zq6BX+&~a~kMbcB%SF(g-5|rOk{(xYpAQ6e;ihXy_k#X}hG}8VX`WHgNRb4o7R5MCh
z?b5CyXH%_tO~>PlHOywl?RmJV`>WT2w;)wlvTxTP#HJwRNo$D!O9TO0dVnRqUEg1F
zXO|Dx@DFn~hwnBrlycVl60(*V+CI@vZz=MW3Qxp-(Ns=*vB{bseWItIhZ0QOMKz!t
zmQnl|zp*6V52AU@b!0{vrKG#KV~r4~`V;%It?56(qgr|qu@2!g*ovNnO8`F$VzA{%
zN0pk<f6h;<o9Gk|YouU1gnr_z(kRuyCW@CC6|LS8W0r+iR<t;XKX(p=V&gLFp@%RB
zXG=Ee!}g+ECTa<P+K(8Y;wbAF411WbC0<Pd{%~`iOAXciyb@*!+m(AbHJJ3~3FKU@
zerF7eZ5lF9XIGMtUH4$c;W`iQ3IcRW__P0<9f9_D^z^5{!#;4fq(qs;XRiWaRdH5o
z9sVAmhYxI`zpR@)<||hYjPsNreql)Tb@xqiJj|u;Kt8+e8t$OY08GvBk%AbddhD<;
zpF;-&O28H>pbGbt*0o6X^1;wTFHKuWFl>-AUn+k1!39m^jpSe#QSws?oi{dv|Du|s
zzr-1yNb~kQc0m&B{F`^XLzt%I?{{Z}@prhHF<^DCQJz7&{r`i4kz%A?)Zdl6qZo1L
zNX%vV<+O`B8Z<v~O6@IaVEli_>D<!|MHUJz3FVpEk<FM_Wpl#_2$|0tP?@G^oXkh<
zLJlMU^<f*|(maNSEP>~+!UW}%F=-<s{T<|V$2U}pD;%#=Ao*RFsN^=LPzt!S7QJar
zbq$rgWFVUVLNlI-VA?|geV`bxByFz8+tK19$tJ<gAVq|r>7@K|I4j?)?a3-Ut2L0&
zxFsp=ZVMiM5kI$=MX87=Q8MzFCUJDZTb!aB<S)|CIj*h!v_^&N@y+U%-~0$m`chq0
zM=yEP@ADU;o3TJ$U7~986o&{~G^@FXsS<q67dQuBI2ZIgDfGNsRH?5$8UN}zpOTl!
zO@E{sd;0h8?$c0DZ6vxh91usZE9Mhho_-b8EVM#LaB;K;S$)}vx*QqXujkWvl8rQ-
zjfCx1xb@w_=m7e~c(b^MVJofgwof6)s&x3bxRPqs`F+ax5Ik5T`AdGXvYG`iz<TVb
zXv`LLs`BolrAq|e*c3|(CU!N<oXud}`<?~lqU5{lmkdI}pmj~=fy|>JrLF4_3w{@w
zc+~$UUNEvr#O`00TyulOM(@#}JEoPakjCdM2}2Vk3Tqd<6(9Xq_E%`vBRB4F!jC(N
ztYwZc%KV*-(J|sgI(pbb?~)oXrl;H|SDY0$P5IX8QSekq44?XQ#yfKYF(??tRMRF#
zEdKk?sp+hymY^O43v2(y@)2zWFWdl~i~Df29PrS-S8tMbqDFF5c8cjHUs{<$*4E4?
zKFY3<S+gt)Y(wGpEL;uE!nI%wIPVAxAZrttqspcH9W8mS7!X{OPug+bNf!MdKem&J
z;vTqsFJAHB1R{_sm(Q^RLeUyjU{3zS@l3op@Vtgp604S}q9v}@e;jS}ngHUCD7i0y
zlCgJ8Hs8!>7$0j0Gy$0JumQG7Fk)HkSU~<pIyWm&Z$hB<w6mUb>+r6n7>5Te9bVR@
zAJA(hNL#dog*+#$@Mkf5uqFnQ1Q7hYP#nW-q5cXm;X0dL1J*wbCm7j7vnmrISE*-=
ztc%hoCZ*h%n0=A>553iEtCItZDZLUC<L#wVq+puxxTf~RL9t%uu8lD!qh5Py_4I^2
zR{xdr-QgmB6RG^vhj(m~gqR0m^nG0f_+y2Mj9ktaGW;d*9IN#*EjZL=D&jg~HF@y9
z&$g7ekD`ITS>X)BbMD5l3;~k19D;5_o&GwnK|dyXUXBYRz-QQ!icVclWo>z_T{Ha|
zsH9H0bBfZ>vr3U^bf4aE?K!Asxsjklx3-6z^q+Mc)?wk5D4n)zp(Cj#Fs2+wJB!!l
z@wMbX*s>`1%?XU|tg~nl{peE?;%yKiU`Zi6M41#9Bnev$j}n#`BVzS(THsRsrod4z
zDY)l;r28w*p!*mlV3BZhd_vtVIW>;#JrVej#|gKPs_$`@{rj1r9QdeOj&`EJQP`-4
zAHQ!3H*oGxg_^2c!*`u%ZK~_vUWM+eeMBtkFwPK<26~_vP}oV+=mYVh{VDI-CbRsN
zm#5*@<x%pyU5qo)eRS;bEk$S6rdRx=`)wo#yGd1Jh|Tg!XJg$VZejNHA_=9l2QQv$
zg^_*mH1e{}rW1l3Ne(LHf>tr+(J1sJmch~$8?tGVf@p^9IkhX1eC0lhn%Y?|#s5^v
z?+PwXKZ@_C{r&y-8<KietPllU1DMp!(D!G_dd6hc>_(9fu-8TbZdH6=H7Fayb~&(K
zSUauJ;%|3em@6YYqEa-@4cDG*Wuj47<Jv>r7)X9FqB2}0_#Fe8W=-kij#%z$DzC3j
zW?MmskJ?=L->d$1=<Hu^j;f^iv50IA!z??W?L|W=*gOk|xcq!l!28vfM`DjgxnCOJ
z<@!CaZj(dIHR>kx>ykk-`euQUrgb}^or$YDvik|j!T5?rqouFcktdW4V9}fPwU^UT
zg^VE80bD5km1gc4qWKdH=?O~cw@z}4Mc0tStJ#<fmN*d3F1TjTe{mvw%KabG%u~k3
z@Q~lA0xr#W*P`xwJ}@PNCNX(9an?rS99@94l`c4j$<dAk1WWH3_bhrSgiadl%{Hl9
z105o512uEqU7?*ATZd-vF?rYsP{o#5Bd0=+_wYKHU`qo>O(V^_25CN>(F|WIICv?h
z8w;K2n2c#qYVeJG(ba2du08}aDaRf2gP{xKZ{*+gCo_Vp2VQXZ8!REtLE(A)WpPzo
z@UVV}yU)dW+Kk@`)7y>nll7VEqmTRonmf?(Bo6?1sqt$F)Iz+^99Y;rhy8uLW$<#z
zW$JrXxjryH4<!29w+nk;t!dk0tF})%H1Gm3XAsJN-<ER7zr_PYzMH$mNoP*uV=$1C
zc0YVHr@A}cSD!j{`rC09&69ap3ULpyqvZVO?o|MtC#P*nHP>dtw0eYWR3YTvStmK-
zher{hCp}MN0e>0C>gAmccV{$56f9}t54%ZkKk&9;sopc}dj5d-#?njYx@_>ESM~u8
zA8xp|2y5AGclJ96M-h#&>LVe^l0Q2cTzqAoXYnn2mcKY*i1KNkL7f+4^MF_9g8a?A
z&w6G>)8DZU|5F|Q80ejUwJ6Mkc1&$i7I2>@3QWdo#_@3F82Ozds05y|x1W1k8JpQY
zzl{Y%HcXUT7TSZA($Zqr_x1w+(KlX9np|5pMRqgCd&X{~IZCpRXI4xps{qs-LOfCB
zdqIY|HTaCc^d{)vM3{GbgV2Vzo8}e|45MH-0v)OQAJdkDf@2d!$>@e|cW5-?2Dx00
z=s(3XmQajcHGa^m(3mJ<PxE(KEH}76ERGgL<~WGl@$$fnCC51s4unV8Yq3Kp>N3c=
zeZb#p`J4asEO8Vtt%re)hyo;?<c^6!9l)Bp4fY}z2V7m<b|f(Cvl^-LZJu7{JE-LM
zzMpmKtIzz#8Wzf?dL}ayZdrRxHt2tO+_4owT6KBmfhWmEYjBRV!-LrX1|9wvS-^wm
zY8AyIXRoI4ui>nGD^GI^39q!tRYRjOZCU|7dfLB<8F9tA1G|Vsk@eN(l1J-unOh_K
zB_dwN+ufTYR$qD)W68$*JW$@x%WMT4-@PBX*mK%&?+gmst^A)|JR?qt-FyveX^bfG
zZ&V7zRcrmXUX}bREylPEPpR3IhY&N8O^?CRjRcKj`T%a=*-6c0u7<dF%dgUG6}!|t
zq%<F@++I}`Exg02ClHi@vo7)6G+7`%z3PzKcic3(Pu=Q>F)!IMM(#v{$iSlu66VEt
zR)X((2Hn=*&U%?MrPB~LrCN2J6tnS8e#=o7KY4v4n}mLvk}PYF#z>`xky3=;Ck&${
zbUkEm+UrEidLn4W$Ea++KJy=(G9`cO6aC$;#i<E<sK->|$%q7(a;bNKu>-gp7~Tvn
zM0x;<H8fcXNwd|shv#`{|L}`KUC#wMbCU=`QLxtesg_kg1#5n$VRGNf)*(P{3Cbxk
z32%&U7d)AT+vRzFTdYYJt=J-EM(#FYc7ceAs5%7O8o`Z%?wlAi%nJU8gpY(B!aIC=
z07JbbWT<yO)O_ojeP)vgEzIu$SDr%W;ajf9MUiQNh%gCJ{^rKhm93!zz6I=InoUni
z|A#2j1fFl>#ujqB=oNkNpHWuth0L!y)v8?csDz}J9EfMXnpkctIIp%LdhUFrwNJf{
zA%gS(tYSx**V)6-oj}fhRvequ5zfVyl6>oC+Gxsb)wtvYXrX$uwl;Jjr9njeJDJNH
z*IQn+lFJkA92QZDTw~6j>-e*X+_``|d_u~^*QWtrIV!D7o=bOjrh!IH0!cu$Dqh_z
zqGbJlrnGTWiEvrPLb2@_I~nJQ%?=`_sa#4V0d{<jy@?9I9mBK25pCnAUNIH!oB_g~
z21?GE!4t71cxiv?s7W5p2hVGYwh0cppVC{a?QJ5}Yx8qwe%zo=|IjwiF)H%oOl0ct
z_KW7E)cm)Q4k%E|h0pr22@#b#$D#*+N$m!U(1tRRTKmr>x+x{SsT!-mPNX?4uIicD
z&(j+C%Ddm>a4IQl4*IpY-u}R4M4Rd;mHkNBr$<Q@csy#2Cvn*I#{en!*An<P#uE)k
zU5A}5Vyl3UYgHi9OwLJ~mQLmTX=xO?2C-^%T!qzbMV?fBK``N%LKY`HirE<1Lp%)1
zh7y%-kZ5$;*ArRRZFcG@%Qn4xa;@V`bsJ-Oqx;?_uNP8lds}*_b*=-e_{w6bqxgdZ
z(}+ND@iq*^WE*JM7A6yqFYdy?p+SlJ(WUIBONib2rX1!zr?Vs++n86hdN02hi&Bya
zwq$=W*-~>Tk8ETM-QfKEsPEauS`V=4IgZW`C+#4?>~j@Wmw+rHr1AdNHT0N6VD^db
zy|Y<@Kk%Dv1^qQMCz`qqJK5YdA!o!2)kHdQIICdjF>3-n-*@fl<_mMTO~z}ufwwJ@
zVx~yx!RZnGx?JQ}A!6AVM(C&3B!J|oervs(aXi1dUHnWH!#CUWNPx>wBrViz5Vk;y
z1QHH(weJiw76{Ep*-vyU=BZx?6%Sv^EcEq-z+woEv0dT*7vnhy5xRBvGD@J?<lU}C
zvd~5T$3X4~i+%hP#lz)&?Y$qfQVD9*V_u80<GAGbFu1c*JLng&Vp6TPM=D%lu3Wiy
zQ(PlAJkGGk>VH96k)vv4l(?clt~UPM8S1}rar4uXPY*rh1i^zITOS--=oRy)T>wrc
z?iHuQX`bG!=xn}JE&7aoidM!X^zVjFl?t-WVK@z_xSbC|=sHm%t)f1*CJlWAN6|(O
zQWpiGHZ!V0lg)v{Gt1w(qivf5Y(bpf6XVHlnL^l;pve%N@*?S*Ox-H8U+ey+4t6#<
zl|ma8!|VKwv8yoqvj3EcsKnYJE*Q;d*`}b2Z}rH*30yo8WSL9DqC~Pus^G5&Z7}B~
zQl&O6X2IX#S5baIUrv^*XEil&&!}ouIj(o+6CBvN-Cj_J2Ie+GGk7wO`J8v3VCNPS
z`k)M|87fGgo&?OkPV~p`=N+?Qc5NJG)Ls8g!1C25DYj5lD^coB3teURv@L3tUZLzh
z+!14k0L$G-<xF>!M~9<V79PT}vE~V*lUM&dj@i<K7MO5XYMnB!gHDE3|J*KdxVwH$
zc#!OdF?HXTYgOG-->2$+GS6RWEbe3Z(zUPncN?nT#H+qbb{OL)CsUXtc1g-o^C;x=
z1?6y$W5(C7a4Y9IjWlD`y1Y++fhb+RL}LJGb&B|Q3KzEHLrj0ag<+ll(xn}DH*JE6
z^p&kUGUcbU1@#u~@`Q@HEYY^BB^khG^L7>xhndYE!`7f7I6-<MwxuTPeGSXdv`vy2
zW6^%?LCW6X>2x+RekhlOYuGpW#7yCjkE)o=T93Kd`YY3@$ZkJ_Zfhv6G!v}^sQRoy
zMpdl@p6v%-iff|3z1`W&zH0h?voG5H-d39S7+p@m+iyX2^?A#+U(S$dnm;cYk0j~&
zEwMh%whB4h1Frlq(S)QXD-(%6pTAMq$5J$4s-?7%cW2-Ua7{@HI0C#<>PhoGi?kYJ
z*vDu!Ja#{=301H?-SQxYI<lH}pY;0@BR4pO=|^W^XQ)D6djHvo#_!2P(r?L<xLkLw
z>8M(_*W3w**83QKr2gw9h7T}s>`u(m6E_f_=w=4qKsPu2eVL~7z6b3EvVQBj!;~b{
zHT}IUj3cu5F*JoU&d_{D!PvDBJCnso07=k0YU%i6&XOb9x47zamiqTPLFnjPZZ}cw
z#e*R0f;m2?KltKj+cFP#vPYh>3yP(9rV!DTD17r2+-4K>ARXQGgCq2pSjXUAzOl+U
z0vIM46uKVZBI+`6xgF*5p2<Ui;VdGK2Kx~6bY7#&#0A4&h_4j}iEkZ`$`TgQP}D&f
zIr`;tTH0{VqSBfgnqV&eQ;ue{EW^@JDJWigd%RpHqFY_C4T=tR=HV*^WC6o^;7}2!
z;&wab-8(VdF`L82oWeJySYWnctRZ+lO?MA<sE)vBIBtX0gPYdOL8CzQn{%=r<)J}D
zcCt%j-#*~%o<Oe?T--$Jd14lS6q3zxGzM8T)5AWtg9v)o_Sk7CD?c!?5XBNN+d`pn
z_qS_gb24<hJ8re1l+rD>eoD6%1@0EBg#M%sct+md%-rVS=||3Q9;>Pfotmcg6oc?X
zT;ZY6Bg=S?)SPQ(HU?8?vnlwXdV`S||M!rL-J2O2edLYxF0M87oYwRdFC~wnUxgWE
ze@EoCqyZ3mk3zz;kQmmYw!M;mr!eyD7H<ATe(QMa`v4)x;iPuhLA5s;myIZ~XqxfN
z{T{Dk!m-Fg5}HP~71LifI577txzop8#B=HoRyCMJdT2ANj#OL(`$YtjRs-q74$U_@
zpq+cu^sqSr{ElBD^+5mwztD@a{j$J(LZFOUtqzN$C55dzId}cT_6)mqBN^!*nQ&x?
zClhsVUoBPh##WK$I2EwH!&c2#5Ut~2!U6?GNFieoMsq^x9pr5N)OO;CA59AS$=-Ja
zW9_8+n*j65e}vM^NROwj->zFjJmJ}p9|q%Zf09C{0FB;1(qNLr<BvgWBaTK}?eZ*i
z=l!`^@`*oPMC7;ZRefP0M$@xPcis3H9yQ)A3E^JvstTSj-Nin#(=-<@s@Mfzq7_@=
zsw5wh!U;|qDTvADc=}fdOBnm)^c|a5N2<c6W@wjlwE};s0x2Q_A%pSHcZXtIOFGr?
z6|QFqJu7^*&iN&hVLF>joLg9q&f_b@JZ^yNr8AQ(k;Uw(wEynTky_fBFOGMpYII3E
zUn`0C&DS*vRVqo@zdc1T$u<*q#4mA&B;TRAW;8N7@K-sC`fpY38$#2Eofyj?l4shg
z8`+tN(MjeR(fMf(y7Gx>!K+_ntMZO7Fap3m7=RGD`%;NiYah}IS*eQSbq(NUzO4>@
zFR8Keu*Cl+W}q*9GLXEZqLdqP3q?0+S95^~a;8Y#bX_25wH?G*niBR$OV;`7PLVe5
zng$#^Be#$4P;g^G3o5C_Q4P-AFNI7DPtzVgZxZg$E(Z4z*_?SCOFgRA425ny3t(b_
z`2;9`7S2Khvi$9TdDJH)hER7N8nYsJ^RU}GSIapRl!a%^;$!4ZE32aD{oG#r#M$6x
ziP=S&KlP@pnOp2SiK#B!SOEE2B+&37_1i<m2PANt9@}^!#8zW19|fdgB2_5o(xmo=
z^BpHQX3faM`3#%U)4q7ZZqxz#HU_%#@4dlPFS{71ItStxB9bl8ga%wxUUBSC_f6ov
zRb8`Q_K+&XURB)+Z}!J58BK-Z!ry4z^m_Igt@jWw!deU|6|YALY0FOi96TA=FG>6+
z7*yHAvdFPG_|!hMB@6l?LO%PwRZpn&u-^S5tp0Vr8MkyxllW~L<Nh_z(aUK5Liwz^
ziBspkDiZj4?6Kz$jL{y%jDC;gF;u)sj*{hW#uWUnT!KS1@Y>iPzRJ^!aW%skM{{rW
zyQ>V|<3(dD#bL78->JPc+;?b4r+oE+^!}7Y#l~_w)$`}}o)977Ka+vZBI})0w<w66
z@}3R(?a^9|yAKFP2rB2S=?Zw`KS%N9i?2ryE%nc~OQzA`wa&|?O|RAi7@@XIl~M57
z$Dbc(E|cwthGnLvh%I5$mQybC+MQL3R6i-}C=D816Cl&Q45_=|NH;7snb;N$O1B~T
zLw_G9z1%x&4oJsaXognlWCl$CP^m=dicn?k5}nP&gIUG0WpQVV_;oIDytSdTyKjFE
zPeg*lG;M;pu%;?S_0Il4H8bgyqO)JW=TL{XcKG5>)&;u|`*E?%Mx@aeC=takWcW1?
z+}A9yN8K&K>p+q@M5ir#ZGk;viy60aSwkQM=EliJ0rV{5l6w^M)oUe|A)Mj|J~?(O
z66W-?czwi11R$m-ai)$G^)X=<tSR{0%3}D)97eb*q@(Jo(CsDlTAgT`E>K+0^SJN~
zPZ^9;zu;tEM!sT^d^x^OcXbmeN#x6m-If%Y$MCmervLu=##29eWf7NafEmXX-K~o-
zFGYNQmM?0#6MGqYrD{V%+@z-2*;W;?u@-o!mZsJlJNz$Ej>28zLG50U>cxD>*$%zi
z9QIl8+KsO}>9}L+%rHR!y<9SR{=PQWxQ)4Cf1i<$JG*^LSaWFoB^2Bx>}!|%KxMql
z%}F`90Ijdj9iq<2u<~dYM>gT)))Uk?VV3Chpp+k^-QNl)y~vFaPc2SF<ZbeVH3;_o
z=BjIkdyUzRYhW1iKEH77J3Mj#CFIjyRvItHMkSnV3jb1d)g6v6kxg<4Ir5iE0;@m|
znBXtx^2dk;P1WS?hC^SNj}+7#Pu;e<ktZ-mvsPlYYljzGkEd*_5Y!`RXY03bI3i5T
z_%pX7ILi4S3S7AgWMi<wq(~iNSG=9JMwL54i(rMQ=9<k&0-_u|a{jN$HvyDxE(MU+
znHknTzr_j~iO(#%aSE>C;?(xo1<Ndfd;eM9;Shk5!{1<yUhrnNVJ~#m^zIdnm;Nng
z52)^U&91$%``dWDK|U4Kd+7gU&^KvP|CKDQA)cjM^7*mQy#tuiio{zKAS-@ZI#UyR
z-pL4iAF2T)q<P5p+Wg_a7kz)c>IfhCdT;K|)0AEvuTKL~zoNWnY`vu|+?^}QYh+8J
zQR$4=-r)X%&^azzt!`LT7!i#SSHPNk&v}M@4*zPdr$qSKW@j%M)vBf*hdj>Leo`C3
ze$>tWvh(Bv_bTXxNfg%;%tE!Fe6T%u3oETb4`myVb2ycaU3o`fOf;B%z@9Cl>WH}7
zrHz$$SCc@f<#V?2Z(kPaC2h~n1qxDmShd$rG57aF^O~OJ=1Y4-^~7Ao-sP;edQbCo
z$BdS}WI-ko0gD;rW~cZahau;MfWd$~Z01}GC3q4~O1?)%85Uc0rfrp9(d)|<()Y$D
zSfaCV?><OZlqmwmYAN&@N~ov7>lLZ38~@FT!<d2Vy07AuArWd_hUc4Ky<)Jn?#l)|
zaCC*W7z}$`uxz0<BIq0*HQfB~#CM<ynbwADsjF4J)gkP~XgSm_ydgUQB74{uaGOeN
z412suym5x8c!*`{V#=?y(dCBpm(UXIXQ~u1NSm1bRx}4sI*?PS5oO4wzo?nUUQA!A
zaq%Y6*T=SOn8x{h29h|jR7?6pOn1}AzvXj?T1HL3p{_tch(yIU3<s)krQ)Jo#FcsN
z^5=1J81IGeFfbraYH1^K7Mh$b6>xg6y{C6F8IqBIB`AnY`{SqCfLKm#c|Xl!rr~(x
z;fUnQ+J1BQPe2Aghjbq^<+ZRj3+zM>!NqGh67LO7LEJ?cx47yxL{n9)JF{AE^(dpL
z6X9kP1gW`t<)8$NExf=AL|mL9wor=_M8r6IJ=*7FqS}`+(i9JSusvi6aDm9!;LE)K
z_yR;*yuiTIvpj4fRXQ`Hv5(djNX=V@+>Yrn;iqd_dhw%o<`2d)?KA#G*s>O!kPR!v
z<F=Ff0OvG_95&5^50a|&0fV%N1I@0peHB@d4aD14aelM$+0mG68Sn7x$+~^rb1B?%
znF1TCX3^b#Y`@mtVf^WeW58VR_B9K{sJ#W{<ce+Xjj-x<$fUcNP>SCmwD0myu;MJ1
z{yPLY=UhNIbk<1Yg?whh1C8v>Q}O;`To-|?F%27Dg*}ov8idCX7Exi++<y1~<MG?>
zV!JXI;`e@nRw~(Gd-nclw7(WJ(a>PD>_E8Xx^bsHx@Qj_ifBsvxZ)L_oRLM?rQ#hA
z6A2BCfpBofTSc4O^o;ef<$OzvO<q6SAAVk!x}M3GbGtdVY1QLiHLWm!f2T5ekS)lu
z%^7h=*GbOo<}<E7-`m*3r@#IpFUpds{>`;GLt4=piEoM0#IKG^(24C=&+3Rl-%yX(
zYT`$aKPYN_AiFmV1~lw@+!^rd8uf`uUE8#9(c|P%IOs+2P*r+xu`pXTQ`q;z&wp)}
znM3WZMoE6Z0j@nuzJ<vO<`jMy$wEle?@TQ7OJB=8b+-gzg*UZ1+Nt@9QdqJWE%^ms
zFMylMm(Bi<ApPkyMDKc`$LQ5{UgH_{4axdv^zCBzaB$fhDU_~a4o~gHp~2YdVPgRd
z;D56KV>uK3-yxIDFkff*Ic7R9%elq&X=c#3=z-B{Wjdqn>6D>YAk&}6G|!Cw4*_a|
zS#urY>c-%ZEWMAw&etY<`>dZ-JnK?)p+|C{6+*n@vy!=iPv{+~twyG6eNF1u72md?
za3}cdCwKi{Yt+SuExo`3qBCtzK+Hig&o5>B^SGCU0(-Y$ifv!=4N!OpZ!p%AiCLM;
zkDtxLl?^J#McJ+!oLKW;x19B2TP#mZ6bqgzjXAyKFL*I4_&?u68Z6P^J=^d1O$1z*
z?NR+;zg@Hz`_80Lxt+zAn!)C$r$9YqPBA8OWse|D*<B!*J+6-}3DpJp_43&`6jDla
zy_T2EeSCuT6eR+(>y;oQ%<I_C?fpgj`pz*UZ`-48Q*M_SNOSLg$%lSy=G%!|tG#cd
zj<BuR(3m4}0s4+t`b@N*hkkA&2(TB!BZ(qzNgGG%$}J7NyF-+VY@tXK{VEsye*r-s
zzTkW>*#ME|U+rPI<=<n9xE)i^p)lH=NA}?{C^n=O4LNk}EQUy{dU-`!ukjipi$}FH
ze|3AwFB*DZFvn65#+8c8#4io<wU3xaHc2R##a_+gaSlx`sbyhqDHKsN=<RM}O$LlQ
z?I-R+J=DKE9B=CRTQD0Ek^u_038P#<$3nCU57PFJNeccx*<jufkygPTwk7InxRVVR
zSB&8s-V0kG=CMBhbqjIy$IR88XR_#4hSYLQO-+#97g#?9|Bn1>ZJSmW-puZm9v&x$
zXkP=jn`#U07S;e_fC_Nu{D~&pwpi|o=tk9yakjH!3TpLf8uw@l-q<n$#NG90?9(Af
z)5&v2e*6|u{6(TDnT;1XTRsa)r|1!>SfN`%@ahcBCa{@6yRPpW)EU6V_-pt$2^9W^
znwl&BGA!i2pc73BR_}h(<YOB<{2YdJ=DPAti0cAA*dIqU2_K><H~^VzDu^xmY_q;%
zGPaGM=E;7MeyQLrw_w_`ubTdoBq6j=!Q}D^tn<Ik+~6YntqpJWd%YC_28P!eJm=rr
zT@bZxwN9-0z0kk@%Gl>97dAU%f>`Ieuiz0jiqxl}dvi|X1nmpHaJ5di;e2et@(9U3
zm7@7C!o`E9ZBr4EpZ?*ShCs2ZeAr8iGTuW!!A_Ei`rrNi{R_&m|L^_@KWV-@ru&gy
zQ@d-nv1{$||BrsWe_dGB2tR)Z>=AhL)I2+y@OOUrb~0Zn9^!3?dph6yt)FcRcO<Ym
zD_YDQMTfFKr7SNp)W^JzZH4a*HK7h|cGt6xkGpY#reF)&dQP_BB)!y8^X)FQjLKto
zb+##R@xsc=Pf_a?A(lUen}kCour1Ba@1+C#DD`AIiV_^P-JK`9P|~bUBAgGamE(aa
z%GDC$*keXJcqnESE+$tIctOU|?-dHCn^i-Fn^%^weKzktkEY-f(4<zY#m}Bax^$iV
zS^lNb9{BDrf8#rYJFy)L7}CeF-G}7((7%7<ulwIK&NOMXq3Ddt`s<H={3a^D{%hZa
z`=Duy++$7f<iGR7Hxc>Uzwu2L;n||=Wi%1_-fzE&$p7+pNT<8^Kaj_n7PKD0|Nomm
z+YdU*=Nx9=1%zCao8&=jJ+wC%+7~>Lh)tb=dPk8N>~Ekwa^P8H8tv)`x|iUTws6b8
zBX&012}uqE{-u&IO9=UIgBj*1RYa*!Z3Nb>*>CwbhMX+2y=a@60X)}wd^IHxWoHbJ
zXYYxC#}?-zn=v&6img&MxDH6${@)kS6g=TxaB%H7%*En?Bj~m0f4Ksnc|JCuAraGc
zn5C)5QZca$FQbPE<0O0kmfDQQIjl~0<$ota_qt=2d6qU^ttz#7v9^Y&KFE7Oo2M&j
zPv-xGVm;$O1v1@#)wKU1zSQ@n!6BDT+8_&B0zZqU;0vl;AZ&7jR#kuYA%?`C<8(rp
z{az9L3x*M48_i>KzUmbaL22|mZR-+bMa>VP#nHwT?Ju77=PYfUh4ADbV2vX;!t?oi
zMq{<FNfA!E8e`}wG_B=l@WmDlX^`&II6+hJg=~FU^^ySCuA<M^u_yL<+W*~K7{wBk
z>tLjLGc{@Y!>v979v>mh5j4Ruztl8W?dHl7LeVsN@ge3IGZuQn-34}1I+;4*?wcUE
z6z$JW<D+yt((Fc*<^OdVq6<RevSBmv8JdDG;EZfx2zd+cidak@;oAj)TQ%+iW&~}m
zD)dJnYwv=9^VH<2_BE7vj*1`rix{Ek+0|o0sr|oq2E--P!a|3`-_gh@-i+w8T}4Cq
zHVpIz9MUwu|8P-xD%1XTABqHmj)&vv*3wvL3g&pwg@(aDbK2g9WT0x=ufokqVoeDA
zYV3qrN<}qnGN@!Oh|ny~7(y<BT)%FOIdhJD2zLX(*F&*F>-j72>B}x)|3s3i6>$CU
zsvXQvm5k6j?Nl4h=@oM$yFK*c+!1<H8o+g@$5I$u1S@`y^KjaW@oCQ6BKG1#np+;2
z_UdAzl9mZ&izcNTFjg$ZY&>W0P)cNX8usJl%XR31aK1Mw>hyet08`X{)y?Y3u<TmP
zkkF_ygM~azv$1kzDEraxYr|#JxJOg4$5?a_dPv*|!C#|b>>R(s6Enxx`DZATE$fm!
ztC_#(5CzFr^d0$y=AhxarTmh(b9}gq7XtR<{$C`iqV$OEH#(vC5I!s~oARSEtFeVa
zc<EmsQ*4r8fr_^NeG2V}@X(Y)Z{Dpc!k_a0o?2<+^V#Jci}4xg)TOoqvN#}tpTM~O
zJ+YBv%JZ2Mi}E!^SnWf&cre}@4B<DAyR6fJaa8=7UkxA+6peu#ZSbR#VOhf1JoZyq
zk*CKZF5NX!GzCu_XgaEb<JE}zE0k&SITB<P8N=eqi(TO?nI6B2c3%XLj_MsrO%F1w
ziG!jGqdu^GZ5eZ2kM@s?pKdZq?#ky+^5#i=9Nlz`R8PkXh4R+8Vgi||jdL^w8?5$J
zeCKRkjomeKLAW^C@|~IYEtV%H`ZJ#&$jlT||9ZWp?r()#54HC~6s#EnIZ55g=8yP)
z<zOySj}4|W86(yn<eJV1$<nd5EFzxjd?Loh9Ae_?B7N@r<3G*F&!8!oBMM&U^Npv8
zr$X1i58QWhfjKtFmMT6ysi<)plSfR=oW^8E!4|Y&hr2IsR0*~^y2_O^4t@D*9mgz4
zCfn5RH@cPlzK^mwkQ-6;%F{&WK3tQ};$^oGM~2BJ+>fCt*x<30QM}lO;Va1)CIoUL
zDT^I!TL%Hni9G3Qozu+Qu0i7Hor@MCjNGn4rC_`-o3XGt1htr!n@vqfHf&hEDY}fX
z_{%*$Z((9y!L4{DtL;J>uRMo6O(!8?81vrt=>iIAn~7gUQ?N%=xG{nYUm+*`KP`71
zMV&}L@Jh2dA?ufApjAqgE^Ot5>r-{7Y;qz(dDchCU4%nd{w@5*gM*@X`1<#y0pKPP
zw56?BgG|Oub{mFEI|N0sE=E?7QTw<X8JdDGP~Sp~lZ8Yt1l@%eWtRzoMyxh)2<dG;
zV2~a~5+p)~sHf-Q)*UnRD0ge^MnFcO$)Eqh`AiN!k;D9N;Q2EsHg)(cYu7ukR$o%&
z`ZZi;`xsP;CD(m)@)lwvqA8f8ej_8sq{u!>Y2v`YkA`mgWUegkIW%H!S1NiFmRn?g
z;Lr|~5I_48lVz5j_3d&~3K{>_vwQ6$65_TKd_cxlmdDusFfH0VhH{AY<6rz#zZ<#r
z;gXtpH~GGbel_F5$<M*jkfmg8avwW$I}Y(}U2UMc-`${FfbAJyI=|b9>h-J3!b)ta
zL&tdBxSI|XXFF|$kq#S&$h`R{bLU%f$pqqg4h4bkZDgz%&aKfrwWq-_*Ty8Q&WZcX
zKWhZfI@TKxv<gn~nxpVRzUYFW{)c_MoD5Fj%n&y=jBxNc!A?Pp1^rvdXH-mPU^vsE
zs@^VF$WzOTuIR#je9w}H8?lqLwMIVS!76R3e>eYzgd=fJar5RedA@n+c!Yb3*R}EZ
zA-dU6rJ476GzDLXZo$&e>m_i=t(u^jH;zO&alwuYmxS<a{T(01;O1)P9bRb$E0m*R
zYpR_OJ8c&0BfHQ(vV1D*Rv$T#{V-+G2bFn67(VuT7DxHhkM#t5*Mb|)x2|DZ;dI%v
z$Ml)MhTIwlXh#HrBi~}3!HxVUKSys?+f>x9v1F1;Ret}cZ!U*{9V4W59DOBplX@jF
z*Xd7w_l@KklY~1pewb(v<lqwi;74z}w)Y+Sm`2(8qQ#-`Ac@Ft|IXV<WE$cgv%5iJ
zwu*@S=J$REX|I1q2-#UJ$~${}Vt=!BB^l9jJjqGZi*dE#m=jcN;}A{38&S4oKV}h+
zXZ&%y1ZN06@~?XR>ny~W>w4_`v0GwK+nr}X$Siutr35idCU!?h@B|mbgOAF9baT7k
z<4FRolY94Qi~vEDeCx6<I2xPP8y^s6NUqvu^LUyHPfBbO@<8olB~NG9KiqD3Yjh8v
zjWe_(!XWy88PAf)CwNf&J8PYfupy?)Wk1ZZ`_eMWEzQTQ?F3WTAIuXt&dpdA-Hw(8
zE+gmg1uk%J0)p<lFclutfcP40y!0NMC&~8-cYl(j*-(%#0PI><(5JGiS4|Nn6GNsL
zZ-mHJFJu0)DA!iFulP8eUn4YliF+g3htVq7;AB-dqh|MWoDtYwvDcZVA0$tjVeBDJ
zB4PftI|Wgxi{L^zuKb7G1vx*~{SUZ1W69X${$T4k=ZG@XIa_wP{hE6`5L#&`$~1Yz
z>Z%L$Z}8SWhpjrD4h=SwCmVn=cI45VL(JzdHe^1zVi#I-@oQ)bzHr!p4u8iX+v4UW
zk8AV?{Og%IS*t;AaI;ls3?5x(?+baKbOZ_#7@ZMFxMsYmJ8+P1Y6pf$j`v@7PX^9c
za|_RnjRCQUNO{Dg8jM+pkB^Q|>qOaw^#q+RCT53|#}<gHa&k6rP_!e0BkM?N-@;n!
zZ}qo;7u@{abVv=nwo=W=o_}RBri2JEGxZEUEW_&N@yLUI-5kq>8`fERq!`R{l^l0^
z?bG-Mhy4aPAKOX>d<y)>EErFj(*yUEe>#x!x=hAAa1o$kak63v7v>zGVfPTY9X`=u
zXNeh-HQr<E-;bavm_YBo&c6K#A)X@_$H<PteB$I6*L`H9ZwN7QY%em>R%cH_s&gOH
za-KomA?r5Rk=IJmJl*yvjAVF&(FMFM)O{M)e?&+KeS*RS?`<5HoAZ)eb3IGOJLS5U
z0kbmeZ#ddnMt1496MynJJhc7v)rsXe&RlNg8L@bG-`#2B0PTq2Nc{ljn6$FmzbRrA
z;*4}=l21manWhT>3rA<AS@@1KeOQH~0`Mwe3{f4<HkX^TEll&{#NWBIef{0hX!JQy
z-$LP<h|%8hF2D<HjLft01JcUb%}DU5e9#w?v{?Tt(|rJ}tt+jP5kcALvTPoMge?D+
zy8Ih=XccTwyAk50_#vKb9vWqfzs!ore25~jMZwXl%fThj-#v8%%`)32P1R%XX%dRz
zU#qWC-1!4GkeHNghH*P7&ZsudS7<bz9o9cS6uTe8HwO2-g6{?6c<+<HPD+opjd8{#
zeTDFy|2*2`?qpS&--5`hXnVufYwpqRMuJws8??=2PR95aM)4AdAt{BBB=a2|=ZPAU
z193VHOT$-4VWHv#HsM6o>xQqlvE%LF!qGCAks$X!h`r5lQzVCFSTvPgI&@!VPjk_k
z6X}bj$S<)v%i)g5I5$IM-R|Vk89Y~CC01R$qnUp+03WJr;aH#5qUD*hr6Enh3s3QH
zM6rewi}8#+%u2T}nkHA@$_+z7DI+YShy36K7QQUGx&*p1jXV%@-ukQLL31|QWZ$ea
zBvE8p<_@>c5TApF=debq_wxP8qJHLMinlpL-hFx8Ha9$z7MDH}W5)Vx$8yCj5XM>L
zZaFEvaf^0DTv#jh3x`69M)vbio&T9gvm3nU?q;h-O=xKz_B(zfQv?l60Hh%FTgG5H
zr0`~AuipQa{OfHiB_ti~sH@?oAjr)<0{0Y_+d{F%@~5yQ@RIqOGnCq+%fYgqj&AqD
z#h77;U6y1%+<i@L4t99M#<2MJy2Knl+b@j!SU&dUjbAjh3N~;}C))*l^daC(blAW(
zpUu*u%uy>;Oj0XqqBA$BIGee>s?Mb=vvfsstn_xd<uyc^Jyq*?<1_i;OIhpL73`no
zHvCorU3)(qW5x`3CNjSPGT)LAsI>YH-1EvDO6U(6dA)6iPxNKELTAi1Bcempb@09u
z4j6KJDefGtf-lhiK>622pn<sUH*-fM>}WoUHiK9yZ0Ks^pw^+KFY~r^nW>a_pvC@X
z6j<Ep0Jkrp2I7k4*vgC;OU!r)l#Yv?627SDizw~c^{*F2x+gMx)G$|oO7V?hU5u5#
zatkAC$kh}bveC5*xOx}gxr}^ijT<xtdtgVU`uFy8?BY7`6e$V0<!P;iQ9I?_QJCs9
z?gh6?ABz~F6UGp0nW4*h;)Nl;-o`3tMfSEA?2luctC_%|x1Qo#z#U!Lc>=yA--WUM
z5j-NJi(U^)V@z9VWHhjWYG)N!ZU?JmbAR1zgKQJ`RX}*w*{wASGzA-+PjV|jms_33
zofeZjSuy94@hUhilbC8^GCnmN9eS@U#*cQABgoL1akpt~tbeUqyp9X{ek448See6@
zB)=Ue5BwV|rc8&%x+xrwxuf46o6*i0I3w&a8%Aq&p0qevx~H#?#<u;xv2_`T#zd6B
zjdL^wZy^6B9tL&sUCgVw+wGQA=w*aeOdh>B&HKM&YL}bQN6rrQjIX=b(Hhz0f*Beb
z38d+2<SGI-16+Sf?xw&Bj1J$^<l$2^ig%>*E|_g`24m{N-lGb)*5vGtm;|PNphG@1
zW5X<A$WTT1H8T=iD{GKYFz=%}<`)J{!5hUUQ|f=XLDVd=#a(1uvK2pZ3U89KW5B!G
z$e8)OVAwCpq8)5<MlvEf3zCXUVW6=5=<if*e+~)hE8`i@o;<6YcRFA|Hgc6wx%(NE
zePTIO6|XRj>Eh!BH$=tcy2rmY2CJn>_pFVzSK4bA#2BW0agTsHpm7nn`S%#TBVr>F
zP)Dta^e$+ouFKISpb<AAM9~^WR$jW!UV$Ck>!Znivlz@2>;y7eMK~>ndEnxBhDar5
z(Gi|S_Tl^yR{RV9z^&7OyiBoNzzV87caQ8UTx@7}GK_u2R5<rwgvMa<Hxb$6!~HWe
zxlQA47Rt$93RjoL=F_zM5?Tdc$oOK65x>QgX;?VltYm48q~-JhbYOed{1c}9M$zC~
zQfiH{Ju@1NrlMls_kyIUhM^u;-s|rixd8)i^HB`ePvPx#(daJV%moo&bXL46XK7Bp
zE;k&^HleGjF+Sjhx-~5)4$rK;sn0TGv>yFnF`Xsh2F>VVmlF=$JPnRk!5*Ox8&!1s
zV>V8?4A-?vrJxd+(dLU;Y5ZUcUJYt%+h)X=NgV9Y1drFM%eJxnA!yjT_gS>A5kkc-
z06pW#M5CZiZW+9m=SpKtqKU2#jUEH2jrb(jk=hJ>QweDd1G1gMXc#Xv4778S^k^4@
zq{Y)!$=SWI(i=5e1slBJLv$gidQdjr_7>jnRu*CFk8{jLeQJ0?d>mXkF{~O)@M(v&
zr1Yqo(_t)Ah;v9K8H#8(Y#XQw`PV;!5PxYy)PtxBRrbxQgxkcedzL8O(~N2T^9@Al
zKJ<<!Cax}n2Yn9NCT@NU{a)PjXbQeCf~dIl0@usgPH2u0&B^E8O_$)3$#G$0uPcd#
zv5_E2PK;MmAm&EKim~H)rtm+m0h#EP7T0iY6>H|kiKok7oF%6|;zt@2hE9f%l_qZ`
zXiVUe!yiSLj!Q`15Sjn1$hehw1;6hnV#X5OK~dIrT0dg!d}-3s*l0(@1$}3Mp#i!{
zuDK+E#LXX{k6-`Ey$U{my0(s<3YOs-1C(6cndq`ctzDpAJ?X@acW|4o1aDi;0+f!^
zZ_$Oo+bzRpH4k$cV=pS^#^B={GG9c-JEG*I;b5KB2G5~xkBvg=FuKL!3L!P%J(NA-
z9kH=|=hsa95?TeHq84aQ>X{?9OteC1Cw2U5nCtO(*nqMB9YT60p3!Bx#|7#Fje2c!
zGvrJ=pc$o{{+fhxC1)IGEMcN7qvIE6#CAn5dqBht*&5S^@`RiQr>G=$%?QefeUY-G
zSL3lA!Mq(;>7wP%4ctv+t#Y26?7VJ7GzELO4*}1>7!Ce~rb#+UK6h5=T`Nu6k|EA2
zSHo3Z77Tke_Cf2yHZynlIpmexof>a4Gh64I4rsuRZNIHWZWnHr*8h40w+`P(%Iank
z@HqF7Gl#U4=wo0ue69+FFQ)INqPsS>1^AkSEyA<}m&(T8rK^p$ldG^9gCFo#`-$bw
zJ82dC8rl&-aLB(|i0O4DK@8iby_SiLsXl_C|1~3~tmn3=xzS>oF@K`RO<+w{<H}8g
z!7@YEueup_Aa@nCWBKW_80Yq@?R2x6Nu{n19<;=2JK%(NFn3O2F18!PZ41#DA<0VV
z__3WV7ysTgZ#H&Hkl2OCrct9+Fu~dgeB1pr@YKG>dOn%J+9}=@f=h6V%HOXXhW(O@
zP7Sq<iH3r~t1#D*{l1PB6Zo2wJHutwv@OOD1h~zJAubxeM_%)9jDs<+3ZcJuvR2!M
zO=3B8M@Fi}%zad6M|H(`nO1ag;KX((R-0o3bI<hgBAf(=j|nN;(tGlrXX7cf3MN=8
zICGE<+}SFomAl0&2fqYIHKv$`zGTZo(Rh-Lp9q*{RsAAx=#?#WCS&1>zg!$MB5_Tw
zbLvtFMW6rp5Ix1@w@Yx$1r&#yDfH~uqE};h$6rVjKAzQx^t}Nt8a|+kxG9JudSi*&
zv3x(uN8Wtbk6L!;B18P(<N_OqXcc_n<mZrgxRt<O2rL~Gm?ip-vFBh~aczDU;X9c5
zyUj}DD8>$$`>}S>Sl&vz@^qT3)O08hjR(fo(#@wEV$NvH9eopm81Yfu<VJk`_6Dzv
zLt1lpHYPrOb@lI}yD^MWo@+Vcg#x55R^h@{bffOA1iqb%G%~abHmKi)!`6xY(9zT-
zWXjHu0w=au4gH+Yf0MNC#CK^3eBX9^(%zywFQR%CP1zF7oIU8t5)dt#5w|1{D>@i(
z3Zg>p`UxrP650^wgk28YU8Oj;2rAX(zHI}ym98S<P?4GC0<w@q1Q?V*c-S~ctKbV`
z{vI&mW|2Wnuh`AEUcndNnw<h#XHl^l!iarUNd(icLlKDwdm8N2`t(A&JHmUkF~p;M
ztu})4>|tbQVVJ3ep-SzkooJh^%paQuD}2}*Q=7=1>M98BQJ-8Of^+yl>oW<RwaW#3
z@9+u9F>?~X_ocmrrr-;zTCBoF4ZHi8+BmEg`qpWAi1z<YdGrf1poRuSrg65#ZdSY9
zh(^4wKh8J00wo!f;w*dEQvvR{6;OrFJxps}(lwsV>dXAPxGy{uTZ@Ua+YWf^oQ+4>
zNfp&hzlOhxFpm#ReOFymgk6I2hehN<@`H-IM%goJWM~TBz*V@clW!p~|Jxl+q1Rcg
zkGbqN?bbS0%ynh*FYmicm4ww8r?XrLW)@}KSeb`C#SfFzn?IwhUS}&xSH56y{i&k@
z1^-h0l}cdi|D8{rKfC4sH6fF0VjbE{O@Y}}i2?bFoZ@}KE1l7$t#EC(?YZB_)yWu8
z4>{<A^3SyQ2tR|SV1mL&=@ihG7rI-Xa{`*S4r<DDek^YOA>u9`1C>|gyKS^aPUDQ6
z8X{`$%V$U5N(gdkjU|5gDHCRONF&Q)Hyb~j5$lQ(t81P+ZRpd++~`$HbAA(rD}DLL
z;hGr{#)^1ZsP<Wy0dEhZnEzPoXY?M88*OE!_VL9!$)8K(5KX~1!2X=l+-$ydPpf&E
zm{J3Do89y?Ig&7eiA*)D*nYn3i@9RInrXK3WBeQC^R6r0HW%npgwSQypCeg2Dd!kr
zMKB^1V$fQ0+R$HjH8>v|b=qJ;Wr6+%<29RGv{0D8-gwAZ^`^5B7TGKhL|G6FZaRJn
zy(7YaimBMhByt%`Zc~_}7((58W*;%6bqnaG^UvHAj{F@c{vo%v?D2$B^uSnGwxUjm
zmF7Q!^D)!8*M99V5WeFukSJSfo~nzrQ^hBOx^?9?D6xhRe1eLp>?8KoYu&HeT?jOR
z$^~?{0p=d{ZuAC2t6&4Sfa6`j7*otuVN~KjrW0w~vVlUJYxbw4=qp|HU6L_DqDGE=
zk$1dh2@piJ<edr0#pz-F$HgQD2;pBj@cfq6!5uA;tEp{jlNrXtBpKDRn9i^`g032K
z8TcE=@o(&CCTn;RId##vc47u$k#c;>&|3Hqt%5I5M%W)BfPS*2$uJF<t}<onXDp7x
z^|S}&y{{Pll3+iz1AycAMF-eP_wNa*nzV4TL&i2^h181G^IR#QBIP-Ds5TuZ10Huu
zgdp!U2=bXYE|wV~t>V+P)>bt%LUo0CkQoEkSB*yauxj9L+hFWNK4-?f@()t<2DMYP
z3a;=DYaQ3zKbcxOrRMeTj%R1(Ztty)>G_fU-dF6>CS)n2M?qtMg=dMcfhU;nLqLJF
ztU9433wle@nXqT;**g@;zYfJK7-{ud+g;|=^<)kqUpF3O4y(<q`q{Ou!MquWPXvn_
zI6}LJHf;Xa>BFuO?`XTx)#T;|c!kJr$n=7`@Gmz?v<f!(^MB6th<%W?Cn~$FJ2FVa
zG`Gjn2a|8tKhgJ)X2@7WcP?ZVPRY?Wj9PnSTgdQ!Nyi5Ii?*;^A1(};n-ZCo+#b&?
z@Q2>@<AT5BMn`u_Lo$=<wC0}CmYdwDy7!SwH0QpCR>1^I4s1rHlP>@O0B=b|K~!BZ
zwy1QGwxE#fbpQJd|Ki2i!NjfhxMU+0dn5@+dzS|ZrFp91U$l)|5j+gh=MalXy#ayt
z;%6oNSKkUY5AX{B!<;&AL6)JJDgn*0@=U4>wGTI%fm_h3+~zzf4xfjPgg85CE`AFC
y4ey<YoU_cVFaQ7m4rN$LW=%~1DgXcg2mk;800000(o>TF0000<MNUMnLSTZxuKsoa

literal 0
HcmV?d00001

diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte
index b504940ca7..6b080747b0 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte
@@ -3,6 +3,7 @@
   import CreationPage from "components/common/CreationPage.svelte"
   import blankImage from "./blank.png"
   import tableImage from "./table.png"
+  import gridImage from "./grid.png"
   import CreateScreenModal from "./CreateScreenModal.svelte"
   import { store } from "builderStore"
 
@@ -43,6 +44,16 @@
           <Body size="XS">View, edit and delete rows on a table</Body>
         </div>
       </div>
+
+      <div class="card" on:click={() => createScreenModal.show("grid")}>
+        <div class="image">
+          <img alt="" src={gridImage} />
+        </div>
+        <div class="text">
+          <Body size="S">Grid</Body>
+          <Body size="XS">View and manipulate rows on a grid</Body>
+        </div>
+      </div>
     </div>
   </CreationPage>
 </div>

From 8db0ff79a6c6d8b5061b1e55a5aa30a6ab0af07e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 26 Oct 2023 12:44:36 +0200
Subject: [PATCH 057/134] Increase yarn timeouts

---
 packages/server/Dockerfile.v2 | 4 ++--
 packages/worker/Dockerfile.v2 | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/server/Dockerfile.v2 b/packages/server/Dockerfile.v2
index 881c21299e..f737570fcd 100644
--- a/packages/server/Dockerfile.v2
+++ b/packages/server/Dockerfile.v2
@@ -44,7 +44,7 @@ RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
 WORKDIR /string-templates
 COPY packages/string-templates/package.json package.json
 RUN ../scripts/removeWorkspaceDependencies.sh package.json
-RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true --network-timeout 1000000
 COPY packages/string-templates .
 
 
@@ -57,7 +57,7 @@ COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.
 RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
 RUN ./scripts/removeWorkspaceDependencies.sh  package.json
 
-RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true \
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true --network-timeout 1000000 \
     # Remove unneeded data from file system to reduce image size
     && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python jq \
     && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp
diff --git a/packages/worker/Dockerfile.v2 b/packages/worker/Dockerfile.v2
index a8be432827..4706ca155a 100644
--- a/packages/worker/Dockerfile.v2
+++ b/packages/worker/Dockerfile.v2
@@ -19,7 +19,7 @@ RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
 WORKDIR /string-templates
 COPY packages/string-templates/package.json package.json
 RUN ../scripts/removeWorkspaceDependencies.sh package.json
-RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true --network-timeout 1000000
 COPY packages/string-templates .
 
 
@@ -30,7 +30,7 @@ RUN cd ../string-templates && yarn link && cd - && yarn link @budibase/string-te
 
 RUN ../scripts/removeWorkspaceDependencies.sh package.json
 
-RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true --network-timeout 1000000
 # Remove unneeded data from file system to reduce image size
 RUN apk del .gyp \
     && yarn cache clean

From 44f9c64ed72b87c8a62e6b4754f107e1545bf07a Mon Sep 17 00:00:00 2001
From: Gerard Burns <Ghrehh@users.noreply.github.com>
Date: Thu, 26 Oct 2023 13:06:25 +0100
Subject: [PATCH 058/134] Prevent the key user being used in rest queries
 (#12072)

* Add warning about unusable user binding

* linting

* remove unnecessary safe nav operators

* change regex to capture property access of user binding
---
 .../integration/RestQueryViewer.svelte        | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/packages/builder/src/components/integration/RestQueryViewer.svelte b/packages/builder/src/components/integration/RestQueryViewer.svelte
index 254f65fcaf..e6913b0953 100644
--- a/packages/builder/src/components/integration/RestQueryViewer.svelte
+++ b/packages/builder/src/components/integration/RestQueryViewer.svelte
@@ -196,8 +196,36 @@
     }
   }
 
+  const validateQuery = async () => {
+    const forbiddenBindings = /{{\s?user(\.(\w|\$)*\s?|\s?)}}/g
+    const bindingError = new Error(
+      "'user' is a protected binding and cannot be used"
+    )
+
+    if (forbiddenBindings.test(url)) {
+      throw bindingError
+    }
+
+    if (forbiddenBindings.test(query.fields.requestBody ?? "")) {
+      throw bindingError
+    }
+
+    Object.values(requestBindings).forEach(bindingValue => {
+      if (forbiddenBindings.test(bindingValue)) {
+        throw bindingError
+      }
+    })
+
+    Object.values(query.fields.headers).forEach(headerValue => {
+      if (forbiddenBindings.test(headerValue)) {
+        throw bindingError
+      }
+    })
+  }
+
   async function runQuery() {
     try {
+      await validateQuery()
       response = await queries.preview(buildQuery())
       if (response.rows.length === 0) {
         notifications.info("Request did not return any data")

From ed0670a00863e27e6ef4f1306792ec5a9ea7cb42 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 26 Oct 2023 13:19:09 +0100
Subject: [PATCH 059/134] Major update to make the table.type always 'table'
 and then adding a new sourceType which states what source the table came
 from, external or internal. Don't want to keep using a type that should be
 static as two different things.

---
 .../src/builderStore/store/frontend.js        |  6 +-
 .../DataTable/RelationshipDataTable.svelte    |  3 +-
 .../backend/DataTable/TableDataTable.svelte   |  8 +-
 .../backend/DataTable/ViewDataTable.svelte    |  2 -
 .../buttons/grid/GridImportButton.svelte      |  2 +-
 .../DataTable/modals/CreateEditColumn.svelte  | 14 ++--
 .../ExistingTableDataImport.svelte            |  4 +-
 .../modals/CreateTableModal.svelte            |  4 +-
 .../popovers/EditTablePopover.svelte          |  9 ++-
 .../QueryViewerSidePanel/PreviewPanel.svelte  |  2 +-
 .../Tables/CreateExternalTableModal.svelte    |  4 +-
 .../data/datasource/bb_internal/index.svelte  |  4 +-
 .../index.svelte                              |  5 +-
 .../server/src/api/controllers/table/index.ts |  7 +-
 .../src/api/controllers/table/internal.ts     |  6 +-
 .../server/src/api/routes/tests/row.spec.ts   |  9 ++-
 .../server/src/api/routes/tests/table.spec.ts | 23 +++---
 .../src/api/routes/tests/viewV2.spec.ts       |  6 +-
 packages/server/src/constants/index.ts        |  7 +-
 .../db/defaultData/datasource_bb_default.ts   | 21 ++++--
 .../src/integration-test/postgres.spec.ts     | 14 ++--
 .../server/src/integrations/googlesheets.ts   |  5 +-
 .../src/integrations/microsoftSqlServer.ts    |  3 +
 packages/server/src/integrations/mysql.ts     |  3 +
 packages/server/src/integrations/oracle.ts    |  3 +
 packages/server/src/integrations/postgres.ts  |  3 +
 .../integrations/tests/googlesheets.spec.ts   |  4 +-
 .../middleware/tests/trimViewRowInfo.spec.ts  |  2 +
 .../server/src/middleware/trimViewRowInfo.ts  |  1 -
 .../app/rows/search/tests/external.spec.ts    |  4 +-
 .../app/rows/search/tests/internal.spec.ts    |  2 +
 .../sdk/app/rows/search/tests/utils.spec.ts   |  9 ++-
 .../src/sdk/app/tables/external/utils.ts      |  5 +-
 packages/server/src/sdk/app/tables/getters.ts | 55 +++++++++-----
 .../src/sdk/app/tables/tests/tables.spec.ts   |  2 +
 .../sdk/app/tables/tests/validation.spec.ts   |  6 +-
 packages/server/src/sdk/app/tables/utils.ts   |  4 +-
 .../src/sdk/app/views/tests/views.spec.ts     |  2 +
 .../src/tests/utilities/TestConfiguration.ts  | 73 ++++++++++---------
 .../server/src/tests/utilities/structures.ts  |  2 +
 .../tests/inputProcessing.spec.ts             |  5 ++
 .../tests/outputProcessing.spec.ts            |  4 +
 packages/server/src/websockets/builder.ts     |  7 +-
 .../types/src/documents/app/table/table.ts    |  8 +-
 44 files changed, 236 insertions(+), 136 deletions(-)

diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js
index a567caf87f..a4729b4a8a 100644
--- a/packages/builder/src/builderStore/store/frontend.js
+++ b/packages/builder/src/builderStore/store/frontend.js
@@ -580,7 +580,7 @@ export const getFrontendStore = () => {
         let table = validTables.find(table => {
           return (
             table.sourceId !== BUDIBASE_INTERNAL_DB_ID &&
-            table.type === DB_TYPE_INTERNAL
+            table.sourceType === DB_TYPE_INTERNAL
           )
         })
         if (table) {
@@ -591,7 +591,7 @@ export const getFrontendStore = () => {
         table = validTables.find(table => {
           return (
             table.sourceId === BUDIBASE_INTERNAL_DB_ID &&
-            table.type === DB_TYPE_INTERNAL
+            table.sourceType === DB_TYPE_INTERNAL
           )
         })
         if (table) {
@@ -599,7 +599,7 @@ export const getFrontendStore = () => {
         }
 
         // Finally try an external table
-        return validTables.find(table => table.type === DB_TYPE_EXTERNAL)
+        return validTables.find(table => table.sourceType === DB_TYPE_EXTERNAL)
       },
       enrichEmptySettings: (component, opts) => {
         if (!component?._component) {
diff --git a/packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte b/packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte
index 8ef870caca..4e67a92443 100644
--- a/packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte
@@ -16,7 +16,6 @@
   $: linkedTable = $tables.list.find(table => table._id === linkedTableId)
   $: schema = linkedTable?.schema
   $: table = $tables.list.find(table => table._id === tableId)
-  $: type = table?.type
   $: fetchData(tableId, rowId)
   $: {
     let rowLabel = row?.[table?.primaryDisplay]
@@ -41,5 +40,5 @@
 </script>
 
 {#if row && row._id === rowId}
-  <Table {title} {schema} {data} {type} />
+  <Table {title} {schema} {data} />
 {/if}
diff --git a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
index 5fee849afb..c2932d3b10 100644
--- a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
@@ -16,6 +16,7 @@
   import GridRelationshipButton from "components/backend/DataTable/buttons/grid/GridRelationshipButton.svelte"
   import GridEditColumnModal from "components/backend/DataTable/modals/grid/GridEditColumnModal.svelte"
   import GridUsersTableButton from "components/backend/DataTable/modals/grid/GridUsersTableButton.svelte"
+  import { DB_TYPE_EXTERNAL } from "constants/backend"
 
   const userSchemaOverrides = {
     firstName: { displayName: "First name", disabled: true },
@@ -27,7 +28,7 @@
 
   $: id = $tables.selected?._id
   $: isUsersTable = id === TableNames.USERS
-  $: isInternal = $tables.selected?.type !== "external"
+  $: isInternal = $tables.selected?.sourceType !== DB_TYPE_EXTERNAL
   $: gridDatasource = {
     type: "table",
     tableId: id,
@@ -46,10 +47,7 @@
     tables.replaceTable(id, e.detail)
 
     // We need to refresh datasources when an external table changes.
-    // Type "external" may exist - sometimes type is "table" and sometimes it
-    // is "external" - it has different meanings in different endpoints.
-    // If we check both these then we hopefully catch all external tables.
-    if (e.detail?.type === "external" || e.detail?.sql) {
+    if (e.detail?.sourceType === DB_TYPE_EXTERNAL || e.detail?.sql) {
       await datasources.fetch()
     }
   }
diff --git a/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte
index f6160e3caa..bdf62f2959 100644
--- a/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte
@@ -17,7 +17,6 @@
   let hideAutocolumns = true
   let data = []
   let loading = false
-  let type = "internal"
 
   $: name = view.name
   $: calculation = view.calculation
@@ -65,7 +64,6 @@
   tableId={view.tableId}
   {data}
   {loading}
-  {type}
   rowCount={10}
   allowEditing={false}
   bind:hideAutocolumns
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte
index 71d971891c..74e255cf7e 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte
@@ -10,6 +10,6 @@
 <ImportButton
   {disabled}
   tableId={$datasource?.tableId}
-  tableType={$definition?.type}
+  tableType={$definition?.sourceType}
   on:importrows={rows.actions.refreshData}
 />
diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 7b51e6c839..8c9b425708 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -26,6 +26,7 @@
     ALLOWABLE_NUMBER_TYPES,
     SWITCHABLE_TYPES,
     PrettyRelationshipDefinitions,
+    DB_TYPE_EXTERNAL,
   } from "constants/backend"
   import { getAutoColumnInformation, buildAutoColumn } from "builderStore/utils"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
@@ -254,10 +255,11 @@
     !uneditable &&
     editableColumn?.type !== AUTO_TYPE &&
     !editableColumn.autocolumn
-  $: external = table.type === "external"
+  $: externalTable = table.sourceType === DB_TYPE_EXTERNAL
   // in the case of internal tables the sourceId will just be undefined
   $: tableOptions = $tables.list.filter(
-    opt => opt.type === table.type && table.sourceId === opt.sourceId
+    opt =>
+      opt.sourceType === table.sourceType && table.sourceId === opt.sourceId
   )
   $: typeEnabled =
     !originalName ||
@@ -409,7 +411,7 @@
       editableColumn.type === FieldType.BB_REFERENCE &&
       editableColumn.subtype === FieldSubtype.USERS
 
-    if (!external) {
+    if (!externalTable) {
       return [
         FIELDS.STRING,
         FIELDS.BARCODEQR,
@@ -441,7 +443,7 @@
         isUsers ? FIELDS.USERS : FIELDS.USER,
       ]
       // no-sql or a spreadsheet
-      if (!external || table.sql) {
+      if (!externalTable || table.sql) {
         fields = [...fields, FIELDS.LINK, FIELDS.ARRAY]
       }
       return fields
@@ -486,7 +488,7 @@
       })
     }
     const newError = {}
-    if (!external && fieldInfo.name?.startsWith("_")) {
+    if (!externalTable && fieldInfo.name?.startsWith("_")) {
       newError.name = `Column name cannot start with an underscore.`
     } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) {
       newError.name = `Illegal character; must be alpha-numeric.`
@@ -498,7 +500,7 @@
       newError.name = `Column name already in use.`
     }
 
-    if (fieldInfo.type == "auto" && !fieldInfo.subtype) {
+    if (fieldInfo.type === "auto" && !fieldInfo.subtype) {
       newError.subtype = `Auto Column requires a type`
     }
 
diff --git a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte
index 43751ad944..eb1e7bc7ff 100644
--- a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte
@@ -1,6 +1,6 @@
 <script>
   import { Select, Toggle, Multiselect } from "@budibase/bbui"
-  import { FIELDS } from "constants/backend"
+  import { DB_TYPE_INTERNAL, FIELDS } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
 
@@ -169,7 +169,7 @@
       </div>
     {/each}
   </div>
-  {#if tableType === "internal"}
+  {#if tableType === DB_TYPE_INTERNAL}
     <br />
     <Toggle
       bind:value={updateExistingRows}
diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
index d13a7c30db..e5227af409 100644
--- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
@@ -8,6 +8,7 @@
   import {
     BUDIBASE_INTERNAL_DB_ID,
     BUDIBASE_DATASOURCE_TYPE,
+    DB_TYPE_INTERNAL,
   } from "constants/backend"
 
   $: tableNames = $tables.list.map(table => table.name)
@@ -55,8 +56,9 @@
       name,
       schema: { ...schema },
       rows,
-      type: "internal",
+      type: "table",
       sourceId: targetDatasourceId,
+      sourceType: DB_TYPE_INTERNAL,
     }
 
     // Only set primary display if defined
diff --git a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte
index 1760938c53..18c0d460a8 100644
--- a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte
@@ -13,6 +13,7 @@
     notifications,
   } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import { DB_TYPE_EXTERNAL } from "constants/backend"
 
   export let table
 
@@ -27,8 +28,8 @@
   let willBeDeleted
   let deleteTableName
 
-  $: external = table?.type === "external"
-  $: allowDeletion = !external || table?.created
+  $: externalTable = table?.sourceType === DB_TYPE_EXTERNAL
+  $: allowDeletion = !externalTable || table?.created
 
   function showDeleteModal() {
     templateScreens = $store.screens.filter(
@@ -48,7 +49,7 @@
       for (let screen of templateScreens) {
         await store.actions.screens.delete(screen)
       }
-      if (table.type === "external") {
+      if (table.sourceType === DB_TYPE_EXTERNAL) {
         await datasources.fetch()
       }
       notifications.success("Table deleted")
@@ -91,7 +92,7 @@
     <div slot="control" class="icon">
       <Icon s hoverable name="MoreSmallList" />
     </div>
-    {#if !external}
+    {#if !externalTable}
       <MenuItem icon="Edit" on:click={editorModal.show}>Edit</MenuItem>
     {/if}
     <MenuItem icon="Delete" on:click={showDeleteModal}>Delete</MenuItem>
diff --git a/packages/builder/src/components/integration/QueryViewerSidePanel/PreviewPanel.svelte b/packages/builder/src/components/integration/QueryViewerSidePanel/PreviewPanel.svelte
index f379ad18a1..3873669b63 100644
--- a/packages/builder/src/components/integration/QueryViewerSidePanel/PreviewPanel.svelte
+++ b/packages/builder/src/components/integration/QueryViewerSidePanel/PreviewPanel.svelte
@@ -23,7 +23,7 @@
 </script>
 
 <div class="table">
-  <Table {schema} data={rowsCopy} type="external" allowEditing={false} />
+  <Table {schema} data={rowsCopy} allowEditing={false} />
 </div>
 
 <style>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Tables/CreateExternalTableModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Tables/CreateExternalTableModal.svelte
index 664b5629d4..faa3611f5d 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Tables/CreateExternalTableModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Tables/CreateExternalTableModal.svelte
@@ -2,6 +2,7 @@
   import { ModalContent, Body, Input, notifications } from "@budibase/bbui"
   import { tables, datasources } from "stores/backend"
   import { goto } from "@roxi/routify"
+  import { DB_TYPE_EXTERNAL } from "constants/backend"
 
   export let datasource
 
@@ -16,9 +17,10 @@
   function buildDefaultTable(tableName, datasourceId) {
     return {
       name: tableName,
-      type: "external",
+      type: "table",
       primary: ["id"],
       sourceId: datasourceId,
+      sourceType: DB_TYPE_EXTERNAL,
       schema: {
         id: {
           autocolumn: true,
diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/index.svelte
index 189141dd39..21214d8840 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/index.svelte
@@ -5,7 +5,7 @@
   import { tables, datasources } from "stores/backend"
   import { goto } from "@roxi/routify"
   import { onMount } from "svelte"
-  import { BUDIBASE_INTERNAL_DB_ID } from "constants/backend"
+  import { BUDIBASE_INTERNAL_DB_ID, DB_TYPE_EXTERNAL } from "constants/backend"
   import { TableNames } from "constants"
   import { store } from "builderStore"
 
@@ -14,7 +14,7 @@
   $: store.actions.websocket.selectResource(BUDIBASE_INTERNAL_DB_ID)
   $: internalTablesBySourceId = $tables.list.filter(
     table =>
-      table.type !== "external" &&
+      table.sourceType !== DB_TYPE_EXTERNAL &&
       table.sourceId === BUDIBASE_INTERNAL_DB_ID &&
       table._id !== TableNames.USERS
   )
diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/datasource_internal_bb_default/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/datasource_internal_bb_default/index.svelte
index f30ffea131..44cb8db3b0 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/datasource_internal_bb_default/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/datasource_internal_bb_default/index.svelte
@@ -4,7 +4,7 @@
   import ICONS from "components/backend/DatasourceNavigator/icons"
   import { tables, datasources } from "stores/backend"
   import { goto } from "@roxi/routify"
-  import { DEFAULT_BB_DATASOURCE_ID } from "constants/backend"
+  import { DEFAULT_BB_DATASOURCE_ID, DB_TYPE_EXTERNAL } from "constants/backend"
   import { onMount } from "svelte"
   import { store } from "builderStore"
 
@@ -13,7 +13,8 @@
   $: store.actions.websocket.selectResource(DEFAULT_BB_DATASOURCE_ID)
   $: internalTablesBySourceId = $tables.list.filter(
     table =>
-      table.type !== "external" && table.sourceId === DEFAULT_BB_DATASOURCE_ID
+      table.sourceType !== DB_TYPE_EXTERNAL &&
+      table.sourceId === DEFAULT_BB_DATASOURCE_ID
   )
 
   onMount(() => {
diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index c814a37ad9..99b1cd73c6 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -13,12 +13,13 @@ import {
   FetchTablesResponse,
   MigrateRequest,
   MigrateResponse,
+  Row,
   SaveTableRequest,
   SaveTableResponse,
   Table,
   TableResponse,
+  TableSourceType,
   UserCtx,
-  Row,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"
@@ -29,7 +30,7 @@ function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
   if (table && !tableId) {
     tableId = table._id
   }
-  if (table && table.type === "external") {
+  if (table && table.sourceType === TableSourceType.EXTERNAL) {
     return external
   } else if (tableId && isExternalTable(tableId)) {
     return external
@@ -48,7 +49,7 @@ export async function fetch(ctx: UserCtx<void, FetchTablesResponse>) {
     if (entities) {
       return Object.values(entities).map<Table>((entity: Table) => ({
         ...entity,
-        type: "external",
+        sourceType: TableSourceType.EXTERNAL,
         sourceId: datasource._id!,
         sql: isSQL(datasource),
       }))
diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts
index 822ff8a75d..bb94f2bc01 100644
--- a/packages/server/src/api/controllers/table/internal.ts
+++ b/packages/server/src/api/controllers/table/internal.ts
@@ -7,6 +7,7 @@ import {
   SaveTableRequest,
   SaveTableResponse,
   Table,
+  TableSourceType,
   UserCtx,
 } from "@budibase/types"
 import sdk from "../../../sdk"
@@ -16,10 +17,11 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   let tableToSave: Table & {
     _rename?: RenameColumn
   } = {
-    type: "table",
     _id: generateTableID(),
-    views: {},
     ...rest,
+    type: "table",
+    sourceType: TableSourceType.INTERNAL,
+    views: {},
   }
   const renaming = tableToSave._rename
   delete tableToSave._rename
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index e5a24c27e0..0fc9e550e3 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -10,6 +10,7 @@ import {
   FieldSchema,
   FieldType,
   FieldTypeSubtypes,
+  INTERNAL_TABLE_SOURCE_ID,
   MonthlyQuotaName,
   PermissionLevel,
   QuotaUsageType,
@@ -21,7 +22,7 @@ import {
   SortType,
   StaticQuotaName,
   Table,
-  INTERNAL_TABLE_SOURCE_ID,
+  TableSourceType,
 } from "@budibase/types"
 import {
   expectAnyExternalColsAttributes,
@@ -66,6 +67,7 @@ describe.each([
       type: "table",
       primary: ["id"],
       primaryDisplay: "name",
+      sourceType: TableSourceType.INTERNAL,
       sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         id: {
@@ -441,6 +443,7 @@ describe.each([
   describe("view save", () => {
     it("views have extra data trimmed", async () => {
       const table = await config.createTable({
+        type: "table",
         name: "orders",
         primary: ["OrderID"],
         schema: {
@@ -883,6 +886,7 @@ describe.each([
       return {
         name: `users_${generator.word()}`,
         sourceId: INTERNAL_TABLE_SOURCE_ID,
+        sourceType: TableSourceType.INTERNAL,
         type: "table",
         primary: ["id"],
         schema: {
@@ -1066,6 +1070,7 @@ describe.each([
         return {
           name: `users_${generator.word()}`,
           sourceId: INTERNAL_TABLE_SOURCE_ID,
+          sourceType: TableSourceType.INTERNAL,
           type: "table",
           primary: ["id"],
           schema: {
@@ -1603,7 +1608,7 @@ describe.each([
       if (config.datasource) {
         tableConfig.sourceId = config.datasource._id!
         if (config.datasource.plus) {
-          tableConfig.type = "external"
+          tableConfig.sourceType = TableSourceType.EXTERNAL
         }
       }
       const table = await config.api.table.create({
diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts
index f988b44e0f..5392954353 100644
--- a/packages/server/src/api/routes/tests/table.spec.ts
+++ b/packages/server/src/api/routes/tests/table.spec.ts
@@ -1,21 +1,23 @@
-import { events, context } from "@budibase/backend-core"
+import { context, events } from "@budibase/backend-core"
 import {
-  FieldType,
-  SaveTableRequest,
-  RelationshipType,
-  Table,
-  ViewCalculation,
   AutoFieldSubTypes,
-  InternalTable,
   FieldSubtype,
-  Row,
+  FieldType,
   INTERNAL_TABLE_SOURCE_ID,
+  InternalTable,
+  RelationshipType,
+  Row,
+  SaveTableRequest,
+  Table,
+  TableSourceType,
+  ViewCalculation,
 } from "@budibase/types"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
-const { basicTable } = setup.structures
 import sdk from "../../../sdk"
 
+const { basicTable } = setup.structures
+
 describe("/tables", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
@@ -434,6 +436,7 @@ describe("/tables", () => {
         name: "table",
         type: "table",
         sourceId: INTERNAL_TABLE_SOURCE_ID,
+        sourceType: TableSourceType.INTERNAL,
         schema: {
           "user relationship": {
             type: FieldType.LINK,
@@ -494,6 +497,7 @@ describe("/tables", () => {
         name: "table",
         type: "table",
         sourceId: INTERNAL_TABLE_SOURCE_ID,
+        sourceType: TableSourceType.INTERNAL,
         schema: {
           "user relationship": {
             type: FieldType.LINK,
@@ -556,6 +560,7 @@ describe("/tables", () => {
         name: "table",
         type: "table",
         sourceId: INTERNAL_TABLE_SOURCE_ID,
+        sourceType: TableSourceType.INTERNAL,
         schema: {
           "user relationship": {
             type: FieldType.LINK,
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index a1ce5ec10e..2f00a9f172 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -3,14 +3,15 @@ import {
   CreateViewRequest,
   FieldSchema,
   FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
   SearchQueryOperators,
   SortOrder,
   SortType,
   Table,
+  TableSourceType,
   UIFieldMetadata,
   UpdateViewRequest,
   ViewV2,
-  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 import { generator } from "@budibase/backend-core/tests"
 import { generateDatasourceID } from "../../../db/utils"
@@ -20,6 +21,7 @@ function priceTable(): Table {
     name: "table",
     type: "table",
     sourceId: INTERNAL_TABLE_SOURCE_ID,
+    sourceType: TableSourceType.INTERNAL,
     schema: {
       Price: {
         type: FieldType.NUMBER,
@@ -59,7 +61,7 @@ describe.each([
       return config.createTable({
         ...priceTable(),
         sourceId: datasource._id,
-        type: "external",
+        sourceType: TableSourceType.EXTERNAL,
       })
     },
   ],
diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts
index fe69b3c9c8..1104fb9b29 100644
--- a/packages/server/src/constants/index.ts
+++ b/packages/server/src/constants/index.ts
@@ -1,9 +1,11 @@
-import { objectStore, roles, constants } from "@budibase/backend-core"
+import { constants, objectStore, roles } from "@budibase/backend-core"
 import {
   FieldType as FieldTypes,
-  Table,
   INTERNAL_TABLE_SOURCE_ID,
+  Table,
+  TableSourceType,
 } from "@budibase/types"
+
 export {
   FieldType as FieldTypes,
   RelationshipType,
@@ -78,6 +80,7 @@ export const USERS_TABLE_SCHEMA: Table = {
   _id: "ta_users",
   type: "table",
   sourceId: INTERNAL_TABLE_SOURCE_ID,
+  sourceType: TableSourceType.INTERNAL,
   views: {},
   name: "Users",
   // TODO: ADMIN PANEL - when implemented this doesn't need to be carried out
diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts
index 48d4876de1..584b76f879 100644
--- a/packages/server/src/db/defaultData/datasource_bb_default.ts
+++ b/packages/server/src/db/defaultData/datasource_bb_default.ts
@@ -1,4 +1,4 @@
-import { FieldTypes, AutoFieldSubTypes } from "../../constants"
+import { AutoFieldSubTypes, FieldTypes } from "../../constants"
 import { importToRows } from "../../api/controllers/table/utils"
 import { cloneDeep } from "lodash/fp"
 import LinkDocument from "../linkedRows/LinkDocument"
@@ -8,11 +8,12 @@ import { jobsImport } from "./jobsImport"
 import { expensesImport } from "./expensesImport"
 import { db as dbCore } from "@budibase/backend-core"
 import {
-  Table,
-  Row,
-  RelationshipType,
   FieldType,
+  RelationshipType,
+  Row,
+  Table,
   TableSchema,
+  TableSourceType,
 } from "@budibase/types"
 
 export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
@@ -89,9 +90,10 @@ const AUTO_COLUMNS: TableSchema = {
 
 export const DEFAULT_INVENTORY_TABLE_SCHEMA: Table = {
   _id: DEFAULT_INVENTORY_TABLE_ID,
-  type: "internal",
+  type: "table",
   views: {},
   sourceId: DEFAULT_BB_DATASOURCE_ID,
+  sourceType: TableSourceType.INTERNAL,
   primaryDisplay: "Item Name",
   name: "Inventory",
   schema: {
@@ -198,10 +200,11 @@ export const DEFAULT_INVENTORY_TABLE_SCHEMA: Table = {
 
 export const DEFAULT_EMPLOYEE_TABLE_SCHEMA: Table = {
   _id: DEFAULT_EMPLOYEE_TABLE_ID,
-  type: "internal",
+  type: "table",
   views: {},
   name: "Employees",
   sourceId: DEFAULT_BB_DATASOURCE_ID,
+  sourceType: TableSourceType.INTERNAL,
   primaryDisplay: "First Name",
   schema: {
     "First Name": {
@@ -346,9 +349,10 @@ export const DEFAULT_EMPLOYEE_TABLE_SCHEMA: Table = {
 
 export const DEFAULT_JOBS_TABLE_SCHEMA: Table = {
   _id: DEFAULT_JOBS_TABLE_ID,
-  type: "internal",
+  type: "table",
   name: "Jobs",
   sourceId: DEFAULT_BB_DATASOURCE_ID,
+  sourceType: TableSourceType.INTERNAL,
   primaryDisplay: "Job ID",
   schema: {
     "Job ID": {
@@ -503,10 +507,11 @@ export const DEFAULT_JOBS_TABLE_SCHEMA: Table = {
 
 export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = {
   _id: DEFAULT_EXPENSES_TABLE_ID,
-  type: "internal",
+  type: "table",
   views: {},
   name: "Expenses",
   sourceId: DEFAULT_BB_DATASOURCE_ID,
+  sourceType: TableSourceType.INTERNAL,
   primaryDisplay: "Expense ID",
   schema: {
     "Expense ID": {
diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 90f0fc9f2c..60416853b3 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -1,6 +1,4 @@
 import fetch from "node-fetch"
-// @ts-ignore
-fetch.mockSearch()
 import {
   generateMakeRequest,
   MakeRequestResponse,
@@ -13,12 +11,15 @@ import {
   RelationshipType,
   Row,
   Table,
+  TableSourceType,
 } from "@budibase/types"
 import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import { utils } from "@budibase/backend-core"
 import { databaseTestProviders } from "../integrations/tests/utils"
 import { Client } from "pg"
+// @ts-ignore
+fetch.mockSearch()
 
 const config = setup.getConfig()!
 
@@ -52,7 +53,7 @@ describe("postgres integrations", () => {
     async function createAuxTable(prefix: string) {
       return await config.createTable({
         name: `${prefix}_${generator.word({ length: 6 })}`,
-        type: "external",
+        type: "table",
         primary: ["id"],
         primaryDisplay: "title",
         schema: {
@@ -67,6 +68,7 @@ describe("postgres integrations", () => {
           },
         },
         sourceId: postgresDatasource._id,
+        sourceType: TableSourceType.EXTERNAL,
       })
     }
 
@@ -88,7 +90,7 @@ describe("postgres integrations", () => {
 
     primaryPostgresTable = await config.createTable({
       name: `p_${generator.word({ length: 6 })}`,
-      type: "external",
+      type: "table",
       primary: ["id"],
       schema: {
         id: {
@@ -143,6 +145,7 @@ describe("postgres integrations", () => {
         },
       },
       sourceId: postgresDatasource._id,
+      sourceType: TableSourceType.EXTERNAL,
     })
   })
 
@@ -249,7 +252,7 @@ describe("postgres integrations", () => {
   async function createDefaultPgTable() {
     return await config.createTable({
       name: generator.word({ length: 10 }),
-      type: "external",
+      type: "table",
       primary: ["id"],
       schema: {
         id: {
@@ -259,6 +262,7 @@ describe("postgres integrations", () => {
         },
       },
       sourceId: postgresDatasource._id,
+      sourceType: TableSourceType.EXTERNAL,
     })
   }
 
diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index 4433f45863..58c867ea0b 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -10,11 +10,12 @@ import {
   QueryJson,
   QueryType,
   Row,
+  Schema,
   SearchFilters,
   SortJson,
   Table,
   TableRequest,
-  Schema,
+  TableSourceType,
 } from "@budibase/types"
 import { OAuth2Client } from "google-auth-library"
 import {
@@ -263,10 +264,12 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   ) {
     // base table
     const table: Table = {
+      type: "table",
       name: title,
       primary: [GOOGLE_SHEETS_PRIMARY_KEY],
       schema: {},
       sourceId: datasourceId,
+      sourceType: TableSourceType.EXTERNAL,
     }
     if (id) {
       table._id = id
diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts
index b86286756c..c615e5ba48 100644
--- a/packages/server/src/integrations/microsoftSqlServer.ts
+++ b/packages/server/src/integrations/microsoftSqlServer.ts
@@ -12,6 +12,7 @@ import {
   ConnectionInfo,
   SourceName,
   Schema,
+  TableSourceType,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -439,7 +440,9 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
       }
       tables[tableName] = {
         _id: buildExternalTableId(datasourceId, tableName),
+        type: "table",
         sourceId: datasourceId,
+        sourceType: TableSourceType.EXTERNAL,
         primary: primaryKeys,
         name: tableName,
         schema,
diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts
index fe7eae51be..e89393d251 100644
--- a/packages/server/src/integrations/mysql.ts
+++ b/packages/server/src/integrations/mysql.ts
@@ -11,6 +11,7 @@ import {
   ConnectionInfo,
   SourceName,
   Schema,
+  TableSourceType,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -317,8 +318,10 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
         }
         if (!tables[tableName]) {
           tables[tableName] = {
+            type: "table",
             _id: buildExternalTableId(datasourceId, tableName),
             sourceId: datasourceId,
+            sourceType: TableSourceType.EXTERNAL,
             primary: primaryKeys,
             name: tableName,
             schema,
diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index 5fde565180..c6a871e41f 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -10,6 +10,7 @@ import {
   DatasourceFeature,
   ConnectionInfo,
   Schema,
+  TableSourceType,
 } from "@budibase/types"
 import {
   buildExternalTableId,
@@ -277,11 +278,13 @@ class OracleIntegration extends Sql implements DatasourcePlus {
       let table = tables[oracleTable.name]
       if (!table) {
         table = {
+          type: "table",
           _id: buildExternalTableId(datasourceId, oracleTable.name),
           primary: [],
           name: oracleTable.name,
           schema: {},
           sourceId: datasourceId,
+          sourceType: TableSourceType.EXTERNAL,
         }
         tables[oracleTable.name] = table
       }
diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts
index 38339499b2..4d7dc33d75 100644
--- a/packages/server/src/integrations/postgres.ts
+++ b/packages/server/src/integrations/postgres.ts
@@ -11,6 +11,7 @@ import {
   ConnectionInfo,
   SourceName,
   Schema,
+  TableSourceType,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -309,11 +310,13 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
         // table key doesn't exist yet
         if (!tables[tableName] || !tables[tableName].schema) {
           tables[tableName] = {
+            type: "table",
             _id: buildExternalTableId(datasourceId, tableName),
             primary: tableKeys[tableName] || [],
             name: tableName,
             schema: {},
             sourceId: datasourceId,
+            sourceType: TableSourceType.EXTERNAL,
           }
         }
 
diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index 842b681867..a7e4217f25 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -30,7 +30,7 @@ GoogleSpreadsheet.mockImplementation(() => mockGoogleIntegration)
 import { structures } from "@budibase/backend-core/tests"
 import TestConfiguration from "../../tests/utilities/TestConfiguration"
 import GoogleSheetsIntegration from "../googlesheets"
-import { FieldType, Table, TableSchema } from "@budibase/types"
+import { FieldType, Table, TableSchema, TableSourceType } from "@budibase/types"
 import { generateDatasourceID } from "../../db/utils"
 
 describe("Google Sheets Integration", () => {
@@ -61,8 +61,10 @@ describe("Google Sheets Integration", () => {
 
   function createBasicTable(name: string, columns: string[]): Table {
     return {
+      type: "table",
       name,
       sourceId: generateDatasourceID(),
+      sourceType: TableSourceType.EXTERNAL,
       schema: {
         ...columns.reduce((p, c) => {
           p[c] = {
diff --git a/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts b/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts
index 106129f6c9..40ff88c1e5 100644
--- a/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts
+++ b/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts
@@ -5,6 +5,7 @@ import {
   Row,
   Table,
   INTERNAL_TABLE_SOURCE_ID,
+  TableSourceType,
 } from "@budibase/types"
 import * as utils from "../../db/utils"
 import trimViewRowInfoMiddleware from "../trimViewRowInfo"
@@ -80,6 +81,7 @@ describe("trimViewRowInfo middleware", () => {
     _id: tableId,
     name: generator.word(),
     sourceId: INTERNAL_TABLE_SOURCE_ID,
+    sourceType: TableSourceType.INTERNAL,
     type: "table",
     schema: {
       name: {
diff --git a/packages/server/src/middleware/trimViewRowInfo.ts b/packages/server/src/middleware/trimViewRowInfo.ts
index 6a7448262b..95b085a08f 100644
--- a/packages/server/src/middleware/trimViewRowInfo.ts
+++ b/packages/server/src/middleware/trimViewRowInfo.ts
@@ -1,7 +1,6 @@
 import { Ctx, Row } from "@budibase/types"
 import * as utils from "../db/utils"
 import sdk from "../sdk"
-import { db } from "@budibase/backend-core"
 import { Next } from "koa"
 import { getTableId } from "../api/controllers/row/utils"
 
diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts
index 1afdca35fa..d0f59cdd83 100644
--- a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts
+++ b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts
@@ -7,6 +7,7 @@ import {
   SourceName,
   Table,
   SearchParams,
+  TableSourceType,
 } from "@budibase/types"
 
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
@@ -59,9 +60,10 @@ describe.skip("external", () => {
 
     tableData = {
       name: generator.word(),
-      type: "external",
+      type: "table",
       primary: ["id"],
       sourceId: externalDatasource._id!,
+      sourceType: TableSourceType.EXTERNAL,
       schema: {
         id: {
           name: "id",
diff --git a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts
index b3cac2321e..d82af66e3d 100644
--- a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts
+++ b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts
@@ -4,6 +4,7 @@ import {
   Table,
   SearchParams,
   INTERNAL_TABLE_SOURCE_ID,
+  TableSourceType,
 } from "@budibase/types"
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
 import { search } from "../internal"
@@ -19,6 +20,7 @@ describe("internal", () => {
     name: generator.word(),
     type: "table",
     sourceId: INTERNAL_TABLE_SOURCE_ID,
+    sourceType: TableSourceType.INTERNAL,
     schema: {
       name: {
         name: "name",
diff --git a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts
index 428c57be64..055628c41c 100644
--- a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts
+++ b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts
@@ -3,16 +3,19 @@ import { db as dbCore } from "@budibase/backend-core"
 import {
   FieldType,
   FieldTypeSubtypes,
-  Table,
-  SearchParams,
   INTERNAL_TABLE_SOURCE_ID,
+  SearchParams,
+  Table,
+  TableSourceType,
 } from "@budibase/types"
 
 const tableId = "ta_a"
 const tableWithUserCol: Table = {
+  type: "table",
   _id: tableId,
   name: "table",
   sourceId: INTERNAL_TABLE_SOURCE_ID,
+  sourceType: TableSourceType.INTERNAL,
   schema: {
     user: {
       name: "user",
@@ -23,9 +26,11 @@ const tableWithUserCol: Table = {
 }
 
 const tableWithUsersCol: Table = {
+  type: "table",
   _id: tableId,
   name: "table",
   sourceId: INTERNAL_TABLE_SOURCE_ID,
+  sourceType: TableSourceType.INTERNAL,
   schema: {
     user: {
       name: "user",
diff --git a/packages/server/src/sdk/app/tables/external/utils.ts b/packages/server/src/sdk/app/tables/external/utils.ts
index a60667f44f..bde812dd3d 100644
--- a/packages/server/src/sdk/app/tables/external/utils.ts
+++ b/packages/server/src/sdk/app/tables/external/utils.ts
@@ -6,6 +6,7 @@ import {
   RelationshipFieldMetadata,
   RelationshipType,
   Table,
+  TableSourceType,
 } from "@budibase/types"
 import { FieldTypes } from "../../../../constants"
 import {
@@ -78,12 +79,14 @@ export function generateManyLinkSchema(
   const jcTblName = generateJunctionTableName(column, table, relatedTable)
   const datasourceId = datasource._id!
   // first create the new table
-  const junctionTable = {
+  const junctionTable: Table = {
+    type: "table",
     _id: buildExternalTableId(datasourceId, jcTblName),
     name: jcTblName,
     primary: [primary, relatedPrimary],
     constrained: [primary, relatedPrimary],
     sourceId: datasourceId,
+    sourceType: TableSourceType.EXTERNAL,
     schema: {
       [primary]: foreignKeyStructure(primary, {
         toTable: table.name,
diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts
index b9c67c573b..176a20a59d 100644
--- a/packages/server/src/sdk/app/tables/getters.ts
+++ b/packages/server/src/sdk/app/tables/getters.ts
@@ -7,24 +7,41 @@ import {
 } from "../../../integrations/utils"
 import {
   Database,
+  INTERNAL_TABLE_SOURCE_ID,
   Table,
   TableResponse,
+  TableSourceType,
   TableViewsResponse,
-  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 import datasources from "../datasources"
 import sdk from "../../../sdk"
 
-function processInternalTables(tables: Table[]): Table[] {
-  return tables.map(processInternalTable)
+export function processTable(table: Table): Table {
+  if (table._id && isExternalTable(table._id)) {
+    return {
+      ...table,
+      type: "table",
+      sourceType: TableSourceType.EXTERNAL,
+    }
+  } else {
+    return {
+      ...table,
+      type: "table",
+      sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
+    }
+  }
 }
 
-export function processInternalTable(table: Table): Table {
-  return {
-    ...table,
-    type: "internal",
-    sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID,
+export function processTables(tables: Table[]): Table[] {
+  return tables.map(table => processTable(table))
+}
+
+function processEntities(tables: Record<string, Table>) {
+  for (let key of Object.keys(tables)) {
+    tables[key] = processTable(tables[key])
   }
+  return tables
 }
 
 export async function getAllInternalTables(db?: Database): Promise<Table[]> {
@@ -36,7 +53,7 @@ export async function getAllInternalTables(db?: Database): Promise<Table[]> {
       include_docs: true,
     })
   )
-  return processInternalTables(internalTables.rows.map(row => row.doc!))
+  return processTables(internalTables.rows.map(row => row.doc!))
 }
 
 async function getAllExternalTables(): Promise<Table[]> {
@@ -48,7 +65,7 @@ async function getAllExternalTables(): Promise<Table[]> {
       final = final.concat(Object.values(entities))
     }
   }
-  return final
+  return processTables(final)
 }
 
 export async function getExternalTable(
@@ -56,19 +73,21 @@ export async function getExternalTable(
   tableName: string
 ): Promise<Table> {
   const entities = await getExternalTablesInDatasource(datasourceId)
-  return entities[tableName]
+  return processTable(entities[tableName])
 }
 
 export async function getTable(tableId: string): Promise<Table> {
   const db = context.getAppDB()
+  let output: Table
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
     const datasource = await datasources.get(datasourceId!)
     const table = await getExternalTable(datasourceId!, tableName!)
-    return { ...table, sql: isSQL(datasource) }
+    output = { ...table, sql: isSQL(datasource) }
   } else {
-    return processInternalTable(await db.get<Table>(tableId))
+    output = await db.get<Table>(tableId)
   }
+  return processTable(output)
 }
 
 export async function getAllTables() {
@@ -76,7 +95,7 @@ export async function getAllTables() {
     getAllInternalTables(),
     getAllExternalTables(),
   ])
-  return [...internal, ...external]
+  return processTables([...internal, ...external])
 }
 
 export async function getExternalTablesInDatasource(
@@ -86,7 +105,7 @@ export async function getExternalTablesInDatasource(
   if (!datasource || !datasource.entities) {
     throw new Error("Datasource is not configured fully.")
   }
-  return datasource.entities
+  return processEntities(datasource.entities)
 }
 
 export async function getTables(tableIds: string[]): Promise<Table[]> {
@@ -106,11 +125,9 @@ export async function getTables(tableIds: string[]): Promise<Table[]> {
     const internalTableDocs = await db.allDocs<Table[]>(
       getMultiIDParams(internalTableIds)
     )
-    tables = tables.concat(
-      processInternalTables(internalTableDocs.rows.map(row => row.doc!))
-    )
+    tables = tables.concat(internalTableDocs.rows.map(row => row.doc!))
   }
-  return tables
+  return processTables(tables)
 }
 
 export function enrichViewSchemas(table: Table): TableResponse {
diff --git a/packages/server/src/sdk/app/tables/tests/tables.spec.ts b/packages/server/src/sdk/app/tables/tests/tables.spec.ts
index ab45eeb8bb..457988c476 100644
--- a/packages/server/src/sdk/app/tables/tests/tables.spec.ts
+++ b/packages/server/src/sdk/app/tables/tests/tables.spec.ts
@@ -2,6 +2,7 @@ import {
   FieldType,
   INTERNAL_TABLE_SOURCE_ID,
   Table,
+  TableSourceType,
   ViewV2,
 } from "@budibase/types"
 import { generator } from "@budibase/backend-core/tests"
@@ -19,6 +20,7 @@ describe("table sdk", () => {
       name: "TestTable",
       type: "table",
       sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
       schema: {
         name: {
           type: FieldType.STRING,
diff --git a/packages/server/src/sdk/app/tables/tests/validation.spec.ts b/packages/server/src/sdk/app/tables/tests/validation.spec.ts
index 62a9c61dbb..66b4222005 100644
--- a/packages/server/src/sdk/app/tables/tests/validation.spec.ts
+++ b/packages/server/src/sdk/app/tables/tests/validation.spec.ts
@@ -7,6 +7,7 @@ import {
   RelationshipType,
   SourceName,
   Table,
+  TableSourceType,
 } from "@budibase/types"
 import { isEqual } from "lodash"
 import { generateDatasourceID } from "../../../../db/utils"
@@ -19,11 +20,13 @@ const SCHEMA: Datasource = {
   _id: datasourceId,
   entities: {
     client: {
+      type: "table",
       _id: "tableA",
       name: "client",
       primary: ["idC"],
       primaryDisplay: "Name",
       sourceId: datasourceId,
+      sourceType: TableSourceType.EXTERNAL,
       schema: {
         idC: {
           autocolumn: true,
@@ -49,11 +52,13 @@ const SCHEMA: Datasource = {
       },
     },
     project: {
+      type: "table",
       _id: "tableB",
       name: "project",
       primary: ["idP"],
       primaryDisplay: "Name",
       sourceId: datasourceId,
+      sourceType: TableSourceType.EXTERNAL,
       schema: {
         idC: {
           externalType: "int unsigned",
@@ -82,7 +87,6 @@ const SCHEMA: Datasource = {
         },
       },
       sql: true,
-      type: "table",
     },
   },
 }
diff --git a/packages/server/src/sdk/app/tables/utils.ts b/packages/server/src/sdk/app/tables/utils.ts
index 88543e7c4c..68a054f6c1 100644
--- a/packages/server/src/sdk/app/tables/utils.ts
+++ b/packages/server/src/sdk/app/tables/utils.ts
@@ -1,8 +1,8 @@
-import { Table } from "@budibase/types"
+import { Table, TableSourceType } from "@budibase/types"
 import { isExternalTable } from "../../../integrations/utils"
 
 export function isExternal(opts: { table?: Table; tableId?: string }): boolean {
-  if (opts.table && opts.table.type === "external") {
+  if (opts.table && opts.table.sourceType === TableSourceType.EXTERNAL) {
     return true
   } else if (opts.tableId && isExternalTable(opts.tableId)) {
     return true
diff --git a/packages/server/src/sdk/app/views/tests/views.spec.ts b/packages/server/src/sdk/app/views/tests/views.spec.ts
index 2068eda4c3..508285651a 100644
--- a/packages/server/src/sdk/app/views/tests/views.spec.ts
+++ b/packages/server/src/sdk/app/views/tests/views.spec.ts
@@ -5,6 +5,7 @@ import {
   INTERNAL_TABLE_SOURCE_ID,
   Table,
   TableSchema,
+  TableSourceType,
   ViewV2,
 } from "@budibase/types"
 import { generator } from "@budibase/backend-core/tests"
@@ -16,6 +17,7 @@ describe("table sdk", () => {
     name: "TestTable",
     type: "table",
     sourceId: INTERNAL_TABLE_SOURCE_ID,
+    sourceType: TableSourceType.INTERNAL,
     schema: {
       name: {
         type: FieldType.STRING,
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 3d4046b91b..a397b00441 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -2,37 +2,31 @@ import { generator, mocks, structures } from "@budibase/backend-core/tests"
 
 // init the licensing mock
 import * as pro from "@budibase/pro"
-mocks.licenses.init(pro)
-
-// use unlimited license by default
-mocks.licenses.useUnlimited()
-
 import { init as dbInit } from "../../db"
-dbInit()
 import env from "../../environment"
 import {
-  basicTable,
-  basicRow,
-  basicRole,
   basicAutomation,
-  basicDatasource,
-  basicQuery,
-  basicScreen,
-  basicLayout,
-  basicWebhook,
   basicAutomationResults,
+  basicDatasource,
+  basicLayout,
+  basicQuery,
+  basicRole,
+  basicRow,
+  basicScreen,
+  basicTable,
+  basicWebhook,
 } from "./structures"
 import {
-  constants,
-  tenancy,
-  sessions,
+  auth,
   cache,
+  constants,
   context,
   db as dbCore,
   encryption,
-  auth,
-  roles,
   env as coreEnv,
+  roles,
+  sessions,
+  tenancy,
 } from "@budibase/backend-core"
 import * as controllers from "./controllers"
 import { cleanup } from "../../utilities/fileSystem"
@@ -43,24 +37,32 @@ import supertest from "supertest"
 import {
   App,
   AuthToken,
+  Automation,
+  CreateViewRequest,
   Datasource,
+  FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
+  RelationshipFieldMetadata,
+  RelationshipType,
   Row,
+  SearchFilters,
   SourceName,
   Table,
-  SearchFilters,
-  UserRoles,
-  Automation,
-  View,
-  FieldType,
-  RelationshipType,
-  CreateViewRequest,
-  RelationshipFieldMetadata,
+  TableSourceType,
   User,
-  INTERNAL_TABLE_SOURCE_ID,
+  UserRoles,
+  View,
 } from "@budibase/types"
 
 import API from "./api"
 
+mocks.licenses.init(pro)
+
+// use unlimited license by default
+mocks.licenses.useUnlimited()
+
+dbInit()
+
 type DefaultUserValues = {
   globalUserId: string
   email: string
@@ -69,8 +71,9 @@ type DefaultUserValues = {
   csrfToken: string
 }
 
-interface TableToBuild extends Omit<Table, "sourceId"> {
+interface TableToBuild extends Omit<Table, "sourceId" | "sourceType"> {
   sourceId?: string
+  sourceType?: TableSourceType
 }
 
 class TestConfiguration {
@@ -547,9 +550,8 @@ class TestConfiguration {
     { skipReassigning } = { skipReassigning: false }
   ): Promise<Table> {
     config = config || basicTable()
-    if (!config.sourceId) {
-      config.sourceId = INTERNAL_TABLE_SOURCE_ID
-    }
+    config.sourceType = config.sourceType || TableSourceType.INTERNAL
+    config.sourceId = config.sourceId || INTERNAL_TABLE_SOURCE_ID
     const response = await this._req(config, null, controllers.table.save)
     if (!skipReassigning) {
       this.table = response
@@ -571,7 +573,7 @@ class TestConfiguration {
     if (this.datasource && !config.sourceId) {
       config.sourceId = this.datasource._id || INTERNAL_TABLE_SOURCE_ID
       if (this.datasource.plus) {
-        config.type = "external"
+        config.sourceType = TableSourceType.EXTERNAL
       }
     }
 
@@ -609,12 +611,11 @@ class TestConfiguration {
     if (this.datasource && !tableConfig.sourceId) {
       tableConfig.sourceId = this.datasource._id || INTERNAL_TABLE_SOURCE_ID
       if (this.datasource.plus) {
-        tableConfig.type = "external"
+        tableConfig.sourceType = TableSourceType.EXTERNAL
       }
     }
 
-    const linkedTable = await this.createTable(tableConfig)
-    return linkedTable
+    return await this.createTable(tableConfig)
   }
 
   async createAttachmentTable() {
diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts
index 52f3450454..be4bff28e3 100644
--- a/packages/server/src/tests/utilities/structures.ts
+++ b/packages/server/src/tests/utilities/structures.ts
@@ -20,6 +20,7 @@ import {
   SourceName,
   Table,
   INTERNAL_TABLE_SOURCE_ID,
+  TableSourceType,
 } from "@budibase/types"
 
 export function basicTable(): Table {
@@ -27,6 +28,7 @@ export function basicTable(): Table {
     name: "TestTable",
     type: "table",
     sourceId: INTERNAL_TABLE_SOURCE_ID,
+    sourceType: TableSourceType.INTERNAL,
     schema: {
       name: {
         type: FieldType.STRING,
diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
index 10274ed4d9..b6c1db9159 100644
--- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
@@ -5,6 +5,7 @@ import {
   FieldTypeSubtypes,
   INTERNAL_TABLE_SOURCE_ID,
   Table,
+  TableSourceType,
 } from "@budibase/types"
 import * as bbReferenceProcessor from "../bbReferenceProcessor"
 
@@ -26,6 +27,7 @@ describe("rowProcessor - inputProcessing", () => {
       name: "TestTable",
       type: "table",
       sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
       schema: {
         name: {
           type: FieldType.STRING,
@@ -77,6 +79,7 @@ describe("rowProcessor - inputProcessing", () => {
       name: "TestTable",
       type: "table",
       sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
       schema: {
         name: {
           type: FieldType.STRING,
@@ -118,6 +121,7 @@ describe("rowProcessor - inputProcessing", () => {
         name: "TestTable",
         type: "table",
         sourceId: INTERNAL_TABLE_SOURCE_ID,
+        sourceType: TableSourceType.INTERNAL,
         schema: {
           name: {
             type: FieldType.STRING,
@@ -159,6 +163,7 @@ describe("rowProcessor - inputProcessing", () => {
       name: "TestTable",
       type: "table",
       sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
       schema: {
         name: {
           type: FieldType.STRING,
diff --git a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts
index 56db285f8b..03584ef53b 100644
--- a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts
@@ -4,6 +4,7 @@ import {
   FieldTypeSubtypes,
   INTERNAL_TABLE_SOURCE_ID,
   Table,
+  TableSourceType,
 } from "@budibase/types"
 import { outputProcessing } from ".."
 import { generator, structures } from "@budibase/backend-core/tests"
@@ -28,6 +29,7 @@ describe("rowProcessor - outputProcessing", () => {
       name: "TestTable",
       type: "table",
       sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
       schema: {
         name: {
           type: FieldType.STRING,
@@ -74,6 +76,7 @@ describe("rowProcessor - outputProcessing", () => {
       name: "TestTable",
       type: "table",
       sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
       schema: {
         name: {
           type: FieldType.STRING,
@@ -112,6 +115,7 @@ describe("rowProcessor - outputProcessing", () => {
       name: "TestTable",
       type: "table",
       sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
       schema: {
         name: {
           type: FieldType.STRING,
diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts
index 0428561bfe..a47d3048d3 100644
--- a/packages/server/src/websockets/builder.ts
+++ b/packages/server/src/websockets/builder.ts
@@ -16,8 +16,7 @@ import { gridSocket } from "./index"
 import { clearLock, updateLock } from "../utilities/redis"
 import { Socket } from "socket.io"
 import { BuilderSocketEvent } from "@budibase/shared-core"
-import { processInternalTable } from "../sdk/app/tables/getters"
-import { isInternalTable } from "../integrations/utils"
+import { processTable } from "../sdk/app/tables/getters"
 
 export default class BuilderSocket extends BaseSocket {
   constructor(app: Koa, server: http.Server) {
@@ -106,9 +105,7 @@ export default class BuilderSocket extends BaseSocket {
     // This was added to make sure that sourceId is always present when
     // sending this message to clients. Without this, tables without a
     // sourceId (e.g. ta_users) won't get correctly updated client-side.
-    if (isInternalTable(table._id!)) {
-      table = processInternalTable(table)
-    }
+    table = processTable(table)
 
     this.emitToRoom(
       ctx,
diff --git a/packages/types/src/documents/app/table/table.ts b/packages/types/src/documents/app/table/table.ts
index df17351c12..f3b8e6df8d 100644
--- a/packages/types/src/documents/app/table/table.ts
+++ b/packages/types/src/documents/app/table/table.ts
@@ -5,8 +5,14 @@ import { TableSchema } from "./schema"
 
 export const INTERNAL_TABLE_SOURCE_ID = "bb_internal"
 
+export enum TableSourceType {
+  EXTERNAL = "external",
+  INTERNAL = "internal",
+}
+
 export interface Table extends Document {
-  type?: string
+  type: "table"
+  sourceType: TableSourceType
   views?: { [key: string]: View | ViewV2 }
   name: string
   sourceId: string

From 536d85d828b48ff5d6028221b958cc5a59869eb4 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 26 Oct 2023 15:04:33 +0100
Subject: [PATCH 060/134] WIP: Improving the use of DatabaseQueryOpts.

---
 packages/backend-core/src/docIds/params.ts    | 29 ++++++++++++++-----
 packages/backend-core/src/users/db.ts         |  6 +---
 packages/backend-core/src/users/users.ts      | 12 +++++---
 .../server/src/db/linkedRows/linkUtils.ts     | 11 +++++--
 .../functions/backfill/global/configs.ts      |  3 +-
 packages/server/src/sdk/app/links/links.ts    | 13 +++++++--
 packages/types/src/sdk/db.ts                  | 10 ++++---
 7 files changed, 56 insertions(+), 28 deletions(-)

diff --git a/packages/backend-core/src/docIds/params.ts b/packages/backend-core/src/docIds/params.ts
index 36fd75622b..d9baee3dc6 100644
--- a/packages/backend-core/src/docIds/params.ts
+++ b/packages/backend-core/src/docIds/params.ts
@@ -6,6 +6,7 @@ import {
   ViewName,
 } from "../constants"
 import { getProdAppID } from "./conversions"
+import { DatabaseQueryOpts } from "@budibase/types"
 
 /**
  * If creating DB allDocs/query params with only a single top level ID this can be used, this
@@ -22,8 +23,8 @@ import { getProdAppID } from "./conversions"
 export function getDocParams(
   docType: string,
   docId?: string | null,
-  otherProps: any = {}
-) {
+  otherProps: Partial<DatabaseQueryOpts> = {}
+): DatabaseQueryOpts {
   if (docId == null) {
     docId = ""
   }
@@ -45,8 +46,8 @@ export function getDocParams(
 export function getRowParams(
   tableId?: string | null,
   rowId?: string | null,
-  otherProps = {}
-) {
+  otherProps: Partial<DatabaseQueryOpts> = {}
+): DatabaseQueryOpts {
   if (tableId == null) {
     return getDocParams(DocumentType.ROW, null, otherProps)
   }
@@ -88,7 +89,10 @@ export const isDatasourceId = (id: string) => {
 /**
  * Gets parameters for retrieving workspaces.
  */
-export function getWorkspaceParams(id = "", otherProps = {}) {
+export function getWorkspaceParams(
+  id = "",
+  otherProps: Partial<DatabaseQueryOpts> = {}
+): DatabaseQueryOpts {
   return {
     ...otherProps,
     startkey: `${DocumentType.WORKSPACE}${SEPARATOR}${id}`,
@@ -99,7 +103,10 @@ export function getWorkspaceParams(id = "", otherProps = {}) {
 /**
  * Gets parameters for retrieving users.
  */
-export function getGlobalUserParams(globalId: any, otherProps: any = {}) {
+export function getGlobalUserParams(
+  globalId: any,
+  otherProps: Partial<DatabaseQueryOpts> = {}
+): DatabaseQueryOpts {
   if (!globalId) {
     globalId = ""
   }
@@ -117,11 +124,17 @@ export function getGlobalUserParams(globalId: any, otherProps: any = {}) {
 /**
  * Gets parameters for retrieving users, this is a utility function for the getDocParams function.
  */
-export function getUserMetadataParams(userId?: string | null, otherProps = {}) {
+export function getUserMetadataParams(
+  userId?: string | null,
+  otherProps: Partial<DatabaseQueryOpts> = {}
+): DatabaseQueryOpts {
   return getRowParams(InternalTable.USER_METADATA, userId, otherProps)
 }
 
-export function getUsersByAppParams(appId: any, otherProps: any = {}) {
+export function getUsersByAppParams(
+  appId: any,
+  otherProps: Partial<DatabaseQueryOpts> = {}
+): DatabaseQueryOpts {
   const prodAppId = getProdAppID(appId)
   return {
     ...otherProps,
diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts
index a2539e836e..78df262a4e 100644
--- a/packages/backend-core/src/users/db.ts
+++ b/packages/backend-core/src/users/db.ts
@@ -160,13 +160,9 @@ export class UserDB {
   }
 
   static async getUsersByAppAccess(opts: { appId?: string; limit?: number }) {
-    const params: any = {
-      include_docs: true,
-      limit: opts.limit || 50,
-    }
     let response: User[] = await usersCore.searchGlobalUsersByAppAccess(
       opts.appId,
-      params
+      { limit: opts.limit || 50 }
     )
     return response
   }
diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts
index 6237c23972..adf134f04a 100644
--- a/packages/backend-core/src/users/users.ts
+++ b/packages/backend-core/src/users/users.ts
@@ -20,6 +20,7 @@ import {
   SearchUsersRequest,
   User,
   DatabaseQueryOpts,
+  CouchFindOptions,
 } from "@budibase/types"
 import { getGlobalDB } from "../context"
 import * as context from "../context"
@@ -140,7 +141,7 @@ export const getGlobalUserByEmail = async (
 
 export const searchGlobalUsersByApp = async (
   appId: any,
-  opts: any,
+  opts: DatabaseQueryOpts,
   getOpts?: GetOpts
 ) => {
   if (typeof appId !== "string") {
@@ -166,7 +167,10 @@ export const searchGlobalUsersByApp = async (
   Return any user who potentially has access to the application
   Admins, developers and app users with the explicitly role.
 */
-export const searchGlobalUsersByAppAccess = async (appId: any, opts: any) => {
+export const searchGlobalUsersByAppAccess = async (
+  appId: any,
+  opts?: { limit?: number }
+) => {
   const roleSelector = `roles.${appId}`
 
   let orQuery: any[] = [
@@ -187,7 +191,7 @@ export const searchGlobalUsersByAppAccess = async (appId: any, opts: any) => {
     orQuery.push(roleCheck)
   }
 
-  let searchOptions = {
+  let searchOptions: CouchFindOptions = {
     selector: {
       $or: orQuery,
       _id: {
@@ -198,7 +202,7 @@ export const searchGlobalUsersByAppAccess = async (appId: any, opts: any) => {
   }
 
   const resp = await directCouchFind(context.getGlobalDBName(), searchOptions)
-  return resp?.rows
+  return resp.rows
 }
 
 export const getGlobalUserByAppPage = (appId: string, user: User) => {
diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts
index c74674a865..db9a0dc7d5 100644
--- a/packages/server/src/db/linkedRows/linkUtils.ts
+++ b/packages/server/src/db/linkedRows/linkUtils.ts
@@ -2,7 +2,12 @@ import { ViewName, getQueryIndex, isRelationshipColumn } from "../utils"
 import { FieldTypes } from "../../constants"
 import { createLinkView } from "../views/staticViews"
 import { context, logging } from "@budibase/backend-core"
-import { LinkDocument, LinkDocumentValue, Table } from "@budibase/types"
+import {
+  DatabaseQueryOpts,
+  LinkDocument,
+  LinkDocumentValue,
+  Table,
+} from "@budibase/types"
 
 export { createLinkView } from "../views/staticViews"
 
@@ -36,13 +41,13 @@ export async function getLinkDocuments(args: {
 }): Promise<LinkDocumentValue[] | LinkDocument[]> {
   const { tableId, rowId, fieldName, includeDocs } = args
   const db = context.getAppDB()
-  let params: any
+  let params: DatabaseQueryOpts
   if (rowId) {
     params = { key: [tableId, rowId] }
   }
   // only table is known
   else {
-    params = { startKey: [tableId], endKey: [tableId, {}] }
+    params = { startkey: [tableId], endkey: [tableId, {}] }
   }
   if (includeDocs) {
     params.include_docs = true
diff --git a/packages/server/src/migrations/functions/backfill/global/configs.ts b/packages/server/src/migrations/functions/backfill/global/configs.ts
index 1b76727bbe..04eb9caff2 100644
--- a/packages/server/src/migrations/functions/backfill/global/configs.ts
+++ b/packages/server/src/migrations/functions/backfill/global/configs.ts
@@ -11,10 +11,11 @@ import {
   isOIDCConfig,
   isSettingsConfig,
   ConfigType,
+  DatabaseQueryOpts,
 } from "@budibase/types"
 import env from "./../../../../environment"
 
-export const getConfigParams = () => {
+export function getConfigParams(): DatabaseQueryOpts {
   return {
     include_docs: true,
     startkey: `${DocumentType.CONFIG}${SEPARATOR}`,
diff --git a/packages/server/src/sdk/app/links/links.ts b/packages/server/src/sdk/app/links/links.ts
index 7f3dcbd4ac..fda07568f9 100644
--- a/packages/server/src/sdk/app/links/links.ts
+++ b/packages/server/src/sdk/app/links/links.ts
@@ -1,6 +1,10 @@
 import { context } from "@budibase/backend-core"
 import { isTableId } from "@budibase/backend-core/src/docIds"
-import { LinkDocument, LinkDocumentValue } from "@budibase/types"
+import {
+  DatabaseQueryOpts,
+  LinkDocument,
+  LinkDocumentValue,
+} from "@budibase/types"
 import { ViewName, getQueryIndex } from "../../../../src/db/utils"
 
 export async function fetch(tableId: string): Promise<LinkDocumentValue[]> {
@@ -9,7 +13,10 @@ export async function fetch(tableId: string): Promise<LinkDocumentValue[]> {
   }
 
   const db = context.getAppDB()
-  const params: any = { startkey: [tableId], endkey: [tableId, {}] }
+  const params: DatabaseQueryOpts = {
+    startkey: [tableId],
+    endkey: [tableId, {}],
+  }
   const linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows
   return linkRows.map(row => row.value as LinkDocumentValue)
 }
@@ -22,7 +29,7 @@ export async function fetchWithDocument(
   }
 
   const db = context.getAppDB()
-  const params: any = {
+  const params: DatabaseQueryOpts = {
     startkey: [tableId],
     endkey: [tableId, {}],
     include_docs: true,
diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts
index 36141cc15b..d3a6396128 100644
--- a/packages/types/src/sdk/db.ts
+++ b/packages/types/src/sdk/db.ts
@@ -54,15 +54,17 @@ export type DatabaseDeleteIndexOpts = {
   type?: string | undefined
 }
 
+export type DatabaseKey = string | number | {} | DatabaseKey[]
+
 export type DatabaseQueryOpts = {
   include_docs?: boolean
-  startkey?: string
-  endkey?: string
+  startkey?: DatabaseKey
+  endkey?: DatabaseKey
   limit?: number
   skip?: number
   descending?: boolean
-  key?: string
-  keys?: string[]
+  key?: DatabaseKey
+  keys?: DatabaseKey[]
   group?: boolean
   startkey_docid?: string
 }

From ae356b6a505694b54b437a9587f6e3cb2b07b010 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 26 Oct 2023 15:51:28 +0100
Subject: [PATCH 061/134] Bump version of @budibase/nano to complete the
 DatabaseQueryOpts type usage.

---
 packages/backend-core/package.json | 2 +-
 packages/types/package.json        | 2 +-
 packages/types/src/sdk/db.ts       | 3 ++-
 yarn.lock                          | 8 ++++----
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index b23cd8e5b1..dc8d71b52c 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -21,7 +21,7 @@
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@budibase/nano": "10.1.2",
+    "@budibase/nano": "10.1.3",
     "@budibase/pouchdb-replication-stream": "1.2.10",
     "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",
diff --git a/packages/types/package.json b/packages/types/package.json
index 1db667e669..1b602097c7 100644
--- a/packages/types/package.json
+++ b/packages/types/package.json
@@ -15,7 +15,7 @@
   },
   "jest": {},
   "devDependencies": {
-    "@budibase/nano": "10.1.2",
+    "@budibase/nano": "10.1.3",
     "@types/koa": "2.13.4",
     "@types/node": "18.17.0",
     "@types/pouchdb": "6.4.0",
diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts
index d3a6396128..05f72f5524 100644
--- a/packages/types/src/sdk/db.ts
+++ b/packages/types/src/sdk/db.ts
@@ -54,7 +54,8 @@ export type DatabaseDeleteIndexOpts = {
   type?: string | undefined
 }
 
-export type DatabaseKey = string | number | {} | DatabaseKey[]
+type DBPrimitiveKey = string | number | {}
+export type DatabaseKey = DBPrimitiveKey | DBPrimitiveKey[]
 
 export type DatabaseQueryOpts = {
   include_docs?: boolean
diff --git a/yarn.lock b/yarn.lock
index e3629a8dbc..f573046394 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2099,10 +2099,10 @@
     striptags "^3.1.1"
     to-gfm-code-block "^0.1.1"
 
-"@budibase/nano@10.1.2":
-  version "10.1.2"
-  resolved "https://registry.yarnpkg.com/@budibase/nano/-/nano-10.1.2.tgz#10fae5a1ab39be6a81261f40e7b7ec6d21cbdd4a"
-  integrity sha512-1w+YN2n/M5aZ9hBKCP4NEjdQbT8BfCLRizkdvm0Je665eEHw3aE1hvo8mon9Ro9QuDdxj1DfDMMFnym6/QUwpQ==
+"@budibase/nano@10.1.3":
+  version "10.1.3"
+  resolved "https://registry.yarnpkg.com/@budibase/nano/-/nano-10.1.3.tgz#81b99d76b5c256a393e6ee0e284a6aecc517e4b8"
+  integrity sha512-UuhwjKCfVO+oVB0dbKpssZfTfb5k3CTrbrjqdx0kd971zzSRMFJ0TwvBB/2Z7kgOOA+Evoq4BSd747meEz21YA==
   dependencies:
     "@types/tough-cookie" "^4.0.2"
     axios "^1.1.3"

From 5266d80afe559a3f964ee8e65a926c0e2dffcf1a Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Thu, 26 Oct 2023 16:09:01 +0100
Subject: [PATCH 062/134] Fix to ensure single integer ids are parsed correctly

---
 packages/client/src/utils/buttonActions.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/client/src/utils/buttonActions.js b/packages/client/src/utils/buttonActions.js
index 18d6b3de3c..9b4640dbb4 100644
--- a/packages/client/src/utils/buttonActions.js
+++ b/packages/client/src/utils/buttonActions.js
@@ -103,7 +103,6 @@ const fetchRowHandler = async action => {
 
 const deleteRowHandler = async action => {
   const { tableId, rowId: rowConfig, notificationOverride } = action.parameters
-
   if (tableId && rowConfig) {
     try {
       let requestConfig
@@ -129,9 +128,11 @@ const deleteRowHandler = async action => {
         requestConfig = [parsedRowConfig]
       } else if (Array.isArray(parsedRowConfig)) {
         requestConfig = parsedRowConfig
+      } else if (Number.isInteger(parsedRowConfig)) {
+        requestConfig = [String(parsedRowConfig)]
       }
 
-      if (!requestConfig.length) {
+      if (!requestConfig && !parsedRowConfig) {
         notificationStore.actions.warning("No valid rows were supplied")
         return false
       }

From 9616e8e5519cc59ca7dabd286a4d97190eecdb9c Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Thu, 26 Oct 2023 16:28:12 +0100
Subject: [PATCH 063/134] Custom role navigation links (#12056)

* Refactor routing unit tests

* Filter out higher level roles in preview

* unit test

* Refactor

---------

Co-authored-by: Michael Drury <me@michaeldrury.co.uk>
---
 packages/server/src/api/controllers/role.ts   | 24 +++++++++++++++++-
 .../server/src/api/routes/tests/role.spec.js  | 20 +++++++++++++++
 .../src/api/routes/tests/routing.spec.js      | 25 ++++++++-----------
 packages/server/src/constants/screens.ts      | 14 ++++++++---
 .../server/src/tests/utilities/structures.ts  | 19 ++++++++++++--
 5 files changed, 81 insertions(+), 21 deletions(-)

diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index ed23009706..3697bbe925 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -1,4 +1,10 @@
-import { context, db as dbCore, events, roles } from "@budibase/backend-core"
+import {
+  context,
+  db as dbCore,
+  events,
+  roles,
+  Header,
+} from "@budibase/backend-core"
 import { getUserMetadataParams, InternalTables } from "../../db/utils"
 import { Database, Role, UserCtx, UserRoles } from "@budibase/types"
 import { sdk as sharedSdk } from "@budibase/shared-core"
@@ -143,4 +149,20 @@ export async function accessible(ctx: UserCtx) {
   } else {
     ctx.body = await roles.getUserRoleIdHierarchy(roleId!)
   }
+
+  // If a custom role is provided in the header, filter out higher level roles
+  const roleHeader = ctx.header?.[Header.PREVIEW_ROLE] as string
+  if (roleHeader && !Object.keys(roles.BUILTIN_ROLE_IDS).includes(roleHeader)) {
+    const inherits = (await roles.getRole(roleHeader))?.inherits
+    const orderedRoles = ctx.body.reverse()
+    let filteredRoles = [roleHeader]
+    for (let role of orderedRoles) {
+      filteredRoles = [role, ...filteredRoles]
+      if (role === inherits) {
+        break
+      }
+    }
+    filteredRoles.pop()
+    ctx.body = [roleHeader, ...filteredRoles]
+  }
 }
diff --git a/packages/server/src/api/routes/tests/role.spec.js b/packages/server/src/api/routes/tests/role.spec.js
index c8e383d5ed..d133a69d64 100644
--- a/packages/server/src/api/routes/tests/role.spec.js
+++ b/packages/server/src/api/routes/tests/role.spec.js
@@ -158,5 +158,25 @@ describe("/roles", () => {
       expect(res.body.length).toBe(1)
       expect(res.body[0]).toBe("PUBLIC")
     })
+
+    it("should not fetch higher level accessible roles when a custom role header is provided", async () => {
+      await createRole({
+        name: `CUSTOM_ROLE`,
+        inherits: roles.BUILTIN_ROLE_IDS.BASIC,
+        permissionId: permissions.BuiltinPermissionID.READ_ONLY,
+        version: "name",
+      })
+      const res = await request
+        .get("/api/roles/accessible")
+        .set({
+          ...config.defaultHeaders(),
+          "x-budibase-role": "CUSTOM_ROLE"
+        })
+        .expect(200)
+      expect(res.body.length).toBe(3)
+      expect(res.body[0]).toBe("CUSTOM_ROLE")
+      expect(res.body[1]).toBe("BASIC")
+      expect(res.body[2]).toBe("PUBLIC")
+    })
   })
 })
diff --git a/packages/server/src/api/routes/tests/routing.spec.js b/packages/server/src/api/routes/tests/routing.spec.js
index ff6d7aba1d..4076f4879c 100644
--- a/packages/server/src/api/routes/tests/routing.spec.js
+++ b/packages/server/src/api/routes/tests/routing.spec.js
@@ -1,5 +1,5 @@
 const setup = require("./utilities")
-const { basicScreen } = setup.structures
+const { basicScreen, powerScreen } = setup.structures
 const { checkBuilderEndpoint, runInProd } = require("./utilities/TestFunctions")
 const { roles } = require("@budibase/backend-core")
 const { BUILTIN_ROLE_IDS } = roles
@@ -12,19 +12,14 @@ const route = "/test"
 describe("/routing", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
-  let screen, screen2
+  let basic, power
 
   afterAll(setup.afterAll)
 
   beforeAll(async () => {
     await config.init()
-    screen = basicScreen()
-    screen.routing.route = route
-    screen = await config.createScreen(screen)
-    screen2 = basicScreen()
-    screen2.routing.roleId = BUILTIN_ROLE_IDS.POWER
-    screen2.routing.route = route
-    screen2 = await config.createScreen(screen2)
+    basic = await config.createScreen(basicScreen(route))
+    power = await config.createScreen(powerScreen(route))
     await config.publish()
   })
 
@@ -61,8 +56,8 @@ describe("/routing", () => {
       expect(res.body.routes[route]).toEqual({
         subpaths: {
           [route]: {
-            screenId: screen._id,
-            roleId: screen.routing.roleId
+            screenId: basic._id,
+            roleId: basic.routing.roleId
           }
         }
       })
@@ -80,8 +75,8 @@ describe("/routing", () => {
       expect(res.body.routes[route]).toEqual({
         subpaths: {
           [route]: {
-            screenId: screen2._id,
-            roleId: screen2.routing.roleId
+            screenId: power._id,
+            roleId: power.routing.roleId
           }
         }
       })
@@ -101,8 +96,8 @@ describe("/routing", () => {
       expect(res.body.routes).toBeDefined()
       expect(res.body.routes[route].subpaths[route]).toBeDefined()
       const subpath = res.body.routes[route].subpaths[route]
-      expect(subpath.screens[screen2.routing.roleId]).toEqual(screen2._id)
-      expect(subpath.screens[screen.routing.roleId]).toEqual(screen._id)
+      expect(subpath.screens[power.routing.roleId]).toEqual(power._id)
+      expect(subpath.screens[basic.routing.roleId]).toEqual(basic._id)
     })
 
     it("make sure it is a builder only endpoint", async () => {
diff --git a/packages/server/src/constants/screens.ts b/packages/server/src/constants/screens.ts
index 23e36a65b8..6c88b0f957 100644
--- a/packages/server/src/constants/screens.ts
+++ b/packages/server/src/constants/screens.ts
@@ -1,7 +1,15 @@
 import { roles } from "@budibase/backend-core"
 import { BASE_LAYOUT_PROP_IDS } from "./layouts"
 
-export function createHomeScreen() {
+export function createHomeScreen(
+  config: {
+    roleId: string
+    route: string
+  } = {
+    roleId: roles.BUILTIN_ROLE_IDS.BASIC,
+    route: "/",
+  }
+) {
   return {
     description: "",
     url: "",
@@ -40,8 +48,8 @@ export function createHomeScreen() {
       gap: "M",
     },
     routing: {
-      route: "/",
-      roleId: roles.BUILTIN_ROLE_IDS.BASIC,
+      route: config.route,
+      roleId: config.roleId,
     },
     name: "home-screen",
   }
diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts
index d3e92ea34d..6d236062a8 100644
--- a/packages/server/src/tests/utilities/structures.ts
+++ b/packages/server/src/tests/utilities/structures.ts
@@ -20,6 +20,7 @@ import {
   SourceName,
   Table,
 } from "@budibase/types"
+const { BUILTIN_ROLE_IDS } = roles
 
 export function basicTable(): Table {
   return {
@@ -322,8 +323,22 @@ export function basicUser(role: string) {
   }
 }
 
-export function basicScreen() {
-  return createHomeScreen()
+export function basicScreen(route: string = "/") {
+  return createHomeScreen({
+    roleId: BUILTIN_ROLE_IDS.BASIC,
+    route,
+  })
+}
+
+export function powerScreen(route: string = "/") {
+  return createHomeScreen({
+    roleId: BUILTIN_ROLE_IDS.POWER,
+    route,
+  })
+}
+
+export function customScreen(config: { roleId: string; route: string }) {
+  return createHomeScreen(config)
 }
 
 export function basicLayout() {

From 19eaafd9465eb886b718de7c5139e1f7f19e0d82 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 26 Oct 2023 16:32:34 +0100
Subject: [PATCH 064/134] Fixing some test cases.

---
 .../server/src/api/controllers/table/index.ts |  9 ++-
 .../server/src/api/routes/tests/table.spec.ts |  3 +-
 packages/server/src/environment.ts            |  1 -
 .../src/integrations/tests/utils/postgres.ts  | 64 +++++++++++--------
 .../src/tests/utilities/TestConfiguration.ts  | 16 ++---
 .../src/utilities/rowProcessor/index.ts       |  2 +-
 6 files changed, 53 insertions(+), 42 deletions(-)

diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index 99b1cd73c6..c68b6e9923 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -10,6 +10,7 @@ import { events } from "@budibase/backend-core"
 import {
   BulkImportRequest,
   BulkImportResponse,
+  DocumentType,
   FetchTablesResponse,
   MigrateRequest,
   MigrateResponse,
@@ -20,6 +21,7 @@ import {
   TableResponse,
   TableSourceType,
   UserCtx,
+  SEPARATOR,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"
@@ -30,7 +32,12 @@ function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
   if (table && !tableId) {
     tableId = table._id
   }
-  if (table && table.sourceType === TableSourceType.EXTERNAL) {
+  if (
+    table?.sourceId &&
+    table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR)
+  ) {
+    return external
+  } else if (table?.sourceType === TableSourceType.EXTERNAL) {
     return external
   } else if (tableId && isExternalTable(tableId)) {
     return external
diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts
index 5392954353..3767e5e912 100644
--- a/packages/server/src/api/routes/tests/table.spec.ts
+++ b/packages/server/src/api/routes/tests/table.spec.ts
@@ -245,7 +245,8 @@ describe("/tables", () => {
         .expect(200)
       const fetchedTable = res.body[0]
       expect(fetchedTable.name).toEqual(testTable.name)
-      expect(fetchedTable.type).toEqual("internal")
+      expect(fetchedTable.type).toEqual("table")
+      expect(fetchedTable.sourceType).toEqual("internal")
     })
 
     it("should apply authorization to endpoint", async () => {
diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts
index a1701535ce..91424113ac 100644
--- a/packages/server/src/environment.ts
+++ b/packages/server/src/environment.ts
@@ -75,7 +75,6 @@ const environment = {
   },
   isTest: coreEnv.isTest,
   isJest: coreEnv.isJest,
-
   isDev: coreEnv.isDev,
   isProd: () => {
     return !coreEnv.isDev()
diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts
index b749551721..f65d33e3e0 100644
--- a/packages/server/src/integrations/tests/utils/postgres.ts
+++ b/packages/server/src/integrations/tests/utils/postgres.ts
@@ -1,39 +1,47 @@
 import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+import env from "../../../environment"
 
 let container: StartedTestContainer | undefined
 
+const isMac = process.platform === "darwin"
+
 export async function getDsConfig(): Promise<Datasource> {
-  if (!container) {
-    container = await new GenericContainer("postgres")
-      .withExposedPorts(5432)
-      .withEnv("POSTGRES_PASSWORD", "password")
-      .withWaitStrategy(
-        Wait.forLogMessage(
-          "PostgreSQL init process complete; ready for start up."
+  try {
+    if (!container) {
+      // postgres 15-bullseye safer bet on Linux
+      const version = isMac ? undefined : "15-bullseye"
+      container = await new GenericContainer("postgres", version)
+        .withExposedPorts(5432)
+        .withEnv("POSTGRES_PASSWORD", "password")
+        .withWaitStrategy(
+          Wait.forLogMessage(
+            "PostgreSQL init process complete; ready for start up."
+          )
         )
-      )
-      .start()
-  }
+        .start()
+    }
+    const host = container.getContainerIpAddress()
+    const port = container.getMappedPort(5432)
 
-  const host = container.getContainerIpAddress()
-  const port = container.getMappedPort(5432)
-
-  return {
-    type: "datasource_plus",
-    source: SourceName.POSTGRES,
-    plus: true,
-    config: {
-      host,
-      port,
-      database: "postgres",
-      user: "postgres",
-      password: "password",
-      schema: "public",
-      ssl: false,
-      rejectUnauthorized: false,
-      ca: false,
-    },
+    return {
+      type: "datasource_plus",
+      source: SourceName.POSTGRES,
+      plus: true,
+      config: {
+        host,
+        port,
+        database: "postgres",
+        user: "postgres",
+        password: "password",
+        schema: "public",
+        ssl: false,
+        rejectUnauthorized: false,
+        ca: false,
+      },
+    }
+  } catch (err) {
+    throw new Error("**UNABLE TO CREATE TO POSTGRES CONTAINER**")
   }
 }
 
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index a397b00441..0c81ba4123 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -570,11 +570,9 @@ class TestConfiguration {
     if (!config.sourceId) {
       config.sourceId = INTERNAL_TABLE_SOURCE_ID
     }
-    if (this.datasource && !config.sourceId) {
-      config.sourceId = this.datasource._id || INTERNAL_TABLE_SOURCE_ID
-      if (this.datasource.plus) {
-        config.sourceType = TableSourceType.EXTERNAL
-      }
+    if (this.datasource?._id) {
+      config.sourceId = this.datasource._id
+      config.sourceType = TableSourceType.EXTERNAL
     }
 
     return this.updateTable(config, options)
@@ -608,11 +606,9 @@ class TestConfiguration {
       } as RelationshipFieldMetadata
     }
 
-    if (this.datasource && !tableConfig.sourceId) {
-      tableConfig.sourceId = this.datasource._id || INTERNAL_TABLE_SOURCE_ID
-      if (this.datasource.plus) {
-        tableConfig.sourceType = TableSourceType.EXTERNAL
-      }
+    if (this.datasource?._id) {
+      tableConfig.sourceId = this.datasource._id
+      tableConfig.sourceType = TableSourceType.EXTERNAL
     }
 
     return await this.createTable(tableConfig)
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index cf3875b2ea..0024e96d50 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -51,7 +51,7 @@ function getRemovedAttachmentKeys(
 /**
  * This will update any auto columns that are found on the row/table with the correct information based on
  * time now and the current logged in user making the request.
- * @param user The user to be used for an appId as well as the createdBy and createdAt fields.
+ * @param userId The user to be used for an appId as well as the createdBy and createdAt fields.
  * @param table The table which is to be used for the schema, as well as handling auto IDs incrementing.
  * @param row The row which is to be updated with information for the auto columns.
  * @param opts specific options for function to carry out optional features.

From 115a67cfd6960c55e5fa7e8f4b95fa2a31fc1669 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 26 Oct 2023 17:03:05 +0100
Subject: [PATCH 065/134] Fixing test case.

---
 .../routes/tests/__snapshots__/datasource.spec.ts.snap   | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap b/packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap
index 2894f597ab..8dc472173c 100644
--- a/packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap
+++ b/packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap
@@ -7,7 +7,7 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = `
     "entities": [
       {
         "_id": "ta_users",
-        "_rev": "1-2375e1bc58aeec664dc1b1f04ad43e44",
+        "_rev": "1-73b7912e6cbdd3d696febc60f3715844",
         "createdAt": "2020-01-01T00:00:00.000Z",
         "name": "Users",
         "primaryDisplay": "email",
@@ -21,7 +21,6 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = `
               "presence": true,
               "type": "string",
             },
-            "fieldName": "email",
             "name": "email",
             "type": "string",
           },
@@ -30,7 +29,6 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = `
               "presence": false,
               "type": "string",
             },
-            "fieldName": "firstName",
             "name": "firstName",
             "type": "string",
           },
@@ -39,7 +37,6 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = `
               "presence": false,
               "type": "string",
             },
-            "fieldName": "lastName",
             "name": "lastName",
             "type": "string",
           },
@@ -54,7 +51,6 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = `
               "presence": false,
               "type": "string",
             },
-            "fieldName": "roleId",
             "name": "roleId",
             "type": "options",
           },
@@ -67,11 +63,12 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = `
               "presence": false,
               "type": "string",
             },
-            "fieldName": "status",
             "name": "status",
             "type": "options",
           },
         },
+        "sourceId": "bb_internal",
+        "sourceType": "internal",
         "type": "table",
         "updatedAt": "2020-01-01T00:00:00.000Z",
         "views": {},

From fd29b8d9490b827f6ea5914b58ac294bff794651 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 26 Oct 2023 17:27:40 +0100
Subject: [PATCH 066/134] PR comments.

---
 packages/builder/.gitignore                                    | 3 ++-
 .../src/components/backend/DataTable/TableDataTable.svelte     | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/builder/.gitignore b/packages/builder/.gitignore
index e5c961d509..2c18582231 100644
--- a/packages/builder/.gitignore
+++ b/packages/builder/.gitignore
@@ -5,4 +5,5 @@ package-lock.json
 release/
 dist/
 routify
-.routify/
\ No newline at end of file
+.routify/
+svelte.config.js
diff --git a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
index c2932d3b10..8dd685e766 100644
--- a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
@@ -47,7 +47,7 @@
     tables.replaceTable(id, e.detail)
 
     // We need to refresh datasources when an external table changes.
-    if (e.detail?.sourceType === DB_TYPE_EXTERNAL || e.detail?.sql) {
+    if (e.detail?.sourceType === DB_TYPE_EXTERNAL) {
       await datasources.fetch()
     }
   }

From d0f989597acb2d2d5ab301d876f3903f6146e961 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 26 Oct 2023 17:27:54 +0100
Subject: [PATCH 067/134] Updating tests to be more consistent in how they pick
 to create an internal or external table.

---
 packages/server/src/api/routes/tests/row.spec.ts   |  7 ++++++-
 .../server/src/api/routes/tests/viewV2.spec.ts     |  2 +-
 .../src/sdk/app/rows/search/tests/external.spec.ts |  2 +-
 .../src/tests/utilities/TestConfiguration.ts       | 14 ++++++++++++++
 4 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 0fc9e550e3..284c60f5b1 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -140,7 +140,12 @@ describe.each([
 
   beforeAll(async () => {
     const tableConfig = generateTableConfig()
-    const table = await config.createTable(tableConfig)
+    let table
+    if (dsProvider) {
+      table = await config.createExternalTable(tableConfig)
+    } else {
+      table = await config.createTable(tableConfig)
+    }
     tableId = table._id!
   })
 
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 2f00a9f172..b03a73ddda 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -58,7 +58,7 @@ describe.each([
         },
       })
 
-      return config.createTable({
+      return config.createExternalTable({
         ...priceTable(),
         sourceId: datasource._id,
         sourceType: TableSourceType.EXTERNAL,
diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts
index d0f59cdd83..c92155230a 100644
--- a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts
+++ b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts
@@ -93,7 +93,7 @@ describe.skip("external", () => {
   describe("search", () => {
     const rows: Row[] = []
     beforeAll(async () => {
-      const table = await config.createTable({
+      const table = await config.createExternalTable({
         ...tableData,
         sourceId: externalDatasource._id,
       })
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 0c81ba4123..5b23c5c353 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -574,7 +574,21 @@ class TestConfiguration {
       config.sourceId = this.datasource._id
       config.sourceType = TableSourceType.EXTERNAL
     }
+    return this.updateTable(config, options)
+  }
 
+  async createExternalTable(
+    config?: TableToBuild,
+    options = { skipReassigning: false }
+  ) {
+    if (config != null && config._id) {
+      delete config._id
+    }
+    config = config || basicTable()
+    if (this.datasource?._id) {
+      config.sourceId = this.datasource._id
+      config.sourceType = TableSourceType.EXTERNAL
+    }
     return this.updateTable(config, options)
   }
 

From 5a80487c77f331ddf054825fa0b04d4e32cec179 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 26 Oct 2023 17:54:26 +0100
Subject: [PATCH 068/134] Fix redirect loop when accessing a group as a global
 builder but not an admin.

---
 packages/pro                                  |  2 +-
 .../api/routes/global/tests/groups.spec.ts    | 37 ++++++++++++++++++-
 .../worker/src/tests/TestConfiguration.ts     | 17 +++++++--
 .../worker/src/tests/structures/groups.ts     |  8 ++--
 4 files changed, 54 insertions(+), 10 deletions(-)

diff --git a/packages/pro b/packages/pro
index d24c0dc3a3..1911442b93 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit d24c0dc3a30014cbe61860252aa48104cad36376
+Subproject commit 1911442b93670d71edb4d9e19b8e0677bbad6c47
diff --git a/packages/worker/src/api/routes/global/tests/groups.spec.ts b/packages/worker/src/api/routes/global/tests/groups.spec.ts
index afeaae952c..8f0739a812 100644
--- a/packages/worker/src/api/routes/global/tests/groups.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/groups.spec.ts
@@ -1,7 +1,7 @@
 import { events } from "@budibase/backend-core"
 import { generator } from "@budibase/backend-core/tests"
 import { structures, TestConfiguration, mocks } from "../../../../tests"
-import { UserGroup } from "@budibase/types"
+import { User, UserGroup } from "@budibase/types"
 
 mocks.licenses.useGroups()
 
@@ -231,4 +231,39 @@ describe("/api/global/groups", () => {
       })
     })
   })
+
+  describe("with global builder role", () => {
+    let builder: User
+    let group: UserGroup
+
+    beforeAll(async () => {
+      builder = await config.createUser({
+        builder: { global: true },
+        admin: { global: false },
+      })
+      await config.createSession(builder)
+
+      let resp = await config.api.groups.saveGroup(
+        structures.groups.UserGroup()
+      )
+      group = resp.body as UserGroup
+    })
+
+    it("find should return 200", async () => {
+      await config.withUser(builder, async () => {
+        await config.api.groups.searchUsers(group._id!, {
+          emailSearch: `user1`,
+        })
+      })
+    })
+
+    it("update should return 200", async () => {
+      await config.withUser(builder, async () => {
+        await config.api.groups.updateGroupUsers(group._id!, {
+          add: [builder._id!],
+          remove: [],
+        })
+      })
+    })
+  })
 })
diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts
index 7e9792c9e3..d4fcbeebd6 100644
--- a/packages/worker/src/tests/TestConfiguration.ts
+++ b/packages/worker/src/tests/TestConfiguration.ts
@@ -190,6 +190,16 @@ class TestConfiguration {
     }
   }
 
+  async withUser(user: User, f: () => Promise<void>) {
+    const oldUser = this.user
+    this.user = user
+    try {
+      await f()
+    } finally {
+      this.user = oldUser
+    }
+  }
+
   authHeaders(user: User) {
     const authToken: AuthToken = {
       userId: user._id!,
@@ -257,9 +267,10 @@ class TestConfiguration {
     })
   }
 
-  async createUser(user?: User) {
-    if (!user) {
-      user = structures.users.user()
+  async createUser(opts?: Partial<User>) {
+    let user = structures.users.user()
+    if (user) {
+      user = { ...user, ...opts }
     }
     const response = await this._req(user, null, controllers.users.save)
     const body = response as SaveUserResponse
diff --git a/packages/worker/src/tests/structures/groups.ts b/packages/worker/src/tests/structures/groups.ts
index b0d6bb8fc0..d39dd74eb8 100644
--- a/packages/worker/src/tests/structures/groups.ts
+++ b/packages/worker/src/tests/structures/groups.ts
@@ -1,8 +1,8 @@
 import { generator } from "@budibase/backend-core/tests"
 import { db } from "@budibase/backend-core"
-import { UserGroupRoles } from "@budibase/types"
+import { UserGroup as UserGroupType, UserGroupRoles } from "@budibase/types"
 
-export const UserGroup = () => {
+export function UserGroup(): UserGroupType {
   const appsCount = generator.integer({ min: 0, max: 3 })
   const roles = Array.from({ length: appsCount }).reduce(
     (p: UserGroupRoles, v) => {
@@ -14,13 +14,11 @@ export const UserGroup = () => {
     {}
   )
 
-  let group = {
-    apps: [],
+  return {
     color: generator.color(),
     icon: generator.word(),
     name: generator.word(),
     roles: roles,
     users: [],
   }
-  return group
 }

From 983091e90161c42f112a501e782620f89dbfd3be Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 26 Oct 2023 18:21:06 +0100
Subject: [PATCH 069/134] Fix for tests - row.spec.ts needs to pick correct
 table create.

---
 .../server/src/api/routes/tests/row.spec.ts   | 68 +++++++++++--------
 .../src/tests/utilities/TestConfiguration.ts  |  2 +-
 2 files changed, 39 insertions(+), 31 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 284c60f5b1..3ae4a6c1e2 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -138,14 +138,22 @@ describe.each([
       }
     : undefined
 
-  beforeAll(async () => {
-    const tableConfig = generateTableConfig()
+  async function createTable(
+    cfg: Omit<SaveTableRequest, "sourceId" | "sourceType">,
+    opts?: { skipReassigning: boolean }
+  ) {
     let table
     if (dsProvider) {
-      table = await config.createExternalTable(tableConfig)
+      table = await config.createExternalTable(cfg, opts)
     } else {
-      table = await config.createTable(tableConfig)
+      table = await config.createTable(cfg, opts)
     }
+    return table
+  }
+
+  beforeAll(async () => {
+    const tableConfig = generateTableConfig()
+    let table = await createTable(tableConfig)
     tableId = table._id!
   })
 
@@ -174,7 +182,7 @@ describe.each([
       const queryUsage = await getQueryUsage()
 
       const tableConfig = generateTableConfig()
-      const newTable = await config.createTable(
+      const newTable = await createTable(
         {
           ...tableConfig,
           name: "TestTableAuto",
@@ -251,7 +259,7 @@ describe.each([
     })
 
     it("should list all rows for given tableId", async () => {
-      const table = await config.createTable(generateTableConfig(), {
+      const table = await createTable(generateTableConfig(), {
         skipReassigning: true,
       })
       const tableId = table._id!
@@ -332,7 +340,7 @@ describe.each([
             inclusion: ["Alpha", "Beta", "Gamma"],
           },
         }
-        const table = await config.createTable({
+        const table = await createTable({
           name: "TestTable2",
           type: "table",
           schema: {
@@ -447,7 +455,7 @@ describe.each([
 
   describe("view save", () => {
     it("views have extra data trimmed", async () => {
-      const table = await config.createTable({
+      const table = await createTable({
         type: "table",
         name: "orders",
         primary: ["OrderID"],
@@ -504,7 +512,7 @@ describe.each([
   describe("patch", () => {
     beforeAll(async () => {
       const tableConfig = generateTableConfig()
-      table = await config.createTable(tableConfig)
+      table = await createTable(tableConfig)
     })
 
     it("should update only the fields that are supplied", async () => {
@@ -558,7 +566,7 @@ describe.each([
   describe("destroy", () => {
     beforeAll(async () => {
       const tableConfig = generateTableConfig()
-      table = await config.createTable(tableConfig)
+      table = await createTable(tableConfig)
     })
 
     it("should be able to delete a row", async () => {
@@ -576,7 +584,7 @@ describe.each([
   describe("validate", () => {
     beforeAll(async () => {
       const tableConfig = generateTableConfig()
-      table = await config.createTable(tableConfig)
+      table = await createTable(tableConfig)
     })
 
     it("should return no errors on valid row", async () => {
@@ -613,7 +621,7 @@ describe.each([
   describe("bulkDelete", () => {
     beforeAll(async () => {
       const tableConfig = generateTableConfig()
-      table = await config.createTable(tableConfig)
+      table = await createTable(tableConfig)
     })
 
     it("should be able to delete a bulk set of rows", async () => {
@@ -697,7 +705,7 @@ describe.each([
     describe("fetchView", () => {
       beforeEach(async () => {
         const tableConfig = generateTableConfig()
-        table = await config.createTable(tableConfig)
+        table = await createTable(tableConfig)
       })
 
       it("should be able to fetch tables contents via 'view'", async () => {
@@ -745,7 +753,7 @@ describe.each([
   describe("fetchEnrichedRows", () => {
     beforeAll(async () => {
       const tableConfig = generateTableConfig()
-      table = await config.createTable(tableConfig)
+      table = await createTable(tableConfig)
     })
 
     it("should allow enriching some linked rows", async () => {
@@ -818,7 +826,7 @@ describe.each([
     describe("attachments", () => {
       beforeAll(async () => {
         const tableConfig = generateTableConfig()
-        table = await config.createTable(tableConfig)
+        table = await createTable(tableConfig)
       })
 
       it("should allow enriching attachment rows", async () => {
@@ -849,7 +857,7 @@ describe.each([
   describe("exportData", () => {
     beforeAll(async () => {
       const tableConfig = generateTableConfig()
-      table = await config.createTable(tableConfig)
+      table = await createTable(tableConfig)
     })
 
     it("should allow exporting all columns", async () => {
@@ -937,7 +945,7 @@ describe.each([
 
     describe("create", () => {
       it("should persist a new row with only the provided view fields", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await createTable(await userTable())
         const view = await config.createView({
           schema: {
             name: { visible: true },
@@ -972,7 +980,7 @@ describe.each([
 
     describe("patch", () => {
       it("should update only the view fields for a row", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await createTable(await userTable())
         const tableId = table._id!
         const view = await config.createView({
           schema: {
@@ -1013,7 +1021,7 @@ describe.each([
 
     describe("destroy", () => {
       it("should be able to delete a row", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await createTable(await userTable())
         const tableId = table._id!
         const view = await config.createView({
           schema: {
@@ -1037,7 +1045,7 @@ describe.each([
       })
 
       it("should be able to delete multiple rows", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await createTable(await userTable())
         const tableId = table._id!
         const view = await config.createView({
           schema: {
@@ -1102,7 +1110,7 @@ describe.each([
       }
 
       it("returns empty rows from view when no schema is passed", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await createTable(await userTable())
         const rows = await Promise.all(
           Array.from({ length: 10 }, () =>
             config.api.row.save(table._id!, { tableId: table._id })
@@ -1133,7 +1141,7 @@ describe.each([
       })
 
       it("searching respects the view filters", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await createTable(await userTable())
 
         await Promise.all(
           Array.from({ length: 10 }, () =>
@@ -1257,7 +1265,7 @@ describe.each([
 
       describe("sorting", () => {
         beforeAll(async () => {
-          const table = await config.createTable(await userTable())
+          const table = await createTable(await userTable())
           const users = [
             { name: "Alice", age: 25 },
             { name: "Bob", age: 30 },
@@ -1324,7 +1332,7 @@ describe.each([
       })
 
       it("when schema is defined, defined columns and row attributes are returned", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await createTable(await userTable())
         const rows = await Promise.all(
           Array.from({ length: 10 }, () =>
             config.api.row.save(table._id!, {
@@ -1355,7 +1363,7 @@ describe.each([
       })
 
       it("views without data can be returned", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await createTable(await userTable())
 
         const createViewResponse = await config.createView()
         const response = await config.api.viewV2.search(createViewResponse.id)
@@ -1364,7 +1372,7 @@ describe.each([
       })
 
       it("respects the limit parameter", async () => {
-        await config.createTable(await userTable())
+        await createTable(await userTable())
         await Promise.all(Array.from({ length: 10 }, () => config.createRow()))
 
         const limit = generator.integer({ min: 1, max: 8 })
@@ -1379,7 +1387,7 @@ describe.each([
       })
 
       it("can handle pagination", async () => {
-        await config.createTable(await userTable())
+        await createTable(await userTable())
         await Promise.all(Array.from({ length: 10 }, () => config.createRow()))
 
         const createViewResponse = await config.createView()
@@ -1457,7 +1465,7 @@ describe.each([
         let tableId: string
 
         beforeAll(async () => {
-          await config.createTable(await userTable())
+          await createTable(await userTable())
           await Promise.all(
             Array.from({ length: 10 }, () => config.createRow())
           )
@@ -1535,13 +1543,13 @@ describe.each([
   let o2mTable: Table
   let m2mTable: Table
   beforeAll(async () => {
-    o2mTable = await config.createTable(
+    o2mTable = await createTable(
       { ...generateTableConfig(), name: "o2m" },
       {
         skipReassigning: true,
       }
     )
-    m2mTable = await config.createTable(
+    m2mTable = await createTable(
       { ...generateTableConfig(), name: "m2m" },
       {
         skipReassigning: true,
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 5b23c5c353..18bfb670a6 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -71,7 +71,7 @@ type DefaultUserValues = {
   csrfToken: string
 }
 
-interface TableToBuild extends Omit<Table, "sourceId" | "sourceType"> {
+export interface TableToBuild extends Omit<Table, "sourceId" | "sourceType"> {
   sourceId?: string
   sourceType?: TableSourceType
 }

From d13cc80022a316762732382f2f1f7573a88a524b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 26 Oct 2023 18:23:39 +0100
Subject: [PATCH 070/134] Removing update to gitignore.

---
 packages/builder/.gitignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/.gitignore b/packages/builder/.gitignore
index 2c18582231..acd1a70579 100644
--- a/packages/builder/.gitignore
+++ b/packages/builder/.gitignore
@@ -6,4 +6,3 @@ release/
 dist/
 routify
 .routify/
-svelte.config.js

From e12cef305ae889e55327136f1a35043790e31b05 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 26 Oct 2023 18:34:21 +0100
Subject: [PATCH 071/134] Fixing export.

---
 .../server/scripts/integrations/postgres/docker-compose.yml     | 2 +-
 packages/server/src/tests/utilities/TestConfiguration.ts        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/scripts/integrations/postgres/docker-compose.yml b/packages/server/scripts/integrations/postgres/docker-compose.yml
index 88efd0301d..0e8e30ecdb 100644
--- a/packages/server/scripts/integrations/postgres/docker-compose.yml
+++ b/packages/server/scripts/integrations/postgres/docker-compose.yml
@@ -2,7 +2,7 @@ version: "3.8"
 services:
   db:
     container_name: postgres
-    image: postgres:15
+    image: postgres:15-bullseye
     restart: unless-stopped
     environment:
       POSTGRES_USER: root
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 18bfb670a6..5b23c5c353 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -71,7 +71,7 @@ type DefaultUserValues = {
   csrfToken: string
 }
 
-export interface TableToBuild extends Omit<Table, "sourceId" | "sourceType"> {
+interface TableToBuild extends Omit<Table, "sourceId" | "sourceType"> {
   sourceId?: string
   sourceType?: TableSourceType
 }

From 854ff47e1c53333a935792a2ddcb0d48fed37d32 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 26 Oct 2023 18:36:07 +0100
Subject: [PATCH 072/134] Final fix - removing external from create table
 (internal).

---
 packages/server/src/tests/utilities/TestConfiguration.ts | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 5b23c5c353..84afa0a729 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -570,10 +570,6 @@ class TestConfiguration {
     if (!config.sourceId) {
       config.sourceId = INTERNAL_TABLE_SOURCE_ID
     }
-    if (this.datasource?._id) {
-      config.sourceId = this.datasource._id
-      config.sourceType = TableSourceType.EXTERNAL
-    }
     return this.updateTable(config, options)
   }
 

From 8b9a8bf5f2ddaaa14b52c74aa00329958f7fca6a Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 27 Oct 2023 10:46:00 +0100
Subject: [PATCH 073/134] Updating pro reference.

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index d24c0dc3a3..5ed0ee2aca 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit d24c0dc3a30014cbe61860252aa48104cad36376
+Subproject commit 5ed0ee2aca9d754d80cd46bae412b24621afa47e

From 09052cb1a6bca50f18dec6578719247c06699dfc Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 27 Oct 2023 10:47:09 +0100
Subject: [PATCH 074/134] Revert "Updating pro reference."

This reverts commit 8b9a8bf5f2ddaaa14b52c74aa00329958f7fca6a.
---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 5ed0ee2aca..d24c0dc3a3 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 5ed0ee2aca9d754d80cd46bae412b24621afa47e
+Subproject commit d24c0dc3a30014cbe61860252aa48104cad36376

From ae69f9dd920a85a48fcc848bf4de1ee28eac2f6b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 27 Oct 2023 10:47:50 +0100
Subject: [PATCH 075/134] Update pro reference.

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 1911442b93..5ed0ee2aca 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 1911442b93670d71edb4d9e19b8e0677bbad6c47
+Subproject commit 5ed0ee2aca9d754d80cd46bae412b24621afa47e

From d9f52295ee88dced451edabc7d32bdd07df63815 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Fri, 27 Oct 2023 11:36:45 +0100
Subject: [PATCH 076/134] Moved the description block out of the header to
 ensure it's visible for the view mode

---
 .../components/app/blocks/form/InnerFormBlock.svelte   | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte
index e65d2cf90b..f7e9a0d2ed 100644
--- a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte
+++ b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte
@@ -220,15 +220,11 @@
               </BlockComponent>
             {/if}
           </BlockComponent>
-          {#if description}
-            <BlockComponent
-              type="text"
-              props={{ text: description }}
-              order={1}
-            />
-          {/if}
         </BlockComponent>
       {/if}
+      {#if description}
+        <BlockComponent type="text" props={{ text: description }} order={1} />
+      {/if}
       {#key fields}
         <BlockComponent type="fieldgroup" props={{ labelPosition }} order={1}>
           {#each fields as field, idx}

From 9b1a7bd8541c4d887f48d3c549628da06cf14fa5 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 27 Oct 2023 11:43:01 +0100
Subject: [PATCH 077/134] PR comments - updating pickAPI to use a common
 function.

---
 .../src/api/controllers/public/utils.ts       |  4 ++--
 .../server/src/api/controllers/row/index.ts   |  6 ++---
 .../server/src/api/controllers/table/index.ts | 19 +++++++--------
 packages/server/src/integrations/utils.ts     | 23 ++++++++++++++++---
 packages/server/src/sdk/app/rows/search.ts    |  4 ++--
 packages/server/src/sdk/app/tables/getters.ts | 12 ++++++----
 .../server/src/sdk/app/tables/migration.ts    |  4 ++--
 packages/server/src/sdk/app/tables/update.ts  |  4 ++--
 packages/server/src/sdk/app/tables/utils.ts   |  4 ++--
 packages/server/src/sdk/app/views/index.ts    |  4 ++--
 .../src/utilities/rowProcessor/index.ts       |  4 ++--
 11 files changed, 52 insertions(+), 36 deletions(-)

diff --git a/packages/server/src/api/controllers/public/utils.ts b/packages/server/src/api/controllers/public/utils.ts
index 1272fcb36a..1d67b49e0d 100644
--- a/packages/server/src/api/controllers/public/utils.ts
+++ b/packages/server/src/api/controllers/public/utils.ts
@@ -1,12 +1,12 @@
 import { context } from "@budibase/backend-core"
-import { isExternalTable } from "../../../integrations/utils"
+import { isExternalTableID } from "../../../integrations/utils"
 import { APP_PREFIX, DocumentType } from "../../../db/utils"
 
 export async function addRev(
   body: { _id?: string; _rev?: string },
   tableId?: string
 ) {
-  if (!body._id || (tableId && isExternalTable(tableId))) {
+  if (!body._id || (tableId && isExternalTableID(tableId))) {
     return body
   }
   let id = body._id
diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts
index 0ccbf5cacf..1a6747a085 100644
--- a/packages/server/src/api/controllers/row/index.ts
+++ b/packages/server/src/api/controllers/row/index.ts
@@ -1,7 +1,7 @@
 import { quotas } from "@budibase/pro"
 import * as internal from "./internal"
 import * as external from "./external"
-import { isExternalTable } from "../../../integrations/utils"
+import { isExternalTableID } from "../../../integrations/utils"
 import {
   Ctx,
   UserCtx,
@@ -30,7 +30,7 @@ import { Format } from "../view/exporters"
 export * as views from "./views"
 
 function pickApi(tableId: any) {
-  if (isExternalTable(tableId)) {
+  if (isExternalTableID(tableId)) {
     return external
   }
   return internal
@@ -227,7 +227,7 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
 export async function validate(ctx: Ctx<Row, ValidateResponse>) {
   const tableId = utils.getTableId(ctx)
   // external tables are hard to validate currently
-  if (isExternalTable(tableId)) {
+  if (isExternalTableID(tableId)) {
     ctx.body = { valid: true, errors: {} }
   } else {
     ctx.body = await sdk.rows.utils.validate({
diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index c68b6e9923..db2bd672d0 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -5,7 +5,11 @@ import {
   isSchema,
   validate as validateSchema,
 } from "../../../utilities/schema"
-import { isExternalTable, isSQL } from "../../../integrations/utils"
+import {
+  isExternalTable,
+  isExternalTableID,
+  isSQL,
+} from "../../../integrations/utils"
 import { events } from "@budibase/backend-core"
 import {
   BulkImportRequest,
@@ -29,17 +33,10 @@ import { builderSocket } from "../../../websockets"
 import { cloneDeep, isEqual } from "lodash"
 
 function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
-  if (table && !tableId) {
-    tableId = table._id
+  if (table && isExternalTable(table)) {
+    return external
   }
-  if (
-    table?.sourceId &&
-    table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR)
-  ) {
-    return external
-  } else if (table?.sourceType === TableSourceType.EXTERNAL) {
-    return external
-  } else if (tableId && isExternalTable(tableId)) {
+  if (tableId && isExternalTableID(tableId)) {
     return external
   }
   return internal
diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts
index 9e6c5ca3af..b4fff0737a 100644
--- a/packages/server/src/integrations/utils.ts
+++ b/packages/server/src/integrations/utils.ts
@@ -4,10 +4,13 @@ import {
   SearchFilters,
   Datasource,
   FieldType,
+  TableSourceType,
 } from "@budibase/types"
 import { DocumentType, SEPARATOR } from "../db/utils"
 import { InvalidColumns, NoEmptyFilterStrings } from "../constants"
 import { helpers } from "@budibase/shared-core"
+import * as external from "../api/controllers/table/external"
+import * as internal from "../api/controllers/table/internal"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
@@ -82,12 +85,26 @@ export enum SqlClient {
   ORACLE = "oracledb",
 }
 
-export function isExternalTable(tableId: string) {
+export function isExternalTableID(tableId: string) {
   return tableId.includes(DocumentType.DATASOURCE)
 }
 
-export function isInternalTable(tableId: string) {
-  return !isExternalTable(tableId)
+export function isInternalTableID(tableId: string) {
+  return !isExternalTableID(tableId)
+}
+
+export function isExternalTable(table: Table) {
+  if (
+    table?.sourceId &&
+    table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR)
+  ) {
+    return true
+  } else if (table?.sourceType === TableSourceType.EXTERNAL) {
+    return true
+  } else if (table?._id && isExternalTableID(table._id)) {
+    return true
+  }
+  return false
 }
 
 export function buildExternalTableId(datasourceId: string, tableName: string) {
diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index caf9ffea02..31f7c74601 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -1,5 +1,5 @@
 import { Row, SearchFilters, SearchParams } from "@budibase/types"
-import { isExternalTable } from "../../../integrations/utils"
+import { isExternalTableID } from "../../../integrations/utils"
 import * as internal from "./search/internal"
 import * as external from "./search/external"
 import { Format } from "../../../api/controllers/view/exporters"
@@ -12,7 +12,7 @@ export interface ViewParams {
 }
 
 function pickApi(tableId: any) {
-  if (isExternalTable(tableId)) {
+  if (isExternalTableID(tableId)) {
     return external
   }
   return internal
diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts
index 176a20a59d..c0d5fe8da8 100644
--- a/packages/server/src/sdk/app/tables/getters.ts
+++ b/packages/server/src/sdk/app/tables/getters.ts
@@ -2,7 +2,7 @@ import { context } from "@budibase/backend-core"
 import { getMultiIDParams, getTableParams } from "../../../db/utils"
 import {
   breakExternalTableId,
-  isExternalTable,
+  isExternalTableID,
   isSQL,
 } from "../../../integrations/utils"
 import {
@@ -17,7 +17,7 @@ import datasources from "../datasources"
 import sdk from "../../../sdk"
 
 export function processTable(table: Table): Table {
-  if (table._id && isExternalTable(table._id)) {
+  if (table._id && isExternalTableID(table._id)) {
     return {
       ...table,
       type: "table",
@@ -79,7 +79,7 @@ export async function getExternalTable(
 export async function getTable(tableId: string): Promise<Table> {
   const db = context.getAppDB()
   let output: Table
-  if (isExternalTable(tableId)) {
+  if (isExternalTableID(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
     const datasource = await datasources.get(datasourceId!)
     const table = await getExternalTable(datasourceId!, tableName!)
@@ -109,8 +109,10 @@ export async function getExternalTablesInDatasource(
 }
 
 export async function getTables(tableIds: string[]): Promise<Table[]> {
-  const externalTableIds = tableIds.filter(tableId => isExternalTable(tableId)),
-    internalTableIds = tableIds.filter(tableId => !isExternalTable(tableId))
+  const externalTableIds = tableIds.filter(tableId =>
+      isExternalTableID(tableId)
+    ),
+    internalTableIds = tableIds.filter(tableId => !isExternalTableID(tableId))
   let tables: Table[] = []
   if (externalTableIds.length) {
     const externalTables = await getAllExternalTables()
diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts
index 293f9184d6..7ab674bbb2 100644
--- a/packages/server/src/sdk/app/tables/migration.ts
+++ b/packages/server/src/sdk/app/tables/migration.ts
@@ -13,7 +13,7 @@ import {
   Table,
 } from "@budibase/types"
 import sdk from "../../../sdk"
-import { isExternalTable } from "../../../integrations/utils"
+import { isExternalTableID } from "../../../integrations/utils"
 import { EventType, updateLinks } from "../../../db/linkedRows"
 import { cloneDeep } from "lodash"
 
@@ -58,7 +58,7 @@ function getColumnMigrator(
   // columns in internal tables. In the future, we may want to support other
   // migrations but for now return an error if we aren't migrating a user
   // relationship.
-  if (isExternalTable(table._id!)) {
+  if (isExternalTableID(table._id!)) {
     throw new BadRequestError("External tables cannot be migrated")
   }
 
diff --git a/packages/server/src/sdk/app/tables/update.ts b/packages/server/src/sdk/app/tables/update.ts
index c59f8a8a42..5c762e628b 100644
--- a/packages/server/src/sdk/app/tables/update.ts
+++ b/packages/server/src/sdk/app/tables/update.ts
@@ -1,5 +1,5 @@
 import { Table, RenameColumn } from "@budibase/types"
-import { isExternalTable } from "../../../integrations/utils"
+import { isExternalTableID } from "../../../integrations/utils"
 import sdk from "../../index"
 import { context } from "@budibase/backend-core"
 import { isExternal } from "./utils"
@@ -14,7 +14,7 @@ export * as internal from "./internal"
 export async function saveTable(table: Table): Promise<Table> {
   const db = context.getAppDB()
   let resp: DocumentInsertResponse
-  if (isExternalTable(table._id!)) {
+  if (isExternalTableID(table._id!)) {
     const datasource = await sdk.datasources.get(table.sourceId!)
     datasource.entities![table.name] = table
     resp = await db.put(datasource)
diff --git a/packages/server/src/sdk/app/tables/utils.ts b/packages/server/src/sdk/app/tables/utils.ts
index 68a054f6c1..b8e3d888af 100644
--- a/packages/server/src/sdk/app/tables/utils.ts
+++ b/packages/server/src/sdk/app/tables/utils.ts
@@ -1,10 +1,10 @@
 import { Table, TableSourceType } from "@budibase/types"
-import { isExternalTable } from "../../../integrations/utils"
+import { isExternalTableID } from "../../../integrations/utils"
 
 export function isExternal(opts: { table?: Table; tableId?: string }): boolean {
   if (opts.table && opts.table.sourceType === TableSourceType.EXTERNAL) {
     return true
-  } else if (opts.tableId && isExternalTable(opts.tableId)) {
+  } else if (opts.tableId && isExternalTableID(opts.tableId)) {
     return true
   }
   return false
diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts
index 927f82cc68..67e7158f21 100644
--- a/packages/server/src/sdk/app/views/index.ts
+++ b/packages/server/src/sdk/app/views/index.ts
@@ -4,13 +4,13 @@ import { cloneDeep } from "lodash"
 
 import sdk from "../../../sdk"
 import * as utils from "../../../db/utils"
-import { isExternalTable } from "../../../integrations/utils"
+import { isExternalTableID } from "../../../integrations/utils"
 
 import * as internal from "./internal"
 import * as external from "./external"
 
 function pickApi(tableId: any) {
-  if (isExternalTable(tableId)) {
+  if (isExternalTableID(tableId)) {
     return external
   }
   return internal
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 0024e96d50..4a91f2c03d 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -17,7 +17,7 @@ import {
   processInputBBReferences,
   processOutputBBReferences,
 } from "./bbReferenceProcessor"
-import { isExternalTable } from "../../integrations/utils"
+import { isExternalTableID } from "../../integrations/utils"
 export * from "./utils"
 
 type AutoColumnProcessingOpts = {
@@ -267,7 +267,7 @@ export async function outputProcessing<T extends Row[] | Row>(
     )) as Row[]
   }
   // remove null properties to match internal API
-  if (isExternalTable(table._id!)) {
+  if (isExternalTableID(table._id!)) {
     for (let row of enriched) {
       for (let key of Object.keys(row)) {
         if (row[key] === null) {

From 5dd61f8994c477a4c804c939557c959e543bef17 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 27 Oct 2023 11:47:01 +0100
Subject: [PATCH 078/134] Remove APIDoc comments.

---
 packages/server/src/api/routes/row.ts   | 190 ------------------------
 packages/server/src/api/routes/table.ts | 115 --------------
 2 files changed, 305 deletions(-)

diff --git a/packages/server/src/api/routes/row.ts b/packages/server/src/api/routes/row.ts
index c29cb65eac..516bfd20c6 100644
--- a/packages/server/src/api/routes/row.ts
+++ b/packages/server/src/api/routes/row.ts
@@ -11,128 +11,24 @@ const { PermissionType, PermissionLevel } = permissions
 const router: Router = new Router()
 
 router
-  /**
-   * @api {get} /api/:sourceId/:rowId/enrich Get an enriched row
-   * @apiName Get an enriched row
-   * @apiGroup rows
-   * @apiPermission table read access
-   * @apiDescription This API is only useful when dealing with rows that have relationships.
-   * Normally when a row is a returned from the API relationships will only have the structure
-   * `{ primaryDisplay: "name", _id: ... }` but this call will return the full related rows
-   * for each relationship instead.
-   *
-   * @apiParam {string} rowId The ID of the row which is to be retrieved and enriched.
-   *
-   * @apiSuccess {object} row The response body will be the enriched row.
-   */
   .get(
     "/api/:sourceId/:rowId/enrich",
     paramSubResource("sourceId", "rowId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.fetchEnrichedRow
   )
-  /**
-   * @api {get} /api/:sourceId/rows Get all rows in a table
-   * @apiName Get all rows in a table
-   * @apiGroup rows
-   * @apiPermission table read access
-   * @apiDescription This is a deprecated endpoint that should not be used anymore, instead use the search endpoint.
-   * This endpoint gets all of the rows within the specified table - it is not heavily used
-   * due to its lack of support for pagination. With SQL tables this will retrieve up to a limit and then
-   * will simply stop.
-   *
-   * @apiParam {string} sourceId The ID of the table to retrieve all rows within.
-   *
-   * @apiSuccess {object[]} rows The response body will be an array of all rows found.
-   */
   .get(
     "/api/:sourceId/rows",
     paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.fetch
   )
-  /**
-   * @api {get} /api/:sourceId/rows/:rowId Retrieve a single row
-   * @apiName Retrieve a single row
-   * @apiGroup rows
-   * @apiPermission table read access
-   * @apiDescription This endpoint retrieves only the specified row. If you wish to retrieve
-   * a row by anything other than its _id field, use the search endpoint.
-   *
-   * @apiParam {string} sourceId The ID of the table to retrieve a row from.
-   * @apiParam {string} rowId The ID of the row to retrieve.
-   *
-   * @apiSuccess {object} body The response body will be the row that was found.
-   */
   .get(
     "/api/:sourceId/rows/:rowId",
     paramSubResource("sourceId", "rowId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.find
   )
-  /**
-   * @api {post} /api/:sourceId/search Search for rows in a table
-   * @apiName Search for rows in a table
-   * @apiGroup rows
-   * @apiPermission table read access
-   * @apiDescription This is the primary method of accessing rows in Budibase, the data provider
-   * and data UI in the builder are built atop this. All filtering, sorting and pagination is
-   * handled through this, for internal and external (datasource plus, e.g. SQL) tables.
-   *
-   * @apiParam {string} sourceId The ID of the table to retrieve rows from.
-   *
-   * @apiParam (Body) {boolean} [paginate] If pagination is required then this should be set to true,
-   * defaults to false.
-   * @apiParam (Body) {object} [query] This contains a set of filters which should be applied, if none
-   * specified then the request will be unfiltered. An example with all of the possible query
-   * options has been supplied below.
-   * @apiParam (Body) {number} [limit] This sets a limit for the number of rows that will be returned,
-   * this will be implemented at the database level if supported for performance reasons. This
-   * is useful when paginating to set exactly how many rows per page.
-   * @apiParam (Body) {string} [bookmark] If pagination is enabled then a bookmark will be returned
-   * with each successful search request, this should be supplied back to get the next page.
-   * @apiParam (Body) {object} [sort] If sort is desired this should contain the name of the column to
-   * sort on.
-   * @apiParam (Body) {string} [sortOrder] If sort is enabled then this can be either "descending" or
-   * "ascending" as required.
-   * @apiParam (Body) {string} [sortType] If sort is enabled then you must specify the type of search
-   * being used, either "string" or "number". This is only used for internal tables.
-   *
-   * @apiParamExample {json} Example:
-   * {
-   *  "tableId": "ta_70260ff0b85c467ca74364aefc46f26d",
-   *  "query": {
-   *    "string": {},
-   *    "fuzzy": {},
-   *    "range": {
-   *      "columnName": {
-   *        "high": 20,
-   *        "low": 10,
-   *      }
-   *    },
-   *    "equal": {
-   *      "columnName": "someValue"
-   *    },
-   *    "notEqual": {},
-   *    "empty": {},
-   *    "notEmpty": {},
-   *    "oneOf": {
-   *      "columnName": ["value"]
-   *    }
-   *  },
-   *  "limit": 10,
-   *  "sort": "name",
-   *  "sortOrder": "descending",
-   *  "sortType": "string",
-   *  "paginate": true
-   * }
-   *
-   * @apiSuccess {object[]} rows An array of rows that was found based on the supplied parameters.
-   * @apiSuccess {boolean} hasNextPage If pagination was enabled then this specifies whether or
-   * not there is another page after this request.
-   * @apiSuccess {string} bookmark The bookmark to be sent with the next request to get the next
-   * page.
-   */
   .post(
     "/api/:sourceId/search",
     internalSearchValidator(),
@@ -148,30 +44,6 @@ router
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.search
   )
-  /**
-   * @api {post} /api/:sourceId/rows Creates a new row
-   * @apiName Creates a new row
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This API will create a new row based on the supplied body. If the
-   * body includes an "_id" field then it will update an existing row if the field
-   * links to one. Please note that "_id", "_rev" and "tableId" are fields that are
-   * already used by Budibase tables and cannot be used for columns.
-   *
-   * @apiParam {string} sourceId The ID of the table to save a row to.
-   *
-   * @apiParam (Body) {string} [_id] If the row exists already then an ID for the row must be provided.
-   * @apiParam (Body) {string} [_rev] If working with an existing row for an internal table its revision
-   * must also be provided.
-   * @apiParam (Body) {string} tableId The ID of the table should also be specified in the row body itself.
-   * @apiParam (Body) {any} [any] Any field supplied in the body will be assessed to see if it matches
-   * a column in the specified table. All other fields will be dropped and not stored.
-   *
-   * @apiSuccess {string} _id The ID of the row that was just saved, if it was just created this
-   * is the rows new ID.
-   * @apiSuccess {string} [_rev] If saving to an internal table a revision will also be returned.
-   * @apiSuccess {object} body The contents of the row that was saved will be returned as well.
-   */
   .post(
     "/api/:sourceId/rows",
     paramResource("sourceId"),
@@ -179,14 +51,6 @@ router
     trimViewRowInfo,
     rowController.save
   )
-  /**
-   * @api {patch} /api/:sourceId/rows Updates a row
-   * @apiName Update a row
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This endpoint is identical to the row creation endpoint but instead it will
-   * error if an _id isn't provided, it will only function for existing rows.
-   */
   .patch(
     "/api/:sourceId/rows",
     paramResource("sourceId"),
@@ -194,52 +58,12 @@ router
     trimViewRowInfo,
     rowController.patch
   )
-  /**
-   * @api {post} /api/:sourceId/rows/validate Validate inputs for a row
-   * @apiName Validate inputs for a row
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription When attempting to save a row you may want to check if the row is valid
-   * given the table schema, this will iterate through all the constraints on the table and
-   * check if the request body is valid.
-   *
-   * @apiParam {string} sourceId The ID of the table the row is to be validated for.
-   *
-   * @apiParam (Body) {any} [any] Any fields provided in the request body will be tested
-   * against the table schema and constraints.
-   *
-   * @apiSuccess {boolean} valid If inputs provided are acceptable within the table schema this
-   * will be true, if it is not then then errors property will be populated.
-   * @apiSuccess {object} [errors] A key value map of information about fields on the input
-   * which do not match the table schema. The key name will be the column names that have breached
-   * the schema.
-   */
   .post(
     "/api/:sourceId/rows/validate",
     paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
     rowController.validate
   )
-  /**
-   * @api {delete} /api/:sourceId/rows Delete rows
-   * @apiName Delete rows
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This endpoint can delete a single row, or delete them in a bulk
-   * fashion.
-   *
-   * @apiParam {string} sourceId The ID of the table the row is to be deleted from.
-   *
-   * @apiParam (Body) {object[]} [rows] If bulk deletion is desired then provide the rows in this
-   * key of the request body that are to be deleted.
-   * @apiParam (Body) {string} [_id] If deleting a single row then provide its ID in this field.
-   * @apiParam (Body) {string} [_rev] If deleting a single row from an internal table then provide its
-   * revision here.
-   *
-   * @apiSuccess {object[]|object} body If deleting bulk then the response body will be an array
-   * of the deleted rows, if deleting a single row then the body will contain a "row" property which
-   * is the deleted row.
-   */
   .delete(
     "/api/:sourceId/rows",
     paramResource("sourceId"),
@@ -247,20 +71,6 @@ router
     trimViewRowInfo,
     rowController.destroy
   )
-
-  /**
-   * @api {post} /api/:sourceId/rows/exportRows Export Rows
-   * @apiName Export rows
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This API can export a number of provided rows
-   *
-   * @apiParam {string} sourceId The ID of the table the row is to be deleted from.
-   *
-   * @apiParam (Body) {object[]} [rows] The row IDs which are to be exported
-   *
-   * @apiSuccess {object[]|object}
-   */
   .post(
     "/api/:sourceId/rows/exportRows",
     paramResource("sourceId"),
diff --git a/packages/server/src/api/routes/table.ts b/packages/server/src/api/routes/table.ts
index 7ffa5acb3e..0172d9844d 100644
--- a/packages/server/src/api/routes/table.ts
+++ b/packages/server/src/api/routes/table.ts
@@ -9,99 +9,13 @@ const { BUILDER, PermissionLevel, PermissionType } = permissions
 const router: Router = new Router()
 
 router
-  /**
-   * @api {get} /api/tables Fetch all tables
-   * @apiName Fetch all tables
-   * @apiGroup tables
-   * @apiPermission table read access
-   * @apiDescription This endpoint retrieves all of the tables which have been created in
-   * an app. This includes all of the external and internal tables; to tell the difference
-   * between these look for the "type" property on each table, either being "internal" or "external".
-   *
-   * @apiSuccess {object[]} body The response body will be the list of tables that was found - as
-   * this does not take any parameters the only error scenario is no access.
-   */
   .get("/api/tables", authorized(BUILDER), tableController.fetch)
-  /**
-   * @api {get} /api/tables/:id Fetch a single table
-   * @apiName Fetch a single table
-   * @apiGroup tables
-   * @apiPermission table read access
-   * @apiDescription Retrieves a single table this could be be internal or external based on
-   * the provided table ID.
-   *
-   * @apiParam {string} id The ID of the table which is to be retrieved.
-   *
-   * @apiSuccess {object[]} body The response body will be the table that was found.
-   */
   .get(
     "/api/tables/:tableId",
     paramResource("tableId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ, { schema: true }),
     tableController.find
   )
-  /**
-   * @api {post} /api/tables Save a table
-   * @apiName Save a table
-   * @apiGroup tables
-   * @apiPermission builder
-   * @apiDescription Create or update a table with this endpoint, this will function for both internal
-   * external tables.
-   *
-   * @apiParam (Body) {string} [_id] If updating an existing table then the ID of the table must be specified.
-   * @apiParam (Body) {string} [_rev] If updating an existing internal table then the revision must also be specified.
-   * @apiParam (Body) {string} type] This should either be "internal" or "external" depending on the table type -
-   * this will default to internal.
-   * @apiParam (Body) {string} [sourceId] If creating an external table then this should be set to the datasource ID. If
-   * building an internal table this does not need to be set, although it will be returned as "bb_internal".
-   * @apiParam (Body) {string} name The name of the table, this will be used in the UI. To rename the table simply
-   * supply the table structure to this endpoint with the name changed.
-   * @apiParam (Body) {object} schema A key value object which has all of the columns in the table as the keys in this
-   * object. For each column a "type" and "constraints" must be specified, with some types requiring further information.
-   * More information about the schema structure can be found in the Typescript definitions.
-   * @apiParam (Body) {string} [primaryDisplay] The name of the column which should be used when displaying rows
-   * from this table as relationships.
-   * @apiParam (Body) {object[]} [indexes] Specifies the search indexes - this is deprecated behaviour with the introduction
-   * of lucene indexes. This functionality is only available for internal tables.
-   * @apiParam (Body) {object} [_rename] If a column is to be renamed then the "old" column name should be set in this
-   * structure, and the "updated", new column name should also be supplied. The schema should also be updated, this field
-   * lets the server know that a field hasn't just been deleted, that the data has moved to a new name, this will fix
-   * the rows in the table. This functionality is only available for internal tables.
-   * @apiParam (Body) {object[]} [rows] When creating a table using a compatible data source, an array of objects to be imported into the new table can be provided.
-   *
-   * @apiParamExample {json} Example:
-   * {
-   *   "_id": "ta_05541307fa0f4044abee071ca2a82119",
-   *   "_rev": "10-0fbe4e78f69b255d79f1017e2eeef807",
-   *   "type": "internal",
-   *   "views": {},
-   *   "name": "tableName",
-   *   "schema": {
-   *     "column": {
-   *       "type": "string",
-   *       "constraints": {
-   *         "type": "string",
-   *         "length": {
-   *           "maximum": null
-   *         },
-   *         "presence": false
-   *       },
-   *       "name": "column"
-   *     },
-   *   },
-   *   "primaryDisplay": "column",
-   *   "indexes": [],
-   *   "sourceId": "bb_internal",
-   *   "_rename": {
-   *     "old": "columnName",
-   *     "updated": "newColumnName",
-   *   },
-   *   "rows": []
-   * }
-   *
-   * @apiSuccess {object} table The response body will contain the table structure after being cleaned up and
-   * saved to the database.
-   */
   .post(
     "/api/tables",
     // allows control over updating a table
@@ -125,41 +39,12 @@ router
     authorized(BUILDER),
     tableController.validateExistingTableImport
   )
-  /**
-   * @api {post} /api/tables/:tableId/:revId Delete a table
-   * @apiName Delete a table
-   * @apiGroup tables
-   * @apiPermission builder
-   * @apiDescription This endpoint will delete a table and all of its associated data, for this reason it is
-   * quite dangerous - it will work for internal and external tables.
-   *
-   * @apiParam {string} tableId The ID of the table which is to be deleted.
-   * @apiParam {string} [revId] If deleting an internal table then the revision must also be supplied (_rev), for
-   * external tables this can simply be set to anything, e.g. "external".
-   *
-   * @apiSuccess {string} message A message stating that the table was deleted successfully.
-   */
   .delete(
     "/api/tables/:tableId/:revId",
     paramResource("tableId"),
     authorized(BUILDER),
     tableController.destroy
   )
-  /**
-   * @api {post} /api/tables/:tableId/:revId Import CSV to existing table
-   * @apiName Import CSV to existing table
-   * @apiGroup tables
-   * @apiPermission builder
-   * @apiDescription This endpoint will import data to existing tables, internal or external. It is used in combination
-   * with the CSV validation endpoint. Take the output of the CSV validation endpoint and pass it to this endpoint to
-   * import the data; please note this will only import fields that already exist on the table/match the type.
-   *
-   * @apiParam {string} tableId The ID of the table which the data should be imported to.
-   *
-   * @apiParam (Body) {object[]} rows An array of objects representing the rows to be imported, key-value pairs not matching the table schema will be ignored.
-   *
-   * @apiSuccess {string} message A message stating that the data was imported successfully.
-   */
   .post(
     "/api/tables/:tableId/import",
     paramResource("tableId"),

From 45543cbc038f5c68e55c925d7c79e100bd505eaf Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 27 Oct 2023 15:59:31 +0100
Subject: [PATCH 079/134] Catch a few more edge cases with column names, add
 tests for them.

---
 packages/backend-core/src/db/constants.ts     |   4 +
 .../grid/controls/MigrationModal.svelte       |   3 +
 .../server/src/api/routes/tests/table.spec.ts | 124 +++++++++++++++---
 .../server/src/sdk/app/tables/migration.ts    |   9 ++
 4 files changed, 123 insertions(+), 17 deletions(-)

diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts
index aea485e3e3..bfa7595d62 100644
--- a/packages/backend-core/src/db/constants.ts
+++ b/packages/backend-core/src/db/constants.ts
@@ -8,3 +8,7 @@ export const CONSTANT_INTERNAL_ROW_COLS = [
 ] as const
 
 export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const
+
+export function isInternalColumnName(name: string): boolean {
+  return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name)
+}
diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
index 1957c3259f..ecef009fe0 100644
--- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
+++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte
@@ -17,6 +17,9 @@
   $: error = checkNewColumnName(newColumnName)
 
   const checkNewColumnName = newColumnName => {
+    if (newColumnName === "") {
+      return "Column name can't be empty."
+    }
     if (newColumnName in $definition.schema) {
       return "New column name can't be the same as an existing column name."
     }
diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts
index 3767e5e912..c239c596fe 100644
--- a/packages/server/src/api/routes/tests/table.spec.ts
+++ b/packages/server/src/api/routes/tests/table.spec.ts
@@ -10,11 +10,13 @@ import {
   SaveTableRequest,
   Table,
   TableSourceType,
+  User,
   ViewCalculation,
 } from "@budibase/types"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 import sdk from "../../../sdk"
+import uuid from "uuid"
 
 const { basicTable } = setup.structures
 
@@ -426,13 +428,16 @@ describe("/tables", () => {
   })
 
   describe("migrate", () => {
-    it("should successfully migrate a one-to-many user relationship to a user column", async () => {
-      const users = await Promise.all([
-        config.createUser({ email: "1@example.com" }),
-        config.createUser({ email: "2@example.com" }),
-        config.createUser({ email: "3@example.com" }),
+    let users: User[]
+    beforeAll(async () => {
+      users = await Promise.all([
+        config.createUser({ email: `${uuid.v4()}@example.com` }),
+        config.createUser({ email: `${uuid.v4()}@example.com` }),
+        config.createUser({ email: `${uuid.v4()}@example.com` }),
       ])
+    })
 
+    it("should successfully migrate a one-to-many user relationship to a user column", async () => {
       const table = await config.api.table.create({
         name: "table",
         type: "table",
@@ -488,12 +493,6 @@ describe("/tables", () => {
     })
 
     it("should successfully migrate a many-to-many user relationship to a users column", async () => {
-      const users = await Promise.all([
-        config.createUser({ email: "1@example.com" }),
-        config.createUser({ email: "2@example.com" }),
-        config.createUser({ email: "3@example.com" }),
-      ])
-
       const table = await config.api.table.create({
         name: "table",
         type: "table",
@@ -551,12 +550,6 @@ describe("/tables", () => {
     })
 
     it("should successfully migrate a many-to-one user relationship to a users column", async () => {
-      const users = await Promise.all([
-        config.createUser({ email: "1@example.com" }),
-        config.createUser({ email: "2@example.com" }),
-        config.createUser({ email: "3@example.com" }),
-      ])
-
       const table = await config.api.table.create({
         name: "table",
         type: "table",
@@ -612,5 +605,102 @@ describe("/tables", () => {
         users[2]._id,
       ])
     })
+
+    describe("unhappy paths", () => {
+      let table: Table
+      beforeAll(async () => {
+        table = await config.api.table.create({
+          name: "table",
+          type: "table",
+          sourceId: INTERNAL_TABLE_SOURCE_ID,
+          sourceType: TableSourceType.INTERNAL,
+          schema: {
+            "user relationship": {
+              type: FieldType.LINK,
+              fieldName: "test",
+              name: "user relationship",
+              constraints: {
+                type: "array",
+                presence: false,
+              },
+              relationshipType: RelationshipType.MANY_TO_ONE,
+              tableId: InternalTable.USER_METADATA,
+            },
+            num: {
+              type: FieldType.NUMBER,
+              name: "num",
+              constraints: {
+                type: "number",
+                presence: false,
+              },
+            },
+          },
+        })
+      })
+
+      it("should fail if the new column name is blank", async () => {
+        await config.api.table.migrate(
+          table._id!,
+          {
+            oldColumn: table.schema["user relationship"],
+            newColumn: {
+              name: "",
+              type: FieldType.BB_REFERENCE,
+              subtype: FieldSubtype.USERS,
+            },
+          },
+          { expectStatus: 400 }
+        )
+      })
+
+      it("should fail if the new column name is a reserved name", async () => {
+        await config.api.table.migrate(
+          table._id!,
+          {
+            oldColumn: table.schema["user relationship"],
+            newColumn: {
+              name: "_id",
+              type: FieldType.BB_REFERENCE,
+              subtype: FieldSubtype.USERS,
+            },
+          },
+          { expectStatus: 400 }
+        )
+      })
+
+      it("should fail if the new column name is the same as an existing column", async () => {
+        await config.api.table.migrate(
+          table._id!,
+          {
+            oldColumn: table.schema["user relationship"],
+            newColumn: {
+              name: "num",
+              type: FieldType.BB_REFERENCE,
+              subtype: FieldSubtype.USERS,
+            },
+          },
+          { expectStatus: 400 }
+        )
+      })
+
+      it("should fail if the old column name isn't a column in the table", async () => {
+        await config.api.table.migrate(
+          table._id!,
+          {
+            oldColumn: {
+              name: "not a column",
+              type: FieldType.BB_REFERENCE,
+              subtype: FieldSubtype.USERS,
+            },
+            newColumn: {
+              name: "new column",
+              type: FieldType.BB_REFERENCE,
+              subtype: FieldSubtype.USERS,
+            },
+          },
+          { expectStatus: 400 }
+        )
+      })
+    })
   })
 })
diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts
index 7ab674bbb2..5a6b0c5bc0 100644
--- a/packages/server/src/sdk/app/tables/migration.ts
+++ b/packages/server/src/sdk/app/tables/migration.ts
@@ -16,6 +16,7 @@ import sdk from "../../../sdk"
 import { isExternalTableID } from "../../../integrations/utils"
 import { EventType, updateLinks } from "../../../db/linkedRows"
 import { cloneDeep } from "lodash"
+import { isInternalColumnName } from "@budibase/backend-core/src/db"
 
 export interface MigrationResult {
   tablesUpdated: Table[]
@@ -30,6 +31,14 @@ export async function migrate(
     throw new BadRequestError(`Column "${newColumn.name}" already exists`)
   }
 
+  if (newColumn.name === "") {
+    throw new BadRequestError(`Column name cannot be empty`)
+  }
+
+  if (isInternalColumnName(newColumn.name)) {
+    throw new BadRequestError(`Column name cannot be a reserved column name`)
+  }
+
   table.schema[newColumn.name] = newColumn
   table = await sdk.tables.saveTable(table)
 

From 2160f4e5e205729e1af2afce6cf586c3f537e6f0 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 27 Oct 2023 16:24:02 +0100
Subject: [PATCH 080/134] Add valid extension list to shared-core.

---
 packages/shared-core/src/constants.ts | 43 +++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/packages/shared-core/src/constants.ts b/packages/shared-core/src/constants.ts
index 725c246e2f..312e69c896 100644
--- a/packages/shared-core/src/constants.ts
+++ b/packages/shared-core/src/constants.ts
@@ -96,3 +96,46 @@ export enum BuilderSocketEvent {
 export const SocketSessionTTL = 60
 export const ValidQueryNameRegex = /^[^()]*$/
 export const ValidColumnNameRegex = /^[_a-zA-Z0-9\s]*$/g
+
+export const ValidFileExtensions = [
+  "avif",
+  "css",
+  "csv",
+  "docx",
+  "drawio",
+  "editorconfig",
+  "edl",
+  "enc",
+  "export",
+  "geojson",
+  "gif",
+  "htm",
+  "html",
+  "ics",
+  "iqy",
+  "jfif",
+  "jpeg",
+  "jpg",
+  "json",
+  "log",
+  "md",
+  "mid",
+  "odt",
+  "pdf",
+  "png",
+  "ris",
+  "rtf",
+  "svg",
+  "tex",
+  "toml",
+  "twig",
+  "txt",
+  "url",
+  "wav",
+  "webp",
+  "xls",
+  "xlsx",
+  "xml",
+  "yaml",
+  "yml",
+]

From 6bb6f106d53ee3e1f866e4317e73f62cba1d207b Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 27 Oct 2023 16:46:30 +0100
Subject: [PATCH 081/134] Apply valid file types to AttachmentCell.

---
 packages/bbui/src/Form/Dropzone.svelte                         | 2 ++
 .../src/components/grid/cells/AttachmentCell.svelte            | 3 +++
 packages/shared-core/src/constants.ts                          | 1 -
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/bbui/src/Form/Dropzone.svelte b/packages/bbui/src/Form/Dropzone.svelte
index 2a6fa1c57b..94742ea08d 100644
--- a/packages/bbui/src/Form/Dropzone.svelte
+++ b/packages/bbui/src/Form/Dropzone.svelte
@@ -17,6 +17,7 @@
   export let fileTags = []
   export let maximum = undefined
   export let compact = false
+  export let extensions = undefined
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -39,6 +40,7 @@
     {fileTags}
     {maximum}
     {compact}
+    {extensions}
     on:change={onChange}
   />
 </Field>
diff --git a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
index a27c31bbe5..4e2a6025e5 100644
--- a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
@@ -2,6 +2,7 @@
   import { onMount } from "svelte"
   import { getContext } from "svelte"
   import { Dropzone } from "@budibase/bbui"
+  import { ValidFileExtensions } from "@budibase/shared-core"
 
   export let value
   export let focused = false
@@ -13,6 +14,7 @@
 
   const { API, notifications } = getContext("grid")
   const imageExtensions = ["png", "tiff", "gif", "raw", "jpg", "jpeg"]
+  const validExtensions = ValidFileExtensions.map(ext => `.${ext}`).join(", ")
 
   let isOpen = false
 
@@ -96,6 +98,7 @@
       {value}
       compact
       on:change={e => onChange(e.detail)}
+      extensions={validExtensions}
       {processFiles}
       {deleteAttachments}
       {handleFileTooLarge}
diff --git a/packages/shared-core/src/constants.ts b/packages/shared-core/src/constants.ts
index 312e69c896..e7c6feb20a 100644
--- a/packages/shared-core/src/constants.ts
+++ b/packages/shared-core/src/constants.ts
@@ -96,7 +96,6 @@ export enum BuilderSocketEvent {
 export const SocketSessionTTL = 60
 export const ValidQueryNameRegex = /^[^()]*$/
 export const ValidColumnNameRegex = /^[_a-zA-Z0-9\s]*$/g
-
 export const ValidFileExtensions = [
   "avif",
   "css",

From 5539ff9c9ce292d844c7d88b4a8384e45310b259 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 27 Oct 2023 16:53:32 +0100
Subject: [PATCH 082/134] Apply valid file types to RowFieldControl and
 AttackmentField.

---
 .../src/components/backend/DataTable/RowFieldControl.svelte  | 5 ++++-
 .../client/src/components/app/forms/AttachmentField.svelte   | 3 ++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte b/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte
index 61b706e28e..9d52cb815e 100644
--- a/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte
+++ b/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte
@@ -13,6 +13,7 @@
   import { capitalise } from "helpers"
   import LinkedRowSelector from "components/common/LinkedRowSelector.svelte"
   import Editor from "../../integration/QueryEditor.svelte"
+  import { ValidFileExtensions } from "@budibase/shared-core"
 
   export let defaultValue
   export let meta
@@ -20,6 +21,8 @@
   export let readonly
   export let error
 
+  let validExtensions = ValidFileExtensions.map(ext => `.${ext}`).join(", ")
+
   const resolveTimeStamp = timestamp => {
     if (!timestamp) {
       return null
@@ -59,7 +62,7 @@
     bind:value
   />
 {:else if type === "attachment"}
-  <Dropzone {label} {error} bind:value />
+  <Dropzone {label} {error} extensions={validExtensions} bind:value />
 {:else if type === "boolean"}
   <Toggle text={label} {error} bind:value />
 {:else if type === "array" && meta.constraints.inclusion.length !== 0}
diff --git a/packages/client/src/components/app/forms/AttachmentField.svelte b/packages/client/src/components/app/forms/AttachmentField.svelte
index e24115ebc0..2effe607ae 100644
--- a/packages/client/src/components/app/forms/AttachmentField.svelte
+++ b/packages/client/src/components/app/forms/AttachmentField.svelte
@@ -2,13 +2,14 @@
   import Field from "./Field.svelte"
   import { CoreDropzone } from "@budibase/bbui"
   import { getContext } from "svelte"
+  import { ValidFileExtensions } from "@budibase/shared-core"
 
   export let field
   export let label
   export let disabled = false
   export let compact = false
   export let validation
-  export let extensions
+  export let extensions = ValidFileExtensions.map(ext => `.${ext}`).join(", ")
   export let onChange
   export let maximum = undefined
 

From 6ecf831f028c650d6df68fa54c9fa518e68f70ac Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 27 Oct 2023 17:10:03 +0100
Subject: [PATCH 083/134] Updating pro submodule reference.

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 5ed0ee2aca..4506399e0d 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 5ed0ee2aca9d754d80cd46bae412b24621afa47e
+Subproject commit 4506399e0d0297554cacbef1f436884aabdb9741

From f1aa32e4461b604c551915953940a990f0670404 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 27 Oct 2023 17:19:39 +0100
Subject: [PATCH 084/134] Truncate file size on the grid, validate extension in
 the attachment API.

---
 packages/bbui/src/Form/Core/Dropzone.svelte        | 10 ++++++----
 .../backend/DataTable/RowFieldControl.svelte       |  5 ++++-
 .../server/src/api/controllers/static/index.ts     | 14 ++++++++++++++
 3 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/packages/bbui/src/Form/Core/Dropzone.svelte b/packages/bbui/src/Form/Core/Dropzone.svelte
index e9ee75bd8b..0b6a9bb94f 100644
--- a/packages/bbui/src/Form/Core/Dropzone.svelte
+++ b/packages/bbui/src/Form/Core/Dropzone.svelte
@@ -159,8 +159,10 @@
           {#if selectedImage.size}
             <div class="filesize">
               {#if selectedImage.size <= BYTES_IN_MB}
-                {`${selectedImage.size / BYTES_IN_KB} KB`}
-              {:else}{`${selectedImage.size / BYTES_IN_MB} MB`}{/if}
+                {`${(selectedImage.size / BYTES_IN_KB).toFixed(1)} KB`}
+              {:else}{`${(selectedImage.size / BYTES_IN_MB).toFixed(
+                  1
+                )} MB`}{/if}
             </div>
           {/if}
           {#if !disabled}
@@ -203,8 +205,8 @@
             {#if file.size}
               <div class="filesize">
                 {#if file.size <= BYTES_IN_MB}
-                  {`${file.size / BYTES_IN_KB} KB`}
-                {:else}{`${file.size / BYTES_IN_MB} MB`}{/if}
+                  {`${(file.size / BYTES_IN_KB).toFixed(1)} KB`}
+                {:else}{`${(file.size / BYTES_IN_MB).toFixed(1)} MB`}{/if}
               </div>
             {/if}
             {#if !disabled}
diff --git a/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte b/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte
index 9d52cb815e..ea1161fe9b 100644
--- a/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte
+++ b/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte
@@ -14,6 +14,7 @@
   import LinkedRowSelector from "components/common/LinkedRowSelector.svelte"
   import Editor from "../../integration/QueryEditor.svelte"
   import { ValidFileExtensions } from "@budibase/shared-core"
+  import { admin } from "stores/portal"
 
   export let defaultValue
   export let meta
@@ -21,7 +22,9 @@
   export let readonly
   export let error
 
-  let validExtensions = ValidFileExtensions.map(ext => `.${ext}`).join(", ")
+  let validExtensions = $admin.cloud
+    ? ValidFileExtensions.map(ext => `.${ext}`).join(", ")
+    : "*"
 
   const resolveTimeStamp = timestamp => {
     if (!timestamp) {
diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index 984cb16c06..e8d403ad12 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -1,3 +1,5 @@
+import { ValidFileExtensions } from "@budibase/shared-core"
+
 require("svelte/register")
 
 import { join } from "../../../utilities/centralPath"
@@ -17,6 +19,7 @@ import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
 import { App, Ctx } from "@budibase/types"
+import environment from "../../../environment"
 
 const send = require("koa-send")
 
@@ -78,6 +81,17 @@ export const uploadFile = async function (ctx: Ctx) {
 
   const uploads = files.map(async (file: any) => {
     const fileExtension = [...file.name.split(".")].pop()
+    if (
+      !environment.SELF_HOSTED &&
+      !ValidFileExtensions.includes(fileExtension)
+    ) {
+      ctx.throw(
+        400,
+        `Invalid file extension. Valid extensions are: ${ValidFileExtensions.join(
+          ", "
+        )}`
+      )
+    }
     // filenames converted to UUIDs so they are unique
     const processedFileName = `${uuid.v4()}.${fileExtension}`
 

From 84ba840dbcfea6e059c3872999f6ac716783eee5 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 27 Oct 2023 17:29:05 +0100
Subject: [PATCH 085/134] Apply valid file type change to AttachmentField in
 cloud only.

---
 .../client/src/components/app/forms/AttachmentField.svelte   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/client/src/components/app/forms/AttachmentField.svelte b/packages/client/src/components/app/forms/AttachmentField.svelte
index 2effe607ae..e57510c770 100644
--- a/packages/client/src/components/app/forms/AttachmentField.svelte
+++ b/packages/client/src/components/app/forms/AttachmentField.svelte
@@ -3,13 +3,16 @@
   import { CoreDropzone } from "@budibase/bbui"
   import { getContext } from "svelte"
   import { ValidFileExtensions } from "@budibase/shared-core"
+  import { environmentStore } from "../../../stores/index.js"
 
   export let field
   export let label
   export let disabled = false
   export let compact = false
   export let validation
-  export let extensions = ValidFileExtensions.map(ext => `.${ext}`).join(", ")
+  export let extensions = $environmentStore.cloud
+    ? ValidFileExtensions.map(ext => `.${ext}`).join(", ")
+    : "*"
   export let onChange
   export let maximum = undefined
 

From d4929ea3b61e5774aabf4f7fe758a66b5e872f21 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 27 Oct 2023 18:03:06 +0100
Subject: [PATCH 086/134] Fixing an issue where unpublished apps with custom
 roles, when used in groups would cause users to be unable to login.

---
 packages/backend-core/src/security/roles.ts | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index b05cf79c8c..02421fd1d0 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -122,7 +122,9 @@ export async function roleToNumber(id?: string) {
   if (isBuiltin(id)) {
     return builtinRoleToNumber(id)
   }
-  const hierarchy = (await getUserRoleHierarchy(id)) as RoleDoc[]
+  const hierarchy = (await getUserRoleHierarchy(id, {
+    defaultPublic: true,
+  })) as RoleDoc[]
   for (let role of hierarchy) {
     if (isBuiltin(role?.inherits)) {
       return builtinRoleToNumber(role.inherits) + 1
@@ -177,7 +179,7 @@ export async function getRole(
     role = Object.assign(role, dbRole)
     // finalise the ID
     role._id = getExternalRoleID(role._id, role.version)
-  } catch (err) {
+  } catch (err: any) {
     if (!isBuiltin(roleId) && opts?.defaultPublic) {
       return cloneDeep(BUILTIN_ROLES.PUBLIC)
     }
@@ -192,12 +194,15 @@ export async function getRole(
 /**
  * Simple function to get all the roles based on the top level user role ID.
  */
-async function getAllUserRoles(userRoleId?: string): Promise<RoleDoc[]> {
+async function getAllUserRoles(
+  userRoleId?: string,
+  opts?: { defaultPublic?: boolean }
+): Promise<RoleDoc[]> {
   // admins have access to all roles
   if (userRoleId === BUILTIN_IDS.ADMIN) {
     return getAllRoles()
   }
-  let currentRole = await getRole(userRoleId)
+  let currentRole = await getRole(userRoleId, opts)
   let roles = currentRole ? [currentRole] : []
   let roleIds = [userRoleId]
   // get all the inherited roles
@@ -226,12 +231,16 @@ export async function getUserRoleIdHierarchy(
  * Returns an ordered array of the user's inherited role IDs, this can be used
  * to determine if a user can access something that requires a specific role.
  * @param userRoleId The user's role ID, this can be found in their access token.
+ * @param opts optional - if want to default to public use this.
  * @returns returns an ordered array of the roles, with the first being their
  * highest level of access and the last being the lowest level.
  */
-export async function getUserRoleHierarchy(userRoleId?: string) {
+export async function getUserRoleHierarchy(
+  userRoleId?: string,
+  opts?: { defaultPublic?: boolean }
+) {
   // special case, if they don't have a role then they are a public user
-  return getAllUserRoles(userRoleId)
+  return getAllUserRoles(userRoleId, opts)
 }
 
 // this function checks that the provided permissions are in an array format

From 330059991e0d6f6f0cf0af5c3ad3199c044b5509 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 27 Oct 2023 18:04:28 +0100
Subject: [PATCH 087/134] Removing any.

---
 packages/backend-core/src/security/roles.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index 02421fd1d0..0d33031de5 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -179,7 +179,7 @@ export async function getRole(
     role = Object.assign(role, dbRole)
     // finalise the ID
     role._id = getExternalRoleID(role._id, role.version)
-  } catch (err: any) {
+  } catch (err) {
     if (!isBuiltin(roleId) && opts?.defaultPublic) {
       return cloneDeep(BUILTIN_ROLES.PUBLIC)
     }

From 887383bdb6a4b54d7ab747571415f112f83f2e14 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Fri, 27 Oct 2023 18:12:21 +0100
Subject: [PATCH 088/134] Update README.md

Removing dead link.
---
 README.md | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/README.md b/README.md
index 9deb16cd4f..7827d4e48a 100644
--- a/README.md
+++ b/README.md
@@ -126,13 +126,6 @@ You can learn more about the Budibase API at the following places:
 
 - [Build an app with Budibase and Next.js](https://budibase.com/blog/building-a-crud-app-with-budibase-and-next.js/)
 
-<p align="center">
-  <img alt="Budibase data" src="https://res.cloudinary.com/daog6scxm/image/upload/v1647858558/Feb%20release/Start_building_with_Budibase_s_API_3_rhlzhv.png">
-</p>
-<br /><br />
-
-<br /><br /><br />
-
 ## 🏁 Get started
 
 Deploy Budibase self-hosted in your existing infrastructure, using Docker, Kubernetes, and Digital Ocean.

From 1221808c67c65989b24a7540ab0b31f44989e9ff Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Mon, 30 Oct 2023 09:00:20 +0100
Subject: [PATCH 089/134] Updata pro submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 39bff12817..3820c0c93a 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 39bff1281715c647f5d0c1db9bbf1d53c9fd4fc6
+Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8

From 725e3aa4ef8431990fa9a54d9857f12f95da9a5d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 10:11:06 +0100
Subject: [PATCH 090/134] Use image v2 on build:docker

---
 packages/server/package.json | 2 +-
 packages/worker/package.json | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/package.json b/packages/server/package.json
index 4a858f3be9..b89fe86a84 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -18,7 +18,7 @@
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
-    "build:docker": "yarn build && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION",
+    "build:docker": "yarn nx build && docker build ../.. -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2",
     "run:docker": "node dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
diff --git a/packages/worker/package.json b/packages/worker/package.json
index 205bf3309a..dd847e6df4 100644
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@@ -20,7 +20,7 @@
     "run:docker": "node dist/index.js",
     "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
-    "build:docker": "yarn build && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION",
+    "build:docker": "yarn nx build && docker build ../.. -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION  -f Dockerfile.v2",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
     "dev:builder": "npm run dev:stack:init && nodemon",
     "dev:built": "yarn run dev:stack:init && yarn run run:docker",

From b542040ad4fa3d379842348633fbe15546c816b0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 10:27:59 +0100
Subject: [PATCH 091/134] Use v2 for single image

---
 .github/workflows/release-singleimage.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml
index f7f87f6e4c..4d35916f4d 100644
--- a/.github/workflows/release-singleimage.yml
+++ b/.github/workflows/release-singleimage.yml
@@ -67,7 +67,7 @@ jobs:
           push: true
           platforms: linux/amd64,linux/arm64
           tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile
+          file: ./hosting/single/Dockerfile.v2
       - name: Tag and release Budibase Azure App Service docker image
         uses: docker/build-push-action@v2
         with:
@@ -76,4 +76,4 @@ jobs:
           platforms: linux/amd64
           build-args: TARGETBUILD=aas
           tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile
+          file: ./hosting/single/Dockerfile.v2

From b0ef79bbd9f9a7ab3a8bbe2f5513426ff2b61362 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 11:36:32 +0100
Subject: [PATCH 092/134] Build for both amd and arm platforms

---
 packages/server/package.json | 2 +-
 packages/worker/package.json | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/package.json b/packages/server/package.json
index b89fe86a84..c37959d33d 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -18,7 +18,7 @@
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
-    "build:docker": "yarn nx build && docker build ../.. -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2",
+    "build:docker": "yarn nx build && docker buildx build ../.. -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2 --platform linux/amd64,linux/arm64",
     "run:docker": "node dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
diff --git a/packages/worker/package.json b/packages/worker/package.json
index dd847e6df4..a391db533b 100644
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@@ -20,7 +20,7 @@
     "run:docker": "node dist/index.js",
     "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
-    "build:docker": "yarn nx build && docker build ../.. -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION  -f Dockerfile.v2",
+    "build:docker": "yarn nx build && docker buildx build ../.. -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION  -f Dockerfile.v2 --platform linux/amd64,linux/arm64",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
     "dev:builder": "npm run dev:stack:init && nodemon",
     "dev:built": "yarn run dev:stack:init && yarn run run:docker",

From 6267d9b601e7b350388b5adb139e6ef6c9cb2b16 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 11:39:25 +0100
Subject: [PATCH 093/134] Test building images

---
 .github/workflows/budibase_ci.yml | 33 +++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 77867c8617..3ec87f7244 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -42,6 +42,39 @@ jobs:
       - run: yarn --frozen-lockfile
       - run: yarn lint
 
+  test-release-images:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          fetch-depth: 0
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 18.x
+
+      - run: yarn install --frozen-lockfile
+      - name: Update versions
+        run: ./scripts/updateVersions.sh
+      - run: yarn lint
+      - run: yarn build
+      - run: yarn build:sdk
+
+      - name: "Get Current tag"
+        id: currenttag
+        run: |
+          version=$(./scripts/getCurrentVersion.sh)
+          echo "Using tag $version"
+          echo "version=$version" >> "$GITHUB_OUTPUT"
+
+      - name: Build/release Docker images
+        run: |
+          yarn lerna run --stream build:docker
+        env:
+          BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.version }}
+
   build:
     runs-on: ubuntu-latest
     steps:

From ac67a17b9393979b65f8bf4335a4f83232872e97 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 11:45:05 +0100
Subject: [PATCH 094/134] Use buildx

---
 .github/workflows/budibase_ci.yml    | 6 +++++-
 .github/workflows/release-master.yml | 3 +++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 3ec87f7244..13245a7fa1 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -51,9 +51,10 @@ jobs:
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
           fetch-depth: 0
 
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v3
         with:
           node-version: 18.x
+          cache: "yarn"
 
       - run: yarn install --frozen-lockfile
       - name: Update versions
@@ -69,6 +70,9 @@ jobs:
           echo "Using tag $version"
           echo "version=$version" >> "$GITHUB_OUTPUT"
 
+      - name: Setup Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
       - name: Build/release Docker images
         run: |
           yarn lerna run --stream build:docker
diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index 9ab8530341..4c5a3f4a1e 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -63,6 +63,9 @@ jobs:
           echo "Using tag $version"
           echo "version=$version" >> "$GITHUB_OUTPUT"
 
+      - name: Setup Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
       - name: Build/release Docker images
         run: |
           docker login -u $DOCKER_USER -p $DOCKER_PASSWORD

From 24eed537edc45d8febb9c119f5a4f54f7f296558 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 12:00:55 +0100
Subject: [PATCH 095/134] Fix timeouts

---
 packages/server/Dockerfile | 2 +-
 packages/worker/Dockerfile | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile
index e1b3b208c7..ea4c5b217a 100644
--- a/packages/server/Dockerfile
+++ b/packages/server/Dockerfile
@@ -38,7 +38,7 @@ RUN apt update && apt upgrade -y \
 
 COPY package.json .
 COPY dist/yarn.lock .
-RUN yarn install --production=true \
+RUN yarn install --production=true --network-timeout 1000000 \
     # Remove unneeded data from file system to reduce image size
     && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python \
     && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp
diff --git a/packages/worker/Dockerfile b/packages/worker/Dockerfile
index 4230ee86f8..50f1bb78b9 100644
--- a/packages/worker/Dockerfile
+++ b/packages/worker/Dockerfile
@@ -14,7 +14,7 @@ RUN yarn global add pm2
 
 COPY package.json .
 COPY dist/yarn.lock .
-RUN yarn install --production=true
+RUN yarn install --production=true --network-timeout 1000000
 # Remove unneeded data from file system to reduce image size
 RUN apk del .gyp \
     && yarn cache clean

From d9c34f3f465476291b2cad1b539e0e5e3ceb649e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 12:40:05 +0100
Subject: [PATCH 096/134] Remove build docker in ci pipelines

---
 .github/workflows/budibase_ci.yml | 37 -------------------------------
 1 file changed, 37 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 13245a7fa1..77867c8617 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -42,43 +42,6 @@ jobs:
       - run: yarn --frozen-lockfile
       - run: yarn lint
 
-  test-release-images:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          fetch-depth: 0
-
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 18.x
-          cache: "yarn"
-
-      - run: yarn install --frozen-lockfile
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - run: yarn lint
-      - run: yarn build
-      - run: yarn build:sdk
-
-      - name: "Get Current tag"
-        id: currenttag
-        run: |
-          version=$(./scripts/getCurrentVersion.sh)
-          echo "Using tag $version"
-          echo "version=$version" >> "$GITHUB_OUTPUT"
-
-      - name: Setup Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Build/release Docker images
-        run: |
-          yarn lerna run --stream build:docker
-        env:
-          BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.version }}
-
   build:
     runs-on: ubuntu-latest
     steps:

From 9229ab6896c81c0bc7d545ba3bb3ee5c70840a87 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Mon, 30 Oct 2023 12:12:56 +0000
Subject: [PATCH 097/134] Bump version to 2.12.0

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 384473120b..3179bc3b2e 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.11.45",
+  "version": "2.12.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From a141598a1cb269ab5deb2b939a861a753d8b236d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 14:26:28 +0100
Subject: [PATCH 098/134] Remove unused scripts

---
 package.json | 2 --
 1 file changed, 2 deletions(-)

diff --git a/package.json b/package.json
index d3f4903e6c..03257d2d06 100644
--- a/package.json
+++ b/package.json
@@ -56,8 +56,6 @@
     "build:specs": "lerna run --stream specs",
     "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
     "build:docker:proxy": "docker build hosting/proxy -t proxy-service",
-    "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -",
-    "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
     "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -",

From 6631a7a11d8704a075bec9fc7516ccfb08aac141 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 14:34:55 +0100
Subject: [PATCH 099/134] Push docker images via docker/build-push-action

---
 .github/workflows/release-master.yml | 39 +++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index 4c5a3f4a1e..a8af6a4da1 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -66,14 +66,47 @@ jobs:
       - name: Setup Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v1
-      - name: Build/release Docker images
+
+      - name: Docker login
         run: |
           docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build:docker
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.version }}
+
+      - name: Build docker (budibase/worker)
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64
+          build-args: |
+            BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }}
+          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
+          file: ./packages/worker/Dockerfile.v2
+          cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
+          cache-to: type=inline
+        env:
+          IMAGE_NAME: budibase/worker
+          IMAGE_TAG: ${{ inputs.image_tag }}
+          BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }}
+
+      - name: Build docker (budibase/apps)
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64
+          build-args: |
+            BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }}
+          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
+          file: ./packages/server/Dockerfile.v2
+          cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
+          cache-to: type=inline
+        env:
+          IMAGE_NAME: budibase/apps
+          IMAGE_TAG: ${{ inputs.image_tag }}
+          BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }}
 
   release-helm-chart:
     needs: [release-images]

From 0cc978f86050ffb40082013fb788fd07e81fb9b5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 14:37:33 +0100
Subject: [PATCH 100/134] Push proxy dockerfile

---
 .github/workflows/release-master.yml | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index a8af6a4da1..4a89d8462f 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -74,7 +74,7 @@ jobs:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
 
-      - name: Build docker (budibase/worker)
+      - name: Build worker docker
         uses: docker/build-push-action@v5
         with:
           context: .
@@ -91,7 +91,7 @@ jobs:
           IMAGE_TAG: ${{ inputs.image_tag }}
           BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }}
 
-      - name: Build docker (budibase/apps)
+      - name: Build server docker
         uses: docker/build-push-action@v5
         with:
           context: .
@@ -108,6 +108,19 @@ jobs:
           IMAGE_TAG: ${{ inputs.image_tag }}
           BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }}
 
+      - name: Build proxy docker
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
+          file: ./hosting/proxy/Dockerfile
+          cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
+          cache-to: type=inline
+        env:
+          IMAGE_NAME: proxy-service
+          IMAGE_TAG: ${{ inputs.image_tag }}
+
   release-helm-chart:
     needs: [release-images]
     runs-on: ubuntu-latest

From a80ea2f2c05747fe4607e1bcafac5506b6d5b4f7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 14:38:26 +0100
Subject: [PATCH 101/134] Clean scripts

---
 package.json                 | 2 --
 packages/server/package.json | 1 -
 packages/worker/package.json | 1 -
 3 files changed, 4 deletions(-)

diff --git a/package.json b/package.json
index 03257d2d06..417fb31e0e 100644
--- a/package.json
+++ b/package.json
@@ -54,8 +54,6 @@
     "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
     "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
     "build:specs": "lerna run --stream specs",
-    "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
-    "build:docker:proxy": "docker build hosting/proxy -t proxy-service",
     "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -",
diff --git a/packages/server/package.json b/packages/server/package.json
index c37959d33d..c845f7889d 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -18,7 +18,6 @@
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
-    "build:docker": "yarn nx build && docker buildx build ../.. -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2 --platform linux/amd64,linux/arm64",
     "run:docker": "node dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
diff --git a/packages/worker/package.json b/packages/worker/package.json
index a391db533b..ec86575395 100644
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@@ -20,7 +20,6 @@
     "run:docker": "node dist/index.js",
     "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
-    "build:docker": "yarn nx build && docker buildx build ../.. -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION  -f Dockerfile.v2 --platform linux/amd64,linux/arm64",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
     "dev:builder": "npm run dev:stack:init && nodemon",
     "dev:built": "yarn run dev:stack:init && yarn run run:docker",

From 0dd655e75dec40415c0e660af74dc2ee72c0802b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 14:39:03 +0100
Subject: [PATCH 102/134] Remove sh

---
 .github/workflows/release-master.yml           |  2 +-
 hosting/scripts/linux/release-to-docker-hub.sh | 18 ------------------
 2 files changed, 1 insertion(+), 19 deletions(-)
 delete mode 100755 hosting/scripts/linux/release-to-docker-hub.sh

diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index 4a89d8462f..24d71a35a6 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -118,7 +118,7 @@ jobs:
           cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
           cache-to: type=inline
         env:
-          IMAGE_NAME: proxy-service
+          IMAGE_NAME: budibase/proxy-service
           IMAGE_TAG: ${{ inputs.image_tag }}
 
   release-helm-chart:
diff --git a/hosting/scripts/linux/release-to-docker-hub.sh b/hosting/scripts/linux/release-to-docker-hub.sh
deleted file mode 100755
index 599a10f914..0000000000
--- a/hosting/scripts/linux/release-to-docker-hub.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-tag=$1
-
-if [[ ! "$tag" ]]; then
-	echo "No tag present. You must pass a tag to this script"
-	exit 1
-fi
-
-echo "Tagging images with tag: $tag"
-
-docker tag proxy-service budibase/proxy:$tag
-docker tag app-service budibase/apps:$tag
-docker tag worker-service budibase/worker:$tag
-
-docker push --all-tags budibase/apps 
-docker push --all-tags budibase/worker
-docker push --all-tags budibase/proxy

From 469114823600edb7f156fae9eef9514efed9b981 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 14:44:48 +0100
Subject: [PATCH 103/134] Cache yarn

---
 .github/workflows/release-master.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index 24d71a35a6..c2bee8da40 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -36,6 +36,7 @@ jobs:
       - uses: actions/setup-node@v1
         with:
           node-version: 18.x
+          cache: yarn
 
       - run: yarn install --frozen-lockfile
       - name: Update versions

From b74f11e0e2d23a67c6c6f30d404b2f719d4cbd32 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 14:53:14 +0100
Subject: [PATCH 104/134] Fix proxy image name

---
 .github/workflows/release-master.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index c2bee8da40..5d67101f4e 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -119,7 +119,7 @@ jobs:
           cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
           cache-to: type=inline
         env:
-          IMAGE_NAME: budibase/proxy-service
+          IMAGE_NAME: budibase/proxy
           IMAGE_TAG: ${{ inputs.image_tag }}
 
   release-helm-chart:

From ac3c9a374cbefa12e51e7dc2354128cc344f5240 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 15:02:36 +0100
Subject: [PATCH 105/134] Fix image tag

---
 .github/workflows/release-master.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index 5d67101f4e..e0adddc6a8 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -89,7 +89,7 @@ jobs:
           cache-to: type=inline
         env:
           IMAGE_NAME: budibase/worker
-          IMAGE_TAG: ${{ inputs.image_tag }}
+          IMAGE_TAG: ${{ steps.currenttag.outputs.version }}
           BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }}
 
       - name: Build server docker
@@ -106,7 +106,7 @@ jobs:
           cache-to: type=inline
         env:
           IMAGE_NAME: budibase/apps
-          IMAGE_TAG: ${{ inputs.image_tag }}
+          IMAGE_TAG: ${{ steps.currenttag.outputs.version }}
           BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }}
 
       - name: Build proxy docker
@@ -120,7 +120,7 @@ jobs:
           cache-to: type=inline
         env:
           IMAGE_NAME: budibase/proxy
-          IMAGE_TAG: ${{ inputs.image_tag }}
+          IMAGE_TAG: ${{ steps.currenttag.outputs.version }}
 
   release-helm-chart:
     needs: [release-images]

From 436d6a1585a9d904b0339e7331d24a43e3ba135f Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 30 Oct 2023 14:39:12 +0000
Subject: [PATCH 106/134] Revert frontend changes to filter out file extensions
 in the upload box.

---
 packages/bbui/src/Form/Dropzone.svelte                    | 2 --
 .../components/backend/DataTable/RowFieldControl.svelte   | 8 +-------
 .../src/components/app/forms/AttachmentField.svelte       | 6 +-----
 .../src/components/grid/cells/AttachmentCell.svelte       | 3 ---
 4 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/packages/bbui/src/Form/Dropzone.svelte b/packages/bbui/src/Form/Dropzone.svelte
index 94742ea08d..2a6fa1c57b 100644
--- a/packages/bbui/src/Form/Dropzone.svelte
+++ b/packages/bbui/src/Form/Dropzone.svelte
@@ -17,7 +17,6 @@
   export let fileTags = []
   export let maximum = undefined
   export let compact = false
-  export let extensions = undefined
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -40,7 +39,6 @@
     {fileTags}
     {maximum}
     {compact}
-    {extensions}
     on:change={onChange}
   />
 </Field>
diff --git a/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte b/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte
index ea1161fe9b..61b706e28e 100644
--- a/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte
+++ b/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte
@@ -13,8 +13,6 @@
   import { capitalise } from "helpers"
   import LinkedRowSelector from "components/common/LinkedRowSelector.svelte"
   import Editor from "../../integration/QueryEditor.svelte"
-  import { ValidFileExtensions } from "@budibase/shared-core"
-  import { admin } from "stores/portal"
 
   export let defaultValue
   export let meta
@@ -22,10 +20,6 @@
   export let readonly
   export let error
 
-  let validExtensions = $admin.cloud
-    ? ValidFileExtensions.map(ext => `.${ext}`).join(", ")
-    : "*"
-
   const resolveTimeStamp = timestamp => {
     if (!timestamp) {
       return null
@@ -65,7 +59,7 @@
     bind:value
   />
 {:else if type === "attachment"}
-  <Dropzone {label} {error} extensions={validExtensions} bind:value />
+  <Dropzone {label} {error} bind:value />
 {:else if type === "boolean"}
   <Toggle text={label} {error} bind:value />
 {:else if type === "array" && meta.constraints.inclusion.length !== 0}
diff --git a/packages/client/src/components/app/forms/AttachmentField.svelte b/packages/client/src/components/app/forms/AttachmentField.svelte
index e57510c770..e24115ebc0 100644
--- a/packages/client/src/components/app/forms/AttachmentField.svelte
+++ b/packages/client/src/components/app/forms/AttachmentField.svelte
@@ -2,17 +2,13 @@
   import Field from "./Field.svelte"
   import { CoreDropzone } from "@budibase/bbui"
   import { getContext } from "svelte"
-  import { ValidFileExtensions } from "@budibase/shared-core"
-  import { environmentStore } from "../../../stores/index.js"
 
   export let field
   export let label
   export let disabled = false
   export let compact = false
   export let validation
-  export let extensions = $environmentStore.cloud
-    ? ValidFileExtensions.map(ext => `.${ext}`).join(", ")
-    : "*"
+  export let extensions
   export let onChange
   export let maximum = undefined
 
diff --git a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
index 4e2a6025e5..a27c31bbe5 100644
--- a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
@@ -2,7 +2,6 @@
   import { onMount } from "svelte"
   import { getContext } from "svelte"
   import { Dropzone } from "@budibase/bbui"
-  import { ValidFileExtensions } from "@budibase/shared-core"
 
   export let value
   export let focused = false
@@ -14,7 +13,6 @@
 
   const { API, notifications } = getContext("grid")
   const imageExtensions = ["png", "tiff", "gif", "raw", "jpg", "jpeg"]
-  const validExtensions = ValidFileExtensions.map(ext => `.${ext}`).join(", ")
 
   let isOpen = false
 
@@ -98,7 +96,6 @@
       {value}
       compact
       on:change={e => onChange(e.detail)}
-      extensions={validExtensions}
       {processFiles}
       {deleteAttachments}
       {handleFileTooLarge}

From 344256a80588543427987e93b32c67d25766783e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 15:57:40 +0100
Subject: [PATCH 107/134] Fix proxy build

---
 .github/workflows/release-master.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index e0adddc6a8..e1feaa1e7c 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -112,7 +112,7 @@ jobs:
       - name: Build proxy docker
         uses: docker/build-push-action@v5
         with:
-          context: .
+          context: ./hosting/proxy
           push: true
           tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
           file: ./hosting/proxy/Dockerfile

From 674f40a06ee49dc1ffa7d56e8b79466e8dab7903 Mon Sep 17 00:00:00 2001
From: Samuel-Martineau <samuel@smartineau.me>
Date: Fri, 27 Oct 2023 17:21:30 -0400
Subject: [PATCH 108/134] Fix how attachment URLs are handled

---
 packages/server/src/utilities/rowProcessor/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index cf3875b2ea..604f872c81 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -241,7 +241,7 @@ export async function outputProcessing<T extends Row[] | Row>(
           continue
         }
         row[property].forEach((attachment: RowAttachment) => {
-          attachment.url = objectStore.getAppFileUrl(attachment.key)
+          attachment.url ??= objectStore.getAppFileUrl(attachment.key)
         })
       }
     } else if (

From b3a4a921aa195aacf68c5dd8dfd84f27b45e1cf1 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 30 Oct 2023 16:47:47 +0100
Subject: [PATCH 109/134] Build multi platform proxy

---
 .github/workflows/release-master.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index e1feaa1e7c..df25182cd6 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -114,6 +114,7 @@ jobs:
         with:
           context: ./hosting/proxy
           push: true
+          platforms: linux/amd64,linux/arm64
           tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
           file: ./hosting/proxy/Dockerfile
           cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest

From af59039d1c8a5735f0325c73c1c0da846c845c31 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 30 Oct 2023 16:46:27 +0000
Subject: [PATCH 110/134] Add tests for attachment processing endpoint.

---
 .../src/api/controllers/static/index.ts       | 100 ++++++++++--------
 .../src/api/routes/tests/attachment.spec.ts   |  49 +++++++++
 .../src/api/routes/tests/static.spec.js       |   8 +-
 .../src/api/routes/tests/webhook.spec.ts      |   8 +-
 .../integrations/tests/googlesheets.spec.ts   |   9 +-
 .../src/tests/utilities/TestConfiguration.ts  |  47 ++++----
 .../src/tests/utilities/api/attachment.ts     |  35 ++++++
 .../server/src/tests/utilities/api/index.ts   |   3 +
 packages/types/src/api/web/app/attachment.ts  |   9 ++
 packages/types/src/api/web/app/index.ts       |   1 +
 10 files changed, 199 insertions(+), 70 deletions(-)
 create mode 100644 packages/server/src/api/routes/tests/attachment.spec.ts
 create mode 100644 packages/server/src/tests/utilities/api/attachment.ts
 create mode 100644 packages/types/src/api/web/app/attachment.ts

diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index e8d403ad12..8fbc0db910 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -13,35 +13,21 @@ import {
 } from "../../../utilities/fileSystem"
 import env from "../../../environment"
 import { DocumentType } from "../../../db/utils"
-import { context, objectStore, utils, configs } from "@budibase/backend-core"
+import {
+  context,
+  objectStore,
+  utils,
+  configs,
+  BadRequestError,
+} from "@budibase/backend-core"
 import AWS from "aws-sdk"
 import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
-import { App, Ctx } from "@budibase/types"
-import environment from "../../../environment"
+import { App, Ctx, ProcessAttachmentResponse, Upload } from "@budibase/types"
 
 const send = require("koa-send")
 
-async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
-  const response = await objectStore.upload({
-    bucket,
-    metadata,
-    filename: s3Key,
-    path: file.path,
-    type: file.type,
-  })
-
-  // don't store a URL, work this out on the way out as the URL could change
-  return {
-    size: file.size,
-    name: file.name,
-    url: objectStore.getAppFileUrl(s3Key),
-    extension: [...file.name.split(".")].pop(),
-    key: response.Key,
-  }
-}
-
 export const toggleBetaUiFeature = async function (ctx: Ctx) {
   const cookieName = `beta:${ctx.params.feature}`
 
@@ -75,34 +61,58 @@ export const serveBuilder = async function (ctx: Ctx) {
   await send(ctx, ctx.file, { root: builderPath })
 }
 
-export const uploadFile = async function (ctx: Ctx) {
+export const uploadFile = async function (
+  ctx: Ctx<{}, ProcessAttachmentResponse>
+) {
   const file = ctx.request?.files?.file
+  if (!file) {
+    throw new BadRequestError("No file provided")
+  }
+
   let files = file && Array.isArray(file) ? Array.from(file) : [file]
 
-  const uploads = files.map(async (file: any) => {
-    const fileExtension = [...file.name.split(".")].pop()
-    if (
-      !environment.SELF_HOSTED &&
-      !ValidFileExtensions.includes(fileExtension)
-    ) {
-      ctx.throw(
-        400,
-        `Invalid file extension. Valid extensions are: ${ValidFileExtensions.join(
-          ", "
-        )}`
-      )
-    }
-    // filenames converted to UUIDs so they are unique
-    const processedFileName = `${uuid.v4()}.${fileExtension}`
+  ctx.body = await Promise.all(
+    files.map(async file => {
+      if (!file.name) {
+        throw new BadRequestError(
+          "Attempted to upload a file without a filename"
+        )
+      }
 
-    return prepareUpload({
-      file,
-      s3Key: `${context.getProdAppId()}/attachments/${processedFileName}`,
-      bucket: ObjectStoreBuckets.APPS,
+      const extension = [...file.name.split(".")].pop()
+      if (!extension) {
+        throw new BadRequestError(
+          `File "${file.name}" has no extension, an extension is required to upload a file`
+        )
+      }
+
+      if (!env.SELF_HOSTED && !ValidFileExtensions.includes(extension)) {
+        throw new BadRequestError(
+          `File "${file.name}" has an invalid extension: "${extension}"`
+        )
+      }
+
+      // filenames converted to UUIDs so they are unique
+      const processedFileName = `${uuid.v4()}.${extension}`
+
+      const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
+
+      const response = await objectStore.upload({
+        bucket: ObjectStoreBuckets.APPS,
+        filename: s3Key,
+        path: file.path,
+        type: file.type,
+      })
+
+      return {
+        size: file.size,
+        name: file.name,
+        url: objectStore.getAppFileUrl(s3Key),
+        extension,
+        key: response.Key,
+      }
     })
-  })
-
-  ctx.body = await Promise.all(uploads)
+  )
 }
 
 export const deleteObjects = async function (ctx: Ctx) {
diff --git a/packages/server/src/api/routes/tests/attachment.spec.ts b/packages/server/src/api/routes/tests/attachment.spec.ts
new file mode 100644
index 0000000000..14d2e845f6
--- /dev/null
+++ b/packages/server/src/api/routes/tests/attachment.spec.ts
@@ -0,0 +1,49 @@
+import * as setup from "./utilities"
+import { APIError } from "@budibase/types"
+
+describe("/api/applications/:appId/sync", () => {
+  let config = setup.getConfig()
+
+  afterAll(setup.afterAll)
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  describe("/api/attachments/process", () => {
+    it("should accept an image file upload", async () => {
+      let resp = await config.api.attachment.process(
+        "1px.jpg",
+        Buffer.from([0])
+      )
+      expect(resp.length).toBe(1)
+
+      let upload = resp[0]
+      expect(upload.url.endsWith(".jpg")).toBe(true)
+      expect(upload.extension).toBe("jpg")
+      expect(upload.size).toBe(1)
+      expect(upload.name).toBe("1px.jpg")
+    })
+
+    it("should reject an upload with a malicious file extension", async () => {
+      await config.withEnv({ SELF_HOSTED: undefined }, async () => {
+        let resp = (await config.api.attachment.process(
+          "ohno.exe",
+          Buffer.from([0]),
+          { expectStatus: 400 }
+        )) as unknown as APIError
+        expect(resp.message).toContain("invalid extension")
+      })
+    })
+
+    it("should reject an upload with no file", async () => {
+      let resp = (await config.api.attachment.process(
+        undefined as any,
+        undefined as any,
+        {
+          expectStatus: 400,
+        }
+      )) as unknown as APIError
+      expect(resp.message).toContain("No file provided")
+    })
+  })
+})
diff --git a/packages/server/src/api/routes/tests/static.spec.js b/packages/server/src/api/routes/tests/static.spec.js
index 13d963d057..a28d9ecd79 100644
--- a/packages/server/src/api/routes/tests/static.spec.js
+++ b/packages/server/src/api/routes/tests/static.spec.js
@@ -5,11 +5,15 @@ describe("/static", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
   let app
+  let cleanupEnv
 
-  afterAll(setup.afterAll)
+  afterAll(() => {
+    setup.afterAll()
+    cleanupEnv()
+  })
 
   beforeAll(async () => {
-    config.modeSelf()
+    cleanupEnv = config.setEnv({ SELF_HOSTED: "true" })
     app = await config.init()
   })
 
diff --git a/packages/server/src/api/routes/tests/webhook.spec.ts b/packages/server/src/api/routes/tests/webhook.spec.ts
index e7046d07c8..118bfca95f 100644
--- a/packages/server/src/api/routes/tests/webhook.spec.ts
+++ b/packages/server/src/api/routes/tests/webhook.spec.ts
@@ -8,11 +8,15 @@ describe("/webhooks", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
   let webhook: Webhook
+  let cleanupEnv: () => void
 
-  afterAll(setup.afterAll)
+  afterAll(() => {
+    setup.afterAll()
+    cleanupEnv()
+  })
 
   const setupTest = async () => {
-    config.modeSelf()
+    cleanupEnv = config.setEnv({ SELF_HOSTED: "true" })
     await config.init()
     const autoConfig = basicAutomation()
     autoConfig.definition.trigger.schema = {
diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index 748baddc39..a38c6bda45 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -35,13 +35,18 @@ import { FieldType, Table, TableSchema } from "@budibase/types"
 describe("Google Sheets Integration", () => {
   let integration: any,
     config = new TestConfiguration()
+  let cleanupEnv: () => void
 
   beforeAll(() => {
-    config.setGoogleAuth("test")
+    cleanupEnv = config.setEnv({
+      GOOGLE_CLIENT_ID: "test",
+      GOOGLE_CLIENT_SECRET: "test",
+    })
   })
 
   afterAll(async () => {
-    await config.end()
+    cleanupEnv()
+    config.end()
   })
 
   beforeEach(async () => {
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index cec8c8aa12..5096b054a6 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -58,6 +58,7 @@ import {
 } from "@budibase/types"
 
 import API from "./api"
+import { cloneDeep } from "lodash"
 
 type DefaultUserValues = {
   globalUserId: string
@@ -188,30 +189,38 @@ class TestConfiguration {
     }
   }
 
-  // MODES
-  setMultiTenancy = (value: boolean) => {
-    env._set("MULTI_TENANCY", value)
-    coreEnv._set("MULTI_TENANCY", value)
+  async withEnv(newEnvVars: Partial<typeof env>, f: () => Promise<void>) {
+    let cleanup = this.setEnv(newEnvVars)
+    try {
+      await f()
+    } finally {
+      cleanup()
+    }
   }
 
-  setSelfHosted = (value: boolean) => {
-    env._set("SELF_HOSTED", value)
-    coreEnv._set("SELF_HOSTED", value)
-  }
+  /*
+   * Sets the environment variables to the given values and returns a function
+   * that can be called to reset the environment variables to their original values.
+   */
+  setEnv(newEnvVars: Partial<typeof env>): () => void {
+    const oldEnv = cloneDeep(env)
+    const oldCoreEnv = cloneDeep(coreEnv)
 
-  setGoogleAuth = (value: string) => {
-    env._set("GOOGLE_CLIENT_ID", value)
-    env._set("GOOGLE_CLIENT_SECRET", value)
-    coreEnv._set("GOOGLE_CLIENT_ID", value)
-    coreEnv._set("GOOGLE_CLIENT_SECRET", value)
-  }
+    let key: keyof typeof newEnvVars
+    for (key in newEnvVars) {
+      env._set(key, newEnvVars[key])
+      coreEnv._set(key, newEnvVars[key])
+    }
 
-  modeCloud = () => {
-    this.setSelfHosted(false)
-  }
+    return () => {
+      for (const [key, value] of Object.entries(oldEnv)) {
+        env._set(key, value)
+      }
 
-  modeSelf = () => {
-    this.setSelfHosted(true)
+      for (const [key, value] of Object.entries(oldCoreEnv)) {
+        coreEnv._set(key, value)
+      }
+    }
   }
 
   // UTILS
diff --git a/packages/server/src/tests/utilities/api/attachment.ts b/packages/server/src/tests/utilities/api/attachment.ts
new file mode 100644
index 0000000000..a466f1a67e
--- /dev/null
+++ b/packages/server/src/tests/utilities/api/attachment.ts
@@ -0,0 +1,35 @@
+import {
+  APIError,
+  Datasource,
+  ProcessAttachmentResponse,
+} from "@budibase/types"
+import TestConfiguration from "../TestConfiguration"
+import { TestAPI } from "./base"
+import fs from "fs"
+
+export class AttachmentAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  process = async (
+    name: string,
+    file: Buffer | fs.ReadStream | string,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<ProcessAttachmentResponse> => {
+    const result = await this.request
+      .post(`/api/attachments/process`)
+      .attach("file", file, name)
+      .set(this.config.defaultHeaders())
+
+    if (result.statusCode !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          result.statusCode
+        }, body: ${JSON.stringify(result.body)}`
+      )
+    }
+
+    return result.body
+  }
+}
diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts
index fce8237760..30ef7c478d 100644
--- a/packages/server/src/tests/utilities/api/index.ts
+++ b/packages/server/src/tests/utilities/api/index.ts
@@ -7,6 +7,7 @@ import { DatasourceAPI } from "./datasource"
 import { LegacyViewAPI } from "./legacyView"
 import { ScreenAPI } from "./screen"
 import { ApplicationAPI } from "./application"
+import { AttachmentAPI } from "./attachment"
 
 export default class API {
   table: TableAPI
@@ -17,6 +18,7 @@ export default class API {
   datasource: DatasourceAPI
   screen: ScreenAPI
   application: ApplicationAPI
+  attachment: AttachmentAPI
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
@@ -27,5 +29,6 @@ export default class API {
     this.datasource = new DatasourceAPI(config)
     this.screen = new ScreenAPI(config)
     this.application = new ApplicationAPI(config)
+    this.attachment = new AttachmentAPI(config)
   }
 }
diff --git a/packages/types/src/api/web/app/attachment.ts b/packages/types/src/api/web/app/attachment.ts
new file mode 100644
index 0000000000..792bdf3885
--- /dev/null
+++ b/packages/types/src/api/web/app/attachment.ts
@@ -0,0 +1,9 @@
+export interface Upload {
+  size: number
+  name: string
+  url: string
+  extension: string
+  key: string
+}
+
+export type ProcessAttachmentResponse = Upload[]
diff --git a/packages/types/src/api/web/app/index.ts b/packages/types/src/api/web/app/index.ts
index 276d7fa7c1..f5b876009b 100644
--- a/packages/types/src/api/web/app/index.ts
+++ b/packages/types/src/api/web/app/index.ts
@@ -5,3 +5,4 @@ export * from "./view"
 export * from "./rows"
 export * from "./table"
 export * from "./permission"
+export * from "./attachment"

From db398a839e056c1db9ecaf8176291f07b72d2831 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Mon, 30 Oct 2023 16:54:07 +0000
Subject: [PATCH 111/134] Bump version to 2.12.1

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 3179bc3b2e..6df4a4c4cd 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.12.0",
+  "version": "2.12.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From ca9491ce67694db85ff941f639310b5b1c9be56a Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 30 Oct 2023 16:55:57 +0000
Subject: [PATCH 112/134] Surface error message from attachments API to user.

---
 packages/builder/src/components/common/Dropzone.svelte          | 2 +-
 .../src/components/grid/cells/AttachmentCell.svelte             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/common/Dropzone.svelte b/packages/builder/src/components/common/Dropzone.svelte
index fd2359fd91..daa6ad1807 100644
--- a/packages/builder/src/components/common/Dropzone.svelte
+++ b/packages/builder/src/components/common/Dropzone.svelte
@@ -23,7 +23,7 @@
     try {
       return await API.uploadBuilderAttachment(data)
     } catch (error) {
-      notifications.error("Failed to upload attachment")
+      notifications.error(error.message || "Failed to upload attachment")
       return []
     }
   }
diff --git a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
index a27c31bbe5..fc0001d55e 100644
--- a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
@@ -55,7 +55,7 @@
     try {
       return await API.uploadBuilderAttachment(data)
     } catch (error) {
-      $notifications.error("Failed to upload attachment")
+      $notifications.error(error.message || "Failed to upload attachment")
       return []
     }
   }

From 41d99f6a582500a500b2246968d624411e697929 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 30 Oct 2023 17:41:08 +0000
Subject: [PATCH 113/134] Updating app backup exports to not include automation
 logs as these bloat the backups.

---
 packages/backend-core/__mocks__/aws-sdk.ts    |  1 +
 .../tests/core/utilities/mocks/date.ts        |  5 ++
 packages/pro                                  |  2 +-
 packages/server/__mocks__/aws-sdk.ts          |  7 +++
 .../src/api/routes/tests/backup.spec.ts       | 37 ++++++++++----
 .../server/src/sdk/app/backups/exports.ts     |  6 ++-
 .../src/tests/utilities/TestConfiguration.ts  |  5 +-
 .../server/src/tests/utilities/api/backup.ts  | 49 +++++++++++++++++++
 .../server/src/tests/utilities/api/index.ts   |  3 ++
 packages/types/src/api/web/app/backup.ts      |  5 ++
 10 files changed, 106 insertions(+), 14 deletions(-)
 create mode 100644 packages/server/src/tests/utilities/api/backup.ts

diff --git a/packages/backend-core/__mocks__/aws-sdk.ts b/packages/backend-core/__mocks__/aws-sdk.ts
index b8d91dbaa9..e3be511d08 100644
--- a/packages/backend-core/__mocks__/aws-sdk.ts
+++ b/packages/backend-core/__mocks__/aws-sdk.ts
@@ -3,6 +3,7 @@ const mockS3 = {
   deleteObject: jest.fn().mockReturnThis(),
   deleteObjects: jest.fn().mockReturnThis(),
   createBucket: jest.fn().mockReturnThis(),
+  getObject: jest.fn().mockReturnThis(),
   listObject: jest.fn().mockReturnThis(),
   getSignedUrl: jest.fn((operation: string, params: any) => {
     return `http://s3.example.com/${params.Bucket}/${params.Key}`
diff --git a/packages/backend-core/tests/core/utilities/mocks/date.ts b/packages/backend-core/tests/core/utilities/mocks/date.ts
index f580b68349..9f33058d29 100644
--- a/packages/backend-core/tests/core/utilities/mocks/date.ts
+++ b/packages/backend-core/tests/core/utilities/mocks/date.ts
@@ -1,2 +1,7 @@
 export const MOCK_DATE = new Date("2020-01-01T00:00:00.000Z")
+
+export const MOCK_DATE_PLUS = (ms: number) => {
+  return new Date(Date.now() + ms)
+}
+
 export const MOCK_DATE_TIMESTAMP = 1577836800000
diff --git a/packages/pro b/packages/pro
index 3820c0c93a..132b080643 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8
+Subproject commit 132b080643f0b1ef1488bb8362d1c41ea0bd263f
diff --git a/packages/server/__mocks__/aws-sdk.ts b/packages/server/__mocks__/aws-sdk.ts
index 8a66f0e213..fa6d099f56 100644
--- a/packages/server/__mocks__/aws-sdk.ts
+++ b/packages/server/__mocks__/aws-sdk.ts
@@ -70,6 +70,13 @@ module AwsMock {
         Contents: {},
       })
     )
+
+    // @ts-ignore
+    this.getObject = jest.fn(
+      response({
+        Body: "",
+      })
+    )
   }
 
   aws.DynamoDB = { DocumentClient }
diff --git a/packages/server/src/api/routes/tests/backup.spec.ts b/packages/server/src/api/routes/tests/backup.spec.ts
index 92e0176060..0c62b32155 100644
--- a/packages/server/src/api/routes/tests/backup.spec.ts
+++ b/packages/server/src/api/routes/tests/backup.spec.ts
@@ -5,6 +5,8 @@ import sdk from "../../../sdk"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import { mocks } from "@budibase/backend-core/tests"
 
+mocks.licenses.useBackups()
+
 describe("/backups", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
@@ -15,13 +17,13 @@ describe("/backups", () => {
     await config.init()
   })
 
-  describe("exportAppDump", () => {
+  describe("/api/backups/export", () => {
     it("should be able to export app", async () => {
-      const res = await request
-        .post(`/api/backups/export?appId=${config.getAppId()}`)
-        .set(config.defaultHeaders())
-        .expect(200)
-      expect(res.headers["content-type"]).toEqual("application/gzip")
+      const { body, headers } = await config.api.backup.exportBasicBackup(
+        config.getAppId()!
+      )
+      expect(body instanceof Buffer).toBe(true)
+      expect(headers["content-type"]).toEqual("application/gzip")
       expect(events.app.exported).toBeCalledTimes(1)
     })
 
@@ -36,11 +38,11 @@ describe("/backups", () => {
     it("should infer the app name from the app", async () => {
       tk.freeze(mocks.date.MOCK_DATE)
 
-      const res = await request
-        .post(`/api/backups/export?appId=${config.getAppId()}`)
-        .set(config.defaultHeaders())
+      const { headers } = await config.api.backup.exportBasicBackup(
+        config.getAppId()!
+      )
 
-      expect(res.headers["content-disposition"]).toEqual(
+      expect(headers["content-disposition"]).toEqual(
         `attachment; filename="${
           config.getApp()!.name
         }-export-${mocks.date.MOCK_DATE.getTime()}.tar.gz"`
@@ -48,6 +50,21 @@ describe("/backups", () => {
     })
   })
 
+  describe("/api/backups/import", () => {
+    it("should be able to import an app", async () => {
+      const appId = config.getAppId()!
+      const automation = await config.createAutomation()
+      await config.createAutomationLog(automation, appId)
+      await config.createScreen()
+      const exportRes = await config.api.backup.createBackup(appId)
+      expect(exportRes.backupId).toBeDefined()
+      const importRes = await config.api.backup.importBackup(
+        appId,
+        exportRes.backupId
+      )
+    })
+  })
+
   describe("calculateBackupStats", () => {
     it("should be able to calculate the backup statistics", async () => {
       await config.createAutomation()
diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts
index d5ea31cdf5..6432bd2e0d 100644
--- a/packages/server/src/sdk/app/backups/exports.ts
+++ b/packages/server/src/sdk/app/backups/exports.ts
@@ -84,7 +84,11 @@ export async function exportDB(
 }
 
 function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) {
-  const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX]
+  const ids = [
+    USER_METDATA_PREFIX,
+    LINK_USER_METADATA_PREFIX,
+    AUTOMATION_LOG_PREFIX,
+  ]
   if (excludeRows) {
     ids.push(TABLE_ROW_PREFIX)
   }
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index cec8c8aa12..869a42012a 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -774,8 +774,9 @@ class TestConfiguration {
 
   // AUTOMATION LOG
 
-  async createAutomationLog(automation: Automation) {
-    return await context.doInAppContext(this.getProdAppId(), async () => {
+  async createAutomationLog(automation: Automation, appId?: string) {
+    appId = appId || this.getProdAppId()
+    return await context.doInAppContext(appId!, async () => {
       return await pro.sdk.automations.logs.storeLog(
         automation,
         basicAutomationResults(automation._id!)
diff --git a/packages/server/src/tests/utilities/api/backup.ts b/packages/server/src/tests/utilities/api/backup.ts
new file mode 100644
index 0000000000..4cee19fdc6
--- /dev/null
+++ b/packages/server/src/tests/utilities/api/backup.ts
@@ -0,0 +1,49 @@
+import {
+  CreateAppBackupResponse,
+  ImportAppBackupResponse,
+} from "@budibase/types"
+import TestConfiguration from "../TestConfiguration"
+import { TestAPI } from "./base"
+import tk from "timekeeper"
+import { mocks } from "@budibase/backend-core/tests"
+
+export class BackupAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  exportBasicBackup = async (appId: string) => {
+    const result = await this.request
+      .post(`/api/backups/export?appId=${appId}`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /application\/gzip/)
+      .expect(200)
+    return {
+      body: result.body as Buffer,
+      headers: result.headers,
+    }
+  }
+
+  createBackup = async (appId: string) => {
+    tk.freeze(mocks.date.MOCK_DATE_PLUS(1))
+    const result = await this.request
+      .post(`/api/apps/${appId}/backups`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+    return result.body as CreateAppBackupResponse
+  }
+
+  importBackup = async (
+    appId: string,
+    backupId: string
+  ): Promise<ImportAppBackupResponse> => {
+    tk.freeze(mocks.date.MOCK_DATE_PLUS(1))
+    const result = await this.request
+      .post(`/api/apps/${appId}/backups/${backupId}/import`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+    return result.body as ImportAppBackupResponse
+  }
+}
diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts
index fce8237760..c344068295 100644
--- a/packages/server/src/tests/utilities/api/index.ts
+++ b/packages/server/src/tests/utilities/api/index.ts
@@ -7,6 +7,7 @@ import { DatasourceAPI } from "./datasource"
 import { LegacyViewAPI } from "./legacyView"
 import { ScreenAPI } from "./screen"
 import { ApplicationAPI } from "./application"
+import { BackupAPI } from "./backup"
 
 export default class API {
   table: TableAPI
@@ -17,6 +18,7 @@ export default class API {
   datasource: DatasourceAPI
   screen: ScreenAPI
   application: ApplicationAPI
+  backup: BackupAPI
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
@@ -27,5 +29,6 @@ export default class API {
     this.datasource = new DatasourceAPI(config)
     this.screen = new ScreenAPI(config)
     this.application = new ApplicationAPI(config)
+    this.backup = new BackupAPI(config)
   }
 }
diff --git a/packages/types/src/api/web/app/backup.ts b/packages/types/src/api/web/app/backup.ts
index c9a8d07f5e..f77707e9c6 100644
--- a/packages/types/src/api/web/app/backup.ts
+++ b/packages/types/src/api/web/app/backup.ts
@@ -20,3 +20,8 @@ export interface CreateAppBackupResponse {
 export interface UpdateAppBackupRequest {
   name: string
 }
+
+export interface ImportAppBackupResponse {
+  restoreId: string
+  message: string
+}

From 15b1f3efe63fb8d4eceb3dc66d4c3ed3f90fd6ee Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 31 Oct 2023 10:51:46 +0000
Subject: [PATCH 114/134] Removing duplicate filtering of automation logs.

---
 packages/server/src/sdk/app/backups/exports.ts | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts
index 6432bd2e0d..c349dcb927 100644
--- a/packages/server/src/sdk/app/backups/exports.ts
+++ b/packages/server/src/sdk/app/backups/exports.ts
@@ -26,7 +26,6 @@ export interface DBDumpOpts {
 export interface ExportOpts extends DBDumpOpts {
   tar?: boolean
   excludeRows?: boolean
-  excludeLogs?: boolean
   encryptPassword?: string
 }
 
@@ -83,7 +82,7 @@ export async function exportDB(
   })
 }
 
-function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) {
+function defineFilter(excludeRows?: boolean) {
   const ids = [
     USER_METDATA_PREFIX,
     LINK_USER_METADATA_PREFIX,
@@ -92,9 +91,6 @@ function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) {
   if (excludeRows) {
     ids.push(TABLE_ROW_PREFIX)
   }
-  if (excludeLogs) {
-    ids.push(AUTOMATION_LOG_PREFIX)
-  }
   return (doc: any) =>
     !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr)
 }
@@ -122,7 +118,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
         fs.writeFileSync(join(tmpPath, path), contents)
       }
     }
-    // get all of the files
+    // get all the files
     else {
       tmpPath = await objectStore.retrieveDirectory(
         ObjectStoreBuckets.APPS,
@@ -145,7 +141,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
   // enforce an export of app DB to the tmp path
   const dbPath = join(tmpPath, DB_EXPORT_FILE)
   await exportDB(appId, {
-    filter: defineFilter(config?.excludeRows, config?.excludeLogs),
+    filter: defineFilter(config?.excludeRows),
     exportPath: dbPath,
   })
 
@@ -195,7 +191,6 @@ export async function streamExportApp({
 }) {
   const tmpPath = await exportApp(appId, {
     excludeRows,
-    excludeLogs: true,
     tar: true,
     encryptPassword,
   })

From 88cc8a19eef23af0ace19b1fab3014d70cc44ba5 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 31 Oct 2023 11:19:53 +0000
Subject: [PATCH 115/134] Resetting timekeeper and resetting pro back to
 normal.

---
 packages/backend-core/tests/core/utilities/mocks/date.ts | 4 ----
 packages/pro                                             | 2 +-
 packages/server/src/api/routes/tests/backup.spec.ts      | 1 +
 packages/server/src/tests/utilities/api/backup.ts        | 4 ----
 4 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/packages/backend-core/tests/core/utilities/mocks/date.ts b/packages/backend-core/tests/core/utilities/mocks/date.ts
index 9f33058d29..1e6d105d93 100644
--- a/packages/backend-core/tests/core/utilities/mocks/date.ts
+++ b/packages/backend-core/tests/core/utilities/mocks/date.ts
@@ -1,7 +1,3 @@
 export const MOCK_DATE = new Date("2020-01-01T00:00:00.000Z")
 
-export const MOCK_DATE_PLUS = (ms: number) => {
-  return new Date(Date.now() + ms)
-}
-
 export const MOCK_DATE_TIMESTAMP = 1577836800000
diff --git a/packages/pro b/packages/pro
index 132b080643..3820c0c93a 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 132b080643f0b1ef1488bb8362d1c41ea0bd263f
+Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8
diff --git a/packages/server/src/api/routes/tests/backup.spec.ts b/packages/server/src/api/routes/tests/backup.spec.ts
index 0c62b32155..d12b5e1507 100644
--- a/packages/server/src/api/routes/tests/backup.spec.ts
+++ b/packages/server/src/api/routes/tests/backup.spec.ts
@@ -14,6 +14,7 @@ describe("/backups", () => {
   afterAll(setup.afterAll)
 
   beforeEach(async () => {
+    tk.reset()
     await config.init()
   })
 
diff --git a/packages/server/src/tests/utilities/api/backup.ts b/packages/server/src/tests/utilities/api/backup.ts
index 4cee19fdc6..f9cbc7086e 100644
--- a/packages/server/src/tests/utilities/api/backup.ts
+++ b/packages/server/src/tests/utilities/api/backup.ts
@@ -4,8 +4,6 @@ import {
 } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
-import tk from "timekeeper"
-import { mocks } from "@budibase/backend-core/tests"
 
 export class BackupAPI extends TestAPI {
   constructor(config: TestConfiguration) {
@@ -25,7 +23,6 @@ export class BackupAPI extends TestAPI {
   }
 
   createBackup = async (appId: string) => {
-    tk.freeze(mocks.date.MOCK_DATE_PLUS(1))
     const result = await this.request
       .post(`/api/apps/${appId}/backups`)
       .set(this.config.defaultHeaders())
@@ -38,7 +35,6 @@ export class BackupAPI extends TestAPI {
     appId: string,
     backupId: string
   ): Promise<ImportAppBackupResponse> => {
-    tk.freeze(mocks.date.MOCK_DATE_PLUS(1))
     const result = await this.request
       .post(`/api/apps/${appId}/backups/${backupId}/import`)
       .set(this.config.defaultHeaders())

From 560c51aed58d244fd03b1a49eaf81ae12f70a749 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 31 Oct 2023 11:47:06 +0000
Subject: [PATCH 116/134] Fix button padding

---
 .../pages/builder/portal/users/users/index.svelte    | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/users/users/index.svelte b/packages/builder/src/pages/builder/portal/users/users/index.svelte
index e1fc0ca7eb..ba6798f3a9 100644
--- a/packages/builder/src/pages/builder/portal/users/users/index.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/index.svelte
@@ -3,7 +3,6 @@
     Heading,
     Body,
     Button,
-    ButtonGroup,
     Table,
     Layout,
     Modal,
@@ -296,7 +295,7 @@
   {/if}
   <div class="controls">
     {#if !readonly}
-      <ButtonGroup>
+      <div class="buttons">
         <Button
           disabled={readonly}
           on:click={$licensing.userLimitReached
@@ -315,7 +314,7 @@
         >
           Import
         </Button>
-      </ButtonGroup>
+      </div>
     {:else}
       <ScimBanner />
     {/if}
@@ -390,12 +389,15 @@
 </Modal>
 
 <style>
+  .buttons {
+    display: flex;
+    gap: 10px;
+  }
   .pagination {
     display: flex;
     flex-direction: row;
     justify-content: flex-end;
   }
-
   .controls {
     display: flex;
     flex-direction: row;
@@ -403,7 +405,6 @@
     align-items: center;
     gap: var(--spacing-xl);
   }
-
   .controls-right {
     display: flex;
     flex-direction: row;
@@ -411,7 +412,6 @@
     align-items: center;
     gap: var(--spacing-xl);
   }
-
   .controls-right :global(.spectrum-Search) {
     width: 200px;
   }

From fe60aced318ff6e6164e1e7a5304c5979c6d30d3 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 31 Oct 2023 11:50:29 +0000
Subject: [PATCH 117/134] Fix 501 when clearing user search field

---
 .../builder/src/pages/builder/portal/users/users/index.svelte   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/portal/users/users/index.svelte b/packages/builder/src/pages/builder/portal/users/users/index.svelte
index ba6798f3a9..e35b90f907 100644
--- a/packages/builder/src/pages/builder/portal/users/users/index.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/index.svelte
@@ -142,7 +142,7 @@
   const updateFetch = email => {
     fetch.update({
       query: {
-        email,
+        email: email || null,
       },
     })
   }

From d5629bb4ec11caa2c8e97380e0583a0369aea1ef Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 31 Oct 2023 11:51:46 +0000
Subject: [PATCH 118/134] Use proper search structure for users fetch in portal
 users page

---
 .../builder/src/pages/builder/portal/users/users/index.svelte | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/portal/users/users/index.svelte b/packages/builder/src/pages/builder/portal/users/users/index.svelte
index e35b90f907..5545765c15 100644
--- a/packages/builder/src/pages/builder/portal/users/users/index.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/index.svelte
@@ -142,7 +142,9 @@
   const updateFetch = email => {
     fetch.update({
       query: {
-        email: email || null,
+        string: {
+          email,
+        },
       },
     })
   }

From 9e18c666ad27d7a09ffa49ad9dc7c5f1ecf7bf15 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 31 Oct 2023 12:10:31 +0000
Subject: [PATCH 119/134] Restructure user list page a bit

---
 .../builder/portal/users/users/index.svelte   | 30 ++++++++++---------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/users/users/index.svelte b/packages/builder/src/pages/builder/portal/users/users/index.svelte
index 5545765c15..ff8b749602 100644
--- a/packages/builder/src/pages/builder/portal/users/users/index.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/index.svelte
@@ -45,6 +45,10 @@
     datasource: {
       type: "user",
     },
+    options: {
+      paginate: true,
+      limit: 10,
+    },
   })
 
   let groupsLoaded = !$licensing.groupsEnabled || $groups?.length
@@ -64,10 +68,12 @@
     { column: "role", component: RoleTableRenderer },
   ]
   let userData = []
+  let invitesLoaded = false
+  let pendingInvites = []
+  let parsedInvites = []
 
   $: isOwner = $auth.accountPortalAccess && $admin.cloud
   $: readonly = !sdk.users.isAdmin($auth.user) || $features.isScimEnabled
-
   $: debouncedUpdateFetch(searchEmail)
   $: schema = {
     email: {
@@ -87,16 +93,6 @@
       width: "1fr",
     },
   }
-
-  const getPendingSchema = tblSchema => {
-    if (!tblSchema) {
-      return {}
-    }
-    let pendingSchema = JSON.parse(JSON.stringify(tblSchema))
-    pendingSchema.email.displayName = "Pending Invites"
-    return pendingSchema
-  }
-
   $: pendingSchema = getPendingSchema(schema)
   $: userData = []
   $: inviteUsersResponse = { successful: [], unsuccessful: [] }
@@ -120,9 +116,15 @@
       }
     })
   }
-  let invitesLoaded = false
-  let pendingInvites = []
-  let parsedInvites = []
+
+  const getPendingSchema = tblSchema => {
+    if (!tblSchema) {
+      return {}
+    }
+    let pendingSchema = JSON.parse(JSON.stringify(tblSchema))
+    pendingSchema.email.displayName = "Pending Invites"
+    return pendingSchema
+  }
 
   const invitesToSchema = invites => {
     return invites.map(invite => {

From 99a0c8b08b1a92397b917463ef9ccb684603d4c9 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 31 Oct 2023 12:10:41 +0000
Subject: [PATCH 120/134] Fix bug breaking pagination in global users search

---
 packages/backend-core/src/users/users.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts
index ea4a17214d..eec4b71edc 100644
--- a/packages/backend-core/src/users/users.ts
+++ b/packages/backend-core/src/users/users.ts
@@ -276,7 +276,7 @@ export const paginatedUsers = async ({
     const response = await db.allDocs(getGlobalUserParams(null, opts))
     userList = response.rows.map((row: any) => row.doc)
   }
-  return pagination(userList, pageLimit, {
+  return pagination(userList, limit ?? PAGE_LIMIT, {
     paginate: true,
     property,
     getKey,

From 68553e85d4dde03bdd6a15945da5aa538d5bb6aa Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 31 Oct 2023 13:01:19 +0000
Subject: [PATCH 121/134] Simplify page size logic in user search

---
 packages/backend-core/src/users/users.ts | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts
index eec4b71edc..6dc8750b62 100644
--- a/packages/backend-core/src/users/users.ts
+++ b/packages/backend-core/src/users/users.ts
@@ -249,7 +249,8 @@ export const paginatedUsers = async ({
   limit,
 }: SearchUsersRequest = {}) => {
   const db = getGlobalDB()
-  const pageLimit = limit ? limit + 1 : PAGE_LIMIT + 1
+  const pageSize = limit ?? PAGE_LIMIT
+  const pageLimit = pageSize + 1
   // get one extra document, to have the next page
   const opts: DatabaseQueryOpts = {
     include_docs: true,
@@ -276,7 +277,7 @@ export const paginatedUsers = async ({
     const response = await db.allDocs(getGlobalUserParams(null, opts))
     userList = response.rows.map((row: any) => row.doc)
   }
-  return pagination(userList, limit ?? PAGE_LIMIT, {
+  return pagination(userList, pageSize, {
     paginate: true,
     property,
     getKey,

From 7e33aacbb1a77b0fa989d8647d9bf680c9b9ddfe Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Tue, 31 Oct 2023 14:48:23 +0000
Subject: [PATCH 122/134] Stop the sample data being identified as an external
 source.

---
 packages/server/src/integrations/utils.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts
index b4fff0737a..33895e4fe1 100644
--- a/packages/server/src/integrations/utils.ts
+++ b/packages/server/src/integrations/utils.ts
@@ -11,6 +11,7 @@ import { InvalidColumns, NoEmptyFilterStrings } from "../constants"
 import { helpers } from "@budibase/shared-core"
 import * as external from "../api/controllers/table/external"
 import * as internal from "../api/controllers/table/internal"
+import { DEFAULT_BB_DATASOURCE_ID } from "../db/defaultData/datasource_bb_default"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
@@ -96,7 +97,8 @@ export function isInternalTableID(tableId: string) {
 export function isExternalTable(table: Table) {
   if (
     table?.sourceId &&
-    table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR)
+    table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) &&
+    table?.sourceId !== DEFAULT_BB_DATASOURCE_ID
   ) {
     return true
   } else if (table?.sourceType === TableSourceType.EXTERNAL) {

From df6f8dad7e725e5c208bd65682e7db61a3a72d03 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 31 Oct 2023 16:56:19 +0000
Subject: [PATCH 123/134] Updating bull parameters to see if this helps with
 queue stalling.

---
 .../backend-core/src/queue/inMemoryQueue.ts     |  2 +-
 packages/backend-core/src/queue/queue.ts        | 17 +++++++++++++++--
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts
index af2ec6dbaa..a8add7ecb6 100644
--- a/packages/backend-core/src/queue/inMemoryQueue.ts
+++ b/packages/backend-core/src/queue/inMemoryQueue.ts
@@ -36,7 +36,7 @@ class InMemoryQueue {
    * @param opts This is not used by the in memory queue as there is no real use
    * case when in memory, but is the same API as Bull
    */
-  constructor(name: string, opts = null) {
+  constructor(name: string, opts?: any) {
     this._name = name
     this._opts = opts
     this._messages = []
diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts
index 0658147709..c6b02921bf 100644
--- a/packages/backend-core/src/queue/queue.ts
+++ b/packages/backend-core/src/queue/queue.ts
@@ -2,10 +2,16 @@ import env from "../environment"
 import { getRedisOptions } from "../redis/utils"
 import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
-import BullQueue from "bull"
+import BullQueue, { QueueOptions } from "bull"
 import { addListeners, StalledFn } from "./listeners"
 import * as timers from "../timers"
+import * as Redis from "ioredis"
 
+// the queue lock is held for 5 minutes
+const QUEUE_LOCK_MS = 300000
+// queue lock is refreshed every 30 seconds
+const QUEUE_LOCK_RENEW_INTERNAL_MS = 30000
+// cleanup the queue every 60 seconds
 const CLEANUP_PERIOD_MS = 60 * 1000
 let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
 let cleanupInterval: NodeJS.Timeout
@@ -21,7 +27,14 @@ export function createQueue<T>(
   opts: { removeStalledCb?: StalledFn } = {}
 ): BullQueue.Queue<T> {
   const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
-  const queueConfig: any = redisProtocolUrl || { redis: redisOpts }
+  const queueConfig: QueueOptions = {
+    redis: redisProtocolUrl! || (redisOpts as Redis.RedisOptions),
+    settings: {
+      maxStalledCount: 0,
+      lockDuration: QUEUE_LOCK_MS,
+      lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS,
+    },
+  }
   let queue: any
   if (!env.isTest()) {
     queue = new BullQueue(jobQueue, queueConfig)

From 17319a6981a79ef41abe8fe4f7a26a6b02d4f9cd Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 31 Oct 2023 17:52:39 +0000
Subject: [PATCH 124/134] Moving things around so that DEFAULT_BB_DATASOURCE_ID
 can be imported without cyclics occurring.

---
 .../server/src/api/controllers/application.ts    |  7 ++-----
 packages/server/src/constants/index.ts           |  5 +++++
 .../src/db/defaultData/datasource_bb_default.ts  | 16 +++++++++-------
 packages/server/src/integrations/utils.ts        |  9 +++++----
 4 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 4afd7b23f9..4e4c66858e 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -32,11 +32,8 @@ import {
   tenancy,
   users,
 } from "@budibase/backend-core"
-import { USERS_TABLE_SCHEMA } from "../../constants"
-import {
-  buildDefaultDocs,
-  DEFAULT_BB_DATASOURCE_ID,
-} from "../../db/defaultData/datasource_bb_default"
+import { USERS_TABLE_SCHEMA, DEFAULT_BB_DATASOURCE_ID } from "../../constants"
+import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default"
 import { removeAppFromUserRoles } from "../../utilities/workerRequests"
 import { stringToReadStream } from "../../utilities"
 import { doesUserHaveLock } from "../../utilities/redis"
diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts
index 1104fb9b29..fb5c42e7b8 100644
--- a/packages/server/src/constants/index.ts
+++ b/packages/server/src/constants/index.ts
@@ -172,3 +172,8 @@ export enum AutomationErrors {
 export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets
 export const MAX_AUTOMATION_RECURRING_ERRORS = 5
 export const GOOGLE_SHEETS_PRIMARY_KEY = "rowNumber"
+export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
+export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
+export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses"
+export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee"
+export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"
diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts
index 584b76f879..b430f9ffb6 100644
--- a/packages/server/src/db/defaultData/datasource_bb_default.ts
+++ b/packages/server/src/db/defaultData/datasource_bb_default.ts
@@ -1,4 +1,12 @@
-import { AutoFieldSubTypes, FieldTypes } from "../../constants"
+import {
+  AutoFieldSubTypes,
+  FieldTypes,
+  DEFAULT_BB_DATASOURCE_ID,
+  DEFAULT_INVENTORY_TABLE_ID,
+  DEFAULT_EMPLOYEE_TABLE_ID,
+  DEFAULT_EXPENSES_TABLE_ID,
+  DEFAULT_JOBS_TABLE_ID,
+} from "../../constants"
 import { importToRows } from "../../api/controllers/table/utils"
 import { cloneDeep } from "lodash/fp"
 import LinkDocument from "../linkedRows/LinkDocument"
@@ -16,12 +24,6 @@ import {
   TableSourceType,
 } from "@budibase/types"
 
-export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
-export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
-export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses"
-export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee"
-export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"
-
 const defaultDatasource = {
   _id: DEFAULT_BB_DATASOURCE_ID,
   type: dbCore.BUDIBASE_DATASOURCE_TYPE,
diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts
index 33895e4fe1..fe8a9055b0 100644
--- a/packages/server/src/integrations/utils.ts
+++ b/packages/server/src/integrations/utils.ts
@@ -7,11 +7,12 @@ import {
   TableSourceType,
 } from "@budibase/types"
 import { DocumentType, SEPARATOR } from "../db/utils"
-import { InvalidColumns, NoEmptyFilterStrings } from "../constants"
+import {
+  InvalidColumns,
+  NoEmptyFilterStrings,
+  DEFAULT_BB_DATASOURCE_ID,
+} from "../constants"
 import { helpers } from "@budibase/shared-core"
-import * as external from "../api/controllers/table/external"
-import * as internal from "../api/controllers/table/internal"
-import { DEFAULT_BB_DATASOURCE_ID } from "../db/defaultData/datasource_bb_default"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g

From e05821d6d7c78320fe427abd8237738c61c88e31 Mon Sep 17 00:00:00 2001
From: Mitch-Budibase <mitch@budibase.com>
Date: Tue, 31 Oct 2023 18:16:58 +0000
Subject: [PATCH 125/134] License Management Test

This test retrieves plans, creates checkout session, and updates license

Essential changes have been made to linkStripeCustomer & updatePlan functions to support the test

Modified "test:self:ci" to include 'licensing' instead of 'license'
Modified environment.ts to include STRIPE_SECRET_KEY
---
 qa-core/package.json                          |   2 +-
 .../src/account-api/api/apis/LicenseAPI.ts    |  10 +-
 qa-core/src/account-api/api/apis/StripeAPI.ts |  13 +-
 .../tests/licensing/license.manage.spec.ts    | 113 ++++++++++++++++++
 qa-core/src/environment.ts                    |   1 +
 5 files changed, 134 insertions(+), 5 deletions(-)
 create mode 100644 qa-core/src/account-api/tests/licensing/license.manage.spec.ts

diff --git a/qa-core/package.json b/qa-core/package.json
index d266ca9def..cfccd5e650 100644
--- a/qa-core/package.json
+++ b/qa-core/package.json
@@ -17,7 +17,7 @@
     "test:notify": "node scripts/testResultsWebhook",
     "test:cloud:prod": "yarn run test --testPathIgnorePatterns=\\.integration\\.",
     "test:cloud:qa": "yarn run test",
-    "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.license\\.",
+    "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.licensing\\.",
     "serve:test:self:ci": "start-server-and-test dev:built http://localhost:4001/health test:self:ci",
     "serve": "start-server-and-test dev:built http://localhost:4001/health",
     "dev:built": "cd ../ && yarn dev:built"
diff --git a/qa-core/src/account-api/api/apis/LicenseAPI.ts b/qa-core/src/account-api/api/apis/LicenseAPI.ts
index b371f00f05..b3b54c5102 100644
--- a/qa-core/src/account-api/api/apis/LicenseAPI.ts
+++ b/qa-core/src/account-api/api/apis/LicenseAPI.ts
@@ -99,9 +99,15 @@ export default class LicenseAPI extends BaseAPI {
     }, opts)
   }
 
-  async updatePlan(opts: APIRequestOpts = { status: 200 }) {
+  async updatePlan(
+      priceId: string,
+      opts: APIRequestOpts = { status: 200 }
+  ) {
     return this.doRequest(() => {
-      return this.client.put(`/api/license/plan`)
+      return this.client.put(`/api/license/plan`,
+          {
+            body: { priceId },
+          })
     }, opts)
   }
 
diff --git a/qa-core/src/account-api/api/apis/StripeAPI.ts b/qa-core/src/account-api/api/apis/StripeAPI.ts
index c9c776e89b..6fef944206 100644
--- a/qa-core/src/account-api/api/apis/StripeAPI.ts
+++ b/qa-core/src/account-api/api/apis/StripeAPI.ts
@@ -38,9 +38,18 @@ export default class StripeAPI extends BaseAPI {
     }, opts)
   }
 
-  async linkStripeCustomer(opts: APIRequestOpts = { status: 200 }) {
+  async linkStripeCustomer(
+      accountId: string,
+      stripeCustomerId: string,
+      opts: APIRequestOpts = { status: 200 }) {
     return this.doRequest(() => {
-      return this.client.post(`/api/stripe/link`)
+      return this.client.post(`/api/stripe/link`, {
+        body: {
+          accountId,
+          stripeCustomerId
+        },
+        internal: true,
+      })
     }, opts)
   }
 
diff --git a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts
new file mode 100644
index 0000000000..a62b733c24
--- /dev/null
+++ b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts
@@ -0,0 +1,113 @@
+import TestConfiguration from "../../config/TestConfiguration"
+import * as fixtures from "../../fixtures"
+import {Hosting, PlanType} from "@budibase/types"
+
+describe("license management", () => {
+    const config = new TestConfiguration()
+
+    beforeAll(async () => {
+        await config.beforeAll()
+    })
+
+    afterAll(async () => {
+        await config.afterAll()
+    })
+
+    it("retrieves plans, creates checkout session, and updates license", async () => {
+        // Create cloud account
+        const createAccountRequest = fixtures.accounts.generateAccount({
+            hosting: Hosting.CLOUD,
+        })
+        const [createAccountRes, account] =
+            await config.accountsApi.accounts.create(createAccountRequest, {
+                autoVerify: true,
+            })
+
+        // Self response has free license
+        await config.doInNewState(async () => {
+            await config.loginAsAccount(createAccountRequest)
+            const [selfRes, selfBody] = await config.api.accounts.self()
+            expect(selfBody.license.plan.type).toBe(PlanType.FREE)
+        })
+
+        // Retrieve plans
+        const [plansRes, planBody] = await config.api.licenses.getPlans()
+
+        // Select priceId from premium plan
+        let premiumPriceId = null
+        let businessPriceId = ''
+        for (const plan of planBody) {
+            if (plan.type === PlanType.PREMIUM) {
+                premiumPriceId = plan.prices[0].priceId
+            }
+            if (plan.type === PlanType.BUSINESS) {
+                businessPriceId = plan.prices[0].priceId
+            }
+        }
+
+        // Create checkout session for price
+        const checkoutSessionRes = await config.api.stripe.createCheckoutSession(
+            premiumPriceId
+        )
+        const checkoutSessionUrl = checkoutSessionRes[1].url
+        expect(checkoutSessionUrl).toContain("checkout.stripe.com")
+
+        // Create stripe customer
+        const stripe = require('stripe')(process.env.STRIPE_SECRET_KEY)
+        const customer = await stripe.customers.create({
+            email: createAccountRequest.email
+        })
+
+        // Create payment method
+        const paymentMethod = await stripe.paymentMethods.create({
+            type: 'card',
+            card: {
+                token: 'tok_visa' // Test Visa Card
+            },
+        })
+
+        // Attach payment method to customer
+        await stripe.paymentMethods.attach(paymentMethod.id, {
+            customer: customer.id,
+        })
+        
+        // Update customer
+        await stripe.customers.update(customer.id, {
+            invoice_settings: {
+                default_payment_method: paymentMethod.id,
+            },
+        })
+
+        // Create subscription for premium plan
+        const subscription = await stripe.subscriptions.create({
+            customer: customer.id,
+            items: [
+                {
+                    price: premiumPriceId,
+                    quantity: 1,
+                },
+            ],
+            default_payment_method: paymentMethod.id,
+            collection_method: 'charge_automatically',
+        })
+
+        await config.doInNewState(async () => {
+            // License updated from Free to Premium
+            await config.loginAsAccount(createAccountRequest)
+            await config.api.stripe.linkStripeCustomer(account.accountId, customer.id)
+            const [_, selfBodyPremium] = await config.api.accounts.self()
+            expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM)
+
+            // Create portal session - Check URL
+            const [portalRes, portalSessionBody] = await config.api.stripe.createPortalSession(customer.id)
+            expect(portalSessionBody.url).toContain("billing.stripe.com")
+
+            // Update subscription from premium to business license
+            await config.api.licenses.updatePlan(businessPriceId)
+
+            // License updated to Business
+            const [selfRes, selfBodyBusiness] = await config.api.accounts.self()
+            expect(selfBodyBusiness.license.plan.type).toBe(PlanType.BUSINESS)
+        })
+    })
+})
diff --git a/qa-core/src/environment.ts b/qa-core/src/environment.ts
index 0257b10831..a805503474 100644
--- a/qa-core/src/environment.ts
+++ b/qa-core/src/environment.ts
@@ -28,6 +28,7 @@ const env = {
   MARIADB_DB: process.env.MARIADB_DB,
   MARIADB_USER: process.env.MARIADB_USER,
   MARIADB_PASSWORD: process.env.MARIADB_PASSWORD,
+  STRIPE_SECRET_KEY: process.env.STRIPE_SECRET_KEY,
 }
 
 export = env

From 84079450f973ad957d8bd031329832d155603b1b Mon Sep 17 00:00:00 2001
From: Mitch-Budibase <mitch@budibase.com>
Date: Tue, 31 Oct 2023 18:20:27 +0000
Subject: [PATCH 126/134] lint

---
 .../src/account-api/api/apis/LicenseAPI.ts    |  12 +-
 qa-core/src/account-api/api/apis/StripeAPI.ts |   9 +-
 .../tests/licensing/license.manage.spec.ts    | 205 +++++++++---------
 3 files changed, 112 insertions(+), 114 deletions(-)

diff --git a/qa-core/src/account-api/api/apis/LicenseAPI.ts b/qa-core/src/account-api/api/apis/LicenseAPI.ts
index b3b54c5102..a9b0a35269 100644
--- a/qa-core/src/account-api/api/apis/LicenseAPI.ts
+++ b/qa-core/src/account-api/api/apis/LicenseAPI.ts
@@ -99,15 +99,11 @@ export default class LicenseAPI extends BaseAPI {
     }, opts)
   }
 
-  async updatePlan(
-      priceId: string,
-      opts: APIRequestOpts = { status: 200 }
-  ) {
+  async updatePlan(priceId: string, opts: APIRequestOpts = { status: 200 }) {
     return this.doRequest(() => {
-      return this.client.put(`/api/license/plan`,
-          {
-            body: { priceId },
-          })
+      return this.client.put(`/api/license/plan`, {
+        body: { priceId },
+      })
     }, opts)
   }
 
diff --git a/qa-core/src/account-api/api/apis/StripeAPI.ts b/qa-core/src/account-api/api/apis/StripeAPI.ts
index 6fef944206..5a4e810655 100644
--- a/qa-core/src/account-api/api/apis/StripeAPI.ts
+++ b/qa-core/src/account-api/api/apis/StripeAPI.ts
@@ -39,14 +39,15 @@ export default class StripeAPI extends BaseAPI {
   }
 
   async linkStripeCustomer(
-      accountId: string,
-      stripeCustomerId: string,
-      opts: APIRequestOpts = { status: 200 }) {
+    accountId: string,
+    stripeCustomerId: string,
+    opts: APIRequestOpts = { status: 200 }
+  ) {
     return this.doRequest(() => {
       return this.client.post(`/api/stripe/link`, {
         body: {
           accountId,
-          stripeCustomerId
+          stripeCustomerId,
         },
         internal: true,
       })
diff --git a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts
index a62b733c24..a9109dad3a 100644
--- a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts
+++ b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts
@@ -1,113 +1,114 @@
 import TestConfiguration from "../../config/TestConfiguration"
 import * as fixtures from "../../fixtures"
-import {Hosting, PlanType} from "@budibase/types"
+import { Hosting, PlanType } from "@budibase/types"
 
 describe("license management", () => {
-    const config = new TestConfiguration()
+  const config = new TestConfiguration()
 
-    beforeAll(async () => {
-        await config.beforeAll()
+  beforeAll(async () => {
+    await config.beforeAll()
+  })
+
+  afterAll(async () => {
+    await config.afterAll()
+  })
+
+  it("retrieves plans, creates checkout session, and updates license", async () => {
+    // Create cloud account
+    const createAccountRequest = fixtures.accounts.generateAccount({
+      hosting: Hosting.CLOUD,
+    })
+    const [createAccountRes, account] =
+      await config.accountsApi.accounts.create(createAccountRequest, {
+        autoVerify: true,
+      })
+
+    // Self response has free license
+    await config.doInNewState(async () => {
+      await config.loginAsAccount(createAccountRequest)
+      const [selfRes, selfBody] = await config.api.accounts.self()
+      expect(selfBody.license.plan.type).toBe(PlanType.FREE)
     })
 
-    afterAll(async () => {
-        await config.afterAll()
+    // Retrieve plans
+    const [plansRes, planBody] = await config.api.licenses.getPlans()
+
+    // Select priceId from premium plan
+    let premiumPriceId = null
+    let businessPriceId = ""
+    for (const plan of planBody) {
+      if (plan.type === PlanType.PREMIUM) {
+        premiumPriceId = plan.prices[0].priceId
+      }
+      if (plan.type === PlanType.BUSINESS) {
+        businessPriceId = plan.prices[0].priceId
+      }
+    }
+
+    // Create checkout session for price
+    const checkoutSessionRes = await config.api.stripe.createCheckoutSession(
+      premiumPriceId
+    )
+    const checkoutSessionUrl = checkoutSessionRes[1].url
+    expect(checkoutSessionUrl).toContain("checkout.stripe.com")
+
+    // Create stripe customer
+    const stripe = require("stripe")(process.env.STRIPE_SECRET_KEY)
+    const customer = await stripe.customers.create({
+      email: createAccountRequest.email,
     })
 
-    it("retrieves plans, creates checkout session, and updates license", async () => {
-        // Create cloud account
-        const createAccountRequest = fixtures.accounts.generateAccount({
-            hosting: Hosting.CLOUD,
-        })
-        const [createAccountRes, account] =
-            await config.accountsApi.accounts.create(createAccountRequest, {
-                autoVerify: true,
-            })
-
-        // Self response has free license
-        await config.doInNewState(async () => {
-            await config.loginAsAccount(createAccountRequest)
-            const [selfRes, selfBody] = await config.api.accounts.self()
-            expect(selfBody.license.plan.type).toBe(PlanType.FREE)
-        })
-
-        // Retrieve plans
-        const [plansRes, planBody] = await config.api.licenses.getPlans()
-
-        // Select priceId from premium plan
-        let premiumPriceId = null
-        let businessPriceId = ''
-        for (const plan of planBody) {
-            if (plan.type === PlanType.PREMIUM) {
-                premiumPriceId = plan.prices[0].priceId
-            }
-            if (plan.type === PlanType.BUSINESS) {
-                businessPriceId = plan.prices[0].priceId
-            }
-        }
-
-        // Create checkout session for price
-        const checkoutSessionRes = await config.api.stripe.createCheckoutSession(
-            premiumPriceId
-        )
-        const checkoutSessionUrl = checkoutSessionRes[1].url
-        expect(checkoutSessionUrl).toContain("checkout.stripe.com")
-
-        // Create stripe customer
-        const stripe = require('stripe')(process.env.STRIPE_SECRET_KEY)
-        const customer = await stripe.customers.create({
-            email: createAccountRequest.email
-        })
-
-        // Create payment method
-        const paymentMethod = await stripe.paymentMethods.create({
-            type: 'card',
-            card: {
-                token: 'tok_visa' // Test Visa Card
-            },
-        })
-
-        // Attach payment method to customer
-        await stripe.paymentMethods.attach(paymentMethod.id, {
-            customer: customer.id,
-        })
-        
-        // Update customer
-        await stripe.customers.update(customer.id, {
-            invoice_settings: {
-                default_payment_method: paymentMethod.id,
-            },
-        })
-
-        // Create subscription for premium plan
-        const subscription = await stripe.subscriptions.create({
-            customer: customer.id,
-            items: [
-                {
-                    price: premiumPriceId,
-                    quantity: 1,
-                },
-            ],
-            default_payment_method: paymentMethod.id,
-            collection_method: 'charge_automatically',
-        })
-
-        await config.doInNewState(async () => {
-            // License updated from Free to Premium
-            await config.loginAsAccount(createAccountRequest)
-            await config.api.stripe.linkStripeCustomer(account.accountId, customer.id)
-            const [_, selfBodyPremium] = await config.api.accounts.self()
-            expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM)
-
-            // Create portal session - Check URL
-            const [portalRes, portalSessionBody] = await config.api.stripe.createPortalSession(customer.id)
-            expect(portalSessionBody.url).toContain("billing.stripe.com")
-
-            // Update subscription from premium to business license
-            await config.api.licenses.updatePlan(businessPriceId)
-
-            // License updated to Business
-            const [selfRes, selfBodyBusiness] = await config.api.accounts.self()
-            expect(selfBodyBusiness.license.plan.type).toBe(PlanType.BUSINESS)
-        })
+    // Create payment method
+    const paymentMethod = await stripe.paymentMethods.create({
+      type: "card",
+      card: {
+        token: "tok_visa", // Test Visa Card
+      },
     })
+
+    // Attach payment method to customer
+    await stripe.paymentMethods.attach(paymentMethod.id, {
+      customer: customer.id,
+    })
+
+    // Update customer
+    await stripe.customers.update(customer.id, {
+      invoice_settings: {
+        default_payment_method: paymentMethod.id,
+      },
+    })
+
+    // Create subscription for premium plan
+    const subscription = await stripe.subscriptions.create({
+      customer: customer.id,
+      items: [
+        {
+          price: premiumPriceId,
+          quantity: 1,
+        },
+      ],
+      default_payment_method: paymentMethod.id,
+      collection_method: "charge_automatically",
+    })
+
+    await config.doInNewState(async () => {
+      // License updated from Free to Premium
+      await config.loginAsAccount(createAccountRequest)
+      await config.api.stripe.linkStripeCustomer(account.accountId, customer.id)
+      const [_, selfBodyPremium] = await config.api.accounts.self()
+      expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM)
+
+      // Create portal session - Check URL
+      const [portalRes, portalSessionBody] =
+        await config.api.stripe.createPortalSession(customer.id)
+      expect(portalSessionBody.url).toContain("billing.stripe.com")
+
+      // Update subscription from premium to business license
+      await config.api.licenses.updatePlan(businessPriceId)
+
+      // License updated to Business
+      const [selfRes, selfBodyBusiness] = await config.api.accounts.self()
+      expect(selfBodyBusiness.license.plan.type).toBe(PlanType.BUSINESS)
+    })
+  })
 })

From 234cdbbf2441c4f3921a7226e14b920ba4506bab Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 1 Nov 2023 10:51:34 +0000
Subject: [PATCH 127/134] Bump version to 2.12.2

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 6df4a4c4cd..cb92b3ba0d 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.12.1",
+  "version": "2.12.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 984d9497811a6daa049c743f7e4095c06eca8b5a Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 1 Nov 2023 15:02:55 +0000
Subject: [PATCH 128/134] Updating single image build - the old method of
 installing Node 18 was deprecated, switching to the new mechanism.

---
 hosting/scripts/install-minio.sh | 10 ----------
 hosting/single/Dockerfile.v2     | 11 +++++------
 scripts/install-minio.sh         |  6 +++---
 scripts/install-node.sh          |  8 ++++++++
 4 files changed, 16 insertions(+), 19 deletions(-)
 delete mode 100755 hosting/scripts/install-minio.sh
 create mode 100644 scripts/install-node.sh

diff --git a/hosting/scripts/install-minio.sh b/hosting/scripts/install-minio.sh
deleted file mode 100755
index 8297593599..0000000000
--- a/hosting/scripts/install-minio.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-if [[ $TARGETARCH == arm* ]] ;
-then
-  echo "INSTALLING ARM64 MINIO"
-  wget https://dl.min.io/server/minio/release/linux-arm64/minio
-else
-  echo "INSTALLING AMD64 MINIO"
-  wget https://dl.min.io/server/minio/release/linux-amd64/minio
-fi
-chmod +x minio
diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2
index 5b07a51b27..ec03a1b5a2 100644
--- a/hosting/single/Dockerfile.v2
+++ b/hosting/single/Dockerfile.v2
@@ -42,6 +42,7 @@ COPY packages/string-templates packages/string-templates
 FROM budibase/couchdb as runner
 ARG TARGETARCH
 ENV TARGETARCH $TARGETARCH
+ENV NODE_MAJOR 18
 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
 # e.g. docker build --build-arg TARGETBUILD=aas ....
 ARG TARGETBUILD=single
@@ -49,10 +50,10 @@ ENV TARGETBUILD $TARGETBUILD
 
 # install base dependencies
 RUN apt-get update && \
-  apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server
+  apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server libaio1
 
 # Install postgres client for pg_dump utils
-RUN apt install software-properties-common apt-transport-https gpg -y \
+RUN apt install -y software-properties-common apt-transport-https ca-certificates gnupg \
     && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \
     && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \
     && apt update -y \
@@ -61,10 +62,8 @@ RUN apt install software-properties-common apt-transport-https gpg -y \
 
 # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx
 WORKDIR /nodejs
-RUN curl -sL https://deb.nodesource.com/setup_18.x -o /tmp/nodesource_setup.sh && \
-  bash /tmp/nodesource_setup.sh && \
-  apt-get install -y --no-install-recommends libaio1 nodejs && \
-  npm install --global yarn pm2
+COPY scripts/install-node.sh ./install.sh
+RUN chmod +x install.sh && ./install.sh
 
 # setup nginx
 COPY hosting/single/nginx/nginx.conf /etc/nginx
diff --git a/scripts/install-minio.sh b/scripts/install-minio.sh
index 8297593599..b1e0d9ee80 100755
--- a/scripts/install-minio.sh
+++ b/scripts/install-minio.sh
@@ -2,9 +2,9 @@
 if [[ $TARGETARCH == arm* ]] ;
 then
   echo "INSTALLING ARM64 MINIO"
-  wget https://dl.min.io/server/minio/release/linux-arm64/minio
+  wget wget https://dl.min.io/server/minio/release/linux-arm64/archive/minio.deb -O minio.deb
 else
   echo "INSTALLING AMD64 MINIO"
-  wget https://dl.min.io/server/minio/release/linux-amd64/minio
+  wget wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio.deb -O minio.deb
 fi
-chmod +x minio
+dpkg -i minio.deb
diff --git a/scripts/install-node.sh b/scripts/install-node.sh
new file mode 100644
index 0000000000..562bdf2cd3
--- /dev/null
+++ b/scripts/install-node.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+apt-get install -y gnupg
+curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor | tee /usr/share/keyrings/nodesource.gpg > /dev/null
+echo "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
+apt-get update
+echo "INSTALLING NODE $NODE_MAJOR"
+apt-get install -y --no-install-recommends nodejs
+npm install --global yarn pm2
\ No newline at end of file

From cf40b87d81f7161f2294c6f00c1768ec8e7aa49f Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 1 Nov 2023 15:10:32 +0000
Subject: [PATCH 129/134] Use installed MinIO rather than binary.

---
 hosting/single/runner.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh
index 9dc7aa25d8..770b23eec1 100644
--- a/hosting/single/runner.sh
+++ b/hosting/single/runner.sh
@@ -77,7 +77,7 @@ mkdir -p ${DATA_DIR}/minio
 chown -R couchdb:couchdb ${DATA_DIR}/couch
 redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
 /bbcouch-runner.sh &
-/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
+minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
 /etc/init.d/nginx restart
 if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then
     # Add monthly cron job to renew certbot certificate

From 85325814d06ab47f3fb8fba6bba3717f66c779de Mon Sep 17 00:00:00 2001
From: Mitch-Budibase <mitch@budibase.com>
Date: Wed, 1 Nov 2023 15:18:20 +0000
Subject: [PATCH 130/134] Moving stripe import to top of test file

---
 qa-core/src/account-api/tests/licensing/license.manage.spec.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts
index a9109dad3a..9cad980038 100644
--- a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts
+++ b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts
@@ -1,6 +1,7 @@
 import TestConfiguration from "../../config/TestConfiguration"
 import * as fixtures from "../../fixtures"
 import { Hosting, PlanType } from "@budibase/types"
+const stripe = require("stripe")(process.env.STRIPE_SECRET_KEY)
 
 describe("license management", () => {
   const config = new TestConfiguration()
@@ -53,7 +54,6 @@ describe("license management", () => {
     expect(checkoutSessionUrl).toContain("checkout.stripe.com")
 
     // Create stripe customer
-    const stripe = require("stripe")(process.env.STRIPE_SECRET_KEY)
     const customer = await stripe.customers.create({
       email: createAccountRequest.email,
     })

From d6a4b71655991f27ee45986f0339bd1e33d8dc2b Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 1 Nov 2023 15:37:08 +0000
Subject: [PATCH 131/134] Bump version to 2.12.3

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index cb92b3ba0d..dc6ffb2da2 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.12.2",
+  "version": "2.12.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 586d539109f38d9a48b8fe23b0d4ea9084d2320d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 1 Nov 2023 16:23:51 +0000
Subject: [PATCH 132/134] Adding release information to single image.

---
 .github/workflows/release-singleimage.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml
index 4d35916f4d..a3444d5e7a 100644
--- a/.github/workflows/release-singleimage.yml
+++ b/.github/workflows/release-singleimage.yml
@@ -66,14 +66,21 @@ jobs:
           context: .
           push: true
           platforms: linux/amd64,linux/arm64
+          build-args: BUDIBASE_VERSION=$BUDIBASE_VERSION
           tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }}
           file: ./hosting/single/Dockerfile.v2
+        env:
+          BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }}
       - name: Tag and release Budibase Azure App Service docker image
         uses: docker/build-push-action@v2
         with:
           context: .
           push: true
           platforms: linux/amd64
-          build-args: TARGETBUILD=aas
+          build-args: |
+            TARGETBUILD=aas
+            BUDIBASE_VERSION=$BUDIBASE_VERSION
           tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }}
           file: ./hosting/single/Dockerfile.v2
+        env:
+          BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }}

From 0477ed64f6edd6b2a2e02a25fb1baaff591584fb Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 1 Nov 2023 16:40:29 +0000
Subject: [PATCH 133/134] Bump version to 2.12.4

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index dc6ffb2da2..f0f51242d1 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.12.3",
+  "version": "2.12.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 17b6985f06aa5faf256c3f5f45b76a1a55278414 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 1 Nov 2023 17:48:13 +0000
Subject: [PATCH 134/134] Adding a duration utility for changing between time
 types.

---
 packages/backend-core/src/index.ts            |  1 +
 packages/backend-core/src/queue/queue.ts      |  7 +--
 packages/backend-core/src/utils/Duration.ts   | 49 +++++++++++++++++++
 packages/backend-core/src/utils/index.ts      |  1 +
 .../src/utils/tests/Duration.spec.ts          | 19 +++++++
 5 files changed, 74 insertions(+), 3 deletions(-)
 create mode 100644 packages/backend-core/src/utils/Duration.ts
 create mode 100644 packages/backend-core/src/utils/tests/Duration.spec.ts

diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts
index ffffd8240a..c7cf9f56cc 100644
--- a/packages/backend-core/src/index.ts
+++ b/packages/backend-core/src/index.ts
@@ -30,6 +30,7 @@ export * as timers from "./timers"
 export { default as env } from "./environment"
 export * as blacklist from "./blacklist"
 export * as docUpdates from "./docUpdates"
+export * from "./utils/Duration"
 export { SearchParams } from "./db"
 // Add context to tenancy for backwards compatibility
 // only do this for external usages to prevent internal
diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts
index c6b02921bf..c0d1861de3 100644
--- a/packages/backend-core/src/queue/queue.ts
+++ b/packages/backend-core/src/queue/queue.ts
@@ -4,15 +4,16 @@ import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
 import BullQueue, { QueueOptions } from "bull"
 import { addListeners, StalledFn } from "./listeners"
+import { Duration } from "../utils"
 import * as timers from "../timers"
 import * as Redis from "ioredis"
 
 // the queue lock is held for 5 minutes
-const QUEUE_LOCK_MS = 300000
+const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs()
 // queue lock is refreshed every 30 seconds
-const QUEUE_LOCK_RENEW_INTERNAL_MS = 30000
+const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs()
 // cleanup the queue every 60 seconds
-const CLEANUP_PERIOD_MS = 60 * 1000
+const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs()
 let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
 let cleanupInterval: NodeJS.Timeout
 
diff --git a/packages/backend-core/src/utils/Duration.ts b/packages/backend-core/src/utils/Duration.ts
new file mode 100644
index 0000000000..f376c2f7c7
--- /dev/null
+++ b/packages/backend-core/src/utils/Duration.ts
@@ -0,0 +1,49 @@
+export enum DurationType {
+  MILLISECONDS = "milliseconds",
+  SECONDS = "seconds",
+  MINUTES = "minutes",
+  HOURS = "hours",
+  DAYS = "days",
+}
+
+const conversion: Record<DurationType, number> = {
+  milliseconds: 1,
+  seconds: 1000,
+  minutes: 60 * 1000,
+  hours: 60 * 60 * 1000,
+  days: 24 * 60 * 60 * 1000,
+}
+
+export class Duration {
+  static convert(from: DurationType, to: DurationType, duration: number) {
+    const milliseconds = duration * conversion[from]
+    return milliseconds / conversion[to]
+  }
+
+  static from(from: DurationType, duration: number) {
+    return {
+      to: (to: DurationType) => {
+        return Duration.convert(from, to, duration)
+      },
+      toMs: () => {
+        return Duration.convert(from, DurationType.MILLISECONDS, duration)
+      },
+    }
+  }
+
+  static fromSeconds(duration: number) {
+    return Duration.from(DurationType.SECONDS, duration)
+  }
+
+  static fromMinutes(duration: number) {
+    return Duration.from(DurationType.MINUTES, duration)
+  }
+
+  static fromHours(duration: number) {
+    return Duration.from(DurationType.HOURS, duration)
+  }
+
+  static fromDays(duration: number) {
+    return Duration.from(DurationType.DAYS, duration)
+  }
+}
diff --git a/packages/backend-core/src/utils/index.ts b/packages/backend-core/src/utils/index.ts
index 318a7f13ba..ac17227459 100644
--- a/packages/backend-core/src/utils/index.ts
+++ b/packages/backend-core/src/utils/index.ts
@@ -1,3 +1,4 @@
 export * from "./hashing"
 export * from "./utils"
 export * from "./stringUtils"
+export * from "./Duration"
diff --git a/packages/backend-core/src/utils/tests/Duration.spec.ts b/packages/backend-core/src/utils/tests/Duration.spec.ts
new file mode 100644
index 0000000000..46b996f788
--- /dev/null
+++ b/packages/backend-core/src/utils/tests/Duration.spec.ts
@@ -0,0 +1,19 @@
+import { Duration, DurationType } from "../Duration"
+
+describe("duration", () => {
+  it("should convert minutes to milliseconds", () => {
+    expect(Duration.fromMinutes(5).toMs()).toBe(300000)
+  })
+
+  it("should convert seconds to milliseconds", () => {
+    expect(Duration.fromSeconds(30).toMs()).toBe(30000)
+  })
+
+  it("should convert days to milliseconds", () => {
+    expect(Duration.fromDays(1).toMs()).toBe(86400000)
+  })
+
+  it("should convert minutes to days", () => {
+    expect(Duration.fromMinutes(1440).to(DurationType.DAYS)).toBe(1)
+  })
+})