From 0921bcf3334e6a77a67b8771005587e2f37a3af4 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 1 Nov 2023 11:40:27 +0000
Subject: [PATCH 001/155] Add button setting to grid and allow removing all
 buttons

---
 .../ButtonConfiguration/ButtonConfiguration.svelte  | 13 ++++++++-----
 packages/client/manifest.json                       | 10 ++++++++++
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
index 324418511b..6fed8abdba 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
@@ -19,7 +19,7 @@
     componentBindings: componentBindings || [],
     bindings,
     removeButton,
-    canRemove: buttonCount > 1,
+    canRemove: true,
   }
 
   const sanitizeValue = val => {
@@ -86,11 +86,11 @@
       focus={focusItem}
       draggable={buttonCount > 1}
     />
-
-    <div class="list-footer" on:click={addButton}>
-      <div class="add-button">Add button</div>
-    </div>
   {/if}
+
+  <div class="list-footer" on:click={addButton} class:empty={!buttonCount}>
+    <div class="add-button">Add button</div>
+  </div>
 </div>
 
 <style>
@@ -120,6 +120,9 @@
       var(--spectrum-table-border-color, var(--spectrum-alias-border-color-mid));
     cursor: pointer;
   }
+  .list-footer.empty {
+    border-radius: 4px;
+  }
 
   .add-button {
     margin: var(--spacing-s);
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index c8ef406472..43b2d1b81d 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -5858,6 +5858,16 @@
         "label": "High contrast",
         "key": "stripeRows",
         "defaultValue": false
+      },
+      {
+        "section": true,
+        "name": "Buttons",
+        "settings": [
+          {
+            "type": "buttonConfiguration",
+            "key": "buttons"
+          }
+        ]
       }
     ]
   },

From 5c049e0465ca27ed078e3ffda4c5c91998ba97cd Mon Sep 17 00:00:00 2001
From: Mel O'Hagan <mel@budibase.com>
Date: Wed, 1 Nov 2023 13:54:18 +0000
Subject: [PATCH 002/155] LongFormField readonly prop

---
 .../bbui/src/Form/Core/RichTextField.svelte   |  2 ++
 packages/bbui/src/Form/Core/TextArea.svelte   |  2 ++
 .../bbui/src/Markdown/MarkdownEditor.svelte   |  5 ++-
 packages/client/manifest.json                 | 33 +++++++++++++++++++
 .../components/app/forms/LongFormField.svelte |  3 ++
 .../components/app/forms/StringField.svelte   |  2 ++
 6 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/packages/bbui/src/Form/Core/RichTextField.svelte b/packages/bbui/src/Form/Core/RichTextField.svelte
index f964405f0d..3e0b0caf4d 100644
--- a/packages/bbui/src/Form/Core/RichTextField.svelte
+++ b/packages/bbui/src/Form/Core/RichTextField.svelte
@@ -4,6 +4,7 @@
   export let value = ""
   export let placeholder = null
   export let disabled = false
+  export let readonly = false
   export let error = null
   export let height = null
   export let id = null
@@ -20,6 +21,7 @@
     {fullScreenOffset}
     {disabled}
     {easyMDEOptions}
+    {readonly}
     on:change
   />
 </div>
diff --git a/packages/bbui/src/Form/Core/TextArea.svelte b/packages/bbui/src/Form/Core/TextArea.svelte
index 465212cd44..be7eed466d 100644
--- a/packages/bbui/src/Form/Core/TextArea.svelte
+++ b/packages/bbui/src/Form/Core/TextArea.svelte
@@ -5,6 +5,7 @@
   export let value = ""
   export let placeholder = null
   export let disabled = false
+  export let readonly = false
   export let error = null
   export let id = null
   export let height = null
@@ -61,6 +62,7 @@
     class="spectrum-Textfield-input"
     style={align ? `text-align: ${align}` : ""}
     {disabled}
+    {readonly}
     {id}
     on:focus={() => (focus = true)}
     on:blur={onChange}
diff --git a/packages/bbui/src/Markdown/MarkdownEditor.svelte b/packages/bbui/src/Markdown/MarkdownEditor.svelte
index 7fb6414ad8..27035d8033 100644
--- a/packages/bbui/src/Markdown/MarkdownEditor.svelte
+++ b/packages/bbui/src/Markdown/MarkdownEditor.svelte
@@ -8,6 +8,7 @@
   export let id = null
   export let fullScreenOffset = 0
   export let disabled = false
+  export let readonly = false
   export let easyMDEOptions
 
   const dispatch = createEventDispatcher()
@@ -19,6 +20,7 @@
   // control
   $: checkValue(value)
   $: mde?.codemirror.on("change", debouncedUpdate)
+  $: mde?.codemirror.setOption("readOnly", readonly)
 
   const checkValue = val => {
     if (mde && val !== latestValue) {
@@ -43,7 +45,7 @@
   const debouncedUpdate = debounce(update, 250)
 </script>
 
-{#key height}
+{#key (height, readonly)}
   <SpectrumMDE
     bind:mde
     scroll={true}
@@ -54,6 +56,7 @@
     easyMDEOptions={{
       initialValue: value,
       placeholder,
+      toolbar: readonly ? false : undefined,
       ...easyMDEOptions,
     }}
   />
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index eef1e50b7c..6e175c44de 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -2589,6 +2589,17 @@
         "key": "disabled",
         "defaultValue": false
       },
+      {
+        "type": "boolean",
+        "label": "Read only",
+        "key": "readonly",
+        "defaultValue": false,
+        "dependsOn": {
+          "setting": "disabled",
+          "value": true,
+          "invert": true
+        }
+      },
       {
         "type": "text",
         "label": "Initial form step",
@@ -2738,6 +2749,17 @@
         "key": "disabled",
         "defaultValue": false
       },
+      {
+        "type": "boolean",
+        "label": "Read only",
+        "key": "readonly",
+        "defaultValue": false,
+        "dependsOn": {
+          "setting": "disabled",
+          "value": true,
+          "invert": true
+        }
+      },
       {
         "type": "validation/string",
         "label": "Validation",
@@ -3427,6 +3449,17 @@
         "key": "disabled",
         "defaultValue": false
       },
+      {
+        "type": "boolean",
+        "label": "Read only",
+        "key": "readonly",
+        "defaultValue": false,
+        "dependsOn": {
+          "setting": "disabled",
+          "value": true,
+          "invert": true
+        }
+      },
       {
         "type": "validation/string",
         "label": "Validation",
diff --git a/packages/client/src/components/app/forms/LongFormField.svelte b/packages/client/src/components/app/forms/LongFormField.svelte
index 8d94f83319..8482a6a68e 100644
--- a/packages/client/src/components/app/forms/LongFormField.svelte
+++ b/packages/client/src/components/app/forms/LongFormField.svelte
@@ -8,6 +8,7 @@
   export let label
   export let placeholder
   export let disabled = false
+  export let readonly = false
   export let validation
   export let defaultValue = ""
   export let format = "auto"
@@ -71,6 +72,7 @@
         value={fieldState.value}
         on:change={handleChange}
         disabled={fieldState.disabled}
+        {readonly}
         error={fieldState.error}
         id={fieldState.fieldId}
         {placeholder}
@@ -88,6 +90,7 @@
         value={fieldState.value}
         on:change={handleChange}
         disabled={fieldState.disabled}
+        {readonly}
         error={fieldState.error}
         id={fieldState.fieldId}
         {placeholder}
diff --git a/packages/client/src/components/app/forms/StringField.svelte b/packages/client/src/components/app/forms/StringField.svelte
index 26136b5d8d..624611c733 100644
--- a/packages/client/src/components/app/forms/StringField.svelte
+++ b/packages/client/src/components/app/forms/StringField.svelte
@@ -11,6 +11,7 @@
   export let defaultValue = ""
   export let align
   export let onChange
+  export let readonly = false
 
   let fieldState
   let fieldApi
@@ -44,6 +45,7 @@
       {placeholder}
       {type}
       {align}
+      {readonly}
     />
   {/if}
 </Field>

From 5c36d70a0116624acadc20865a7bba6f76d46d1d Mon Sep 17 00:00:00 2001
From: Mel O'Hagan <mel@budibase.com>
Date: Wed, 1 Nov 2023 14:56:28 +0000
Subject: [PATCH 003/155] Pickers readonly prop

---
 .../bbui/src/Form/Core/CheckboxGroup.svelte   |  6 +++++
 packages/bbui/src/Form/Core/RadioGroup.svelte |  6 +++++
 packages/client/manifest.json                 | 22 +++++++++++++++++++
 .../app/forms/MultiFieldSelect.svelte         |  3 +++
 .../components/app/forms/OptionsField.svelte  |  3 +++
 5 files changed, 40 insertions(+)

diff --git a/packages/bbui/src/Form/Core/CheckboxGroup.svelte b/packages/bbui/src/Form/Core/CheckboxGroup.svelte
index 2b8a1e438a..faf37f3ad8 100644
--- a/packages/bbui/src/Form/Core/CheckboxGroup.svelte
+++ b/packages/bbui/src/Form/Core/CheckboxGroup.svelte
@@ -8,6 +8,7 @@
   export let options = []
   export let error = null
   export let disabled = false
+  export let readonly = false
   export let getOptionLabel = option => option
   export let getOptionValue = option => option
 
@@ -40,6 +41,11 @@
         >
           <input
             on:change={onChange}
+            on:click={e => {
+              if (readonly) {
+                e.preventDefault()
+              }
+            }}
             type="checkbox"
             class="spectrum-Checkbox-input"
             value={optionValue}
diff --git a/packages/bbui/src/Form/Core/RadioGroup.svelte b/packages/bbui/src/Form/Core/RadioGroup.svelte
index f7afc10bbc..fc99fafd40 100644
--- a/packages/bbui/src/Form/Core/RadioGroup.svelte
+++ b/packages/bbui/src/Form/Core/RadioGroup.svelte
@@ -8,6 +8,7 @@
   export let options = []
   export let error = null
   export let disabled = false
+  export let readonly = false
   export let getOptionLabel = option => option
   export let getOptionValue = option => option
   export let getOptionTitle = option => option
@@ -43,6 +44,11 @@
       >
         <input
           on:change={onChange}
+          on:click={e => {
+            if (readonly) {
+              e.preventDefault()
+            }
+          }}
           bind:group={value}
           value={getOptionValue(option)}
           type="radio"
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 6e175c44de..90c897139c 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -3071,6 +3071,17 @@
         "key": "disabled",
         "defaultValue": false
       },
+      {
+        "type": "boolean",
+        "label": "Read only",
+        "key": "readonly",
+        "defaultValue": false,
+        "dependsOn": {
+          "setting": "disabled",
+          "value": true,
+          "invert": true
+        }
+      },
       {
         "type": "select",
         "label": "Options source",
@@ -3196,6 +3207,17 @@
         "key": "disabled",
         "defaultValue": false
       },
+      {
+        "type": "boolean",
+        "label": "Read only",
+        "key": "readonly",
+        "defaultValue": false,
+        "dependsOn": {
+          "setting": "disabled",
+          "value": true,
+          "invert": true
+        }
+      },
       {
         "type": "select",
         "label": "Type",
diff --git a/packages/client/src/components/app/forms/MultiFieldSelect.svelte b/packages/client/src/components/app/forms/MultiFieldSelect.svelte
index 88e1ec5a8e..cb4879f86e 100644
--- a/packages/client/src/components/app/forms/MultiFieldSelect.svelte
+++ b/packages/client/src/components/app/forms/MultiFieldSelect.svelte
@@ -6,6 +6,7 @@
   export let label
   export let placeholder
   export let disabled = false
+  export let readonly = false
   export let validation
   export let defaultValue
   export let optionsSource = "schema"
@@ -71,6 +72,7 @@
         getOptionValue={flatOptions ? x => x : x => x.value}
         id={fieldState.fieldId}
         disabled={fieldState.disabled}
+        {readonly}
         on:change={handleChange}
         {placeholder}
         {options}
@@ -81,6 +83,7 @@
         value={fieldState.value || []}
         id={fieldState.fieldId}
         disabled={fieldState.disabled}
+        {readonly}
         error={fieldState.error}
         {options}
         {direction}
diff --git a/packages/client/src/components/app/forms/OptionsField.svelte b/packages/client/src/components/app/forms/OptionsField.svelte
index 3c229c0509..c01827471a 100644
--- a/packages/client/src/components/app/forms/OptionsField.svelte
+++ b/packages/client/src/components/app/forms/OptionsField.svelte
@@ -6,6 +6,7 @@
   export let label
   export let placeholder
   export let disabled = false
+  export let readonly = false
   export let optionsType = "select"
   export let validation
   export let defaultValue
@@ -58,6 +59,7 @@
         value={fieldState.value}
         id={fieldState.fieldId}
         disabled={fieldState.disabled}
+        {readonly}
         error={fieldState.error}
         {options}
         {placeholder}
@@ -72,6 +74,7 @@
         value={fieldState.value}
         id={fieldState.fieldId}
         disabled={fieldState.disabled}
+        {readonly}
         error={fieldState.error}
         {options}
         {direction}

From 56d5a0b8f6fbc0e265bdc0068f253d601d56c62a Mon Sep 17 00:00:00 2001
From: Mel O'Hagan <mel@budibase.com>
Date: Wed, 1 Nov 2023 16:01:45 +0000
Subject: [PATCH 004/155] Further read only settings

---
 packages/bbui/src/Form/Core/Checkbox.svelte   |  6 ++
 packages/bbui/src/Form/Core/DatePicker.svelte |  4 +-
 packages/bbui/src/Form/DatePicker.svelte      |  2 +
 packages/client/manifest.json                 | 57 ++++++++++++++++++-
 .../components/app/forms/BooleanField.svelte  |  2 +
 .../components/app/forms/DateTimeField.svelte |  2 +
 .../src/components/app/forms/JSONField.svelte |  2 +
 .../app/forms/RelationshipField.svelte        |  2 +
 8 files changed, 75 insertions(+), 2 deletions(-)

diff --git a/packages/bbui/src/Form/Core/Checkbox.svelte b/packages/bbui/src/Form/Core/Checkbox.svelte
index 3efc737bfb..3eaaf4dede 100644
--- a/packages/bbui/src/Form/Core/Checkbox.svelte
+++ b/packages/bbui/src/Form/Core/Checkbox.svelte
@@ -8,6 +8,7 @@
   export let id = null
   export let text = null
   export let disabled = false
+  export let readonly = false
   export let size
   export let indeterminate = false
 
@@ -29,6 +30,11 @@
     checked={value}
     {disabled}
     on:change={onChange}
+    on:click={e => {
+      if (readonly) {
+        e.preventDefault()
+      }
+    }}
     type="checkbox"
     class="spectrum-Checkbox-input"
     {id}
diff --git a/packages/bbui/src/Form/Core/DatePicker.svelte b/packages/bbui/src/Form/Core/DatePicker.svelte
index 7ce15292be..786aee40b6 100644
--- a/packages/bbui/src/Form/Core/DatePicker.svelte
+++ b/packages/bbui/src/Form/Core/DatePicker.svelte
@@ -9,6 +9,7 @@
 
   export let id = null
   export let disabled = false
+  export let readonly = false
   export let error = null
   export let enableTime = true
   export let value = null
@@ -186,7 +187,7 @@
   >
     <div
       id={flatpickrId}
-      class:is-disabled={disabled}
+      class:is-disabled={disabled || readonly}
       class:is-invalid={!!error}
       class="flatpickr spectrum-InputGroup spectrum-Datepicker"
       class:is-focused={open}
@@ -211,6 +212,7 @@
         {/if}
         <input
           {disabled}
+          {readonly}
           data-input
           type="text"
           class="spectrum-Textfield-input spectrum-InputGroup-input"
diff --git a/packages/bbui/src/Form/DatePicker.svelte b/packages/bbui/src/Form/DatePicker.svelte
index 04ce8b5467..f17871a576 100644
--- a/packages/bbui/src/Form/DatePicker.svelte
+++ b/packages/bbui/src/Form/DatePicker.svelte
@@ -7,6 +7,7 @@
   export let label = null
   export let labelPosition = "above"
   export let disabled = false
+  export let readonly = false
   export let error = null
   export let enableTime = true
   export let timeOnly = false
@@ -33,6 +34,7 @@
   <DatePicker
     {error}
     {disabled}
+    {readonly}
     {value}
     {placeholder}
     {enableTime}
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 90c897139c..7559825267 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -2851,6 +2851,17 @@
         "key": "disabled",
         "defaultValue": false
       },
+      {
+        "type": "boolean",
+        "label": "Read only",
+        "key": "readonly",
+        "defaultValue": false,
+        "dependsOn": {
+          "setting": "disabled",
+          "value": true,
+          "invert": true
+        }
+      },
       {
         "type": "validation/number",
         "label": "Validation",
@@ -3392,6 +3403,17 @@
         "key": "disabled",
         "defaultValue": false
       },
+      {
+        "type": "boolean",
+        "label": "Read only",
+        "key": "readonly",
+        "defaultValue": false,
+        "dependsOn": {
+          "setting": "disabled",
+          "value": true,
+          "invert": true
+        }
+      },
       {
         "type": "validation/boolean",
         "label": "Validation",
@@ -3563,6 +3585,17 @@
         "key": "disabled",
         "defaultValue": false
       },
+      {
+        "type": "boolean",
+        "label": "Read only",
+        "key": "readonly",
+        "defaultValue": false,
+        "dependsOn": {
+          "setting": "disabled",
+          "value": true,
+          "invert": true
+        }
+      },
       {
         "type": "validation/datetime",
         "label": "Validation",
@@ -3836,7 +3869,7 @@
       },
       {
         "type": "boolean",
-        "label": "Disabled",
+        "label": "Read only",
         "key": "disabled",
         "defaultValue": false
       },
@@ -3912,6 +3945,17 @@
         "label": "Disabled",
         "key": "disabled",
         "defaultValue": false
+      },
+      {
+        "type": "boolean",
+        "label": "Read only",
+        "key": "readonly",
+        "defaultValue": false,
+        "dependsOn": {
+          "setting": "disabled",
+          "value": true,
+          "invert": true
+        }
       }
     ]
   },
@@ -3964,6 +4008,17 @@
         "label": "Disabled",
         "key": "disabled",
         "defaultValue": false
+      },
+      {
+        "type": "boolean",
+        "label": "Read only",
+        "key": "readonly",
+        "defaultValue": false,
+        "dependsOn": {
+          "setting": "disabled",
+          "value": true,
+          "invert": true
+        }
       }
     ]
   },
diff --git a/packages/client/src/components/app/forms/BooleanField.svelte b/packages/client/src/components/app/forms/BooleanField.svelte
index a65d041c29..9635a7d7b5 100644
--- a/packages/client/src/components/app/forms/BooleanField.svelte
+++ b/packages/client/src/components/app/forms/BooleanField.svelte
@@ -6,6 +6,7 @@
   export let label
   export let text
   export let disabled = false
+  export let readonly = false
   export let size
   export let validation
   export let defaultValue
@@ -49,6 +50,7 @@
     <CoreCheckbox
       value={fieldState.value}
       disabled={fieldState.disabled}
+      {readonly}
       error={fieldState.error}
       id={fieldState.fieldId}
       {size}
diff --git a/packages/client/src/components/app/forms/DateTimeField.svelte b/packages/client/src/components/app/forms/DateTimeField.svelte
index 6bcd20d250..8124c7861a 100644
--- a/packages/client/src/components/app/forms/DateTimeField.svelte
+++ b/packages/client/src/components/app/forms/DateTimeField.svelte
@@ -6,6 +6,7 @@
   export let label
   export let placeholder
   export let disabled = false
+  export let readonly = false
   export let enableTime = true
   export let timeOnly = false
   export let time24hr = false
@@ -40,6 +41,7 @@
       value={fieldState.value}
       on:change={handleChange}
       disabled={fieldState.disabled}
+      {readonly}
       error={fieldState.error}
       id={fieldState.fieldId}
       appendTo={document.getElementById("flatpickr-root")}
diff --git a/packages/client/src/components/app/forms/JSONField.svelte b/packages/client/src/components/app/forms/JSONField.svelte
index c80060d3d6..1ee09c8460 100644
--- a/packages/client/src/components/app/forms/JSONField.svelte
+++ b/packages/client/src/components/app/forms/JSONField.svelte
@@ -7,6 +7,7 @@
   export let label
   export let placeholder
   export let disabled = false
+  export let readonly = false
   export let defaultValue = ""
   export let onChange
 
@@ -60,6 +61,7 @@
         value={serialiseValue(fieldState.value)}
         on:change={handleChange}
         disabled={fieldState.disabled}
+        {readonly}
         error={fieldState.error}
         id={fieldState.fieldId}
         {placeholder}
diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte
index 544a1a8434..9d2d6adf0f 100644
--- a/packages/client/src/components/app/forms/RelationshipField.svelte
+++ b/packages/client/src/components/app/forms/RelationshipField.svelte
@@ -11,6 +11,7 @@
   export let label
   export let placeholder
   export let disabled = false
+  export let readonly = false
   export let validation
   export let autocomplete = true
   export let defaultValue
@@ -200,6 +201,7 @@
       on:loadMore={loadMore}
       id={fieldState.fieldId}
       disabled={fieldState.disabled}
+      {readonly}
       error={fieldState.error}
       getOptionLabel={getDisplayName}
       getOptionValue={option => option._id}

From 33e37261b2e8a16404efb68467eb27eea7e42ac7 Mon Sep 17 00:00:00 2001
From: Mel O'Hagan <mel@budibase.com>
Date: Wed, 1 Nov 2023 16:27:52 +0000
Subject: [PATCH 005/155] Use fieldState

---
 .../client/src/components/app/forms/AttachmentField.svelte   | 4 +++-
 packages/client/src/components/app/forms/BooleanField.svelte | 3 ++-
 .../client/src/components/app/forms/CodeScannerField.svelte  | 4 +++-
 .../client/src/components/app/forms/DateTimeField.svelte     | 3 ++-
 packages/client/src/components/app/forms/Field.svelte        | 2 ++
 packages/client/src/components/app/forms/Form.svelte         | 4 +++-
 packages/client/src/components/app/forms/InnerForm.svelte    | 3 +++
 packages/client/src/components/app/forms/JSONField.svelte    | 3 ++-
 .../client/src/components/app/forms/LongFormField.svelte     | 5 +++--
 .../client/src/components/app/forms/MultiFieldSelect.svelte  | 5 +++--
 packages/client/src/components/app/forms/OptionsField.svelte | 5 +++--
 .../client/src/components/app/forms/RelationshipField.svelte | 3 ++-
 packages/client/src/components/app/forms/StringField.svelte  | 5 +++--
 13 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/packages/client/src/components/app/forms/AttachmentField.svelte b/packages/client/src/components/app/forms/AttachmentField.svelte
index e24115ebc0..861d881733 100644
--- a/packages/client/src/components/app/forms/AttachmentField.svelte
+++ b/packages/client/src/components/app/forms/AttachmentField.svelte
@@ -6,6 +6,7 @@
   export let field
   export let label
   export let disabled = false
+  export let readonly = false
   export let compact = false
   export let validation
   export let extensions
@@ -71,6 +72,7 @@
   {label}
   {field}
   {disabled}
+  {readonly}
   {validation}
   type="attachment"
   bind:fieldState
@@ -81,7 +83,7 @@
     {#if fieldState}
       <CoreDropzone
         value={fieldState.value}
-        disabled={fieldState.disabled}
+        disabled={fieldState.disabled || fieldState.readonly}
         error={fieldState.error}
         on:change={handleChange}
         {processFiles}
diff --git a/packages/client/src/components/app/forms/BooleanField.svelte b/packages/client/src/components/app/forms/BooleanField.svelte
index 9635a7d7b5..1f59ddcfa6 100644
--- a/packages/client/src/components/app/forms/BooleanField.svelte
+++ b/packages/client/src/components/app/forms/BooleanField.svelte
@@ -40,6 +40,7 @@
   {label}
   {field}
   {disabled}
+  {readonly}
   {validation}
   defaultValue={isTruthy(defaultValue)}
   type="boolean"
@@ -50,7 +51,7 @@
     <CoreCheckbox
       value={fieldState.value}
       disabled={fieldState.disabled}
-      {readonly}
+      readonly={fieldState.readonly}
       error={fieldState.error}
       id={fieldState.fieldId}
       {size}
diff --git a/packages/client/src/components/app/forms/CodeScannerField.svelte b/packages/client/src/components/app/forms/CodeScannerField.svelte
index c408f78d7c..a9c115e852 100644
--- a/packages/client/src/components/app/forms/CodeScannerField.svelte
+++ b/packages/client/src/components/app/forms/CodeScannerField.svelte
@@ -6,6 +6,7 @@
   export let label
   export let type = "barcodeqr"
   export let disabled = false
+  export let readonly = false
   export let validation
   export let defaultValue = ""
   export let onChange
@@ -32,6 +33,7 @@
   {label}
   {field}
   {disabled}
+  {readonly}
   {validation}
   {defaultValue}
   {type}
@@ -42,7 +44,7 @@
     <CodeScanner
       value={fieldState.value}
       on:change={handleUpdate}
-      disabled={fieldState.disabled}
+      disabled={fieldState.disabled || fieldState.readonly}
       {allowManualEntry}
       scanButtonText={scanText}
       {beepOnScan}
diff --git a/packages/client/src/components/app/forms/DateTimeField.svelte b/packages/client/src/components/app/forms/DateTimeField.svelte
index 8124c7861a..22a76e5ef2 100644
--- a/packages/client/src/components/app/forms/DateTimeField.svelte
+++ b/packages/client/src/components/app/forms/DateTimeField.svelte
@@ -30,6 +30,7 @@
   {label}
   {field}
   {disabled}
+  {readonly}
   {validation}
   {defaultValue}
   type="datetime"
@@ -41,7 +42,7 @@
       value={fieldState.value}
       on:change={handleChange}
       disabled={fieldState.disabled}
-      {readonly}
+      readonly={fieldState.readonly}
       error={fieldState.error}
       id={fieldState.fieldId}
       appendTo={document.getElementById("flatpickr-root")}
diff --git a/packages/client/src/components/app/forms/Field.svelte b/packages/client/src/components/app/forms/Field.svelte
index 5d4da5afef..16115583a7 100644
--- a/packages/client/src/components/app/forms/Field.svelte
+++ b/packages/client/src/components/app/forms/Field.svelte
@@ -11,6 +11,7 @@
   export let defaultValue
   export let type
   export let disabled = false
+  export let readonly = false
   export let validation
 
   // Get contexts
@@ -29,6 +30,7 @@
     type,
     defaultValue,
     disabled,
+    readonly,
     validation,
     formStep
   )
diff --git a/packages/client/src/components/app/forms/Form.svelte b/packages/client/src/components/app/forms/Form.svelte
index 87883fe4b6..1a740585f3 100644
--- a/packages/client/src/components/app/forms/Form.svelte
+++ b/packages/client/src/components/app/forms/Form.svelte
@@ -8,6 +8,7 @@
   export let theme
   export let size
   export let disabled = false
+  export let readonly = false
   export let actionType = "Create"
   export let initialFormStep = 1
 
@@ -39,7 +40,7 @@
   $: schemaKey = generateSchemaKey(schema)
   $: initialValues = getInitialValues(actionType, dataSource, $context)
   $: resetKey = Helpers.hashString(
-    schemaKey + JSON.stringify(initialValues) + disabled
+    schemaKey + JSON.stringify(initialValues) + disabled + readonly
   )
 
   // Returns the closes data context which isn't a built in context
@@ -97,6 +98,7 @@
       {theme}
       {size}
       {disabled}
+      {readonly}
       {actionType}
       {schema}
       {table}
diff --git a/packages/client/src/components/app/forms/InnerForm.svelte b/packages/client/src/components/app/forms/InnerForm.svelte
index 4dacf36244..6ebe9de7ec 100644
--- a/packages/client/src/components/app/forms/InnerForm.svelte
+++ b/packages/client/src/components/app/forms/InnerForm.svelte
@@ -6,6 +6,7 @@
 
   export let dataSource
   export let disabled = false
+  export let readonly = false
   export let initialValues
   export let size
   export let schema
@@ -148,6 +149,7 @@
       type,
       defaultValue = null,
       fieldDisabled = false,
+      fieldReadOnly = false,
       validationRules,
       step = 1
     ) => {
@@ -205,6 +207,7 @@
           error: initialError,
           disabled:
             disabled || fieldDisabled || (isAutoColumn && !editAutoColumns),
+          readonly: readonly || fieldReadOnly,
           defaultValue,
           validator,
           lastUpdate: Date.now(),
diff --git a/packages/client/src/components/app/forms/JSONField.svelte b/packages/client/src/components/app/forms/JSONField.svelte
index 1ee09c8460..cf96f54a23 100644
--- a/packages/client/src/components/app/forms/JSONField.svelte
+++ b/packages/client/src/components/app/forms/JSONField.svelte
@@ -49,6 +49,7 @@
   {label}
   {field}
   {disabled}
+  {readonly}
   {validation}
   {defaultValue}
   type="json"
@@ -61,7 +62,7 @@
         value={serialiseValue(fieldState.value)}
         on:change={handleChange}
         disabled={fieldState.disabled}
-        {readonly}
+        readonly={fieldState.readonly}
         error={fieldState.error}
         id={fieldState.fieldId}
         {placeholder}
diff --git a/packages/client/src/components/app/forms/LongFormField.svelte b/packages/client/src/components/app/forms/LongFormField.svelte
index 8482a6a68e..a9087a0a9c 100644
--- a/packages/client/src/components/app/forms/LongFormField.svelte
+++ b/packages/client/src/components/app/forms/LongFormField.svelte
@@ -59,6 +59,7 @@
   {label}
   {field}
   {disabled}
+  {readonly}
   {validation}
   {defaultValue}
   type="longform"
@@ -72,7 +73,7 @@
         value={fieldState.value}
         on:change={handleChange}
         disabled={fieldState.disabled}
-        {readonly}
+        readonly={fieldState.readonly}
         error={fieldState.error}
         id={fieldState.fieldId}
         {placeholder}
@@ -90,7 +91,7 @@
         value={fieldState.value}
         on:change={handleChange}
         disabled={fieldState.disabled}
-        {readonly}
+        readonly={fieldState.readonly}
         error={fieldState.error}
         id={fieldState.fieldId}
         {placeholder}
diff --git a/packages/client/src/components/app/forms/MultiFieldSelect.svelte b/packages/client/src/components/app/forms/MultiFieldSelect.svelte
index cb4879f86e..4ee691061a 100644
--- a/packages/client/src/components/app/forms/MultiFieldSelect.svelte
+++ b/packages/client/src/components/app/forms/MultiFieldSelect.svelte
@@ -56,6 +56,7 @@
   {field}
   {label}
   {disabled}
+  {readonly}
   {validation}
   defaultValue={expandedDefaultValue}
   type="array"
@@ -72,7 +73,7 @@
         getOptionValue={flatOptions ? x => x : x => x.value}
         id={fieldState.fieldId}
         disabled={fieldState.disabled}
-        {readonly}
+        readonly={fieldState.readonly}
         on:change={handleChange}
         {placeholder}
         {options}
@@ -83,7 +84,7 @@
         value={fieldState.value || []}
         id={fieldState.fieldId}
         disabled={fieldState.disabled}
-        {readonly}
+        readonly={fieldState.readonly}
         error={fieldState.error}
         {options}
         {direction}
diff --git a/packages/client/src/components/app/forms/OptionsField.svelte b/packages/client/src/components/app/forms/OptionsField.svelte
index c01827471a..dc18df8dbe 100644
--- a/packages/client/src/components/app/forms/OptionsField.svelte
+++ b/packages/client/src/components/app/forms/OptionsField.svelte
@@ -46,6 +46,7 @@
   {field}
   {label}
   {disabled}
+  {readonly}
   {validation}
   {defaultValue}
   type="options"
@@ -59,7 +60,7 @@
         value={fieldState.value}
         id={fieldState.fieldId}
         disabled={fieldState.disabled}
-        {readonly}
+        readonly={fieldState.readonly}
         error={fieldState.error}
         {options}
         {placeholder}
@@ -74,7 +75,7 @@
         value={fieldState.value}
         id={fieldState.fieldId}
         disabled={fieldState.disabled}
-        {readonly}
+        readonly={fieldState.readonly}
         error={fieldState.error}
         {options}
         {direction}
diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte
index 9d2d6adf0f..10fcaa904f 100644
--- a/packages/client/src/components/app/forms/RelationshipField.svelte
+++ b/packages/client/src/components/app/forms/RelationshipField.svelte
@@ -184,6 +184,7 @@
   {label}
   {field}
   {disabled}
+  {readonly}
   {validation}
   defaultValue={expandedDefaultValue}
   {type}
@@ -201,7 +202,7 @@
       on:loadMore={loadMore}
       id={fieldState.fieldId}
       disabled={fieldState.disabled}
-      {readonly}
+      readonly={fieldState.readonly}
       error={fieldState.error}
       getOptionLabel={getDisplayName}
       getOptionValue={option => option._id}
diff --git a/packages/client/src/components/app/forms/StringField.svelte b/packages/client/src/components/app/forms/StringField.svelte
index 624611c733..674be9f1b2 100644
--- a/packages/client/src/components/app/forms/StringField.svelte
+++ b/packages/client/src/components/app/forms/StringField.svelte
@@ -7,11 +7,11 @@
   export let placeholder
   export let type = "text"
   export let disabled = false
+  export let readonly = false
   export let validation
   export let defaultValue = ""
   export let align
   export let onChange
-  export let readonly = false
 
   let fieldState
   let fieldApi
@@ -28,6 +28,7 @@
   {label}
   {field}
   {disabled}
+  {readonly}
   {validation}
   {defaultValue}
   type={type === "number" ? "number" : "string"}
@@ -40,12 +41,12 @@
       value={fieldState.value}
       on:change={handleChange}
       disabled={fieldState.disabled}
+      readonly={fieldState.readonly}
       error={fieldState.error}
       id={fieldState.fieldId}
       {placeholder}
       {type}
       {align}
-      {readonly}
     />
   {/if}
 </Field>

From 5923ae2983d588d1f1d5a5fd3550a04cda47d671 Mon Sep 17 00:00:00 2001
From: Mel O'Hagan <mel@budibase.com>
Date: Wed, 1 Nov 2023 16:40:23 +0000
Subject: [PATCH 006/155] Make form block view readonly

---
 packages/bbui/src/Markdown/MarkdownEditor.svelte           | 2 +-
 packages/client/manifest.json                              | 7 +------
 .../src/components/app/blocks/form/InnerFormBlock.svelte   | 3 ++-
 3 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/packages/bbui/src/Markdown/MarkdownEditor.svelte b/packages/bbui/src/Markdown/MarkdownEditor.svelte
index 27035d8033..225d25a0eb 100644
--- a/packages/bbui/src/Markdown/MarkdownEditor.svelte
+++ b/packages/bbui/src/Markdown/MarkdownEditor.svelte
@@ -56,7 +56,7 @@
     easyMDEOptions={{
       initialValue: value,
       placeholder,
-      toolbar: readonly ? false : undefined,
+      toolbar: disabled || readonly ? false : undefined,
       ...easyMDEOptions,
     }}
   />
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 7559825267..749e2cf194 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -5640,12 +5640,7 @@
             "type": "boolean",
             "label": "Disabled",
             "key": "disabled",
-            "defaultValue": false,
-            "dependsOn": {
-              "setting": "actionType",
-              "value": "View",
-              "invert": true
-            }
+            "defaultValue": false
           }
         ]
       },
diff --git a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte
index f7e9a0d2ed..b2c9888c73 100644
--- a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte
+++ b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte
@@ -136,7 +136,8 @@
       actionType: actionType === "Create" ? "Create" : "Update",
       dataSource,
       size,
-      disabled: disabled || actionType === "View",
+      disabled,
+      readonly: !disabled && actionType === "View",
     }}
     styles={{
       normal: {

From 2b8bbafcac4cb9aba6b79e89e2007ca4c6b708f0 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 1 Nov 2023 16:45:37 +0000
Subject: [PATCH 007/155] Add support for buttons inside grids

---
 .../src/components/app/GridBlock.svelte       |  27 +++-
 packages/client/src/sdk.js                    |  14 +-
 .../src/components/grid/cells/GridCell.svelte |   2 +-
 .../grid/layout/ButtonColumn.svelte           | 137 ++++++++++++++++++
 .../src/components/grid/layout/Grid.svelte    |   2 +
 .../components/grid/layout/GridBody.svelte    |   5 +
 .../src/components/grid/stores/scroll.js      |  18 ++-
 .../src/components/grid/stores/ui.js          |   2 +
 8 files changed, 198 insertions(+), 9 deletions(-)
 create mode 100644 packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte

diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index 8febaf44ff..9f93b6c4ff 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -2,6 +2,7 @@
   // NOTE: this is not a block - it's just named as such to avoid confusing users,
   // because it functions similarly to one
   import { getContext } from "svelte"
+  import { get } from "svelte/store"
   import { Grid } from "@budibase/frontend-core"
 
   // table is actually any datasource, but called table for legacy compatibility
@@ -16,12 +17,21 @@
   export let fixedRowHeight = null
   export let columns = null
   export let onRowClick = null
+  export let buttons = null
 
+  const context = getContext("context")
   const component = getContext("component")
-  const { styleable, API, builderStore, notificationStore } = getContext("sdk")
+  const {
+    styleable,
+    API,
+    builderStore,
+    notificationStore,
+    enrichButtonActions,
+  } = getContext("sdk")
 
   $: columnWhitelist = columns?.map(col => col.name)
   $: schemaOverrides = getSchemaOverrides(columns)
+  $: parsedButtons = parseButtons(buttons)
 
   const getSchemaOverrides = columns => {
     let overrides = {}
@@ -33,6 +43,20 @@
     })
     return overrides
   }
+
+  const parseButtons = buttons => {
+    if (!buttons?.length) {
+      return null
+    }
+    return buttons.map(settings => ({
+      size: "M",
+      text: settings.text,
+      type: settings.type,
+      onClick: async () => {
+        return await enrichButtonActions(settings.onClick, get(context))()
+      },
+    }))
+  }
 </script>
 
 <div
@@ -58,6 +82,7 @@
     showControls={false}
     notifySuccess={notificationStore.actions.success}
     notifyError={notificationStore.actions.error}
+    buttons={parsedButtons}
     on:rowclick={e => onRowClick?.({ row: e.detail })}
   />
 </div>
diff --git a/packages/client/src/sdk.js b/packages/client/src/sdk.js
index 1d77f3e2da..77089d648b 100644
--- a/packages/client/src/sdk.js
+++ b/packages/client/src/sdk.js
@@ -24,9 +24,12 @@ import BlockComponent from "components/BlockComponent.svelte"
 import { ActionTypes } from "./constants"
 import { fetchDatasourceSchema } from "./utils/schema.js"
 import { getAPIKey } from "./utils/api.js"
+import { enrichButtonActions } from "./utils/buttonActions.js"
 
 export default {
   API,
+
+  // Stores
   authStore,
   notificationStore,
   routeStore,
@@ -41,13 +44,20 @@ export default {
   currentRole,
   confirmationStore,
   roleStore,
+
+  // Utils
   styleable,
   linkable,
   getAction,
   fetchDatasourceSchema,
-  Provider,
-  ActionTypes,
   getAPIKey,
+  enrichButtonActions,
+
+  // Components
+  Provider,
   Block,
   BlockComponent,
+
+  // Constants
+  ActionTypes,
 }
diff --git a/packages/frontend-core/src/components/grid/cells/GridCell.svelte b/packages/frontend-core/src/components/grid/cells/GridCell.svelte
index dcc76b9c75..d10eb25528 100644
--- a/packages/frontend-core/src/components/grid/cells/GridCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/GridCell.svelte
@@ -15,7 +15,7 @@
   $: style = getStyle(width, selectedUser)
 
   const getStyle = (width, selectedUser) => {
-    let style = `flex: 0 0 ${width}px;`
+    let style = width === "auto" ? "width: auto;" : `flex: 0 0 ${width}px;`
     if (selectedUser) {
       style += `--user-color:${selectedUser.color};`
     }
diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
new file mode 100644
index 0000000000..4acd8b6b70
--- /dev/null
+++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
@@ -0,0 +1,137 @@
+<script>
+  import { getContext, onMount } from "svelte"
+  import { Button } from "@budibase/bbui"
+  import GridCell from "../cells/GridCell.svelte"
+  import GridScrollWrapper from "./GridScrollWrapper.svelte"
+
+  const {
+    renderedRows,
+    hoveredRowId,
+    props,
+    width,
+    focusedRow,
+    selectedRows,
+    visibleColumns,
+    scroll,
+    isDragging,
+    buttonColumnWidth,
+  } = getContext("grid")
+
+  let measureContainer
+
+  $: buttons = $props.buttons?.slice(0, 3) || []
+  $: columnsWidth = $visibleColumns.reduce(
+    (total, col) => (total += col.width),
+    0
+  )
+  $: end = columnsWidth - 1 - $scroll.left
+  $: left = Math.min($width - $buttonColumnWidth, end)
+
+  onMount(() => {
+    const observer = new ResizeObserver(entries => {
+      const width = entries?.[0]?.contentRect?.width ?? 0
+      buttonColumnWidth.set(width)
+    })
+    observer.observe(measureContainer)
+  })
+</script>
+
+<!-- Hidden copy of buttons to measure -->
+<div class="measure" bind:this={measureContainer}>
+  <GridCell width="auto">
+    <div class="buttons">
+      {#each buttons as button}
+        <Button size="S">
+          {button.text || "Button"}
+        </Button>
+      {/each}
+    </div>
+  </GridCell>
+</div>
+
+<div
+  class="button-column"
+  style="left:{left}px"
+  class:hidden={$buttonColumnWidth === 0}
+>
+  <div class="content" on:mouseleave={() => ($hoveredRowId = null)}>
+    <GridScrollWrapper scrollVertically attachHandlers>
+      {#each $renderedRows as row}
+        {@const rowSelected = !!$selectedRows[row._id]}
+        {@const rowHovered = $hoveredRowId === row._id}
+        {@const rowFocused = $focusedRow?._id === row._id}
+        <div
+          class="row"
+          on:mouseenter={$isDragging ? null : () => ($hoveredRowId = row._id)}
+          on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
+        >
+          <GridCell
+            width="auto"
+            rowIdx={row.__idx}
+            selected={rowSelected}
+            highlighted={rowHovered || rowFocused}
+          >
+            <div class="buttons">
+              {#each buttons as button}
+                <Button
+                  newStyles
+                  size="S"
+                  cta={button.type === "cta"}
+                  primary={button.type === "primary"}
+                  secondary={button.type === "secondary"}
+                  warning={button.type === "warning"}
+                  overBackground={button.type === "overBackground"}
+                  on:click={() => button.onClick?.(row)}
+                >
+                  {button.text || "Button"}
+                </Button>
+              {/each}
+            </div>
+          </GridCell>
+        </div>
+      {/each}
+    </GridScrollWrapper>
+  </div>
+</div>
+
+<style>
+  .button-column {
+    display: flex;
+    flex-direction: column;
+    background: var(--cell-background);
+    position: absolute;
+    top: 0;
+  }
+  .button-column.hidden {
+    opacity: 0;
+  }
+  .content {
+    position: relative;
+    flex: 1 1 auto;
+  }
+  .row {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: stretch;
+  }
+  .buttons {
+    display: flex;
+    align-items: center;
+    padding: 0 var(--cell-padding);
+    gap: var(--cell-padding);
+    height: inherit;
+  }
+
+  /* Add left cell border */
+  .button-column :global(.cell) {
+    border-left: var(--cell-border);
+  }
+
+  /* Hidden copy of buttons to measure width against */
+  .measure {
+    position: absolute;
+    opacity: 0;
+    pointer-events: none;
+  }
+</style>
diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte
index e2ecd0f968..285282ddf7 100644
--- a/packages/frontend-core/src/components/grid/layout/Grid.svelte
+++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte
@@ -48,6 +48,7 @@
   export let fixedRowHeight = null
   export let notifySuccess = null
   export let notifyError = null
+  export let buttons = null
 
   // Unique identifier for DOM nodes inside this instance
   const rand = Math.random()
@@ -99,6 +100,7 @@
     fixedRowHeight,
     notifySuccess,
     notifyError,
+    buttons,
   })
 
   // Set context for children to consume
diff --git a/packages/frontend-core/src/components/grid/layout/GridBody.svelte b/packages/frontend-core/src/components/grid/layout/GridBody.svelte
index 0bb2a51fb4..559c465b86 100644
--- a/packages/frontend-core/src/components/grid/layout/GridBody.svelte
+++ b/packages/frontend-core/src/components/grid/layout/GridBody.svelte
@@ -3,6 +3,7 @@
   import GridScrollWrapper from "./GridScrollWrapper.svelte"
   import GridRow from "./GridRow.svelte"
   import { BlankRowID } from "../lib/constants"
+  import ButtonColumn from "./ButtonColumn.svelte"
 
   const {
     bounds,
@@ -13,6 +14,7 @@
     dispatch,
     isDragging,
     config,
+    props,
   } = getContext("grid")
 
   let body
@@ -54,6 +56,9 @@
       />
     {/if}
   </GridScrollWrapper>
+  {#if $props.buttons?.length}
+    <ButtonColumn />
+  {/if}
 </div>
 
 <style>
diff --git a/packages/frontend-core/src/components/grid/stores/scroll.js b/packages/frontend-core/src/components/grid/stores/scroll.js
index 89e759854f..88a676f078 100644
--- a/packages/frontend-core/src/components/grid/stores/scroll.js
+++ b/packages/frontend-core/src/components/grid/stores/scroll.js
@@ -20,8 +20,15 @@ export const createStores = () => {
 }
 
 export const deriveStores = context => {
-  const { rows, visibleColumns, stickyColumn, rowHeight, width, height } =
-    context
+  const {
+    rows,
+    visibleColumns,
+    stickyColumn,
+    rowHeight,
+    width,
+    height,
+    buttonColumnWidth,
+  } = context
 
   // Memoize store primitives
   const stickyColumnWidth = derived(stickyColumn, $col => $col?.width || 0, 0)
@@ -40,9 +47,10 @@ export const deriveStores = context => {
 
   // Derive horizontal limits
   const contentWidth = derived(
-    [visibleColumns, stickyColumnWidth],
-    ([$visibleColumns, $stickyColumnWidth]) => {
-      let width = GutterWidth + Padding + $stickyColumnWidth
+    [visibleColumns, stickyColumnWidth, buttonColumnWidth],
+    ([$visibleColumns, $stickyColumnWidth, $buttonColumnWidth]) => {
+      const space = Math.max(Padding, $buttonColumnWidth - 1)
+      let width = GutterWidth + space + $stickyColumnWidth
       $visibleColumns.forEach(col => {
         width += col.width
       })
diff --git a/packages/frontend-core/src/components/grid/stores/ui.js b/packages/frontend-core/src/components/grid/stores/ui.js
index e10e96a722..129d6614e5 100644
--- a/packages/frontend-core/src/components/grid/stores/ui.js
+++ b/packages/frontend-core/src/components/grid/stores/ui.js
@@ -18,6 +18,7 @@ export const createStores = context => {
   const previousFocusedRowId = writable(null)
   const gridFocused = writable(false)
   const isDragging = writable(false)
+  const buttonColumnWidth = writable(0)
 
   // Derive the current focused row ID
   const focusedRowId = derived(
@@ -51,6 +52,7 @@ export const createStores = context => {
     rowHeight,
     gridFocused,
     isDragging,
+    buttonColumnWidth,
     selectedRows: {
       ...selectedRows,
       actions: {

From 4ab9af53a099405b4f5db138af386e565702644f Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 2 Nov 2023 08:53:48 +0000
Subject: [PATCH 008/155] Add row to grid button click callback

---
 packages/client/src/components/app/GridBlock.svelte | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index 9f93b6c4ff..2b06e41986 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -31,7 +31,7 @@
 
   $: columnWhitelist = columns?.map(col => col.name)
   $: schemaOverrides = getSchemaOverrides(columns)
-  $: parsedButtons = parseButtons(buttons)
+  $: enrichedButtons = enrichButtons(buttons)
 
   const getSchemaOverrides = columns => {
     let overrides = {}
@@ -44,7 +44,7 @@
     return overrides
   }
 
-  const parseButtons = buttons => {
+  const enrichButtons = buttons => {
     if (!buttons?.length) {
       return null
     }
@@ -52,8 +52,9 @@
       size: "M",
       text: settings.text,
       type: settings.type,
-      onClick: async () => {
-        return await enrichButtonActions(settings.onClick, get(context))()
+      onClick: async row => {
+        const fn = enrichButtonActions(settings.onClick, get(context))
+        return await fn({ row })
       },
     }))
   }
@@ -82,7 +83,7 @@
     showControls={false}
     notifySuccess={notificationStore.actions.success}
     notifyError={notificationStore.actions.error}
-    buttons={parsedButtons}
+    buttons={enrichedButtons}
     on:rowclick={e => onRowClick?.({ row: e.detail })}
   />
 </div>

From 738082dc27e564530e7af107c99beb7c27956733 Mon Sep 17 00:00:00 2001
From: Mel O'Hagan <mel@budibase.com>
Date: Thu, 2 Nov 2023 15:49:16 +0000
Subject: [PATCH 009/155] Remove readonly key

---
 packages/bbui/src/Markdown/MarkdownEditor.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/bbui/src/Markdown/MarkdownEditor.svelte b/packages/bbui/src/Markdown/MarkdownEditor.svelte
index 225d25a0eb..6c711c9d28 100644
--- a/packages/bbui/src/Markdown/MarkdownEditor.svelte
+++ b/packages/bbui/src/Markdown/MarkdownEditor.svelte
@@ -45,7 +45,7 @@
   const debouncedUpdate = debounce(update, 250)
 </script>
 
-{#key (height, readonly)}
+{#key height}
   <SpectrumMDE
     bind:mde
     scroll={true}

From 12a7811847fd15cdd8e2e5b3c64e939460f54a56 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Fri, 3 Nov 2023 13:05:23 +0000
Subject: [PATCH 010/155] Revert "Reverting changes to bull parameters"

---
 packages/backend-core/src/index.ts            |  1 +
 .../backend-core/src/queue/inMemoryQueue.ts   |  2 +-
 packages/backend-core/src/queue/queue.ts      | 20 ++++++--
 packages/backend-core/src/utils/Duration.ts   | 49 +++++++++++++++++++
 packages/backend-core/src/utils/index.ts      |  1 +
 .../src/utils/tests/Duration.spec.ts          | 19 +++++++
 6 files changed, 88 insertions(+), 4 deletions(-)
 create mode 100644 packages/backend-core/src/utils/Duration.ts
 create mode 100644 packages/backend-core/src/utils/tests/Duration.spec.ts

diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts
index ffffd8240a..c7cf9f56cc 100644
--- a/packages/backend-core/src/index.ts
+++ b/packages/backend-core/src/index.ts
@@ -30,6 +30,7 @@ export * as timers from "./timers"
 export { default as env } from "./environment"
 export * as blacklist from "./blacklist"
 export * as docUpdates from "./docUpdates"
+export * from "./utils/Duration"
 export { SearchParams } from "./db"
 // Add context to tenancy for backwards compatibility
 // only do this for external usages to prevent internal
diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts
index af2ec6dbaa..a8add7ecb6 100644
--- a/packages/backend-core/src/queue/inMemoryQueue.ts
+++ b/packages/backend-core/src/queue/inMemoryQueue.ts
@@ -36,7 +36,7 @@ class InMemoryQueue {
    * @param opts This is not used by the in memory queue as there is no real use
    * case when in memory, but is the same API as Bull
    */
-  constructor(name: string, opts = null) {
+  constructor(name: string, opts?: any) {
     this._name = name
     this._opts = opts
     this._messages = []
diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts
index 0658147709..c0d1861de3 100644
--- a/packages/backend-core/src/queue/queue.ts
+++ b/packages/backend-core/src/queue/queue.ts
@@ -2,11 +2,18 @@ import env from "../environment"
 import { getRedisOptions } from "../redis/utils"
 import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
-import BullQueue from "bull"
+import BullQueue, { QueueOptions } from "bull"
 import { addListeners, StalledFn } from "./listeners"
+import { Duration } from "../utils"
 import * as timers from "../timers"
+import * as Redis from "ioredis"
 
-const CLEANUP_PERIOD_MS = 60 * 1000
+// the queue lock is held for 5 minutes
+const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs()
+// queue lock is refreshed every 30 seconds
+const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs()
+// cleanup the queue every 60 seconds
+const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs()
 let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
 let cleanupInterval: NodeJS.Timeout
 
@@ -21,7 +28,14 @@ export function createQueue<T>(
   opts: { removeStalledCb?: StalledFn } = {}
 ): BullQueue.Queue<T> {
   const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
-  const queueConfig: any = redisProtocolUrl || { redis: redisOpts }
+  const queueConfig: QueueOptions = {
+    redis: redisProtocolUrl! || (redisOpts as Redis.RedisOptions),
+    settings: {
+      maxStalledCount: 0,
+      lockDuration: QUEUE_LOCK_MS,
+      lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS,
+    },
+  }
   let queue: any
   if (!env.isTest()) {
     queue = new BullQueue(jobQueue, queueConfig)
diff --git a/packages/backend-core/src/utils/Duration.ts b/packages/backend-core/src/utils/Duration.ts
new file mode 100644
index 0000000000..f376c2f7c7
--- /dev/null
+++ b/packages/backend-core/src/utils/Duration.ts
@@ -0,0 +1,49 @@
+export enum DurationType {
+  MILLISECONDS = "milliseconds",
+  SECONDS = "seconds",
+  MINUTES = "minutes",
+  HOURS = "hours",
+  DAYS = "days",
+}
+
+const conversion: Record<DurationType, number> = {
+  milliseconds: 1,
+  seconds: 1000,
+  minutes: 60 * 1000,
+  hours: 60 * 60 * 1000,
+  days: 24 * 60 * 60 * 1000,
+}
+
+export class Duration {
+  static convert(from: DurationType, to: DurationType, duration: number) {
+    const milliseconds = duration * conversion[from]
+    return milliseconds / conversion[to]
+  }
+
+  static from(from: DurationType, duration: number) {
+    return {
+      to: (to: DurationType) => {
+        return Duration.convert(from, to, duration)
+      },
+      toMs: () => {
+        return Duration.convert(from, DurationType.MILLISECONDS, duration)
+      },
+    }
+  }
+
+  static fromSeconds(duration: number) {
+    return Duration.from(DurationType.SECONDS, duration)
+  }
+
+  static fromMinutes(duration: number) {
+    return Duration.from(DurationType.MINUTES, duration)
+  }
+
+  static fromHours(duration: number) {
+    return Duration.from(DurationType.HOURS, duration)
+  }
+
+  static fromDays(duration: number) {
+    return Duration.from(DurationType.DAYS, duration)
+  }
+}
diff --git a/packages/backend-core/src/utils/index.ts b/packages/backend-core/src/utils/index.ts
index 318a7f13ba..ac17227459 100644
--- a/packages/backend-core/src/utils/index.ts
+++ b/packages/backend-core/src/utils/index.ts
@@ -1,3 +1,4 @@
 export * from "./hashing"
 export * from "./utils"
 export * from "./stringUtils"
+export * from "./Duration"
diff --git a/packages/backend-core/src/utils/tests/Duration.spec.ts b/packages/backend-core/src/utils/tests/Duration.spec.ts
new file mode 100644
index 0000000000..46b996f788
--- /dev/null
+++ b/packages/backend-core/src/utils/tests/Duration.spec.ts
@@ -0,0 +1,19 @@
+import { Duration, DurationType } from "../Duration"
+
+describe("duration", () => {
+  it("should convert minutes to milliseconds", () => {
+    expect(Duration.fromMinutes(5).toMs()).toBe(300000)
+  })
+
+  it("should convert seconds to milliseconds", () => {
+    expect(Duration.fromSeconds(30).toMs()).toBe(30000)
+  })
+
+  it("should convert days to milliseconds", () => {
+    expect(Duration.fromDays(1).toMs()).toBe(86400000)
+  })
+
+  it("should convert minutes to days", () => {
+    expect(Duration.fromMinutes(1440).to(DurationType.DAYS)).toBe(1)
+  })
+})

From a03a00c4af0e3deffb55c16c24c0a37748fae594 Mon Sep 17 00:00:00 2001
From: Mel O'Hagan <mel@budibase.com>
Date: Fri, 3 Nov 2023 14:08:46 +0000
Subject: [PATCH 011/155] Toggle preview

---
 packages/bbui/src/Markdown/MarkdownEditor.svelte | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/bbui/src/Markdown/MarkdownEditor.svelte b/packages/bbui/src/Markdown/MarkdownEditor.svelte
index 6c711c9d28..888187c8da 100644
--- a/packages/bbui/src/Markdown/MarkdownEditor.svelte
+++ b/packages/bbui/src/Markdown/MarkdownEditor.svelte
@@ -20,7 +20,9 @@
   // control
   $: checkValue(value)
   $: mde?.codemirror.on("change", debouncedUpdate)
-  $: mde?.codemirror.setOption("readOnly", readonly)
+  $: if (readonly || disabled) {
+    mde?.togglePreview()
+  }
 
   const checkValue = val => {
     if (mde && val !== latestValue) {

From 86f7bd192fb4c6bbe12636e94cca69dc28957d23 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 3 Nov 2023 14:55:56 +0000
Subject: [PATCH 012/155] Moving audit log init to be part of the server
 startup.

---
 packages/worker/src/index.ts | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts
index 3905a21c73..4b1d11ecf7 100644
--- a/packages/worker/src/index.ts
+++ b/packages/worker/src/index.ts
@@ -31,10 +31,6 @@ import destroyable from "server-destroy"
 import { initPro } from "./initPro"
 import { handleScimBody } from "./middleware/handleScimBody"
 
-// configure events to use the pro audit log write
-// can't integrate directly into backend-core due to cyclic issues
-events.processors.init(proSdk.auditLogs.write)
-
 if (coreEnv.ENABLE_SSO_MAINTENANCE_MODE) {
   console.warn(
     "Warning: ENABLE_SSO_MAINTENANCE_MODE is set. It is recommended this flag is disabled if maintenance is not in progress"
@@ -93,6 +89,9 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => {
   console.log(`Worker running on ${JSON.stringify(server.address())}`)
   await initPro()
   await redis.init()
+  // configure events to use the pro audit log write
+  // can't integrate directly into backend-core due to cyclic issues
+  await events.processors.init(proSdk.auditLogs.write)
 })
 
 process.on("uncaughtException", err => {

From 97ca361a6a53ec8109a103ed50cf9cb197473fe6 Mon Sep 17 00:00:00 2001
From: Julio Morimoto <jhem@tjpr.jus.br>
Date: Fri, 3 Nov 2023 14:05:58 -0300
Subject: [PATCH 013/155] Change minio liveness probe to use native httpGet
 instead of curl.

Fixes #12277
---
 charts/budibase/templates/minio-service-deployment.yaml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/charts/budibase/templates/minio-service-deployment.yaml b/charts/budibase/templates/minio-service-deployment.yaml
index 41af2624bf..f98ecc139d 100644
--- a/charts/budibase/templates/minio-service-deployment.yaml
+++ b/charts/budibase/templates/minio-service-deployment.yaml
@@ -46,11 +46,9 @@ spec:
         image: minio/minio
         imagePullPolicy: ""
         livenessProbe:
-          exec:
-            command:
-            - curl
-            - -f
-            - http://localhost:9000/minio/health/live
+          httpGet:
+            path: /minio/health/live
+            port: 9000
           failureThreshold: 3
           periodSeconds: 30
           timeoutSeconds: 20

From f8f1ec4ce930f93e57a74fa57679b9cc582168db Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 3 Nov 2023 17:17:20 +0000
Subject: [PATCH 014/155] Removing old redisProtocol string - it is causing
 confusion and should not be necessary.

---
 packages/backend-core/src/queue/queue.ts |  5 ++--
 packages/backend-core/src/redis/redis.ts |  4 +--
 packages/backend-core/src/redis/utils.ts | 38 +++++++++++++-----------
 3 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts
index c0d1861de3..b460a7312b 100644
--- a/packages/backend-core/src/queue/queue.ts
+++ b/packages/backend-core/src/queue/queue.ts
@@ -6,7 +6,6 @@ import BullQueue, { QueueOptions } from "bull"
 import { addListeners, StalledFn } from "./listeners"
 import { Duration } from "../utils"
 import * as timers from "../timers"
-import * as Redis from "ioredis"
 
 // the queue lock is held for 5 minutes
 const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs()
@@ -27,9 +26,9 @@ export function createQueue<T>(
   jobQueue: JobQueue,
   opts: { removeStalledCb?: StalledFn } = {}
 ): BullQueue.Queue<T> {
-  const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
+  const { opts: redisOpts } = getRedisOptions()
   const queueConfig: QueueOptions = {
-    redis: redisProtocolUrl! || (redisOpts as Redis.RedisOptions),
+    redis: redisOpts,
     settings: {
       maxStalledCount: 0,
       lockDuration: QUEUE_LOCK_MS,
diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts
index d1e2d8989e..1ae2bd7794 100644
--- a/packages/backend-core/src/redis/redis.ts
+++ b/packages/backend-core/src/redis/redis.ts
@@ -91,12 +91,10 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   if (client) {
     client.disconnect()
   }
-  const { redisProtocolUrl, opts, host, port } = getRedisOptions()
+  const { opts, host, port } = getRedisOptions()
 
   if (CLUSTERED) {
     client = new RedisCore.Cluster([{ host, port }], opts)
-  } else if (redisProtocolUrl) {
-    client = new RedisCore(redisProtocolUrl)
   } else {
     client = new RedisCore(opts)
   }
diff --git a/packages/backend-core/src/redis/utils.ts b/packages/backend-core/src/redis/utils.ts
index 34b7275a2b..6cac7b2633 100644
--- a/packages/backend-core/src/redis/utils.ts
+++ b/packages/backend-core/src/redis/utils.ts
@@ -1,4 +1,5 @@
 import env from "../environment"
+import * as Redis from "ioredis"
 
 const SLOT_REFRESH_MS = 2000
 const CONNECT_TIMEOUT_MS = 10000
@@ -74,28 +75,29 @@ export function getRedisOptions() {
   }
   const [host, port] = url.split(":")
 
-  let redisProtocolUrl
-
-  // fully qualified redis URL
-  if (/rediss?:\/\//.test(env.REDIS_URL)) {
-    redisProtocolUrl = env.REDIS_URL
-  }
-
-  const opts: any = {
+  let redisOpts: Redis.RedisOptions = {
     connectTimeout: CONNECT_TIMEOUT_MS,
+    port: parseInt(port),
+    host,
+    password,
   }
+  let opts: Redis.ClusterOptions | Redis.RedisOptions = redisOpts
   if (env.REDIS_CLUSTERED) {
-    opts.redisOptions = {}
-    opts.redisOptions.tls = {}
-    opts.redisOptions.password = password
-    opts.slotsRefreshTimeout = SLOT_REFRESH_MS
-    opts.dnsLookup = (address: string, callback: any) => callback(null, address)
-  } else {
-    opts.host = host
-    opts.port = port
-    opts.password = password
+    opts = {
+      connectTimeout: CONNECT_TIMEOUT_MS,
+      redisOptions: {
+        ...redisOpts,
+        tls: {},
+      },
+      slotsRefreshTimeout: SLOT_REFRESH_MS,
+      dnsLookup: (address: string, callback: any) => callback(null, address),
+    } as Redis.ClusterOptions
+  }
+  return {
+    opts,
+    host,
+    port: parseInt(port),
   }
-  return { opts, host, port: parseInt(port), redisProtocolUrl }
 }
 
 export function addDbPrefix(db: string, key: string) {

From 001cf0130360ea5874fc4979bad65c8c557a1e74 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 3 Nov 2023 18:00:13 +0000
Subject: [PATCH 015/155] Updating public API rate limiting functionality to be
 better typed as well.

---
 packages/backend-core/src/redis/utils.ts      |  3 +-
 .../server/src/api/routes/public/index.ts     | 47 +++++++++----------
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/packages/backend-core/src/redis/utils.ts b/packages/backend-core/src/redis/utils.ts
index 6cac7b2633..e0bdcfcd20 100644
--- a/packages/backend-core/src/redis/utils.ts
+++ b/packages/backend-core/src/redis/utils.ts
@@ -43,7 +43,7 @@ export enum Databases {
 export enum SelectableDatabase {
   DEFAULT = 0,
   SOCKET_IO = 1,
-  UNUSED_1 = 2,
+  RATE_LIMITING = 2,
   UNUSED_2 = 3,
   UNUSED_3 = 4,
   UNUSED_4 = 5,
@@ -96,6 +96,7 @@ export function getRedisOptions() {
   return {
     opts,
     host,
+    password,
     port: parseInt(port),
   }
 }
diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts
index 4cc1eff8a4..ab10b2ed74 100644
--- a/packages/server/src/api/routes/public/index.ts
+++ b/packages/server/src/api/routes/public/index.ts
@@ -15,6 +15,16 @@ import env from "../../../environment"
 const Router = require("@koa/router")
 const { RateLimit, Stores } = require("koa2-ratelimit")
 import { middleware, redis } from "@budibase/backend-core"
+import { SelectableDatabase } from "@budibase/backend-core/src/redis/utils"
+
+interface KoaRateLimitOptions {
+  socket: {
+    host: string
+    port: number
+  }
+  password?: string
+  database?: number
+}
 
 const PREFIX = "/api/public/v1"
 // allow a lot more requests when in test
@@ -29,32 +39,21 @@ function getApiLimitPerSecond(): number {
 
 let rateLimitStore: any = null
 if (!env.isTest()) {
-  const REDIS_OPTS = redis.utils.getRedisOptions()
-  let options
-  if (REDIS_OPTS.redisProtocolUrl) {
-    // fully qualified redis URL
-    options = {
-      url: REDIS_OPTS.redisProtocolUrl,
-    }
-  } else {
-    options = {
-      socket: {
-        host: REDIS_OPTS.host,
-        port: REDIS_OPTS.port,
-      },
-    }
+  const { password, host, port } = redis.utils.getRedisOptions()
+  let options: KoaRateLimitOptions = {
+    socket: {
+      host: host,
+      port: port,
+    },
+  }
 
-    if (REDIS_OPTS.opts?.password || REDIS_OPTS.opts.redisOptions?.password) {
-      // @ts-ignore
-      options.password =
-        REDIS_OPTS.opts.password || REDIS_OPTS.opts.redisOptions.password
-    }
+  if (password) {
+    options.password = password
+  }
 
-    if (!env.REDIS_CLUSTERED) {
-      // @ts-ignore
-      // Can't set direct redis db in clustered env
-      options.database = 1
-    }
+  if (!env.REDIS_CLUSTERED) {
+    // Can't set direct redis db in clustered env
+    options.database = SelectableDatabase.RATE_LIMITING
   }
   rateLimitStore = new Stores.Redis(options)
   RateLimit.defaultOptions({

From 7bf307b0c2e86284b93741e94e0e106261d2e638 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 3 Nov 2023 18:03:11 +0000
Subject: [PATCH 016/155] Further updates to typing.

---
 packages/backend-core/src/redis/utils.ts      | 20 +++++++++++--------
 .../server/src/api/routes/public/index.ts     |  2 +-
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/packages/backend-core/src/redis/utils.ts b/packages/backend-core/src/redis/utils.ts
index e0bdcfcd20..5187fe13f8 100644
--- a/packages/backend-core/src/redis/utils.ts
+++ b/packages/backend-core/src/redis/utils.ts
@@ -59,7 +59,7 @@ export enum SelectableDatabase {
   UNUSED_14 = 15,
 }
 
-export function getRedisOptions() {
+export function getRedisConnectionDetails() {
   let password = env.REDIS_PASSWORD
   let url: string[] | string = env.REDIS_URL.split("//")
   // get rid of the protocol
@@ -75,9 +75,18 @@ export function getRedisOptions() {
   }
   const [host, port] = url.split(":")
 
+  return {
+    host,
+    password,
+    port: parseInt(port),
+  }
+}
+
+export function getRedisOptions() {
+  const { host, password, port } = getRedisConnectionDetails()
   let redisOpts: Redis.RedisOptions = {
     connectTimeout: CONNECT_TIMEOUT_MS,
-    port: parseInt(port),
+    port: port,
     host,
     password,
   }
@@ -93,12 +102,7 @@ export function getRedisOptions() {
       dnsLookup: (address: string, callback: any) => callback(null, address),
     } as Redis.ClusterOptions
   }
-  return {
-    opts,
-    host,
-    password,
-    port: parseInt(port),
-  }
+  return opts
 }
 
 export function addDbPrefix(db: string, key: string) {
diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts
index ab10b2ed74..b37ed931fc 100644
--- a/packages/server/src/api/routes/public/index.ts
+++ b/packages/server/src/api/routes/public/index.ts
@@ -39,7 +39,7 @@ function getApiLimitPerSecond(): number {
 
 let rateLimitStore: any = null
 if (!env.isTest()) {
-  const { password, host, port } = redis.utils.getRedisOptions()
+  const { password, host, port } = redis.utils.getRedisConnectionDetails()
   let options: KoaRateLimitOptions = {
     socket: {
       host: host,

From 08c4ba00975d074c10f01cabc79aa92089f8022b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 3 Nov 2023 18:06:12 +0000
Subject: [PATCH 017/155] Updating redis option functions usage, as it is no
 longer returned as a part of the getRedisOptions response.

---
 packages/backend-core/src/queue/queue.ts | 2 +-
 packages/backend-core/src/redis/redis.ts | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts
index b460a7312b..0657437a3b 100644
--- a/packages/backend-core/src/queue/queue.ts
+++ b/packages/backend-core/src/queue/queue.ts
@@ -26,7 +26,7 @@ export function createQueue<T>(
   jobQueue: JobQueue,
   opts: { removeStalledCb?: StalledFn } = {}
 ): BullQueue.Queue<T> {
-  const { opts: redisOpts } = getRedisOptions()
+  const redisOpts = getRedisOptions()
   const queueConfig: QueueOptions = {
     redis: redisOpts,
     settings: {
diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts
index 1ae2bd7794..6f1b573718 100644
--- a/packages/backend-core/src/redis/redis.ts
+++ b/packages/backend-core/src/redis/redis.ts
@@ -16,6 +16,7 @@ import {
   getRedisOptions,
   SEPARATOR,
   SelectableDatabase,
+  getRedisConnectionDetails,
 } from "./utils"
 import * as timers from "../timers"
 
@@ -91,7 +92,8 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   if (client) {
     client.disconnect()
   }
-  const { opts, host, port } = getRedisOptions()
+  const { host, port } = getRedisConnectionDetails()
+  const opts = getRedisOptions()
 
   if (CLUSTERED) {
     client = new RedisCore.Cluster([{ host, port }], opts)

From b07382fa2e0efb0009bf98f14f3140ebd07da591 Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Sat, 4 Nov 2023 11:04:09 +0100
Subject: [PATCH 018/155] WIP - Migrating worker

---
 packages/types/src/sdk/licensing/plan.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/types/src/sdk/licensing/plan.ts b/packages/types/src/sdk/licensing/plan.ts
index 1604dfb8af..5ac8b1c9f6 100644
--- a/packages/types/src/sdk/licensing/plan.ts
+++ b/packages/types/src/sdk/licensing/plan.ts
@@ -7,7 +7,9 @@ export enum PlanType {
   /** @deprecated */
   PREMIUM = "premium",
   PREMIUM_PLUS = "premium_plus",
+  /** @deprecated */
   BUSINESS = "business",
+  ENTERPRISE_BASIC = "enterprise_basic",
   ENTERPRISE = "enterprise",
 }
 

From 5088caf0f944c82cad3b292df3c06896a0583893 Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Mon, 6 Nov 2023 08:37:53 +0100
Subject: [PATCH 019/155] Update pro submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 3820c0c93a..34cce46d9c 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8
+Subproject commit 34cce46d9c67f03aa66272e079e1d460a731225e

From c43bfda45df9acac45ca136156a41046547283b3 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 6 Nov 2023 12:38:10 +0000
Subject: [PATCH 020/155] Fix for user invitations throwing a 501 error due to
 the way the search was being performed.

---
 .../app/[application]/_components/BuilderSidePanel.svelte     | 4 ++--
 packages/frontend-core/src/fetch/UserFetch.js                 | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
index f9a40b09a6..a67c2d3c61 100644
--- a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
@@ -112,9 +112,9 @@
     }
     await usersFetch.update({
       query: {
-        appId: query || !filterByAppAccess ? null : prodAppId,
-        email: query,
+        string: { email: query },
       },
+      appId: query || !filterByAppAccess ? null : prodAppId,
       limit: 50,
       paginate: query || !filterByAppAccess ? null : false,
     })
diff --git a/packages/frontend-core/src/fetch/UserFetch.js b/packages/frontend-core/src/fetch/UserFetch.js
index b1478c3a6d..65bfe36058 100644
--- a/packages/frontend-core/src/fetch/UserFetch.js
+++ b/packages/frontend-core/src/fetch/UserFetch.js
@@ -33,7 +33,7 @@ export default class UserFetch extends DataFetch {
     let finalQuery
     // convert old format to new one - we now allow use of the lucene format
     const { appId, paginated, ...rest } = query
-    if (!LuceneUtils.hasFilters(query) && rest.email) {
+    if (!LuceneUtils.hasFilters(query) && rest.email != null) {
       finalQuery = { string: { email: rest.email } }
     } else {
       finalQuery = rest

From 561778cf6a59f542571b248779e582e90fea5368 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Mon, 6 Nov 2023 12:58:47 +0000
Subject: [PATCH 021/155] Fixes for exported data UI. Filtered out unnecessary
 onEmptyFilter filter.

---
 .../DataTable/modals/ExportModal.svelte       | 28 +++++++++++--------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte
index 09f76d3522..61c81c86eb 100644
--- a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte
@@ -17,6 +17,8 @@
   export let selectedRows = []
   export let formats
 
+  $: appliedFilters = filters?.filter(filter => !filter.onEmptyFilter)
+
   const FORMATS = [
     {
       name: "CSV",
@@ -46,8 +48,12 @@
     exportFormat = Array.isArray(options) ? options[0]?.key : []
   }
 
-  $: luceneFilter = LuceneUtils.buildLuceneQuery(filters)
-  $: exportOpDisplay = buildExportOpDisplay(sorting, filterDisplay, filters)
+  $: luceneFilter = LuceneUtils.buildLuceneQuery(appliedFilters)
+  $: exportOpDisplay = buildExportOpDisplay(
+    sorting,
+    filterDisplay,
+    appliedFilters
+  )
 
   const buildFilterLookup = () => {
     return Object.keys(Constants.OperatorOptions).reduce((acc, key) => {
@@ -59,10 +65,10 @@
   filterLookup = buildFilterLookup()
 
   const filterDisplay = () => {
-    if (!filters) {
+    if (!appliedFilters) {
       return []
     }
-    return filters.map(filter => {
+    return appliedFilters.map(filter => {
       let newFieldName = filter.field + ""
       const parts = newFieldName.split(":")
       parts.shift()
@@ -77,7 +83,7 @@
 
   const buildExportOpDisplay = (sorting, filterDisplay) => {
     let filterDisplayConfig = filterDisplay()
-    if (sorting) {
+    if (sorting?.sortColumn) {
       filterDisplayConfig = [
         ...filterDisplayConfig,
         {
@@ -132,7 +138,7 @@
         format: exportFormat,
       })
       downloadWithBlob(data, `export.${exportFormat}`)
-    } else if (filters || sorting) {
+    } else if (appliedFilters || sorting) {
       let response
       try {
         response = await API.exportRows({
@@ -163,16 +169,16 @@
   title="Export Data"
   confirmText="Export"
   onConfirm={exportRows}
-  size={filters?.length || sorting ? "M" : "S"}
+  size={appliedFilters?.length || sorting ? "M" : "S"}
 >
   {#if selectedRows?.length}
     <Body size="S">
       <strong>{selectedRows?.length}</strong>
       {`row${selectedRows?.length > 1 ? "s" : ""} will be exported`}
     </Body>
-  {:else if filters || (sorting?.sortOrder && sorting?.sortColumn)}
+  {:else if appliedFilters?.length || (sorting?.sortOrder && sorting?.sortColumn)}
     <Body size="S">
-      {#if !filters}
+      {#if !appliedFilters}
         Exporting <strong>all</strong> rows
       {:else}
         Filters applied
@@ -183,9 +189,9 @@
       <Table
         schema={displaySchema}
         data={exportOpDisplay}
-        {filters}
+        {appliedFilters}
         loading={false}
-        rowCount={filters?.length + 1}
+        rowCount={appliedFilters?.length + 1}
         disableSorting={true}
         allowSelectRows={false}
         allowEditRows={false}

From 9bb9fb55496b05ac81f379a106c1de9dc2b48f4d Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Mon, 6 Nov 2023 17:00:47 +0000
Subject: [PATCH 022/155] Reintroduce filtering and sorting for exported row
 data

---
 packages/server/src/api/controllers/row/index.ts    |  4 +++-
 packages/server/src/sdk/app/rows/search.ts          | 10 +++++++++-
 packages/server/src/sdk/app/rows/search/external.ts | 10 ++++++----
 packages/server/src/sdk/app/rows/search/internal.ts |  9 +++++++--
 packages/types/src/api/web/app/rows.ts              |  3 +++
 5 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts
index 1a6747a085..018283c8c5 100644
--- a/packages/server/src/api/controllers/row/index.ts
+++ b/packages/server/src/api/controllers/row/index.ts
@@ -254,7 +254,7 @@ export const exportRows = async (
 
   const format = ctx.query.format
 
-  const { rows, columns, query } = ctx.request.body
+  const { rows, columns, query, sort, sortOrder } = ctx.request.body
   if (typeof format !== "string" || !exporters.isFormat(format)) {
     ctx.throw(
       400,
@@ -272,6 +272,8 @@ export const exportRows = async (
         rowIds: rows,
         columns,
         query,
+        sort,
+        sortOrder,
       })
       ctx.attachment(fileName)
       return apiFileReturn(content)
diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index 31f7c74601..36ef538d8b 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -1,4 +1,10 @@
-import { Row, SearchFilters, SearchParams } from "@budibase/types"
+import {
+  Row,
+  SearchFilters,
+  SearchParams,
+  SortOrder,
+  SortType,
+} from "@budibase/types"
 import { isExternalTableID } from "../../../integrations/utils"
 import * as internal from "./search/internal"
 import * as external from "./search/external"
@@ -32,6 +38,8 @@ export interface ExportRowsParams {
   rowIds?: string[]
   columns?: string[]
   query?: SearchFilters
+  sort?: string
+  sortOrder?: SortOrder
 }
 
 export interface ExportRowsResult {
diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts
index 981ae1bf8d..974900ba6d 100644
--- a/packages/server/src/sdk/app/rows/search/external.ts
+++ b/packages/server/src/sdk/app/rows/search/external.ts
@@ -98,12 +98,12 @@ export async function search(options: SearchParams) {
 export async function exportRows(
   options: ExportRowsParams
 ): Promise<ExportRowsResult> {
-  const { tableId, format, columns, rowIds } = options
+  const { tableId, format, columns, rowIds, query, sort, sortOrder } = options
   const { datasourceId, tableName } = breakExternalTableId(tableId)
 
-  let query: SearchFilters = {}
+  let requestQuery: SearchFilters = {}
   if (rowIds?.length) {
-    query = {
+    requestQuery = {
       oneOf: {
         _id: rowIds.map((row: string) => {
           const ids = JSON.parse(
@@ -119,6 +119,8 @@ export async function exportRows(
         }),
       },
     }
+  } else {
+    requestQuery = query || {}
   }
 
   const datasource = await sdk.datasources.get(datasourceId!)
@@ -126,7 +128,7 @@ export async function exportRows(
     throw new HTTPError("Datasource has not been configured for plus API.", 400)
   }
 
-  let result = await search({ tableId, query })
+  let result = await search({ tableId, query: requestQuery, sort, sortOrder })
   let rows: Row[] = []
 
   // Filter data to only specified columns if required
diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts
index 1aec8a321e..e31bda1a15 100644
--- a/packages/server/src/sdk/app/rows/search/internal.ts
+++ b/packages/server/src/sdk/app/rows/search/internal.ts
@@ -84,7 +84,7 @@ export async function search(options: SearchParams) {
 export async function exportRows(
   options: ExportRowsParams
 ): Promise<ExportRowsResult> {
-  const { tableId, format, rowIds, columns, query } = options
+  const { tableId, format, rowIds, columns, query, sort, sortOrder } = options
   const db = context.getAppDB()
   const table = await sdk.tables.getTable(tableId)
 
@@ -99,7 +99,12 @@ export async function exportRows(
 
     result = await outputProcessing(table, response)
   } else if (query) {
-    let searchResponse = await search({ tableId, query })
+    let searchResponse = await search({
+      tableId,
+      query,
+      sort,
+      sortOrder,
+    })
     result = searchResponse.rows
   }
 
diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts
index 62ea90a6a4..dad3286754 100644
--- a/packages/types/src/api/web/app/rows.ts
+++ b/packages/types/src/api/web/app/rows.ts
@@ -1,5 +1,6 @@
 import { SearchFilters, SearchParams } from "../../../sdk"
 import { Row } from "../../../documents"
+import { SortOrder } from "../../../api"
 import { ReadStream } from "fs"
 
 export interface SaveRowRequest extends Row {}
@@ -34,6 +35,8 @@ export interface ExportRowsRequest {
   rows: string[]
   columns?: string[]
   query?: SearchFilters
+  sort?: string
+  sortOrder?: SortOrder
 }
 
 export type ExportRowsResponse = ReadStream

From a5246bc08aa8a335d1d06efa7bfbd395cdd39d49 Mon Sep 17 00:00:00 2001
From: Greg <129239095+gcglinton@users.noreply.github.com>
Date: Mon, 6 Nov 2023 13:53:22 -0500
Subject: [PATCH 023/155] Update nginx-ssl.conf

---
 hosting/letsencrypt/nginx-ssl.conf | 61 +++++++++++++++++++++++++++---
 1 file changed, 56 insertions(+), 5 deletions(-)

diff --git a/hosting/letsencrypt/nginx-ssl.conf b/hosting/letsencrypt/nginx-ssl.conf
index 50c5e0198a..b3f51e5cc5 100644
--- a/hosting/letsencrypt/nginx-ssl.conf
+++ b/hosting/letsencrypt/nginx-ssl.conf
@@ -2,16 +2,18 @@ server {
     listen       443 ssl default_server;
     listen  [::]:443 ssl default_server;
     server_name  _;
-    ssl_certificate /etc/letsencrypt/live/CUSTOM_DOMAIN/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/CUSTOM_DOMAIN/privkey.pem;
-    include /etc/letsencrypt/options-ssl-nginx.conf;
-    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
-
+    error_log            /dev/stderr warn;
+    access_log           /dev/stdout main;
     client_max_body_size 1000m;
     ignore_invalid_headers off;
     proxy_buffering off;
     # port_in_redirect off;
 
+    ssl_certificate /etc/letsencrypt/live/CUSTOM_DOMAIN/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/CUSTOM_DOMAIN/privkey.pem;
+    include /etc/letsencrypt/options-ssl-nginx.conf;
+    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
+
     location ^~ /.well-known/acme-challenge/ {
         default_type "text/plain";
         root         /var/www/html;
@@ -47,6 +49,24 @@ server {
         rewrite ^/worker/(.*)$ /$1 break;
     }
 
+    location /api/backups/ {
+        # calls to export apps are limited
+        limit_req zone=ratelimit burst=20 nodelay;
+
+        # 1800s timeout for app export requests
+        proxy_read_timeout 1800s;
+        proxy_connect_timeout 1800s;
+        proxy_send_timeout 1800s;
+
+        proxy_http_version 1.1;
+        proxy_set_header    Connection          $connection_upgrade;
+        proxy_set_header    Upgrade             $http_upgrade;
+        proxy_set_header    X-Real-IP           $remote_addr;
+        proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+
+        proxy_pass      http://127.0.0.1:4001;
+    }
+
     location /api/ {
         # calls to the API are rate limited with bursting
         limit_req zone=ratelimit burst=20 nodelay;
@@ -70,18 +90,49 @@ server {
         rewrite ^/db/(.*)$ /$1 break;
     }
 
+    location /socket/ {
+        proxy_http_version  1.1;
+        proxy_set_header    Upgrade     $http_upgrade;
+        proxy_set_header    Connection  'upgrade';
+        proxy_set_header    Host        $host;
+        proxy_cache_bypass  $http_upgrade;
+        proxy_pass          http://127.0.0.1:4001;
+    }
+
     location / {
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Host $http_host;
 
         proxy_connect_timeout 300;
         proxy_http_version 1.1;
         proxy_set_header Connection "";
         chunked_transfer_encoding off;
+        
         proxy_pass      http://127.0.0.1:9000;
     }
 
+    location /files/signed/ {
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+
+        # IMPORTANT: Signed urls will inspect the host header of the request.
+        # Normally a signed url will need to be generated with a specified client host in mind.
+        # To support dynamic hosts, e.g. some unknown self-hosted installation url,
+        # use a predefined host header. The host 'minio-service' is also used at the time of url signing.
+        proxy_set_header Host minio-service;
+
+        proxy_connect_timeout 300;
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
+        chunked_transfer_encoding off;
+
+        proxy_pass      http://127.0.0.1:9000;
+        rewrite ^/files/signed/(.*)$ /$1 break;
+    }
+
     client_header_timeout 60;
     client_body_timeout   60;
     keepalive_timeout     60;

From 635af0f76aec1c8da2139701ecd3ab99f606a38e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 6 Nov 2023 19:03:37 +0000
Subject: [PATCH 024/155] Getting external rows to be more consistent with the
 internal API - the formulas should be processed using the outputProcessing as
 they were being processed, but was causing an overwrite. The problem was that
 the external system internally processed the formulas/relationships, then
 squashed the relationships. Once it got to the external API, it would go
 through normal output processing, which would run over the squashed rows,
 causing an inconsistent behaviour.

---
 .../src/api/controllers/deploy/index.ts       |  1 -
 .../api/controllers/row/ExternalRequest.ts    | 33 ++++----
 .../src/api/controllers/row/external.ts       | 14 +++-
 .../src/api/controllers/row/staticFormula.ts  | 14 ++--
 packages/server/src/db/linkedRows/index.ts    |  1 -
 .../server/src/db/linkedRows/linkUtils.ts     |  4 +-
 .../src/sdk/app/rows/search/external.ts       |  5 +-
 .../src/utilities/rowProcessor/index.ts       |  2 +-
 .../src/utilities/rowProcessor/utils.ts       | 76 ++++++++++---------
 9 files changed, 79 insertions(+), 71 deletions(-)

diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts
index 66439d3411..2cf3da3dda 100644
--- a/packages/server/src/api/controllers/deploy/index.ts
+++ b/packages/server/src/api/controllers/deploy/index.ts
@@ -106,7 +106,6 @@ export async function fetchDeployments(ctx: any) {
     }
     ctx.body = Object.values(deployments.history).reverse()
   } catch (err) {
-    console.error(err)
     ctx.body = []
   }
 }
diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index c3c5468840..7c98fecb9b 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -23,7 +23,6 @@ import {
   breakRowIdField,
   convertRowId,
   generateRowIdField,
-  getPrimaryDisplay,
   isRowId,
   isSQL,
 } from "../../../integrations/utils"
@@ -237,7 +236,7 @@ function basicProcessing({
   thisRow._id = generateIdForRow(row, table, isLinked)
   thisRow.tableId = table._id
   thisRow._rev = "rev"
-  return processFormulas(table, thisRow)
+  return thisRow
 }
 
 function fixArrayTypes(row: Row, table: Table) {
@@ -392,7 +391,7 @@ export class ExternalRequest<T extends Operation> {
     return { row: newRow, manyRelationships }
   }
 
-  squashRelationshipColumns(
+  processRelationshipFields(
     table: Table,
     row: Row,
     relationships: RelationshipsJson[]
@@ -402,7 +401,6 @@ export class ExternalRequest<T extends Operation> {
       if (!linkedTable || !row[relationship.column]) {
         continue
       }
-      const display = linkedTable.primaryDisplay
       for (let key of Object.keys(row[relationship.column])) {
         let relatedRow: Row = row[relationship.column][key]
         // add this row as context for the relationship
@@ -411,15 +409,10 @@ export class ExternalRequest<T extends Operation> {
             relatedRow[col.name] = [row]
           }
         }
+        // process additional types
+        relatedRow = processDates(table, relatedRow)
         relatedRow = processFormulas(linkedTable, relatedRow)
-        let relatedDisplay
-        if (display) {
-          relatedDisplay = getPrimaryDisplay(relatedRow[display])
-        }
-        row[relationship.column][key] = {
-          primaryDisplay: relatedDisplay || "Invalid display column",
-          _id: relatedRow._id,
-        }
+        row[relationship.column][key] = relatedRow
       }
     }
     return row
@@ -521,14 +514,14 @@ export class ExternalRequest<T extends Operation> {
       )
     }
 
-    // Process some additional data types
-    let finalRowArray = Object.values(finalRows)
-    finalRowArray = processDates(table, finalRowArray)
-    finalRowArray = processFormulas(table, finalRowArray) as Row[]
-
-    return finalRowArray.map((row: Row) =>
-      this.squashRelationshipColumns(table, row, relationships)
+    // make sure all related rows are correct
+    let finalRowArray = Object.values(finalRows).map(row =>
+      this.processRelationshipFields(table, row, relationships)
     )
+
+    // process some additional types
+    finalRowArray = processDates(table, finalRowArray)
+    return finalRowArray
   }
 
   /**
@@ -663,7 +656,7 @@ export class ExternalRequest<T extends Operation> {
         linkPrimary,
         linkSecondary,
       }: {
-        row: { [key: string]: any }
+        row: Row
         linkPrimary: string
         linkSecondary?: string
       }) {
diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 0515b6b97e..287b2ae6aa 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -76,6 +76,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
     relationships: true,
   })
   const enrichedRow = await outputProcessing(table, row, {
+    squash: true,
     preserveLinks: true,
   })
   return {
@@ -119,7 +120,10 @@ export async function save(ctx: UserCtx) {
     })
     return {
       ...response,
-      row: await outputProcessing(table, row, { preserveLinks: true }),
+      row: await outputProcessing(table, row, {
+        preserveLinks: true,
+        squash: true,
+      }),
     }
   } else {
     return response
@@ -140,7 +144,7 @@ export async function find(ctx: UserCtx): Promise<Row> {
   const table = await sdk.tables.getTable(tableId)
   // Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case
   return await outputProcessing(table, row, {
-    squash: false,
+    squash: true,
     preserveLinks: true,
   })
 }
@@ -207,7 +211,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
     // don't support composite keys right now
     const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0])
     const primaryLink = linkedTable.primary?.[0] as string
-    row[fieldName] = await handleRequest(Operation.READ, linkedTableId!, {
+    const relatedRows = await handleRequest(Operation.READ, linkedTableId!, {
       tables,
       filters: {
         oneOf: {
@@ -216,6 +220,10 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
       },
       includeSqlRelationships: IncludeRelationship.INCLUDE,
     })
+    row[fieldName] = await outputProcessing(linkedTable, relatedRows, {
+      squash: true,
+      preserveLinks: true,
+    })
   }
   return row
 }
diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts
index 6f426c6fa0..87cbdad23f 100644
--- a/packages/server/src/api/controllers/row/staticFormula.ts
+++ b/packages/server/src/api/controllers/row/staticFormula.ts
@@ -86,12 +86,12 @@ export async function updateAllFormulasInTable(table: Table) {
   const db = context.getAppDB()
   // start by getting the raw rows (which will be written back to DB after update)
   let rows = (
-    await db.allDocs(
+    await db.allDocs<Row>(
       getRowParams(table._id, null, {
         include_docs: true,
       })
     )
-  ).rows.map(row => row.doc)
+  ).rows.map(row => row.doc as Row)
   // now enrich the rows, note the clone so that we have the base state of the
   // rows so that we don't write any of the enriched information back
   let enrichedRows = await outputProcessing(table, cloneDeep(rows), {
@@ -101,12 +101,12 @@ export async function updateAllFormulasInTable(table: Table) {
   for (let row of rows) {
     // find the enriched row, if found process the formulas
     const enrichedRow = enrichedRows.find(
-      (enriched: any) => enriched._id === row._id
+      (enriched: Row) => enriched._id === row._id
     )
     if (enrichedRow) {
       const processed = processFormulas(table, cloneDeep(row), {
         dynamic: false,
-        contextRows: enrichedRow,
+        contextRows: [enrichedRow],
       })
       // values have changed, need to add to bulk docs to update
       if (!isEqual(processed, row)) {
@@ -139,7 +139,7 @@ export async function finaliseRow(
   // use enriched row to generate formulas for saving, specifically only use as context
   row = processFormulas(table, row, {
     dynamic: false,
-    contextRows: enrichedRow,
+    contextRows: [enrichedRow],
   })
   // don't worry about rev, tables handle rev/lastID updates
   // if another row has been written since processing this will
@@ -163,7 +163,9 @@ export async function finaliseRow(
   const response = await db.put(row)
   // for response, calculate the formulas for the enriched row
   enrichedRow._rev = response.rev
-  enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
+  enrichedRow = processFormulas(table, enrichedRow, {
+    dynamic: false,
+  })
   // this updates the related formulas in other rows based on the relations to this row
   if (updateFormula) {
     await updateRelatedFormula(table, enrichedRow)
diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts
index 7a7a06551e..0dba773ad7 100644
--- a/packages/server/src/db/linkedRows/index.ts
+++ b/packages/server/src/db/linkedRows/index.ts
@@ -2,7 +2,6 @@ import LinkController from "./LinkController"
 import {
   IncludeDocs,
   getLinkDocuments,
-  createLinkView,
   getUniqueByProp,
   getRelatedTableForField,
   getLinkedTableIDs,
diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts
index db9a0dc7d5..5942e7e5a1 100644
--- a/packages/server/src/db/linkedRows/linkUtils.ts
+++ b/packages/server/src/db/linkedRows/linkUtils.ts
@@ -8,6 +8,7 @@ import {
   LinkDocumentValue,
   Table,
 } from "@budibase/types"
+import sdk from "../../sdk"
 
 export { createLinkView } from "../views/staticViews"
 
@@ -110,12 +111,11 @@ export function getLinkedTableIDs(table: Table): string[] {
 }
 
 export async function getLinkedTable(id: string, tables: Table[]) {
-  const db = context.getAppDB()
   let linkedTable = tables.find(table => table._id === id)
   if (linkedTable) {
     return linkedTable
   }
-  linkedTable = await db.get(id)
+  linkedTable = await sdk.tables.getTable(id)
   if (linkedTable) {
     tables.push(linkedTable)
   }
diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts
index 981ae1bf8d..71c12f0a79 100644
--- a/packages/server/src/sdk/app/rows/search/external.ts
+++ b/packages/server/src/sdk/app/rows/search/external.ts
@@ -80,7 +80,10 @@ export async function search(options: SearchParams) {
       rows = rows.map((r: any) => pick(r, fields))
     }
 
-    rows = await outputProcessing(table, rows, { preserveLinks: true })
+    rows = await outputProcessing(table, rows, {
+      preserveLinks: true,
+      squash: true,
+    })
 
     // need wrapper object for bookmarks etc when paginating
     return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 }
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 098962c646..6070d40f7d 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -258,7 +258,7 @@ export async function outputProcessing<T extends Row[] | Row>(
   }
 
   // process formulas after the complex types had been processed
-  enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]
+  enriched = processFormulas(table, enriched, { dynamic: true })
 
   if (opts.squash) {
     enriched = (await linkRows.squashLinksToPrimaryDisplay(
diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts
index 48697af6a9..8d4af29a6b 100644
--- a/packages/server/src/utilities/rowProcessor/utils.ts
+++ b/packages/server/src/utilities/rowProcessor/utils.ts
@@ -12,6 +12,11 @@ import {
   Table,
 } from "@budibase/types"
 
+interface FormulaOpts {
+  dynamic?: boolean
+  contextRows?: Row[]
+}
+
 /**
  * If the subtype has been lost for any reason this works out what
  * subtype the auto column should be.
@@ -40,52 +45,50 @@ export function fixAutoColumnSubType(
 /**
  * Looks through the rows provided and finds formulas - which it then processes.
  */
-export function processFormulas(
+export function processFormulas<T extends Row | Row[]>(
   table: Table,
-  rows: Row[] | Row,
-  { dynamic, contextRows }: any = { dynamic: true }
-) {
-  const single = !Array.isArray(rows)
-  let rowArray: Row[]
-  if (single) {
-    rowArray = [rows]
-    contextRows = contextRows ? [contextRows] : contextRows
-  } else {
-    rowArray = rows
-  }
-  for (let [column, schema] of Object.entries(table.schema)) {
-    if (schema.type !== FieldTypes.FORMULA) {
-      continue
-    }
+  inputRows: T,
+  { dynamic, contextRows }: FormulaOpts = { dynamic: true }
+): Promise<T> {
+  const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
+  if (rows)
+    for (let [column, schema] of Object.entries(table.schema)) {
+      if (schema.type !== FieldTypes.FORMULA) {
+        continue
+      }
 
-    const isStatic = schema.formulaType === FormulaTypes.STATIC
+      const isStatic = schema.formulaType === FormulaTypes.STATIC
 
-    if (
-      schema.formula == null ||
-      (dynamic && isStatic) ||
-      (!dynamic && !isStatic)
-    ) {
-      continue
-    }
-    // iterate through rows and process formula
-    for (let i = 0; i < rowArray.length; i++) {
-      let row = rowArray[i]
-      let context = contextRows ? contextRows[i] : row
-      rowArray[i] = {
-        ...row,
-        [column]: processStringSync(schema.formula, context),
+      if (
+        schema.formula == null ||
+        (dynamic && isStatic) ||
+        (!dynamic && !isStatic)
+      ) {
+        continue
+      }
+      // iterate through rows and process formula
+      for (let i = 0; i < rows.length; i++) {
+        let row = rows[i]
+        let context = contextRows ? contextRows[i] : row
+        rows[i] = {
+          ...row,
+          [column]: processStringSync(schema.formula, context),
+        }
       }
     }
-  }
-  return single ? rowArray[0] : rowArray
+  return Array.isArray(inputRows) ? rows : rows[0]
 }
 
 /**
  * Processes any date columns and ensures that those without the ignoreTimezones
  * flag set are parsed as UTC rather than local time.
  */
-export function processDates(table: Table, rows: Row[]) {
-  let datesWithTZ = []
+export function processDates<T extends Row | Row[]>(
+  table: Table,
+  inputRows: T
+): T {
+  let rows = Array.isArray(inputRows) ? inputRows : [inputRows]
+  let datesWithTZ: string[] = []
   for (let [column, schema] of Object.entries(table.schema)) {
     if (schema.type !== FieldTypes.DATETIME) {
       continue
@@ -102,5 +105,6 @@ export function processDates(table: Table, rows: Row[]) {
       }
     }
   }
-  return rows
+
+  return Array.isArray(inputRows) ? rows : rows[0]
 }

From 466d5350e23ea8cd5ac67eaa35e2bfb6374c691d Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Mon, 6 Nov 2023 19:50:27 +0000
Subject: [PATCH 025/155] Bump version to 2.13.1

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 611cf7d32b..2126adac29 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.13.0",
+  "version": "2.13.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From c169a7376499747dace7471320737cb22c121319 Mon Sep 17 00:00:00 2001
From: Mel O'Hagan <mel@budibase.com>
Date: Tue, 7 Nov 2023 09:19:29 +0000
Subject: [PATCH 026/155] readonly css

---
 packages/bbui/src/Form/Core/Checkbox.svelte           | 9 ++++-----
 packages/bbui/src/Form/Core/CheckboxGroup.svelte      | 9 ++++-----
 packages/bbui/src/Form/Core/RadioGroup.svelte         | 9 ++++-----
 packages/client/src/components/app/forms/Field.svelte | 4 ++++
 4 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/packages/bbui/src/Form/Core/Checkbox.svelte b/packages/bbui/src/Form/Core/Checkbox.svelte
index 3eaaf4dede..e24f5669eb 100644
--- a/packages/bbui/src/Form/Core/Checkbox.svelte
+++ b/packages/bbui/src/Form/Core/Checkbox.svelte
@@ -25,16 +25,12 @@
   class:is-invalid={!!error}
   class:checked={value}
   class:is-indeterminate={indeterminate}
+  class:readonly
 >
   <input
     checked={value}
     {disabled}
     on:change={onChange}
-    on:click={e => {
-      if (readonly) {
-        e.preventDefault()
-      }
-    }}
     type="checkbox"
     class="spectrum-Checkbox-input"
     {id}
@@ -74,4 +70,7 @@
   .spectrum-Checkbox-input {
     opacity: 0;
   }
+  .readonly {
+    pointer-events: none;
+  }
 </style>
diff --git a/packages/bbui/src/Form/Core/CheckboxGroup.svelte b/packages/bbui/src/Form/Core/CheckboxGroup.svelte
index faf37f3ad8..66ac55561b 100644
--- a/packages/bbui/src/Form/Core/CheckboxGroup.svelte
+++ b/packages/bbui/src/Form/Core/CheckboxGroup.svelte
@@ -35,17 +35,13 @@
         title={getOptionLabel(option)}
         class="spectrum-Checkbox spectrum-FieldGroup-item"
         class:is-invalid={!!error}
+        class:readonly
       >
         <label
           class="spectrum-Checkbox spectrum-Checkbox--sizeM spectrum-FieldGroup-item"
         >
           <input
             on:change={onChange}
-            on:click={e => {
-              if (readonly) {
-                e.preventDefault()
-              }
-            }}
             type="checkbox"
             class="spectrum-Checkbox-input"
             value={optionValue}
@@ -72,4 +68,7 @@
   .spectrum-Checkbox-input {
     opacity: 0;
   }
+  .readonly {
+    pointer-events: none;
+  }
 </style>
diff --git a/packages/bbui/src/Form/Core/RadioGroup.svelte b/packages/bbui/src/Form/Core/RadioGroup.svelte
index fc99fafd40..648c46b5fc 100644
--- a/packages/bbui/src/Form/Core/RadioGroup.svelte
+++ b/packages/bbui/src/Form/Core/RadioGroup.svelte
@@ -41,14 +41,10 @@
         title={getOptionTitle(option)}
         class="spectrum-Radio spectrum-FieldGroup-item spectrum-Radio--emphasized"
         class:is-invalid={!!error}
+        class:readonly
       >
         <input
           on:change={onChange}
-          on:click={e => {
-            if (readonly) {
-              e.preventDefault()
-            }
-          }}
           bind:group={value}
           value={getOptionValue(option)}
           type="radio"
@@ -68,4 +64,7 @@
   .spectrum-Radio-input {
     opacity: 0;
   }
+  .readonly {
+    pointer-events: none;
+  }
 </style>
diff --git a/packages/client/src/components/app/forms/Field.svelte b/packages/client/src/components/app/forms/Field.svelte
index 1fa1aeda5a..8878b25989 100644
--- a/packages/client/src/components/app/forms/Field.svelte
+++ b/packages/client/src/components/app/forms/Field.svelte
@@ -78,6 +78,7 @@
       contenteditable={$component.editing}
       on:blur={$component.editing ? updateLabel : null}
       class:hidden={!label}
+      class:readonly
       for={fieldState?.fieldId}
       class={`spectrum-FieldLabel spectrum-FieldLabel--sizeM spectrum-Form-itemLabel ${labelClass}`}
     >
@@ -138,4 +139,7 @@
   .spectrum-FieldLabel--left {
     padding-right: var(--spectrum-global-dimension-size-200);
   }
+  .readonly {
+    pointer-events: none;
+  }
 </style>

From 49b00ee7eeb23717b5f075064b021e0aa74e6429 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 7 Nov 2023 11:07:25 +0000
Subject: [PATCH 027/155] Fixing build issue.

---
 packages/server/src/utilities/rowProcessor/utils.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts
index 8d4af29a6b..9eb725dd7c 100644
--- a/packages/server/src/utilities/rowProcessor/utils.ts
+++ b/packages/server/src/utilities/rowProcessor/utils.ts
@@ -49,7 +49,7 @@ export function processFormulas<T extends Row | Row[]>(
   table: Table,
   inputRows: T,
   { dynamic, contextRows }: FormulaOpts = { dynamic: true }
-): Promise<T> {
+): T {
   const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
   if (rows)
     for (let [column, schema] of Object.entries(table.schema)) {

From f28eb054f38b66c40b33d9b9f946481c15b3556f Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Tue, 7 Nov 2023 11:59:51 +0000
Subject: [PATCH 028/155] Set the relationship type (#12275)

* Set the relationship type

* Add users type

* Set user relationship type saveColumn

* Set relationshipType for users type on save

* Add relatioshipType to schema

* Refactor

* Check isUsersColumn

* Make relationshipType optional

---------

Co-authored-by: Michael Drury <me@michaeldrury.co.uk>
---
 .../backend/DataTable/modals/CreateEditColumn.svelte   | 10 ++++++++++
 packages/types/src/documents/app/table/schema.ts       |  1 +
 2 files changed, 11 insertions(+)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index d5a9aba488..4eb1f962f0 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -44,6 +44,8 @@
   const NUMBER_TYPE = FIELDS.NUMBER.type
   const JSON_TYPE = FIELDS.JSON.type
   const DATE_TYPE = FIELDS.DATETIME.type
+  const USER_TYPE = FIELDS.USER.subtype
+  const USERS_TYPE = FIELDS.USERS.subtype
 
   const dispatch = createEventDispatcher()
   const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
@@ -287,6 +289,14 @@
     if (saveColumn.type !== LINK_TYPE) {
       delete saveColumn.fieldName
     }
+    if (isUsersColumn(saveColumn)) {
+      if (saveColumn.subtype === USER_TYPE) {
+        saveColumn.relationshipType = RelationshipType.ONE_TO_MANY
+      } else if (saveColumn.subtype === USERS_TYPE) {
+        saveColumn.relationshipType = RelationshipType.MANY_TO_MANY
+      }
+    }
+
     try {
       await tables.saveField({
         originalName,
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 755ccf61e7..19a7303072 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -102,6 +102,7 @@ export interface BBReferenceFieldMetadata
   extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.BB_REFERENCE
   subtype: FieldSubtype.USER | FieldSubtype.USERS
+  relationshipType?: RelationshipType
 }
 
 export interface FieldConstraints {

From dc59245b39da5c4b8ed95cebba5931adf8028d4c Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 7 Nov 2023 12:20:21 +0000
Subject: [PATCH 029/155] Bump version to 2.13.2

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 2126adac29..bfcac5633c 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.13.1",
+  "version": "2.13.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 7dd4521ea83a3089902c66c8c9968668b7cbcf7a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 7 Nov 2023 13:55:34 +0100
Subject: [PATCH 030/155] Fix fetching tag

---
 .github/workflows/tag-release.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tag-release.yml b/.github/workflows/tag-release.yml
index 78c07a037c..13d59d1019 100644
--- a/.github/workflows/tag-release.yml
+++ b/.github/workflows/tag-release.yml
@@ -45,8 +45,8 @@ jobs:
           BUMP_TYPE=${BUMP_TYPE_INPUT:-"patch"}
           ./versionCommit.sh $BUMP_TYPE
 
-
-          new_version=$(./getCurrentVersion.sh)
+          cd ..
+          new_version=$(./scripts/getCurrentVersion.sh)
           echo "version=$new_version" >> $GITHUB_OUTPUT
 
   trigger-release:

From 57e3b1ad083cdc5fe8d25464c79619c72da651d1 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 7 Nov 2023 13:01:50 +0000
Subject: [PATCH 031/155] Fix issue with buttons inside grids when no actions
 are defined

---
 packages/client/src/components/app/GridBlock.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index 2b06e41986..490caa622f 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -54,7 +54,7 @@
       type: settings.type,
       onClick: async row => {
         const fn = enrichButtonActions(settings.onClick, get(context))
-        return await fn({ row })
+        return await fn?.({ row })
       },
     }))
   }

From 9f285b32a73ba9b09dbd6d9aea51f206ec114418 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 7 Nov 2023 13:21:13 +0000
Subject: [PATCH 032/155] Allow using ephemeral action context for 'buttons'
 type settings

---
 .../ButtonConfiguration/ButtonConfiguration.svelte     | 10 +++++++++-
 .../FieldConfiguration/FieldConfiguration.svelte       |  1 +
 packages/client/manifest.json                          |  9 ++++++++-
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
index 6fed8abdba..9126fa0f77 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
@@ -4,10 +4,13 @@
   import { createEventDispatcher } from "svelte"
   import { store } from "builderStore"
   import { Helpers } from "@budibase/bbui"
+  import { getEventContextBindings } from "builderStore/dataBinding"
 
+  export let componentInstance
   export let componentBindings
   export let bindings
   export let value
+  export let key
 
   const dispatch = createEventDispatcher()
 
@@ -15,9 +18,14 @@
 
   $: buttonList = sanitizeValue(value) || []
   $: buttonCount = buttonList.length
+  $: eventContextBindings = getEventContextBindings({
+    componentInstance,
+    settingKey: key,
+  })
+  $: allBindings = [...bindings, ...eventContextBindings]
   $: itemProps = {
     componentBindings: componentBindings || [],
-    bindings,
+    bindings: allBindings,
     removeButton,
     canRemove: true,
   }
diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
index 6c74705ab0..c1bf903584 100644
--- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
@@ -17,6 +17,7 @@
   export let value
 
   const dispatch = createEventDispatcher()
+
   let sanitisedFields
   let fieldList
   let schema
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 0627646c6f..6bd1643756 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -6346,7 +6346,14 @@
         "settings": [
           {
             "type": "buttonConfiguration",
-            "key": "buttons"
+            "key": "buttons",
+            "nested": true,
+            "context": [
+              {
+                "label": "Clicked row",
+                "key": "row"
+              }
+            ]
           }
         ]
       }

From fdae7ab913ec22e7ce22ff01e3356c13e1d53d05 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 7 Nov 2023 13:37:45 +0000
Subject: [PATCH 033/155] Fixing issues discovered by tests.

---
 packages/server/src/integration-test/postgres.spec.ts | 2 --
 packages/server/src/sdk/app/rows/search/external.ts   | 1 +
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 60416853b3..8dc49a9489 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -923,7 +923,6 @@ describe("postgres integrations", () => {
             [m2mFieldName]: [
               {
                 _id: row._id,
-                primaryDisplay: "Invalid display column",
               },
             ],
           })
@@ -932,7 +931,6 @@ describe("postgres integrations", () => {
             [m2mFieldName]: [
               {
                 _id: row._id,
-                primaryDisplay: "Invalid display column",
               },
             ],
           })
diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts
index 71c12f0a79..1edd6b30f2 100644
--- a/packages/server/src/sdk/app/rows/search/external.ts
+++ b/packages/server/src/sdk/app/rows/search/external.ts
@@ -186,6 +186,7 @@ export async function fetch(tableId: string): Promise<Row[]> {
   const table = await sdk.tables.getTable(tableId)
   return await outputProcessing<Row[]>(table, response, {
     preserveLinks: true,
+    squash: true,
   })
 }
 

From 312415ca7d1e99f71d3442f0b9674b1f73d8b349 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 7 Nov 2023 13:47:21 +0000
Subject: [PATCH 034/155] Enforcing squash for rows which contain circular
 structures.

---
 packages/backend-core/src/utils/utils.ts      | 14 +++++++++++++
 .../src/utilities/rowProcessor/index.ts       | 20 +++++++++++--------
 2 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts
index b92471a7a4..1c1ca8473b 100644
--- a/packages/backend-core/src/utils/utils.ts
+++ b/packages/backend-core/src/utils/utils.ts
@@ -237,3 +237,17 @@ export function timeout(timeMs: number) {
 export function isAudited(event: Event) {
   return !!AuditedEventFriendlyName[event]
 }
+
+export function hasCircularStructure(json: any) {
+  if (typeof json !== "object") {
+    return false
+  }
+  try {
+    JSON.stringify(json)
+  } catch (err) {
+    if (err instanceof Error && err?.message.includes("circular structure")) {
+      return true
+    }
+  }
+  return false
+}
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 6070d40f7d..0e53422a4f 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -2,16 +2,15 @@ import * as linkRows from "../../db/linkedRows"
 import { FieldTypes, AutoFieldSubTypes } from "../../constants"
 import { processFormulas, fixAutoColumnSubType } from "./utils"
 import { ObjectStoreBuckets } from "../../constants"
-import { context, db as dbCore, objectStore } from "@budibase/backend-core"
+import {
+  context,
+  db as dbCore,
+  objectStore,
+  utils,
+} from "@budibase/backend-core"
 import { InternalTables } from "../../db/utils"
 import { TYPE_TRANSFORM_MAP } from "./map"
-import {
-  AutoColumnFieldMetadata,
-  FieldSubtype,
-  Row,
-  RowAttachment,
-  Table,
-} from "@budibase/types"
+import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import {
   processInputBBReferences,
@@ -233,6 +232,11 @@ export async function outputProcessing<T extends Row[] | Row>(
       })
     : safeRows
 
+  // make sure squash is enabled if needed
+  if (!opts.squash && utils.hasCircularStructure(rows)) {
+    opts.squash = true
+  }
+
   // process complex types: attachements, bb references...
   for (let [property, column] of Object.entries(table.schema)) {
     if (column.type === FieldTypes.ATTACHMENT) {

From 8d35453f015b53d4f9a159ea6a494bfb227cd702 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 7 Nov 2023 13:56:42 +0000
Subject: [PATCH 035/155] Adding test case for circular detection.

---
 packages/backend-core/src/utils/tests/utils.spec.ts | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/packages/backend-core/src/utils/tests/utils.spec.ts b/packages/backend-core/src/utils/tests/utils.spec.ts
index 5a0ac4f283..7b411e801c 100644
--- a/packages/backend-core/src/utils/tests/utils.spec.ts
+++ b/packages/backend-core/src/utils/tests/utils.spec.ts
@@ -188,4 +188,17 @@ describe("utils", () => {
       expectResult(false)
     })
   })
+
+  describe("hasCircularStructure", () => {
+    it("should detect a circular structure", () => {
+      const a: any = { b: "b" }
+      const b = { a }
+      a.b = b
+      expect(utils.hasCircularStructure(b)).toBe(true)
+    })
+
+    it("should allow none circular structures", () => {
+      expect(utils.hasCircularStructure({ a: "b" })).toBe(false)
+    })
+  })
 })

From c25b8a1448bf44b462fbf0a83268709c6dfdbab3 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 7 Nov 2023 14:10:16 +0000
Subject: [PATCH 036/155] Expose schema context from grid and allow usage in
 buttons

---
 .../ButtonActionEditor/actions/SaveRow.svelte  |  3 ++-
 .../ButtonConfiguration.svelte                 |  2 ++
 .../ButtonConfiguration/ButtonSetting.svelte   | 18 ++++++++++++++++++
 packages/client/manifest.json                  |  5 ++++-
 .../client/src/components/app/GridBlock.svelte |  6 +++++-
 .../components/grid/layout/ButtonColumn.svelte |  9 ++++++++-
 .../src/components/grid/stores/rows.js         |  8 ++++++--
 7 files changed, 45 insertions(+), 6 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte
index c1917ad90f..27b6463ffa 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte
@@ -21,7 +21,8 @@
   $: schemaComponents = getContextProviderComponents(
     $currentAsset,
     $store.selectedComponentId,
-    "schema"
+    "schema",
+    { includeSelf: nested }
   )
   $: providerOptions = getProviderOptions(formComponents, schemaComponents)
   $: schemaFields = getSchemaFields(parameters?.tableId)
diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
index 9126fa0f77..3f1ea83d8c 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
@@ -11,6 +11,7 @@
   export let bindings
   export let value
   export let key
+  export let nested
 
   const dispatch = createEventDispatcher()
 
@@ -28,6 +29,7 @@
     bindings: allBindings,
     removeButton,
     canRemove: true,
+    nested,
   }
 
   const sanitizeValue = val => {
diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
index a05fd9a39b..17c9667c26 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
@@ -10,10 +10,27 @@
   export let anchor
   export let removeButton
   export let canRemove
+  export let nested
 
   $: readableText = isJSBinding(item.text)
     ? "(JavaScript function)"
     : runtimeToReadableBinding([...bindings, componentBindings], item.text)
+
+  // If this is a nested setting (for example inside a grid or form block) then
+  // we need to mark all the settings of the actual buttons as nested too, to
+  // allow up to reference context provided by the block
+  const updatedNestedFlags = settings => {
+    if (!nested) {
+      return settings
+    }
+    // Buttons do not currently have any sections, so this works.
+    // We will need to update this in future if the normal button component
+    // gets broken into multiple settings sections
+    return settings?.map(setting => ({
+      ...setting,
+      nested: true,
+    }))
+  }
 </script>
 
 <div class="list-item-body">
@@ -24,6 +41,7 @@
       {componentBindings}
       {bindings}
       on:change
+      parseSettings={updatedNestedFlags}
     />
     <div class="field-label">{readableText || "Button"}</div>
   </div>
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 6bd1643756..d131cda8c3 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -6357,7 +6357,10 @@
           }
         ]
       }
-    ]
+    ],
+    "context": {
+      "type": "schema"
+    }
   },
   "bbreferencefield": {
     "devComment": "As bb reference is only used for user subtype for now, we are using user for icon and labels",
diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index 490caa622f..847f1e0768 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -53,7 +53,11 @@
       text: settings.text,
       type: settings.type,
       onClick: async row => {
-        const fn = enrichButtonActions(settings.onClick, get(context))
+        // We add a fake context binding in here, which allows us to pretend
+        // that the grid provides a "schema" binding - that lets us use the
+        // clicked row in things like save row actions
+        const enrichedContext = { ...get(context), [$component.id]: row }
+        const fn = enrichButtonActions(settings.onClick, enrichedContext)
         return await fn?.({ row })
       },
     }))
diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
index 4acd8b6b70..65b9d212ca 100644
--- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
+++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
@@ -9,6 +9,7 @@
     hoveredRowId,
     props,
     width,
+    rows,
     focusedRow,
     selectedRows,
     visibleColumns,
@@ -27,6 +28,12 @@
   $: end = columnsWidth - 1 - $scroll.left
   $: left = Math.min($width - $buttonColumnWidth, end)
 
+  const handleClick = async (button, row) => {
+    await button.onClick?.(row)
+    // Refresh the row in case it changed
+    await rows.actions.refreshRow(row._id)
+  }
+
   onMount(() => {
     const observer = new ResizeObserver(entries => {
       const width = entries?.[0]?.contentRect?.width ?? 0
@@ -81,7 +88,7 @@
                   secondary={button.type === "secondary"}
                   warning={button.type === "warning"}
                   overBackground={button.type === "overBackground"}
-                  on:click={() => button.onClick?.(row)}
+                  on:click={() => handleClick(button, row)}
                 >
                   {button.text || "Button"}
                 </Button>
diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 82185d6b91..34ba77afa2 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -314,8 +314,12 @@ export const createActions = context => {
 
   // Refreshes a specific row
   const refreshRow = async id => {
-    const row = await datasource.actions.getRow(id)
-    replaceRow(id, row)
+    try {
+      const row = await datasource.actions.getRow(id)
+      replaceRow(id, row)
+    } catch {
+      // Do nothing - we probably just don't support refreshing individual rows
+    }
   }
 
   // Refreshes all data

From 8017dd592b518c695ba2dde3961b6bacbc10a40f Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 7 Nov 2023 14:12:06 +0000
Subject: [PATCH 037/155] Ensure rows are cleaned before being passed to button
 actions in grids

---
 .../src/components/grid/layout/ButtonColumn.svelte              | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
index 65b9d212ca..d484f9408d 100644
--- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
+++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
@@ -29,7 +29,7 @@
   $: left = Math.min($width - $buttonColumnWidth, end)
 
   const handleClick = async (button, row) => {
-    await button.onClick?.(row)
+    await button.onClick?.(rows.actions.cleanRow(row))
     // Refresh the row in case it changed
     await rows.actions.refreshRow(row._id)
   }

From 3f91d62b272c2de2a447b6a573eca85c45518ecf Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 7 Nov 2023 14:19:17 +0000
Subject: [PATCH 038/155] Allow button configuration settings to be cleared
 completely

---
 .../controls/ButtonConfiguration/ButtonConfiguration.svelte     | 1 -
 .../settings/controls/ButtonConfiguration/ButtonSetting.svelte  | 2 --
 2 files changed, 3 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
index 3f1ea83d8c..a24d18909f 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
@@ -28,7 +28,6 @@
     componentBindings: componentBindings || [],
     bindings: allBindings,
     removeButton,
-    canRemove: true,
     nested,
   }
 
diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
index 17c9667c26..c84e86c8cc 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
@@ -9,7 +9,6 @@
   export let bindings
   export let anchor
   export let removeButton
-  export let canRemove
   export let nested
 
   $: readableText = isJSBinding(item.text)
@@ -47,7 +46,6 @@
   </div>
   <div class="list-item-right">
     <Icon
-      disabled={!canRemove}
       size="S"
       name="Close"
       hoverable

From 19a9318eac778cccdd77ccdc8ee014a4cab88f53 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 7 Nov 2023 14:19:52 +0000
Subject: [PATCH 039/155] Type

---
 .../settings/controls/ButtonConfiguration/ButtonSetting.svelte  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
index c84e86c8cc..8e8c89ac14 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
@@ -17,7 +17,7 @@
 
   // If this is a nested setting (for example inside a grid or form block) then
   // we need to mark all the settings of the actual buttons as nested too, to
-  // allow up to reference context provided by the block
+  // allow us to reference context provided by the block
   const updatedNestedFlags = settings => {
     if (!nested) {
       return settings

From 3d2cd9cc0bf7810d5584b399f840353eb852aaf1 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 7 Nov 2023 14:22:09 +0000
Subject: [PATCH 040/155] Use explicit svelte store 'get' when clicking buttons
 in grids

---
 packages/client/src/components/app/GridBlock.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index 847f1e0768..801e1a4d0a 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -56,7 +56,7 @@
         // We add a fake context binding in here, which allows us to pretend
         // that the grid provides a "schema" binding - that lets us use the
         // clicked row in things like save row actions
-        const enrichedContext = { ...get(context), [$component.id]: row }
+        const enrichedContext = { ...get(context), [get(component).id]: row }
         const fn = enrichButtonActions(settings.onClick, enrichedContext)
         return await fn?.({ row })
       },

From a3435728150c9a981fdb3b3f8b0919dcbd1eef27 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 7 Nov 2023 14:36:57 +0000
Subject: [PATCH 041/155] Add utils to client lib

---
 packages/client/src/sdk.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/client/src/sdk.js b/packages/client/src/sdk.js
index 77089d648b..b83039df2a 100644
--- a/packages/client/src/sdk.js
+++ b/packages/client/src/sdk.js
@@ -14,6 +14,7 @@ import {
   dndIsDragging,
   confirmationStore,
   roleStore,
+  stateStore,
 } from "stores"
 import { styleable } from "utils/styleable"
 import { linkable } from "utils/linkable"
@@ -25,6 +26,7 @@ import { ActionTypes } from "./constants"
 import { fetchDatasourceSchema } from "./utils/schema.js"
 import { getAPIKey } from "./utils/api.js"
 import { enrichButtonActions } from "./utils/buttonActions.js"
+import { processStringSync } from "@budibase/string-templates"
 
 export default {
   API,
@@ -44,6 +46,7 @@ export default {
   currentRole,
   confirmationStore,
   roleStore,
+  stateStore,
 
   // Utils
   styleable,
@@ -52,6 +55,7 @@ export default {
   fetchDatasourceSchema,
   getAPIKey,
   enrichButtonActions,
+  processStringSync,
 
   // Components
   Provider,

From ace3b69971c67537cac012cfb6c4d06683757605 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 7 Nov 2023 14:55:02 +0000
Subject: [PATCH 042/155] Limit grid buttons at 3

---
 .../ButtonConfiguration.svelte                | 20 ++++++++++++++-----
 packages/client/manifest.json                 |  1 +
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
index a24d18909f..ce91c8f7b5 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
@@ -12,6 +12,7 @@
   export let value
   export let key
   export let nested
+  export let max
 
   const dispatch = createEventDispatcher()
 
@@ -30,6 +31,7 @@
     removeButton,
     nested,
   }
+  $: canAddButtons = max == null || buttonList.length < max
 
   const sanitizeValue = val => {
     return val?.map(button => {
@@ -97,7 +99,12 @@
     />
   {/if}
 
-  <div class="list-footer" on:click={addButton} class:empty={!buttonCount}>
+  <div
+    class="list-footer"
+    class:disabled={!canAddButtons}
+    on:click={addButton}
+    class:empty={!buttonCount}
+  >
     <div class="add-button">Add button</div>
   </div>
 </div>
@@ -132,15 +139,18 @@
   .list-footer.empty {
     border-radius: 4px;
   }
-
-  .add-button {
-    margin: var(--spacing-s);
+  .list-footer.disabled {
+    color: var(--spectrum-global-color-gray-500);
+    pointer-events: none;
   }
-
   .list-footer:hover {
     background-color: var(
       --spectrum-table-row-background-color-hover,
       var(--spectrum-alias-highlight-hover)
     );
   }
+
+  .add-button {
+    margin: var(--spacing-s);
+  }
 </style>
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index d131cda8c3..378d8c3493 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -6348,6 +6348,7 @@
             "type": "buttonConfiguration",
             "key": "buttons",
             "nested": true,
+            "max": 3,
             "context": [
               {
                 "label": "Clicked row",

From 34693481b13c858b5f8e6e45831ed75d6ad322dc Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 7 Nov 2023 15:02:45 +0000
Subject: [PATCH 043/155] Fix bug in SectionSelect

---
 .../components/design/settings/controls/SectionSelect.svelte  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/design/settings/controls/SectionSelect.svelte b/packages/builder/src/components/design/settings/controls/SectionSelect.svelte
index 4394f47cef..5b4eb8d2a9 100644
--- a/packages/builder/src/components/design/settings/controls/SectionSelect.svelte
+++ b/packages/builder/src/components/design/settings/controls/SectionSelect.svelte
@@ -32,7 +32,9 @@
   }
 </script>
 
-<ActionButton on:click={modal.show}>{layoutMap[value].name}</ActionButton>
+<ActionButton on:click={modal.show}>
+  {layoutMap[value || "mainSidebar"].name}
+</ActionButton>
 <Modal bind:this={modal}>
   <ModalContent
     onConfirm={() => dispatch("change", selected)}

From a39accdb42ce6921113506dda9dc0533b76da12f Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Tue, 7 Nov 2023 15:07:00 +0000
Subject: [PATCH 044/155] Test coverage added for the component. Minor refactor

---
 .../DataTable/modals/ExportModal.svelte       |  53 ++--
 .../DataTable/modals/ExportModal.test.js      | 240 ++++++++++++++++++
 packages/shared-core/src/utils.ts             |  14 +
 3 files changed, 281 insertions(+), 26 deletions(-)
 create mode 100644 packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js

diff --git a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte
index 61c81c86eb..c2a7ec33fe 100644
--- a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte
@@ -8,7 +8,8 @@
   } from "@budibase/bbui"
   import download from "downloadjs"
   import { API } from "api"
-  import { Constants, LuceneUtils } from "@budibase/frontend-core"
+  import { LuceneUtils } from "@budibase/frontend-core"
+  import { utils } from "@budibase/shared-core"
   import { ROW_EXPORT_FORMATS } from "constants/backend"
 
   export let view
@@ -17,8 +18,6 @@
   export let selectedRows = []
   export let formats
 
-  $: appliedFilters = filters?.filter(filter => !filter.onEmptyFilter)
-
   const FORMATS = [
     {
       name: "CSV",
@@ -34,6 +33,8 @@
     },
   ]
 
+  $: appliedFilters = filters?.filter(filter => !filter.onEmptyFilter)
+
   $: options = FORMATS.filter(format => {
     if (formats && !formats.includes(format.key)) {
       return false
@@ -55,14 +56,7 @@
     appliedFilters
   )
 
-  const buildFilterLookup = () => {
-    return Object.keys(Constants.OperatorOptions).reduce((acc, key) => {
-      const op = Constants.OperatorOptions[key]
-      acc[op.value] = op.label
-      return acc
-    }, {})
-  }
-  filterLookup = buildFilterLookup()
+  filterLookup = utils.filterValueToLabel()
 
   const filterDisplay = () => {
     if (!appliedFilters) {
@@ -173,19 +167,23 @@
 >
   {#if selectedRows?.length}
     <Body size="S">
-      <strong>{selectedRows?.length}</strong>
-      {`row${selectedRows?.length > 1 ? "s" : ""} will be exported`}
+      <span data-testid="exporting-n-rows">
+        <strong>{selectedRows?.length}</strong>
+        {`row${selectedRows?.length > 1 ? "s" : ""} will be exported`}
+      </span>
     </Body>
   {:else if appliedFilters?.length || (sorting?.sortOrder && sorting?.sortColumn)}
     <Body size="S">
       {#if !appliedFilters}
-        Exporting <strong>all</strong> rows
+        <span data-testid="exporting-rows">
+          Exporting <strong>all</strong> rows
+        </span>
       {:else}
-        Filters applied
+        <span data-testid="filters-applied">Filters applied</span>
       {/if}
     </Body>
 
-    <div class="table-wrap">
+    <div class="table-wrap" data-testid="export-config-table">
       <Table
         schema={displaySchema}
         data={exportOpDisplay}
@@ -202,18 +200,21 @@
     </div>
   {:else}
     <Body size="S">
-      Exporting <strong>all</strong> rows
+      <span data-testid="export-all-rows">
+        Exporting <strong>all</strong> rows
+      </span>
     </Body>
   {/if}
-
-  <Select
-    label="Format"
-    bind:value={exportFormat}
-    {options}
-    placeholder={null}
-    getOptionLabel={x => x.name}
-    getOptionValue={x => x.key}
-  />
+  <span data-testid="format-select">
+    <Select
+      label="Format"
+      bind:value={exportFormat}
+      {options}
+      placeholder={null}
+      getOptionLabel={x => x.name}
+      getOptionValue={x => x.key}
+    />
+  </span>
 </ModalContent>
 
 <style>
diff --git a/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js b/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js
new file mode 100644
index 0000000000..feb776eb5e
--- /dev/null
+++ b/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js
@@ -0,0 +1,240 @@
+import { it, expect, describe } from "vitest"
+import { render, screen } from "@testing-library/svelte"
+import "@testing-library/jest-dom"
+
+import ExportModal from "./ExportModal.svelte"
+import { utils } from "@budibase/shared-core"
+
+const labelLookup = utils.filterValueToLabel()
+
+const rowText = filter => {
+  let readableField = filter.field.split(":")[1]
+  let rowLabel = labelLookup[filter.operator]
+  let value = Array.isArray(filter.value)
+    ? JSON.stringify(filter.value)
+    : filter.value
+  return `${readableField}${rowLabel}${value}`.trim()
+}
+
+const defaultFilters = [
+  {
+    onEmptyFilter: "all",
+  },
+]
+
+vi.mock("svelte", async () => {
+  return {
+    getContext: () => {
+      return {
+        hide: vi.fn(),
+        cancel: vi.fn(),
+      }
+    },
+    createEventDispatcher: vi.fn(),
+    onDestroy: vi.fn(),
+  }
+})
+
+vi.mock("api", async () => {
+  return {
+    API: {
+      exportView: vi.fn(),
+      exportRows: vi.fn(),
+    },
+  }
+})
+
+describe("Export Modal", () => {
+  it("show default messaging with no export config specified", () => {
+    render(ExportModal, {
+      props: {},
+    })
+
+    expect(screen.getByTestId("export-all-rows")).toBeVisible()
+    expect(screen.getByTestId("export-all-rows")).toHaveTextContent(
+      "Exporting all rows"
+    )
+
+    expect(screen.queryByTestId("export-config-table")).toBe(null)
+  })
+
+  it("indicate that a filter is being applied to the export", () => {
+    const propsCfg = {
+      filters: [
+        {
+          id: "MOQkMx9p9",
+          field: "1:Cost",
+          operator: "rangeHigh",
+          value: "100",
+          valueType: "Value",
+          type: "number",
+          noValue: false,
+        },
+        ...defaultFilters,
+      ],
+    }
+
+    render(ExportModal, {
+      props: propsCfg,
+    })
+
+    expect(screen.getByTestId("filters-applied")).toBeVisible()
+    expect(screen.getByTestId("filters-applied").textContent).toBe(
+      "Filters applied"
+    )
+
+    const ele = screen.queryByTestId("export-config-table")
+    expect(ele).toBeVisible()
+
+    const rows = ele.getElementsByClassName("spectrum-Table-row")
+
+    expect(rows.length).toBe(1)
+    let rowTextContent = rowText(propsCfg.filters[0])
+
+    //"CostLess than or equal to100"
+    expect(rows[0].textContent?.trim()).toEqual(rowTextContent)
+  })
+
+  it("Show only selected row messaging if rows are supplied", () => {
+    const propsCfg = {
+      filters: [
+        {
+          id: "MOQkMx9p9",
+          field: "1:Cost",
+          operator: "rangeHigh",
+          value: "100",
+          valueType: "Value",
+          type: "number",
+          noValue: false,
+        },
+        ...defaultFilters,
+      ],
+      sorting: {
+        sortColumn: "Cost",
+        sortOrder: "descending",
+      },
+      selectedRows: [
+        {
+          _id: "ro_ta_bb_expenses_57d5f6fe1b6640d8bb22b15f5eae62cd",
+        },
+        {
+          _id: "ro_ta_bb_expenses_99ce5760a53a430bab4349cd70335a07",
+        },
+      ],
+    }
+
+    render(ExportModal, {
+      props: propsCfg,
+    })
+
+    expect(screen.queryByTestId("export-config-table")).toBeNull()
+    expect(screen.queryByTestId("filters-applied")).toBeNull()
+
+    expect(screen.queryByTestId("exporting-n-rows")).toBeVisible()
+    expect(screen.queryByTestId("exporting-n-rows").textContent).toEqual(
+      "2 rows will be exported"
+    )
+  })
+
+  it("Show only the configured sort when no filters are specified", () => {
+    const propsCfg = {
+      filters: [...defaultFilters],
+      sorting: {
+        sortColumn: "Cost",
+        sortOrder: "descending",
+      },
+    }
+    render(ExportModal, {
+      props: propsCfg,
+    })
+
+    expect(screen.queryByTestId("export-config-table")).toBeVisible()
+    const ele = screen.queryByTestId("export-config-table")
+    const rows = ele.getElementsByClassName("spectrum-Table-row")
+
+    expect(rows.length).toBe(1)
+    expect(rows[0].textContent?.trim()).toEqual(
+      `${propsCfg.sorting.sortColumn}Order By${propsCfg.sorting.sortOrder}`
+    )
+  })
+
+  it("Display all currently configured filters and applied sort", () => {
+    const propsCfg = {
+      filters: [
+        {
+          id: "MOQkMx9p9",
+          field: "1:Cost",
+          operator: "rangeHigh",
+          value: "100",
+          valueType: "Value",
+          type: "number",
+          noValue: false,
+        },
+        {
+          id: "2ot-aB0gE",
+          field: "2:Expense Tags",
+          operator: "contains",
+          value: ["Equipment", "Services"],
+          valueType: "Value",
+          type: "array",
+          noValue: false,
+        },
+        ...defaultFilters,
+      ],
+      sorting: {
+        sortColumn: "Payment Due",
+        sortOrder: "ascending",
+      },
+    }
+
+    render(ExportModal, {
+      props: propsCfg,
+    })
+
+    const ele = screen.queryByTestId("export-config-table")
+    expect(ele).toBeVisible()
+
+    const rows = ele.getElementsByClassName("spectrum-Table-row")
+    expect(rows.length).toBe(3)
+
+    let rowTextContent1 = rowText(propsCfg.filters[0])
+    expect(rows[0].textContent?.trim()).toEqual(rowTextContent1)
+
+    let rowTextContent2 = rowText(propsCfg.filters[1])
+    expect(rows[1].textContent?.trim()).toEqual(rowTextContent2)
+
+    expect(rows[2].textContent?.trim()).toEqual(
+      `${propsCfg.sorting.sortColumn}Order By${propsCfg.sorting.sortOrder}`
+    )
+  })
+
+  it("show only the valid, configured download formats", () => {
+    const propsCfg = {
+      formats: ["badger", "json"],
+    }
+
+    render(ExportModal, {
+      props: propsCfg,
+    })
+
+    let ele = screen.getByTestId("format-select")
+    expect(ele).toBeVisible()
+
+    let formatDisplay = ele.getElementsByTagName("button")[0]
+
+    expect(formatDisplay.textContent.trim()).toBe("JSON")
+  })
+
+  it("Load the default format config when no explicit formats are configured", () => {
+    render(ExportModal, {
+      props: {},
+    })
+
+    let ele = screen.getByTestId("format-select")
+    expect(ele).toBeVisible()
+
+    let formatDisplay = ele.getElementsByTagName("button")[0]
+
+    expect(formatDisplay.textContent.trim()).toBe("CSV")
+  })
+})
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 46a6585f89..c775010909 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -1,3 +1,5 @@
+import * as Constants from "./constants"
+
 export function unreachable(
   value: never,
   message = `No such case in exhaustive switch: ${value}`
@@ -43,3 +45,15 @@ export async function parallelForeach<T>(
 
   await Promise.all(promises)
 }
+
+export function filterValueToLabel() {
+  return Object.keys(Constants.OperatorOptions).reduce(
+    (acc: { [key: string]: string }, key: string) => {
+      const ops: { [key: string]: any } = Constants.OperatorOptions
+      const op: { [key: string]: string } = ops[key]
+      acc[op["value"]] = op.label
+      return acc
+    },
+    {}
+  )
+}

From cc5418e16e942267217ec62658ce191039634e37 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 7 Nov 2023 15:10:27 +0000
Subject: [PATCH 045/155] Update pull_request_template.md

Remove colon in pull request template which break styling
---
 pull_request_template.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pull_request_template.md b/pull_request_template.md
index 1dd1a1d45d..405059b2ab 100644
--- a/pull_request_template.md
+++ b/pull_request_template.md
@@ -1,7 +1,7 @@
 ## Description
 _Describe the problem or feature in addition to a link to the relevant github issues._
 
-### Addresses:
+## Addresses
 - `<Enter the Link to the issue(s) this PR addresses>`
 - ...more if required
 

From 856fe14ae9f877c83bf353a47fe11d6c6cdf8c8a Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Tue, 7 Nov 2023 15:14:33 +0000
Subject: [PATCH 046/155] Missing import

---
 .../src/components/backend/DataTable/modals/ExportModal.test.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js b/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js
index feb776eb5e..f4c2cdc8b2 100644
--- a/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js
+++ b/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js
@@ -1,4 +1,4 @@
-import { it, expect, describe } from "vitest"
+import { it, expect, describe, vi } from "vitest"
 import { render, screen } from "@testing-library/svelte"
 import "@testing-library/jest-dom"
 

From 86cfd76b5a7edc02614431200a6640fc43b5421d Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 7 Nov 2023 16:37:31 +0000
Subject: [PATCH 047/155] Bump version to 2.13.3

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index bfcac5633c..25ec556c56 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.13.2",
+  "version": "2.13.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 9d29e3dddab4daeb3b656e71a7fe1d5b7c726d74 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 7 Nov 2023 18:31:44 +0100
Subject: [PATCH 048/155] Update pro ref

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 3820c0c93a..cffa01ac67 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8
+Subproject commit cffa01ac67a2ca7401e848593f95f881e103a33d

From 4d5d9a0685a9c340c9859414363bb8654b6e0ef6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 7 Nov 2023 18:42:41 +0100
Subject: [PATCH 049/155] Update pro ref

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index cffa01ac67..ad9a0085be 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit cffa01ac67a2ca7401e848593f95f881e103a33d
+Subproject commit ad9a0085bee0c4f3184acd86cadd872ea9917e88

From fdfda100c1f019067bda247924d452eb4344c1a2 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 7 Nov 2023 18:14:52 +0000
Subject: [PATCH 050/155] Ensure that the DB always returns Documents.

---
 .../backend-core/src/db/couch/DatabaseImpl.ts |  6 ++-
 packages/backend-core/src/db/views.ts         | 27 ++++++-----
 packages/backend-core/src/users/db.ts         | 10 ++--
 packages/backend-core/src/users/users.ts      |  2 +-
 packages/server/src/api/controllers/layout.ts |  4 +-
 .../server/src/api/controllers/permission.ts  |  6 +--
 packages/server/src/api/controllers/role.ts   | 12 +++--
 .../src/api/controllers/row/internal.ts       |  9 ++--
 .../src/api/controllers/row/staticFormula.ts  |  4 +-
 .../server/src/api/controllers/view/utils.ts  |  4 +-
 packages/server/src/automations/triggers.ts   | 10 ++--
 packages/server/src/db/linkedRows/index.ts    | 23 +++++----
 .../server/src/sdk/app/applications/import.ts |  2 +-
 .../server/src/sdk/app/applications/sync.ts   | 10 ++--
 .../src/sdk/app/datasources/datasources.ts    |  6 +--
 packages/server/src/sdk/app/tables/getters.ts |  4 +-
 packages/server/src/sdk/users/utils.ts        | 30 +++++++-----
 packages/server/src/utilities/global.ts       | 48 +++++++++----------
 .../server/src/utilities/routing/index.ts     |  4 +-
 packages/types/src/documents/app/user.ts      | 10 ++--
 packages/types/src/documents/pouch.ts         |  8 ++--
 packages/types/src/sdk/db.ts                  |  6 ++-
 .../worker/src/constants/templates/index.ts   |  6 +--
 packages/worker/src/utilities/email.ts        | 11 +----
 24 files changed, 140 insertions(+), 122 deletions(-)

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index 29ca4123f5..330b15e680 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -175,12 +175,14 @@ export class DatabaseImpl implements Database {
     return this.updateOutput(() => db.bulk({ docs: documents }))
   }
 
-  async allDocs<T>(params: DatabaseQueryOpts): Promise<AllDocsResponse<T>> {
+  async allDocs<T extends Document>(
+    params: DatabaseQueryOpts
+  ): Promise<AllDocsResponse<T>> {
     const db = await this.checkSetup()
     return this.updateOutput(() => db.list(params))
   }
 
-  async query<T>(
+  async query<T extends Document>(
     viewName: string,
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>> {
diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts
index f0980ad217..fbcdbde5fc 100644
--- a/packages/backend-core/src/db/views.ts
+++ b/packages/backend-core/src/db/views.ts
@@ -7,7 +7,12 @@ import {
 } from "../constants"
 import { getGlobalDB } from "../context"
 import { doWithDB } from "./"
-import { AllDocsResponse, Database, DatabaseQueryOpts } from "@budibase/types"
+import {
+  AllDocsResponse,
+  Database,
+  DatabaseQueryOpts,
+  Document,
+} from "@budibase/types"
 import env from "../environment"
 
 const DESIGN_DB = "_design/database"
@@ -109,7 +114,7 @@ export interface QueryViewOptions {
   arrayResponse?: boolean
 }
 
-export async function queryViewRaw<T>(
+export async function queryViewRaw<T extends Document>(
   viewName: ViewName,
   params: DatabaseQueryOpts,
   db: Database,
@@ -137,18 +142,16 @@ export async function queryViewRaw<T>(
   }
 }
 
-export const queryView = async <T>(
+export const queryView = async <T extends Document>(
   viewName: ViewName,
   params: DatabaseQueryOpts,
   db: Database,
   createFunc: any,
   opts?: QueryViewOptions
-): Promise<T[] | T | undefined> => {
+): Promise<T[] | T> => {
   const response = await queryViewRaw<T>(viewName, params, db, createFunc, opts)
   const rows = response.rows
-  const docs = rows.map((row: any) =>
-    params.include_docs ? row.doc : row.value
-  )
+  const docs = rows.map(row => (params.include_docs ? row.doc! : row.value))
 
   // if arrayResponse has been requested, always return array regardless of length
   if (opts?.arrayResponse) {
@@ -198,11 +201,11 @@ export const createPlatformUserView = async () => {
   await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
 }
 
-export const queryPlatformView = async <T>(
+export const queryPlatformView = async <T extends Document>(
   viewName: ViewName,
   params: DatabaseQueryOpts,
   opts?: QueryViewOptions
-): Promise<T[] | T | undefined> => {
+): Promise<T[] | T> => {
   const CreateFuncByName: any = {
     [ViewName.ACCOUNT_BY_EMAIL]: createPlatformAccountEmailView,
     [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView,
@@ -220,7 +223,7 @@ const CreateFuncByName: any = {
   [ViewName.USER_BY_APP]: createUserAppView,
 }
 
-export const queryGlobalView = async <T>(
+export const queryGlobalView = async <T extends Document>(
   viewName: ViewName,
   params: DatabaseQueryOpts,
   db?: Database,
@@ -231,10 +234,10 @@ export const queryGlobalView = async <T>(
     db = getGlobalDB()
   }
   const createFn = CreateFuncByName[viewName]
-  return queryView(viewName, params, db!, createFn, opts)
+  return queryView<T>(viewName, params, db!, createFn, opts)
 }
 
-export async function queryGlobalViewRaw<T>(
+export async function queryGlobalViewRaw<T extends Document>(
   viewName: ViewName,
   params: DatabaseQueryOpts,
   opts?: QueryViewOptions
diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts
index c071064713..59f698d99c 100644
--- a/packages/backend-core/src/users/db.ts
+++ b/packages/backend-core/src/users/db.ts
@@ -413,15 +413,13 @@ export class UserDB {
     }
 
     // Get users and delete
-    const allDocsResponse: AllDocsResponse<User> = await db.allDocs({
+    const allDocsResponse = await db.allDocs<User>({
       include_docs: true,
       keys: userIds,
     })
-    const usersToDelete: User[] = allDocsResponse.rows.map(
-      (user: RowResponse<User>) => {
-        return user.doc
-      }
-    )
+    const usersToDelete = allDocsResponse.rows.map(user => {
+      return user.doc!
+    })
 
     // Delete from DB
     const toDelete = usersToDelete.map(user => ({
diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts
index 6dc8750b62..9f4a41f6df 100644
--- a/packages/backend-core/src/users/users.ts
+++ b/packages/backend-core/src/users/users.ts
@@ -151,7 +151,7 @@ export const searchGlobalUsersByApp = async (
     include_docs: true,
   })
   params.startkey = opts && opts.startkey ? opts.startkey : params.startkey
-  let response = await queryGlobalView(ViewName.USER_BY_APP, params)
+  let response = await queryGlobalView<User>(ViewName.USER_BY_APP, params)
 
   if (!response) {
     response = []
diff --git a/packages/server/src/api/controllers/layout.ts b/packages/server/src/api/controllers/layout.ts
index c00252d643..2a359592c7 100644
--- a/packages/server/src/api/controllers/layout.ts
+++ b/packages/server/src/api/controllers/layout.ts
@@ -30,12 +30,12 @@ export async function destroy(ctx: BBContext) {
     layoutRev = ctx.params.layoutRev
 
   const layoutsUsedByScreens = (
-    await db.allDocs(
+    await db.allDocs<any>(
       getScreenParams(null, {
         include_docs: true,
       })
     )
-  ).rows.map(element => element.doc.layoutId)
+  ).rows.map(element => element.doc!.layoutId)
   if (layoutsUsedByScreens.includes(layoutId)) {
     ctx.throw(400, "Cannot delete a layout that's being used by a screen")
   }
diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts
index a9cd686674..e2bd6c40e5 100644
--- a/packages/server/src/api/controllers/permission.ts
+++ b/packages/server/src/api/controllers/permission.ts
@@ -25,12 +25,12 @@ const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS
 
 // utility function to stop this repetition - permissions always stored under roles
 async function getAllDBRoles(db: Database) {
-  const body = await db.allDocs(
+  const body = await db.allDocs<Role>(
     getRoleParams(null, {
       include_docs: true,
     })
   )
-  return body.rows.map(row => row.doc)
+  return body.rows.map(row => row.doc!)
 }
 
 async function updatePermissionOnRole(
@@ -79,7 +79,7 @@ async function updatePermissionOnRole(
     ) {
       rolePermissions[resourceId] =
         typeof rolePermissions[resourceId] === "string"
-          ? [rolePermissions[resourceId]]
+          ? [rolePermissions[resourceId] as unknown as string]
           : []
     }
     // handle the removal/updating the role which has this permission first
diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index 3697bbe925..ae6b89e6d4 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -6,7 +6,13 @@ import {
   Header,
 } from "@budibase/backend-core"
 import { getUserMetadataParams, InternalTables } from "../../db/utils"
-import { Database, Role, UserCtx, UserRoles } from "@budibase/types"
+import {
+  Database,
+  Role,
+  UserCtx,
+  UserMetadata,
+  UserRoles,
+} from "@budibase/types"
 import { sdk as sharedSdk } from "@budibase/shared-core"
 import sdk from "../../sdk"
 
@@ -115,12 +121,12 @@ export async function destroy(ctx: UserCtx) {
   const role = await db.get<Role>(roleId)
   // first check no users actively attached to role
   const users = (
-    await db.allDocs(
+    await db.allDocs<UserMetadata>(
       getUserMetadataParams(undefined, {
         include_docs: true,
       })
     )
-  ).rows.map(row => row.doc)
+  ).rows.map(row => row.doc!)
   const usersWithRole = users.filter(user => user.roleId === roleId)
   if (usersWithRole.length !== 0) {
     ctx.throw(400, "Cannot delete role when it is in use.")
diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts
index 80a69cf92b..d53345a239 100644
--- a/packages/server/src/api/controllers/row/internal.ts
+++ b/packages/server/src/api/controllers/row/internal.ts
@@ -233,17 +233,16 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
   const tableId = utils.getTableId(ctx)
   const rowId = ctx.params.rowId as string
   // need table to work out where links go in row, as well as the link docs
-  let response = await Promise.all([
+  const [table, row, links] = await Promise.all([
     sdk.tables.getTable(tableId),
     utils.findRow(ctx, tableId, rowId),
     linkRows.getLinkDocuments({ tableId, rowId, fieldName }),
   ])
-  const table = response[0] as Table
-  const row = response[1] as Row
-  const linkVals = response[2] as LinkDocumentValue[]
+  const linkVals = links as LinkDocumentValue[]
+
   // look up the actual rows based on the ids
   const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id))
-  let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc)
+  let linkedRows = (await db.allDocs<Row>(params)).rows.map(row => row.doc!)
 
   // get the linked tables
   const linkTableIds = getLinkedTableIDs(table as Table)
diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts
index 6f426c6fa0..87d551bc4b 100644
--- a/packages/server/src/api/controllers/row/staticFormula.ts
+++ b/packages/server/src/api/controllers/row/staticFormula.ts
@@ -86,12 +86,12 @@ export async function updateAllFormulasInTable(table: Table) {
   const db = context.getAppDB()
   // start by getting the raw rows (which will be written back to DB after update)
   let rows = (
-    await db.allDocs(
+    await db.allDocs<Row>(
       getRowParams(table._id, null, {
         include_docs: true,
       })
     )
-  ).rows.map(row => row.doc)
+  ).rows.map(row => row.doc!)
   // now enrich the rows, note the clone so that we have the base state of the
   // rows so that we don't write any of the enriched information back
   let enrichedRows = await outputProcessing(table, cloneDeep(rows), {
diff --git a/packages/server/src/api/controllers/view/utils.ts b/packages/server/src/api/controllers/view/utils.ts
index 189c4ede51..9ffa091a8c 100644
--- a/packages/server/src/api/controllers/view/utils.ts
+++ b/packages/server/src/api/controllers/view/utils.ts
@@ -53,12 +53,12 @@ export async function getViews() {
     }
   } else {
     const views = (
-      await db.allDocs(
+      await db.allDocs<any>(
         getMemoryViewParams({
           include_docs: true,
         })
       )
-    ).rows.map(row => row.doc)
+    ).rows.map(row => row.doc!)
     for (let viewDoc of views) {
       response.push({
         name: viewDoc.name,
diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts
index 922bc10343..f0eca759f5 100644
--- a/packages/server/src/automations/triggers.ts
+++ b/packages/server/src/automations/triggers.ts
@@ -20,10 +20,10 @@ const JOB_OPTS = {
 
 async function getAllAutomations() {
   const db = context.getAppDB()
-  let automations = await db.allDocs(
+  let automations = await db.allDocs<Automation>(
     getAutomationParams(null, { include_docs: true })
   )
-  return automations.rows.map(row => row.doc)
+  return automations.rows.map(row => row.doc!)
 }
 
 async function queueRelevantRowAutomations(
@@ -45,19 +45,19 @@ async function queueRelevantRowAutomations(
 
     for (let automation of automations) {
       let automationDef = automation.definition
-      let automationTrigger = automationDef ? automationDef.trigger : {}
+      let automationTrigger = automationDef?.trigger
       // don't queue events which are for dev apps, only way to test automations is
       // running tests on them, in production the test flag will never
       // be checked due to lazy evaluation (first always false)
       if (
         !env.ALLOW_DEV_AUTOMATIONS &&
         isDevAppID(event.appId) &&
-        !(await checkTestFlag(automation._id))
+        !(await checkTestFlag(automation._id!))
       ) {
         continue
       }
       if (
-        automationTrigger.inputs &&
+        automationTrigger?.inputs &&
         automationTrigger.inputs.tableId === event.row.tableId
       ) {
         await automationQueue.add({ automation, event }, JOB_OPTS)
diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts
index 7a7a06551e..8cccf1b96a 100644
--- a/packages/server/src/db/linkedRows/index.ts
+++ b/packages/server/src/db/linkedRows/index.ts
@@ -14,7 +14,14 @@ import partition from "lodash/partition"
 import { getGlobalUsersFromMetadata } from "../../utilities/global"
 import { processFormulas } from "../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
-import { Table, Row, LinkDocumentValue, FieldType } from "@budibase/types"
+import {
+  Table,
+  Row,
+  LinkDocumentValue,
+  FieldType,
+  LinkDocument,
+  ContextUser,
+} from "@budibase/types"
 import sdk from "../../sdk"
 
 export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils"
@@ -73,18 +80,18 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) {
   const db = context.getAppDB()
   const linkedRowIds = links.map(link => link.id)
   const uniqueRowIds = [...new Set(linkedRowIds)]
-  let dbRows = (await db.allDocs(getMultiIDParams(uniqueRowIds))).rows.map(
-    row => row.doc
+  let dbRows = (await db.allDocs<Row>(getMultiIDParams(uniqueRowIds))).rows.map(
+    row => row.doc!
   )
   // convert the unique db rows back to a full list of linked rows
   const linked = linkedRowIds
     .map(id => dbRows.find(row => row && row._id === id))
-    .filter(row => row != null)
+    .filter(row => row != null) as Row[]
   // need to handle users as specific cases
   let [users, other] = partition(linked, linkRow =>
-    linkRow._id.startsWith(USER_METDATA_PREFIX)
+    linkRow._id!.startsWith(USER_METDATA_PREFIX)
   )
-  users = await getGlobalUsersFromMetadata(users)
+  users = await getGlobalUsersFromMetadata(users as ContextUser[])
   return [...other, ...users]
 }
 
@@ -176,7 +183,7 @@ export async function attachFullLinkedDocs(
   // clear any existing links that could be dupe'd
   rows = clearRelationshipFields(table, rows)
   // now get the docs and combine into the rows
-  let linked = []
+  let linked: Row[] = []
   if (linksWithoutFromRow.length > 0) {
     linked = await getFullLinkedDocs(linksWithoutFromRow)
   }
@@ -189,7 +196,7 @@ export async function attachFullLinkedDocs(
       if (opts?.fromRow && opts?.fromRow?._id === link.id) {
         linkedRow = opts.fromRow!
       } else {
-        linkedRow = linked.find(row => row._id === link.id)
+        linkedRow = linked.find(row => row._id === link.id)!
       }
       if (linkedRow) {
         const linkedTableId =
diff --git a/packages/server/src/sdk/app/applications/import.ts b/packages/server/src/sdk/app/applications/import.ts
index 158e4772b2..c3415bdb36 100644
--- a/packages/server/src/sdk/app/applications/import.ts
+++ b/packages/server/src/sdk/app/applications/import.ts
@@ -91,7 +91,7 @@ async function getImportableDocuments(db: Database) {
   // map the responses to the document itself
   let documents: Document[] = []
   for (let response of await Promise.all(docPromises)) {
-    documents = documents.concat(response.rows.map(row => row.doc))
+    documents = documents.concat(response.rows.map(row => row.doc!))
   }
   // remove the _rev, stops it being written
   documents.forEach(doc => {
diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts
index 6e1e6747e1..4813f6dd21 100644
--- a/packages/server/src/sdk/app/applications/sync.ts
+++ b/packages/server/src/sdk/app/applications/sync.ts
@@ -3,7 +3,11 @@ import { db as dbCore, context, logging, roles } from "@budibase/backend-core"
 import { User, ContextUser, UserGroup } from "@budibase/types"
 import { sdk as proSdk } from "@budibase/pro"
 import sdk from "../../"
-import { getGlobalUsers, processUser } from "../../../utilities/global"
+import {
+  getGlobalUsers,
+  getRawGlobalUsers,
+  processUser,
+} from "../../../utilities/global"
 import { generateUserMetadataID, InternalTables } from "../../../db/utils"
 
 type DeletedUser = { _id: string; deleted: boolean }
@@ -77,9 +81,7 @@ async function syncUsersToApp(
 
 export async function syncUsersToAllApps(userIds: string[]) {
   // list of users, if one has been deleted it will be undefined in array
-  const users = (await getGlobalUsers(userIds, {
-    noProcessing: true,
-  })) as User[]
+  const users = (await getRawGlobalUsers(userIds)) as User[]
   const groups = await proSdk.groups.fetch()
   const finalUsers: (User | DeletedUser)[] = []
   for (let userId of userIds) {
diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts
index fb5d04b03e..51cceeab94 100644
--- a/packages/server/src/sdk/app/datasources/datasources.ts
+++ b/packages/server/src/sdk/app/datasources/datasources.ts
@@ -51,12 +51,12 @@ export async function fetch(opts?: {
 
   // Get external datasources
   const datasources = (
-    await db.allDocs(
+    await db.allDocs<Datasource>(
       getDatasourceParams(null, {
         include_docs: true,
       })
     )
-  ).rows.map(row => row.doc)
+  ).rows.map(row => row.doc!)
 
   const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([
     bbInternalDb,
@@ -271,5 +271,5 @@ export async function getExternalDatasources(): Promise<Datasource[]> {
     })
   )
 
-  return externalDatasources.rows.map(r => r.doc)
+  return externalDatasources.rows.map(r => r.doc!)
 }
diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts
index c0d5fe8da8..a7074f95b2 100644
--- a/packages/server/src/sdk/app/tables/getters.ts
+++ b/packages/server/src/sdk/app/tables/getters.ts
@@ -48,7 +48,7 @@ export async function getAllInternalTables(db?: Database): Promise<Table[]> {
   if (!db) {
     db = context.getAppDB()
   }
-  const internalTables = await db.allDocs<Table[]>(
+  const internalTables = await db.allDocs<Table>(
     getTableParams(null, {
       include_docs: true,
     })
@@ -124,7 +124,7 @@ export async function getTables(tableIds: string[]): Promise<Table[]> {
   }
   if (internalTableIds.length) {
     const db = context.getAppDB()
-    const internalTableDocs = await db.allDocs<Table[]>(
+    const internalTableDocs = await db.allDocs<Table>(
       getMultiIDParams(internalTableIds)
     )
     tables = tables.concat(internalTableDocs.rows.map(row => row.doc!))
diff --git a/packages/server/src/sdk/users/utils.ts b/packages/server/src/sdk/users/utils.ts
index 03ddc954e9..9632ac29d8 100644
--- a/packages/server/src/sdk/users/utils.ts
+++ b/packages/server/src/sdk/users/utils.ts
@@ -7,12 +7,17 @@ import {
   InternalTables,
 } from "../../db/utils"
 import isEqual from "lodash/isEqual"
-import { ContextUser, UserMetadata, User, Database } from "@budibase/types"
+import {
+  ContextUser,
+  UserMetadata,
+  Database,
+  ContextUserMetadata,
+} from "@budibase/types"
 
 export function combineMetadataAndUser(
   user: ContextUser,
   metadata: UserMetadata | UserMetadata[]
-) {
+): ContextUserMetadata | null {
   const metadataId = generateUserMetadataID(user._id!)
   const found = Array.isArray(metadata)
     ? metadata.find(doc => doc._id === metadataId)
@@ -51,33 +56,33 @@ export function combineMetadataAndUser(
   return null
 }
 
-export async function rawUserMetadata(db?: Database) {
+export async function rawUserMetadata(db?: Database): Promise<UserMetadata[]> {
   if (!db) {
     db = context.getAppDB()
   }
   return (
-    await db.allDocs(
+    await db.allDocs<UserMetadata>(
       getUserMetadataParams(null, {
         include_docs: true,
       })
     )
-  ).rows.map(row => row.doc)
+  ).rows.map(row => row.doc!)
 }
 
-export async function fetchMetadata() {
+export async function fetchMetadata(): Promise<ContextUserMetadata[]> {
   const global = await getGlobalUsers()
   const metadata = await rawUserMetadata()
-  const users = []
+  const users: ContextUserMetadata[] = []
   for (let user of global) {
     // find the metadata that matches up to the global ID
-    const info = metadata.find(meta => meta._id.includes(user._id))
+    const info = metadata.find(meta => meta._id!.includes(user._id!))
     // remove these props, not for the correct DB
     users.push({
       ...user,
       ...info,
       tableId: InternalTables.USER_METADATA,
       // make sure the ID is always a local ID, not a global one
-      _id: generateUserMetadataID(user._id),
+      _id: generateUserMetadataID(user._id!),
     })
   }
   return users
@@ -90,9 +95,10 @@ export async function syncGlobalUsers() {
     if (!(await db.exists())) {
       continue
     }
-    const resp = await Promise.all([getGlobalUsers(), rawUserMetadata(db)])
-    const users = resp[0] as User[]
-    const metadata = resp[1] as UserMetadata[]
+    const [users, metadata] = await Promise.all([
+      getGlobalUsers(),
+      rawUserMetadata(db),
+    ])
     const toWrite = []
     for (let user of users) {
       const combined = combineMetadataAndUser(user, metadata)
diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts
index 5aa201990c..cdc2d84513 100644
--- a/packages/server/src/utilities/global.ts
+++ b/packages/server/src/utilities/global.ts
@@ -71,69 +71,67 @@ export async function processUser(
   return user
 }
 
-export async function getCachedSelf(ctx: UserCtx, appId: string) {
+export async function getCachedSelf(
+  ctx: UserCtx,
+  appId: string
+): Promise<ContextUser> {
   // this has to be tenant aware, can't depend on the context to find it out
   // running some middlewares before the tenancy causes context to break
   const user = await cache.user.getUser(ctx.user?._id!)
   return processUser(user, { appId })
 }
 
-export async function getRawGlobalUser(userId: string) {
+export async function getRawGlobalUser(userId: string): Promise<User> {
   const db = tenancy.getGlobalDB()
   return db.get<User>(getGlobalIDFromUserMetadataID(userId))
 }
 
-export async function getGlobalUser(userId: string) {
+export async function getGlobalUser(userId: string): Promise<ContextUser> {
   const appId = context.getAppId()
   let user = await getRawGlobalUser(userId)
   return processUser(user, { appId })
 }
 
-export async function getGlobalUsers(
-  userIds?: string[],
-  opts?: { noProcessing?: boolean }
-) {
-  const appId = context.getAppId()
+export async function getRawGlobalUsers(userIds?: string[]): Promise<User[]> {
   const db = tenancy.getGlobalDB()
-  let globalUsers
+  let globalUsers: User[]
   if (userIds) {
-    globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map(
-      row => row.doc
+    globalUsers = (await db.allDocs<User>(getMultiIDParams(userIds))).rows.map(
+      row => row.doc!
     )
   } else {
     globalUsers = (
-      await db.allDocs(
+      await db.allDocs<User>(
         dbCore.getGlobalUserParams(null, {
           include_docs: true,
         })
       )
-    ).rows.map(row => row.doc)
+    ).rows.map(row => row.doc!)
   }
-  globalUsers = globalUsers
+  return globalUsers
     .filter(user => user != null)
     .map(user => {
       delete user.password
       delete user.forceResetPassword
       return user
     })
+}
 
-  if (opts?.noProcessing || !appId) {
-    return globalUsers
-  } else {
-    // pass in the groups, meaning we don't actually need to retrieve them for
-    // each user individually
-    const allGroups = await groups.fetch()
-    return Promise.all(
-      globalUsers.map(user => processUser(user, { groups: allGroups }))
-    )
-  }
+export async function getGlobalUsers(
+  userIds?: string[]
+): Promise<ContextUser[]> {
+  const users = await getRawGlobalUsers(userIds)
+  const allGroups = await groups.fetch()
+  return Promise.all(
+    users.map(user => processUser(user, { groups: allGroups }))
+  )
 }
 
 export async function getGlobalUsersFromMetadata(users: ContextUser[]) {
   const globalUsers = await getGlobalUsers(users.map(user => user._id!))
   return users.map(user => {
     const globalUser = globalUsers.find(
-      globalUser => globalUser && user._id?.includes(globalUser._id)
+      globalUser => globalUser && user._id?.includes(globalUser._id!)
     )
     return {
       ...globalUser,
diff --git a/packages/server/src/utilities/routing/index.ts b/packages/server/src/utilities/routing/index.ts
index de966a946b..82d45743ce 100644
--- a/packages/server/src/utilities/routing/index.ts
+++ b/packages/server/src/utilities/routing/index.ts
@@ -1,9 +1,9 @@
 import { createRoutingView } from "../../db/views/staticViews"
 import { ViewName, getQueryIndex, UNICODE_MAX } from "../../db/utils"
 import { context } from "@budibase/backend-core"
-import { ScreenRouting } from "@budibase/types"
+import { ScreenRouting, Document } from "@budibase/types"
 
-type ScreenRoutesView = {
+interface ScreenRoutesView extends Document {
   id: string
   routing: ScreenRouting
 }
diff --git a/packages/types/src/documents/app/user.ts b/packages/types/src/documents/app/user.ts
index 4defd4a414..207997245e 100644
--- a/packages/types/src/documents/app/user.ts
+++ b/packages/types/src/documents/app/user.ts
@@ -1,6 +1,6 @@
-import { Document } from "../document"
+import { User } from "../global"
+import { Row } from "./row"
+import { ContextUser } from "../../sdk"
 
-export interface UserMetadata extends Document {
-  roleId: string
-  email?: string
-}
+export type UserMetadata = User & Row
+export type ContextUserMetadata = ContextUser & Row
diff --git a/packages/types/src/documents/pouch.ts b/packages/types/src/documents/pouch.ts
index d484f4700d..11efc502be 100644
--- a/packages/types/src/documents/pouch.ts
+++ b/packages/types/src/documents/pouch.ts
@@ -1,17 +1,19 @@
+import { Document } from "../"
+
 export interface RowValue {
   rev: string
   deleted: boolean
 }
 
-export interface RowResponse<T> {
+export interface RowResponse<T extends Document> {
   id: string
   key: string
   error: string
   value: T | RowValue
-  doc?: T | any
+  doc?: T
 }
 
-export interface AllDocsResponse<T> {
+export interface AllDocsResponse<T extends Document> {
   offset: number
   total_rows: number
   rows: RowResponse<T>[]
diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts
index 05f72f5524..12c53a9561 100644
--- a/packages/types/src/sdk/db.ts
+++ b/packages/types/src/sdk/db.ts
@@ -101,8 +101,10 @@ export interface Database {
     opts?: DatabasePutOpts
   ): Promise<Nano.DocumentInsertResponse>
   bulkDocs(documents: AnyDocument[]): Promise<Nano.DocumentBulkResponse[]>
-  allDocs<T>(params: DatabaseQueryOpts): Promise<AllDocsResponse<T>>
-  query<T>(
+  allDocs<T extends Document>(
+    params: DatabaseQueryOpts
+  ): Promise<AllDocsResponse<T>>
+  query<T extends Document>(
     viewName: string,
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>>
diff --git a/packages/worker/src/constants/templates/index.ts b/packages/worker/src/constants/templates/index.ts
index 1feac62040..6dd3f556a6 100644
--- a/packages/worker/src/constants/templates/index.ts
+++ b/packages/worker/src/constants/templates/index.ts
@@ -56,12 +56,12 @@ export async function getTemplates({
   id,
 }: { ownerId?: string; type?: string; id?: string } = {}) {
   const db = tenancy.getGlobalDB()
-  const response = await db.allDocs(
+  const response = await db.allDocs<Template>(
     dbCore.getTemplateParams(ownerId || GLOBAL_OWNER, id, {
       include_docs: true,
     })
   )
-  let templates = response.rows.map(row => row.doc)
+  let templates = response.rows.map(row => row.doc!)
   // should only be one template with ID
   if (id) {
     return templates[0]
@@ -73,6 +73,6 @@ export async function getTemplates({
 }
 
 export async function getTemplateByPurpose(type: string, purpose: string) {
-  const templates = await getTemplates({ type })
+  const templates = (await getTemplates({ type })) as Template[]
   return templates.find((template: Template) => template.purpose === purpose)
 }
diff --git a/packages/worker/src/utilities/email.ts b/packages/worker/src/utilities/email.ts
index 38a367c4d5..c5b1d9d8ab 100644
--- a/packages/worker/src/utilities/email.ts
+++ b/packages/worker/src/utilities/email.ts
@@ -99,8 +99,6 @@ async function buildEmail(
   if (!base || !body || !core) {
     throw "Unable to build email, missing base components"
   }
-  base = base.contents
-  body = body.contents
 
   let name = user ? user.name : undefined
   if (user && !name && user.firstName) {
@@ -114,15 +112,10 @@ async function buildEmail(
     user: user || {},
   }
 
-  // Prepend the core template
-  const fullBody = core + body
-
-  body = await processString(fullBody, context)
-
   // this should now be the core email HTML
-  return processString(base, {
+  return processString(base.contents, {
     ...context,
-    body,
+    body: await processString(core + body?.contents, context),
   })
 }
 

From 68a3e030c2df46f81975ea81af9c3919bb6de0f6 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 7 Nov 2023 18:34:51 +0000
Subject: [PATCH 051/155] Adding test case to check that relationships can be
 used in external table formulas.

---
 .../server/src/api/routes/tests/row.spec.ts   | 47 +++++++++++++++++++
 .../server/src/tests/utilities/api/row.ts     |  3 +-
 2 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 48f7ab4f09..97b06f33c8 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -10,6 +10,7 @@ import {
   FieldSchema,
   FieldType,
   FieldTypeSubtypes,
+  FormulaTypes,
   INTERNAL_TABLE_SOURCE_ID,
   MonthlyQuotaName,
   PermissionLevel,
@@ -1998,4 +1999,50 @@ describe.each([
       })
     })
   })
+
+  describe("Formula fields", () => {
+    let relationshipTable: Table, tableId: string, relatedRow: Row
+
+    beforeAll(async () => {
+      const otherTableId = config.table!._id!
+      const cfg = generateTableConfig()
+      relationshipTable = await config.createLinkedTable(
+        RelationshipType.ONE_TO_MANY,
+        ["links"],
+        {
+          ...cfg,
+          schema: {
+            ...cfg.schema,
+            formula: {
+              name: "formula",
+              type: FieldType.FORMULA,
+              formula: "{{ links.0.name }}",
+              formulaType: FormulaTypes.DYNAMIC,
+            },
+          },
+        }
+      )
+
+      tableId = relationshipTable._id!
+
+      relatedRow = await config.api.row.save(otherTableId, {
+        name: generator.word(),
+        description: generator.paragraph(),
+      })
+      await config.api.row.save(tableId, {
+        name: generator.word(),
+        description: generator.paragraph(),
+        tableId,
+        links: [relatedRow._id],
+      })
+    })
+
+    it("should be able to search for rows containing formulas", async () => {
+      const { rows } = await config.api.row.search(tableId)
+      expect(rows.length).toBe(1)
+      expect(rows[0].links.length).toBe(1)
+      const row = rows[0]
+      expect(row.formula).toBe(relatedRow.name)
+    })
+  })
 })
diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts
index 20b1d6f9ee..3d4cf6c82c 100644
--- a/packages/server/src/tests/utilities/api/row.ts
+++ b/packages/server/src/tests/utilities/api/row.ts
@@ -6,6 +6,7 @@ import {
   ExportRowsRequest,
   BulkImportRequest,
   BulkImportResponse,
+  SearchRowResponse,
 } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
@@ -154,7 +155,7 @@ export class RowAPI extends TestAPI {
   search = async (
     sourceId: string,
     { expectStatus } = { expectStatus: 200 }
-  ): Promise<Row[]> => {
+  ): Promise<SearchRowResponse> => {
     const request = this.request
       .post(`/api/${sourceId}/search`)
       .set(this.config.defaultHeaders())

From 57eb23dc5015daeb41158ee04c3e966a79ccae56 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 8 Nov 2023 09:24:51 +0000
Subject: [PATCH 052/155] Remove unnecessary type cast.

---
 packages/server/src/sdk/app/applications/sync.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts
index 4813f6dd21..e73b3396d9 100644
--- a/packages/server/src/sdk/app/applications/sync.ts
+++ b/packages/server/src/sdk/app/applications/sync.ts
@@ -81,7 +81,7 @@ async function syncUsersToApp(
 
 export async function syncUsersToAllApps(userIds: string[]) {
   // list of users, if one has been deleted it will be undefined in array
-  const users = (await getRawGlobalUsers(userIds)) as User[]
+  const users = await getRawGlobalUsers(userIds)
   const groups = await proSdk.groups.fetch()
   const finalUsers: (User | DeletedUser)[] = []
   for (let userId of userIds) {

From 0633a3de65922beeea097d933706406bc125e642 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 8 Nov 2023 10:03:22 +0000
Subject: [PATCH 053/155] WIP

---
 packages/server/src/api/controllers/auth.ts   |  2 +-
 packages/server/src/api/controllers/user.ts   |  6 +--
 .../tests/{user.spec.js => user.spec.ts}      | 27 ++++++-----
 .../server/src/tests/utilities/api/index.ts   |  3 ++
 .../server/src/tests/utilities/api/user.ts    | 48 +++++++++++++++++++
 packages/server/src/utilities/users.ts        | 12 +++--
 packages/types/src/api/web/app/index.ts       |  1 +
 packages/types/src/api/web/app/user.ts        |  3 ++
 8 files changed, 81 insertions(+), 21 deletions(-)
 rename packages/server/src/api/routes/tests/{user.spec.js => user.spec.ts} (89%)
 create mode 100644 packages/server/src/tests/utilities/api/user.ts
 create mode 100644 packages/types/src/api/web/app/user.ts

diff --git a/packages/server/src/api/controllers/auth.ts b/packages/server/src/api/controllers/auth.ts
index eabfe10bab..9b1b78ed9e 100644
--- a/packages/server/src/api/controllers/auth.ts
+++ b/packages/server/src/api/controllers/auth.ts
@@ -26,7 +26,7 @@ export async function fetchSelf(ctx: UserCtx) {
   }
 
   const appId = context.getAppId()
-  let user: ContextUser = await getFullUser(ctx, userId)
+  let user: ContextUser = await getFullUser(userId)
   // this shouldn't be returned by the app self
   delete user.roles
   // forward the csrf token from the session
diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts
index b6c3e7c6bd..bfe206e98a 100644
--- a/packages/server/src/api/controllers/user.ts
+++ b/packages/server/src/api/controllers/user.ts
@@ -1,10 +1,10 @@
 import { generateUserFlagID, InternalTables } from "../../db/utils"
 import { getFullUser } from "../../utilities/users"
 import { context } from "@budibase/backend-core"
-import { Ctx, UserCtx } from "@budibase/types"
+import { Ctx, FetchUserMetadataResponse, UserCtx } from "@budibase/types"
 import sdk from "../../sdk"
 
-export async function fetchMetadata(ctx: Ctx) {
+export async function fetchMetadata(ctx: Ctx<null, FetchUserMetadataResponse>) {
   ctx.body = await sdk.users.fetchMetadata()
 }
 
@@ -44,7 +44,7 @@ export async function destroyMetadata(ctx: UserCtx) {
 }
 
 export async function findMetadata(ctx: UserCtx) {
-  ctx.body = await getFullUser(ctx, ctx.params.id)
+  ctx.body = await getFullUser(ctx.params.id)
 }
 
 export async function setFlag(ctx: UserCtx) {
diff --git a/packages/server/src/api/routes/tests/user.spec.js b/packages/server/src/api/routes/tests/user.spec.ts
similarity index 89%
rename from packages/server/src/api/routes/tests/user.spec.js
rename to packages/server/src/api/routes/tests/user.spec.ts
index e8ffd8df2b..caae54bc68 100644
--- a/packages/server/src/api/routes/tests/user.spec.js
+++ b/packages/server/src/api/routes/tests/user.spec.ts
@@ -1,7 +1,6 @@
-const { roles, utils } = require("@budibase/backend-core")
-const { checkPermissionsEndpoint } = require("./utilities/TestFunctions")
-const setup = require("./utilities")
-const { BUILTIN_ROLE_IDS } = roles
+import { roles, utils } from "@budibase/backend-core"
+import { checkPermissionsEndpoint } from "./utilities/TestFunctions"
+import * as setup from "./utilities"
 
 jest.setTimeout(30000)
 
@@ -47,8 +46,8 @@ describe("/users", () => {
         request,
         method: "GET",
         url: `/api/users/metadata`,
-        passRole: BUILTIN_ROLE_IDS.ADMIN,
-        failRole: BUILTIN_ROLE_IDS.PUBLIC,
+        passRole: roles.BUILTIN_ROLE_IDS.ADMIN,
+        failRole: roles.BUILTIN_ROLE_IDS.PUBLIC,
       })
     })
   })
@@ -56,7 +55,7 @@ describe("/users", () => {
   describe("update", () => {
     it("should be able to update the user", async () => {
       const user = await config.createUser({ id: `us_update${utils.newid()}` })
-      user.roleId = BUILTIN_ROLE_IDS.BASIC
+      user.roleId = roles.BUILTIN_ROLE_IDS.BASIC
       delete user._rev
       const res = await request
         .put(`/api/users/metadata`)
@@ -74,14 +73,18 @@ describe("/users", () => {
       const res1 = await request
         .put(`/api/users/metadata`)
         .set(config.defaultHeaders())
-        .send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC })
+        .send({ ...user, roleId: roles.BUILTIN_ROLE_IDS.BASIC })
         .expect(200)
         .expect("Content-Type", /json/)
 
       const res = await request
         .put(`/api/users/metadata`)
         .set(config.defaultHeaders())
-        .send({ ...user, _rev: res1.body.rev, roleId: BUILTIN_ROLE_IDS.POWER })
+        .send({
+          ...user,
+          _rev: res1.body.rev,
+          roleId: roles.BUILTIN_ROLE_IDS.POWER,
+        })
         .expect(200)
         .expect("Content-Type", /json/)
 
@@ -95,14 +98,14 @@ describe("/users", () => {
       await request
         .put(`/api/users/metadata`)
         .set(config.defaultHeaders())
-        .send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC })
+        .send({ ...user, roleId: roles.BUILTIN_ROLE_IDS.BASIC })
         .expect(200)
         .expect("Content-Type", /json/)
 
       await request
         .put(`/api/users/metadata`)
         .set(config.defaultHeaders())
-        .send({ ...user, roleId: BUILTIN_ROLE_IDS.POWER })
+        .send({ ...user, roleId: roles.BUILTIN_ROLE_IDS.POWER })
         .expect(409)
         .expect("Content-Type", /json/)
     })
@@ -129,7 +132,7 @@ describe("/users", () => {
         .expect(200)
         .expect("Content-Type", /json/)
       expect(res.body._id).toEqual(user._id)
-      expect(res.body.roleId).toEqual(BUILTIN_ROLE_IDS.ADMIN)
+      expect(res.body.roleId).toEqual(roles.BUILTIN_ROLE_IDS.ADMIN)
       expect(res.body.tableId).toBeDefined()
     })
   })
diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts
index c553e7b8f4..20b96f7a99 100644
--- a/packages/server/src/tests/utilities/api/index.ts
+++ b/packages/server/src/tests/utilities/api/index.ts
@@ -9,6 +9,7 @@ import { ScreenAPI } from "./screen"
 import { ApplicationAPI } from "./application"
 import { BackupAPI } from "./backup"
 import { AttachmentAPI } from "./attachment"
+import { UserAPI } from "./user"
 
 export default class API {
   table: TableAPI
@@ -21,6 +22,7 @@ export default class API {
   application: ApplicationAPI
   backup: BackupAPI
   attachment: AttachmentAPI
+  user: UserAPI
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
@@ -33,5 +35,6 @@ export default class API {
     this.application = new ApplicationAPI(config)
     this.backup = new BackupAPI(config)
     this.attachment = new AttachmentAPI(config)
+    this.user = new UserAPI(config)
   }
 }
diff --git a/packages/server/src/tests/utilities/api/user.ts b/packages/server/src/tests/utilities/api/user.ts
new file mode 100644
index 0000000000..2066315778
--- /dev/null
+++ b/packages/server/src/tests/utilities/api/user.ts
@@ -0,0 +1,48 @@
+import { FetchUserMetadataResponse } from "@budibase/types"
+import TestConfiguration from "../TestConfiguration"
+import { TestAPI } from "./base"
+
+export class UserAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  fetch = async (
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<FetchUserMetadataResponse> => {
+    const res = await this.request
+      .get(`/api/users/metadata`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+
+    if (res.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          res.status
+        } with body ${JSON.stringify(res.body)}`
+      )
+    }
+
+    return res.body
+  }
+
+  get = async (
+    id: string,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<GetUserMetadataResponse> => {
+    const res = await this.request
+      .get(`/api/users/metadata/${id}`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+
+    if (res.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          res.status
+        } with body ${JSON.stringify(res.body)}`
+      )
+    }
+
+    return res.body
+  }
+}
diff --git a/packages/server/src/utilities/users.ts b/packages/server/src/utilities/users.ts
index bbc1370355..73b2f48b15 100644
--- a/packages/server/src/utilities/users.ts
+++ b/packages/server/src/utilities/users.ts
@@ -1,11 +1,13 @@
 import { InternalTables } from "../db/utils"
 import { getGlobalUser } from "./global"
 import { context, roles } from "@budibase/backend-core"
-import { UserCtx } from "@budibase/types"
+import { ContextUserMetadata, UserCtx, UserMetadata } from "@budibase/types"
 
-export async function getFullUser(ctx: UserCtx, userId: string) {
+export async function getFullUser(
+  userId: string
+): Promise<ContextUserMetadata> {
   const global = await getGlobalUser(userId)
-  let metadata: any = {}
+  let metadata: UserMetadata | undefined = undefined
 
   // always prefer the user metadata _id and _rev
   delete global._id
@@ -14,11 +16,11 @@ export async function getFullUser(ctx: UserCtx, userId: string) {
   try {
     // this will throw an error if the db doesn't exist, or there is no appId
     const db = context.getAppDB()
-    metadata = await db.get(userId)
+    metadata = await db.get<UserMetadata>(userId)
+    delete metadata.csrfToken
   } catch (err) {
     // it is fine if there is no user metadata yet
   }
-  delete metadata.csrfToken
   return {
     ...metadata,
     ...global,
diff --git a/packages/types/src/api/web/app/index.ts b/packages/types/src/api/web/app/index.ts
index f5b876009b..cb1cea2b08 100644
--- a/packages/types/src/api/web/app/index.ts
+++ b/packages/types/src/api/web/app/index.ts
@@ -6,3 +6,4 @@ export * from "./rows"
 export * from "./table"
 export * from "./permission"
 export * from "./attachment"
+export * from "./user"
diff --git a/packages/types/src/api/web/app/user.ts b/packages/types/src/api/web/app/user.ts
new file mode 100644
index 0000000000..2e0eeee7f8
--- /dev/null
+++ b/packages/types/src/api/web/app/user.ts
@@ -0,0 +1,3 @@
+import { ContextUserMetadata } from "src/documents"
+
+export type FetchUserMetadataResponse = ContextUserMetadata[]

From 19b4ed070cb93359113b390edc8b384c0a8c76bb Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 8 Nov 2023 10:25:28 +0000
Subject: [PATCH 054/155] Reduce the likelihood of collisions in table names by
 using a UUID instead of a random word.

---
 packages/server/src/api/routes/tests/row.spec.ts | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 48f7ab4f09..c426d59f4e 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -32,6 +32,7 @@ import {
   structures,
 } from "@budibase/backend-core/tests"
 import _ from "lodash"
+import * as uuid from "uuid"
 
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
@@ -68,7 +69,7 @@ describe.each([
 
   const generateTableConfig: () => SaveTableRequest = () => {
     return {
-      name: generator.word(),
+      name: uuid.v4(),
       type: "table",
       primary: ["id"],
       primaryDisplay: "name",
@@ -481,7 +482,7 @@ describe.each([
       })
 
       const createViewResponse = await config.createView({
-        name: generator.word(),
+        name: uuid.v4(),
         schema: {
           Country: {
             visible: true,
@@ -816,7 +817,8 @@ describe.each([
             RelationshipType.ONE_TO_MANY,
             ["link"],
             {
-              name: generator.word(),
+              // Making sure that the combined table name + column name is within postgres limits
+              name: uuid.v4().replace(/-/g, "").substring(0, 16),
               type: "table",
               primary: ["id"],
               primaryDisplay: "id",
@@ -949,7 +951,7 @@ describe.each([
   describe("view 2.0", () => {
     async function userTable(): Promise<Table> {
       return {
-        name: `users_${generator.word()}`,
+        name: `users_${uuid.v4()}`,
         sourceId: INTERNAL_TABLE_SOURCE_ID,
         sourceType: TableSourceType.INTERNAL,
         type: "table",
@@ -1133,7 +1135,7 @@ describe.each([
       const viewSchema = { age: { visible: true }, name: { visible: true } }
       async function userTable(): Promise<Table> {
         return {
-          name: `users_${generator.word()}`,
+          name: `users_${uuid.v4()}`,
           sourceId: INTERNAL_TABLE_SOURCE_ID,
           sourceType: TableSourceType.INTERNAL,
           type: "table",
@@ -1630,7 +1632,7 @@ describe.each([
       }),
       (tableId: string) =>
         config.api.row.save(tableId, {
-          name: generator.word(),
+          name: uuid.v4(),
           description: generator.paragraph(),
           tableId,
         }),

From d86e94376586870a8b18dc0d08dca60ee86ee0fa Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Wed, 8 Nov 2023 10:27:15 +0000
Subject: [PATCH 055/155] Automation history issue fix

---
 .../FlowChart/FlowItemHeader.svelte           | 68 +++++++++++--------
 .../AutomationBuilder/TestDisplay.svelte      |  1 +
 .../_components/HistoryDetailsPanel.svelte    |  2 +-
 3 files changed, 43 insertions(+), 28 deletions(-)

diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte
index 0c7ac3d27d..3c9e1a13b1 100644
--- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte
+++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte
@@ -13,13 +13,13 @@
   export let idx
   export let addLooping
   export let deleteStep
-
+  export let enableNaming = true
   let validRegex = /^[A-Za-z0-9_\s]+$/
   let typing = false
 
   const dispatch = createEventDispatcher()
 
-  $: stepNames = $selectedAutomation.definition.stepNames
+  $: stepNames = $selectedAutomation?.definition.stepNames
   $: automationName = stepNames?.[block.id] || block?.name || ""
   $: automationNameError = getAutomationNameError(automationName)
   $: status = updateStatus(testResult, isTrigger)
@@ -32,7 +32,7 @@
       )?.[0]
     }
   }
-  $: loopBlock = $selectedAutomation.definition.steps.find(
+  $: loopBlock = $selectedAutomation?.definition.steps.find(
     x => x.blockToLoop === block?.id
   )
 
@@ -126,24 +126,33 @@
             <Body size="XS"><b>Step {idx}</b></Body>
           </div>
         {/if}
-        <input
-          placeholder="Enter some text"
-          name="name"
-          autocomplete="off"
-          value={automationName}
-          on:input={e => {
-            automationName = e.target.value.trim()
-          }}
-          on:click={startTyping}
-          on:blur={async () => {
-            typing = false
-            if (automationNameError) {
-              automationName = stepNames[block.id] || block?.name
-            } else {
-              await saveName()
-            }
-          }}
-        />
+
+        {#if enableNaming}
+          <input
+            class="input-text"
+            disabled={!enableNaming}
+            placeholder="Enter some text"
+            name="name"
+            autocomplete="off"
+            value={automationName}
+            on:input={e => {
+              automationName = e.target.value.trim()
+            }}
+            on:click={startTyping}
+            on:blur={async () => {
+              typing = false
+              if (automationNameError) {
+                automationName = stepNames[block.id] || block?.name
+              } else {
+                await saveName()
+              }
+            }}
+          />
+        {:else}
+          <div class="input-text">
+            {automationName}
+          </div>
+        {/if}
       </div>
     </div>
     <div class="blockTitle">
@@ -178,9 +187,11 @@
               <Icon on:click={addLooping} hoverable name="RotateCW" />
             </AbsTooltip>
           {/if}
-          <AbsTooltip type="negative" text="Delete step">
-            <Icon on:click={deleteStep} hoverable name="DeleteOutline" />
-          </AbsTooltip>
+          {#if !isHeaderTrigger}
+            <AbsTooltip type="negative" text="Delete step">
+              <Icon on:click={deleteStep} hoverable name="DeleteOutline" />
+            </AbsTooltip>
+          {/if}
         {/if}
         {#if !showTestStatus}
           <Icon
@@ -244,18 +255,21 @@
     display: none;
   }
   input {
-    font-family: var(--font-sans);
     color: var(--ink);
     background-color: transparent;
     border: 1px solid transparent;
-    font-size: var(--spectrum-alias-font-size-default);
     width: 230px;
     box-sizing: border-box;
     overflow: hidden;
-    text-overflow: ellipsis;
     white-space: nowrap;
   }
 
+  .input-text {
+    font-size: var(--spectrum-alias-font-size-default);
+    font-family: var(--font-sans);
+    text-overflow: ellipsis;
+  }
+
   input:focus {
     outline: none;
   }
diff --git a/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte b/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte
index aa441cec40..89406f15d8 100644
--- a/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte
+++ b/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte
@@ -48,6 +48,7 @@
     <div class="block" style={width ? `width: ${width}` : ""}>
       {#if block.stepId !== ActionStepID.LOOP}
         <FlowItemHeader
+          enableNaming={false}
           open={!!openBlocks[block.id]}
           on:toggle={() => (openBlocks[block.id] = !openBlocks[block.id])}
           isTrigger={idx === 0}
diff --git a/packages/builder/src/pages/builder/app/[application]/settings/automation-history/_components/HistoryDetailsPanel.svelte b/packages/builder/src/pages/builder/app/[application]/settings/automation-history/_components/HistoryDetailsPanel.svelte
index 5b9c925130..cde76fa1c0 100644
--- a/packages/builder/src/pages/builder/app/[application]/settings/automation-history/_components/HistoryDetailsPanel.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/settings/automation-history/_components/HistoryDetailsPanel.svelte
@@ -56,7 +56,7 @@
     {/if}
     {#key history}
       <div class="history">
-        <TestDisplay testResults={history} width="100%" />
+        <TestDisplay testResults={history} width="320px" />
       </div>
     {/key}
   </Layout>

From 555f8fc22ccdd77da0c44fbab58ad8725153d770 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 8 Nov 2023 11:29:28 +0000
Subject: [PATCH 056/155] Bump version to 2.13.4

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 25ec556c56..765395227c 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.13.3",
+  "version": "2.13.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From d146df5f7367959bccc6d3ce3b02c9098e8e4e4e Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 8 Nov 2023 11:53:00 +0000
Subject: [PATCH 057/155] Convert user.spec.js to user.spec.ts

---
 .../src/api/controllers/row/internal.ts       |   3 +-
 packages/server/src/api/controllers/user.ts   |  48 ++++--
 .../server/src/api/routes/tests/user.spec.ts  | 149 +++++-------------
 .../src/tests/utilities/TestConfiguration.ts  |   2 +-
 .../server/src/tests/utilities/api/user.ts    | 115 +++++++++++++-
 packages/types/src/api/web/app/user.ts        |   8 +-
 packages/types/src/documents/account/flag.ts  |   5 +
 packages/types/src/documents/account/index.ts |   1 +
 8 files changed, 201 insertions(+), 130 deletions(-)
 create mode 100644 packages/types/src/documents/account/flag.ts

diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts
index d53345a239..fe7d94547a 100644
--- a/packages/server/src/api/controllers/row/internal.ts
+++ b/packages/server/src/api/controllers/row/internal.ts
@@ -2,7 +2,6 @@ import * as linkRows from "../../../db/linkedRows"
 import {
   generateRowID,
   getMultiIDParams,
-  getTableIDFromRowID,
   InternalTables,
 } from "../../../db/utils"
 import * as userController from "../user"
@@ -89,7 +88,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   if (isUserTable) {
     // the row has been updated, need to put it into the ctx
     ctx.request.body = row as any
-    await userController.updateMetadata(ctx)
+    await userController.updateMetadata(ctx as any)
     return { row: ctx.body as Row, table }
   }
 
diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts
index bfe206e98a..108e29fd3d 100644
--- a/packages/server/src/api/controllers/user.ts
+++ b/packages/server/src/api/controllers/user.ts
@@ -1,14 +1,26 @@
 import { generateUserFlagID, InternalTables } from "../../db/utils"
 import { getFullUser } from "../../utilities/users"
 import { context } from "@budibase/backend-core"
-import { Ctx, FetchUserMetadataResponse, UserCtx } from "@budibase/types"
+import {
+  ContextUserMetadata,
+  Ctx,
+  FetchUserMetadataResponse,
+  FindUserMetadataResponse,
+  Flags,
+  SetFlagRequest,
+  UserCtx,
+  UserMetadata,
+} from "@budibase/types"
 import sdk from "../../sdk"
+import { DocumentInsertResponse } from "@budibase/nano"
 
-export async function fetchMetadata(ctx: Ctx<null, FetchUserMetadataResponse>) {
+export async function fetchMetadata(ctx: Ctx<void, FetchUserMetadataResponse>) {
   ctx.body = await sdk.users.fetchMetadata()
 }
 
-export async function updateSelfMetadata(ctx: UserCtx) {
+export async function updateSelfMetadata(
+  ctx: UserCtx<UserMetadata, DocumentInsertResponse>
+) {
   // overwrite the ID with current users
   ctx.request.body._id = ctx.user?._id
   // make sure no stale rev
@@ -18,19 +30,21 @@ export async function updateSelfMetadata(ctx: UserCtx) {
   await updateMetadata(ctx)
 }
 
-export async function updateMetadata(ctx: UserCtx) {
+export async function updateMetadata(
+  ctx: UserCtx<UserMetadata, DocumentInsertResponse>
+) {
   const db = context.getAppDB()
   const user = ctx.request.body
-  // this isn't applicable to the user
-  delete user.roles
-  const metadata = {
+  const metadata: ContextUserMetadata = {
     tableId: InternalTables.USER_METADATA,
     ...user,
   }
+  // this isn't applicable to the user
+  delete metadata.roles
   ctx.body = await db.put(metadata)
 }
 
-export async function destroyMetadata(ctx: UserCtx) {
+export async function destroyMetadata(ctx: UserCtx<void, { message: string }>) {
   const db = context.getAppDB()
   try {
     const dbUser = await sdk.users.get(ctx.params.id)
@@ -43,11 +57,15 @@ export async function destroyMetadata(ctx: UserCtx) {
   }
 }
 
-export async function findMetadata(ctx: UserCtx) {
+export async function findMetadata(
+  ctx: UserCtx<void, FindUserMetadataResponse>
+) {
   ctx.body = await getFullUser(ctx.params.id)
 }
 
-export async function setFlag(ctx: UserCtx) {
+export async function setFlag(
+  ctx: UserCtx<SetFlagRequest, { message: string }>
+) {
   const userId = ctx.user?._id
   const { flag, value } = ctx.request.body
   if (!flag) {
@@ -55,9 +73,9 @@ export async function setFlag(ctx: UserCtx) {
   }
   const flagDocId = generateUserFlagID(userId!)
   const db = context.getAppDB()
-  let doc
+  let doc: Flags
   try {
-    doc = await db.get<any>(flagDocId)
+    doc = await db.get<Flags>(flagDocId)
   } catch (err) {
     doc = { _id: flagDocId }
   }
@@ -66,13 +84,13 @@ export async function setFlag(ctx: UserCtx) {
   ctx.body = { message: "Flag set successfully" }
 }
 
-export async function getFlags(ctx: UserCtx) {
+export async function getFlags(ctx: UserCtx<void, Flags>) {
   const userId = ctx.user?._id
   const docId = generateUserFlagID(userId!)
   const db = context.getAppDB()
-  let doc
+  let doc: Flags
   try {
-    doc = await db.get(docId)
+    doc = await db.get<Flags>(docId)
   } catch (err) {
     doc = { _id: docId }
   }
diff --git a/packages/server/src/api/routes/tests/user.spec.ts b/packages/server/src/api/routes/tests/user.spec.ts
index caae54bc68..e6349099d7 100644
--- a/packages/server/src/api/routes/tests/user.spec.ts
+++ b/packages/server/src/api/routes/tests/user.spec.ts
@@ -1,6 +1,7 @@
 import { roles, utils } from "@budibase/backend-core"
 import { checkPermissionsEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
+import { UserMetadata } from "@budibase/types"
 
 jest.setTimeout(30000)
 
@@ -28,15 +29,13 @@ describe("/users", () => {
     it("returns a list of users from an instance db", async () => {
       await config.createUser({ id: "uuidx" })
       await config.createUser({ id: "uuidy" })
-      const res = await request
-        .get(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
 
-      expect(res.body.length).toBe(3)
-      expect(res.body.find(u => u._id === `ro_ta_users_us_uuidx`)).toBeDefined()
-      expect(res.body.find(u => u._id === `ro_ta_users_us_uuidy`)).toBeDefined()
+      const res = await config.api.user.fetch()
+      expect(res.length).toBe(3)
+
+      const ids = res.map(u => u._id)
+      expect(ids).toContain(`ro_ta_users_us_uuidx`)
+      expect(ids).toContain(`ro_ta_users_us_uuidy`)
     })
 
     it("should apply authorization to endpoint", async () => {
@@ -54,86 +53,61 @@ describe("/users", () => {
 
   describe("update", () => {
     it("should be able to update the user", async () => {
-      const user = await config.createUser({ id: `us_update${utils.newid()}` })
+      const user: UserMetadata = await config.createUser({
+        id: `us_update${utils.newid()}`,
+      })
       user.roleId = roles.BUILTIN_ROLE_IDS.BASIC
       delete user._rev
-      const res = await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send(user)
-        .expect(200)
-        .expect("Content-Type", /json/)
-      expect(res.body.ok).toEqual(true)
+      const res = await config.api.user.update(user)
+      expect(res.ok).toEqual(true)
     })
 
     it("should be able to update the user multiple times", async () => {
       const user = await config.createUser()
       delete user._rev
 
-      const res1 = await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send({ ...user, roleId: roles.BUILTIN_ROLE_IDS.BASIC })
-        .expect(200)
-        .expect("Content-Type", /json/)
-
-      const res = await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send({
-          ...user,
-          _rev: res1.body.rev,
-          roleId: roles.BUILTIN_ROLE_IDS.POWER,
-        })
-        .expect(200)
-        .expect("Content-Type", /json/)
-
-      expect(res.body.ok).toEqual(true)
+      const res1 = await config.api.user.update({
+        ...user,
+        roleId: roles.BUILTIN_ROLE_IDS.BASIC,
+      })
+      const res2 = await config.api.user.update({
+        ...user,
+        _rev: res1.rev,
+        roleId: roles.BUILTIN_ROLE_IDS.POWER,
+      })
+      expect(res2.ok).toEqual(true)
     })
 
     it("should require the _rev field for multiple updates", async () => {
       const user = await config.createUser()
       delete user._rev
 
-      await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send({ ...user, roleId: roles.BUILTIN_ROLE_IDS.BASIC })
-        .expect(200)
-        .expect("Content-Type", /json/)
-
-      await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send({ ...user, roleId: roles.BUILTIN_ROLE_IDS.POWER })
-        .expect(409)
-        .expect("Content-Type", /json/)
+      await config.api.user.update({
+        ...user,
+        roleId: roles.BUILTIN_ROLE_IDS.BASIC,
+      })
+      await config.api.user.update(
+        { ...user, roleId: roles.BUILTIN_ROLE_IDS.POWER },
+        { expectStatus: 409 }
+      )
     })
   })
 
   describe("destroy", () => {
     it("should be able to delete the user", async () => {
       const user = await config.createUser()
-      const res = await request
-        .delete(`/api/users/metadata/${user._id}`)
-        .set(config.defaultHeaders())
-        .expect(200)
-        .expect("Content-Type", /json/)
-      expect(res.body.message).toBeDefined()
+      const res = await config.api.user.destroy(user._id!)
+      expect(res.message).toBeDefined()
     })
   })
 
   describe("find", () => {
     it("should be able to find the user", async () => {
       const user = await config.createUser()
-      const res = await request
-        .get(`/api/users/metadata/${user._id}`)
-        .set(config.defaultHeaders())
-        .expect(200)
-        .expect("Content-Type", /json/)
-      expect(res.body._id).toEqual(user._id)
-      expect(res.body.roleId).toEqual(roles.BUILTIN_ROLE_IDS.ADMIN)
-      expect(res.body.tableId).toBeDefined()
+      const res = await config.api.user.find(user._id!)
+      expect(res._id).toEqual(user._id)
+      expect(res.roleId).toEqual(roles.BUILTIN_ROLE_IDS.ADMIN)
+      expect(res.tableId).toBeDefined()
     })
   })
 
@@ -153,59 +127,18 @@ describe("/users", () => {
 
     it("should be able to set a flag on the user", async () => {
       await config.createUser()
-      const res = await request
-        .post(`/api/users/flags`)
-        .set(config.defaultHeaders())
-        .send({ value: "test", flag: "test" })
-        .expect(200)
-        .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual("Flag set successfully")
+      const res = await config.api.user.setFlag("test", true)
+      expect(res.message).toEqual("Flag set successfully")
     })
   })
 
   describe("getFlags", () => {
     it("should get flags for a specific user", async () => {
-      let flagData = { value: "test", flag: "test" }
       await config.createUser()
-      await request
-        .post(`/api/users/flags`)
-        .set(config.defaultHeaders())
-        .send(flagData)
-        .expect(200)
-        .expect("Content-Type", /json/)
+      await config.api.user.setFlag("test", "test")
 
-      const res = await request
-        .get(`/api/users/flags`)
-        .set(config.defaultHeaders())
-        .expect(200)
-        .expect("Content-Type", /json/)
-      expect(res.body[flagData.value]).toEqual(flagData.flag)
-    })
-  })
-
-  describe("setFlag", () => {
-    it("should throw an error if a flag is not provided", async () => {
-      await config.createUser()
-      const res = await request
-        .post(`/api/users/flags`)
-        .set(config.defaultHeaders())
-        .send({ value: "test" })
-        .expect(400)
-        .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual(
-        "Must supply a 'flag' field in request body."
-      )
-    })
-
-    it("should be able to set a flag on the user", async () => {
-      await config.createUser()
-      const res = await request
-        .post(`/api/users/flags`)
-        .set(config.defaultHeaders())
-        .send({ value: "test", flag: "test" })
-        .expect(200)
-        .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual("Flag set successfully")
+      const res = await config.api.user.getFlags()
+      expect(res.test).toEqual("test")
     })
   })
 })
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 04c0552457..6877561fcb 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -264,7 +264,7 @@ class TestConfiguration {
     admin = false,
     email = this.defaultUserValues.email,
     roles,
-  }: any = {}) {
+  }: any = {}): Promise<User> {
     const db = tenancy.getTenantDB(this.getTenantId())
     let existing
     try {
diff --git a/packages/server/src/tests/utilities/api/user.ts b/packages/server/src/tests/utilities/api/user.ts
index 2066315778..2ed23c0461 100644
--- a/packages/server/src/tests/utilities/api/user.ts
+++ b/packages/server/src/tests/utilities/api/user.ts
@@ -1,6 +1,12 @@
-import { FetchUserMetadataResponse } from "@budibase/types"
+import {
+  FetchUserMetadataResponse,
+  FindUserMetadataResponse,
+  Flags,
+  UserMetadata,
+} from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
+import { DocumentInsertResponse } from "@budibase/nano"
 
 export class UserAPI extends TestAPI {
   constructor(config: TestConfiguration) {
@@ -26,10 +32,10 @@ export class UserAPI extends TestAPI {
     return res.body
   }
 
-  get = async (
+  find = async (
     id: string,
     { expectStatus } = { expectStatus: 200 }
-  ): Promise<GetUserMetadataResponse> => {
+  ): Promise<FindUserMetadataResponse> => {
     const res = await this.request
       .get(`/api/users/metadata/${id}`)
       .set(this.config.defaultHeaders())
@@ -45,4 +51,107 @@ export class UserAPI extends TestAPI {
 
     return res.body
   }
+
+  update = async (
+    user: UserMetadata,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<DocumentInsertResponse> => {
+    const res = await this.request
+      .put(`/api/users/metadata`)
+      .set(this.config.defaultHeaders())
+      .send(user)
+      .expect("Content-Type", /json/)
+
+    if (res.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          res.status
+        } with body ${JSON.stringify(res.body)}`
+      )
+    }
+
+    return res.body as DocumentInsertResponse
+  }
+
+  updateSelf = async (
+    user: UserMetadata,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<DocumentInsertResponse> => {
+    const res = await this.request
+      .post(`/api/users/metadata/self`)
+      .set(this.config.defaultHeaders())
+      .send(user)
+      .expect("Content-Type", /json/)
+
+    if (res.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          res.status
+        } with body ${JSON.stringify(res.body)}`
+      )
+    }
+
+    return res.body as DocumentInsertResponse
+  }
+
+  destroy = async (
+    id: string,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<{ message: string }> => {
+    const res = await this.request
+      .delete(`/api/users/metadata/${id}`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+
+    if (res.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          res.status
+        } with body ${JSON.stringify(res.body)}`
+      )
+    }
+
+    return res.body as { message: string }
+  }
+
+  setFlag = async (
+    flag: string,
+    value: any,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<{ message: string }> => {
+    const res = await this.request
+      .post(`/api/users/flags`)
+      .set(this.config.defaultHeaders())
+      .send({ flag, value })
+      .expect("Content-Type", /json/)
+
+    if (res.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          res.status
+        } with body ${JSON.stringify(res.body)}`
+      )
+    }
+
+    return res.body as { message: string }
+  }
+
+  getFlags = async (
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Flags> => {
+    const res = await this.request
+      .get(`/api/users/flags`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+
+    if (res.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          res.status
+        } with body ${JSON.stringify(res.body)}`
+      )
+    }
+
+    return res.body as Flags
+  }
 }
diff --git a/packages/types/src/api/web/app/user.ts b/packages/types/src/api/web/app/user.ts
index 2e0eeee7f8..7faec83e9c 100644
--- a/packages/types/src/api/web/app/user.ts
+++ b/packages/types/src/api/web/app/user.ts
@@ -1,3 +1,9 @@
-import { ContextUserMetadata } from "src/documents"
+import { ContextUserMetadata } from "../../../"
 
 export type FetchUserMetadataResponse = ContextUserMetadata[]
+export type FindUserMetadataResponse = ContextUserMetadata
+
+export interface SetFlagRequest {
+  flag: string
+  value: any
+}
diff --git a/packages/types/src/documents/account/flag.ts b/packages/types/src/documents/account/flag.ts
new file mode 100644
index 0000000000..a214348fe7
--- /dev/null
+++ b/packages/types/src/documents/account/flag.ts
@@ -0,0 +1,5 @@
+import { Document } from "../../"
+
+export interface Flags extends Document {
+  [key: string]: any
+}
diff --git a/packages/types/src/documents/account/index.ts b/packages/types/src/documents/account/index.ts
index 663fb91b58..1e0c800f39 100644
--- a/packages/types/src/documents/account/index.ts
+++ b/packages/types/src/documents/account/index.ts
@@ -1,2 +1,3 @@
 export * from "./account"
 export * from "./user"
+export * from "./flag"

From e8fb43d30c2f7527c7473b3ebb0ae783084058fd Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 8 Nov 2023 12:46:00 +0000
Subject: [PATCH 058/155] Updating DesignDocument and View typing - making it
 more clear how view and lucene indexing works.

---
 packages/backend-core/src/constants/db.ts     |  2 +-
 packages/backend-core/src/db/views.ts         | 18 ++--
 .../server/src/api/controllers/table/utils.ts | 51 +++++-----
 .../server/src/api/controllers/view/utils.ts  | 94 +++++++++++--------
 .../src/api/controllers/view/viewBuilder.ts   | 13 +--
 .../server/src/api/controllers/view/views.ts  | 14 ++-
 packages/server/src/db/inMemoryView.ts        |  4 +-
 .../src/sdk/app/rows/search/internal.ts       |  6 +-
 packages/types/src/documents/app/view.ts      | 21 ++++-
 packages/types/src/sdk/db.ts                  | 33 ++++++-
 10 files changed, 160 insertions(+), 96 deletions(-)

diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts
index b33b4835a9..bb944556af 100644
--- a/packages/backend-core/src/constants/db.ts
+++ b/packages/backend-core/src/constants/db.ts
@@ -28,7 +28,7 @@ export enum ViewName {
   APP_BACKUP_BY_TRIGGER = "by_trigger",
 }
 
-export const DeprecatedViews = {
+export const DeprecatedViews: Record<string, string[]> = {
   [ViewName.USER_BY_EMAIL]: [
     // removed due to inaccuracy in view doc filter logic
     "by_email",
diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts
index fbcdbde5fc..5d9c5b74d3 100644
--- a/packages/backend-core/src/db/views.ts
+++ b/packages/backend-core/src/db/views.ts
@@ -12,12 +12,14 @@ import {
   Database,
   DatabaseQueryOpts,
   Document,
+  DesignDocument,
+  DBView,
 } from "@budibase/types"
 import env from "../environment"
 
 const DESIGN_DB = "_design/database"
 
-function DesignDoc() {
+function DesignDoc(): DesignDocument {
   return {
     _id: DESIGN_DB,
     // view collation information, read before writing any complex views:
@@ -26,20 +28,14 @@ function DesignDoc() {
   }
 }
 
-interface DesignDocument {
-  views: any
-}
-
 async function removeDeprecated(db: Database, viewName: ViewName) {
-  // @ts-ignore
   if (!DeprecatedViews[viewName]) {
     return
   }
   try {
     const designDoc = await db.get<DesignDocument>(DESIGN_DB)
-    // @ts-ignore
     for (let deprecatedNames of DeprecatedViews[viewName]) {
-      delete designDoc.views[deprecatedNames]
+      delete designDoc.views?.[deprecatedNames]
     }
     await db.put(designDoc)
   } catch (err) {
@@ -48,18 +44,18 @@ async function removeDeprecated(db: Database, viewName: ViewName) {
 }
 
 export async function createView(
-  db: any,
+  db: Database,
   viewJs: string,
   viewName: string
 ): Promise<void> {
   let designDoc
   try {
-    designDoc = (await db.get(DESIGN_DB)) as DesignDocument
+    designDoc = await db.get<DesignDocument>(DESIGN_DB)
   } catch (err) {
     // no design doc, make one
     designDoc = DesignDoc()
   }
-  const view = {
+  const view: DBView = {
     map: viewJs,
   }
   designDoc.views = {
diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts
index da1573715c..ee295ff7af 100644
--- a/packages/server/src/api/controllers/table/utils.ts
+++ b/packages/server/src/api/controllers/table/utils.ts
@@ -333,29 +333,33 @@ export async function checkForViewUpdates(
   columnRename?: RenameColumn
 ) {
   const views = await getViews()
-  const tableViews = views.filter(view => view.meta.tableId === table._id)
+  const tableViews = views.filter(view => view.meta?.tableId === table._id)
 
   // Check each table view to see if impacted by this table action
   for (let view of tableViews) {
     let needsUpdated = false
+    const viewMetadata = view.meta as any
+    if (!viewMetadata) {
+      continue
+    }
 
     // First check for renames, otherwise check for deletions
     if (columnRename) {
       // Update calculation field if required
-      if (view.meta.field === columnRename.old) {
-        view.meta.field = columnRename.updated
+      if (viewMetadata.field === columnRename.old) {
+        viewMetadata.field = columnRename.updated
         needsUpdated = true
       }
 
       // Update group by field if required
-      if (view.meta.groupBy === columnRename.old) {
-        view.meta.groupBy = columnRename.updated
+      if (viewMetadata.groupBy === columnRename.old) {
+        viewMetadata.groupBy = columnRename.updated
         needsUpdated = true
       }
 
       // Update filters if required
-      if (view.meta.filters) {
-        view.meta.filters.forEach((filter: any) => {
+      if (viewMetadata.filters) {
+        viewMetadata.filters.forEach((filter: any) => {
           if (filter.key === columnRename.old) {
             filter.key = columnRename.updated
             needsUpdated = true
@@ -365,26 +369,26 @@ export async function checkForViewUpdates(
     } else if (deletedColumns) {
       deletedColumns.forEach((column: string) => {
         // Remove calculation statement if required
-        if (view.meta.field === column) {
-          delete view.meta.field
-          delete view.meta.calculation
-          delete view.meta.groupBy
+        if (viewMetadata.field === column) {
+          delete viewMetadata.field
+          delete viewMetadata.calculation
+          delete viewMetadata.groupBy
           needsUpdated = true
         }
 
         // Remove group by field if required
-        if (view.meta.groupBy === column) {
-          delete view.meta.groupBy
+        if (viewMetadata.groupBy === column) {
+          delete viewMetadata.groupBy
           needsUpdated = true
         }
 
         // Remove filters referencing deleted field if required
-        if (view.meta.filters && view.meta.filters.length) {
-          const initialLength = view.meta.filters.length
-          view.meta.filters = view.meta.filters.filter((filter: any) => {
+        if (viewMetadata.filters && viewMetadata.filters.length) {
+          const initialLength = viewMetadata.filters.length
+          viewMetadata.filters = viewMetadata.filters.filter((filter: any) => {
             return filter.key !== column
           })
-          if (initialLength !== view.meta.filters.length) {
+          if (initialLength !== viewMetadata.filters.length) {
             needsUpdated = true
           }
         }
@@ -397,15 +401,16 @@ export async function checkForViewUpdates(
         (field: any) => field.name == view.groupBy
       )
       const newViewTemplate = viewTemplate(
-        view.meta,
+        viewMetadata,
         groupByField?.type === FieldTypes.ARRAY
       )
-      await saveView(null, view.name, newViewTemplate)
-      if (!newViewTemplate.meta.schema) {
-        newViewTemplate.meta.schema = table.schema
+      const viewName = view.name!
+      await saveView(null, viewName, newViewTemplate)
+      if (!newViewTemplate.meta?.schema) {
+        newViewTemplate.meta!.schema = table.schema
       }
-      if (table.views?.[view.name]) {
-        table.views[view.name] = newViewTemplate.meta as View
+      if (table.views?.[viewName]) {
+        table.views[viewName] = newViewTemplate.meta as View
       }
     }
   }
diff --git a/packages/server/src/api/controllers/view/utils.ts b/packages/server/src/api/controllers/view/utils.ts
index 9ffa091a8c..7f9ae1a9bc 100644
--- a/packages/server/src/api/controllers/view/utils.ts
+++ b/packages/server/src/api/controllers/view/utils.ts
@@ -8,13 +8,13 @@ import {
 import env from "../../../environment"
 import { context } from "@budibase/backend-core"
 import viewBuilder from "./viewBuilder"
-import { Database } from "@budibase/types"
+import { Database, DBView, DesignDocument, InMemoryView } from "@budibase/types"
 
 export async function getView(viewName: string) {
   const db = context.getAppDB()
   if (env.SELF_HOSTED) {
-    const designDoc = await db.get<any>("_design/database")
-    return designDoc.views[viewName]
+    const designDoc = await db.get<DesignDocument>("_design/database")
+    return designDoc.views?.[viewName]
   } else {
     // This is a table view, don't read the view from the DB
     if (viewName.startsWith(DocumentType.TABLE + SEPARATOR)) {
@@ -22,7 +22,7 @@ export async function getView(viewName: string) {
     }
 
     try {
-      const viewDoc = await db.get<any>(generateMemoryViewID(viewName))
+      const viewDoc = await db.get<InMemoryView>(generateMemoryViewID(viewName))
       return viewDoc.view
     } catch (err: any) {
       // Return null when PouchDB doesn't found the view
@@ -35,25 +35,28 @@ export async function getView(viewName: string) {
   }
 }
 
-export async function getViews() {
+export async function getViews(): Promise<DBView[]> {
   const db = context.getAppDB()
-  const response = []
+  const response: DBView[] = []
   if (env.SELF_HOSTED) {
-    const designDoc = await db.get<any>("_design/database")
-    for (let name of Object.keys(designDoc.views)) {
+    const designDoc = await db.get<DesignDocument>("_design/database")
+    for (let name of Object.keys(designDoc.views || {})) {
       // Only return custom views, not built ins
       const viewNames = Object.values(ViewName) as string[]
       if (viewNames.indexOf(name) !== -1) {
         continue
       }
-      response.push({
-        name,
-        ...designDoc.views[name],
-      })
+      const view = designDoc.views?.[name]
+      if (view) {
+        response.push({
+          name,
+          ...view,
+        })
+      }
     }
   } else {
     const views = (
-      await db.allDocs<any>(
+      await db.allDocs<InMemoryView>(
         getMemoryViewParams({
           include_docs: true,
         })
@@ -72,11 +75,11 @@ export async function getViews() {
 export async function saveView(
   originalName: string | null,
   viewName: string,
-  viewTemplate: any
+  viewTemplate: DBView
 ) {
   const db = context.getAppDB()
   if (env.SELF_HOSTED) {
-    const designDoc = await db.get<any>("_design/database")
+    const designDoc = await db.get<DesignDocument>("_design/database")
     designDoc.views = {
       ...designDoc.views,
       [viewName]: viewTemplate,
@@ -89,17 +92,17 @@ export async function saveView(
   } else {
     const id = generateMemoryViewID(viewName)
     const originalId = originalName ? generateMemoryViewID(originalName) : null
-    const viewDoc: any = {
+    const viewDoc: InMemoryView = {
       _id: id,
       view: viewTemplate,
       name: viewName,
-      tableId: viewTemplate.meta.tableId,
+      tableId: viewTemplate.meta!.tableId,
     }
     try {
-      const old = await db.get<any>(id)
+      const old = await db.get<InMemoryView>(id)
       if (originalId) {
-        const originalDoc = await db.get<any>(originalId)
-        await db.remove(originalDoc._id, originalDoc._rev)
+        const originalDoc = await db.get<InMemoryView>(originalId)
+        await db.remove(originalDoc._id!, originalDoc._rev)
       }
       if (old && old._rev) {
         viewDoc._rev = old._rev
@@ -114,52 +117,65 @@ export async function saveView(
 export async function deleteView(viewName: string) {
   const db = context.getAppDB()
   if (env.SELF_HOSTED) {
-    const designDoc = await db.get<any>("_design/database")
-    const view = designDoc.views[viewName]
-    delete designDoc.views[viewName]
+    const designDoc = await db.get<DesignDocument>("_design/database")
+    const view = designDoc.views?.[viewName]
+    delete designDoc.views?.[viewName]
     await db.put(designDoc)
     return view
   } else {
     const id = generateMemoryViewID(viewName)
-    const viewDoc = await db.get<any>(id)
-    await db.remove(viewDoc._id, viewDoc._rev)
+    const viewDoc = await db.get<InMemoryView>(id)
+    await db.remove(viewDoc._id!, viewDoc._rev)
     return viewDoc.view
   }
 }
 
 export async function migrateToInMemoryView(db: Database, viewName: string) {
   // delete the view initially
-  const designDoc = await db.get<any>("_design/database")
+  const designDoc = await db.get<DesignDocument>("_design/database")
+  const meta = designDoc.views?.[viewName].meta
+  if (!meta) {
+    throw new Error("Unable to migrate view - no metadata")
+  }
   // run the view back through the view builder to update it
-  const view = viewBuilder(designDoc.views[viewName].meta)
-  delete designDoc.views[viewName]
+  const view = viewBuilder(meta)
+  delete designDoc.views?.[viewName]
   await db.put(designDoc)
-  await exports.saveView(db, null, viewName, view)
+  await saveView(null, viewName, view)
 }
 
 export async function migrateToDesignView(db: Database, viewName: string) {
-  let view = await db.get<any>(generateMemoryViewID(viewName))
-  const designDoc = await db.get<any>("_design/database")
-  designDoc.views[viewName] = viewBuilder(view.view.meta)
+  let view = await db.get<InMemoryView>(generateMemoryViewID(viewName))
+  const designDoc = await db.get<DesignDocument>("_design/database")
+  const meta = view.view.meta
+  if (!meta) {
+    throw new Error("Unable to migrate view - no metadata")
+  }
+  if (!designDoc.views) {
+    designDoc.views = {}
+  }
+  designDoc.views[viewName] = viewBuilder(meta)
   await db.put(designDoc)
-  await db.remove(view._id, view._rev)
+  await db.remove(view._id!, view._rev)
 }
 
 export async function getFromDesignDoc(db: Database, viewName: string) {
-  const designDoc = await db.get<any>("_design/database")
-  let view = designDoc.views[viewName]
+  const designDoc = await db.get<DesignDocument>("_design/database")
+  let view = designDoc.views?.[viewName]
   if (view == null) {
     throw { status: 404, message: "Unable to get view" }
   }
   return view
 }
 
-export async function getFromMemoryDoc(db: Database, viewName: string) {
-  let view = await db.get<any>(generateMemoryViewID(viewName))
+export async function getFromMemoryDoc(
+  db: Database,
+  viewName: string
+): Promise<DBView> {
+  let view = await db.get<InMemoryView>(generateMemoryViewID(viewName))
   if (view) {
-    view = view.view
+    return view.view
   } else {
     throw { status: 404, message: "Unable to get view" }
   }
-  return view
 }
diff --git a/packages/server/src/api/controllers/view/viewBuilder.ts b/packages/server/src/api/controllers/view/viewBuilder.ts
index cbe7e72d04..3df9df6657 100644
--- a/packages/server/src/api/controllers/view/viewBuilder.ts
+++ b/packages/server/src/api/controllers/view/viewBuilder.ts
@@ -1,13 +1,4 @@
-import { ViewFilter } from "@budibase/types"
-
-type ViewTemplateOpts = {
-  field: string
-  tableId: string
-  groupBy: string
-  filters: ViewFilter[]
-  calculation: string
-  groupByMulti: boolean
-}
+import { ViewFilter, ViewTemplateOpts, DBView } from "@budibase/types"
 
 const TOKEN_MAP: Record<string, string> = {
   EQUALS: "===",
@@ -146,7 +137,7 @@ function parseEmitExpression(field: string, groupBy: string) {
 export default function (
   { field, tableId, groupBy, filters = [], calculation }: ViewTemplateOpts,
   groupByMulti?: boolean
-) {
+): DBView {
   // first filter can't have a conjunction
   if (filters && filters.length > 0 && filters[0].conjunction) {
     delete filters[0].conjunction
diff --git a/packages/server/src/api/controllers/view/views.ts b/packages/server/src/api/controllers/view/views.ts
index 86deff09ef..d311565524 100644
--- a/packages/server/src/api/controllers/view/views.ts
+++ b/packages/server/src/api/controllers/view/views.ts
@@ -47,8 +47,11 @@ export async function save(ctx: Ctx) {
 
   // add views to table document
   if (!table.views) table.views = {}
-  if (!view.meta.schema) {
-    view.meta.schema = table.schema
+  if (!view.meta?.schema) {
+    view.meta = {
+      ...view.meta!,
+      schema: table.schema,
+    }
   }
   table.views[viewName] = { ...view.meta, name: viewName }
   if (originalName) {
@@ -125,10 +128,13 @@ export async function destroy(ctx: Ctx) {
   const db = context.getAppDB()
   const viewName = decodeURIComponent(ctx.params.viewName)
   const view = await deleteView(viewName)
+  if (!view || !view.meta) {
+    ctx.throw(400, "Unable to delete view - no metadata/view not found.")
+  }
   const table = await sdk.tables.getTable(view.meta.tableId)
   delete table.views![viewName]
   await db.put(table)
-  await events.view.deleted(view)
+  await events.view.deleted(view as View)
 
   ctx.body = view
   builderSocket?.emitTableUpdate(ctx, table)
@@ -147,7 +153,7 @@ export async function exportView(ctx: Ctx) {
     )
   }
 
-  if (view) {
+  if (view && view.meta) {
     ctx.params.viewName = viewName
     // Fetch view rows
     ctx.query = {
diff --git a/packages/server/src/db/inMemoryView.ts b/packages/server/src/db/inMemoryView.ts
index 4e9301f4ee..724bc725ce 100644
--- a/packages/server/src/db/inMemoryView.ts
+++ b/packages/server/src/db/inMemoryView.ts
@@ -1,5 +1,5 @@
 import newid from "./newid"
-import { Row, View, Document } from "@budibase/types"
+import { Row, Document, DBView } from "@budibase/types"
 
 // bypass the main application db config
 // use in memory pouchdb directly
@@ -7,7 +7,7 @@ import { db as dbCore } from "@budibase/backend-core"
 const Pouch = dbCore.getPouch({ inMemory: true })
 
 export async function runView(
-  view: View,
+  view: DBView,
   calculation: string,
   group: boolean,
   data: Row[]
diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts
index e31bda1a15..87a33c0ba0 100644
--- a/packages/server/src/sdk/app/rows/search/internal.ts
+++ b/packages/server/src/sdk/app/rows/search/internal.ts
@@ -185,8 +185,8 @@ export async function fetchView(
       group: !!group,
     })
   } else {
-    const tableId = viewInfo.meta.tableId
-    const data = await fetchRaw(tableId)
+    const tableId = viewInfo.meta!.tableId
+    const data = await fetchRaw(tableId!)
     response = await inMemoryViews.runView(
       viewInfo,
       calculation as string,
@@ -200,7 +200,7 @@ export async function fetchView(
     response.rows = response.rows.map(row => row.doc)
     let table: Table
     try {
-      table = await sdk.tables.getTable(viewInfo.meta.tableId)
+      table = await sdk.tables.getTable(viewInfo.meta!.tableId)
     } catch (err) {
       throw new Error("Unable to retrieve view table.")
     }
diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts
index 0d79e2c505..b5a22ec592 100644
--- a/packages/types/src/documents/app/view.ts
+++ b/packages/types/src/documents/app/view.ts
@@ -1,5 +1,24 @@
 import { SearchFilter, SortOrder, SortType } from "../../api"
 import { UIFieldMetadata } from "./table"
+import { Document } from "../document"
+import { DBView } from "../../sdk"
+
+export type ViewTemplateOpts = {
+  field: string
+  tableId: string
+  groupBy: string
+  filters: ViewFilter[]
+  schema: any
+  calculation: string
+  groupByMulti?: boolean
+}
+
+export interface InMemoryView extends Document {
+  view: DBView
+  name: string
+  tableId: string
+  groupBy?: string
+}
 
 export interface View {
   name?: string
@@ -10,7 +29,7 @@ export interface View {
   calculation?: ViewCalculation
   map?: string
   reduce?: any
-  meta?: Record<string, any>
+  meta?: ViewTemplateOpts
 }
 
 export interface ViewV2 {
diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts
index 12c53a9561..26807d99ce 100644
--- a/packages/types/src/sdk/db.ts
+++ b/packages/types/src/sdk/db.ts
@@ -1,5 +1,5 @@
 import Nano from "@budibase/nano"
-import { AllDocsResponse, AnyDocument, Document } from "../"
+import { AllDocsResponse, AnyDocument, Document, ViewTemplateOpts } from "../"
 import { Writable } from "stream"
 
 export enum SearchIndex {
@@ -20,6 +20,37 @@ export enum SortOption {
   DESCENDING = "desc",
 }
 
+export type IndexAnalyzer = {
+  name: string
+  default?: string
+  fields?: Record<string, string>
+}
+
+export type DBView = {
+  name?: string
+  map: string
+  reduce?: string
+  meta?: ViewTemplateOpts
+  groupBy?: string
+}
+
+export interface DesignDocument extends Document {
+  // we use this static reference for all design documents
+  _id: "_design/database"
+  language?: string
+  // CouchDB views
+  views?: {
+    [viewName: string]: DBView
+  }
+  // Lucene indexes
+  indexes?: {
+    [indexName: string]: {
+      index: string
+      analyzer?: string | IndexAnalyzer
+    }
+  }
+}
+
 export type CouchFindOptions = {
   selector: PouchDB.Find.Selector
   fields?: string[]

From 935ec4c47ef6035ad163d9b39e1c6f686f197e16 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 8 Nov 2023 12:57:07 +0000
Subject: [PATCH 059/155] Updating layout typing.

---
 packages/server/src/api/controllers/layout.ts | 4 ++--
 packages/types/src/documents/app/layout.ts    | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/layout.ts b/packages/server/src/api/controllers/layout.ts
index 2a359592c7..69e4ad91ed 100644
--- a/packages/server/src/api/controllers/layout.ts
+++ b/packages/server/src/api/controllers/layout.ts
@@ -1,7 +1,7 @@
 import { EMPTY_LAYOUT } from "../../constants/layouts"
 import { generateLayoutID, getScreenParams } from "../../db/utils"
 import { events, context } from "@budibase/backend-core"
-import { BBContext } from "@budibase/types"
+import { BBContext, Layout } from "@budibase/types"
 
 export async function save(ctx: BBContext) {
   const db = context.getAppDB()
@@ -30,7 +30,7 @@ export async function destroy(ctx: BBContext) {
     layoutRev = ctx.params.layoutRev
 
   const layoutsUsedByScreens = (
-    await db.allDocs<any>(
+    await db.allDocs<Layout>(
       getScreenParams(null, {
         include_docs: true,
       })
diff --git a/packages/types/src/documents/app/layout.ts b/packages/types/src/documents/app/layout.ts
index db046e3d92..06542f680d 100644
--- a/packages/types/src/documents/app/layout.ts
+++ b/packages/types/src/documents/app/layout.ts
@@ -2,4 +2,5 @@ import { Document } from "../document"
 
 export interface Layout extends Document {
   props: any
+  layoutId?: string
 }

From b167fd08e1b4c2f3f1abb02f0583ddc1b78ff6cd Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Wed, 8 Nov 2023 13:27:25 +0000
Subject: [PATCH 060/155] Fix issues with automation layout styling

---
 .../SetupPanel/QueryParamSelector.svelte      |  8 +-
 .../automation/SetupPanel/RowSelector.svelte  | 84 ++++++++++---------
 .../SetupPanel/RowSelectorTypes.svelte        |  1 +
 .../common/LinkedRowSelector.svelte           |  4 +-
 4 files changed, 50 insertions(+), 47 deletions(-)

diff --git a/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte b/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte
index 6b3433babc..e68ab1feac 100644
--- a/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte
@@ -41,8 +41,8 @@
 </div>
 
 {#if parameters.length}
-  <div class="schema-fields">
-    {#each parameters as field}
+  {#each parameters as field}
+    <div class="schema-fields">
       <Label>{field.name}</Label>
       <div class="field-width">
         <DrawerBindableInput
@@ -56,8 +56,8 @@
           updateOnChange={false}
         />
       </div>
-    {/each}
-  </div>
+    </div>
+  {/each}
 {/if}
 
 <style>
diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte
index 21d90944e8..e4751e0f6a 100644
--- a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte
@@ -114,33 +114,12 @@
 </div>
 {#if schemaFields.length}
   {#each schemaFields as [field, schema]}
-    <div class="schema-fields">
-      <Label>{field}</Label>
-      <div class="field-width">
-        {#if !schema.autocolumn && schema.type !== "attachment"}
-          {#if isTestModal}
-            <RowSelectorTypes
-              {isTestModal}
-              {field}
-              {schema}
-              bindings={parsedBindings}
-              {value}
-              {onChange}
-            />
-          {:else}
-            <DrawerBindableSlot
-              fillWidth
-              title={value.title}
-              panel={AutomationBindingPanel}
-              type={schema.type}
-              {schema}
-              value={value[field]}
-              on:change={e => onChange(e, field)}
-              {bindings}
-              allowJS={true}
-              updateOnChange={false}
-              drawerLeft="260px"
-            >
+    {#if !schema.autocolumn}
+      {#if schema.type !== "attachment"}
+        <div class="schema-fields">
+          <Label>{field}</Label>
+          <div class="field-width">
+            {#if isTestModal}
               <RowSelectorTypes
                 {isTestModal}
                 {field}
@@ -149,22 +128,45 @@
                 {value}
                 {onChange}
               />
-            </DrawerBindableSlot>
-          {/if}
-        {/if}
+            {:else}
+              <DrawerBindableSlot
+                fillWidth
+                title={value.title}
+                panel={AutomationBindingPanel}
+                type={schema.type}
+                {schema}
+                value={value[field]}
+                on:change={e => onChange(e, field)}
+                {bindings}
+                allowJS={true}
+                updateOnChange={false}
+                drawerLeft="260px"
+              >
+                <RowSelectorTypes
+                  {isTestModal}
+                  {field}
+                  {schema}
+                  bindings={parsedBindings}
+                  {value}
+                  {onChange}
+                />
+              </DrawerBindableSlot>
+            {/if}
 
-        {#if isUpdateRow && schema.type === "link"}
-          <div class="checkbox-field">
-            <Checkbox
-              value={meta.fields?.[field]?.clearRelationships}
-              text={"Clear relationships if empty?"}
-              size={"S"}
-              on:change={e => onChangeSetting(e, field)}
-            />
+            {#if isUpdateRow && schema.type === "link"}
+              <div class="checkbox-field">
+                <Checkbox
+                  value={meta.fields?.[field]?.clearRelationships}
+                  text={"Clear relationships if empty?"}
+                  size={"S"}
+                  on:change={e => onChangeSetting(e, field)}
+                />
+              </div>
+            {/if}
           </div>
-        {/if}
-      </div>
-    </div>
+        </div>
+      {/if}
+    {/if}
   {/each}
 {/if}
 
diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte
index 373d174541..86235297ae 100644
--- a/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte
@@ -67,6 +67,7 @@
     bind:linkedRows={value[field]}
     {schema}
     on:change={e => onChange(e, field)}
+    useLabel={false}
   />
 {:else if schema.type === "string" || schema.type === "number"}
   <svelte:component
diff --git a/packages/builder/src/components/common/LinkedRowSelector.svelte b/packages/builder/src/components/common/LinkedRowSelector.svelte
index d357a0a54d..839075ba89 100644
--- a/packages/builder/src/components/common/LinkedRowSelector.svelte
+++ b/packages/builder/src/components/common/LinkedRowSelector.svelte
@@ -7,7 +7,7 @@
 
   export let schema
   export let linkedRows = []
-
+  export let useLabel = true
   const dispatch = createEventDispatcher()
 
   let rows = []
@@ -51,7 +51,7 @@
       linkedIds = e.detail ? [e.detail] : []
       dispatch("change", linkedIds)
     }}
-    {label}
+    label={useLabel ? label : null}
     sort
   />
 {:else}

From cf4e29ef311ed4974cbee7cf3e97f9f8c7e1d37e Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 8 Nov 2023 13:35:43 +0000
Subject: [PATCH 061/155] allow for custom DATA_DIR on azure

---
 hosting/couchdb/build-target-paths.sh | 4 ++--
 hosting/scripts/build-target-paths.sh | 4 ++--
 hosting/single/runner.sh              | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/hosting/couchdb/build-target-paths.sh b/hosting/couchdb/build-target-paths.sh
index 67e1765ca8..34227011f4 100644
--- a/hosting/couchdb/build-target-paths.sh
+++ b/hosting/couchdb/build-target-paths.sh
@@ -2,8 +2,8 @@
 
 echo ${TARGETBUILD} > /buildtarget.txt
 if [[ "${TARGETBUILD}" = "aas" ]]; then
-    # Azure AppService uses /home for persisent data & SSH on port 2222
-    DATA_DIR=/home
+    # Azure AppService uses /home for persistent data & SSH on port 2222
+    DATA_DIR="${DATA_DIR:-/home}"
     WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
     mkdir -p $DATA_DIR/{search,minio,couch}
     mkdir -p $DATA_DIR/couch/{dbs,views}
diff --git a/hosting/scripts/build-target-paths.sh b/hosting/scripts/build-target-paths.sh
index 67e1765ca8..34227011f4 100644
--- a/hosting/scripts/build-target-paths.sh
+++ b/hosting/scripts/build-target-paths.sh
@@ -2,8 +2,8 @@
 
 echo ${TARGETBUILD} > /buildtarget.txt
 if [[ "${TARGETBUILD}" = "aas" ]]; then
-    # Azure AppService uses /home for persisent data & SSH on port 2222
-    DATA_DIR=/home
+    # Azure AppService uses /home for persistent data & SSH on port 2222
+    DATA_DIR="${DATA_DIR:-/home}"
     WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
     mkdir -p $DATA_DIR/{search,minio,couch}
     mkdir -p $DATA_DIR/couch/{dbs,views}
diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh
index 9dc7aa25d8..87201c95c0 100644
--- a/hosting/single/runner.sh
+++ b/hosting/single/runner.sh
@@ -22,7 +22,7 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME
 
 # Azure App Service customisations
 if [[ "${TARGETBUILD}" = "aas" ]]; then
-    DATA_DIR=/home
+    DATA_DIR="${DATA_DIR:-/home}"
     WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
     /etc/init.d/ssh start
 else

From 1016fa97f53178b7ea4bcf546f4526a4cb9e7f6a Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 8 Nov 2023 13:49:53 +0000
Subject: [PATCH 062/155] Bump version to 2.13.5

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 765395227c..5605642877 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.13.4",
+  "version": "2.13.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From f7fabf152d0c3e8e41562f78077eb0eaf6f92dbf Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Wed, 8 Nov 2023 13:55:34 +0000
Subject: [PATCH 063/155] css fixes

---
 .../FlowChart/FlowChart.svelte                | 15 ++--
 .../SetupPanel/QueryParamSelector.svelte      |  8 +-
 .../automation/SetupPanel/RowSelector.svelte  | 82 +++++++++----------
 3 files changed, 51 insertions(+), 54 deletions(-)

diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte
index a3c6a06690..54e098c9d5 100644
--- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte
+++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte
@@ -48,15 +48,14 @@
     <UndoRedoControl store={automationHistoryStore} />
   </div>
   <div class="controls">
-    <div class="buttons">
+    <div
+      on:click={() => {
+        testDataModal.show()
+      }}
+      class="buttons"
+    >
       <Icon hoverable size="M" name="Play" />
-      <div
-        on:click={() => {
-          testDataModal.show()
-        }}
-      >
-        Run test
-      </div>
+      <div>Run test</div>
     </div>
     <div class="buttons">
       <Icon
diff --git a/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte b/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte
index e68ab1feac..fcf8e7fddb 100644
--- a/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte
@@ -27,7 +27,7 @@
   $: if (value?.queryId == null) value = { queryId: "" }
 </script>
 
-<div class="schema-fields">
+<div class="schema-field">
   <Label>Query</Label>
   <div class="field-width">
     <Select
@@ -42,7 +42,7 @@
 
 {#if parameters.length}
   {#each parameters as field}
-    <div class="schema-fields">
+    <div class="schema-field">
       <Label>{field.name}</Label>
       <div class="field-width">
         <DrawerBindableInput
@@ -65,7 +65,7 @@
     width: 320px;
   }
 
-  .schema-fields {
+  .schema-field {
     display: flex;
     justify-content: space-between;
     align-items: center;
@@ -76,7 +76,7 @@
     margin-bottom: 10px;
   }
 
-  .schema-fields :global(label) {
+  .schema-field :global(label) {
     text-transform: capitalize;
   }
 </style>
diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte
index e4751e0f6a..c4a67ab025 100644
--- a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte
@@ -114,12 +114,33 @@
 </div>
 {#if schemaFields.length}
   {#each schemaFields as [field, schema]}
-    {#if !schema.autocolumn}
-      {#if schema.type !== "attachment"}
-        <div class="schema-fields">
-          <Label>{field}</Label>
-          <div class="field-width">
-            {#if isTestModal}
+    {#if !schema.autocolumn && schema.type !== "attachment"}
+      <div class="schema-fields">
+        <Label>{field}</Label>
+        <div class="field-width">
+          {#if isTestModal}
+            <RowSelectorTypes
+              {isTestModal}
+              {field}
+              {schema}
+              bindings={parsedBindings}
+              {value}
+              {onChange}
+            />
+          {:else}
+            <DrawerBindableSlot
+              fillWidth
+              title={value.title}
+              panel={AutomationBindingPanel}
+              type={schema.type}
+              {schema}
+              value={value[field]}
+              on:change={e => onChange(e, field)}
+              {bindings}
+              allowJS={true}
+              updateOnChange={false}
+              drawerLeft="260px"
+            >
               <RowSelectorTypes
                 {isTestModal}
                 {field}
@@ -128,44 +149,21 @@
                 {value}
                 {onChange}
               />
-            {:else}
-              <DrawerBindableSlot
-                fillWidth
-                title={value.title}
-                panel={AutomationBindingPanel}
-                type={schema.type}
-                {schema}
-                value={value[field]}
-                on:change={e => onChange(e, field)}
-                {bindings}
-                allowJS={true}
-                updateOnChange={false}
-                drawerLeft="260px"
-              >
-                <RowSelectorTypes
-                  {isTestModal}
-                  {field}
-                  {schema}
-                  bindings={parsedBindings}
-                  {value}
-                  {onChange}
-                />
-              </DrawerBindableSlot>
-            {/if}
+            </DrawerBindableSlot>
+          {/if}
 
-            {#if isUpdateRow && schema.type === "link"}
-              <div class="checkbox-field">
-                <Checkbox
-                  value={meta.fields?.[field]?.clearRelationships}
-                  text={"Clear relationships if empty?"}
-                  size={"S"}
-                  on:change={e => onChangeSetting(e, field)}
-                />
-              </div>
-            {/if}
-          </div>
+          {#if isUpdateRow && schema.type === "link"}
+            <div class="checkbox-field">
+              <Checkbox
+                value={meta.fields?.[field]?.clearRelationships}
+                text={"Clear relationships if empty?"}
+                size={"S"}
+                on:change={e => onChangeSetting(e, field)}
+              />
+            </div>
+          {/if}
         </div>
-      {/if}
+      </div>
     {/if}
   {/each}
 {/if}

From 47292b8ab4fd95a7d1ffaada5baf28bd394b977d Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 8 Nov 2023 14:37:19 +0000
Subject: [PATCH 064/155] Make DB name non-optional.

---
 packages/backend-core/src/context/mainContext.ts   |  3 +++
 packages/backend-core/src/db/couch/DatabaseImpl.ts |  5 +----
 packages/backend-core/src/db/db.ts                 | 12 +-----------
 packages/server/src/api/controllers/datasource.ts  |  2 +-
 packages/server/src/db/utils.ts                    | 12 +++++++++---
 5 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts
index 609c18abb5..52ab17d31e 100644
--- a/packages/backend-core/src/context/mainContext.ts
+++ b/packages/backend-core/src/context/mainContext.ts
@@ -276,6 +276,9 @@ export function getAuditLogsDB(): Database {
  */
 export function getAppDB(opts?: any): Database {
   const appId = getAppId()
+  if (!appId) {
+    throw new Error("Unable to retrieve app DB - no app ID.")
+  }
   return getDB(appId, opts)
 }
 
diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index 330b15e680..1ace60ed5b 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -48,10 +48,7 @@ export class DatabaseImpl implements Database {
 
   private readonly couchInfo = getCouchInfo()
 
-  constructor(dbName?: string, opts?: DatabaseOpts, connection?: string) {
-    if (dbName == null) {
-      throw new Error("Database name cannot be undefined.")
-    }
+  constructor(dbName: string, opts?: DatabaseOpts, connection?: string) {
     this.name = dbName
     this.pouchOpts = opts || {}
     if (connection) {
diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts
index 9aae64b892..54c4d3ae0f 100644
--- a/packages/backend-core/src/db/db.ts
+++ b/packages/backend-core/src/db/db.ts
@@ -1,10 +1,7 @@
-import env from "../environment"
 import { directCouchQuery, DatabaseImpl } from "./couch"
 import { CouchFindOptions, Database } from "@budibase/types"
 
-const dbList = new Set()
-
-export function getDB(dbName?: string, opts?: any): Database {
+export function getDB(dbName: string, opts?: any): Database {
   return new DatabaseImpl(dbName, opts)
 }
 
@@ -22,13 +19,6 @@ export async function doWithDB<T>(
   return await cb(db)
 }
 
-export function allDbs() {
-  if (!env.isTest()) {
-    throw new Error("Cannot be used outside test environment.")
-  }
-  return [...dbList]
-}
-
 export async function directCouchAllDbs(queryString?: string) {
   let couchPath = "/_all_dbs"
   if (queryString) {
diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts
index 5d024d51b6..39bc612b32 100644
--- a/packages/server/src/api/controllers/datasource.ts
+++ b/packages/server/src/api/controllers/datasource.ts
@@ -337,7 +337,7 @@ export async function destroy(ctx: UserCtx) {
   if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
     await destroyInternalTablesBySourceId(datasourceId)
   } else {
-    const queries = await db.allDocs(getQueryParams(datasourceId, null))
+    const queries = await db.allDocs(getQueryParams(datasourceId))
     await db.bulkDocs(
       queries.rows.map((row: any) => ({
         _id: row.id,
diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts
index d532d8a8b2..715db552c9 100644
--- a/packages/server/src/db/utils.ts
+++ b/packages/server/src/db/utils.ts
@@ -6,6 +6,7 @@ import {
   RelationshipFieldMetadata,
   VirtualDocumentType,
   INTERNAL_TABLE_SOURCE_ID,
+  DatabaseQueryOpts,
 } from "@budibase/types"
 import { FieldTypes } from "../constants"
 export { DocumentType, VirtualDocumentType } from "@budibase/types"
@@ -229,7 +230,10 @@ export function getAutomationMetadataParams(otherProps: any = {}) {
 /**
  * Gets parameters for retrieving a query, this is a utility function for the getDocParams function.
  */
-export function getQueryParams(datasourceId?: Optional, otherProps: any = {}) {
+export function getQueryParams(
+  datasourceId?: Optional,
+  otherProps: Partial<DatabaseQueryOpts> = {}
+) {
   if (datasourceId == null) {
     return getDocParams(DocumentType.QUERY, null, otherProps)
   }
@@ -256,7 +260,7 @@ export function generateMetadataID(type: string, entityId: string) {
 export function getMetadataParams(
   type: string,
   entityId?: Optional,
-  otherProps: any = {}
+  otherProps: Partial<DatabaseQueryOpts> = {}
 ) {
   let docId = `${type}${SEPARATOR}`
   if (entityId != null) {
@@ -269,7 +273,9 @@ export function generateMemoryViewID(viewName: string) {
   return `${DocumentType.MEM_VIEW}${SEPARATOR}${viewName}`
 }
 
-export function getMemoryViewParams(otherProps: any = {}) {
+export function getMemoryViewParams(
+  otherProps: Partial<DatabaseQueryOpts> = {}
+) {
   return getDocParams(DocumentType.MEM_VIEW, null, otherProps)
 }
 

From 8d98da6c5773add2372cb070724571e7622a2126 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 8 Nov 2023 14:57:30 +0000
Subject: [PATCH 065/155] Fixing issue with table names.

---
 packages/server/src/api/routes/tests/row.spec.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 3598376a76..060f6e46c1 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -2013,6 +2013,8 @@ describe.each([
         ["links"],
         {
           ...cfg,
+          // needs to be a short name
+          name: "b",
           schema: {
             ...cfg.schema,
             formula: {

From e3a4c34f8d2b7094965b1ff10f23fb3a8cd8144f Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 8 Nov 2023 16:17:24 +0000
Subject: [PATCH 066/155] Pass types through context callbacks.

---
 .../backend-core/src/cache/appMetadata.ts     |  2 +-
 packages/backend-core/src/context/Context.ts  |  2 +-
 .../backend-core/src/context/mainContext.ts   | 20 +++++++++----------
 packages/backend-core/src/db/db.ts            |  6 +++---
 packages/server/src/automations/triggers.ts   |  2 +-
 .../src/tests/utilities/TestConfiguration.ts  | 11 +++++-----
 packages/server/src/threads/automation.ts     |  2 +-
 7 files changed, 23 insertions(+), 22 deletions(-)

diff --git a/packages/backend-core/src/cache/appMetadata.ts b/packages/backend-core/src/cache/appMetadata.ts
index bd3efc20db..d442511fb8 100644
--- a/packages/backend-core/src/cache/appMetadata.ts
+++ b/packages/backend-core/src/cache/appMetadata.ts
@@ -19,7 +19,7 @@ async function populateFromDB(appId: string) {
   return doWithDB(
     appId,
     (db: Database) => {
-      return db.get(DocumentType.APP_METADATA)
+      return db.get<App>(DocumentType.APP_METADATA)
     },
     { skip_setup: true }
   )
diff --git a/packages/backend-core/src/context/Context.ts b/packages/backend-core/src/context/Context.ts
index d29b6935a8..a59f5c6503 100644
--- a/packages/backend-core/src/context/Context.ts
+++ b/packages/backend-core/src/context/Context.ts
@@ -4,7 +4,7 @@ import { ContextMap } from "./types"
 export default class Context {
   static storage = new AsyncLocalStorage<ContextMap>()
 
-  static run(context: ContextMap, func: any) {
+  static run<T>(context: ContextMap, func: () => T) {
     return Context.storage.run(context, () => func())
   }
 
diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts
index 52ab17d31e..d2259cfcab 100644
--- a/packages/backend-core/src/context/mainContext.ts
+++ b/packages/backend-core/src/context/mainContext.ts
@@ -98,17 +98,17 @@ function updateContext(updates: ContextMap): ContextMap {
   return context
 }
 
-async function newContext(updates: ContextMap, task: any) {
+async function newContext<T>(updates: ContextMap, task: () => T) {
   // see if there already is a context setup
   let context: ContextMap = updateContext(updates)
   return Context.run(context, task)
 }
 
-export async function doInAutomationContext(params: {
+export async function doInAutomationContext<T>(params: {
   appId: string
   automationId: string
-  task: any
-}): Promise<any> {
+  task: () => T
+}): Promise<T> {
   const tenantId = getTenantIDFromAppID(params.appId)
   return newContext(
     {
@@ -144,10 +144,10 @@ export async function doInTenant<T>(
   return newContext(updates, task)
 }
 
-export async function doInAppContext(
+export async function doInAppContext<T>(
   appId: string | null,
-  task: any
-): Promise<any> {
+  task: () => T
+): Promise<T> {
   if (!appId && !env.isTest()) {
     throw new Error("appId is required")
   }
@@ -165,10 +165,10 @@ export async function doInAppContext(
   return newContext(updates, task)
 }
 
-export async function doInIdentityContext(
+export async function doInIdentityContext<T>(
   identity: IdentityContext,
-  task: any
-): Promise<any> {
+  task: () => T
+): Promise<T> {
   if (!identity) {
     throw new Error("identity is required")
   }
diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts
index 54c4d3ae0f..3e69d49f0e 100644
--- a/packages/backend-core/src/db/db.ts
+++ b/packages/backend-core/src/db/db.ts
@@ -1,7 +1,7 @@
 import { directCouchQuery, DatabaseImpl } from "./couch"
-import { CouchFindOptions, Database } from "@budibase/types"
+import { CouchFindOptions, Database, DatabaseOpts } from "@budibase/types"
 
-export function getDB(dbName: string, opts?: any): Database {
+export function getDB(dbName: string, opts?: DatabaseOpts): Database {
   return new DatabaseImpl(dbName, opts)
 }
 
@@ -11,7 +11,7 @@ export function getDB(dbName: string, opts?: any): Database {
 export async function doWithDB<T>(
   dbName: string,
   cb: (db: Database) => Promise<T>,
-  opts = {}
+  opts?: DatabaseOpts
 ) {
   const db = getDB(dbName, opts)
   // need this to be async so that we can correctly close DB after all
diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts
index f0eca759f5..ac977bbefb 100644
--- a/packages/server/src/automations/triggers.ts
+++ b/packages/server/src/automations/triggers.ts
@@ -94,7 +94,7 @@ export async function externalTrigger(
   automation: Automation,
   params: { fields: Record<string, any>; timeout?: number },
   { getResponses }: { getResponses?: boolean } = {}
-) {
+): Promise<any> {
   if (
     automation.definition != null &&
     automation.definition.trigger != null &&
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 6877561fcb..3a14a87d2a 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -510,13 +510,14 @@ class TestConfiguration {
     // create dev app
     // clear any old app
     this.appId = null
-    await context.doInAppContext(null, async () => {
-      this.app = await this._req(
+    this.app = await context.doInAppContext(null, async () => {
+      const app = await this._req(
         { name: appName },
         null,
         controllers.app.create
       )
-      this.appId = this.app?.appId!
+      this.appId = app.appId!
+      return app
     })
     return await context.doInAppContext(this.appId, async () => {
       // create production app
@@ -525,7 +526,7 @@ class TestConfiguration {
       this.allApps.push(this.prodApp)
       this.allApps.push(this.app)
 
-      return this.app
+      return this.app!
     })
   }
 
@@ -537,7 +538,7 @@ class TestConfiguration {
 
     return context.doInAppContext(prodAppId, async () => {
       const db = context.getProdAppDB()
-      return await db.get(dbCore.DocumentType.APP_METADATA)
+      return await db.get<App>(dbCore.DocumentType.APP_METADATA)
     })
   }
 
diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts
index 9241289e86..d1fcc2be72 100644
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@@ -241,7 +241,7 @@ class Orchestrator {
     })
   }
 
-  async execute() {
+  async execute(): Promise<any> {
     // this will retrieve from context created at start of thread
     this._context.env = await sdkUtils.getEnvironmentVariables()
     let automation = this._automation

From b03719ff1c72509e7edd75f7a1eb82695c658c80 Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Wed, 8 Nov 2023 17:34:28 +0100
Subject: [PATCH 067/155] Update pro submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 34cce46d9c..cbd3c7e333 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 34cce46d9c67f03aa66272e079e1d460a731225e
+Subproject commit cbd3c7e33389c976dc73934667beb65d60c605f3

From f35daee07f43120554991be35456a31a804df2c1 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Wed, 8 Nov 2023 16:59:00 +0000
Subject: [PATCH 068/155] Added toggle-all functionality to the formblock and
 update context to the ComponentSettingsSection

---
 .../controls/EditComponentPopover.svelte      |   2 +
 .../FieldConfiguration/CellEditor.svelte      |  26 ---
 .../FieldConfiguration/ColumnDrawer.svelte    | 202 ------------------
 .../FieldConfiguration.svelte                 |  58 ++++-
 .../FieldConfiguration/FieldSetting.svelte    |  10 +-
 .../Component/ComponentSettingsSection.svelte |  19 ++
 6 files changed, 84 insertions(+), 233 deletions(-)
 delete mode 100644 packages/builder/src/components/design/settings/controls/FieldConfiguration/CellEditor.svelte
 delete mode 100644 packages/builder/src/components/design/settings/controls/FieldConfiguration/ColumnDrawer.svelte

diff --git a/packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte b/packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte
index 26c1ced502..d0f6f24362 100644
--- a/packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte
+++ b/packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte
@@ -11,6 +11,7 @@
   export let componentBindings
   export let bindings
   export let parseSettings
+  export let disabled
 
   const draggable = getContext("draggable")
   const dispatch = createEventDispatcher()
@@ -90,6 +91,7 @@
       open = true
     }
   }}
+  {disabled}
 />
 
 <Popover
diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/CellEditor.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/CellEditor.svelte
deleted file mode 100644
index e70decc035..0000000000
--- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/CellEditor.svelte
+++ /dev/null
@@ -1,26 +0,0 @@
-<script>
-  import { DrawerContent, Drawer, Button, Icon } from "@budibase/bbui"
-  import ValidationDrawer from "components/design/settings/controls/ValidationEditor/ValidationDrawer.svelte"
-  export let column
-  export let type
-
-  let drawer
-</script>
-
-<Icon name="Settings" hoverable size="S" on:click={drawer.show} />
-<Drawer bind:this={drawer} title="Field Validation">
-  <svelte:fragment slot="description">
-    "{column.name}" field validation
-  </svelte:fragment>
-  <Button cta slot="buttons" on:click={drawer.hide}>Save</Button>
-  <DrawerContent slot="body">
-    <div class="container">
-      <ValidationDrawer
-        slot="body"
-        bind:rules={column.validation}
-        fieldName={column.name}
-        {type}
-      />
-    </div>
-  </DrawerContent>
-</Drawer>
diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/ColumnDrawer.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/ColumnDrawer.svelte
deleted file mode 100644
index 316bf56da3..0000000000
--- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/ColumnDrawer.svelte
+++ /dev/null
@@ -1,202 +0,0 @@
-<script>
-  import {
-    Button,
-    Icon,
-    DrawerContent,
-    Layout,
-    Select,
-    Label,
-    Body,
-    Input,
-  } from "@budibase/bbui"
-  import { flip } from "svelte/animate"
-  import { dndzone } from "svelte-dnd-action"
-  import { generate } from "shortid"
-  import CellEditor from "./CellEditor.svelte"
-
-  export let columns = []
-  export let options = []
-  export let schema = {}
-
-  const flipDurationMs = 150
-  let dragDisabled = true
-
-  $: unselectedColumns = getUnselectedColumns(options, columns)
-  $: columns.forEach(column => {
-    if (!column.id) {
-      column.id = generate()
-    }
-  })
-
-  const getUnselectedColumns = (allColumns, selectedColumns) => {
-    let optionsObj = {}
-    allColumns.forEach(option => {
-      optionsObj[option] = true
-    })
-    selectedColumns?.forEach(column => {
-      delete optionsObj[column.name]
-    })
-    return Object.keys(optionsObj)
-  }
-
-  const getRemainingColumnOptions = selectedColumn => {
-    if (!selectedColumn || unselectedColumns.includes(selectedColumn)) {
-      return unselectedColumns
-    }
-    return [selectedColumn, ...unselectedColumns]
-  }
-
-  const addColumn = () => {
-    columns = [...columns, {}]
-  }
-
-  const removeColumn = id => {
-    columns = columns.filter(column => column.id !== id)
-  }
-
-  const updateColumnOrder = e => {
-    columns = e.detail.items
-  }
-
-  const handleFinalize = e => {
-    updateColumnOrder(e)
-    dragDisabled = true
-  }
-
-  const addAllColumns = () => {
-    let newColumns = columns || []
-    options.forEach(field => {
-      const fieldSchema = schema[field]
-      const hasCol = columns && columns.findIndex(x => x.name === field) !== -1
-      if (!fieldSchema?.autocolumn && !hasCol) {
-        newColumns.push({
-          name: field,
-          displayName: field,
-        })
-      }
-    })
-    columns = newColumns
-  }
-
-  const reset = () => {
-    columns = []
-  }
-
-  const getFieldType = column => {
-    return `validation/${schema[column.name]?.type}`
-  }
-</script>
-
-<DrawerContent>
-  <div class="container">
-    <Layout noPadding gap="S">
-      {#if columns?.length}
-        <Layout noPadding gap="XS">
-          <div class="column">
-            <div />
-            <Label size="L">Column</Label>
-            <Label size="L">Label</Label>
-            <div />
-            <div />
-          </div>
-          <div
-            class="columns"
-            use:dndzone={{
-              items: columns,
-              flipDurationMs,
-              dropTargetStyle: { outline: "none" },
-              dragDisabled,
-            }}
-            on:finalize={handleFinalize}
-            on:consider={updateColumnOrder}
-          >
-            {#each columns as column (column.id)}
-              <div class="column" animate:flip={{ duration: flipDurationMs }}>
-                <div
-                  class="handle"
-                  aria-label="drag-handle"
-                  style={dragDisabled ? "cursor: grab" : "cursor: grabbing"}
-                  on:mousedown={() => (dragDisabled = false)}
-                >
-                  <Icon name="DragHandle" size="XL" />
-                </div>
-                <Select
-                  bind:value={column.name}
-                  placeholder="Column"
-                  options={getRemainingColumnOptions(column.name)}
-                  on:change={e => (column.displayName = e.detail)}
-                />
-                <Input bind:value={column.displayName} placeholder="Label" />
-                <CellEditor type={getFieldType(column)} bind:column />
-                <Icon
-                  name="Close"
-                  hoverable
-                  size="S"
-                  on:click={() => removeColumn(column.id)}
-                  disabled={columns.length === 1}
-                />
-              </div>
-            {/each}
-          </div>
-        </Layout>
-      {:else}
-        <div class="column">
-          <div class="wide">
-            <Body size="S">Add columns to be included in your form below.</Body>
-          </div>
-        </div>
-      {/if}
-      <div class="column">
-        <div class="buttons wide">
-          <Button secondary icon="Add" on:click={addColumn}>Add column</Button>
-          <Button secondary quiet on:click={addAllColumns}>
-            Add all columns
-          </Button>
-          {#if columns?.length}
-            <Button secondary quiet on:click={reset}>Reset columns</Button>
-          {/if}
-        </div>
-      </div>
-    </Layout>
-  </div>
-</DrawerContent>
-
-<style>
-  .container {
-    width: 100%;
-    max-width: 600px;
-    margin: 0 auto;
-  }
-  .columns {
-    display: flex;
-    flex-direction: column;
-    justify-content: flex-start;
-    align-items: stretch;
-    gap: var(--spacing-m);
-  }
-  .column {
-    gap: var(--spacing-l);
-    display: grid;
-    grid-template-columns: 20px 1fr 1fr 16px 16px;
-    align-items: center;
-    border-radius: var(--border-radius-s);
-    transition: background-color ease-in-out 130ms;
-  }
-  .column:hover {
-    background-color: var(--spectrum-global-color-gray-100);
-  }
-  .handle {
-    display: grid;
-    place-items: center;
-  }
-  .wide {
-    grid-column: 2 / -1;
-  }
-  .buttons {
-    display: flex;
-    flex-direction: row;
-    justify-content: flex-start;
-    align-items: center;
-    gap: var(--spacing-m);
-  }
-</style>
diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
index 6c74705ab0..2211f4a41b 100644
--- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
@@ -1,4 +1,5 @@
 <script>
+  import { Toggle } from "@budibase/bbui"
   import { cloneDeep, isEqual } from "lodash/fp"
   import {
     getDatasourceForProvider,
@@ -8,7 +9,7 @@
   } from "builderStore/dataBinding"
   import { currentAsset } from "builderStore"
   import DraggableList from "../DraggableList/DraggableList.svelte"
-  import { createEventDispatcher } from "svelte"
+  import { createEventDispatcher, getContext } from "svelte"
   import { store, selectedScreen } from "builderStore"
   import FieldSetting from "./FieldSetting.svelte"
   import { convertOldFieldFormat, getComponentForField } from "./utils"
@@ -16,6 +17,8 @@
   export let componentInstance
   export let value
 
+  const updates = getContext("settings")
+
   const dispatch = createEventDispatcher()
   let sanitisedFields
   let fieldList
@@ -25,6 +28,9 @@
   let sanitisedValue
   let unconfigured
 
+  let selectAll = true
+  let updating = false
+
   $: bindings = getBindableProperties($selectedScreen, componentInstance._id)
   $: actionType = componentInstance.actionType
   let componentBindings = []
@@ -43,6 +49,10 @@
     cachedValue = cloneDeep(value)
   }
 
+  $: if (typeof $updates.resp == "string") {
+    updating = false
+  }
+
   const updateState = value => {
     schema = getSchema($currentAsset, datasource)
     options = Object.keys(schema || {})
@@ -145,16 +155,37 @@
     dispatch("change", getValidColumns(parentFieldsUpdated, options))
   }
 
-  const listUpdated = e => {
-    const parsedColumns = getValidColumns(e.detail, options)
+  const listUpdated = columns => {
+    const parsedColumns = getValidColumns(columns, options)
     dispatch("change", parsedColumns)
   }
+
+  const toggleAll = update => {
+    updating = true
+    listUpdated(update)
+  }
 </script>
 
 <div class="field-configuration">
+  <div class="toggle-all">
+    <span />
+    <Toggle
+      on:change={() => {
+        let update = fieldList.map(field => ({
+          ...field,
+          active: selectAll,
+        }))
+        toggleAll(update)
+      }}
+      text=""
+      bind:value={selectAll}
+      thin
+      disabled={updating}
+    />
+  </div>
   {#if fieldList?.length}
     <DraggableList
-      on:change={listUpdated}
+      on:change={e => listUpdated(e.detail)}
       on:itemChange={processItemUpdate}
       items={fieldList}
       listItemKey={"_id"}
@@ -162,7 +193,9 @@
       listTypeProps={{
         componentBindings,
         bindings,
+        updating,
       }}
+      draggable={!updating}
     />
   {/if}
 </div>
@@ -171,4 +204,21 @@
   .field-configuration :global(.spectrum-ActionButton) {
     width: 100%;
   }
+  .toggle-all {
+    display: flex;
+    justify-content: space-between;
+  }
+  .toggle-all :global(.spectrum-Switch) {
+    margin-right: 0px;
+    padding-right: calc(var(--spacing-s) - 1px);
+    min-height: unset;
+  }
+  .toggle-all :global(.spectrum-Switch .spectrum-Switch-switch) {
+    margin-top: 0px;
+  }
+  .toggle-all span {
+    color: var(--spectrum-global-color-gray-700);
+    font-size: 12px;
+    margin-left: calc(var(--spacing-s) - 1px);
+  }
 </style>
diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte
index 1d9ce733b8..75e6599a6c 100644
--- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte
+++ b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte
@@ -11,6 +11,7 @@
   export let componentBindings
   export let bindings
   export let anchor
+  export let updating //or disabled
 
   const dispatch = createEventDispatcher()
   const onToggle = item => {
@@ -49,6 +50,7 @@
       {bindings}
       {parseSettings}
       on:change
+      disabled={updating}
     >
       <div slot="header" class="type-icon">
         <Icon name={componentDef.icon} />
@@ -58,7 +60,13 @@
     <div class="field-label">{readableText}</div>
   </div>
   <div class="list-item-right">
-    <Toggle on:change={onToggle(item)} text="" value={item.active} thin />
+    <Toggle
+      on:change={onToggle(item)}
+      text=""
+      value={item.active}
+      thin
+      disabled={updating}
+    />
   </div>
 </div>
 
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
index 65f010e4ec..3483f6d26f 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
@@ -8,6 +8,8 @@
   import { getComponentForSetting } from "components/design/settings/componentSettings"
   import InfoDisplay from "./InfoDisplay.svelte"
   import analytics, { Events } from "analytics"
+  import { setContext } from "svelte"
+  import { writable } from "svelte/store"
 
   export let componentDefinition
   export let componentInstance
@@ -19,6 +21,21 @@
   export let includeHidden = false
   export let tag
 
+  let status = writable({
+    status: null,
+    lastUpdate: null,
+  })
+
+  const updateStatus = resp => {
+    status.update(state => ({
+      ...state,
+      resp,
+      lastUpdate: new Date().getTime(),
+    }))
+  }
+
+  setContext("settings", status)
+
   $: sections = getSections(
     componentInstance,
     componentDefinition,
@@ -76,6 +93,7 @@
       } else {
         await store.actions.components.updateSetting(setting.key, value)
       }
+      updateStatus("success")
       // Send event if required
       if (setting.sendEvents) {
         analytics.captureEvent(Events.COMPONENT_UPDATED, {
@@ -85,6 +103,7 @@
         })
       }
     } catch (error) {
+      updateStatus("failed")
       notifications.error("Error updating component prop")
     }
   }

From e4bc6a5fc78448b1c447874a321bf2dd021ec647 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 8 Nov 2023 18:41:15 +0000
Subject: [PATCH 069/155] Attempting to fix QA-core inconsistencies in CI -
 disabling rate limiting during test runs for public API.

---
 .../server/src/api/routes/public/index.ts     | 72 ++++++++++---------
 packages/server/src/environment.ts            |  1 +
 qa-core/package.json                          |  2 +-
 3 files changed, 42 insertions(+), 33 deletions(-)

diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts
index b37ed931fc..f27f3f8857 100644
--- a/packages/server/src/api/routes/public/index.ts
+++ b/packages/server/src/api/routes/public/index.ts
@@ -27,51 +27,59 @@ interface KoaRateLimitOptions {
 }
 
 const PREFIX = "/api/public/v1"
-// allow a lot more requests when in test
-const DEFAULT_API_REQ_LIMIT_PER_SEC = env.isTest() ? 100 : 10
 
-function getApiLimitPerSecond(): number {
-  if (!env.API_REQ_LIMIT_PER_SEC) {
-    return DEFAULT_API_REQ_LIMIT_PER_SEC
-  }
-  return parseInt(env.API_REQ_LIMIT_PER_SEC)
-}
+// type can't be known - untyped libraries
+let limiter: any, rateLimitStore: any
+if (!env.DISABLE_RATE_LIMITING) {
+  // allow a lot more requests when in test
+  const DEFAULT_API_REQ_LIMIT_PER_SEC = env.isTest() ? 100 : 10
 
-let rateLimitStore: any = null
-if (!env.isTest()) {
-  const { password, host, port } = redis.utils.getRedisConnectionDetails()
-  let options: KoaRateLimitOptions = {
-    socket: {
-      host: host,
-      port: port,
-    },
+  function getApiLimitPerSecond(): number {
+    if (!env.API_REQ_LIMIT_PER_SEC) {
+      return DEFAULT_API_REQ_LIMIT_PER_SEC
+    }
+    return parseInt(env.API_REQ_LIMIT_PER_SEC)
   }
 
-  if (password) {
-    options.password = password
-  }
+  if (!env.isTest()) {
+    const { password, host, port } = redis.utils.getRedisConnectionDetails()
+    let options: KoaRateLimitOptions = {
+      socket: {
+        host: host,
+        port: port,
+      },
+    }
 
-  if (!env.REDIS_CLUSTERED) {
-    // Can't set direct redis db in clustered env
-    options.database = SelectableDatabase.RATE_LIMITING
+    if (password) {
+      options.password = password
+    }
+
+    if (!env.REDIS_CLUSTERED) {
+      // Can't set direct redis db in clustered env
+      options.database = SelectableDatabase.RATE_LIMITING
+    }
+    rateLimitStore = new Stores.Redis(options)
+    RateLimit.defaultOptions({
+      store: rateLimitStore,
+    })
   }
-  rateLimitStore = new Stores.Redis(options)
-  RateLimit.defaultOptions({
-    store: rateLimitStore,
+  // rate limiting, allows for 2 requests per second
+  limiter = RateLimit.middleware({
+    interval: { sec: 1 },
+    // per ip, per interval
+    max: getApiLimitPerSecond(),
   })
+} else {
+  console.log("**** PUBLIC API RATE LIMITING DISABLED ****")
 }
-// rate limiting, allows for 2 requests per second
-const limiter = RateLimit.middleware({
-  interval: { sec: 1 },
-  // per ip, per interval
-  max: getApiLimitPerSecond(),
-})
 
 const publicRouter = new Router({
   prefix: PREFIX,
 })
 
-publicRouter.use(limiter)
+if (limiter) {
+  publicRouter.use(limiter)
+}
 
 function addMiddleware(
   endpoints: any,
diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts
index 91424113ac..c126a61c22 100644
--- a/packages/server/src/environment.ts
+++ b/packages/server/src/environment.ts
@@ -61,6 +61,7 @@ const environment = {
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,
   DISABLE_AUTOMATION_LOGS: process.env.DISABLE_AUTOMATION_LOGS,
+  DISABLE_RATE_LIMITING: process.env.DISABLE_RATE_LIMITING,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   SELF_HOSTED: process.env.SELF_HOSTED,
diff --git a/qa-core/package.json b/qa-core/package.json
index cfccd5e650..87172d2ed9 100644
--- a/qa-core/package.json
+++ b/qa-core/package.json
@@ -20,7 +20,7 @@
     "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.licensing\\.",
     "serve:test:self:ci": "start-server-and-test dev:built http://localhost:4001/health test:self:ci",
     "serve": "start-server-and-test dev:built http://localhost:4001/health",
-    "dev:built": "cd ../ && yarn dev:built"
+    "dev:built": "cd ../ && DISABLE_RATE_LIMITING=1 yarn dev:built"
   },
   "devDependencies": {
     "@budibase/types": "^2.3.17",

From 8436a8d22055d47cf5f331cc4e65259ae5d87e5d Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Thu, 9 Nov 2023 09:54:05 +0100
Subject: [PATCH 070/155] Day passes are optional

---
 packages/types/src/sdk/licensing/quota.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts
index 85700f167b..2d33c14182 100644
--- a/packages/types/src/sdk/licensing/quota.ts
+++ b/packages/types/src/sdk/licensing/quota.ts
@@ -61,7 +61,7 @@ export type PlanQuotas = { [key in PlanType]: Quotas | undefined }
 export type MonthlyQuotas = {
   [MonthlyQuotaName.QUERIES]: Quota
   [MonthlyQuotaName.AUTOMATIONS]: Quota
-  [MonthlyQuotaName.DAY_PASSES]: Quota
+  [MonthlyQuotaName.DAY_PASSES]?: Quota
 }
 
 export type StaticQuotas = {

From e1ca1d67322779eae0857028ea715a899933da14 Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Thu, 9 Nov 2023 09:55:09 +0100
Subject: [PATCH 071/155] Update pro submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index cbd3c7e333..ab17824872 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit cbd3c7e33389c976dc73934667beb65d60c605f3
+Subproject commit ab17824872a08010886abdc5893b14d3430efe00

From 311ad6b6a5010435fd56778151f85356e4ee01aa Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Thu, 9 Nov 2023 10:10:04 +0100
Subject: [PATCH 072/155] Update pro submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index ab17824872..18f558efb0 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit ab17824872a08010886abdc5893b14d3430efe00
+Subproject commit 18f558efb0f2d4a021a46d6236d12d93f6ce474f

From 7afcfbac93fc172e3b0a754e101f06de90d600bc Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 9 Nov 2023 09:27:06 +0000
Subject: [PATCH 073/155] Bump version to 2.13.6

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 5605642877..ce3128165f 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.13.5",
+  "version": "2.13.6",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 63b3bee311e4f329b6b4bb28e114944e3963ca49 Mon Sep 17 00:00:00 2001
From: Mel O'Hagan <mel@budibase.com>
Date: Thu, 9 Nov 2023 09:52:07 +0000
Subject: [PATCH 074/155] fix

---
 packages/client/src/components/app/Text.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/components/app/Text.svelte b/packages/client/src/components/app/Text.svelte
index 6c16db25fd..1037725ff8 100644
--- a/packages/client/src/components/app/Text.svelte
+++ b/packages/client/src/components/app/Text.svelte
@@ -94,7 +94,7 @@
   .align--right {
     text-align: right;
   }
-  .align-justify {
+  .align--justify {
     text-align: justify;
   }
 </style>

From 1f85b56a99400b3cda5735379a98a359279180a5 Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Thu, 9 Nov 2023 11:44:36 +0100
Subject: [PATCH 075/155] Revert "Day passes are optional"

This reverts commit 8436a8d22055d47cf5f331cc4e65259ae5d87e5d.
---
 packages/types/src/sdk/licensing/quota.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts
index 2d33c14182..85700f167b 100644
--- a/packages/types/src/sdk/licensing/quota.ts
+++ b/packages/types/src/sdk/licensing/quota.ts
@@ -61,7 +61,7 @@ export type PlanQuotas = { [key in PlanType]: Quotas | undefined }
 export type MonthlyQuotas = {
   [MonthlyQuotaName.QUERIES]: Quota
   [MonthlyQuotaName.AUTOMATIONS]: Quota
-  [MonthlyQuotaName.DAY_PASSES]?: Quota
+  [MonthlyQuotaName.DAY_PASSES]: Quota
 }
 
 export type StaticQuotas = {

From 5ddc3faf22a7100cf83a286ae372967b83a268bb Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Thu, 9 Nov 2023 11:45:38 +0100
Subject: [PATCH 076/155] Upgrade pro submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 18f558efb0..837bbc7b33 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 18f558efb0f2d4a021a46d6236d12d93f6ce474f
+Subproject commit 837bbc7b3317c5681fa22f2b92a8d1a7d39c3c95

From 14c6e1bc4a4cf9c3f68cda4b300e8e511e15e73d Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Thu, 9 Nov 2023 12:00:21 +0100
Subject: [PATCH 077/155] Update pro submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 837bbc7b33..e202f415d9 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 837bbc7b3317c5681fa22f2b92a8d1a7d39c3c95
+Subproject commit e202f415d9fa540d08cc2ba6e27394fbc22f357b

From 3f69b17c94a6ca45f88764d1ecfb1306b1afc07f Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 9 Nov 2023 11:05:42 +0000
Subject: [PATCH 078/155] Fully type the worker redis utils file.

---
 .../tests/core/utilities/structures/users.ts  |   2 +-
 .../src/api/controllers/global/users.ts       |  15 +-
 .../src/api/routes/global/tests/users.spec.ts |  21 ++-
 packages/worker/src/sdk/auth/auth.ts          |   2 +-
 packages/worker/src/sdk/users/users.ts        |   2 +
 packages/worker/src/utilities/email.ts        |   6 +-
 packages/worker/src/utilities/redis.ts        | 162 ++++++++++--------
 7 files changed, 124 insertions(+), 86 deletions(-)

diff --git a/packages/backend-core/tests/core/utilities/structures/users.ts b/packages/backend-core/tests/core/utilities/structures/users.ts
index 66d23696e0..68ee29686c 100644
--- a/packages/backend-core/tests/core/utilities/structures/users.ts
+++ b/packages/backend-core/tests/core/utilities/structures/users.ts
@@ -12,7 +12,7 @@ import { generator } from "./generator"
 import { tenant } from "."
 
 export const newEmail = () => {
-  return `${uuid()}@test.com`
+  return `${uuid()}@example.com`
 }
 
 export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts
index de1a605890..1f4168c00b 100644
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@@ -1,5 +1,6 @@
 import {
-  checkInviteCode,
+  getInviteCode,
+  deleteInviteCode,
   getInviteCodes,
   updateInviteCode,
 } from "../../../utilities/redis"
@@ -336,10 +337,11 @@ export const checkInvite = async (ctx: any) => {
   const { code } = ctx.params
   let invite
   try {
-    invite = await checkInviteCode(code, false)
+    invite = await getInviteCode(code)
   } catch (e) {
     console.warn("Error getting invite from code", e)
     ctx.throw(400, "There was a problem with the invite")
+    return
   }
   ctx.body = {
     email: invite.email,
@@ -365,12 +367,10 @@ export const updateInvite = async (ctx: any) => {
 
   let invite
   try {
-    invite = await checkInviteCode(code, false)
-    if (!invite) {
-      throw new Error("The invite could not be retrieved")
-    }
+    invite = await getInviteCode(code)
   } catch (e) {
     ctx.throw(400, "There was a problem with the invite")
+    return
   }
 
   let updated = {
@@ -405,7 +405,8 @@ export const inviteAccept = async (
   const { inviteCode, password, firstName, lastName } = ctx.request.body
   try {
     // info is an extension of the user object that was stored by global
-    const { email, info }: any = await checkInviteCode(inviteCode)
+    const { email, info }: any = await getInviteCode(inviteCode)
+    await deleteInviteCode(inviteCode)
     const user = await tenancy.doInTenant(info.tenantId, async () => {
       let request: any = {
         firstName,
diff --git a/packages/worker/src/api/routes/global/tests/users.spec.ts b/packages/worker/src/api/routes/global/tests/users.spec.ts
index 846b98a7ae..adcbca9d29 100644
--- a/packages/worker/src/api/routes/global/tests/users.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/users.spec.ts
@@ -1,11 +1,12 @@
 import { InviteUsersResponse, User } from "@budibase/types"
 
-jest.mock("nodemailer")
 import { TestConfiguration, mocks, structures } from "../../../../tests"
-const sendMailMock = mocks.email.mock()
 import { events, tenancy, accounts as _accounts } from "@budibase/backend-core"
 import * as userSdk from "../../../../sdk/users"
 
+jest.mock("nodemailer")
+const sendMailMock = mocks.email.mock()
+
 const accounts = jest.mocked(_accounts)
 
 describe("/api/global/users", () => {
@@ -54,6 +55,22 @@ describe("/api/global/users", () => {
       expect(events.user.invited).toBeCalledTimes(0)
     })
 
+    it("should not invite the same user twice", async () => {
+      const email = structures.users.newEmail()
+      await config.api.users.sendUserInvite(sendMailMock, email)
+
+      const { code, res } = await config.api.users.sendUserInvite(
+        sendMailMock,
+        email,
+        400
+      )
+
+      expect(res.body.message).toBe(`Unavailable`)
+      expect(sendMailMock).toHaveBeenCalledTimes(0)
+      expect(code).toBeUndefined()
+      expect(events.user.invited).toBeCalledTimes(0)
+    })
+
     it("should be able to create new user from invite", async () => {
       const email = structures.users.newEmail()
       const { code } = await config.api.users.sendUserInvite(
diff --git a/packages/worker/src/sdk/auth/auth.ts b/packages/worker/src/sdk/auth/auth.ts
index d989113c3d..e25d34fa5e 100644
--- a/packages/worker/src/sdk/auth/auth.ts
+++ b/packages/worker/src/sdk/auth/auth.ts
@@ -73,7 +73,7 @@ export const reset = async (email: string) => {
  * Perform the user password update if the provided reset code is valid.
  */
 export const resetUpdate = async (resetCode: string, password: string) => {
-  const { userId } = await redis.checkResetPasswordCode(resetCode)
+  const { userId } = await redis.getResetPasswordCode(resetCode)
 
   let user = await userSdk.db.getUser(userId)
   user.password = password
diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts
index 230a8fa146..8f04bb1941 100644
--- a/packages/worker/src/sdk/users/users.ts
+++ b/packages/worker/src/sdk/users/users.ts
@@ -2,6 +2,7 @@ import { events, tenancy, users as usersCore } from "@budibase/backend-core"
 import { InviteUsersRequest, InviteUsersResponse } from "@budibase/types"
 import { sendEmail } from "../../utilities/email"
 import { EmailTemplatePurpose } from "../../constants"
+import { getInviteCodes } from "../..//utilities/redis"
 
 export async function invite(
   users: InviteUsersRequest
@@ -14,6 +15,7 @@ export async function invite(
   const matchedEmails = await usersCore.searchExistingEmails(
     users.map(u => u.email)
   )
+  const existingInvites = await getInviteCodes()
   const newUsers = []
 
   // separate duplicates from new users
diff --git a/packages/worker/src/utilities/email.ts b/packages/worker/src/utilities/email.ts
index c5b1d9d8ab..a0c02d335c 100644
--- a/packages/worker/src/utilities/email.ts
+++ b/packages/worker/src/utilities/email.ts
@@ -3,7 +3,7 @@ import { EmailTemplatePurpose, TemplateType } from "../constants"
 import { getTemplateByPurpose, EmailTemplates } from "../constants/templates"
 import { getSettingsTemplateContext } from "./templates"
 import { processString } from "@budibase/string-templates"
-import { getResetPasswordCode, getInviteCode } from "./redis"
+import { createResetPasswordCode, createInviteCode } from "./redis"
 import { User, SendEmailOpts, SMTPInnerConfig } from "@budibase/types"
 import { configs } from "@budibase/backend-core"
 import ical from "ical-generator"
@@ -61,9 +61,9 @@ async function getLinkCode(
 ) {
   switch (purpose) {
     case EmailTemplatePurpose.PASSWORD_RECOVERY:
-      return getResetPasswordCode(user._id!, info)
+      return createResetPasswordCode(user._id!, info)
     case EmailTemplatePurpose.INVITATION:
-      return getInviteCode(email, info)
+      return createInviteCode(email, info)
     default:
       return null
   }
diff --git a/packages/worker/src/utilities/redis.ts b/packages/worker/src/utilities/redis.ts
index 993cdf97ce..9852fb0467 100644
--- a/packages/worker/src/utilities/redis.ts
+++ b/packages/worker/src/utilities/redis.ts
@@ -1,60 +1,24 @@
 import { redis, utils, tenancy } from "@budibase/backend-core"
 import env from "../environment"
 
-function getExpirySecondsForDB(db: string) {
-  switch (db) {
-    case redis.utils.Databases.PW_RESETS:
-      // a hour
-      return 3600
-    case redis.utils.Databases.INVITATIONS:
-      // a week
-      return 604800
-  }
+interface Invite {
+  email: string
+  info: any
 }
 
-let pwResetClient: any, invitationClient: any
-
-function getClient(db: string) {
-  switch (db) {
-    case redis.utils.Databases.PW_RESETS:
-      return pwResetClient
-    case redis.utils.Databases.INVITATIONS:
-      return invitationClient
-  }
+interface InviteWithCode extends Invite {
+  code: string
 }
 
-async function writeACode(db: string, value: any) {
-  const client = await getClient(db)
-  const code = utils.newid()
-  await client.store(code, value, getExpirySecondsForDB(db))
-  return code
+interface PasswordReset {
+  userId: string
+  info: any
 }
 
-async function updateACode(db: string, code: string, value: any) {
-  const client = await getClient(db)
-  await client.store(code, value, getExpirySecondsForDB(db))
-}
-
-/**
- * Given an invite code and invite body, allow the update an existing/valid invite in redis
- * @param inviteCode The invite code for an invite in redis
- * @param value The body of the updated user invitation
- */
-export async function updateInviteCode(inviteCode: string, value: string) {
-  await updateACode(redis.utils.Databases.INVITATIONS, inviteCode, value)
-}
-
-async function getACode(db: string, code: string, deleteCode = true) {
-  const client = await getClient(db)
-  const value = await client.get(code)
-  if (!value) {
-    throw new Error("Invalid code.")
-  }
-  if (deleteCode) {
-    await client.delete(code)
-  }
-  return value
-}
+type RedisDBName =
+  | redis.utils.Databases.PW_RESETS
+  | redis.utils.Databases.INVITATIONS
+let pwResetClient: redis.Client, invitationClient: redis.Client
 
 export async function init() {
   pwResetClient = new redis.Client(redis.utils.Databases.PW_RESETS)
@@ -63,9 +27,6 @@ export async function init() {
   await invitationClient.init()
 }
 
-/**
- * make sure redis connection is closed.
- */
 export async function shutdown() {
   if (pwResetClient) await pwResetClient.finish()
   if (invitationClient) await invitationClient.finish()
@@ -74,6 +35,65 @@ export async function shutdown() {
   console.log("Redis shutdown")
 }
 
+function getExpirySecondsForDB(db: RedisDBName) {
+  switch (db) {
+    case redis.utils.Databases.PW_RESETS:
+      // a hour
+      return 3600
+    case redis.utils.Databases.INVITATIONS:
+      // a week
+      return 604800
+    default:
+      throw new Error(`Unknown redis database: ${db}`)
+  }
+}
+
+function getClient(db: RedisDBName): redis.Client {
+  switch (db) {
+    case redis.utils.Databases.PW_RESETS:
+      return pwResetClient
+    case redis.utils.Databases.INVITATIONS:
+      return invitationClient
+    default:
+      throw new Error(`Unknown redis database: ${db}`)
+  }
+}
+
+async function writeCode(db: RedisDBName, value: Invite | PasswordReset) {
+  const client = getClient(db)
+  const code = utils.newid()
+  await client.store(code, value, getExpirySecondsForDB(db))
+  return code
+}
+
+async function updateCode(db: RedisDBName, code: string, value: any) {
+  const client = getClient(db)
+  await client.store(code, value, getExpirySecondsForDB(db))
+}
+
+/**
+ * Given an invite code and invite body, allow the update an existing/valid invite in redis
+ * @param inviteCode The invite code for an invite in redis
+ * @param value The body of the updated user invitation
+ */
+export async function updateInviteCode(code: string, value: Invite) {
+  await updateCode(redis.utils.Databases.INVITATIONS, code, value)
+}
+
+async function deleteCode(db: RedisDBName, code: string) {
+  const client = getClient(db)
+  await client.delete(code)
+}
+
+async function getCode(db: RedisDBName, code: string) {
+  const client = getClient(db)
+  const value = await client.get(code)
+  if (!value) {
+    throw new Error(`Could not find code: ${code}`)
+  }
+  return value
+}
+
 /**
  * Given a user ID this will store a code (that is returned) for an hour in redis.
  * The user can then return this code for resetting their password (through their reset link).
@@ -81,22 +101,20 @@ export async function shutdown() {
  * @param info Info about the user/the reset process.
  * @return returns the code that was stored to redis.
  */
-export async function getResetPasswordCode(userId: string, info: any) {
-  return writeACode(redis.utils.Databases.PW_RESETS, { userId, info })
+export async function createResetPasswordCode(userId: string, info: any) {
+  return writeCode(redis.utils.Databases.PW_RESETS, { userId, info })
 }
 
 /**
- * Given a reset code this will lookup to redis, check if the code is valid and delete if required.
+ * Given a reset code this will lookup to redis, check if the code is valid.
  * @param resetCode The code provided via the email link.
- * @param deleteCode If the code is used/finished with this will delete it - defaults to true.
  * @return returns the user ID if it is found
  */
-export async function checkResetPasswordCode(
-  resetCode: string,
-  deleteCode = true
-) {
+export async function getResetPasswordCode(
+  code: string
+): Promise<PasswordReset> {
   try {
-    return getACode(redis.utils.Databases.PW_RESETS, resetCode, deleteCode)
+    return getCode(redis.utils.Databases.PW_RESETS, code)
   } catch (err) {
     throw "Provided information is not valid, cannot reset password - please try again."
   }
@@ -108,35 +126,35 @@ export async function checkResetPasswordCode(
  * @param info Information to be carried along with the invitation.
  * @return returns the code that was stored to redis.
  */
-export async function getInviteCode(email: string, info: any) {
-  return writeACode(redis.utils.Databases.INVITATIONS, { email, info })
+export async function createInviteCode(email: string, info: any) {
+  return writeCode(redis.utils.Databases.INVITATIONS, { email, info })
 }
 
 /**
  * Checks that the provided invite code is valid - will return the email address of user that was invited.
  * @param inviteCode the invite code that was provided as part of the link.
- * @param deleteCode whether or not the code should be deleted after retrieval - defaults to true.
  * @return If the code is valid then an email address will be returned.
  */
-export async function checkInviteCode(
-  inviteCode: string,
-  deleteCode: boolean = true
-) {
+export async function getInviteCode(code: string): Promise<Invite> {
   try {
-    return getACode(redis.utils.Databases.INVITATIONS, inviteCode, deleteCode)
+    return getCode(redis.utils.Databases.INVITATIONS, code)
   } catch (err) {
     throw "Invitation is not valid or has expired, please request a new one."
   }
 }
 
+export async function deleteInviteCode(code: string) {
+  return deleteCode(redis.utils.Databases.INVITATIONS, code)
+}
+
 /** 
   Get all currently available user invitations for the current tenant.
 **/
-export async function getInviteCodes() {
-  const client = await getClient(redis.utils.Databases.INVITATIONS)
-  const invites: any[] = await client.scan()
+export async function getInviteCodes(): Promise<InviteWithCode[]> {
+  const client = getClient(redis.utils.Databases.INVITATIONS)
+  const invites: { key: string; value: Invite }[] = await client.scan()
 
-  const results = invites.map(invite => {
+  const results: InviteWithCode[] = invites.map(invite => {
     return {
       ...invite.value,
       code: invite.key,

From a6a75b533c0ea587516200691d75afb3b33c387c Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 9 Nov 2023 11:15:44 +0000
Subject: [PATCH 079/155] Reject inviting the same user twice.

---
 .../worker/src/api/controllers/global/users.ts  |  4 +---
 .../src/api/routes/global/tests/users.spec.ts   |  2 ++
 packages/worker/src/sdk/users/users.ts          | 17 +++++++++++++----
 packages/worker/src/utilities/redis.ts          |  6 +++++-
 4 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts
index 1f4168c00b..fae42acdfe 100644
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@@ -349,14 +349,12 @@ export const checkInvite = async (ctx: any) => {
 }
 
 export const getUserInvites = async (ctx: any) => {
-  let invites
   try {
     // Restricted to the currently authenticated tenant
-    invites = await getInviteCodes()
+    ctx.body = await getInviteCodes()
   } catch (e) {
     ctx.throw(400, "There was a problem fetching invites")
   }
-  ctx.body = invites
 }
 
 export const updateInvite = async (ctx: any) => {
diff --git a/packages/worker/src/api/routes/global/tests/users.spec.ts b/packages/worker/src/api/routes/global/tests/users.spec.ts
index adcbca9d29..09cd4d358f 100644
--- a/packages/worker/src/api/routes/global/tests/users.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/users.spec.ts
@@ -59,6 +59,8 @@ describe("/api/global/users", () => {
       const email = structures.users.newEmail()
       await config.api.users.sendUserInvite(sendMailMock, email)
 
+      jest.clearAllMocks()
+
       const { code, res } = await config.api.users.sendUserInvite(
         sendMailMock,
         email,
diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts
index 8f04bb1941..4cca0b8fa6 100644
--- a/packages/worker/src/sdk/users/users.ts
+++ b/packages/worker/src/sdk/users/users.ts
@@ -1,5 +1,9 @@
 import { events, tenancy, users as usersCore } from "@budibase/backend-core"
-import { InviteUsersRequest, InviteUsersResponse } from "@budibase/types"
+import {
+  InviteUserRequest,
+  InviteUsersRequest,
+  InviteUsersResponse,
+} from "@budibase/types"
 import { sendEmail } from "../../utilities/email"
 import { EmailTemplatePurpose } from "../../constants"
 import { getInviteCodes } from "../..//utilities/redis"
@@ -15,12 +19,17 @@ export async function invite(
   const matchedEmails = await usersCore.searchExistingEmails(
     users.map(u => u.email)
   )
-  const existingInvites = await getInviteCodes()
-  const newUsers = []
+  const invitedEmails = (await getInviteCodes()).map(invite => invite.email)
+  const newUsers: InviteUserRequest[] = []
 
   // separate duplicates from new users
   for (let user of users) {
-    if (matchedEmails.includes(user.email)) {
+    if (
+      matchedEmails.includes(user.email) ||
+      invitedEmails.includes(user.email)
+    ) {
+      // This "Unavailable" is load bearing. The tests and frontend both check for it
+      // specifically
       response.unsuccessful.push({ email: user.email, reason: "Unavailable" })
     } else {
       newUsers.push(user)
diff --git a/packages/worker/src/utilities/redis.ts b/packages/worker/src/utilities/redis.ts
index 9852fb0467..337d34e376 100644
--- a/packages/worker/src/utilities/redis.ts
+++ b/packages/worker/src/utilities/redis.ts
@@ -66,7 +66,11 @@ async function writeCode(db: RedisDBName, value: Invite | PasswordReset) {
   return code
 }
 
-async function updateCode(db: RedisDBName, code: string, value: any) {
+async function updateCode(
+  db: RedisDBName,
+  code: string,
+  value: Invite | PasswordReset
+) {
   const client = getClient(db)
   await client.store(code, value, getExpirySecondsForDB(db))
 }

From b2841b30b23270139cd9560b9520758d78671f44 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 9 Nov 2023 11:17:30 +0000
Subject: [PATCH 080/155] Add a test for the multi-invite endpoint.

---
 .../src/api/routes/global/tests/users.spec.ts   | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/packages/worker/src/api/routes/global/tests/users.spec.ts b/packages/worker/src/api/routes/global/tests/users.spec.ts
index 09cd4d358f..ce2c9347b4 100644
--- a/packages/worker/src/api/routes/global/tests/users.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/users.spec.ts
@@ -120,6 +120,23 @@ describe("/api/global/users", () => {
       expect(sendMailMock).toHaveBeenCalledTimes(0)
       expect(events.user.invited).toBeCalledTimes(0)
     })
+
+    it("should not be able to generate an invitation for user that has already been invited", async () => {
+      const email = structures.users.newEmail()
+      await config.api.users.sendUserInvite(sendMailMock, email)
+
+      jest.clearAllMocks()
+
+      const request = [{ email: email, userInfo: {} }]
+      const res = await config.api.users.sendMultiUserInvite(request)
+
+      const body = res.body as InviteUsersResponse
+      expect(body.successful.length).toBe(0)
+      expect(body.unsuccessful.length).toBe(1)
+      expect(body.unsuccessful[0].reason).toBe("Unavailable")
+      expect(sendMailMock).toHaveBeenCalledTimes(0)
+      expect(events.user.invited).toBeCalledTimes(0)
+    })
   })
 
   describe("POST /api/global/users/bulk", () => {

From ee91d6753c438229affc681e13cf54b4d8ab1760 Mon Sep 17 00:00:00 2001
From: Mel O'Hagan <mel@budibase.com>
Date: Thu, 9 Nov 2023 11:55:10 +0000
Subject: [PATCH 081/155] Encode query string

---
 .../builder/src/components/integration/RestQueryViewer.svelte | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/integration/RestQueryViewer.svelte b/packages/builder/src/components/integration/RestQueryViewer.svelte
index e6913b0953..9634cd9746 100644
--- a/packages/builder/src/components/integration/RestQueryViewer.svelte
+++ b/packages/builder/src/components/integration/RestQueryViewer.svelte
@@ -404,7 +404,7 @@
     datasource = $datasources.list.find(ds => ds._id === query?.datasourceId)
     const datasourceUrl = datasource?.config.url
     const qs = query?.fields.queryString
-    breakQs = restUtils.breakQueryString(qs)
+    breakQs = restUtils.breakQueryString(encodeURI(qs))
     breakQs = runtimeToReadableMap(mergedBindings, breakQs)
 
     const path = query.fields.path
@@ -652,7 +652,7 @@
     <div class="bottom">
       <Layout paddingY="S" gap="S">
         <Divider />
-        {#if !response && Object.keys(schema).length === 0}
+        {#if !response && Object.keys(schema || {}).length === 0}
           <Heading size="M">Response</Heading>
           <div class="placeholder">
             <div class="placeholder-internal">

From 08b96ad2131dacd72e7042cd5c98110afc8f1fe0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Nov 2023 12:55:50 +0100
Subject: [PATCH 082/155] Remove unnused pipelines

---
 .github/workflows/deploy-cloud.yaml       |  48 ------
 .github/workflows/release-master.yml      | 178 ----------------------
 .github/workflows/release-selfhost.yml    | 125 ---------------
 .github/workflows/release-singleimage.yml |  86 -----------
 4 files changed, 437 deletions(-)
 delete mode 100644 .github/workflows/deploy-cloud.yaml
 delete mode 100644 .github/workflows/release-master.yml
 delete mode 100644 .github/workflows/release-selfhost.yml
 delete mode 100644 .github/workflows/release-singleimage.yml

diff --git a/.github/workflows/deploy-cloud.yaml b/.github/workflows/deploy-cloud.yaml
deleted file mode 100644
index 389b10f7d3..0000000000
--- a/.github/workflows/deploy-cloud.yaml
+++ /dev/null
@@ -1,48 +0,0 @@
-name: Budibase Deploy Production
-
-on:
-  workflow_dispatch:
-    inputs:
-      version:
-        description: Budibase release version. For example - 1.0.0
-        required: false
-
-jobs:
-  release:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Fail if not a tag
-        run: |
-          if [[ $GITHUB_REF != refs/tags/* ]]; then
-            echo "Workflow Dispatch can only be run on tags"
-            exit 1
-          fi
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Fail if tag is not in master
-        run: |
-          if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
-            echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
-            exit 1
-          fi
-
-      - name: Get the latest budibase release version
-        id: version
-        run: |
-          if [ -z "${{ github.event.inputs.version }}" ]; then
-            release_version=$(cat lerna.json | jq -r '.version')
-          else
-            release_version=${{ github.event.inputs.version }}
-          fi
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-
-      - uses: passeidireto/trigger-external-workflow-action@main
-        env:
-          PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }}
-        with:
-          repository: budibase/budibase-deploys
-          event: budicloud-prod-deploy
-          github_pat: ${{ secrets.GH_ACCESS_TOKEN }}
diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
deleted file mode 100644
index 2edb470405..0000000000
--- a/.github/workflows/release-master.yml
+++ /dev/null
@@ -1,178 +0,0 @@
-name: Budibase Release
-concurrency:
-  group: release
-  cancel-in-progress: false
-
-on:
-  push:
-    tags:
-      - "[0-9]+.[0-9]+.[0-9]+"
-      # Exclude all pre-releases
-      - "!*[0-9]+.[0-9]+.[0-9]+-*"
-
-env:
-  # Posthog token used by ui at build time
-  POSTHOG_TOKEN: phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
-  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
-  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-
-jobs:
-  release-images:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          fetch-depth: 0
-
-      - name: Fail if tag is not in master
-        run: |
-          if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
-            echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
-            exit 1
-          fi
-
-      - uses: actions/setup-node@v1
-        with:
-          node-version: 18.x
-          cache: yarn
-
-      - run: yarn install --frozen-lockfile
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - run: yarn lint
-      - run: yarn build
-      - run: yarn build:sdk
-
-      - name: Publish budibase packages to NPM
-        env:
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-        run: |
-          # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default
-          git config --global user.name "Budibase Release Bot"
-          git config --global user.email "<>"
-          git submodule foreach git commit -a -m 'Release process'
-          git commit -a -m 'Release process'
-          echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc
-          yarn release
-
-      - name: "Get Current tag"
-        id: currenttag
-        run: |
-          version=$(./scripts/getCurrentVersion.sh)
-          echo "Using tag $version"
-          echo "version=$version" >> "$GITHUB_OUTPUT"
-
-      - name: Setup Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Docker login
-        run: |
-          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-
-      - name: Build worker docker
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          platforms: linux/amd64,linux/arm64
-          build-args: |
-            BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }}
-          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
-          file: ./packages/worker/Dockerfile.v2
-          cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
-          cache-to: type=inline
-        env:
-          IMAGE_NAME: budibase/worker
-          IMAGE_TAG: ${{ steps.currenttag.outputs.version }}
-          BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }}
-
-      - name: Build server docker
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          platforms: linux/amd64,linux/arm64
-          build-args: |
-            BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }}
-          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
-          file: ./packages/server/Dockerfile.v2
-          cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
-          cache-to: type=inline
-        env:
-          IMAGE_NAME: budibase/apps
-          IMAGE_TAG: ${{ steps.currenttag.outputs.version }}
-          BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }}
-
-      - name: Build proxy docker
-        uses: docker/build-push-action@v5
-        with:
-          context: ./hosting/proxy
-          push: true
-          platforms: linux/amd64,linux/arm64
-          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
-          file: ./hosting/proxy/Dockerfile
-          cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
-          cache-to: type=inline
-        env:
-          IMAGE_NAME: budibase/proxy
-          IMAGE_TAG: ${{ steps.currenttag.outputs.version }}
-
-  release-helm-chart:
-    needs: [release-images]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup Helm
-        uses: azure/setup-helm@v1
-        id: helm-install
-
-      - name: Get the latest budibase release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-
-      # due to helm repo index issue: https://github.com/helm/helm/issues/7363
-      # we need to create new package in a different dir, merge the index and move the package back
-      - name: Build and release helm chart
-        run: |
-          git config user.name "Budibase Helm Bot"
-          git config user.email "<>"
-          git reset --hard
-          git fetch
-          mkdir sync
-          echo "Packaging chart to sync dir"
-          helm package charts/budibase --version 0.0.0-master --app-version "$RELEASE_VERSION" --destination sync
-          echo "Packaging successful"
-          git checkout gh-pages
-          echo "Indexing helm repo"         
-          helm repo index --merge docs/index.yaml sync
-          mv -f sync/* docs
-          rm -rf sync
-          echo "Pushing new helm release"
-          git add -A
-          git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}"
-          git push
-
-  trigger-deploy-to-qa-env:
-    needs: [release-helm-chart]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - uses: peter-evans/repository-dispatch@v2
-        with:
-          repository: budibase/budibase-deploys
-          event-type: budicloud-qa-deploy
-          token: ${{ secrets.GH_ACCESS_TOKEN }}
-          client-payload: |-
-            {
-              "VERSION": "${{ github.ref_name }}",
-              "REF_NAME": "${{ github.ref_name}}"
-            }
diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml
deleted file mode 100644
index d2689a0ea0..0000000000
--- a/.github/workflows/release-selfhost.yml
+++ /dev/null
@@ -1,125 +0,0 @@
-name: Budibase Release Selfhost
-
-on:
-  workflow_dispatch:
-
-jobs:
-  release:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Fail if not a tag
-        run: |
-          if [[ $GITHUB_REF != refs/tags/* ]]; then 
-            echo "Workflow Dispatch can only be run on tags" 
-            exit 1 
-          fi
-
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          fetch-depth: 0
-
-      - name: Fail if tag is not in master
-        run: |
-          if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
-            echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
-            exit 1
-          fi
-
-      - name: Use Node.js 18.x
-        uses: actions/setup-node@v1
-        with:
-          node-version: 18.x
-
-      - name: Get the latest budibase release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-
-      - name: Tag and release Docker images (Self Host)
-        run: |
-          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-
-          release_tag=${{ env.RELEASE_VERSION }}
-
-          # Pull apps and worker images
-          docker pull budibase/apps:$release_tag
-          docker pull budibase/worker:$release_tag
-          docker pull budibase/proxy:$release_tag
-
-          # Tag apps and worker images
-          docker tag budibase/apps:$release_tag budibase/apps:$SELFHOST_TAG
-          docker tag budibase/worker:$release_tag budibase/worker:$SELFHOST_TAG
-          docker tag budibase/proxy:$release_tag budibase/proxy:$SELFHOST_TAG
-
-          # Push images
-          docker push budibase/apps:$SELFHOST_TAG
-          docker push budibase/worker:$SELFHOST_TAG
-          docker push budibase/proxy:$SELFHOST_TAG
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          SELFHOST_TAG: latest
-
-      - name: Bootstrap and build (CLI)
-        run: |
-          yarn
-          yarn build
-
-      - name: Build OpenAPI spec
-        run: |
-          pushd packages/server
-          yarn
-          yarn specs
-          popd
-
-      - name: Setup Helm
-        uses: azure/setup-helm@v1
-        id: helm-install
-
-      # due to helm repo index issue: https://github.com/helm/helm/issues/7363
-      # we need to create new package in a different dir, merge the index and move the package back
-      - name: Build and release helm chart
-        run: |
-          git config user.name "Budibase Helm Bot"
-          git config user.email "<>"
-          git reset --hard
-          git fetch
-          mkdir sync
-          echo "Packaging chart to sync dir"
-          helm package charts/budibase --version "$RELEASE_VERSION" --app-version "$RELEASE_VERSION" --destination sync
-          echo "Packaging successful"
-          git checkout gh-pages
-          echo "Indexing helm repo"
-          helm repo index --merge docs/index.yaml sync
-          mv -f sync/* docs
-          rm -rf sync
-          echo "Pushing new helm release"
-          git add -A
-          git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}"
-          git push
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Perform Github Release
-        uses: softprops/action-gh-release@v1
-        with:
-          name: ${{ env.RELEASE_VERSION }}
-          tag_name: ${{ env.RELEASE_VERSION }}
-          generate_release_notes: true
-          files: |
-            packages/cli/build/cli-win.exe
-            packages/cli/build/cli-linux
-            packages/cli/build/cli-macos 
-            packages/server/specs/openapi.yaml
-            packages/server/specs/openapi.json
-
-      - name: Discord Webhook Action
-        uses: tsickert/discord-webhook@v4.0.0
-        with:
-          webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
-          content: "Self Host Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Self Host."
-          embed-title: ${{ env.RELEASE_VERSION }}
diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml
deleted file mode 100644
index 16b1da186a..0000000000
--- a/.github/workflows/release-singleimage.yml
+++ /dev/null
@@ -1,86 +0,0 @@
-name: Deploy Budibase Single Container Image to DockerHub
-
-on:
-  workflow_dispatch:
-
-env:
-  CI: true
-  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-  REGISTRY_URL: registry.hub.docker.com
-jobs:
-  build:
-    name: "build"
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [18.x]
-    steps:
-      - name: Maximize build space
-        uses: easimon/maximize-build-space@master
-        with:
-          root-reserve-mb: 30000
-          swap-size-mb: 1024
-          remove-android: "true"
-          remove-dotnet: "true"
-      - name: Fail if not a tag
-        run: |
-          if [[ $GITHUB_REF != refs/tags/* ]]; then 
-            echo "Workflow Dispatch can only be run on tags" 
-            exit 1 
-          fi
-      - name: "Checkout"
-        uses: actions/checkout@v2
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
-        with:
-          node-version: ${{ matrix.node-version }}
-      - name: Setup QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Setup Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Run Yarn
-        run: yarn
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - name: Run Yarn Build
-        run: yarn build
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_API_KEY }}
-      - name: Get the latest release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo $release_version
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-      - name: Tag and release Budibase service docker image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          platforms: linux/amd64,linux/arm64
-          build-args: BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }}
-          tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile.v2
-        env:
-          BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }}
-      - name: Tag and release Budibase Azure App Service docker image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          platforms: linux/amd64
-          build-args: |
-            TARGETBUILD=aas
-            BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }}
-          tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile.v2
-        env:
-          BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }}

From a26f2e83e49a1acc24f3b262666ec273e7c4d470 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 9 Nov 2023 14:45:23 +0000
Subject: [PATCH 083/155] Some of the new table getters did not account for
 table IDs not being found, adding a new function which properly accounts for
 IDs being missing and manages it correctly, rather than leaving tables in an
 undefined state.

---
 packages/backend-core/src/db/couch/DatabaseImpl.ts | 13 ++++++++++++-
 .../server/src/api/controllers/row/internal.ts     | 11 ++++-------
 packages/server/src/api/controllers/row/utils.ts   | 14 +-------------
 packages/server/src/db/linkedRows/index.ts         |  6 ++----
 packages/server/src/db/utils.ts                    | 10 ----------
 packages/server/src/sdk/app/tables/getters.ts      | 14 +++++++++-----
 packages/server/src/utilities/global.ts            |  6 ++----
 packages/types/src/sdk/db.ts                       |  3 ++-
 8 files changed, 32 insertions(+), 45 deletions(-)

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index 1ace60ed5b..96b946fe16 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -109,7 +109,7 @@ export class DatabaseImpl implements Database {
     }
   }
 
-  async get<T>(id?: string): Promise<T | any> {
+  async get<T extends Document>(id?: string): Promise<T> {
     const db = await this.checkSetup()
     if (!id) {
       throw new Error("Unable to get doc without a valid _id.")
@@ -117,6 +117,17 @@ export class DatabaseImpl implements Database {
     return this.updateOutput(() => db.get(id))
   }
 
+  async getMultiple<T extends Document>(ids: string[]): Promise<T[]> {
+    // get unique
+    ids = [...new Set(ids)]
+    const response = await this.allDocs<T>({
+      keys: ids,
+      include_docs: true,
+    })
+    const rows = response.rows.filter(row => row.error !== "not_found")
+    return rows.map(row => row.doc!)
+  }
+
   async remove(idOrDoc: string | Document, rev?: string) {
     const db = await this.checkSetup()
     let _id: string
diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts
index fe7d94547a..0edbe4ea86 100644
--- a/packages/server/src/api/controllers/row/internal.ts
+++ b/packages/server/src/api/controllers/row/internal.ts
@@ -1,9 +1,5 @@
 import * as linkRows from "../../../db/linkedRows"
-import {
-  generateRowID,
-  getMultiIDParams,
-  InternalTables,
-} from "../../../db/utils"
+import { generateRowID, InternalTables } from "../../../db/utils"
 import * as userController from "../user"
 import {
   cleanupAttachments,
@@ -240,8 +236,9 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
   const linkVals = links as LinkDocumentValue[]
 
   // look up the actual rows based on the ids
-  const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id))
-  let linkedRows = (await db.allDocs<Row>(params)).rows.map(row => row.doc!)
+  let linkedRows = await db.getMultiple<Row>(
+    linkVals.map(linkVal => linkVal.id)
+  )
 
   // get the linked tables
   const linkTableIds = getLinkedTableIDs(table as Table)
diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts
index cd311fdf0f..ed6ccd4c53 100644
--- a/packages/server/src/api/controllers/row/utils.ts
+++ b/packages/server/src/api/controllers/row/utils.ts
@@ -1,21 +1,9 @@
 import { InternalTables } from "../../../db/utils"
 import * as userController from "../user"
 import { context } from "@budibase/backend-core"
-import {
-  Ctx,
-  FieldType,
-  ManyToOneRelationshipFieldMetadata,
-  OneToManyRelationshipFieldMetadata,
-  Row,
-  SearchFilters,
-  Table,
-  UserCtx,
-} from "@budibase/types"
-import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
-import sdk from "../../../sdk"
+import { Ctx, Row, UserCtx } from "@budibase/types"
 
 import validateJs from "validate.js"
-import { cloneDeep } from "lodash/fp"
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts
index 7324fa1d94..24ff47ead3 100644
--- a/packages/server/src/db/linkedRows/index.ts
+++ b/packages/server/src/db/linkedRows/index.ts
@@ -8,7 +8,7 @@ import {
   getLinkedTable,
 } from "./linkUtils"
 import flatten from "lodash/flatten"
-import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils"
+import { USER_METDATA_PREFIX } from "../utils"
 import partition from "lodash/partition"
 import { getGlobalUsersFromMetadata } from "../../utilities/global"
 import { processFormulas } from "../../utilities/rowProcessor"
@@ -79,9 +79,7 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) {
   const db = context.getAppDB()
   const linkedRowIds = links.map(link => link.id)
   const uniqueRowIds = [...new Set(linkedRowIds)]
-  let dbRows = (await db.allDocs<Row>(getMultiIDParams(uniqueRowIds))).rows.map(
-    row => row.doc!
-  )
+  let dbRows = await db.getMultiple<Row>(uniqueRowIds)
   // convert the unique db rows back to a full list of linked rows
   const linked = linkedRowIds
     .map(id => dbRows.find(row => row && row._id === id))
diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts
index 715db552c9..a5569f8166 100644
--- a/packages/server/src/db/utils.ts
+++ b/packages/server/src/db/utils.ts
@@ -283,16 +283,6 @@ export function generatePluginID(name: string) {
   return `${DocumentType.PLUGIN}${SEPARATOR}${name}`
 }
 
-/**
- * This can be used with the db.allDocs to get a list of IDs
- */
-export function getMultiIDParams(ids: string[]) {
-  return {
-    keys: ids,
-    include_docs: true,
-  }
-}
-
 /**
  * Generates a new view ID.
  * @returns The new view ID which the view doc can be stored under.
diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts
index a7074f95b2..a09d686b6d 100644
--- a/packages/server/src/sdk/app/tables/getters.ts
+++ b/packages/server/src/sdk/app/tables/getters.ts
@@ -1,5 +1,5 @@
 import { context } from "@budibase/backend-core"
-import { getMultiIDParams, getTableParams } from "../../../db/utils"
+import { getTableParams } from "../../../db/utils"
 import {
   breakExternalTableId,
   isExternalTableID,
@@ -17,6 +17,9 @@ import datasources from "../datasources"
 import sdk from "../../../sdk"
 
 export function processTable(table: Table): Table {
+  if (!table) {
+    return table
+  }
   if (table._id && isExternalTableID(table._id)) {
     return {
       ...table,
@@ -73,6 +76,9 @@ export async function getExternalTable(
   tableName: string
 ): Promise<Table> {
   const entities = await getExternalTablesInDatasource(datasourceId)
+  if (!entities[tableName]) {
+    throw new Error(`Unable to find table named "${tableName}"`)
+  }
   return processTable(entities[tableName])
 }
 
@@ -124,10 +130,8 @@ export async function getTables(tableIds: string[]): Promise<Table[]> {
   }
   if (internalTableIds.length) {
     const db = context.getAppDB()
-    const internalTableDocs = await db.allDocs<Table>(
-      getMultiIDParams(internalTableIds)
-    )
-    tables = tables.concat(internalTableDocs.rows.map(row => row.doc!))
+    const internalTables = await db.getMultiple<Table>(internalTableIds)
+    tables = tables.concat(internalTables)
   }
   return processTables(tables)
 }
diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts
index cdc2d84513..762d1fb7a0 100644
--- a/packages/server/src/utilities/global.ts
+++ b/packages/server/src/utilities/global.ts
@@ -1,4 +1,4 @@
-import { getMultiIDParams, getGlobalIDFromUserMetadataID } from "../db/utils"
+import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import {
   roles,
   db as dbCore,
@@ -96,9 +96,7 @@ export async function getRawGlobalUsers(userIds?: string[]): Promise<User[]> {
   const db = tenancy.getGlobalDB()
   let globalUsers: User[]
   if (userIds) {
-    globalUsers = (await db.allDocs<User>(getMultiIDParams(userIds))).rows.map(
-      row => row.doc!
-    )
+    globalUsers = await db.getMultiple<User>(userIds)
   } else {
     globalUsers = (
       await db.allDocs<User>(
diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts
index 26807d99ce..9ea854b3aa 100644
--- a/packages/types/src/sdk/db.ts
+++ b/packages/types/src/sdk/db.ts
@@ -122,7 +122,8 @@ export interface Database {
 
   exists(): Promise<boolean>
   checkSetup(): Promise<Nano.DocumentScope<any>>
-  get<T>(id?: string): Promise<T>
+  get<T extends Document>(id?: string): Promise<T>
+  getMultiple<T extends Document>(ids: string[]): Promise<T[]>
   remove(
     id: string | Document,
     rev?: string

From b29cfc600cf628fae8b4740bdabc530cbb8f2e21 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 9 Nov 2023 14:51:07 +0000
Subject: [PATCH 084/155] Move Invite and PasswordReset code into backend-core.

---
 packages/backend-core/src/redis/index.ts      |   2 +
 packages/backend-core/src/redis/invite.ts     |  96 ++++++++++
 .../backend-core/src/redis/passwordReset.ts   |  51 ++++++
 packages/backend-core/src/users/lookup.ts     |   4 +
 .../src/api/controllers/global/users.ts       |   2 +-
 packages/worker/src/index.ts                  |   3 -
 packages/worker/src/sdk/auth/auth.ts          |   4 +-
 packages/worker/src/sdk/users/users.ts        |   7 +-
 packages/worker/src/utilities/email.ts        |   3 +-
 packages/worker/src/utilities/redis.ts        | 172 ------------------
 10 files changed, 159 insertions(+), 185 deletions(-)
 create mode 100644 packages/backend-core/src/redis/invite.ts
 create mode 100644 packages/backend-core/src/redis/passwordReset.ts
 delete mode 100644 packages/worker/src/utilities/redis.ts

diff --git a/packages/backend-core/src/redis/index.ts b/packages/backend-core/src/redis/index.ts
index 6585d6e4fa..8d153ea5a1 100644
--- a/packages/backend-core/src/redis/index.ts
+++ b/packages/backend-core/src/redis/index.ts
@@ -4,3 +4,5 @@ export { default as Client } from "./redis"
 export * as utils from "./utils"
 export * as clients from "./init"
 export * as locks from "./redlockImpl"
+export * as invite from "./invite"
+export * as passwordReset from "./passwordReset"
diff --git a/packages/backend-core/src/redis/invite.ts b/packages/backend-core/src/redis/invite.ts
new file mode 100644
index 0000000000..db36d3dfa6
--- /dev/null
+++ b/packages/backend-core/src/redis/invite.ts
@@ -0,0 +1,96 @@
+import { redis, utils, tenancy } from "../"
+import env from "../environment"
+
+const TTL_SECONDS = 60 * 60 * 24 * 7
+
+interface Invite {
+  email: string
+  info: any
+}
+
+interface InviteWithCode extends Invite {
+  code: string
+}
+
+let client: redis.Client
+
+async function getClient(): Promise<redis.Client> {
+  if (!client) {
+    client = new redis.Client(redis.utils.Databases.INVITATIONS)
+    await client.init()
+  }
+  return client
+}
+
+/**
+ * Given an invite code and invite body, allow the update an existing/valid invite in redis
+ * @param inviteCode The invite code for an invite in redis
+ * @param value The body of the updated user invitation
+ */
+export async function updateInviteCode(code: string, value: Invite) {
+  const client = await getClient()
+  await client.store(code, value, TTL_SECONDS)
+}
+
+/**
+ * Generates an invitation code and writes it to redis - which can later be checked for user creation.
+ * @param email the email address which the code is being sent to (for use later).
+ * @param info Information to be carried along with the invitation.
+ * @return returns the code that was stored to redis.
+ */
+export async function createInviteCode(
+  email: string,
+  info: any
+): Promise<string> {
+  const client = await getClient()
+  const code = utils.newid()
+  await client.store(code, { email, info }, TTL_SECONDS)
+  return code
+}
+
+/**
+ * Checks that the provided invite code is valid - will return the email address of user that was invited.
+ * @param inviteCode the invite code that was provided as part of the link.
+ * @return If the code is valid then an email address will be returned.
+ */
+export async function getInviteCode(code: string): Promise<Invite> {
+  const client = await getClient()
+  const value = (await client.get(code)) as Invite | undefined
+  if (!value) {
+    throw "Invitation is not valid or has expired, please request a new one."
+  }
+  return value
+}
+
+export async function deleteInviteCode(code: string) {
+  const client = await getClient()
+  await client.delete(code)
+}
+
+/** 
+  Get all currently available user invitations for the current tenant.
+**/
+export async function getInviteCodes(): Promise<InviteWithCode[]> {
+  const client = await getClient()
+  const invites: { key: string; value: Invite }[] = await client.scan()
+
+  const results: InviteWithCode[] = invites.map(invite => {
+    return {
+      ...invite.value,
+      code: invite.key,
+    }
+  })
+  if (!env.MULTI_TENANCY) {
+    return results
+  }
+  const tenantId = tenancy.getTenantId()
+  return results.filter(invite => tenantId === invite.info.tenantId)
+}
+
+export async function getExistingInvites(
+  emails: string[]
+): Promise<InviteWithCode[]> {
+  return (await getInviteCodes()).filter(invite =>
+    emails.includes(invite.email)
+  )
+}
diff --git a/packages/backend-core/src/redis/passwordReset.ts b/packages/backend-core/src/redis/passwordReset.ts
new file mode 100644
index 0000000000..63c3371bba
--- /dev/null
+++ b/packages/backend-core/src/redis/passwordReset.ts
@@ -0,0 +1,51 @@
+import { redis, utils } from "../"
+
+const TTL_SECONDS = 60 * 60
+
+interface PasswordReset {
+  userId: string
+  info: any
+}
+
+let client: redis.Client
+
+async function getClient(): Promise<redis.Client> {
+  if (!client) {
+    client = new redis.Client(redis.utils.Databases.PW_RESETS)
+    await client.init()
+  }
+  return client
+}
+
+/**
+ * Given a user ID this will store a code (that is returned) for an hour in redis.
+ * The user can then return this code for resetting their password (through their reset link).
+ * @param userId the ID of the user which is to be reset.
+ * @param info Info about the user/the reset process.
+ * @return returns the code that was stored to redis.
+ */
+export async function createResetPasswordCode(
+  userId: string,
+  info: any
+): Promise<string> {
+  const client = await getClient()
+  const code = utils.newid()
+  await client.store(code, { userId, info }, TTL_SECONDS)
+  return code
+}
+
+/**
+ * Given a reset code this will lookup to redis, check if the code is valid.
+ * @param code The code provided via the email link.
+ * @return returns the user ID if it is found
+ */
+export async function getResetPasswordCode(
+  code: string
+): Promise<PasswordReset> {
+  const client = await getClient()
+  const value = (await client.get(code)) as PasswordReset | undefined
+  if (!value) {
+    throw "Provided information is not valid, cannot reset password - please try again."
+  }
+  return value
+}
diff --git a/packages/backend-core/src/users/lookup.ts b/packages/backend-core/src/users/lookup.ts
index 17d0e91d88..2c0c66276a 100644
--- a/packages/backend-core/src/users/lookup.ts
+++ b/packages/backend-core/src/users/lookup.ts
@@ -6,6 +6,7 @@ import {
 } from "@budibase/types"
 import * as dbUtils from "../db"
 import { ViewName } from "../constants"
+import { getExistingInvites } from "../redis/invite"
 
 /**
  * Apply a system-wide search on emails:
@@ -26,6 +27,9 @@ export async function searchExistingEmails(emails: string[]) {
   const existingAccounts = await getExistingAccounts(emails)
   matchedEmails.push(...existingAccounts.map(account => account.email))
 
+  const invitedEmails = await getExistingInvites(emails)
+  matchedEmails.push(...invitedEmails.map(invite => invite.email))
+
   return [...new Set(matchedEmails.map(email => email.toLowerCase()))]
 }
 
diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts
index fae42acdfe..8fb2b5675d 100644
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@@ -3,7 +3,7 @@ import {
   deleteInviteCode,
   getInviteCodes,
   updateInviteCode,
-} from "../../../utilities/redis"
+} from "@budibase/backend-core/src/redis/invite"
 import * as userSdk from "../../../sdk/users"
 import env from "../../../environment"
 import {
diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts
index 4b1d11ecf7..1c101983df 100644
--- a/packages/worker/src/index.ts
+++ b/packages/worker/src/index.ts
@@ -22,7 +22,6 @@ import Koa from "koa"
 import koaBody from "koa-body"
 import http from "http"
 import api from "./api"
-import * as redis from "./utilities/redis"
 
 const koaSession = require("koa-session")
 import { userAgent } from "koa-useragent"
@@ -72,7 +71,6 @@ server.on("close", async () => {
   shuttingDown = true
   console.log("Server Closed")
   timers.cleanup()
-  await redis.shutdown()
   await events.shutdown()
   await queue.shutdown()
   if (!env.isTest()) {
@@ -88,7 +86,6 @@ const shutdown = () => {
 export default server.listen(parseInt(env.PORT || "4002"), async () => {
   console.log(`Worker running on ${JSON.stringify(server.address())}`)
   await initPro()
-  await redis.init()
   // configure events to use the pro audit log write
   // can't integrate directly into backend-core due to cyclic issues
   await events.processors.init(proSdk.auditLogs.write)
diff --git a/packages/worker/src/sdk/auth/auth.ts b/packages/worker/src/sdk/auth/auth.ts
index e25d34fa5e..57131294b3 100644
--- a/packages/worker/src/sdk/auth/auth.ts
+++ b/packages/worker/src/sdk/auth/auth.ts
@@ -6,12 +6,12 @@ import {
   sessions,
   tenancy,
   utils as coreUtils,
+  redis,
 } from "@budibase/backend-core"
 import { PlatformLogoutOpts, User } from "@budibase/types"
 import jwt from "jsonwebtoken"
 import * as userSdk from "../users"
 import * as emails from "../../utilities/email"
-import * as redis from "../../utilities/redis"
 import { EmailTemplatePurpose } from "../../constants"
 
 // LOGIN / LOGOUT
@@ -73,7 +73,7 @@ export const reset = async (email: string) => {
  * Perform the user password update if the provided reset code is valid.
  */
 export const resetUpdate = async (resetCode: string, password: string) => {
-  const { userId } = await redis.getResetPasswordCode(resetCode)
+  const { userId } = await redis.passwordReset.getResetPasswordCode(resetCode)
 
   let user = await userSdk.db.getUser(userId)
   user.password = password
diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts
index 4cca0b8fa6..9dc1bac0e9 100644
--- a/packages/worker/src/sdk/users/users.ts
+++ b/packages/worker/src/sdk/users/users.ts
@@ -6,7 +6,6 @@ import {
 } from "@budibase/types"
 import { sendEmail } from "../../utilities/email"
 import { EmailTemplatePurpose } from "../../constants"
-import { getInviteCodes } from "../..//utilities/redis"
 
 export async function invite(
   users: InviteUsersRequest
@@ -19,15 +18,11 @@ export async function invite(
   const matchedEmails = await usersCore.searchExistingEmails(
     users.map(u => u.email)
   )
-  const invitedEmails = (await getInviteCodes()).map(invite => invite.email)
   const newUsers: InviteUserRequest[] = []
 
   // separate duplicates from new users
   for (let user of users) {
-    if (
-      matchedEmails.includes(user.email) ||
-      invitedEmails.includes(user.email)
-    ) {
+    if (matchedEmails.includes(user.email)) {
       // This "Unavailable" is load bearing. The tests and frontend both check for it
       // specifically
       response.unsuccessful.push({ email: user.email, reason: "Unavailable" })
diff --git a/packages/worker/src/utilities/email.ts b/packages/worker/src/utilities/email.ts
index a0c02d335c..f3f1fd40c4 100644
--- a/packages/worker/src/utilities/email.ts
+++ b/packages/worker/src/utilities/email.ts
@@ -3,7 +3,8 @@ import { EmailTemplatePurpose, TemplateType } from "../constants"
 import { getTemplateByPurpose, EmailTemplates } from "../constants/templates"
 import { getSettingsTemplateContext } from "./templates"
 import { processString } from "@budibase/string-templates"
-import { createResetPasswordCode, createInviteCode } from "./redis"
+import { createResetPasswordCode } from "@budibase/backend-core/src/redis/passwordReset"
+import { createInviteCode } from "@budibase/backend-core/src/redis/invite"
 import { User, SendEmailOpts, SMTPInnerConfig } from "@budibase/types"
 import { configs } from "@budibase/backend-core"
 import ical from "ical-generator"
diff --git a/packages/worker/src/utilities/redis.ts b/packages/worker/src/utilities/redis.ts
deleted file mode 100644
index 337d34e376..0000000000
--- a/packages/worker/src/utilities/redis.ts
+++ /dev/null
@@ -1,172 +0,0 @@
-import { redis, utils, tenancy } from "@budibase/backend-core"
-import env from "../environment"
-
-interface Invite {
-  email: string
-  info: any
-}
-
-interface InviteWithCode extends Invite {
-  code: string
-}
-
-interface PasswordReset {
-  userId: string
-  info: any
-}
-
-type RedisDBName =
-  | redis.utils.Databases.PW_RESETS
-  | redis.utils.Databases.INVITATIONS
-let pwResetClient: redis.Client, invitationClient: redis.Client
-
-export async function init() {
-  pwResetClient = new redis.Client(redis.utils.Databases.PW_RESETS)
-  invitationClient = new redis.Client(redis.utils.Databases.INVITATIONS)
-  await pwResetClient.init()
-  await invitationClient.init()
-}
-
-export async function shutdown() {
-  if (pwResetClient) await pwResetClient.finish()
-  if (invitationClient) await invitationClient.finish()
-  // shutdown core clients
-  await redis.clients.shutdown()
-  console.log("Redis shutdown")
-}
-
-function getExpirySecondsForDB(db: RedisDBName) {
-  switch (db) {
-    case redis.utils.Databases.PW_RESETS:
-      // a hour
-      return 3600
-    case redis.utils.Databases.INVITATIONS:
-      // a week
-      return 604800
-    default:
-      throw new Error(`Unknown redis database: ${db}`)
-  }
-}
-
-function getClient(db: RedisDBName): redis.Client {
-  switch (db) {
-    case redis.utils.Databases.PW_RESETS:
-      return pwResetClient
-    case redis.utils.Databases.INVITATIONS:
-      return invitationClient
-    default:
-      throw new Error(`Unknown redis database: ${db}`)
-  }
-}
-
-async function writeCode(db: RedisDBName, value: Invite | PasswordReset) {
-  const client = getClient(db)
-  const code = utils.newid()
-  await client.store(code, value, getExpirySecondsForDB(db))
-  return code
-}
-
-async function updateCode(
-  db: RedisDBName,
-  code: string,
-  value: Invite | PasswordReset
-) {
-  const client = getClient(db)
-  await client.store(code, value, getExpirySecondsForDB(db))
-}
-
-/**
- * Given an invite code and invite body, allow the update an existing/valid invite in redis
- * @param inviteCode The invite code for an invite in redis
- * @param value The body of the updated user invitation
- */
-export async function updateInviteCode(code: string, value: Invite) {
-  await updateCode(redis.utils.Databases.INVITATIONS, code, value)
-}
-
-async function deleteCode(db: RedisDBName, code: string) {
-  const client = getClient(db)
-  await client.delete(code)
-}
-
-async function getCode(db: RedisDBName, code: string) {
-  const client = getClient(db)
-  const value = await client.get(code)
-  if (!value) {
-    throw new Error(`Could not find code: ${code}`)
-  }
-  return value
-}
-
-/**
- * Given a user ID this will store a code (that is returned) for an hour in redis.
- * The user can then return this code for resetting their password (through their reset link).
- * @param userId the ID of the user which is to be reset.
- * @param info Info about the user/the reset process.
- * @return returns the code that was stored to redis.
- */
-export async function createResetPasswordCode(userId: string, info: any) {
-  return writeCode(redis.utils.Databases.PW_RESETS, { userId, info })
-}
-
-/**
- * Given a reset code this will lookup to redis, check if the code is valid.
- * @param resetCode The code provided via the email link.
- * @return returns the user ID if it is found
- */
-export async function getResetPasswordCode(
-  code: string
-): Promise<PasswordReset> {
-  try {
-    return getCode(redis.utils.Databases.PW_RESETS, code)
-  } catch (err) {
-    throw "Provided information is not valid, cannot reset password - please try again."
-  }
-}
-
-/**
- * Generates an invitation code and writes it to redis - which can later be checked for user creation.
- * @param email the email address which the code is being sent to (for use later).
- * @param info Information to be carried along with the invitation.
- * @return returns the code that was stored to redis.
- */
-export async function createInviteCode(email: string, info: any) {
-  return writeCode(redis.utils.Databases.INVITATIONS, { email, info })
-}
-
-/**
- * Checks that the provided invite code is valid - will return the email address of user that was invited.
- * @param inviteCode the invite code that was provided as part of the link.
- * @return If the code is valid then an email address will be returned.
- */
-export async function getInviteCode(code: string): Promise<Invite> {
-  try {
-    return getCode(redis.utils.Databases.INVITATIONS, code)
-  } catch (err) {
-    throw "Invitation is not valid or has expired, please request a new one."
-  }
-}
-
-export async function deleteInviteCode(code: string) {
-  return deleteCode(redis.utils.Databases.INVITATIONS, code)
-}
-
-/** 
-  Get all currently available user invitations for the current tenant.
-**/
-export async function getInviteCodes(): Promise<InviteWithCode[]> {
-  const client = getClient(redis.utils.Databases.INVITATIONS)
-  const invites: { key: string; value: Invite }[] = await client.scan()
-
-  const results: InviteWithCode[] = invites.map(invite => {
-    return {
-      ...invite.value,
-      code: invite.key,
-    }
-  })
-  if (!env.MULTI_TENANCY) {
-    return results
-  }
-  const tenantId = tenancy.getTenantId()
-  return results.filter(invite => tenantId === invite.info.tenantId)
-}

From 37e34c8ed2e79c1c5981a95e6402736643d94d1e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 9 Nov 2023 14:53:14 +0000
Subject: [PATCH 085/155] Adding the ability to fail on getMultiple if needed.

---
 packages/backend-core/src/db/couch/DatabaseImpl.ts | 14 ++++++++++++--
 packages/types/src/sdk/db.ts                       |  5 ++++-
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index 96b946fe16..fa505847af 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -117,14 +117,24 @@ export class DatabaseImpl implements Database {
     return this.updateOutput(() => db.get(id))
   }
 
-  async getMultiple<T extends Document>(ids: string[]): Promise<T[]> {
+  async getMultiple<T extends Document>(
+    ids: string[],
+    opts?: { failIfMissing?: boolean }
+  ): Promise<T[]> {
     // get unique
     ids = [...new Set(ids)]
     const response = await this.allDocs<T>({
       keys: ids,
       include_docs: true,
     })
-    const rows = response.rows.filter(row => row.error !== "not_found")
+    const NOT_FOUND = "not_found"
+    const rows = response.rows.filter(row => row.error !== NOT_FOUND)
+    // some were filtered out - means some missing
+    if (opts?.failIfMissing && rows.length !== response.rows.length) {
+      const missing = response.rows.filter(row => row.error === NOT_FOUND)
+      const missingIds = missing.map(row => row.key).join(", ")
+      throw new Error(`Unable to get documents: ${missingIds}`)
+    }
     return rows.map(row => row.doc!)
   }
 
diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts
index 9ea854b3aa..ff5742bcfc 100644
--- a/packages/types/src/sdk/db.ts
+++ b/packages/types/src/sdk/db.ts
@@ -123,7 +123,10 @@ export interface Database {
   exists(): Promise<boolean>
   checkSetup(): Promise<Nano.DocumentScope<any>>
   get<T extends Document>(id?: string): Promise<T>
-  getMultiple<T extends Document>(ids: string[]): Promise<T[]>
+  getMultiple<T extends Document>(
+    ids: string[],
+    opts?: { failIfMissing?: boolean }
+  ): Promise<T[]>
   remove(
     id: string | Document,
     rev?: string

From 822c03b0ef2d2bbbba996831a4c5cedf19e0d0bc Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 9 Nov 2023 15:02:44 +0000
Subject: [PATCH 086/155] Refactor onboardUsers endpoint.

---
 packages/types/src/api/web/user.ts            |  1 +
 .../src/api/controllers/global/users.ts       | 79 ++++++-------------
 2 files changed, 27 insertions(+), 53 deletions(-)

diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts
index 3a5bd16bdf..6db70f20d0 100644
--- a/packages/types/src/api/web/user.ts
+++ b/packages/types/src/api/web/user.ts
@@ -10,6 +10,7 @@ export interface SaveUserResponse {
 export interface UserDetails {
   _id: string
   email: string
+  password?: string
 }
 
 export interface BulkUserRequest {
diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts
index 8fb2b5675d..6608fcde05 100644
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@@ -251,58 +251,32 @@ export const tenantUserLookup = async (ctx: any) => {
   Encapsulate the app user onboarding flows here.
 */
 export const onboardUsers = async (ctx: Ctx<InviteUsersRequest>) => {
-  const request = ctx.request.body
-  const isBulkCreate = "create" in request
-
-  const emailConfigured = await isEmailConfigured()
-
-  let onboardingResponse
-
-  if (isBulkCreate) {
-    // @ts-ignore
-    const { users, groups, roles } = request.create
-    const assignUsers = users.map((user: User) => (user.roles = roles))
-    onboardingResponse = await userSdk.db.bulkCreate(assignUsers, groups)
-    ctx.body = onboardingResponse
-  } else if (emailConfigured) {
-    onboardingResponse = await inviteMultiple(ctx)
-  } else if (!emailConfigured) {
-    const inviteRequest = ctx.request.body
-
-    let createdPasswords: any = {}
-
-    const users: User[] = inviteRequest.map(invite => {
-      let password = Math.random().toString(36).substring(2, 22)
-
-      // Temp password to be passed to the user.
-      createdPasswords[invite.email] = password
-
-      return {
-        email: invite.email,
-        password,
-        forceResetPassword: true,
-        roles: invite.userInfo.apps,
-        admin: invite.userInfo.admin,
-        builder: invite.userInfo.builder,
-        tenantId: tenancy.getTenantId(),
-      }
-    })
-    let bulkCreateReponse = await userSdk.db.bulkCreate(users, [])
-
-    // Apply temporary credentials
-    ctx.body = {
-      ...bulkCreateReponse,
-      successful: bulkCreateReponse?.successful.map(user => {
-        return {
-          ...user,
-          password: createdPasswords[user.email],
-        }
-      }),
-      created: true,
-    }
-  } else {
-    ctx.throw(400, "User onboarding failed")
+  if (await isEmailConfigured()) {
+    await inviteMultiple(ctx)
+    return
   }
+
+  let createdPasswords: Record<string, string> = {}
+  const users: User[] = ctx.request.body.map(invite => {
+    let password = Math.random().toString(36).substring(2, 22)
+    createdPasswords[invite.email] = password
+
+    return {
+      email: invite.email,
+      password,
+      forceResetPassword: true,
+      roles: invite.userInfo.apps,
+      admin: invite.userInfo.admin,
+      builder: invite.userInfo.builder,
+      tenantId: tenancy.getTenantId(),
+    }
+  })
+
+  let resp = await userSdk.db.bulkCreate(users, [])
+  resp.successful.forEach(user => {
+    user.password = createdPasswords[user.email]
+  })
+  ctx.body = { ...resp, created: true }
 }
 
 export const invite = async (ctx: Ctx<InviteUserRequest>) => {
@@ -329,8 +303,7 @@ export const invite = async (ctx: Ctx<InviteUserRequest>) => {
 }
 
 export const inviteMultiple = async (ctx: Ctx<InviteUsersRequest>) => {
-  const request = ctx.request.body
-  ctx.body = await userSdk.invite(request)
+  ctx.body = await userSdk.invite(ctx.request.body)
 }
 
 export const checkInvite = async (ctx: any) => {

From 7f530eeab5970feda052576ad3c786b1f25673cf Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 9 Nov 2023 15:13:59 +0000
Subject: [PATCH 087/155] Add tests for the onboarding endpoint.

---
 packages/backend-core/src/users/db.ts         |  4 ++--
 packages/types/src/api/web/user.ts            |  1 +
 .../src/api/controllers/global/users.ts       | 11 ++++++----
 .../src/api/routes/global/tests/users.spec.ts | 21 +++++++++++++++++++
 packages/worker/src/tests/api/users.ts        | 21 +++++++++++++++++++
 5 files changed, 52 insertions(+), 6 deletions(-)

diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts
index 59f698d99c..bd85097bbd 100644
--- a/packages/backend-core/src/users/db.ts
+++ b/packages/backend-core/src/users/db.ts
@@ -303,7 +303,7 @@ export class UserDB {
 
   static async bulkCreate(
     newUsersRequested: User[],
-    groups: string[]
+    groups?: string[]
   ): Promise<BulkUserCreated> {
     const tenantId = getTenantId()
 
@@ -328,7 +328,7 @@ export class UserDB {
         })
         continue
       }
-      newUser.userGroups = groups
+      newUser.userGroups = groups || []
       newUsers.push(newUser)
       if (isCreator(newUser)) {
         newCreators.push(newUser)
diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts
index 6db70f20d0..0de42622e6 100644
--- a/packages/types/src/api/web/user.ts
+++ b/packages/types/src/api/web/user.ts
@@ -50,6 +50,7 @@ export type InviteUsersRequest = InviteUserRequest[]
 export interface InviteUsersResponse {
   successful: { email: string }[]
   unsuccessful: { email: string; reason: string }[]
+  created?: boolean
 }
 
 export interface SearchUsersRequest {
diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts
index 6608fcde05..e904567674 100644
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@@ -17,6 +17,7 @@ import {
   Ctx,
   InviteUserRequest,
   InviteUsersRequest,
+  InviteUsersResponse,
   MigrationType,
   SaveUserResponse,
   SearchUsersRequest,
@@ -250,7 +251,9 @@ export const tenantUserLookup = async (ctx: any) => {
 /* 
   Encapsulate the app user onboarding flows here.
 */
-export const onboardUsers = async (ctx: Ctx<InviteUsersRequest>) => {
+export const onboardUsers = async (
+  ctx: Ctx<InviteUsersRequest, InviteUsersResponse>
+) => {
   if (await isEmailConfigured()) {
     await inviteMultiple(ctx)
     return
@@ -272,10 +275,10 @@ export const onboardUsers = async (ctx: Ctx<InviteUsersRequest>) => {
     }
   })
 
-  let resp = await userSdk.db.bulkCreate(users, [])
-  resp.successful.forEach(user => {
+  let resp = await userSdk.db.bulkCreate(users)
+  for (const user of resp.successful) {
     user.password = createdPasswords[user.email]
-  })
+  }
   ctx.body = { ...resp, created: true }
 }
 
diff --git a/packages/worker/src/api/routes/global/tests/users.spec.ts b/packages/worker/src/api/routes/global/tests/users.spec.ts
index ce2c9347b4..a85933255a 100644
--- a/packages/worker/src/api/routes/global/tests/users.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/users.spec.ts
@@ -669,4 +669,25 @@ describe("/api/global/users", () => {
       expect(response.body.message).toBe("Unable to delete self.")
     })
   })
+
+  describe("POST /api/global/users/onboard", () => {
+    it("should successfully onboard a user", async () => {
+      const response = await config.api.users.onboardUser([
+        { email: structures.users.newEmail(), userInfo: {} },
+      ])
+      expect(response.successful.length).toBe(1)
+      expect(response.unsuccessful.length).toBe(0)
+    })
+
+    it("should not onboard a user who has been invited", async () => {
+      const email = structures.users.newEmail()
+      await config.api.users.sendUserInvite(sendMailMock, email)
+
+      const response = await config.api.users.onboardUser([
+        { email, userInfo: {} },
+      ])
+      expect(response.successful.length).toBe(0)
+      expect(response.unsuccessful.length).toBe(1)
+    })
+  })
 })
diff --git a/packages/worker/src/tests/api/users.ts b/packages/worker/src/tests/api/users.ts
index ca25e2f9ca..5ecd1447ca 100644
--- a/packages/worker/src/tests/api/users.ts
+++ b/packages/worker/src/tests/api/users.ts
@@ -5,6 +5,7 @@ import {
   User,
   CreateAdminUserRequest,
   SearchQuery,
+  InviteUsersResponse,
 } from "@budibase/types"
 import structures from "../structures"
 import { generator } from "@budibase/backend-core/tests"
@@ -176,4 +177,24 @@ export class UserAPI extends TestAPI {
       .expect("Content-Type", /json/)
       .expect(200)
   }
+
+  onboardUser = async (
+    req: InviteUsersRequest
+  ): Promise<InviteUsersResponse> => {
+    const resp = await this.request
+      .post(`/api/global/users/onboard`)
+      .send(req)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+
+    if (resp.status !== 200) {
+      throw new Error(
+        `request failed with status ${resp.status} and body ${JSON.stringify(
+          resp.body
+        )}`
+      )
+    }
+
+    return resp.body as InviteUsersResponse
+  }
 }

From 419c5ced70d9f25cc037132b2c6737976b59af5f Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Thu, 9 Nov 2023 15:31:20 +0000
Subject: [PATCH 088/155] fix issue with relationship selection

---
 .../Datasources/CreateEditRelationship.svelte        | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte
index f6621c1508..abdc95a6da 100644
--- a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte
+++ b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte
@@ -33,6 +33,10 @@
       part1: PrettyRelationshipDefinitions.MANY,
       part2: PrettyRelationshipDefinitions.ONE,
     },
+    [RelationshipType.MANY_TO_ONE]: {
+      part1: PrettyRelationshipDefinitions.ONE,
+      part2: PrettyRelationshipDefinitions.MANY,
+    },
   }
   let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions)
   let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions)
@@ -58,7 +62,7 @@
   let fromPrimary, fromForeign, fromColumn, toColumn
 
   let throughId, throughToKey, throughFromKey
-  let isManyToMany, isManyToOne, relationshipType
+  let relationshipType
   let hasValidated = false
 
   $: fromId = null
@@ -85,8 +89,10 @@
   $: valid =
     getErrorCount(errors) === 0 && allRequiredAttributesSet(relationshipType)
   $: isManyToMany = relationshipType === RelationshipType.MANY_TO_MANY
-  $: isManyToOne = relationshipType === RelationshipType.MANY_TO_ONE
-
+  $: isManyToOne =
+    relationshipType === RelationshipType.MANY_TO_ONE ||
+    relationshipType === RelationshipType.ONE_TO_MANY
+  $: console.log(relationshipType)
   function getTable(id) {
     return plusTables.find(table => table._id === id)
   }

From d92df57daa35ce814b224495b056d21b098976ae Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 9 Nov 2023 16:20:05 +0000
Subject: [PATCH 089/155] Remove nested flag from buttongroup component since
 it isn't needed

---
 packages/client/manifest.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 378d8c3493..5979d177f4 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -270,7 +270,6 @@
           {
             "type": "buttonConfiguration",
             "key": "buttons",
-            "nested": true,
             "defaultValue": [
               {
                 "type": "cta",

From bb47b91064e17dd1a6ff2b4ddd1c71ff367b832b Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 9 Nov 2023 16:20:44 +0000
Subject: [PATCH 090/155] Add new disableBindings property to prevent a text
 field from being bindable when required

---
 .../common/bindings/DrawerBindableInput.svelte |  3 ++-
 .../ButtonConfiguration/ButtonSetting.svelte   | 18 ++++++++++++------
 .../settings/controls/PropertyControl.svelte   |  2 ++
 .../Component/ComponentSettingsSection.svelte  |  1 +
 4 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte b/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte
index 5c4f90606d..09732ce4f2 100644
--- a/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte
+++ b/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte
@@ -21,6 +21,7 @@
   export let allowHelpers = true
   export let updateOnChange = true
   export let drawerLeft
+  export let disableBindings = false
 
   const dispatch = createEventDispatcher()
   let bindingDrawer
@@ -62,7 +63,7 @@
     {placeholder}
     {updateOnChange}
   />
-  {#if !disabled}
+  {#if !disabled && !disableBindings}
     <div
       class="icon"
       on:click={() => {
diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
index 8e8c89ac14..56b5deace4 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte
@@ -17,18 +17,24 @@
 
   // If this is a nested setting (for example inside a grid or form block) then
   // we need to mark all the settings of the actual buttons as nested too, to
-  // allow us to reference context provided by the block
+  // allow us to reference context provided by the block.
+  // We will need to update this in future if the normal button component
+  // gets broken into multiple settings sections, as we assume a flat array.
   const updatedNestedFlags = settings => {
-    if (!nested) {
+    if (!nested || !settings?.length) {
       return settings
     }
-    // Buttons do not currently have any sections, so this works.
-    // We will need to update this in future if the normal button component
-    // gets broken into multiple settings sections
-    return settings?.map(setting => ({
+    let newSettings = settings.map(setting => ({
       ...setting,
       nested: true,
     }))
+    // We need to prevent bindings for the button names because of how grid
+    // blocks work. This is an edge case but unavoidable.
+    let name = newSettings.find(x => x.key === "text")
+    if (name) {
+      name.disableBindings = true
+    }
+    return newSettings
   }
 </script>
 
diff --git a/packages/builder/src/components/design/settings/controls/PropertyControl.svelte b/packages/builder/src/components/design/settings/controls/PropertyControl.svelte
index c8135b4f61..a6f3d1b218 100644
--- a/packages/builder/src/components/design/settings/controls/PropertyControl.svelte
+++ b/packages/builder/src/components/design/settings/controls/PropertyControl.svelte
@@ -23,6 +23,7 @@
   export let highlighted = false
   export let propertyFocus = false
   export let info = null
+  export let disableBindings = false
 
   $: nullishValue = value == null || value === ""
   $: allBindings = getAllBindings(bindings, componentBindings, nested)
@@ -99,6 +100,7 @@
       {nested}
       {key}
       {type}
+      {disableBindings}
       {...props}
       on:drawerHide
       on:drawerShow
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
index 65f010e4ec..6093d2a45e 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
@@ -179,6 +179,7 @@
               highlighted={$store.highlightedSettingKey === setting.key}
               propertyFocus={$store.propertyFocus === setting.key}
               info={setting.info}
+              disableBindings={setting.disableBindings}
               props={{
                 // Generic settings
                 placeholder: setting.placeholder || null,

From dde446286de6922d79953881b11abb1033d33e0e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 9 Nov 2023 17:08:14 +0000
Subject: [PATCH 091/155] Switching getMultiple to default to failure if not
 all entries found, then updating usages.

---
 packages/backend-core/src/db/couch/DatabaseImpl.ts  | 5 +++--
 packages/server/src/api/controllers/row/internal.ts | 3 ++-
 packages/server/src/db/linkedRows/index.ts          | 2 +-
 packages/server/src/sdk/app/tables/getters.ts       | 4 +++-
 packages/server/src/utilities/global.ts             | 2 +-
 packages/types/src/sdk/db.ts                        | 2 +-
 6 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index fa505847af..6a1e575ac9 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -119,7 +119,7 @@ export class DatabaseImpl implements Database {
 
   async getMultiple<T extends Document>(
     ids: string[],
-    opts?: { failIfMissing?: boolean }
+    opts?: { allowMissing?: boolean }
   ): Promise<T[]> {
     // get unique
     ids = [...new Set(ids)]
@@ -129,8 +129,9 @@ export class DatabaseImpl implements Database {
     })
     const NOT_FOUND = "not_found"
     const rows = response.rows.filter(row => row.error !== NOT_FOUND)
+    const someMissing = rows.length !== response.rows.length
     // some were filtered out - means some missing
-    if (opts?.failIfMissing && rows.length !== response.rows.length) {
+    if (!opts?.allowMissing && someMissing) {
       const missing = response.rows.filter(row => row.error === NOT_FOUND)
       const missingIds = missing.map(row => row.key).join(", ")
       throw new Error(`Unable to get documents: ${missingIds}`)
diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts
index 0edbe4ea86..0907c22f0e 100644
--- a/packages/server/src/api/controllers/row/internal.ts
+++ b/packages/server/src/api/controllers/row/internal.ts
@@ -237,7 +237,8 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
 
   // look up the actual rows based on the ids
   let linkedRows = await db.getMultiple<Row>(
-    linkVals.map(linkVal => linkVal.id)
+    linkVals.map(linkVal => linkVal.id),
+    { allowMissing: true }
   )
 
   // get the linked tables
diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts
index 24ff47ead3..7af3f9392f 100644
--- a/packages/server/src/db/linkedRows/index.ts
+++ b/packages/server/src/db/linkedRows/index.ts
@@ -79,7 +79,7 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) {
   const db = context.getAppDB()
   const linkedRowIds = links.map(link => link.id)
   const uniqueRowIds = [...new Set(linkedRowIds)]
-  let dbRows = await db.getMultiple<Row>(uniqueRowIds)
+  let dbRows = await db.getMultiple<Row>(uniqueRowIds, { allowMissing: true })
   // convert the unique db rows back to a full list of linked rows
   const linked = linkedRowIds
     .map(id => dbRows.find(row => row && row._id === id))
diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts
index a09d686b6d..72a6ab61f1 100644
--- a/packages/server/src/sdk/app/tables/getters.ts
+++ b/packages/server/src/sdk/app/tables/getters.ts
@@ -130,7 +130,9 @@ export async function getTables(tableIds: string[]): Promise<Table[]> {
   }
   if (internalTableIds.length) {
     const db = context.getAppDB()
-    const internalTables = await db.getMultiple<Table>(internalTableIds)
+    const internalTables = await db.getMultiple<Table>(internalTableIds, {
+      allowMissing: true,
+    })
     tables = tables.concat(internalTables)
   }
   return processTables(tables)
diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts
index 762d1fb7a0..bbb84c1882 100644
--- a/packages/server/src/utilities/global.ts
+++ b/packages/server/src/utilities/global.ts
@@ -96,7 +96,7 @@ export async function getRawGlobalUsers(userIds?: string[]): Promise<User[]> {
   const db = tenancy.getGlobalDB()
   let globalUsers: User[]
   if (userIds) {
-    globalUsers = await db.getMultiple<User>(userIds)
+    globalUsers = await db.getMultiple<User>(userIds, { allowMissing: true })
   } else {
     globalUsers = (
       await db.allDocs<User>(
diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts
index ff5742bcfc..7613ac6aeb 100644
--- a/packages/types/src/sdk/db.ts
+++ b/packages/types/src/sdk/db.ts
@@ -125,7 +125,7 @@ export interface Database {
   get<T extends Document>(id?: string): Promise<T>
   getMultiple<T extends Document>(
     ids: string[],
-    opts?: { failIfMissing?: boolean }
+    opts?: { allowMissing?: boolean }
   ): Promise<T[]>
   remove(
     id: string | Document,

From 77f0eebdfdb5afec83bf1fc75d49b19209c81ed0 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 9 Nov 2023 17:30:40 +0000
Subject: [PATCH 092/155] Removing stray console.log.

---
 .../components/backend/Datasources/CreateEditRelationship.svelte | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte
index abdc95a6da..3f2253c754 100644
--- a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte
+++ b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte
@@ -92,7 +92,6 @@
   $: isManyToOne =
     relationshipType === RelationshipType.MANY_TO_ONE ||
     relationshipType === RelationshipType.ONE_TO_MANY
-  $: console.log(relationshipType)
   function getTable(id) {
     return plusTables.find(table => table._id === id)
   }

From b68607b0485844e004a2a481300048421d0ef33d Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Thu, 9 Nov 2023 18:21:28 +0000
Subject: [PATCH 093/155] Redis query command doesn't accept spaced values
 (#12357)

* Handle string phrase with spaces value

* Unit test
---
 packages/server/src/integrations/redis.ts       | 14 +++++++++++++-
 .../server/src/integrations/tests/redis.spec.ts | 17 +++++++++++++++++
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/integrations/redis.ts b/packages/server/src/integrations/redis.ts
index 879a790550..6a6331ccd4 100644
--- a/packages/server/src/integrations/redis.ts
+++ b/packages/server/src/integrations/redis.ts
@@ -165,10 +165,22 @@ class RedisIntegration {
       // commands split line by line
       const commands = query.json.trim().split("\n")
       let pipelineCommands = []
+      let tokenised
 
       // process each command separately
       for (let command of commands) {
-        const tokenised = command.trim().split(" ")
+        const valueToken = command.trim().match(/".*"/)
+        if (valueToken?.[0]) {
+          tokenised = [
+            ...command
+              .substring(0, command.indexOf(valueToken[0]) - 1)
+              .trim()
+              .split(" "),
+            valueToken?.[0],
+          ]
+        } else {
+          tokenised = command.trim().split(" ")
+        }
         // Pipeline only accepts lower case commands
         tokenised[0] = tokenised[0].toLowerCase()
         pipelineCommands.push(tokenised)
diff --git a/packages/server/src/integrations/tests/redis.spec.ts b/packages/server/src/integrations/tests/redis.spec.ts
index 9521d58a51..942da99530 100644
--- a/packages/server/src/integrations/tests/redis.spec.ts
+++ b/packages/server/src/integrations/tests/redis.spec.ts
@@ -85,4 +85,21 @@ describe("Redis Integration", () => {
       ["get", "foo"],
     ])
   })
+
+  it("calls the pipeline method with double quoted phrase values", async () => {
+    const body = {
+      json: 'SET foo "What a wonderful world!"\nGET foo',
+    }
+
+    // ioredis-mock doesn't support pipelines
+    config.integration.client.pipeline = jest.fn(() => ({
+      exec: jest.fn(() => [[]]),
+    }))
+
+    await config.integration.command(body)
+    expect(config.integration.client.pipeline).toHaveBeenCalledWith([
+      ["set", "foo", '"What a wonderful world!"'],
+      ["get", "foo"],
+    ])
+  })
 })

From 20895cf4266f3accc30aba8bfd5678793195de04 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 9 Nov 2023 18:22:06 +0000
Subject: [PATCH 094/155] Adding test case.

---
 packages/server/src/sdk/tests/tables.spec.ts | 39 ++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 packages/server/src/sdk/tests/tables.spec.ts

diff --git a/packages/server/src/sdk/tests/tables.spec.ts b/packages/server/src/sdk/tests/tables.spec.ts
new file mode 100644
index 0000000000..0e3cd73cfd
--- /dev/null
+++ b/packages/server/src/sdk/tests/tables.spec.ts
@@ -0,0 +1,39 @@
+import TestConfig from "../../tests/utilities/TestConfiguration"
+import { basicTable } from "../../tests/utilities/structures"
+import { Table } from "@budibase/types"
+import sdk from "../"
+
+describe("tables", () => {
+  const config = new TestConfig()
+  let table: Table
+
+  beforeAll(async () => {
+    await config.init()
+    table = await config.api.table.create(basicTable())
+  })
+
+  describe("getTables", () => {
+    it("should be able to retrieve tables", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tables = await sdk.tables.getTables([table._id!])
+        expect(tables.length).toBe(1)
+        expect(tables[0]._id).toBe(table._id)
+        expect(tables[0].name).toBe(table.name)
+      })
+    })
+
+    it("shouldn't fail when retrieving tables that don't exist", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tables = await sdk.tables.getTables(["unknown"])
+        expect(tables.length).toBe(0)
+      })
+    })
+
+    it("should de-duplicate the IDs", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tables = await sdk.tables.getTables([table._id!, table._id!])
+        expect(tables.length).toBe(1)
+      })
+    })
+  })
+})

From bcfab2d0a30655f6a959e584f003557ecd85783d Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 10 Nov 2023 08:03:07 +0000
Subject: [PATCH 095/155] Bump version to 2.13.7

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index ce3128165f..c0559d8346 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.13.6",
+  "version": "2.13.7",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 4fd9d2262ec0433580e81b55b838ea12646e1179 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Fri, 10 Nov 2023 09:48:19 +0000
Subject: [PATCH 096/155] PR Feedback

---
 .../FieldConfiguration.svelte                 | 13 +------------
 .../Component/ComponentSettingsSection.svelte | 19 -------------------
 2 files changed, 1 insertion(+), 31 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
index 2211f4a41b..965c1c2881 100644
--- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
@@ -9,7 +9,7 @@
   } from "builderStore/dataBinding"
   import { currentAsset } from "builderStore"
   import DraggableList from "../DraggableList/DraggableList.svelte"
-  import { createEventDispatcher, getContext } from "svelte"
+  import { createEventDispatcher } from "svelte"
   import { store, selectedScreen } from "builderStore"
   import FieldSetting from "./FieldSetting.svelte"
   import { convertOldFieldFormat, getComponentForField } from "./utils"
@@ -17,8 +17,6 @@
   export let componentInstance
   export let value
 
-  const updates = getContext("settings")
-
   const dispatch = createEventDispatcher()
   let sanitisedFields
   let fieldList
@@ -29,7 +27,6 @@
   let unconfigured
 
   let selectAll = true
-  let updating = false
 
   $: bindings = getBindableProperties($selectedScreen, componentInstance._id)
   $: actionType = componentInstance.actionType
@@ -49,10 +46,6 @@
     cachedValue = cloneDeep(value)
   }
 
-  $: if (typeof $updates.resp == "string") {
-    updating = false
-  }
-
   const updateState = value => {
     schema = getSchema($currentAsset, datasource)
     options = Object.keys(schema || {})
@@ -161,7 +154,6 @@
   }
 
   const toggleAll = update => {
-    updating = true
     listUpdated(update)
   }
 </script>
@@ -180,7 +172,6 @@
       text=""
       bind:value={selectAll}
       thin
-      disabled={updating}
     />
   </div>
   {#if fieldList?.length}
@@ -193,9 +184,7 @@
       listTypeProps={{
         componentBindings,
         bindings,
-        updating,
       }}
-      draggable={!updating}
     />
   {/if}
 </div>
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
index 3483f6d26f..65f010e4ec 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
@@ -8,8 +8,6 @@
   import { getComponentForSetting } from "components/design/settings/componentSettings"
   import InfoDisplay from "./InfoDisplay.svelte"
   import analytics, { Events } from "analytics"
-  import { setContext } from "svelte"
-  import { writable } from "svelte/store"
 
   export let componentDefinition
   export let componentInstance
@@ -21,21 +19,6 @@
   export let includeHidden = false
   export let tag
 
-  let status = writable({
-    status: null,
-    lastUpdate: null,
-  })
-
-  const updateStatus = resp => {
-    status.update(state => ({
-      ...state,
-      resp,
-      lastUpdate: new Date().getTime(),
-    }))
-  }
-
-  setContext("settings", status)
-
   $: sections = getSections(
     componentInstance,
     componentDefinition,
@@ -93,7 +76,6 @@
       } else {
         await store.actions.components.updateSetting(setting.key, value)
       }
-      updateStatus("success")
       // Send event if required
       if (setting.sendEvents) {
         analytics.captureEvent(Events.COMPONENT_UPDATED, {
@@ -103,7 +85,6 @@
         })
       }
     } catch (error) {
-      updateStatus("failed")
       notifications.error("Error updating component prop")
     }
   }

From 6282582bb0426b7e473db02287df4b21da9e68a9 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 10 Nov 2023 11:02:50 +0000
Subject: [PATCH 097/155] PR comments.

---
 .../backend/Datasources/CreateEditRelationship.svelte           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte
index 3f2253c754..af678a88ba 100644
--- a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte
+++ b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte
@@ -33,7 +33,7 @@
       part1: PrettyRelationshipDefinitions.MANY,
       part2: PrettyRelationshipDefinitions.ONE,
     },
-    [RelationshipType.MANY_TO_ONE]: {
+    [RelationshipType.ONE_TO_MANY]: {
       part1: PrettyRelationshipDefinitions.ONE,
       part2: PrettyRelationshipDefinitions.MANY,
     },

From d98e217c6c1ba90ccddfc9833609e145ed6785f5 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 10 Nov 2023 11:21:36 +0000
Subject: [PATCH 098/155] Fix backend-core redis imports.

---
 packages/backend-core/src/redis/index.ts      |  5 +++--
 .../src/api/controllers/global/users.ts       | 19 +++++++------------
 packages/worker/src/sdk/auth/auth.ts          |  2 +-
 packages/worker/src/utilities/email.ts        |  4 ++--
 4 files changed, 13 insertions(+), 17 deletions(-)

diff --git a/packages/backend-core/src/redis/index.ts b/packages/backend-core/src/redis/index.ts
index 8d153ea5a1..419f1db700 100644
--- a/packages/backend-core/src/redis/index.ts
+++ b/packages/backend-core/src/redis/index.ts
@@ -4,5 +4,6 @@ export { default as Client } from "./redis"
 export * as utils from "./utils"
 export * as clients from "./init"
 export * as locks from "./redlockImpl"
-export * as invite from "./invite"
-export * as passwordReset from "./passwordReset"
+
+export * from "./invite"
+export * from "./passwordReset"
diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts
index e904567674..cd11fc74b6 100644
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@@ -1,9 +1,4 @@
-import {
-  getInviteCode,
-  deleteInviteCode,
-  getInviteCodes,
-  updateInviteCode,
-} from "@budibase/backend-core/src/redis/invite"
+import { redis } from "@budibase/backend-core"
 import * as userSdk from "../../../sdk/users"
 import env from "../../../environment"
 import {
@@ -313,7 +308,7 @@ export const checkInvite = async (ctx: any) => {
   const { code } = ctx.params
   let invite
   try {
-    invite = await getInviteCode(code)
+    invite = await redis.getInviteCode(code)
   } catch (e) {
     console.warn("Error getting invite from code", e)
     ctx.throw(400, "There was a problem with the invite")
@@ -327,7 +322,7 @@ export const checkInvite = async (ctx: any) => {
 export const getUserInvites = async (ctx: any) => {
   try {
     // Restricted to the currently authenticated tenant
-    ctx.body = await getInviteCodes()
+    ctx.body = await redis.getInviteCodes()
   } catch (e) {
     ctx.throw(400, "There was a problem fetching invites")
   }
@@ -341,7 +336,7 @@ export const updateInvite = async (ctx: any) => {
 
   let invite
   try {
-    invite = await getInviteCode(code)
+    invite = await redis.getInviteCode(code)
   } catch (e) {
     ctx.throw(400, "There was a problem with the invite")
     return
@@ -369,7 +364,7 @@ export const updateInvite = async (ctx: any) => {
     }
   }
 
-  await updateInviteCode(code, updated)
+  await redis.updateInviteCode(code, updated)
   ctx.body = { ...invite }
 }
 
@@ -379,8 +374,8 @@ export const inviteAccept = async (
   const { inviteCode, password, firstName, lastName } = ctx.request.body
   try {
     // info is an extension of the user object that was stored by global
-    const { email, info }: any = await getInviteCode(inviteCode)
-    await deleteInviteCode(inviteCode)
+    const { email, info }: any = await redis.getInviteCode(inviteCode)
+    await redis.deleteInviteCode(inviteCode)
     const user = await tenancy.doInTenant(info.tenantId, async () => {
       let request: any = {
         firstName,
diff --git a/packages/worker/src/sdk/auth/auth.ts b/packages/worker/src/sdk/auth/auth.ts
index 57131294b3..2140b89ce3 100644
--- a/packages/worker/src/sdk/auth/auth.ts
+++ b/packages/worker/src/sdk/auth/auth.ts
@@ -73,7 +73,7 @@ export const reset = async (email: string) => {
  * Perform the user password update if the provided reset code is valid.
  */
 export const resetUpdate = async (resetCode: string, password: string) => {
-  const { userId } = await redis.passwordReset.getResetPasswordCode(resetCode)
+  const { userId } = await redis.getResetPasswordCode(resetCode)
 
   let user = await userSdk.db.getUser(userId)
   user.password = password
diff --git a/packages/worker/src/utilities/email.ts b/packages/worker/src/utilities/email.ts
index f3f1fd40c4..a4d2c296e5 100644
--- a/packages/worker/src/utilities/email.ts
+++ b/packages/worker/src/utilities/email.ts
@@ -4,7 +4,7 @@ import { getTemplateByPurpose, EmailTemplates } from "../constants/templates"
 import { getSettingsTemplateContext } from "./templates"
 import { processString } from "@budibase/string-templates"
 import { createResetPasswordCode } from "@budibase/backend-core/src/redis/passwordReset"
-import { createInviteCode } from "@budibase/backend-core/src/redis/invite"
+import { redis } from "@budibase/backend-core"
 import { User, SendEmailOpts, SMTPInnerConfig } from "@budibase/types"
 import { configs } from "@budibase/backend-core"
 import ical from "ical-generator"
@@ -64,7 +64,7 @@ async function getLinkCode(
     case EmailTemplatePurpose.PASSWORD_RECOVERY:
       return createResetPasswordCode(user._id!, info)
     case EmailTemplatePurpose.INVITATION:
-      return createInviteCode(email, info)
+      return redis.createInviteCode(email, info)
     default:
       return null
   }

From dd2f68d0990b7019ac0faa8caf4c74f09dcae07b Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 10 Nov 2023 11:24:55 +0000
Subject: [PATCH 099/155] Hook new Redis clients into init/shutdown flow.

---
 packages/backend-core/src/redis/init.ts       | 22 +++++++++++++++++-
 packages/backend-core/src/redis/invite.ts     | 23 ++++++-------------
 .../backend-core/src/redis/passwordReset.ts   | 17 ++++----------
 3 files changed, 32 insertions(+), 30 deletions(-)

diff --git a/packages/backend-core/src/redis/init.ts b/packages/backend-core/src/redis/init.ts
index 55ffe3dd12..a4f1fecc17 100644
--- a/packages/backend-core/src/redis/init.ts
+++ b/packages/backend-core/src/redis/init.ts
@@ -7,7 +7,9 @@ let userClient: Client,
   cacheClient: Client,
   writethroughClient: Client,
   lockClient: Client,
-  socketClient: Client
+  socketClient: Client,
+  inviteClient: Client,
+  passwordResetClient: Client
 
 async function init() {
   userClient = await new Client(utils.Databases.USER_CACHE).init()
@@ -20,6 +22,8 @@ async function init() {
     utils.Databases.SOCKET_IO,
     utils.SelectableDatabase.SOCKET_IO
   ).init()
+  inviteClient = await new Client(utils.Databases.INVITATIONS).init()
+  passwordResetClient = await new Client(utils.Databases.PW_RESETS).init()
 }
 
 export async function shutdown() {
@@ -30,6 +34,8 @@ export async function shutdown() {
   if (writethroughClient) await writethroughClient.finish()
   if (lockClient) await lockClient.finish()
   if (socketClient) await socketClient.finish()
+  if (inviteClient) await inviteClient.finish()
+  if (passwordResetClient) await passwordResetClient.finish()
 }
 
 process.on("exit", async () => {
@@ -84,3 +90,17 @@ export async function getSocketClient() {
   }
   return socketClient
 }
+
+export async function getInviteClient() {
+  if (!inviteClient) {
+    await init()
+  }
+  return inviteClient
+}
+
+export async function getPasswordResetClient() {
+  if (!passwordResetClient) {
+    await init()
+  }
+  return passwordResetClient
+}
diff --git a/packages/backend-core/src/redis/invite.ts b/packages/backend-core/src/redis/invite.ts
index db36d3dfa6..6e6a1fd9e9 100644
--- a/packages/backend-core/src/redis/invite.ts
+++ b/packages/backend-core/src/redis/invite.ts
@@ -1,5 +1,6 @@
-import { redis, utils, tenancy } from "../"
+import { utils, tenancy } from "../"
 import env from "../environment"
+import { getInviteClient } from "./init"
 
 const TTL_SECONDS = 60 * 60 * 24 * 7
 
@@ -12,23 +13,13 @@ interface InviteWithCode extends Invite {
   code: string
 }
 
-let client: redis.Client
-
-async function getClient(): Promise<redis.Client> {
-  if (!client) {
-    client = new redis.Client(redis.utils.Databases.INVITATIONS)
-    await client.init()
-  }
-  return client
-}
-
 /**
  * Given an invite code and invite body, allow the update an existing/valid invite in redis
  * @param inviteCode The invite code for an invite in redis
  * @param value The body of the updated user invitation
  */
 export async function updateInviteCode(code: string, value: Invite) {
-  const client = await getClient()
+  const client = await getInviteClient()
   await client.store(code, value, TTL_SECONDS)
 }
 
@@ -42,7 +33,7 @@ export async function createInviteCode(
   email: string,
   info: any
 ): Promise<string> {
-  const client = await getClient()
+  const client = await getInviteClient()
   const code = utils.newid()
   await client.store(code, { email, info }, TTL_SECONDS)
   return code
@@ -54,7 +45,7 @@ export async function createInviteCode(
  * @return If the code is valid then an email address will be returned.
  */
 export async function getInviteCode(code: string): Promise<Invite> {
-  const client = await getClient()
+  const client = await getInviteClient()
   const value = (await client.get(code)) as Invite | undefined
   if (!value) {
     throw "Invitation is not valid or has expired, please request a new one."
@@ -63,7 +54,7 @@ export async function getInviteCode(code: string): Promise<Invite> {
 }
 
 export async function deleteInviteCode(code: string) {
-  const client = await getClient()
+  const client = await getInviteClient()
   await client.delete(code)
 }
 
@@ -71,7 +62,7 @@ export async function deleteInviteCode(code: string) {
   Get all currently available user invitations for the current tenant.
 **/
 export async function getInviteCodes(): Promise<InviteWithCode[]> {
-  const client = await getClient()
+  const client = await getInviteClient()
   const invites: { key: string; value: Invite }[] = await client.scan()
 
   const results: InviteWithCode[] = invites.map(invite => {
diff --git a/packages/backend-core/src/redis/passwordReset.ts b/packages/backend-core/src/redis/passwordReset.ts
index 63c3371bba..243b73c529 100644
--- a/packages/backend-core/src/redis/passwordReset.ts
+++ b/packages/backend-core/src/redis/passwordReset.ts
@@ -1,4 +1,5 @@
-import { redis, utils } from "../"
+import { utils } from "../"
+import { getPasswordResetClient } from "./init"
 
 const TTL_SECONDS = 60 * 60
 
@@ -7,16 +8,6 @@ interface PasswordReset {
   info: any
 }
 
-let client: redis.Client
-
-async function getClient(): Promise<redis.Client> {
-  if (!client) {
-    client = new redis.Client(redis.utils.Databases.PW_RESETS)
-    await client.init()
-  }
-  return client
-}
-
 /**
  * Given a user ID this will store a code (that is returned) for an hour in redis.
  * The user can then return this code for resetting their password (through their reset link).
@@ -28,7 +19,7 @@ export async function createResetPasswordCode(
   userId: string,
   info: any
 ): Promise<string> {
-  const client = await getClient()
+  const client = await getPasswordResetClient()
   const code = utils.newid()
   await client.store(code, { userId, info }, TTL_SECONDS)
   return code
@@ -42,7 +33,7 @@ export async function createResetPasswordCode(
 export async function getResetPasswordCode(
   code: string
 ): Promise<PasswordReset> {
-  const client = await getClient()
+  const client = await getPasswordResetClient()
   const value = (await client.get(code)) as PasswordReset | undefined
   if (!value) {
     throw "Provided information is not valid, cannot reset password - please try again."

From 94983c289fc578033e2d20466388adf05858dfa3 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 10 Nov 2023 11:39:26 +0000
Subject: [PATCH 100/155] Hook redis init flow into overall worker init flow.

---
 packages/backend-core/src/index.ts       | 5 ++++-
 packages/backend-core/src/redis/index.ts | 1 +
 packages/backend-core/src/redis/init.ts  | 2 +-
 packages/worker/src/db/index.ts          | 4 ++--
 4 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts
index c7cf9f56cc..d41f2b9384 100644
--- a/packages/backend-core/src/index.ts
+++ b/packages/backend-core/src/index.ts
@@ -37,6 +37,7 @@ export { SearchParams } from "./db"
 // circular dependencies
 import * as context from "./context"
 import * as _tenancy from "./tenancy"
+import * as redis from "./redis"
 export const tenancy = {
   ..._tenancy,
   ...context,
@@ -50,6 +51,8 @@ export * from "./constants"
 
 // expose package init function
 import * as db from "./db"
-export const init = (opts: any = {}) => {
+
+export const init = async (opts: any = {}) => {
   db.init(opts.db)
+  await redis.init()
 }
diff --git a/packages/backend-core/src/redis/index.ts b/packages/backend-core/src/redis/index.ts
index 419f1db700..cc9eb854b4 100644
--- a/packages/backend-core/src/redis/index.ts
+++ b/packages/backend-core/src/redis/index.ts
@@ -4,6 +4,7 @@ export { default as Client } from "./redis"
 export * as utils from "./utils"
 export * as clients from "./init"
 export * as locks from "./redlockImpl"
+export * from "./init"
 
 export * from "./invite"
 export * from "./passwordReset"
diff --git a/packages/backend-core/src/redis/init.ts b/packages/backend-core/src/redis/init.ts
index a4f1fecc17..8f2d2914b5 100644
--- a/packages/backend-core/src/redis/init.ts
+++ b/packages/backend-core/src/redis/init.ts
@@ -11,7 +11,7 @@ let userClient: Client,
   inviteClient: Client,
   passwordResetClient: Client
 
-async function init() {
+export async function init() {
   userClient = await new Client(utils.Databases.USER_CACHE).init()
   sessionClient = await new Client(utils.Databases.SESSIONS).init()
   appClient = await new Client(utils.Databases.APP_METADATA).init()
diff --git a/packages/worker/src/db/index.ts b/packages/worker/src/db/index.ts
index 157c2f4fb3..19f8f8acee 100644
--- a/packages/worker/src/db/index.ts
+++ b/packages/worker/src/db/index.ts
@@ -1,7 +1,7 @@
 import * as core from "@budibase/backend-core"
 import env from "../environment"
 
-export function init() {
+export async function init() {
   const dbConfig: any = {
     replication: true,
     find: true,
@@ -12,5 +12,5 @@ export function init() {
     dbConfig.allDbs = true
   }
 
-  core.init({ db: dbConfig })
+  await core.init({ db: dbConfig })
 }

From a427d990a11eea5910526b378e4cd70b8a5e5aba Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 10 Nov 2023 11:58:07 +0000
Subject: [PATCH 101/155] Quick addition - if the object has been deleted but
 the key is still known, then CouchDB will alert us to the fact that it is
 deleted, leaving the response in a weird state.

---
 packages/backend-core/src/db/couch/DatabaseImpl.ts | 14 +++++++++++---
 packages/backend-core/src/redis/redis.ts           |  1 -
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index 6a1e575ac9..8588a7157a 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -10,6 +10,7 @@ import {
   DatabaseDeleteIndexOpts,
   Document,
   isDocument,
+  RowResponse,
 } from "@budibase/types"
 import { getCouchInfo } from "./connections"
 import { directCouchUrlCall } from "./utils"
@@ -127,12 +128,19 @@ export class DatabaseImpl implements Database {
       keys: ids,
       include_docs: true,
     })
-    const NOT_FOUND = "not_found"
-    const rows = response.rows.filter(row => row.error !== NOT_FOUND)
+    const rowUnavailable = (row: RowResponse<T>) => {
+      // row is deleted - key lookup can return this
+      if (row.doc == null || ("deleted" in row.value && row.value.deleted)) {
+        return true
+      }
+      return row.error === "not_found"
+    }
+
+    const rows = response.rows.filter(row => !rowUnavailable(row))
     const someMissing = rows.length !== response.rows.length
     // some were filtered out - means some missing
     if (!opts?.allowMissing && someMissing) {
-      const missing = response.rows.filter(row => row.error === NOT_FOUND)
+      const missing = response.rows.filter(row => rowUnavailable(row))
       const missingIds = missing.map(row => row.key).join(", ")
       throw new Error(`Unable to get documents: ${missingIds}`)
     }
diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts
index 6f1b573718..701e262091 100644
--- a/packages/backend-core/src/redis/redis.ts
+++ b/packages/backend-core/src/redis/redis.ts
@@ -28,7 +28,6 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT
 // for testing just generate the client once
 let CLOSED = false
 let CLIENTS: { [key: number]: any } = {}
-0
 let CONNECTED = false
 
 // mock redis always connected

From f808b3e1117ee78f8d4a4cac5ee871d0b69985e3 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Fri, 10 Nov 2023 12:02:11 +0000
Subject: [PATCH 102/155] PR Feedback

---
 .../settings/controls/EditComponentPopover.svelte      |  1 -
 .../FieldConfiguration/FieldConfiguration.svelte       |  6 +-----
 .../controls/FieldConfiguration/FieldSetting.svelte    | 10 +---------
 3 files changed, 2 insertions(+), 15 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte b/packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte
index d0f6f24362..0ba502bbcb 100644
--- a/packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte
+++ b/packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte
@@ -91,7 +91,6 @@
       open = true
     }
   }}
-  {disabled}
 />
 
 <Popover
diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
index 965c1c2881..2b76bc6591 100644
--- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte
@@ -152,10 +152,6 @@
     const parsedColumns = getValidColumns(columns, options)
     dispatch("change", parsedColumns)
   }
-
-  const toggleAll = update => {
-    listUpdated(update)
-  }
 </script>
 
 <div class="field-configuration">
@@ -167,7 +163,7 @@
           ...field,
           active: selectAll,
         }))
-        toggleAll(update)
+        listUpdated(update)
       }}
       text=""
       bind:value={selectAll}
diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte
index 75e6599a6c..1d9ce733b8 100644
--- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte
+++ b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte
@@ -11,7 +11,6 @@
   export let componentBindings
   export let bindings
   export let anchor
-  export let updating //or disabled
 
   const dispatch = createEventDispatcher()
   const onToggle = item => {
@@ -50,7 +49,6 @@
       {bindings}
       {parseSettings}
       on:change
-      disabled={updating}
     >
       <div slot="header" class="type-icon">
         <Icon name={componentDef.icon} />
@@ -60,13 +58,7 @@
     <div class="field-label">{readableText}</div>
   </div>
   <div class="list-item-right">
-    <Toggle
-      on:change={onToggle(item)}
-      text=""
-      value={item.active}
-      thin
-      disabled={updating}
-    />
+    <Toggle on:change={onToggle(item)} text="" value={item.active} thin />
   </div>
 </div>
 

From 90aaa1d9c19762fc0993957b7631e6c532f1af79 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 10 Nov 2023 12:13:45 +0000
Subject: [PATCH 103/155] Bump version to 2.13.8

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index c0559d8346..e06cae80fd 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.13.7",
+  "version": "2.13.8",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From e1f8b575689a730a9d38146b1b19e09107cea846 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 10 Nov 2023 12:54:49 +0000
Subject: [PATCH 104/155] Bump version to 2.13.9

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index e06cae80fd..aafb6b22ce 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.13.8",
+  "version": "2.13.9",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From ac57cdbf028b207a4ce943508de2da7fa0f33129 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Nov 2023 16:16:54 +0100
Subject: [PATCH 105/155] Encode view ids on paths

---
 packages/frontend-core/src/api/viewsV2.js | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/packages/frontend-core/src/api/viewsV2.js b/packages/frontend-core/src/api/viewsV2.js
index 3580d08229..a2072c2e1d 100644
--- a/packages/frontend-core/src/api/viewsV2.js
+++ b/packages/frontend-core/src/api/viewsV2.js
@@ -5,7 +5,7 @@ export const buildViewV2Endpoints = API => ({
    */
   fetchDefinition: async viewId => {
     return await API.get({
-      url: `/api/v2/views/${viewId}`,
+      url: `/api/v2/views/${encodeURIComponent(viewId)}`,
     })
   },
   /**
@@ -24,7 +24,7 @@ export const buildViewV2Endpoints = API => ({
    */
   update: async view => {
     return await API.put({
-      url: `/api/v2/views/${view.id}`,
+      url: `/api/v2/views/${encodeURIComponent(view.id)}`,
       body: view,
     })
   },
@@ -50,7 +50,7 @@ export const buildViewV2Endpoints = API => ({
     sortType,
   }) => {
     return await API.post({
-      url: `/api/v2/views/${viewId}/search`,
+      url: `/api/v2/views/${encodeURIComponent(viewId)}/search`,
       body: {
         query,
         paginate,
@@ -67,6 +67,8 @@ export const buildViewV2Endpoints = API => ({
    * @param viewId the id of the view
    */
   delete: async viewId => {
-    return await API.delete({ url: `/api/v2/views/${viewId}` })
+    return await API.delete({
+      url: `/api/v2/views/${encodeURIComponent(viewId)}`,
+    })
   },
 })

From 24774f0836f6f9c9723f0cc6356d629474cab641 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Nov 2023 16:33:59 +0100
Subject: [PATCH 106/155] Fix navigation

---
 .../src/components/backend/TableNavigator/TableNavigator.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte b/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte
index 056a36c4a7..712d74889c 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte
@@ -53,7 +53,7 @@
           selected={isViewActive(view, $isActive, $views, $viewsV2)}
           on:click={() => {
             if (view.version === 2) {
-              $goto(`./view/v2/${view.id}`)
+              $goto(`./view/v2/${encodeURIComponent(view.id)}`)
             } else {
               $goto(`./view/v1/${encodeURIComponent(name)}`)
             }

From d6eb2b945223c5ef72113b2db87d30d795c02a3b Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 10 Nov 2023 15:43:06 +0000
Subject: [PATCH 107/155] Attempting to get integration tests passing again.

---
 packages/worker/src/index.ts                  |  3 +-
 .../worker/src/tests/TestConfiguration.ts     |  2 +-
 qa-core/yarn.lock                             | 28 +++++++++----------
 3 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts
index 1c101983df..d40c7f0668 100644
--- a/packages/worker/src/index.ts
+++ b/packages/worker/src/index.ts
@@ -17,7 +17,7 @@ import {
   env as coreEnv,
   timers,
 } from "@budibase/backend-core"
-db.init()
+
 import Koa from "koa"
 import koaBody from "koa-body"
 import http from "http"
@@ -85,6 +85,7 @@ const shutdown = () => {
 
 export default server.listen(parseInt(env.PORT || "4002"), async () => {
   console.log(`Worker running on ${JSON.stringify(server.address())}`)
+  await db.init()
   await initPro()
   // configure events to use the pro audit log write
   // can't integrate directly into backend-core due to cyclic issues
diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts
index d4fcbeebd6..1961d22c34 100644
--- a/packages/worker/src/tests/TestConfiguration.ts
+++ b/packages/worker/src/tests/TestConfiguration.ts
@@ -7,7 +7,6 @@ mocks.licenses.init(mocks.pro)
 mocks.licenses.useUnlimited()
 
 import * as dbConfig from "../db"
-dbConfig.init()
 import env from "../environment"
 import * as controllers from "./controllers"
 const supertest = require("supertest")
@@ -109,6 +108,7 @@ class TestConfiguration {
   // SETUP / TEARDOWN
 
   async beforeAll() {
+    await dbConfig.init()
     try {
       await this.createDefaultUser()
       await this.createSession(this.user!)
diff --git a/qa-core/yarn.lock b/qa-core/yarn.lock
index d2cde27530..1de9d75e60 100644
--- a/qa-core/yarn.lock
+++ b/qa-core/yarn.lock
@@ -983,10 +983,10 @@
     expect "^29.0.0"
     pretty-format "^29.0.0"
 
-"@types/node-fetch@2.6.2":
-  version "2.6.2"
-  resolved "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.2.tgz"
-  integrity sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==
+"@types/node-fetch@2.6.4":
+  version "2.6.4"
+  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.4.tgz#1bc3a26de814f6bf466b25aeb1473fa1afe6a660"
+  integrity sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==
   dependencies:
     "@types/node" "*"
     form-data "^3.0.0"
@@ -3587,18 +3587,18 @@ node-duration@^1.0.4:
   resolved "https://registry.npmjs.org/node-duration/-/node-duration-1.0.4.tgz"
   integrity sha512-eUXYNSY7DL53vqfTosggWkvyIW3bhAcqBDIlolgNYlZhianXTrCL50rlUJWD1eRqkIxMppXTfiFbp+9SjpPrgA==
 
-node-fetch@2, node-fetch@2.6.7, node-fetch@^2.6.7:
+node-fetch@2.6.0:
+  version "2.6.0"
+  resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz"
+  integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
+
+node-fetch@2.6.7, node-fetch@^2.6.7:
   version "2.6.7"
   resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz"
   integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
   dependencies:
     whatwg-url "^5.0.0"
 
-node-fetch@2.6.0:
-  version "2.6.0"
-  resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz"
-  integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
-
 node-gyp-build-optional-packages@5.0.7:
   version "5.0.7"
   resolved "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz"
@@ -4893,10 +4893,10 @@ type-is@^1.6.16, type-is@^1.6.18:
     media-typer "0.3.0"
     mime-types "~2.1.24"
 
-typescript@4.7.3:
-  version "4.7.3"
-  resolved "https://registry.npmjs.org/typescript/-/typescript-4.7.3.tgz"
-  integrity sha512-WOkT3XYvrpXx4vMMqlD+8R8R37fZkjyLGlxavMc4iB8lrl8L0DeTcHbYgw/v0N/z9wAFsgBhcsF0ruoySS22mA==
+typescript@5.2.2:
+  version "5.2.2"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78"
+  integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==
 
 uid2@0.0.x:
   version "0.0.4"

From 4c7c10b121682ad53acbd8ae3aec1765d08060c9 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 10 Nov 2023 16:17:18 +0000
Subject: [PATCH 108/155] Set Redis initialisation back to how it was before I
 started messing with it.

---
 packages/backend-core/src/index.ts            |  4 +--
 packages/backend-core/src/redis/index.ts      |  6 ++--
 packages/backend-core/src/redis/init.ts       | 24 ++------------
 packages/backend-core/src/redis/invite.ts     | 32 +++++++++++--------
 .../backend-core/src/redis/passwordReset.ts   | 27 +++++++++-------
 .../src/api/controllers/global/users.ts       | 12 +++----
 packages/worker/src/db/index.ts               |  4 +--
 packages/worker/src/index.ts                  |  8 +++--
 packages/worker/src/sdk/auth/auth.ts          |  2 +-
 .../worker/src/tests/TestConfiguration.ts     |  2 +-
 packages/worker/src/utilities/email.ts        |  5 ++-
 11 files changed, 57 insertions(+), 69 deletions(-)

diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts
index d41f2b9384..f6c091d679 100644
--- a/packages/backend-core/src/index.ts
+++ b/packages/backend-core/src/index.ts
@@ -37,7 +37,6 @@ export { SearchParams } from "./db"
 // circular dependencies
 import * as context from "./context"
 import * as _tenancy from "./tenancy"
-import * as redis from "./redis"
 export const tenancy = {
   ..._tenancy,
   ...context,
@@ -52,7 +51,6 @@ export * from "./constants"
 // expose package init function
 import * as db from "./db"
 
-export const init = async (opts: any = {}) => {
+export const init = (opts: any = {}) => {
   db.init(opts.db)
-  await redis.init()
 }
diff --git a/packages/backend-core/src/redis/index.ts b/packages/backend-core/src/redis/index.ts
index cc9eb854b4..8d153ea5a1 100644
--- a/packages/backend-core/src/redis/index.ts
+++ b/packages/backend-core/src/redis/index.ts
@@ -4,7 +4,5 @@ export { default as Client } from "./redis"
 export * as utils from "./utils"
 export * as clients from "./init"
 export * as locks from "./redlockImpl"
-export * from "./init"
-
-export * from "./invite"
-export * from "./passwordReset"
+export * as invite from "./invite"
+export * as passwordReset from "./passwordReset"
diff --git a/packages/backend-core/src/redis/init.ts b/packages/backend-core/src/redis/init.ts
index 8f2d2914b5..55ffe3dd12 100644
--- a/packages/backend-core/src/redis/init.ts
+++ b/packages/backend-core/src/redis/init.ts
@@ -7,11 +7,9 @@ let userClient: Client,
   cacheClient: Client,
   writethroughClient: Client,
   lockClient: Client,
-  socketClient: Client,
-  inviteClient: Client,
-  passwordResetClient: Client
+  socketClient: Client
 
-export async function init() {
+async function init() {
   userClient = await new Client(utils.Databases.USER_CACHE).init()
   sessionClient = await new Client(utils.Databases.SESSIONS).init()
   appClient = await new Client(utils.Databases.APP_METADATA).init()
@@ -22,8 +20,6 @@ export async function init() {
     utils.Databases.SOCKET_IO,
     utils.SelectableDatabase.SOCKET_IO
   ).init()
-  inviteClient = await new Client(utils.Databases.INVITATIONS).init()
-  passwordResetClient = await new Client(utils.Databases.PW_RESETS).init()
 }
 
 export async function shutdown() {
@@ -34,8 +30,6 @@ export async function shutdown() {
   if (writethroughClient) await writethroughClient.finish()
   if (lockClient) await lockClient.finish()
   if (socketClient) await socketClient.finish()
-  if (inviteClient) await inviteClient.finish()
-  if (passwordResetClient) await passwordResetClient.finish()
 }
 
 process.on("exit", async () => {
@@ -90,17 +84,3 @@ export async function getSocketClient() {
   }
   return socketClient
 }
-
-export async function getInviteClient() {
-  if (!inviteClient) {
-    await init()
-  }
-  return inviteClient
-}
-
-export async function getPasswordResetClient() {
-  if (!passwordResetClient) {
-    await init()
-  }
-  return passwordResetClient
-}
diff --git a/packages/backend-core/src/redis/invite.ts b/packages/backend-core/src/redis/invite.ts
index 6e6a1fd9e9..006a06fe26 100644
--- a/packages/backend-core/src/redis/invite.ts
+++ b/packages/backend-core/src/redis/invite.ts
@@ -1,6 +1,5 @@
-import { utils, tenancy } from "../"
+import { utils, tenancy, redis } from "../"
 import env from "../environment"
-import { getInviteClient } from "./init"
 
 const TTL_SECONDS = 60 * 60 * 24 * 7
 
@@ -13,13 +12,25 @@ interface InviteWithCode extends Invite {
   code: string
 }
 
+let client: redis.Client
+
+export async function init() {
+  if (!client) {
+    client = new redis.Client(redis.utils.Databases.INVITATIONS)
+  }
+  return client
+}
+
+export async function shutdown() {
+  if (client) await client.finish()
+}
+
 /**
  * Given an invite code and invite body, allow the update an existing/valid invite in redis
  * @param inviteCode The invite code for an invite in redis
  * @param value The body of the updated user invitation
  */
-export async function updateInviteCode(code: string, value: Invite) {
-  const client = await getInviteClient()
+export async function updateCode(code: string, value: Invite) {
   await client.store(code, value, TTL_SECONDS)
 }
 
@@ -29,11 +40,7 @@ export async function updateInviteCode(code: string, value: Invite) {
  * @param info Information to be carried along with the invitation.
  * @return returns the code that was stored to redis.
  */
-export async function createInviteCode(
-  email: string,
-  info: any
-): Promise<string> {
-  const client = await getInviteClient()
+export async function createCode(email: string, info: any): Promise<string> {
   const code = utils.newid()
   await client.store(code, { email, info }, TTL_SECONDS)
   return code
@@ -44,8 +51,7 @@ export async function createInviteCode(
  * @param inviteCode the invite code that was provided as part of the link.
  * @return If the code is valid then an email address will be returned.
  */
-export async function getInviteCode(code: string): Promise<Invite> {
-  const client = await getInviteClient()
+export async function getCode(code: string): Promise<Invite> {
   const value = (await client.get(code)) as Invite | undefined
   if (!value) {
     throw "Invitation is not valid or has expired, please request a new one."
@@ -53,8 +59,7 @@ export async function getInviteCode(code: string): Promise<Invite> {
   return value
 }
 
-export async function deleteInviteCode(code: string) {
-  const client = await getInviteClient()
+export async function deleteCode(code: string) {
   await client.delete(code)
 }
 
@@ -62,7 +67,6 @@ export async function deleteInviteCode(code: string) {
   Get all currently available user invitations for the current tenant.
 **/
 export async function getInviteCodes(): Promise<InviteWithCode[]> {
-  const client = await getInviteClient()
   const invites: { key: string; value: Invite }[] = await client.scan()
 
   const results: InviteWithCode[] = invites.map(invite => {
diff --git a/packages/backend-core/src/redis/passwordReset.ts b/packages/backend-core/src/redis/passwordReset.ts
index 243b73c529..13c1b1d2e6 100644
--- a/packages/backend-core/src/redis/passwordReset.ts
+++ b/packages/backend-core/src/redis/passwordReset.ts
@@ -1,5 +1,4 @@
-import { utils } from "../"
-import { getPasswordResetClient } from "./init"
+import { redis, utils } from "../"
 
 const TTL_SECONDS = 60 * 60
 
@@ -8,6 +7,19 @@ interface PasswordReset {
   info: any
 }
 
+let client: redis.Client
+
+export async function init() {
+  if (!client) {
+    client = new redis.Client(redis.utils.Databases.PW_RESETS)
+  }
+  return client
+}
+
+export async function shutdown() {
+  if (client) await client.finish()
+}
+
 /**
  * Given a user ID this will store a code (that is returned) for an hour in redis.
  * The user can then return this code for resetting their password (through their reset link).
@@ -15,11 +27,7 @@ interface PasswordReset {
  * @param info Info about the user/the reset process.
  * @return returns the code that was stored to redis.
  */
-export async function createResetPasswordCode(
-  userId: string,
-  info: any
-): Promise<string> {
-  const client = await getPasswordResetClient()
+export async function createCode(userId: string, info: any): Promise<string> {
   const code = utils.newid()
   await client.store(code, { userId, info }, TTL_SECONDS)
   return code
@@ -30,10 +38,7 @@ export async function createResetPasswordCode(
  * @param code The code provided via the email link.
  * @return returns the user ID if it is found
  */
-export async function getResetPasswordCode(
-  code: string
-): Promise<PasswordReset> {
-  const client = await getPasswordResetClient()
+export async function getCode(code: string): Promise<PasswordReset> {
   const value = (await client.get(code)) as PasswordReset | undefined
   if (!value) {
     throw "Provided information is not valid, cannot reset password - please try again."
diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts
index cd11fc74b6..affdaa9938 100644
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@@ -308,7 +308,7 @@ export const checkInvite = async (ctx: any) => {
   const { code } = ctx.params
   let invite
   try {
-    invite = await redis.getInviteCode(code)
+    invite = await redis.invite.getCode(code)
   } catch (e) {
     console.warn("Error getting invite from code", e)
     ctx.throw(400, "There was a problem with the invite")
@@ -322,7 +322,7 @@ export const checkInvite = async (ctx: any) => {
 export const getUserInvites = async (ctx: any) => {
   try {
     // Restricted to the currently authenticated tenant
-    ctx.body = await redis.getInviteCodes()
+    ctx.body = await redis.invite.getInviteCodes()
   } catch (e) {
     ctx.throw(400, "There was a problem fetching invites")
   }
@@ -336,7 +336,7 @@ export const updateInvite = async (ctx: any) => {
 
   let invite
   try {
-    invite = await redis.getInviteCode(code)
+    invite = await redis.invite.getCode(code)
   } catch (e) {
     ctx.throw(400, "There was a problem with the invite")
     return
@@ -364,7 +364,7 @@ export const updateInvite = async (ctx: any) => {
     }
   }
 
-  await redis.updateInviteCode(code, updated)
+  await redis.invite.updateCode(code, updated)
   ctx.body = { ...invite }
 }
 
@@ -374,8 +374,8 @@ export const inviteAccept = async (
   const { inviteCode, password, firstName, lastName } = ctx.request.body
   try {
     // info is an extension of the user object that was stored by global
-    const { email, info }: any = await redis.getInviteCode(inviteCode)
-    await redis.deleteInviteCode(inviteCode)
+    const { email, info }: any = await redis.invite.getCode(inviteCode)
+    await redis.invite.deleteCode(inviteCode)
     const user = await tenancy.doInTenant(info.tenantId, async () => {
       let request: any = {
         firstName,
diff --git a/packages/worker/src/db/index.ts b/packages/worker/src/db/index.ts
index 19f8f8acee..157c2f4fb3 100644
--- a/packages/worker/src/db/index.ts
+++ b/packages/worker/src/db/index.ts
@@ -1,7 +1,7 @@
 import * as core from "@budibase/backend-core"
 import env from "../environment"
 
-export async function init() {
+export function init() {
   const dbConfig: any = {
     replication: true,
     find: true,
@@ -12,5 +12,5 @@ export async function init() {
     dbConfig.allDbs = true
   }
 
-  await core.init({ db: dbConfig })
+  core.init({ db: dbConfig })
 }
diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts
index d40c7f0668..e486a67433 100644
--- a/packages/worker/src/index.ts
+++ b/packages/worker/src/index.ts
@@ -16,8 +16,9 @@ import {
   queue,
   env as coreEnv,
   timers,
+  redis,
 } from "@budibase/backend-core"
-
+db.init()
 import Koa from "koa"
 import koaBody from "koa-body"
 import http from "http"
@@ -71,6 +72,8 @@ server.on("close", async () => {
   shuttingDown = true
   console.log("Server Closed")
   timers.cleanup()
+  await redis.invite.shutdown()
+  await redis.passwordReset.shutdown()
   await events.shutdown()
   await queue.shutdown()
   if (!env.isTest()) {
@@ -85,8 +88,9 @@ const shutdown = () => {
 
 export default server.listen(parseInt(env.PORT || "4002"), async () => {
   console.log(`Worker running on ${JSON.stringify(server.address())}`)
-  await db.init()
   await initPro()
+  await redis.invite.init()
+  await redis.passwordReset.init()
   // configure events to use the pro audit log write
   // can't integrate directly into backend-core due to cyclic issues
   await events.processors.init(proSdk.auditLogs.write)
diff --git a/packages/worker/src/sdk/auth/auth.ts b/packages/worker/src/sdk/auth/auth.ts
index 2140b89ce3..704de9e4b2 100644
--- a/packages/worker/src/sdk/auth/auth.ts
+++ b/packages/worker/src/sdk/auth/auth.ts
@@ -73,7 +73,7 @@ export const reset = async (email: string) => {
  * Perform the user password update if the provided reset code is valid.
  */
 export const resetUpdate = async (resetCode: string, password: string) => {
-  const { userId } = await redis.getResetPasswordCode(resetCode)
+  const { userId } = await redis.passwordReset.getCode(resetCode)
 
   let user = await userSdk.db.getUser(userId)
   user.password = password
diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts
index 1961d22c34..d4fcbeebd6 100644
--- a/packages/worker/src/tests/TestConfiguration.ts
+++ b/packages/worker/src/tests/TestConfiguration.ts
@@ -7,6 +7,7 @@ mocks.licenses.init(mocks.pro)
 mocks.licenses.useUnlimited()
 
 import * as dbConfig from "../db"
+dbConfig.init()
 import env from "../environment"
 import * as controllers from "./controllers"
 const supertest = require("supertest")
@@ -108,7 +109,6 @@ class TestConfiguration {
   // SETUP / TEARDOWN
 
   async beforeAll() {
-    await dbConfig.init()
     try {
       await this.createDefaultUser()
       await this.createSession(this.user!)
diff --git a/packages/worker/src/utilities/email.ts b/packages/worker/src/utilities/email.ts
index a4d2c296e5..530b6ce87f 100644
--- a/packages/worker/src/utilities/email.ts
+++ b/packages/worker/src/utilities/email.ts
@@ -3,7 +3,6 @@ import { EmailTemplatePurpose, TemplateType } from "../constants"
 import { getTemplateByPurpose, EmailTemplates } from "../constants/templates"
 import { getSettingsTemplateContext } from "./templates"
 import { processString } from "@budibase/string-templates"
-import { createResetPasswordCode } from "@budibase/backend-core/src/redis/passwordReset"
 import { redis } from "@budibase/backend-core"
 import { User, SendEmailOpts, SMTPInnerConfig } from "@budibase/types"
 import { configs } from "@budibase/backend-core"
@@ -62,9 +61,9 @@ async function getLinkCode(
 ) {
   switch (purpose) {
     case EmailTemplatePurpose.PASSWORD_RECOVERY:
-      return createResetPasswordCode(user._id!, info)
+      return redis.passwordReset.createCode(user._id!, info)
     case EmailTemplatePurpose.INVITATION:
-      return redis.createInviteCode(email, info)
+      return redis.invite.createCode(email, info)
     default:
       return null
   }

From a4cac14914df968c241f487f530b1bef515526f6 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 17 Nov 2023 10:59:55 +0000
Subject: [PATCH 109/155] Bump version to 2.13.10

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index aafb6b22ce..a12b1238b3 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.13.9",
+  "version": "2.13.10",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 051551f876bb5f0f7f29b8d472d336e09c2ee25a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 12:51:22 +0100
Subject: [PATCH 110/155] Unify .eslintrc

---
 packages/cli/.eslintrc              | 12 ------------
 packages/string-templates/.eslintrc | 12 ------------
 2 files changed, 24 deletions(-)
 delete mode 100644 packages/cli/.eslintrc
 delete mode 100644 packages/string-templates/.eslintrc

diff --git a/packages/cli/.eslintrc b/packages/cli/.eslintrc
deleted file mode 100644
index 3431bf04fb..0000000000
--- a/packages/cli/.eslintrc
+++ /dev/null
@@ -1,12 +0,0 @@
-{
-  "globals": {
-    "emit": true,
-    "key": true
-  },
-  "env": {
-    "node": true
-  },
-  "extends": ["eslint:recommended"],
-  "rules": {
-  }
-}
\ No newline at end of file
diff --git a/packages/string-templates/.eslintrc b/packages/string-templates/.eslintrc
deleted file mode 100644
index 3431bf04fb..0000000000
--- a/packages/string-templates/.eslintrc
+++ /dev/null
@@ -1,12 +0,0 @@
-{
-  "globals": {
-    "emit": true,
-    "key": true
-  },
-  "env": {
-    "node": true
-  },
-  "extends": ["eslint:recommended"],
-  "rules": {
-  }
-}
\ No newline at end of file

From ab56228192f1e307f49a95ff1bea932237056050 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 13:03:28 +0100
Subject: [PATCH 111/155] Use eslint-plugin-import and
 import/no-relative-packages

---
 .eslintrc.json |  12 +-
 package.json   |  13 +-
 yarn.lock      | 412 ++++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 426 insertions(+), 11 deletions(-)

diff --git a/.eslintrc.json b/.eslintrc.json
index 75584b8163..783a212f11 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -19,6 +19,7 @@
     "bundle.js"
   ],
   "extends": ["eslint:recommended"],
+  "plugins": ["import"],
   "overrides": [
     {
       "files": ["**/*.svelte"],
@@ -30,7 +31,6 @@
         "sourceType": "module",
         "allowImportExportEverywhere": true
       }
-
     },
     {
       "files": ["**/*.ts"],
@@ -48,7 +48,15 @@
   ],
   "rules": {
     "no-self-assign": "off",
-    "no-unused-vars": ["error", { "varsIgnorePattern": "^_", "argsIgnorePattern": "^_", "destructuredArrayIgnorePattern": "^_" }]
+    "no-unused-vars": [
+      "error",
+      {
+        "varsIgnorePattern": "^_",
+        "argsIgnorePattern": "^_",
+        "destructuredArrayIgnorePattern": "^_"
+      }
+    ],
+    "import/no-relative-packages": "error"
   },
   "globals": {
     "GeolocationPositionError": true
diff --git a/package.json b/package.json
index 8a27cde104..29d5560955 100644
--- a/package.json
+++ b/package.json
@@ -2,11 +2,16 @@
   "name": "root",
   "private": true,
   "devDependencies": {
+    "@babel/core": "^7.22.5",
+    "@babel/eslint-parser": "^7.22.5",
+    "@babel/preset-env": "^7.22.5",
     "@esbuild-plugins/tsconfig-paths": "^0.1.2",
     "@typescript-eslint/parser": "6.7.2",
     "esbuild": "^0.18.17",
     "esbuild-node-externals": "^1.8.0",
     "eslint": "^8.44.0",
+    "eslint-plugin-import": "^2.29.0",
+    "eslint-plugin-svelte": "^2.32.2",
     "husky": "^8.0.3",
     "kill-port": "^1.6.1",
     "lerna": "7.1.1",
@@ -17,12 +22,8 @@
     "prettier": "2.8.8",
     "prettier-plugin-svelte": "^2.3.0",
     "svelte": "3.49.0",
-    "typescript": "5.2.2",
-    "@babel/core": "^7.22.5",
-    "@babel/eslint-parser": "^7.22.5",
-    "@babel/preset-env": "^7.22.5",
-    "eslint-plugin-svelte": "^2.32.2",
-    "svelte-eslint-parser": "^0.32.0"
+    "svelte-eslint-parser": "^0.32.0",
+    "typescript": "5.2.2"
   },
   "scripts": {
     "preinstall": "node scripts/syncProPackage.js",
diff --git a/yarn.lock b/yarn.lock
index f573046394..50c497c6c2 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6799,6 +6799,14 @@ array-back@^6.2.0, array-back@^6.2.2:
   resolved "https://registry.yarnpkg.com/array-back/-/array-back-6.2.2.tgz#f567d99e9af88a6d3d2f9dfcc21db6f9ba9fd157"
   integrity sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==
 
+array-buffer-byte-length@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz#fabe8bc193fea865f317fe7807085ee0dee5aead"
+  integrity sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A==
+  dependencies:
+    call-bind "^1.0.2"
+    is-array-buffer "^3.0.1"
+
 array-differ@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/array-differ/-/array-differ-3.0.0.tgz#3cbb3d0f316810eafcc47624734237d6aee4ae6b"
@@ -6809,6 +6817,17 @@ array-ify@^1.0.0:
   resolved "https://registry.yarnpkg.com/array-ify/-/array-ify-1.0.0.tgz#9e528762b4a9066ad163a6962a364418e9626ece"
   integrity sha512-c5AMf34bKdvPhQ7tBGhqkgKNUzMr4WUs+WDtC2ZUGOUncbxKMTvqxYctiseW3+L4bA8ec+GcZ6/A/FW4m8ukng==
 
+array-includes@^3.1.7:
+  version "3.1.7"
+  resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.7.tgz#8cd2e01b26f7a3086cbc87271593fe921c62abda"
+  integrity sha512-dlcsNBIiWhPkHdOEEKnehA+RNUWDc4UqFtnIXU4uuYDPtA4LDkr7qip2p0VvFAEXNDr0yWZ9PJyIRiGjRLQzwQ==
+  dependencies:
+    call-bind "^1.0.2"
+    define-properties "^1.2.0"
+    es-abstract "^1.22.1"
+    get-intrinsic "^1.2.1"
+    is-string "^1.0.7"
+
 array-sort@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/array-sort/-/array-sort-1.0.0.tgz#e4c05356453f56f53512a7d1d6123f2c54c0a88a"
@@ -6833,6 +6852,50 @@ array-unique@^0.3.2:
   resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428"
   integrity sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ==
 
+array.prototype.findlastindex@^1.2.3:
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.3.tgz#b37598438f97b579166940814e2c0493a4f50207"
+  integrity sha512-LzLoiOMAxvy+Gd3BAq3B7VeIgPdo+Q8hthvKtXybMvRV0jrXfJM/t8mw7nNlpEcVlVUnCnM2KSX4XU5HmpodOA==
+  dependencies:
+    call-bind "^1.0.2"
+    define-properties "^1.2.0"
+    es-abstract "^1.22.1"
+    es-shim-unscopables "^1.0.0"
+    get-intrinsic "^1.2.1"
+
+array.prototype.flat@^1.3.2:
+  version "1.3.2"
+  resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.3.2.tgz#1476217df8cff17d72ee8f3ba06738db5b387d18"
+  integrity sha512-djYB+Zx2vLewY8RWlNCUdHjDXs2XOgm602S9E7P/UpHgfeHL00cRiIF+IN/G/aUJ7kGPb6yO/ErDI5V2s8iycA==
+  dependencies:
+    call-bind "^1.0.2"
+    define-properties "^1.2.0"
+    es-abstract "^1.22.1"
+    es-shim-unscopables "^1.0.0"
+
+array.prototype.flatmap@^1.3.2:
+  version "1.3.2"
+  resolved "https://registry.yarnpkg.com/array.prototype.flatmap/-/array.prototype.flatmap-1.3.2.tgz#c9a7c6831db8e719d6ce639190146c24bbd3e527"
+  integrity sha512-Ewyx0c9PmpcsByhSW4r+9zDU7sGjFc86qf/kKtuSCRdhfbk0SNLLkaT5qvcHnRGgc5NP/ly/y+qkXkqONX54CQ==
+  dependencies:
+    call-bind "^1.0.2"
+    define-properties "^1.2.0"
+    es-abstract "^1.22.1"
+    es-shim-unscopables "^1.0.0"
+
+arraybuffer.prototype.slice@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.2.tgz#98bd561953e3e74bb34938e77647179dfe6e9f12"
+  integrity sha512-yMBKppFur/fbHu9/6USUe03bZ4knMYiwFBcyiaXB8Go0qNehwX6inYPzK9U0NeQvGxKthcmHcaR8P5MStSRBAw==
+  dependencies:
+    array-buffer-byte-length "^1.0.0"
+    call-bind "^1.0.2"
+    define-properties "^1.2.0"
+    es-abstract "^1.22.1"
+    get-intrinsic "^1.2.1"
+    is-array-buffer "^3.0.2"
+    is-shared-array-buffer "^1.0.2"
+
 arrify@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d"
@@ -7727,6 +7790,15 @@ call-bind@^1.0.0, call-bind@^1.0.2:
     function-bind "^1.1.1"
     get-intrinsic "^1.0.2"
 
+call-bind@^1.0.4, call-bind@^1.0.5:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.5.tgz#6fa2b7845ce0ea49bf4d8b9ef64727a2c2e2e513"
+  integrity sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==
+  dependencies:
+    function-bind "^1.1.2"
+    get-intrinsic "^1.2.1"
+    set-function-length "^1.1.1"
+
 call-me-maybe@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/call-me-maybe/-/call-me-maybe-1.0.1.tgz#26d208ea89e37b5cbde60250a15f031c16a4d66b"
@@ -9087,6 +9159,15 @@ deferred-leveldown@~5.3.0:
     abstract-leveldown "~6.2.1"
     inherits "^2.0.3"
 
+define-data-property@^1.0.1, define-data-property@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/define-data-property/-/define-data-property-1.1.1.tgz#c35f7cd0ab09883480d12ac5cb213715587800b3"
+  integrity sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==
+  dependencies:
+    get-intrinsic "^1.2.1"
+    gopd "^1.0.1"
+    has-property-descriptors "^1.0.0"
+
 define-lazy-prop@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz#3f7ae421129bcaaac9bc74905c98a0009ec9ee7f"
@@ -9100,6 +9181,15 @@ define-properties@^1.1.3, define-properties@^1.1.4:
     has-property-descriptors "^1.0.0"
     object-keys "^1.1.1"
 
+define-properties@^1.2.0:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.2.1.tgz#10781cc616eb951a80a034bafcaa7377f6af2b6c"
+  integrity sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==
+  dependencies:
+    define-data-property "^1.0.1"
+    has-property-descriptors "^1.0.0"
+    object-keys "^1.1.1"
+
 define-property@^0.2.5:
   version "0.2.5"
   resolved "https://registry.yarnpkg.com/define-property/-/define-property-0.2.5.tgz#c35b1ef918ec3c990f9a5bc57be04aacec5c8116"
@@ -9465,6 +9555,13 @@ doctrine@3.0.0, doctrine@^3.0.0:
   dependencies:
     esutils "^2.0.2"
 
+doctrine@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.1.0.tgz#5cd01fc101621b42c4cd7f5d1a66243716d3f39d"
+  integrity sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==
+  dependencies:
+    esutils "^2.0.2"
+
 dom-accessibility-api@^0.5.6, dom-accessibility-api@^0.5.9:
   version "0.5.16"
   resolved "https://registry.yarnpkg.com/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz#5a7429e6066eb3664d911e33fb0e45de8eb08453"
@@ -9890,6 +9987,51 @@ es-abstract@^1.17.5, es-abstract@^1.19.0, es-abstract@^1.20.4:
     unbox-primitive "^1.0.2"
     which-typed-array "^1.1.9"
 
+es-abstract@^1.22.1:
+  version "1.22.3"
+  resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.22.3.tgz#48e79f5573198de6dee3589195727f4f74bc4f32"
+  integrity sha512-eiiY8HQeYfYH2Con2berK+To6GrK2RxbPawDkGq4UiCQQfZHb6wX9qQqkbpPqaxQFcl8d9QzZqo0tGE0VcrdwA==
+  dependencies:
+    array-buffer-byte-length "^1.0.0"
+    arraybuffer.prototype.slice "^1.0.2"
+    available-typed-arrays "^1.0.5"
+    call-bind "^1.0.5"
+    es-set-tostringtag "^2.0.1"
+    es-to-primitive "^1.2.1"
+    function.prototype.name "^1.1.6"
+    get-intrinsic "^1.2.2"
+    get-symbol-description "^1.0.0"
+    globalthis "^1.0.3"
+    gopd "^1.0.1"
+    has-property-descriptors "^1.0.0"
+    has-proto "^1.0.1"
+    has-symbols "^1.0.3"
+    hasown "^2.0.0"
+    internal-slot "^1.0.5"
+    is-array-buffer "^3.0.2"
+    is-callable "^1.2.7"
+    is-negative-zero "^2.0.2"
+    is-regex "^1.1.4"
+    is-shared-array-buffer "^1.0.2"
+    is-string "^1.0.7"
+    is-typed-array "^1.1.12"
+    is-weakref "^1.0.2"
+    object-inspect "^1.13.1"
+    object-keys "^1.1.1"
+    object.assign "^4.1.4"
+    regexp.prototype.flags "^1.5.1"
+    safe-array-concat "^1.0.1"
+    safe-regex-test "^1.0.0"
+    string.prototype.trim "^1.2.8"
+    string.prototype.trimend "^1.0.7"
+    string.prototype.trimstart "^1.0.7"
+    typed-array-buffer "^1.0.0"
+    typed-array-byte-length "^1.0.0"
+    typed-array-byte-offset "^1.0.0"
+    typed-array-length "^1.0.4"
+    unbox-primitive "^1.0.2"
+    which-typed-array "^1.1.13"
+
 es-aggregate-error@^1.0.8:
   version "1.0.9"
   resolved "https://registry.yarnpkg.com/es-aggregate-error/-/es-aggregate-error-1.0.9.tgz#b50925cdf78c8a634bd766704f6f7825902be3d9"
@@ -9932,6 +10074,13 @@ es-set-tostringtag@^2.0.1:
     has "^1.0.3"
     has-tostringtag "^1.0.0"
 
+es-shim-unscopables@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/es-shim-unscopables/-/es-shim-unscopables-1.0.2.tgz#1f6942e71ecc7835ed1c8a83006d8771a63a3763"
+  integrity sha512-J3yBRXCzDu4ULnQwxyToo/OjdMx6akgVC7K6few0a7F/0wLtmKKN7I73AH5T2836UuXRqN7Qg+IIUw/+YJksRw==
+  dependencies:
+    hasown "^2.0.0"
+
 es-to-primitive@^1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.1.tgz#e55cd4c9cdc188bcefb03b366c736323fc5c898a"
@@ -10141,6 +10290,45 @@ escodegen@^2.0.0:
   optionalDependencies:
     source-map "~0.6.1"
 
+eslint-import-resolver-node@^0.3.9:
+  version "0.3.9"
+  resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz#d4eaac52b8a2e7c3cd1903eb00f7e053356118ac"
+  integrity sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g==
+  dependencies:
+    debug "^3.2.7"
+    is-core-module "^2.13.0"
+    resolve "^1.22.4"
+
+eslint-module-utils@^2.8.0:
+  version "2.8.0"
+  resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.8.0.tgz#e439fee65fc33f6bba630ff621efc38ec0375c49"
+  integrity sha512-aWajIYfsqCKRDgUfjEXNN/JlrzauMuSEy5sbd7WXbtW3EH6A6MpwEh42c7qD+MqQo9QMJ6fWLAeIJynx0g6OAw==
+  dependencies:
+    debug "^3.2.7"
+
+eslint-plugin-import@^2.29.0:
+  version "2.29.0"
+  resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.29.0.tgz#8133232e4329ee344f2f612885ac3073b0b7e155"
+  integrity sha512-QPOO5NO6Odv5lpoTkddtutccQjysJuFxoPS7fAHO+9m9udNHvTCPSAMW9zGAYj8lAIdr40I8yPCdUYrncXtrwg==
+  dependencies:
+    array-includes "^3.1.7"
+    array.prototype.findlastindex "^1.2.3"
+    array.prototype.flat "^1.3.2"
+    array.prototype.flatmap "^1.3.2"
+    debug "^3.2.7"
+    doctrine "^2.1.0"
+    eslint-import-resolver-node "^0.3.9"
+    eslint-module-utils "^2.8.0"
+    hasown "^2.0.0"
+    is-core-module "^2.13.1"
+    is-glob "^4.0.3"
+    minimatch "^3.1.2"
+    object.fromentries "^2.0.7"
+    object.groupby "^1.0.1"
+    object.values "^1.1.7"
+    semver "^6.3.1"
+    tsconfig-paths "^3.14.2"
+
 eslint-plugin-svelte@^2.32.2:
   version "2.32.2"
   resolved "https://registry.yarnpkg.com/eslint-plugin-svelte/-/eslint-plugin-svelte-2.32.2.tgz#d8f1352b55967445ee8d57aaee55f99712696a30"
@@ -11068,6 +11256,11 @@ function-bind@^1.1.1:
   resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d"
   integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==
 
+function-bind@^1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c"
+  integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==
+
 function.prototype.name@^1.1.5:
   version "1.1.5"
   resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.5.tgz#cce0505fe1ffb80503e6f9e46cc64e46a12a9621"
@@ -11078,6 +11271,16 @@ function.prototype.name@^1.1.5:
     es-abstract "^1.19.0"
     functions-have-names "^1.2.2"
 
+function.prototype.name@^1.1.6:
+  version "1.1.6"
+  resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.6.tgz#cdf315b7d90ee77a4c6ee216c3c3362da07533fd"
+  integrity sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg==
+  dependencies:
+    call-bind "^1.0.2"
+    define-properties "^1.2.0"
+    es-abstract "^1.22.1"
+    functions-have-names "^1.2.3"
+
 functional-red-black-tree@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327"
@@ -11234,6 +11437,16 @@ get-intrinsic@^1.0.2, get-intrinsic@^1.1.1, get-intrinsic@^1.1.3, get-intrinsic@
     has "^1.0.3"
     has-symbols "^1.0.3"
 
+get-intrinsic@^1.2.1, get-intrinsic@^1.2.2:
+  version "1.2.2"
+  resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.2.tgz#281b7622971123e1ef4b3c90fd7539306da93f3b"
+  integrity sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==
+  dependencies:
+    function-bind "^1.1.2"
+    has-proto "^1.0.1"
+    has-symbols "^1.0.3"
+    hasown "^2.0.0"
+
 get-object@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/get-object/-/get-object-0.2.0.tgz#d92ff7d5190c64530cda0543dac63a3d47fe8c0c"
@@ -11937,6 +12150,13 @@ hash.js@^1.0.0, hash.js@^1.0.3:
     inherits "^2.0.3"
     minimalistic-assert "^1.0.1"
 
+hasown@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.0.tgz#f4c513d454a57b7c7e1650778de226b11700546c"
+  integrity sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==
+  dependencies:
+    function-bind "^1.1.2"
+
 help-me@^4.0.1:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/help-me/-/help-me-4.2.0.tgz#50712bfd799ff1854ae1d312c36eafcea85b0563"
@@ -12407,6 +12627,15 @@ internal-slot@^1.0.4:
     has "^1.0.3"
     side-channel "^1.0.4"
 
+internal-slot@^1.0.5:
+  version "1.0.6"
+  resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.6.tgz#37e756098c4911c5e912b8edbf71ed3aa116f930"
+  integrity sha512-Xj6dv+PsbtwyPpEflsejS+oIZxmMlV44zAhG479uYu89MsjcYOhCFnNyKrkJrihbsiasQyY0afoCl/9BLR65bg==
+  dependencies:
+    get-intrinsic "^1.2.2"
+    hasown "^2.0.0"
+    side-channel "^1.0.4"
+
 interpret@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/interpret/-/interpret-2.2.0.tgz#1a78a0b5965c40a5416d007ad6f50ad27c417df9"
@@ -12508,7 +12737,7 @@ is-arguments@^1.1.1:
     call-bind "^1.0.2"
     has-tostringtag "^1.0.0"
 
-is-array-buffer@^3.0.1:
+is-array-buffer@^3.0.1, is-array-buffer@^3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/is-array-buffer/-/is-array-buffer-3.0.2.tgz#f2653ced8412081638ecb0ebbd0c41c6e0aecbbe"
   integrity sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==
@@ -12585,6 +12814,13 @@ is-core-module@2.9.0, is-core-module@^2.5.0, is-core-module@^2.8.1, is-core-modu
   dependencies:
     has "^1.0.3"
 
+is-core-module@^2.13.0, is-core-module@^2.13.1:
+  version "2.13.1"
+  resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.13.1.tgz#ad0d7532c6fea9da1ebdc82742d74525c6273384"
+  integrity sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==
+  dependencies:
+    hasown "^2.0.0"
+
 is-data-descriptor@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz#0b5ee648388e2c860282e793f1856fec3f301b56"
@@ -12935,6 +13171,13 @@ is-typed-array@^1.1.10, is-typed-array@^1.1.9:
     gopd "^1.0.1"
     has-tostringtag "^1.0.0"
 
+is-typed-array@^1.1.12:
+  version "1.1.12"
+  resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.12.tgz#d0bab5686ef4a76f7a73097b95470ab199c57d4a"
+  integrity sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==
+  dependencies:
+    which-typed-array "^1.1.11"
+
 is-typedarray@^1.0.0, is-typedarray@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a"
@@ -13794,10 +14037,10 @@ json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1:
   resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
   integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==
 
-json5@^1.0.1:
+json5@^1.0.1, json5@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593"
-  integrity "sha1-Y9mNYPIbMTt3xNbaGL+mnYDh1ZM= sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA=="
+  integrity sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==
   dependencies:
     minimist "^1.2.0"
 
@@ -16298,6 +16541,11 @@ object-inspect@^1.12.2, object-inspect@^1.9.0:
   resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.3.tgz#ba62dffd67ee256c8c086dfae69e016cd1f198b9"
   integrity sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==
 
+object-inspect@^1.13.1:
+  version "1.13.1"
+  resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.1.tgz#b96c6109324ccfef6b12216a956ca4dc2ff94bc2"
+  integrity sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==
+
 object-is@^1.1.5:
   version "1.1.5"
   resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.5.tgz#b9deeaa5fc7f1846a0faecdceec138e5778f53ac"
@@ -16349,6 +16597,25 @@ object.assign@^4.1.4:
     has-symbols "^1.0.3"
     object-keys "^1.1.1"
 
+object.fromentries@^2.0.7:
+  version "2.0.7"
+  resolved "https://registry.yarnpkg.com/object.fromentries/-/object.fromentries-2.0.7.tgz#71e95f441e9a0ea6baf682ecaaf37fa2a8d7e616"
+  integrity sha512-UPbPHML6sL8PI/mOqPwsH4G6iyXcCGzLin8KvEPenOZN5lpCNBZZQ+V62vdjB1mQHrmqGQt5/OJzemUA+KJmEA==
+  dependencies:
+    call-bind "^1.0.2"
+    define-properties "^1.2.0"
+    es-abstract "^1.22.1"
+
+object.groupby@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/object.groupby/-/object.groupby-1.0.1.tgz#d41d9f3c8d6c778d9cbac86b4ee9f5af103152ee"
+  integrity sha512-HqaQtqLnp/8Bn4GL16cj+CUYbnpe1bh0TtEaWvybszDG4tgxCJuRpV8VGuvNaI1fAnI4lUJzDG55MXcOH4JZcQ==
+  dependencies:
+    call-bind "^1.0.2"
+    define-properties "^1.2.0"
+    es-abstract "^1.22.1"
+    get-intrinsic "^1.2.1"
+
 object.pick@^1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/object.pick/-/object.pick-1.3.0.tgz#87a10ac4c1694bd2e1cbf53591a66141fb5dd747"
@@ -16356,6 +16623,15 @@ object.pick@^1.3.0:
   dependencies:
     isobject "^3.0.1"
 
+object.values@^1.1.7:
+  version "1.1.7"
+  resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.7.tgz#617ed13272e7e1071b43973aa1655d9291b8442a"
+  integrity sha512-aU6xnDFYT3x17e/f0IiiwlGPTy2jzMySGfUB4fq6z7CV8l85CWHDk5ErhyhpfDHhrOMwGFhSQkhMGHaIotA6Ng==
+  dependencies:
+    call-bind "^1.0.2"
+    define-properties "^1.2.0"
+    es-abstract "^1.22.1"
+
 octal@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/octal/-/octal-1.0.0.tgz#63e7162a68efbeb9e213588d58e989d1e5c4530b"
@@ -18668,6 +18944,15 @@ regexp.prototype.flags@^1.4.3:
     define-properties "^1.1.3"
     functions-have-names "^1.2.2"
 
+regexp.prototype.flags@^1.5.1:
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.1.tgz#90ce989138db209f81492edd734183ce99f9677e"
+  integrity sha512-sy6TXMN+hnP/wMy+ISxg3krXx7BAtWVO4UouuCN/ziM9UEne0euamVNafDfvC83bRNr95y0V5iijeDQFUNpvrg==
+  dependencies:
+    call-bind "^1.0.2"
+    define-properties "^1.2.0"
+    set-function-name "^2.0.0"
+
 regexparam@2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-2.0.1.tgz#c912f5dae371e3798100b3c9ce22b7414d0889fa"
@@ -18879,6 +19164,15 @@ resolve@^1.10.0, resolve@^1.11.1, resolve@^1.14.2, resolve@^1.17.0, resolve@^1.1
     path-parse "^1.0.7"
     supports-preserve-symlinks-flag "^1.0.0"
 
+resolve@^1.22.4:
+  version "1.22.8"
+  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d"
+  integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==
+  dependencies:
+    is-core-module "^2.13.0"
+    path-parse "^1.0.7"
+    supports-preserve-symlinks-flag "^1.0.0"
+
 responselike@1.0.2, responselike@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/responselike/-/responselike-1.0.2.tgz#918720ef3b631c5642be068f15ade5a46f4ba1e7"
@@ -19153,6 +19447,16 @@ rxjs@^7.5.5:
   dependencies:
     tslib "^2.1.0"
 
+safe-array-concat@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/safe-array-concat/-/safe-array-concat-1.0.1.tgz#91686a63ce3adbea14d61b14c99572a8ff84754c"
+  integrity sha512-6XbUAseYE2KtOuGueyeobCySj9L4+66Tn6KQMOPQJrAJEowYKW/YR/MGJZl7FdydUdaFu4LYyDZjxf4/Nmo23Q==
+  dependencies:
+    call-bind "^1.0.2"
+    get-intrinsic "^1.2.1"
+    has-symbols "^1.0.3"
+    isarray "^2.0.5"
+
 safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@^5.2.0, safe-buffer@~5.2.0:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
@@ -19334,6 +19638,11 @@ semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0:
   resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d"
   integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==
 
+semver@^6.3.1:
+  version "6.3.1"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
+  integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==
+
 semver@^7.0.0, semver@^7.1.1, semver@^7.1.2, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4:
   version "7.5.4"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e"
@@ -19387,6 +19696,25 @@ set-blocking@^2.0.0, set-blocking@~2.0.0:
   resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"
   integrity sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==
 
+set-function-length@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.1.1.tgz#4bc39fafb0307224a33e106a7d35ca1218d659ed"
+  integrity sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ==
+  dependencies:
+    define-data-property "^1.1.1"
+    get-intrinsic "^1.2.1"
+    gopd "^1.0.1"
+    has-property-descriptors "^1.0.0"
+
+set-function-name@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/set-function-name/-/set-function-name-2.0.1.tgz#12ce38b7954310b9f61faa12701620a0c882793a"
+  integrity sha512-tMNCiqYVkXIZgc2Hnoy2IvC/f8ezc5koaRFkCjrpWzGpCd3qbZXPzVy9MAZzK1ch/X0jvSkojys3oqJN0qCmdA==
+  dependencies:
+    define-data-property "^1.0.1"
+    functions-have-names "^1.2.3"
+    has-property-descriptors "^1.0.0"
+
 set-value@^2.0.0, set-value@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/set-value/-/set-value-2.0.1.tgz#a18d40530e6f07de4228c7defe4227af8cad005b"
@@ -20109,6 +20437,15 @@ string.prototype.startswith@^1.0.0:
     define-properties "^1.1.3"
     es-abstract "^1.17.5"
 
+string.prototype.trim@^1.2.8:
+  version "1.2.8"
+  resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.2.8.tgz#f9ac6f8af4bd55ddfa8895e6aea92a96395393bd"
+  integrity sha512-lfjY4HcixfQXOfaqCvcBuOIapyaroTXhbkfJN3gcB1OtyupngWK4sEET9Knd0cXd28kTUqu/kHoV4HKSJdnjiQ==
+  dependencies:
+    call-bind "^1.0.2"
+    define-properties "^1.2.0"
+    es-abstract "^1.22.1"
+
 string.prototype.trimend@^1.0.6:
   version "1.0.6"
   resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.6.tgz#c4a27fa026d979d79c04f17397f250a462944533"
@@ -20118,6 +20455,15 @@ string.prototype.trimend@^1.0.6:
     define-properties "^1.1.4"
     es-abstract "^1.20.4"
 
+string.prototype.trimend@^1.0.7:
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.7.tgz#1bb3afc5008661d73e2dc015cd4853732d6c471e"
+  integrity sha512-Ni79DqeB72ZFq1uH/L6zJ+DKZTkOtPIHovb3YZHQViE+HDouuU4mBrLOLDn5Dde3RF8qw5qVETEjhu9locMLvA==
+  dependencies:
+    call-bind "^1.0.2"
+    define-properties "^1.2.0"
+    es-abstract "^1.22.1"
+
 string.prototype.trimstart@^1.0.6:
   version "1.0.6"
   resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.6.tgz#e90ab66aa8e4007d92ef591bbf3cd422c56bdcf4"
@@ -20127,6 +20473,15 @@ string.prototype.trimstart@^1.0.6:
     define-properties "^1.1.4"
     es-abstract "^1.20.4"
 
+string.prototype.trimstart@^1.0.7:
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.7.tgz#d4cdb44b83a4737ffbac2d406e405d43d0184298"
+  integrity sha512-NGhtDFu3jCEm7B4Fy0DpLewdJQOZcQ0rGbwQ/+stjnrp2i+rlKeCvos9hOIeCmqwratM47OBxY7uFZzjxHXmrg==
+  dependencies:
+    call-bind "^1.0.2"
+    define-properties "^1.2.0"
+    es-abstract "^1.22.1"
+
 string_decoder@^1.1.1:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
@@ -21079,6 +21434,16 @@ tsconfig-paths@^3.10.1:
     minimist "^1.2.6"
     strip-bom "^3.0.0"
 
+tsconfig-paths@^3.14.2:
+  version "3.14.2"
+  resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz#6e32f1f79412decd261f92d633a9dc1cfa99f088"
+  integrity sha512-o/9iXgCYc5L/JxCHPe3Hvh8Q/2xm5Z+p18PESBU6Ff33695QnCHBEjcytY2q19ua7Mbl/DavtBOLq+oG0RCL+g==
+  dependencies:
+    "@types/json5" "^0.0.29"
+    json5 "^1.0.2"
+    minimist "^1.2.6"
+    strip-bom "^3.0.0"
+
 tsconfig-paths@^4.1.2, tsconfig-paths@^4.2.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-4.2.0.tgz#ef78e19039133446d244beac0fd6a1632e2d107c"
@@ -21203,6 +21568,36 @@ type-is@^1.6.14, type-is@^1.6.16, type-is@^1.6.18:
     media-typer "0.3.0"
     mime-types "~2.1.24"
 
+typed-array-buffer@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/typed-array-buffer/-/typed-array-buffer-1.0.0.tgz#18de3e7ed7974b0a729d3feecb94338d1472cd60"
+  integrity sha512-Y8KTSIglk9OZEr8zywiIHG/kmQ7KWyjseXs1CbSo8vC42w7hg2HgYTxSWwP0+is7bWDc1H+Fo026CpHFwm8tkw==
+  dependencies:
+    call-bind "^1.0.2"
+    get-intrinsic "^1.2.1"
+    is-typed-array "^1.1.10"
+
+typed-array-byte-length@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/typed-array-byte-length/-/typed-array-byte-length-1.0.0.tgz#d787a24a995711611fb2b87a4052799517b230d0"
+  integrity sha512-Or/+kvLxNpeQ9DtSydonMxCx+9ZXOswtwJn17SNLvhptaXYDJvkFFP5zbfU/uLmvnBJlI4yrnXRxpdWH/M5tNA==
+  dependencies:
+    call-bind "^1.0.2"
+    for-each "^0.3.3"
+    has-proto "^1.0.1"
+    is-typed-array "^1.1.10"
+
+typed-array-byte-offset@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/typed-array-byte-offset/-/typed-array-byte-offset-1.0.0.tgz#cbbe89b51fdef9cd6aaf07ad4707340abbc4ea0b"
+  integrity sha512-RD97prjEt9EL8YgAgpOkf3O4IF9lhJFr9g0htQkm0rchFp/Vx7LW5Q8fSXXub7BXAODyUQohRMyOc3faCPd0hg==
+  dependencies:
+    available-typed-arrays "^1.0.5"
+    call-bind "^1.0.2"
+    for-each "^0.3.3"
+    has-proto "^1.0.1"
+    is-typed-array "^1.1.10"
+
 typed-array-length@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/typed-array-length/-/typed-array-length-1.0.4.tgz#89d83785e5c4098bec72e08b319651f0eac9c1bb"
@@ -21978,6 +22373,17 @@ which-module@^2.0.0:
   resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a"
   integrity sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q==
 
+which-typed-array@^1.1.11, which-typed-array@^1.1.13:
+  version "1.1.13"
+  resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.13.tgz#870cd5be06ddb616f504e7b039c4c24898184d36"
+  integrity sha512-P5Nra0qjSncduVPEAr7xhoF5guty49ArDTwzJ/yNuPIbZppyRxFQsRCWrocxIY+CnMVG+qfbU2FmDKyvSGClow==
+  dependencies:
+    available-typed-arrays "^1.0.5"
+    call-bind "^1.0.4"
+    for-each "^0.3.3"
+    gopd "^1.0.1"
+    has-tostringtag "^1.0.0"
+
 which-typed-array@^1.1.9:
   version "1.1.9"
   resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.9.tgz#307cf898025848cf995e795e8423c7f337efbde6"

From e8abb5cb46b071d813731b8390cd31ecd6759818 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 14:39:52 +0100
Subject: [PATCH 112/155] Detect non-barrel workspace usages

---
 .eslintrc.json                     |  5 +++--
 eslint-local-rules/index.js        | 21 +++++++++++++++++++++
 package.json                       |  1 +
 packages/backend-core/src/index.ts |  1 +
 yarn.lock                          |  5 +++++
 5 files changed, 31 insertions(+), 2 deletions(-)
 create mode 100644 eslint-local-rules/index.js

diff --git a/.eslintrc.json b/.eslintrc.json
index 783a212f11..f6f03c6523 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -19,7 +19,7 @@
     "bundle.js"
   ],
   "extends": ["eslint:recommended"],
-  "plugins": ["import"],
+  "plugins": ["import", "eslint-plugin-local-rules"],
   "overrides": [
     {
       "files": ["**/*.svelte"],
@@ -42,7 +42,8 @@
         "no-case-declarations": "off",
         "no-useless-escape": "off",
         "no-undef": "off",
-        "no-prototype-builtins": "off"
+        "no-prototype-builtins": "off",
+        "local-rules/no-budibase-imports": "error"
       }
     }
   ],
diff --git a/eslint-local-rules/index.js b/eslint-local-rules/index.js
new file mode 100644
index 0000000000..af02599c90
--- /dev/null
+++ b/eslint-local-rules/index.js
@@ -0,0 +1,21 @@
+module.exports = {
+  "no-budibase-imports": {
+    create: function (context) {
+      return {
+        ImportDeclaration(node) {
+          const importPath = node.source.value
+
+          if (
+            /^@budibase\/[^/]+\/.*$/.test(importPath) &&
+            importPath !== "@budibase/backend-core/tests"
+          ) {
+            context.report({
+              node,
+              message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests.`,
+            })
+          }
+        },
+      }
+    },
+  },
+}
diff --git a/package.json b/package.json
index 29d5560955..2978483448 100644
--- a/package.json
+++ b/package.json
@@ -11,6 +11,7 @@
     "esbuild-node-externals": "^1.8.0",
     "eslint": "^8.44.0",
     "eslint-plugin-import": "^2.29.0",
+    "eslint-plugin-local-rules": "^2.0.0",
     "eslint-plugin-svelte": "^2.32.2",
     "husky": "^8.0.3",
     "kill-port": "^1.6.1",
diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts
index c7cf9f56cc..e569f27b88 100644
--- a/packages/backend-core/src/index.ts
+++ b/packages/backend-core/src/index.ts
@@ -32,6 +32,7 @@ export * as blacklist from "./blacklist"
 export * as docUpdates from "./docUpdates"
 export * from "./utils/Duration"
 export { SearchParams } from "./db"
+export * as docIds from "./docIds"
 // Add context to tenancy for backwards compatibility
 // only do this for external usages to prevent internal
 // circular dependencies
diff --git a/yarn.lock b/yarn.lock
index 50c497c6c2..700c3f9456 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -10329,6 +10329,11 @@ eslint-plugin-import@^2.29.0:
     semver "^6.3.1"
     tsconfig-paths "^3.14.2"
 
+eslint-plugin-local-rules@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/eslint-plugin-local-rules/-/eslint-plugin-local-rules-2.0.0.tgz#cda95d7616cc0e2609d76c347c187ca2be1e252e"
+  integrity sha512-sWueme0kUcP0JC1+6OBDQ9edBDVFJR92WJHSRbhiRExlenMEuUisdaVBPR+ItFBFXo2Pdw6FD2UfGZWkz8e93g==
+
 eslint-plugin-svelte@^2.32.2:
   version "2.32.2"
   resolved "https://registry.yarnpkg.com/eslint-plugin-svelte/-/eslint-plugin-svelte-2.32.2.tgz#d8f1352b55967445ee8d57aaee55f99712696a30"

From d0e40afbcb9cb8bfe82e2d5865ff4a310e02dae8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 14:42:37 +0100
Subject: [PATCH 113/155] Fix lint issues

---
 packages/client/src/components/app/forms/S3Upload.svelte | 2 +-
 packages/pro                                             | 2 +-
 packages/server/src/sdk/app/links/links.ts               | 8 ++++----
 packages/server/src/sdk/app/tables/migration.ts          | 3 +--
 4 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/packages/client/src/components/app/forms/S3Upload.svelte b/packages/client/src/components/app/forms/S3Upload.svelte
index 9985c83bb8..5ead872863 100644
--- a/packages/client/src/components/app/forms/S3Upload.svelte
+++ b/packages/client/src/components/app/forms/S3Upload.svelte
@@ -2,7 +2,7 @@
   import Field from "./Field.svelte"
   import { CoreDropzone, ProgressCircle } from "@budibase/bbui"
   import { getContext, onMount, onDestroy } from "svelte"
-  import { cloneDeep } from "../../../../../bbui/src/helpers"
+  import { cloneDeep } from "@budibase/bbui/src/helpers"
 
   export let datasourceId
   export let bucket
diff --git a/packages/pro b/packages/pro
index e202f415d9..d017f81efd 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit e202f415d9fa540d08cc2ba6e27394fbc22f357b
+Subproject commit d017f81efdfc5fef3ec6c185cbccba54213be0b1
diff --git a/packages/server/src/sdk/app/links/links.ts b/packages/server/src/sdk/app/links/links.ts
index fda07568f9..5d3420341f 100644
--- a/packages/server/src/sdk/app/links/links.ts
+++ b/packages/server/src/sdk/app/links/links.ts
@@ -1,5 +1,5 @@
-import { context } from "@budibase/backend-core"
-import { isTableId } from "@budibase/backend-core/src/docIds"
+import { context, docIds } from "@budibase/backend-core"
+
 import {
   DatabaseQueryOpts,
   LinkDocument,
@@ -8,7 +8,7 @@ import {
 import { ViewName, getQueryIndex } from "../../../../src/db/utils"
 
 export async function fetch(tableId: string): Promise<LinkDocumentValue[]> {
-  if (!isTableId(tableId)) {
+  if (!docIds.isTableId(tableId)) {
     throw new Error(`Invalid tableId: ${tableId}`)
   }
 
@@ -24,7 +24,7 @@ export async function fetch(tableId: string): Promise<LinkDocumentValue[]> {
 export async function fetchWithDocument(
   tableId: string
 ): Promise<LinkDocument[]> {
-  if (!isTableId(tableId)) {
+  if (!docIds.isTableId(tableId)) {
     throw new Error(`Invalid tableId: ${tableId}`)
   }
 
diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts
index 718223dbeb..e282251bfb 100644
--- a/packages/server/src/sdk/app/tables/migration.ts
+++ b/packages/server/src/sdk/app/tables/migration.ts
@@ -17,7 +17,6 @@ import sdk from "../../../sdk"
 import { isExternalTableID } from "../../../integrations/utils"
 import { EventType, updateLinks } from "../../../db/linkedRows"
 import { cloneDeep } from "lodash"
-import { isInternalColumnName } from "@budibase/backend-core/src/db"
 
 export interface MigrationResult {
   tablesUpdated: Table[]
@@ -36,7 +35,7 @@ export async function migrate(
     throw new BadRequestError(`Column name cannot be empty`)
   }
 
-  if (isInternalColumnName(newColumn.name)) {
+  if (dbCore.isInternalColumnName(newColumn.name)) {
     throw new BadRequestError(`Column name cannot be a reserved column name`)
   }
 

From 9a097c37f2f12cb06ff4d80406a86cdec0aa22c5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 14:59:25 +0100
Subject: [PATCH 114/155] Update master ref

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index d017f81efd..2cf6f28380 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit d017f81efdfc5fef3ec6c185cbccba54213be0b1
+Subproject commit 2cf6f28380d3ab22128b8a889d622fd5adfa31fc

From f70bb967e69211d5c62b6aaa461a87ea3920918c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 15:40:26 +0100
Subject: [PATCH 115/155] Fix references

---
 packages/client/src/components/app/forms/S3Upload.svelte | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/client/src/components/app/forms/S3Upload.svelte b/packages/client/src/components/app/forms/S3Upload.svelte
index 5ead872863..fd9af0489c 100644
--- a/packages/client/src/components/app/forms/S3Upload.svelte
+++ b/packages/client/src/components/app/forms/S3Upload.svelte
@@ -1,8 +1,7 @@
 <script>
   import Field from "./Field.svelte"
-  import { CoreDropzone, ProgressCircle } from "@budibase/bbui"
+  import { CoreDropzone, ProgressCircle, Helpers } from "@budibase/bbui"
   import { getContext, onMount, onDestroy } from "svelte"
-  import { cloneDeep } from "@budibase/bbui/src/helpers"
 
   export let datasourceId
   export let bucket
@@ -99,8 +98,9 @@
   }
 
   const handleChange = e => {
+    debugger
     localFiles = e.detail
-    let files = cloneDeep(e.detail) || []
+    let files = Helpers.cloneDeep(e.detail) || []
     // remove URL as it contains the full base64 image data
     files.forEach(file => {
       if (file.type?.startsWith("image")) {

From 4b5fa1dfd901d6488241479e1342877e2b04d82f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 15:53:20 +0100
Subject: [PATCH 116/155] Fix qa-core lint

---
 .eslintrc.json                                     | 14 ++++++++++++++
 .../src/components/app/forms/S3Upload.svelte       |  1 -
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/.eslintrc.json b/.eslintrc.json
index f6f03c6523..3e7d40d2c6 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -34,6 +34,7 @@
     },
     {
       "files": ["**/*.ts"],
+      "excludedFiles": "qa-core/**/*",
       "parser": "@typescript-eslint/parser",
       "extends": ["eslint:recommended"],
       "rules": {
@@ -45,6 +46,19 @@
         "no-prototype-builtins": "off",
         "local-rules/no-budibase-imports": "error"
       }
+    },
+    {
+      "files": ["**/*.ts"],
+      "parser": "@typescript-eslint/parser",
+      "extends": ["eslint:recommended"],
+      "rules": {
+        "no-unused-vars": "off",
+        "no-inner-declarations": "off",
+        "no-case-declarations": "off",
+        "no-useless-escape": "off",
+        "no-undef": "off",
+        "no-prototype-builtins": "off"
+      }
     }
   ],
   "rules": {
diff --git a/packages/client/src/components/app/forms/S3Upload.svelte b/packages/client/src/components/app/forms/S3Upload.svelte
index fd9af0489c..0147cbca6e 100644
--- a/packages/client/src/components/app/forms/S3Upload.svelte
+++ b/packages/client/src/components/app/forms/S3Upload.svelte
@@ -98,7 +98,6 @@
   }
 
   const handleChange = e => {
-    debugger
     localFiles = e.detail
     let files = Helpers.cloneDeep(e.detail) || []
     // remove URL as it contains the full base64 image data

From 27ba888ba0c627651193c8eb580a6d86e832ec3e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 16:06:01 +0100
Subject: [PATCH 117/155] Fix qa-core lint

---
 .eslintrc.json                              | 14 --------------
 qa-core/src/public-api/api/apis/RowAPI.ts   |  6 +-----
 qa-core/src/public-api/api/apis/TableAPI.ts |  6 +-----
 qa-core/src/public-api/api/apis/UserAPI.ts  |  6 +-----
 qa-core/src/public-api/fixtures/users.ts    |  5 +----
 qa-core/src/types/appPackage.ts             |  2 +-
 6 files changed, 5 insertions(+), 34 deletions(-)

diff --git a/.eslintrc.json b/.eslintrc.json
index 3e7d40d2c6..f6f03c6523 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -34,7 +34,6 @@
     },
     {
       "files": ["**/*.ts"],
-      "excludedFiles": "qa-core/**/*",
       "parser": "@typescript-eslint/parser",
       "extends": ["eslint:recommended"],
       "rules": {
@@ -46,19 +45,6 @@
         "no-prototype-builtins": "off",
         "local-rules/no-budibase-imports": "error"
       }
-    },
-    {
-      "files": ["**/*.ts"],
-      "parser": "@typescript-eslint/parser",
-      "extends": ["eslint:recommended"],
-      "rules": {
-        "no-unused-vars": "off",
-        "no-inner-declarations": "off",
-        "no-case-declarations": "off",
-        "no-useless-escape": "off",
-        "no-undef": "off",
-        "no-prototype-builtins": "off"
-      }
     }
   ],
   "rules": {
diff --git a/qa-core/src/public-api/api/apis/RowAPI.ts b/qa-core/src/public-api/api/apis/RowAPI.ts
index 68649db5a3..00c2aec2e0 100644
--- a/qa-core/src/public-api/api/apis/RowAPI.ts
+++ b/qa-core/src/public-api/api/apis/RowAPI.ts
@@ -1,8 +1,4 @@
-import {
-  CreateRowParams,
-  Row,
-  SearchInputParams,
-} from "@budibase/server/api/controllers/public/mapping/types"
+import { CreateRowParams, Row, SearchInputParams } from "../../../types"
 import { HeadersInit, Response } from "node-fetch"
 import BudibasePublicAPIClient from "../BudibasePublicAPIClient"
 import * as fixtures from "../../fixtures"
diff --git a/qa-core/src/public-api/api/apis/TableAPI.ts b/qa-core/src/public-api/api/apis/TableAPI.ts
index b7bafb2b67..4849953211 100644
--- a/qa-core/src/public-api/api/apis/TableAPI.ts
+++ b/qa-core/src/public-api/api/apis/TableAPI.ts
@@ -1,8 +1,4 @@
-import {
-  Table,
-  SearchInputParams,
-  CreateTableParams,
-} from "@budibase/server/api/controllers/public/mapping/types"
+import { Table, SearchInputParams, CreateTableParams } from "../../../types"
 import { HeadersInit, Response } from "node-fetch"
 import { generateTable } from "../../fixtures/tables"
 import BudibasePublicAPIClient from "../BudibasePublicAPIClient"
diff --git a/qa-core/src/public-api/api/apis/UserAPI.ts b/qa-core/src/public-api/api/apis/UserAPI.ts
index 7676a0fdd6..589bec1d40 100644
--- a/qa-core/src/public-api/api/apis/UserAPI.ts
+++ b/qa-core/src/public-api/api/apis/UserAPI.ts
@@ -1,8 +1,4 @@
-import {
-  CreateUserParams,
-  SearchInputParams,
-  User,
-} from "@budibase/server/api/controllers/public/mapping/types"
+import { CreateUserParams, SearchInputParams, User } from "../../../types"
 import { Response } from "node-fetch"
 import BudibasePublicAPIClient from "../BudibasePublicAPIClient"
 import * as fixtures from "../../fixtures"
diff --git a/qa-core/src/public-api/fixtures/users.ts b/qa-core/src/public-api/fixtures/users.ts
index 14c525fc0f..e20c464b34 100644
--- a/qa-core/src/public-api/fixtures/users.ts
+++ b/qa-core/src/public-api/fixtures/users.ts
@@ -1,7 +1,4 @@
-import {
-  CreateUserParams,
-  User,
-} from "@budibase/server/api/controllers/public/mapping/types"
+import { CreateUserParams, User } from "../../types"
 import { generator } from "../../shared"
 
 export const generateUser = (
diff --git a/qa-core/src/types/appPackage.ts b/qa-core/src/types/appPackage.ts
index 0285565dea..42c89b6138 100644
--- a/qa-core/src/types/appPackage.ts
+++ b/qa-core/src/types/appPackage.ts
@@ -1,4 +1,4 @@
-import { Application } from "@budibase/server/api/controllers/public/mapping/types"
+import { Application } from "../types"
 import { Layout } from "@budibase/types"
 import { Screen } from "@budibase/types"
 // Create type for getAppPackage response

From 5489a25fabe3b95a9ca8397ed844583232673863 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 16:59:09 +0100
Subject: [PATCH 118/155] Lint action

---
 .github/workflows/budibase_ci.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 77867c8617..818fb07145 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -38,7 +38,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: "yarn"
+          cache: yarn
       - run: yarn --frozen-lockfile
       - run: yarn lint
 
@@ -62,7 +62,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: "yarn"
+          cache: yarn
       - run: yarn --frozen-lockfile
 
       # Run build all the projects
@@ -98,7 +98,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: "yarn"
+          cache: yarn
       - run: yarn --frozen-lockfile
       - name: Test
         run: |
@@ -133,7 +133,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: "yarn"
+          cache: yarn
       - run: yarn --frozen-lockfile
       - name: Test worker
         run: |
@@ -169,7 +169,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: "yarn"
+          cache: yarn
       - run: yarn --frozen-lockfile
       - name: Test server
         run: |
@@ -200,7 +200,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: "yarn"
+          cache: yarn
       - run: yarn --frozen-lockfile
       - name: Test
         run: |
@@ -227,7 +227,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: "yarn"
+          cache: yarn
       - run: yarn --frozen-lockfile
       - name: Build packages
         run: yarn build --scope @budibase/server --scope @budibase/worker

From 44c436c25b4333e75c7e56d01dd300c72055c5ec Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 17:08:41 +0100
Subject: [PATCH 119/155] Unify checkouts

---
 .github/workflows/budibase_ci.yml | 57 +++++++------------------------
 1 file changed, 13 insertions(+), 44 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 818fb07145..3aad335ddf 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -19,20 +19,17 @@ env:
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   NX_BASE_BRANCH: origin/${{ github.base_ref }}
   USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' }}
+  IS_OSS_CONTRIBUTOR: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' }}
 
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repo and submodules
+      - name: Checkout repo
         uses: actions/checkout@v3
-        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
-          submodules: true
+          submodules: ${{ !env.IS_OSS_CONTRIBUTOR }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
-      - name: Checkout repo only
-        uses: actions/checkout@v3
-        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3
@@ -45,18 +42,12 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repo and submodules
+      - name: Checkout repo
         uses: actions/checkout@v3
-        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
-          submodules: true
+          submodules: ${{ !env.IS_OSS_CONTRIBUTOR }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
           fetch-depth: 0
-      - name: Checkout repo only
-        uses: actions/checkout@v3
-        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
-        with:
-          fetch-depth: 0
 
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3
@@ -81,18 +72,12 @@ jobs:
   test-libraries:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repo and submodules
+      - name: Checkout repo
         uses: actions/checkout@v3
-        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
-          submodules: true
+          submodules: ${{ !env.IS_OSS_CONTRIBUTOR }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
           fetch-depth: 0
-      - name: Checkout repo only
-        uses: actions/checkout@v3
-        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
-        with:
-          fetch-depth: 0
 
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3
@@ -116,18 +101,12 @@ jobs:
   test-worker:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repo and submodules
+      - name: Checkout repo
         uses: actions/checkout@v3
-        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
-          submodules: true
+          submodules: ${{ !env.IS_OSS_CONTRIBUTOR }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
           fetch-depth: 0
-      - name: Checkout repo only
-        uses: actions/checkout@v3
-        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
-        with:
-          fetch-depth: 0
 
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3
@@ -152,18 +131,12 @@ jobs:
   test-server:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repo and submodules
+      - name: Checkout repo
         uses: actions/checkout@v3
-        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
-          submodules: true
+          submodules: ${{ !env.IS_OSS_CONTRIBUTOR }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
           fetch-depth: 0
-      - name: Checkout repo only
-        uses: actions/checkout@v3
-        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
-        with:
-          fetch-depth: 0
 
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3
@@ -213,15 +186,11 @@ jobs:
   integration-test:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repo and submodules
+      - name: Checkout repo
         uses: actions/checkout@v3
-        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
-          submodules: true
+          submodules: ${{ !env.IS_OSS_CONTRIBUTOR }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
-      - name: Checkout repo only
-        uses: actions/checkout@v3
-        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3

From 40ff3b88afae3baaf3d51bb2579f81214b9808df Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 17:15:54 +0100
Subject: [PATCH 120/155] Conditional cache

---
 .github/workflows/budibase_ci.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 3aad335ddf..c731e8f4c7 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -35,7 +35,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: yarn
+          cache: ${{ !env.is_oss_contributor && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - run: yarn lint
 
@@ -53,7 +53,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: yarn
+          cache: ${{ !env.is_oss_contributor && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
 
       # Run build all the projects
@@ -83,7 +83,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: yarn
+          cache: ${{ !env.is_oss_contributor && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Test
         run: |
@@ -112,7 +112,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: yarn
+          cache: ${{ !env.is_oss_contributor && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Test worker
         run: |
@@ -142,7 +142,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: yarn
+          cache: ${{ !env.is_oss_contributor && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Test server
         run: |
@@ -173,7 +173,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: yarn
+          cache: ${{ !env.is_oss_contributor && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Test
         run: |
@@ -196,7 +196,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: yarn
+          cache: ${{ !env.is_oss_contributor && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Build packages
         run: yarn build --scope @budibase/server --scope @budibase/worker

From 1641b5a18e380852dd130eee0415d681cdacb560 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 17:16:36 +0100
Subject: [PATCH 121/155] Fix condition

---
 .github/workflows/budibase_ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index c731e8f4c7..3109183989 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -19,7 +19,7 @@ env:
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   NX_BASE_BRANCH: origin/${{ github.base_ref }}
   USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' }}
-  IS_OSS_CONTRIBUTOR: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' }}
+  IS_OSS_CONTRIBUTOR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' }}
 
 jobs:
   lint:

From 7fb6c0927a16f394dfdbe881298462f1c4ebd719 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 17 Nov 2023 16:20:10 +0000
Subject: [PATCH 122/155] Moving things around to get rid of cyclics created by
 moving invite/passwordReset into backend-core - also updating currentapp test
 case to mock a bit better.

---
 packages/backend-core/src/cache/generic.ts    | 20 ++--
 packages/backend-core/src/cache/index.ts      |  2 +
 .../src/{redis => cache}/invite.ts            | 37 +++----
 .../src/{redis => cache}/passwordReset.ts     | 21 ++--
 packages/backend-core/src/redis/index.ts      |  2 -
 packages/backend-core/src/redis/init.ts       | 22 ++++-
 packages/backend-core/src/users/lookup.ts     |  2 +-
 .../src/middleware/tests/currentapp.spec.js   | 97 +++++--------------
 .../src/api/controllers/global/users.ts       | 13 ++-
 packages/worker/src/index.ts                  |  9 +-
 packages/worker/src/sdk/auth/auth.ts          |  4 +-
 packages/worker/src/utilities/email.ts        |  7 +-
 12 files changed, 95 insertions(+), 141 deletions(-)
 rename packages/backend-core/src/{redis => cache}/invite.ts (74%)
 rename packages/backend-core/src/{redis => cache}/passwordReset.ts (77%)

diff --git a/packages/backend-core/src/cache/generic.ts b/packages/backend-core/src/cache/generic.ts
index 7cd5d6227f..19365e0e3f 100644
--- a/packages/backend-core/src/cache/generic.ts
+++ b/packages/backend-core/src/cache/generic.ts
@@ -1,6 +1,6 @@
-const BaseCache = require("./base")
+import BaseCache from "./base"
 
-const GENERIC = new BaseCache.default()
+const GENERIC = new BaseCache()
 
 export enum CacheKey {
   CHECKLIST = "checklist",
@@ -18,13 +18,9 @@ export enum TTL {
   ONE_DAY = 86400,
 }
 
-function performExport(funcName: string) {
-  return (...args: any) => GENERIC[funcName](...args)
-}
-
-export const keys = performExport("keys")
-export const get = performExport("get")
-export const store = performExport("store")
-export const destroy = performExport("delete")
-export const withCache = performExport("withCache")
-export const bustCache = performExport("bustCache")
+export const keys = GENERIC.keys
+export const get = GENERIC.get
+export const store = GENERIC.store
+export const destroy = GENERIC.delete
+export const withCache = GENERIC.withCache
+export const bustCache = GENERIC.bustCache
diff --git a/packages/backend-core/src/cache/index.ts b/packages/backend-core/src/cache/index.ts
index 58928c271a..4fa986e4e2 100644
--- a/packages/backend-core/src/cache/index.ts
+++ b/packages/backend-core/src/cache/index.ts
@@ -2,4 +2,6 @@ export * as generic from "./generic"
 export * as user from "./user"
 export * as app from "./appMetadata"
 export * as writethrough from "./writethrough"
+export * as invite from "./invite"
+export * as passwordReset from "./passwordReset"
 export * from "./generic"
diff --git a/packages/backend-core/src/redis/invite.ts b/packages/backend-core/src/cache/invite.ts
similarity index 74%
rename from packages/backend-core/src/redis/invite.ts
rename to packages/backend-core/src/cache/invite.ts
index 006a06fe26..1fec52122e 100644
--- a/packages/backend-core/src/redis/invite.ts
+++ b/packages/backend-core/src/cache/invite.ts
@@ -1,7 +1,10 @@
-import { utils, tenancy, redis } from "../"
+import * as utils from "../utils"
+import { Duration, DurationType } from "../utils"
 import env from "../environment"
+import { getTenantId } from "../context"
+import * as redis from "../redis/init"
 
-const TTL_SECONDS = 60 * 60 * 24 * 7
+const TTL_SECONDS = Duration.fromDays(7).to(DurationType.SECONDS)
 
 interface Invite {
   email: string
@@ -12,25 +15,13 @@ interface InviteWithCode extends Invite {
   code: string
 }
 
-let client: redis.Client
-
-export async function init() {
-  if (!client) {
-    client = new redis.Client(redis.utils.Databases.INVITATIONS)
-  }
-  return client
-}
-
-export async function shutdown() {
-  if (client) await client.finish()
-}
-
 /**
  * Given an invite code and invite body, allow the update an existing/valid invite in redis
- * @param inviteCode The invite code for an invite in redis
+ * @param code The invite code for an invite in redis
  * @param value The body of the updated user invitation
  */
 export async function updateCode(code: string, value: Invite) {
+  const client = await redis.getInviteClient()
   await client.store(code, value, TTL_SECONDS)
 }
 
@@ -42,16 +33,18 @@ export async function updateCode(code: string, value: Invite) {
  */
 export async function createCode(email: string, info: any): Promise<string> {
   const code = utils.newid()
+  const client = await redis.getInviteClient()
   await client.store(code, { email, info }, TTL_SECONDS)
   return code
 }
 
 /**
  * Checks that the provided invite code is valid - will return the email address of user that was invited.
- * @param inviteCode the invite code that was provided as part of the link.
+ * @param code the invite code that was provided as part of the link.
  * @return If the code is valid then an email address will be returned.
  */
 export async function getCode(code: string): Promise<Invite> {
+  const client = await redis.getInviteClient()
   const value = (await client.get(code)) as Invite | undefined
   if (!value) {
     throw "Invitation is not valid or has expired, please request a new one."
@@ -60,13 +53,15 @@ export async function getCode(code: string): Promise<Invite> {
 }
 
 export async function deleteCode(code: string) {
+  const client = await redis.getInviteClient()
   await client.delete(code)
 }
 
-/** 
-  Get all currently available user invitations for the current tenant.
-**/
+/**
+ Get all currently available user invitations for the current tenant.
+ **/
 export async function getInviteCodes(): Promise<InviteWithCode[]> {
+  const client = await redis.getInviteClient()
   const invites: { key: string; value: Invite }[] = await client.scan()
 
   const results: InviteWithCode[] = invites.map(invite => {
@@ -78,7 +73,7 @@ export async function getInviteCodes(): Promise<InviteWithCode[]> {
   if (!env.MULTI_TENANCY) {
     return results
   }
-  const tenantId = tenancy.getTenantId()
+  const tenantId = getTenantId()
   return results.filter(invite => tenantId === invite.info.tenantId)
 }
 
diff --git a/packages/backend-core/src/redis/passwordReset.ts b/packages/backend-core/src/cache/passwordReset.ts
similarity index 77%
rename from packages/backend-core/src/redis/passwordReset.ts
rename to packages/backend-core/src/cache/passwordReset.ts
index 13c1b1d2e6..9ee50c634c 100644
--- a/packages/backend-core/src/redis/passwordReset.ts
+++ b/packages/backend-core/src/cache/passwordReset.ts
@@ -1,25 +1,14 @@
-import { redis, utils } from "../"
+import * as redis from "../redis/init"
+import * as utils from "../utils"
+import { Duration, DurationType } from "../utils"
 
-const TTL_SECONDS = 60 * 60
+const TTL_SECONDS = Duration.fromHours(1).to(DurationType.SECONDS)
 
 interface PasswordReset {
   userId: string
   info: any
 }
 
-let client: redis.Client
-
-export async function init() {
-  if (!client) {
-    client = new redis.Client(redis.utils.Databases.PW_RESETS)
-  }
-  return client
-}
-
-export async function shutdown() {
-  if (client) await client.finish()
-}
-
 /**
  * Given a user ID this will store a code (that is returned) for an hour in redis.
  * The user can then return this code for resetting their password (through their reset link).
@@ -29,6 +18,7 @@ export async function shutdown() {
  */
 export async function createCode(userId: string, info: any): Promise<string> {
   const code = utils.newid()
+  const client = await redis.getPasswordResetClient()
   await client.store(code, { userId, info }, TTL_SECONDS)
   return code
 }
@@ -39,6 +29,7 @@ export async function createCode(userId: string, info: any): Promise<string> {
  * @return returns the user ID if it is found
  */
 export async function getCode(code: string): Promise<PasswordReset> {
+  const client = await redis.getPasswordResetClient()
   const value = (await client.get(code)) as PasswordReset | undefined
   if (!value) {
     throw "Provided information is not valid, cannot reset password - please try again."
diff --git a/packages/backend-core/src/redis/index.ts b/packages/backend-core/src/redis/index.ts
index 8d153ea5a1..6585d6e4fa 100644
--- a/packages/backend-core/src/redis/index.ts
+++ b/packages/backend-core/src/redis/index.ts
@@ -4,5 +4,3 @@ export { default as Client } from "./redis"
 export * as utils from "./utils"
 export * as clients from "./init"
 export * as locks from "./redlockImpl"
-export * as invite from "./invite"
-export * as passwordReset from "./passwordReset"
diff --git a/packages/backend-core/src/redis/init.ts b/packages/backend-core/src/redis/init.ts
index 55ffe3dd12..9ec3246fac 100644
--- a/packages/backend-core/src/redis/init.ts
+++ b/packages/backend-core/src/redis/init.ts
@@ -7,7 +7,9 @@ let userClient: Client,
   cacheClient: Client,
   writethroughClient: Client,
   lockClient: Client,
-  socketClient: Client
+  socketClient: Client,
+  inviteClient: Client,
+  passwordResetClient: Client
 
 async function init() {
   userClient = await new Client(utils.Databases.USER_CACHE).init()
@@ -16,6 +18,8 @@ async function init() {
   cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init()
   lockClient = await new Client(utils.Databases.LOCKS).init()
   writethroughClient = await new Client(utils.Databases.WRITE_THROUGH).init()
+  inviteClient = await new Client(utils.Databases.INVITATIONS).init()
+  passwordResetClient = await new Client(utils.Databases.PW_RESETS).init()
   socketClient = await new Client(
     utils.Databases.SOCKET_IO,
     utils.SelectableDatabase.SOCKET_IO
@@ -29,6 +33,8 @@ export async function shutdown() {
   if (cacheClient) await cacheClient.finish()
   if (writethroughClient) await writethroughClient.finish()
   if (lockClient) await lockClient.finish()
+  if (inviteClient) await inviteClient.finish()
+  if (passwordResetClient) await passwordResetClient.finish()
   if (socketClient) await socketClient.finish()
 }
 
@@ -84,3 +90,17 @@ export async function getSocketClient() {
   }
   return socketClient
 }
+
+export async function getInviteClient() {
+  if (!inviteClient) {
+    await init()
+  }
+  return inviteClient
+}
+
+export async function getPasswordResetClient() {
+  if (!passwordResetClient) {
+    await init()
+  }
+  return passwordResetClient
+}
diff --git a/packages/backend-core/src/users/lookup.ts b/packages/backend-core/src/users/lookup.ts
index 2c0c66276a..355be74dab 100644
--- a/packages/backend-core/src/users/lookup.ts
+++ b/packages/backend-core/src/users/lookup.ts
@@ -6,7 +6,7 @@ import {
 } from "@budibase/types"
 import * as dbUtils from "../db"
 import { ViewName } from "../constants"
-import { getExistingInvites } from "../redis/invite"
+import { getExistingInvites } from "../cache/invite"
 
 /**
  * Apply a system-wide search on emails:
diff --git a/packages/server/src/middleware/tests/currentapp.spec.js b/packages/server/src/middleware/tests/currentapp.spec.js
index b80800fd96..d287683869 100644
--- a/packages/server/src/middleware/tests/currentapp.spec.js
+++ b/packages/server/src/middleware/tests/currentapp.spec.js
@@ -9,11 +9,11 @@ function mockWorker() {
       return {
         _id: "us_uuid1",
         roles: {
-          "app_test": "BASIC",
+          app_test: "BASIC",
         },
         roleId: "BASIC",
       }
-    }
+    },
   }))
 }
 
@@ -37,27 +37,12 @@ function mockAuthWithNoCookie() {
   mockWorker()
   jest.mock("@budibase/backend-core", () => {
     const core = jest.requireActual("@budibase/backend-core")
-    return {
-      ...core,
-      db: {
-        ...core.db,
-        dbExists: () => true,
-      },
-      cache: {
-        user: {
-          getUser: async id => {
-            return {
-              _id: "us_uuid1",
-            }
-          },
-        },
-      },
-      utils: {
-        getAppIdFromCtx: jest.fn(),
-        setCookie: jest.fn(),
-        getCookie: jest.fn(),
-      },
-    }
+    core.db.dbExists = () => true
+    core.cache.user.getUser = () => ({ _id: "us_uuid1" })
+    core.utils.getAppIdFromCtx = jest.fn()
+    core.utils.setCookie = jest.fn()
+    core.utils.getCookie = jest.fn()
+    return core
   })
 }
 
@@ -66,30 +51,13 @@ function mockAuthWithCookie() {
   mockWorker()
   jest.mock("@budibase/backend-core", () => {
     const core = jest.requireActual("@budibase/backend-core")
-    return {
-      ...core,
-      db: {
-        ...core.db,
-        dbExists: () => true,
-      },
-      utils: {
-        getAppIdFromCtx: () => {
-          return "app_test"
-        },
-        setCookie: jest.fn(),
-        clearCookie: jest.fn(),
-        getCookie: () => ({ appId: "app_different", roleId: "PUBLIC" }),
-      },
-      cache: {
-        user: {
-          getUser: async id => {
-            return {
-              _id: "us_uuid1",
-            }
-          },
-        },
-      },
-    }
+    core.db.dbExists = () => true
+    core.cache.user.getUser = () => ({ _id: "us_uuid1" })
+    core.utils.getAppIdFromCtx = () => "app_test"
+    core.utils.getCookie = () => ({ appId: "app_different", roleId: "PUBLIC" })
+    core.utils.setCookie = jest.fn()
+    core.utils.clearCookie = jest.fn()
+    return core
   })
 }
 
@@ -109,7 +77,7 @@ class TestConfiguration {
       path: "",
       cookies: {
         set: jest.fn(),
-      }
+      },
     }
   }
 
@@ -177,29 +145,16 @@ describe("Current app middleware", () => {
       mockReset()
       jest.mock("@budibase/backend-core", () => {
         const core = jest.requireActual("@budibase/backend-core")
-        return {
-          ...core,
-          db: {
-            ...core.db,
-            dbExists: () => true,
-          },
-          utils: {
-            getAppIdFromCtx: () => {
-              return "app_test"
-            },
-            setCookie: jest.fn(),
-            getCookie: jest.fn(),
-          },
-          cache: {
-            user: {
-              getUser: async id => {
-                return {
-                  _id: "us_uuid1",
-                }
-              },
-            },
-          },
-        }
+        core.db.dbExists = () => true
+        core.cache.user.getUser = () => ({ _id: "us_uuid1" })
+        core.utils.getAppIdFromCtx = () => "app_test"
+        core.utils.getCookie = () => ({
+          appId: "app_different",
+          roleId: "PUBLIC",
+        })
+        core.utils.setCookie = jest.fn()
+        core.utils.clearCookie = jest.fn()
+        return core
       })
       await checkExpected()
     })
diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts
index affdaa9938..82a1578c88 100644
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@@ -1,4 +1,3 @@
-import { redis } from "@budibase/backend-core"
 import * as userSdk from "../../../sdk/users"
 import env from "../../../environment"
 import {
@@ -308,7 +307,7 @@ export const checkInvite = async (ctx: any) => {
   const { code } = ctx.params
   let invite
   try {
-    invite = await redis.invite.getCode(code)
+    invite = await cache.invite.getCode(code)
   } catch (e) {
     console.warn("Error getting invite from code", e)
     ctx.throw(400, "There was a problem with the invite")
@@ -322,7 +321,7 @@ export const checkInvite = async (ctx: any) => {
 export const getUserInvites = async (ctx: any) => {
   try {
     // Restricted to the currently authenticated tenant
-    ctx.body = await redis.invite.getInviteCodes()
+    ctx.body = await cache.invite.getInviteCodes()
   } catch (e) {
     ctx.throw(400, "There was a problem fetching invites")
   }
@@ -336,7 +335,7 @@ export const updateInvite = async (ctx: any) => {
 
   let invite
   try {
-    invite = await redis.invite.getCode(code)
+    invite = await cache.invite.getCode(code)
   } catch (e) {
     ctx.throw(400, "There was a problem with the invite")
     return
@@ -364,7 +363,7 @@ export const updateInvite = async (ctx: any) => {
     }
   }
 
-  await redis.invite.updateCode(code, updated)
+  await cache.invite.updateCode(code, updated)
   ctx.body = { ...invite }
 }
 
@@ -374,8 +373,8 @@ export const inviteAccept = async (
   const { inviteCode, password, firstName, lastName } = ctx.request.body
   try {
     // info is an extension of the user object that was stored by global
-    const { email, info }: any = await redis.invite.getCode(inviteCode)
-    await redis.invite.deleteCode(inviteCode)
+    const { email, info }: any = await cache.invite.getCode(inviteCode)
+    await cache.invite.deleteCode(inviteCode)
     const user = await tenancy.doInTenant(info.tenantId, async () => {
       let request: any = {
         firstName,
diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts
index e486a67433..34dc90ccd6 100644
--- a/packages/worker/src/index.ts
+++ b/packages/worker/src/index.ts
@@ -72,9 +72,8 @@ server.on("close", async () => {
   shuttingDown = true
   console.log("Server Closed")
   timers.cleanup()
-  await redis.invite.shutdown()
-  await redis.passwordReset.shutdown()
-  await events.shutdown()
+  events.shutdown()
+  await redis.clients.shutdown()
   await queue.shutdown()
   if (!env.isTest()) {
     process.exit(errCode)
@@ -89,8 +88,8 @@ const shutdown = () => {
 export default server.listen(parseInt(env.PORT || "4002"), async () => {
   console.log(`Worker running on ${JSON.stringify(server.address())}`)
   await initPro()
-  await redis.invite.init()
-  await redis.passwordReset.init()
+  await redis.clients.getInviteClient()
+  await redis.clients.getPasswordResetClient()
   // configure events to use the pro audit log write
   // can't integrate directly into backend-core due to cyclic issues
   await events.processors.init(proSdk.auditLogs.write)
diff --git a/packages/worker/src/sdk/auth/auth.ts b/packages/worker/src/sdk/auth/auth.ts
index 704de9e4b2..1f9da8a260 100644
--- a/packages/worker/src/sdk/auth/auth.ts
+++ b/packages/worker/src/sdk/auth/auth.ts
@@ -6,7 +6,7 @@ import {
   sessions,
   tenancy,
   utils as coreUtils,
-  redis,
+  cache,
 } from "@budibase/backend-core"
 import { PlatformLogoutOpts, User } from "@budibase/types"
 import jwt from "jsonwebtoken"
@@ -73,7 +73,7 @@ export const reset = async (email: string) => {
  * Perform the user password update if the provided reset code is valid.
  */
 export const resetUpdate = async (resetCode: string, password: string) => {
-  const { userId } = await redis.passwordReset.getCode(resetCode)
+  const { userId } = await cache.passwordReset.getCode(resetCode)
 
   let user = await userSdk.db.getUser(userId)
   user.password = password
diff --git a/packages/worker/src/utilities/email.ts b/packages/worker/src/utilities/email.ts
index 530b6ce87f..e57a17a0f4 100644
--- a/packages/worker/src/utilities/email.ts
+++ b/packages/worker/src/utilities/email.ts
@@ -3,9 +3,8 @@ import { EmailTemplatePurpose, TemplateType } from "../constants"
 import { getTemplateByPurpose, EmailTemplates } from "../constants/templates"
 import { getSettingsTemplateContext } from "./templates"
 import { processString } from "@budibase/string-templates"
-import { redis } from "@budibase/backend-core"
 import { User, SendEmailOpts, SMTPInnerConfig } from "@budibase/types"
-import { configs } from "@budibase/backend-core"
+import { configs, cache } from "@budibase/backend-core"
 import ical from "ical-generator"
 const nodemailer = require("nodemailer")
 
@@ -61,9 +60,9 @@ async function getLinkCode(
 ) {
   switch (purpose) {
     case EmailTemplatePurpose.PASSWORD_RECOVERY:
-      return redis.passwordReset.createCode(user._id!, info)
+      return cache.passwordReset.createCode(user._id!, info)
     case EmailTemplatePurpose.INVITATION:
-      return redis.invite.createCode(email, info)
+      return cache.invite.createCode(email, info)
     default:
       return null
   }

From 34d6a4104259f96fc18dc443cb906a1118ef942a Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 17 Nov 2023 16:23:41 +0000
Subject: [PATCH 123/155] Resetting test.

---
 .../src/middleware/tests/currentapp.spec.js   | 91 ++++++++++++++-----
 1 file changed, 68 insertions(+), 23 deletions(-)

diff --git a/packages/server/src/middleware/tests/currentapp.spec.js b/packages/server/src/middleware/tests/currentapp.spec.js
index d287683869..22e47b0a6e 100644
--- a/packages/server/src/middleware/tests/currentapp.spec.js
+++ b/packages/server/src/middleware/tests/currentapp.spec.js
@@ -37,12 +37,27 @@ function mockAuthWithNoCookie() {
   mockWorker()
   jest.mock("@budibase/backend-core", () => {
     const core = jest.requireActual("@budibase/backend-core")
-    core.db.dbExists = () => true
-    core.cache.user.getUser = () => ({ _id: "us_uuid1" })
-    core.utils.getAppIdFromCtx = jest.fn()
-    core.utils.setCookie = jest.fn()
-    core.utils.getCookie = jest.fn()
-    return core
+    return {
+      ...core,
+      db: {
+        ...core.db,
+        dbExists: () => true,
+      },
+      cache: {
+        user: {
+          getUser: async id => {
+            return {
+              _id: "us_uuid1",
+            }
+          },
+        },
+      },
+      utils: {
+        getAppIdFromCtx: jest.fn(),
+        setCookie: jest.fn(),
+        getCookie: jest.fn(),
+      },
+    }
   })
 }
 
@@ -51,13 +66,30 @@ function mockAuthWithCookie() {
   mockWorker()
   jest.mock("@budibase/backend-core", () => {
     const core = jest.requireActual("@budibase/backend-core")
-    core.db.dbExists = () => true
-    core.cache.user.getUser = () => ({ _id: "us_uuid1" })
-    core.utils.getAppIdFromCtx = () => "app_test"
-    core.utils.getCookie = () => ({ appId: "app_different", roleId: "PUBLIC" })
-    core.utils.setCookie = jest.fn()
-    core.utils.clearCookie = jest.fn()
-    return core
+    return {
+      ...core,
+      db: {
+        ...core.db,
+        dbExists: () => true,
+      },
+      utils: {
+        getAppIdFromCtx: () => {
+          return "app_test"
+        },
+        setCookie: jest.fn(),
+        clearCookie: jest.fn(),
+        getCookie: () => ({ appId: "app_different", roleId: "PUBLIC" }),
+      },
+      cache: {
+        user: {
+          getUser: async id => {
+            return {
+              _id: "us_uuid1",
+            }
+          },
+        },
+      },
+    }
   })
 }
 
@@ -145,16 +177,29 @@ describe("Current app middleware", () => {
       mockReset()
       jest.mock("@budibase/backend-core", () => {
         const core = jest.requireActual("@budibase/backend-core")
-        core.db.dbExists = () => true
-        core.cache.user.getUser = () => ({ _id: "us_uuid1" })
-        core.utils.getAppIdFromCtx = () => "app_test"
-        core.utils.getCookie = () => ({
-          appId: "app_different",
-          roleId: "PUBLIC",
-        })
-        core.utils.setCookie = jest.fn()
-        core.utils.clearCookie = jest.fn()
-        return core
+        return {
+          ...core,
+          db: {
+            ...core.db,
+            dbExists: () => true,
+          },
+          utils: {
+            getAppIdFromCtx: () => {
+              return "app_test"
+            },
+            setCookie: jest.fn(),
+            getCookie: jest.fn(),
+          },
+          cache: {
+            user: {
+              getUser: async id => {
+                return {
+                  _id: "us_uuid1",
+                }
+              },
+            },
+          },
+        }
       })
       await checkExpected()
     })

From bdc297d4eca37dd5d320dda93481cbc33df348a0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 17:29:13 +0100
Subject: [PATCH 124/155] Fix

---
 .github/workflows/budibase_ci.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 3109183989..3960ce09e6 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -35,7 +35,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ !env.is_oss_contributor && 'yarn' || '' }}
+          cache: ${{ env.is_oss_contributor == false && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - run: yarn lint
 
@@ -53,7 +53,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ !env.is_oss_contributor && 'yarn' || '' }}
+          cache: ${{ env.is_oss_contributor == false && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
 
       # Run build all the projects
@@ -83,7 +83,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ !env.is_oss_contributor && 'yarn' || '' }}
+          cache: ${{ env.is_oss_contributor == false && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Test
         run: |
@@ -112,7 +112,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ !env.is_oss_contributor && 'yarn' || '' }}
+          cache: ${{ env.is_oss_contributor == false && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Test worker
         run: |
@@ -142,7 +142,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ !env.is_oss_contributor && 'yarn' || '' }}
+          cache: ${{ env.is_oss_contributor == false && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Test server
         run: |
@@ -173,7 +173,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ !env.is_oss_contributor && 'yarn' || '' }}
+          cache: ${{ env.is_oss_contributor == false && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Test
         run: |
@@ -196,7 +196,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ !env.is_oss_contributor && 'yarn' || '' }}
+          cache: ${{ env.is_oss_contributor == false && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Build packages
         run: yarn build --scope @budibase/server --scope @budibase/worker

From 57fa9baef8c16cf9482a37b401dc72f4e7243bb0 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 17 Nov 2023 16:33:58 +0000
Subject: [PATCH 125/155] PR comments and possible fix for test failures.

---
 packages/backend-core/src/cache/generic.ts      | 17 +++++++++++------
 packages/backend-core/src/cache/invite.ts       |  2 +-
 .../backend-core/src/cache/passwordReset.ts     |  2 +-
 packages/backend-core/src/utils/Duration.ts     |  3 +++
 4 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/packages/backend-core/src/cache/generic.ts b/packages/backend-core/src/cache/generic.ts
index 19365e0e3f..7a2be5a0f0 100644
--- a/packages/backend-core/src/cache/generic.ts
+++ b/packages/backend-core/src/cache/generic.ts
@@ -18,9 +18,14 @@ export enum TTL {
   ONE_DAY = 86400,
 }
 
-export const keys = GENERIC.keys
-export const get = GENERIC.get
-export const store = GENERIC.store
-export const destroy = GENERIC.delete
-export const withCache = GENERIC.withCache
-export const bustCache = GENERIC.bustCache
+function performExport(funcName: string) {
+  // @ts-ignore
+  return (...args: any) => GENERIC[funcName](...args)
+}
+
+export const keys = performExport("keys")
+export const get = performExport("get")
+export const store = performExport("store")
+export const destroy = performExport("delete")
+export const withCache = performExport("withCache")
+export const bustCache = performExport("bustCache")
diff --git a/packages/backend-core/src/cache/invite.ts b/packages/backend-core/src/cache/invite.ts
index 1fec52122e..e43ebc4aa8 100644
--- a/packages/backend-core/src/cache/invite.ts
+++ b/packages/backend-core/src/cache/invite.ts
@@ -4,7 +4,7 @@ import env from "../environment"
 import { getTenantId } from "../context"
 import * as redis from "../redis/init"
 
-const TTL_SECONDS = Duration.fromDays(7).to(DurationType.SECONDS)
+const TTL_SECONDS = Duration.fromDays(7).toSeconds()
 
 interface Invite {
   email: string
diff --git a/packages/backend-core/src/cache/passwordReset.ts b/packages/backend-core/src/cache/passwordReset.ts
index 9ee50c634c..7f5a93f149 100644
--- a/packages/backend-core/src/cache/passwordReset.ts
+++ b/packages/backend-core/src/cache/passwordReset.ts
@@ -2,7 +2,7 @@ import * as redis from "../redis/init"
 import * as utils from "../utils"
 import { Duration, DurationType } from "../utils"
 
-const TTL_SECONDS = Duration.fromHours(1).to(DurationType.SECONDS)
+const TTL_SECONDS = Duration.fromHours(1).toSeconds()
 
 interface PasswordReset {
   userId: string
diff --git a/packages/backend-core/src/utils/Duration.ts b/packages/backend-core/src/utils/Duration.ts
index f376c2f7c7..3c7ef23b11 100644
--- a/packages/backend-core/src/utils/Duration.ts
+++ b/packages/backend-core/src/utils/Duration.ts
@@ -28,6 +28,9 @@ export class Duration {
       toMs: () => {
         return Duration.convert(from, DurationType.MILLISECONDS, duration)
       },
+      toSeconds: () => {
+        return Duration.convert(from, DurationType.SECONDS, duration)
+      },
     }
   }
 

From e78a68fb031f7f102dc0feb1577779010533ee1c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 17:35:26 +0100
Subject: [PATCH 126/155] Fix

---
 .github/workflows/budibase_ci.yml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 3960ce09e6..e8133dd1ce 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -28,14 +28,14 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
-          submodules: ${{ !env.IS_OSS_CONTRIBUTOR }}
+          submodules: ${{ env.IS_OSS_CONTRIBUTOR == false }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
 
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.is_oss_contributor == false && 'yarn' || '' }}
+          cache: ${{ env.IS_OSS_CONTRIBUTOR == false && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - run: yarn lint
 
@@ -45,7 +45,7 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
-          submodules: ${{ !env.IS_OSS_CONTRIBUTOR }}
+          submodules: ${{ env.IS_OSS_CONTRIBUTOR == false }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
           fetch-depth: 0
 
@@ -53,7 +53,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.is_oss_contributor == false && 'yarn' || '' }}
+          cache: ${{ env.IS_OSS_CONTRIBUTOR == false && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
 
       # Run build all the projects
@@ -75,7 +75,7 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
-          submodules: ${{ !env.IS_OSS_CONTRIBUTOR }}
+          submodules: ${{ env.IS_OSS_CONTRIBUTOR == false }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
           fetch-depth: 0
 
@@ -83,7 +83,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.is_oss_contributor == false && 'yarn' || '' }}
+          cache: ${{ env.IS_OSS_CONTRIBUTOR == false && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Test
         run: |
@@ -104,7 +104,7 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
-          submodules: ${{ !env.IS_OSS_CONTRIBUTOR }}
+          submodules: ${{ env.IS_OSS_CONTRIBUTOR == false }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
           fetch-depth: 0
 
@@ -112,7 +112,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.is_oss_contributor == false && 'yarn' || '' }}
+          cache: ${{ env.IS_OSS_CONTRIBUTOR == false && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Test worker
         run: |
@@ -134,7 +134,7 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
-          submodules: ${{ !env.IS_OSS_CONTRIBUTOR }}
+          submodules: ${{ env.IS_OSS_CONTRIBUTOR == false }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
           fetch-depth: 0
 
@@ -142,7 +142,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.is_oss_contributor == false && 'yarn' || '' }}
+          cache: ${{ env.IS_OSS_CONTRIBUTOR == false && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Test server
         run: |
@@ -173,7 +173,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.is_oss_contributor == false && 'yarn' || '' }}
+          cache: ${{ env.IS_OSS_CONTRIBUTOR == false && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Test
         run: |
@@ -189,14 +189,14 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
-          submodules: ${{ !env.IS_OSS_CONTRIBUTOR }}
+          submodules: ${{ env.IS_OSS_CONTRIBUTOR == false }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
 
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.is_oss_contributor == false && 'yarn' || '' }}
+          cache: ${{ env.IS_OSS_CONTRIBUTOR == false && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Build packages
         run: yarn build --scope @budibase/server --scope @budibase/worker

From 258ea803bc20020c49e20d0a81342e548d2de299 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 17 Nov 2023 16:37:00 +0000
Subject: [PATCH 127/155] Further PR comments.

---
 packages/backend-core/src/redis/init.ts | 2 +-
 packages/worker/src/index.ts            | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/backend-core/src/redis/init.ts b/packages/backend-core/src/redis/init.ts
index 9ec3246fac..f3bcee3209 100644
--- a/packages/backend-core/src/redis/init.ts
+++ b/packages/backend-core/src/redis/init.ts
@@ -11,7 +11,7 @@ let userClient: Client,
   inviteClient: Client,
   passwordResetClient: Client
 
-async function init() {
+export async function init() {
   userClient = await new Client(utils.Databases.USER_CACHE).init()
   sessionClient = await new Client(utils.Databases.SESSIONS).init()
   appClient = await new Client(utils.Databases.APP_METADATA).init()
diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts
index 34dc90ccd6..1c7591a7e8 100644
--- a/packages/worker/src/index.ts
+++ b/packages/worker/src/index.ts
@@ -88,8 +88,7 @@ const shutdown = () => {
 export default server.listen(parseInt(env.PORT || "4002"), async () => {
   console.log(`Worker running on ${JSON.stringify(server.address())}`)
   await initPro()
-  await redis.clients.getInviteClient()
-  await redis.clients.getPasswordResetClient()
+  await redis.clients.init()
   // configure events to use the pro audit log write
   // can't integrate directly into backend-core due to cyclic issues
   await events.processors.init(proSdk.auditLogs.write)

From 52f2e5633c2d5f0e1115904afb6ed42163125197 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 17:41:07 +0100
Subject: [PATCH 128/155] Fixes

---
 .github/workflows/budibase_ci.yml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index e8133dd1ce..d68e9d4aa1 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -28,14 +28,14 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
-          submodules: ${{ env.IS_OSS_CONTRIBUTOR == false }}
+          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
 
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.IS_OSS_CONTRIBUTOR == false && 'yarn' || '' }}
+          cache: ${{ env.IS_OSS_CONTRIBUTOR == 'false' && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - run: yarn lint
 
@@ -45,7 +45,7 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
-          submodules: ${{ env.IS_OSS_CONTRIBUTOR == false }}
+          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
           fetch-depth: 0
 
@@ -53,7 +53,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.IS_OSS_CONTRIBUTOR == false && 'yarn' || '' }}
+          cache: ${{ env.IS_OSS_CONTRIBUTOR == 'false' && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
 
       # Run build all the projects
@@ -75,7 +75,7 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
-          submodules: ${{ env.IS_OSS_CONTRIBUTOR == false }}
+          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
           fetch-depth: 0
 
@@ -83,7 +83,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.IS_OSS_CONTRIBUTOR == false && 'yarn' || '' }}
+          cache: ${{ env.IS_OSS_CONTRIBUTOR == 'false' && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Test
         run: |
@@ -104,7 +104,7 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
-          submodules: ${{ env.IS_OSS_CONTRIBUTOR == false }}
+          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
           fetch-depth: 0
 
@@ -112,7 +112,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.IS_OSS_CONTRIBUTOR == false && 'yarn' || '' }}
+          cache: ${{ env.IS_OSS_CONTRIBUTOR == 'false' && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Test worker
         run: |
@@ -134,7 +134,7 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
-          submodules: ${{ env.IS_OSS_CONTRIBUTOR == false }}
+          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
           fetch-depth: 0
 
@@ -142,7 +142,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.IS_OSS_CONTRIBUTOR == false && 'yarn' || '' }}
+          cache: ${{ env.IS_OSS_CONTRIBUTOR == 'false' && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Test server
         run: |
@@ -173,7 +173,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.IS_OSS_CONTRIBUTOR == false && 'yarn' || '' }}
+          cache: ${{ env.IS_OSS_CONTRIBUTOR == 'false' && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Test
         run: |
@@ -189,14 +189,14 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
-          submodules: ${{ env.IS_OSS_CONTRIBUTOR == false }}
+          submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
 
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.IS_OSS_CONTRIBUTOR == false && 'yarn' || '' }}
+          cache: ${{ env.IS_OSS_CONTRIBUTOR == 'false' && 'yarn' || '' }}
       - run: yarn --frozen-lockfile
       - name: Build packages
         run: yarn build --scope @budibase/server --scope @budibase/worker

From 347b6ccd4e865f8ff0e7116fd59576e465851ce2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 17:44:50 +0100
Subject: [PATCH 129/155] Fix yarn version

---
 .github/workflows/budibase_ci.yml | 14 +++++++-------
 scripts/syncProPackage.js         |  9 +--------
 2 files changed, 8 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index d68e9d4aa1..cb71d0702b 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -35,7 +35,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.IS_OSS_CONTRIBUTOR == 'false' && 'yarn' || '' }}
+          cache: yarn
       - run: yarn --frozen-lockfile
       - run: yarn lint
 
@@ -53,7 +53,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.IS_OSS_CONTRIBUTOR == 'false' && 'yarn' || '' }}
+          cache: yarn
       - run: yarn --frozen-lockfile
 
       # Run build all the projects
@@ -83,7 +83,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.IS_OSS_CONTRIBUTOR == 'false' && 'yarn' || '' }}
+          cache: yarn
       - run: yarn --frozen-lockfile
       - name: Test
         run: |
@@ -112,7 +112,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.IS_OSS_CONTRIBUTOR == 'false' && 'yarn' || '' }}
+          cache: yarn
       - run: yarn --frozen-lockfile
       - name: Test worker
         run: |
@@ -142,7 +142,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.IS_OSS_CONTRIBUTOR == 'false' && 'yarn' || '' }}
+          cache: yarn
       - run: yarn --frozen-lockfile
       - name: Test server
         run: |
@@ -173,7 +173,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.IS_OSS_CONTRIBUTOR == 'false' && 'yarn' || '' }}
+          cache: yarn
       - run: yarn --frozen-lockfile
       - name: Test
         run: |
@@ -196,7 +196,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18.x
-          cache: ${{ env.IS_OSS_CONTRIBUTOR == 'false' && 'yarn' || '' }}
+          cache: yarn
       - run: yarn --frozen-lockfile
       - name: Build packages
         run: yarn build --scope @budibase/server --scope @budibase/worker
diff --git a/scripts/syncProPackage.js b/scripts/syncProPackage.js
index 46a55c291b..14ee222ce7 100755
--- a/scripts/syncProPackage.js
+++ b/scripts/syncProPackage.js
@@ -5,14 +5,7 @@ const { execSync } = require("child_process")
 let version = "0.0.0"
 const localPro = fs.existsSync("packages/pro/src")
 if (!localPro) {
-  const branchName = execSync("git rev-parse --abbrev-ref HEAD")
-    .toString()
-    .trim()
-  if (branchName === "master") {
-    version = "latest"
-  } else {
-    version = "develop"
-  }
+  version = "latest"
 }
 
 // Get the list of workspaces with mismatched dependencies

From bed1b3d3307f685eaa35e026e26f61eb2a840be2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 18:01:49 +0100
Subject: [PATCH 130/155] Test oss

---
 .github/workflows/budibase_ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index cb71d0702b..d0a0860b40 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -19,7 +19,7 @@ env:
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   NX_BASE_BRANCH: origin/${{ github.base_ref }}
   USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' }}
-  IS_OSS_CONTRIBUTOR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' }}
+  IS_OSS_CONTRIBUTOR: true
 
 jobs:
   lint:

From f4c20b864d414e667f5dfbceb95c07c33cfb2831 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 18:16:30 +0100
Subject: [PATCH 131/155] Fix integration test for OSS

---
 .github/workflows/budibase_ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index d0a0860b40..5b56bd7edc 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -200,6 +200,9 @@ jobs:
       - run: yarn --frozen-lockfile
       - name: Build packages
         run: yarn build --scope @budibase/server --scope @budibase/worker
+      - name: Build pro for OSS contributor
+        if: ${{ env.IS_OSS_CONTRIBUTOR == 'true' }}
+        run: yarn build --scope @budibase/backend-core
       - name: Run tests
         run: |
           cd qa-core

From c5930ec0b646d061c2d9f286f6c13e25221fa1aa Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 18:23:31 +0100
Subject: [PATCH 132/155] Revert fixing oss contributor

---
 .github/workflows/budibase_ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 5b56bd7edc..7929da12e7 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -19,7 +19,7 @@ env:
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   NX_BASE_BRANCH: origin/${{ github.base_ref }}
   USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' }}
-  IS_OSS_CONTRIBUTOR: true
+  IS_OSS_CONTRIBUTOR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' }}
 
 jobs:
   lint:
@@ -200,7 +200,7 @@ jobs:
       - run: yarn --frozen-lockfile
       - name: Build packages
         run: yarn build --scope @budibase/server --scope @budibase/worker
-      - name: Build pro for OSS contributor
+      - name: Build backend-core for OSS contributor (required for pro)
         if: ${{ env.IS_OSS_CONTRIBUTOR == 'true' }}
         run: yarn build --scope @budibase/backend-core
       - name: Run tests

From 68a5d1a09d2b166bbb63276e7413ce02ea1a9003 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 18:44:06 +0100
Subject: [PATCH 133/155] Run oss checks

---
 .github/workflows/budibase_ci.yml           |  9 ++++++++-
 .github/workflows/check-oss-contributor.yml | 13 +++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/check-oss-contributor.yml

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 7929da12e7..004a326c15 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -12,6 +12,13 @@ on:
       - master
   pull_request:
   workflow_dispatch:
+  workflow_call:
+    inputs:
+      run_as_oss:
+        type: boolean
+        required: false
+        description: Force running checks as if it was an OSS contributor
+        default: false
 
 env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
@@ -19,7 +26,7 @@ env:
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   NX_BASE_BRANCH: origin/${{ github.base_ref }}
   USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' }}
-  IS_OSS_CONTRIBUTOR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' }}
+  IS_OSS_CONTRIBUTOR: ${{ inputs.run_as_oss == 'true' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase') }}
 
 jobs:
   lint:
diff --git a/.github/workflows/check-oss-contributor.yml b/.github/workflows/check-oss-contributor.yml
new file mode 100644
index 0000000000..afa3ff7af9
--- /dev/null
+++ b/.github/workflows/check-oss-contributor.yml
@@ -0,0 +1,13 @@
+name: OSS contributor checks
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 8,16 * * 1-5" # on weekdays at 8am and 4pm
+
+jobs:
+  run:
+    name: Publish server and worker docker images
+    uses: ./.github/workflows/budibase_ci.yml
+    with:
+      run_as_oss: true
+    secrets: inherit

From 50e22b54fe10892ef0727407728488e6072b09a2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 18:44:39 +0100
Subject: [PATCH 134/155] Run branch for test

---
 .github/workflows/check-oss-contributor.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/check-oss-contributor.yml b/.github/workflows/check-oss-contributor.yml
index afa3ff7af9..c2ece9254a 100644
--- a/.github/workflows/check-oss-contributor.yml
+++ b/.github/workflows/check-oss-contributor.yml
@@ -3,6 +3,9 @@ on:
   workflow_dispatch:
   schedule:
     - cron: "0 8,16 * * 1-5" # on weekdays at 8am and 4pm
+  push:
+    branches:
+      - chore/run_oss_checks # TODO: remove
 
 jobs:
   run:

From c41e339d9db6841250e5f0466b321410af37999f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 17 Nov 2023 18:49:41 +0100
Subject: [PATCH 135/155] Attempt

---
 .github/workflows/budibase_ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 004a326c15..6e04ca6f67 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -26,7 +26,7 @@ env:
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   NX_BASE_BRANCH: origin/${{ github.base_ref }}
   USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' }}
-  IS_OSS_CONTRIBUTOR: ${{ inputs.run_as_oss == 'true' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase') }}
+  IS_OSS_CONTRIBUTOR: ${{ inputs.run_as_oss == true || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase') }}
 
 jobs:
   lint:

From e04ea1a620ca5878b8affa043ff72fd3b70dbb5a Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 20 Nov 2023 08:30:42 +0000
Subject: [PATCH 136/155] Add makePropSafe to client SDK

---
 packages/client/src/sdk.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/client/src/sdk.js b/packages/client/src/sdk.js
index b83039df2a..69c2cc9fd4 100644
--- a/packages/client/src/sdk.js
+++ b/packages/client/src/sdk.js
@@ -26,7 +26,7 @@ import { ActionTypes } from "./constants"
 import { fetchDatasourceSchema } from "./utils/schema.js"
 import { getAPIKey } from "./utils/api.js"
 import { enrichButtonActions } from "./utils/buttonActions.js"
-import { processStringSync } from "@budibase/string-templates"
+import { processStringSync, makePropSafe } from "@budibase/string-templates"
 
 export default {
   API,
@@ -56,6 +56,7 @@ export default {
   getAPIKey,
   enrichButtonActions,
   processStringSync,
+  makePropSafe,
 
   // Components
   Provider,

From d4060c0f78c0afeb4a52f8b5bbbd322b7162e872 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 Nov 2023 09:52:00 +0100
Subject: [PATCH 137/155] Add alerts

---
 .github/workflows/check-oss-contributor.yml | 22 ++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/check-oss-contributor.yml b/.github/workflows/check-oss-contributor.yml
index c2ece9254a..f9bd54248e 100644
--- a/.github/workflows/check-oss-contributor.yml
+++ b/.github/workflows/check-oss-contributor.yml
@@ -8,9 +8,29 @@ on:
       - chore/run_oss_checks # TODO: remove
 
 jobs:
-  run:
+  run-checks:
     name: Publish server and worker docker images
     uses: ./.github/workflows/budibase_ci.yml
     with:
       run_as_oss: true
     secrets: inherit
+
+  notify-error:
+    needs: ["run-checks"]
+    if: ${{ failure() }}
+    name: Notify error
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set commit SHA
+        id: set_sha
+        run: echo "::set-output name=sha::$(git rev-parse --short ${{ github.sha }})"
+
+      - name: Notify error
+        uses: tsickert/discord-webhook@v5.3.0
+        with:
+          webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
+          embed-title: "[TEST] OSS checks failed in master"
+          content: "The OSS build is failing on master. SHA: ${{ steps.set_sha.outputs.sha }}, link: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

From c0ae8640c0fae0ed10b357003724501d3e289a1f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 Nov 2023 09:52:18 +0100
Subject: [PATCH 138/155] Run always for testing

---
 .github/workflows/check-oss-contributor.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/check-oss-contributor.yml b/.github/workflows/check-oss-contributor.yml
index f9bd54248e..01cb87656b 100644
--- a/.github/workflows/check-oss-contributor.yml
+++ b/.github/workflows/check-oss-contributor.yml
@@ -16,8 +16,8 @@ jobs:
     secrets: inherit
 
   notify-error:
-    needs: ["run-checks"]
-    if: ${{ failure() }}
+    # needs: ["run-checks"]
+    if: ${{ always() }}
     name: Notify error
     runs-on: ubuntu-latest
     steps:

From 0f2d95cd2e2d29ee3409e7eaab4d9f594638c6d8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 Nov 2023 09:56:37 +0100
Subject: [PATCH 139/155] Test message

---
 .github/workflows/check-oss-contributor.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/check-oss-contributor.yml b/.github/workflows/check-oss-contributor.yml
index 01cb87656b..da5036b530 100644
--- a/.github/workflows/check-oss-contributor.yml
+++ b/.github/workflows/check-oss-contributor.yml
@@ -33,4 +33,7 @@ jobs:
         with:
           webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
           embed-title: "[TEST] OSS checks failed in master"
-          content: "The OSS build is failing on master. SHA: ${{ steps.set_sha.outputs.sha }}, link: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          embed-url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          embed-description: |
+            🚨 The OSS build is failing on master ⚠️
+            Git sha: `${{ steps.set_sha.outputs.sha }}`

From cf30097710badfeeebb63df48202521b8fbe71ec Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 Nov 2023 09:58:09 +0100
Subject: [PATCH 140/155] Title change

---
 .github/workflows/check-oss-contributor.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/check-oss-contributor.yml b/.github/workflows/check-oss-contributor.yml
index da5036b530..93f6bcadab 100644
--- a/.github/workflows/check-oss-contributor.yml
+++ b/.github/workflows/check-oss-contributor.yml
@@ -32,8 +32,7 @@ jobs:
         uses: tsickert/discord-webhook@v5.3.0
         with:
           webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
-          embed-title: "[TEST] OSS checks failed in master"
+          embed-title: 🚨 OSS checks failed in master
           embed-url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           embed-description: |
-            🚨 The OSS build is failing on master ⚠️
             Git sha: `${{ steps.set_sha.outputs.sha }}`

From 370d12965cc75331249cfd618bd925fcb87fe7e4 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 Nov 2023 09:58:43 +0100
Subject: [PATCH 141/155] Cleanup pipeline

---
 .github/workflows/check-oss-contributor.yml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/check-oss-contributor.yml b/.github/workflows/check-oss-contributor.yml
index 93f6bcadab..ec40cad399 100644
--- a/.github/workflows/check-oss-contributor.yml
+++ b/.github/workflows/check-oss-contributor.yml
@@ -3,9 +3,6 @@ on:
   workflow_dispatch:
   schedule:
     - cron: "0 8,16 * * 1-5" # on weekdays at 8am and 4pm
-  push:
-    branches:
-      - chore/run_oss_checks # TODO: remove
 
 jobs:
   run-checks:
@@ -16,8 +13,8 @@ jobs:
     secrets: inherit
 
   notify-error:
-    # needs: ["run-checks"]
-    if: ${{ always() }}
+    needs: ["run-checks"]
+    if: ${{ failure() }}
     name: Notify error
     runs-on: ubuntu-latest
     steps:

From 5a608b0873f6bf5ce1ef3e4b99b552071816c7b7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 Nov 2023 10:05:25 +0100
Subject: [PATCH 142/155] Change webhook url

---
 .github/workflows/check-oss-contributor.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/check-oss-contributor.yml b/.github/workflows/check-oss-contributor.yml
index ec40cad399..398f07a130 100644
--- a/.github/workflows/check-oss-contributor.yml
+++ b/.github/workflows/check-oss-contributor.yml
@@ -28,7 +28,7 @@ jobs:
       - name: Notify error
         uses: tsickert/discord-webhook@v5.3.0
         with:
-          webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
+          webhook-url: ${{ secrets.OSS_CHECKS_WEBHOOK_URL }}
           embed-title: 🚨 OSS checks failed in master
           embed-url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           embed-description: |

From d9ed5fb3d09f73437c997ac64213562a29a26aa0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 Nov 2023 10:07:33 +0100
Subject: [PATCH 143/155] Do not release fb for OSS

---
 .github/workflows/deploy-featurebranch.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml
index f1fb12c087..c70f2fff20 100644
--- a/.github/workflows/deploy-featurebranch.yml
+++ b/.github/workflows/deploy-featurebranch.yml
@@ -7,6 +7,7 @@ on:
 
 jobs:
   release:
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

From 8297564029e3af7efb186b32424b28806da0ce9c Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Mon, 20 Nov 2023 11:25:51 +0000
Subject: [PATCH 144/155] Bump version to 2.13.11

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index a12b1238b3..64d6b745e5 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.13.10",
+  "version": "2.13.11",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From c71644a1e7e8dc81b27618759a975c70c4f7cb94 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 Nov 2023 15:36:48 +0100
Subject: [PATCH 145/155] Remove .spec.js from prettier ignore

---
 .prettierignore | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.prettierignore b/.prettierignore
index 64607d74ab..ce7617224b 100644
--- a/.prettierignore
+++ b/.prettierignore
@@ -1,13 +1,11 @@
 node_modules
 dist
-*.spec.js
-packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte
 packages/server/builder
 packages/server/coverage
-packages/worker/coverage
-packages/backend-core/coverage
 packages/server/client
 packages/server/src/definitions/openapi.ts
+packages/worker/coverage
+packages/backend-core/coverage
 packages/builder/.routify
 packages/sdk/sdk
 packages/pro/coverage
\ No newline at end of file

From 742eb4ff46a37e819fad8185335d2fd0f95dc0d7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 20 Nov 2023 15:36:55 +0100
Subject: [PATCH 146/155] Lint fix

---
 .../backend-core/src/db/tests/index.spec.js   |   1 -
 .../tests/core/users/users.spec.js            |  19 +-
 .../import/sources/tests/curl/curl.spec.js    |  32 ++-
 .../sources/tests/openapi2/openapi2.spec.js   | 165 +++++------
 .../query/import/tests/index.spec.js          |  85 +++---
 .../view/tests/viewBuilder.spec.js            |  98 ++++---
 .../api/routes/public/tests/metrics.spec.js   |   7 +-
 .../src/api/routes/tests/analytics.spec.js    |  29 +-
 .../src/api/routes/tests/apikeys.spec.js      |   4 +-
 .../server/src/api/routes/tests/auth.spec.js  |   2 +-
 .../src/api/routes/tests/component.spec.js    |   2 +-
 .../server/src/api/routes/tests/dev.spec.js   |   7 +-
 .../src/api/routes/tests/integration.spec.js  |   2 +-
 .../src/api/routes/tests/layout.spec.js       |   2 +-
 .../src/api/routes/tests/metadata.spec.js     |   9 +-
 .../src/api/routes/tests/query.seq.spec.js    | 265 +++++++++++-------
 .../server/src/api/routes/tests/role.spec.js  |   2 +-
 .../src/api/routes/tests/routing.spec.js      |  50 ++--
 .../src/api/routes/tests/screen.spec.js       |   4 +-
 .../src/api/routes/tests/templates.spec.js    |   2 +-
 .../server/src/api/routes/tests/view.spec.js  |  68 +++--
 .../server/src/automations/tests/bash.spec.js |  53 ++--
 .../src/automations/tests/delay.spec.js       |   2 +-
 .../src/automations/tests/discord.spec.js     |  33 ++-
 .../automations/tests/executeQuery.spec.js    |  75 +++--
 .../automations/tests/executeScript.spec.js   |  75 +++--
 .../automations/tests/outgoingWebhook.spec.js |   8 +-
 .../src/automations/tests/serverLog.spec.js   |  29 +-
 .../src/db/tests/linkController.spec.js       |  83 ++++--
 .../server/src/db/tests/linkTests.spec.js     |   6 +-
 .../src/middleware/tests/resourceId.spec.js   |  14 +-
 .../src/middleware/tests/selfhost.spec.js     |   4 +-
 .../functions/tests/appUrls.spec.js           |   2 +-
 .../tests/userEmailViewCasing.spec.js         |  30 +-
 packages/string-templates/test/basic.spec.js  |  38 ++-
 .../string-templates/test/escapes.spec.js     |  13 +-
 .../string-templates/test/hbsToJs.spec.js     |  48 ++--
 .../string-templates/test/helpers.spec.js     |   5 +-
 .../string-templates/test/javascript.spec.js  |   2 +-
 39 files changed, 768 insertions(+), 607 deletions(-)

diff --git a/packages/backend-core/src/db/tests/index.spec.js b/packages/backend-core/src/db/tests/index.spec.js
index 0d257f7ed7..e03c9a5b0e 100644
--- a/packages/backend-core/src/db/tests/index.spec.js
+++ b/packages/backend-core/src/db/tests/index.spec.js
@@ -5,7 +5,6 @@ const { getDB } = require("../db")
 describe("db", () => {
   describe("getDB", () => {
     it("returns a db", async () => {
-      
       const dbName = structures.db.id()
       const db = getDB(dbName)
       expect(db).toBeDefined()
diff --git a/packages/backend-core/tests/core/users/users.spec.js b/packages/backend-core/tests/core/users/users.spec.js
index ae7109344a..f08c435b95 100644
--- a/packages/backend-core/tests/core/users/users.spec.js
+++ b/packages/backend-core/tests/core/users/users.spec.js
@@ -1,5 +1,5 @@
-const _ = require('lodash/fp')
-const {structures} = require("../../../tests")
+const _ = require("lodash/fp")
+const { structures } = require("../../../tests")
 
 jest.mock("../../../src/context")
 jest.mock("../../../src/db")
@@ -7,10 +7,9 @@ jest.mock("../../../src/db")
 const context = require("../../../src/context")
 const db = require("../../../src/db")
 
-const {getCreatorCount} = require('../../../src/users/users')
+const { getCreatorCount } = require("../../../src/users/users")
 
 describe("Users", () => {
-
   let getGlobalDBMock
   let getGlobalUserParamsMock
   let paginationMock
@@ -26,26 +25,26 @@ describe("Users", () => {
   it("Retrieves the number of creators", async () => {
     const getUsers = (offset, limit, creators = false) => {
       const range = _.range(offset, limit)
-      const opts = creators ? {builder: {global: true}} : undefined
+      const opts = creators ? { builder: { global: true } } : undefined
       return range.map(() => structures.users.user(opts))
     }
     const page1Data = getUsers(0, 8)
     const page2Data = getUsers(8, 12, true)
     getGlobalDBMock.mockImplementation(() => ({
-      name   : "fake-db",
+      name: "fake-db",
       allDocs: () => ({
-        rows: [...page1Data, ...page2Data]
-      })
+        rows: [...page1Data, ...page2Data],
+      }),
     }))
     paginationMock.mockImplementationOnce(() => ({
       data: page1Data,
       hasNextPage: true,
-      nextPage: "1"
+      nextPage: "1",
     }))
     paginationMock.mockImplementation(() => ({
       data: page2Data,
       hasNextPage: false,
-      nextPage: undefined
+      nextPage: undefined,
     }))
     const creatorsCount = await getCreatorCount()
     expect(creatorsCount).toBe(4)
diff --git a/packages/server/src/api/controllers/query/import/sources/tests/curl/curl.spec.js b/packages/server/src/api/controllers/query/import/sources/tests/curl/curl.spec.js
index 2b17685f24..7ea2f8932b 100644
--- a/packages/server/src/api/controllers/query/import/sources/tests/curl/curl.spec.js
+++ b/packages/server/src/api/controllers/query/import/sources/tests/curl/curl.spec.js
@@ -1,13 +1,13 @@
 const { Curl } = require("../../curl")
 const fs = require("fs")
-const path = require('path')
+const path = require("path")
 
-const getData = (file) => {
+const getData = file => {
   return fs.readFileSync(path.join(__dirname, `./data/${file}.txt`), "utf8")
 }
 
 describe("Curl Import", () => {
-  let curl 
+  let curl
 
   beforeEach(() => {
     curl = new Curl()
@@ -28,7 +28,7 @@ describe("Curl Import", () => {
     expect(supported).toBe(false)
   })
 
-  const init = async (file) => {
+  const init = async file => {
     await curl.isSupported(getData(file))
   }
 
@@ -37,10 +37,9 @@ describe("Curl Import", () => {
     const info = await curl.getInfo()
     expect(info.name).toBe("example.com")
   })
-  
+
   describe("Returns queries", () => {
-    
-    const getQueries = async (file) => {
+    const getQueries = async file => {
       await init(file)
       const queries = await curl.getQueries()
       expect(queries.length).toBe(1)
@@ -58,7 +57,7 @@ describe("Curl Import", () => {
       await testVerb("put", "update")
       await testVerb("delete", "delete")
       await testVerb("patch", "patch")
-    }) 
+    })
 
     const testPath = async (file, urlPath) => {
       const queries = await getQueries(file)
@@ -66,8 +65,8 @@ describe("Curl Import", () => {
     }
 
     it("populates path", async () => {
-        await testPath("get", "http://example.com/")
-        await testPath("path", "http://example.com/paths/abc")
+      await testPath("get", "http://example.com/")
+      await testPath("path", "http://example.com/paths/abc")
     })
 
     const testHeaders = async (file, headers) => {
@@ -77,7 +76,10 @@ describe("Curl Import", () => {
 
     it("populates headers", async () => {
       await testHeaders("get", {})
-      await testHeaders("headers", { "x-bb-header-1" : "123", "x-bb-header-2" : "456"} )
+      await testHeaders("headers", {
+        "x-bb-header-1": "123",
+        "x-bb-header-2": "456",
+      })
     })
 
     const testQuery = async (file, queryString) => {
@@ -91,13 +93,15 @@ describe("Curl Import", () => {
 
     const testBody = async (file, body) => {
       const queries = await getQueries(file)
-      expect(queries[0].fields.requestBody).toStrictEqual(JSON.stringify(body, null, 2))
+      expect(queries[0].fields.requestBody).toStrictEqual(
+        JSON.stringify(body, null, 2)
+      )
     }
 
     it("populates body", async () => {
       await testBody("get", undefined)
-      await testBody("post", { "key" : "val" })
+      await testBody("post", { key: "val" })
       await testBody("empty-body", {})
     })
   })
-})
\ No newline at end of file
+})
diff --git a/packages/server/src/api/controllers/query/import/sources/tests/openapi2/openapi2.spec.js b/packages/server/src/api/controllers/query/import/sources/tests/openapi2/openapi2.spec.js
index 3c5aa89e3c..b61991f7d6 100644
--- a/packages/server/src/api/controllers/query/import/sources/tests/openapi2/openapi2.spec.js
+++ b/packages/server/src/api/controllers/query/import/sources/tests/openapi2/openapi2.spec.js
@@ -1,13 +1,16 @@
 const { OpenAPI2 } = require("../../openapi2")
 const fs = require("fs")
-const path = require('path')
+const path = require("path")
 
 const getData = (file, extension) => {
-  return fs.readFileSync(path.join(__dirname, `./data/${file}/${file}.${extension}`), "utf8")
+  return fs.readFileSync(
+    path.join(__dirname, `./data/${file}/${file}.${extension}`),
+    "utf8"
+  )
 }
 
 describe("OpenAPI2 Import", () => {
-  let openapi2 
+  let openapi2
 
   beforeEach(() => {
     openapi2 = new OpenAPI2()
@@ -47,9 +50,9 @@ describe("OpenAPI2 Import", () => {
   it("returns import info", async () => {
     await runTests("petstore", testImportInfo)
   })
-  
+
   describe("Returns queries", () => {
-    const indexQueries = (queries) => {
+    const indexQueries = queries => {
       return queries.reduce((acc, query) => {
         acc[query.name] = query
         return acc
@@ -72,15 +75,15 @@ describe("OpenAPI2 Import", () => {
 
     it("populates verb", async () => {
       const assertions = {
-        "createEntity" : "create",
-        "getEntities" : "read",
-        "getEntity" : "read",
-        "updateEntity" : "update",
-        "patchEntity" : "patch",
-        "deleteEntity" : "delete"
+        createEntity: "create",
+        getEntities: "read",
+        getEntity: "read",
+        updateEntity: "update",
+        patchEntity: "patch",
+        deleteEntity: "delete",
       }
       await runTests("crud", testVerb, assertions)
-    }) 
+    })
 
     const testPath = async (file, extension, assertions) => {
       const queries = await getQueries(file, extension)
@@ -91,12 +94,12 @@ describe("OpenAPI2 Import", () => {
 
     it("populates path", async () => {
       const assertions = {
-        "createEntity" : "http://example.com/entities",
-        "getEntities" : "http://example.com/entities",
-        "getEntity" : "http://example.com/entities/{{entityId}}",
-        "updateEntity" : "http://example.com/entities/{{entityId}}",
-        "patchEntity" : "http://example.com/entities/{{entityId}}",
-        "deleteEntity" : "http://example.com/entities/{{entityId}}"
+        createEntity: "http://example.com/entities",
+        getEntities: "http://example.com/entities",
+        getEntity: "http://example.com/entities/{{entityId}}",
+        updateEntity: "http://example.com/entities/{{entityId}}",
+        patchEntity: "http://example.com/entities/{{entityId}}",
+        deleteEntity: "http://example.com/entities/{{entityId}}",
       }
       await runTests("crud", testPath, assertions)
     })
@@ -109,27 +112,25 @@ describe("OpenAPI2 Import", () => {
     }
 
     const contentTypeHeader = {
-      "Content-Type" : "application/json",
+      "Content-Type": "application/json",
     }
 
     it("populates headers", async () => {
       const assertions = {
-        "createEntity" : {
-         ...contentTypeHeader
+        createEntity: {
+          ...contentTypeHeader,
         },
-        "getEntities" : {
+        getEntities: {},
+        getEntity: {},
+        updateEntity: {
+          ...contentTypeHeader,
         },
-        "getEntity" : {
+        patchEntity: {
+          ...contentTypeHeader,
         },
-        "updateEntity" : {
-          ...contentTypeHeader
+        deleteEntity: {
+          "x-api-key": "{{x-api-key}}",
         },
-        "patchEntity" : {
-          ...contentTypeHeader
-        },
-        "deleteEntity" : {
-            "x-api-key" : "{{x-api-key}}",
-        }
       }
 
       await runTests("crud", testHeaders, assertions)
@@ -138,18 +139,20 @@ describe("OpenAPI2 Import", () => {
     const testQuery = async (file, extension, assertions) => {
       const queries = await getQueries(file, extension)
       for (let [operationId, queryString] of Object.entries(assertions)) {
-        expect(queries[operationId].fields.queryString).toStrictEqual(queryString)
+        expect(queries[operationId].fields.queryString).toStrictEqual(
+          queryString
+        )
       }
     }
 
     it("populates query", async () => {
       const assertions = {
-        "createEntity" : "",
-        "getEntities" : "page={{page}}&size={{size}}",
-        "getEntity" : "",
-        "updateEntity" : "",
-        "patchEntity" : "",
-        "deleteEntity" : ""
+        createEntity: "",
+        getEntities: "page={{page}}&size={{size}}",
+        getEntity: "",
+        updateEntity: "",
+        patchEntity: "",
+        deleteEntity: "",
       }
       await runTests("crud", testQuery, assertions)
     })
@@ -163,45 +166,45 @@ describe("OpenAPI2 Import", () => {
 
     it("populates parameters", async () => {
       const assertions = {
-        "createEntity" : [],
-        "getEntities" : [
+        createEntity: [],
+        getEntities: [
           {
-            "name" : "page",
-            "default" : "",
+            name: "page",
+            default: "",
           },
           {
-            "name" : "size",
-            "default" : "",
-          }
+            name: "size",
+            default: "",
+          },
         ],
-        "getEntity" : [
+        getEntity: [
           {
-            "name" : "entityId",
-            "default" : "",
-          }
+            name: "entityId",
+            default: "",
+          },
         ],
-        "updateEntity" : [
+        updateEntity: [
           {
-            "name" : "entityId",
-            "default" : "",
-          }
+            name: "entityId",
+            default: "",
+          },
         ],
-        "patchEntity" : [
+        patchEntity: [
           {
-            "name" : "entityId",
-            "default" : "",
-          }
+            name: "entityId",
+            default: "",
+          },
         ],
-        "deleteEntity" : [
+        deleteEntity: [
           {
-            "name" : "entityId",
-            "default" : "",
+            name: "entityId",
+            default: "",
           },
           {
-            "name" : "x-api-key",
-            "default" : "",
-          }
-        ]
+            name: "x-api-key",
+            default: "",
+          },
+        ],
       }
       await runTests("crud", testParameters, assertions)
     })
@@ -209,30 +212,32 @@ describe("OpenAPI2 Import", () => {
     const testBody = async (file, extension, assertions) => {
       const queries = await getQueries(file, extension)
       for (let [operationId, body] of Object.entries(assertions)) {
-        expect(queries[operationId].fields.requestBody).toStrictEqual(JSON.stringify(body, null, 2))
+        expect(queries[operationId].fields.requestBody).toStrictEqual(
+          JSON.stringify(body, null, 2)
+        )
       }
     }
     it("populates body", async () => {
       const assertions = {
-        "createEntity" : {
-          "name" : "name",
-          "type" : "type",
+        createEntity: {
+          name: "name",
+          type: "type",
         },
-        "getEntities" : undefined,
-        "getEntity" : undefined,
-        "updateEntity" : {
-          "id": 1,
-          "name" : "name",
-          "type" : "type",
+        getEntities: undefined,
+        getEntity: undefined,
+        updateEntity: {
+          id: 1,
+          name: "name",
+          type: "type",
         },
-        "patchEntity" : {
-          "id": 1,
-          "name" : "name",
-          "type" : "type",
+        patchEntity: {
+          id: 1,
+          name: "name",
+          type: "type",
         },
-        "deleteEntity" : undefined
+        deleteEntity: undefined,
       }
       await runTests("crud", testBody, assertions)
     })
   })
-})
\ No newline at end of file
+})
diff --git a/packages/server/src/api/controllers/query/import/tests/index.spec.js b/packages/server/src/api/controllers/query/import/tests/index.spec.js
index 60030ae7f3..cef27abb08 100644
--- a/packages/server/src/api/controllers/query/import/tests/index.spec.js
+++ b/packages/server/src/api/controllers/query/import/tests/index.spec.js
@@ -1,11 +1,14 @@
 const TestConfig = require("../../../../../tests/utilities/TestConfiguration")
 const { RestImporter } = require("../index")
 const fs = require("fs")
-const path = require('path')
-const { events} = require("@budibase/backend-core")
+const path = require("path")
+const { events } = require("@budibase/backend-core")
 
-const getData = (file) => {
-  return fs.readFileSync(path.join(__dirname, `../sources/tests/${file}`), "utf8")
+const getData = file => {
+  return fs.readFileSync(
+    path.join(__dirname, `../sources/tests/${file}`),
+    "utf8"
+  )
 }
 
 // openapi2 (swagger)
@@ -14,7 +17,7 @@ const oapi2CrudYaml = getData("openapi2/data/crud/crud.json")
 const oapi2PetstoreJson = getData("openapi2/data/petstore/petstore.json")
 const oapi2PetstoreYaml = getData("openapi2/data/petstore/petstore.json")
 
-// openapi3 
+// openapi3
 const oapi3CrudJson = getData("openapi3/data/crud/crud.json")
 const oapi3CrudYaml = getData("openapi3/data/crud/crud.json")
 const oapi3PetstoreJson = getData("openapi3/data/petstore/petstore.json")
@@ -35,7 +38,7 @@ const datasets = {
   oapi3PetstoreJson,
   oapi3PetstoreYaml,
   // curl
-  curl
+  curl,
 }
 
 describe("Rest Importer", () => {
@@ -45,9 +48,9 @@ describe("Rest Importer", () => {
     await config.init()
   })
 
-  let restImporter 
+  let restImporter
 
-  const init = async (data) => {
+  const init = async data => {
     restImporter = new RestImporter(data)
     await restImporter.init()
   }
@@ -69,35 +72,35 @@ describe("Rest Importer", () => {
   it("gets info", async () => {
     const assertions = {
       // openapi2 (swagger)
-      "oapi2CrudJson" : {
+      oapi2CrudJson: {
         name: "CRUD",
       },
-      "oapi2CrudYaml" : {
+      oapi2CrudYaml: {
         name: "CRUD",
       },
-      "oapi2PetstoreJson" : {
+      oapi2PetstoreJson: {
         name: "Swagger Petstore",
       },
-      "oapi2PetstoreYaml" :{
+      oapi2PetstoreYaml: {
         name: "Swagger Petstore",
       },
       // openapi3
-      "oapi3CrudJson" : {
+      oapi3CrudJson: {
         name: "CRUD",
       },
-      "oapi3CrudYaml" : {
+      oapi3CrudYaml: {
         name: "CRUD",
       },
-      "oapi3PetstoreJson" : {
+      oapi3PetstoreJson: {
         name: "Swagger Petstore - OpenAPI 3.0",
       },
-      "oapi3PetstoreYaml" :{
+      oapi3PetstoreYaml: {
         name: "Swagger Petstore - OpenAPI 3.0",
       },
       // curl
-      "curl": {
+      curl: {
         name: "example.com",
-      }
+      },
     }
     await runTest(testGetInfo, assertions)
   })
@@ -109,54 +112,58 @@ describe("Rest Importer", () => {
     expect(importResult.errorQueries.length).toBe(0)
     expect(importResult.queries.length).toBe(assertions[key].count)
     expect(events.query.imported).toBeCalledTimes(1)
-    expect(events.query.imported).toBeCalledWith(datasource, assertions[key].source, assertions[key].count)
+    expect(events.query.imported).toBeCalledWith(
+      datasource,
+      assertions[key].source,
+      assertions[key].count
+    )
     jest.clearAllMocks()
   }
 
   it("imports queries", async () => {
-    // simple sanity assertions that the whole dataset 
+    // simple sanity assertions that the whole dataset
     // makes it through the importer
     const assertions = {
       // openapi2 (swagger)
-      "oapi2CrudJson" : {
+      oapi2CrudJson: {
         count: 6,
         source: "openapi2.0",
       },
-      "oapi2CrudYaml" :{
+      oapi2CrudYaml: {
         count: 6,
-        source: "openapi2.0"
+        source: "openapi2.0",
       },
-      "oapi2PetstoreJson" : {
+      oapi2PetstoreJson: {
         count: 20,
-        source: "openapi2.0"
+        source: "openapi2.0",
       },
-      "oapi2PetstoreYaml" :{
+      oapi2PetstoreYaml: {
         count: 20,
-        source: "openapi2.0"
+        source: "openapi2.0",
       },
       // openapi3
-      "oapi3CrudJson" : {
+      oapi3CrudJson: {
         count: 6,
-        source: "openapi3.0"
+        source: "openapi3.0",
       },
-      "oapi3CrudYaml" :{
+      oapi3CrudYaml: {
         count: 6,
-        source: "openapi3.0"
+        source: "openapi3.0",
       },
-      "oapi3PetstoreJson" : {
+      oapi3PetstoreJson: {
         count: 19,
-        source: "openapi3.0"
+        source: "openapi3.0",
       },
-      "oapi3PetstoreYaml" :{
+      oapi3PetstoreYaml: {
         count: 19,
-        source: "openapi3.0"
+        source: "openapi3.0",
       },
       // curl
-      "curl": {
+      curl: {
         count: 1,
-        source: "curl"
-      }
+        source: "curl",
+      },
     }
     await runTest(testImportQueries, assertions)
   })
-})
\ No newline at end of file
+})
diff --git a/packages/server/src/api/controllers/view/tests/viewBuilder.spec.js b/packages/server/src/api/controllers/view/tests/viewBuilder.spec.js
index 303bb09cf6..7f4298b2ff 100644
--- a/packages/server/src/api/controllers/view/tests/viewBuilder.spec.js
+++ b/packages/server/src/api/controllers/view/tests/viewBuilder.spec.js
@@ -1,65 +1,75 @@
-const viewTemplate = require("../viewBuilder").default;
+const viewTemplate = require("../viewBuilder").default
 
 describe("viewBuilder", () => {
-
   describe("Filter", () => {
     it("creates a view with multiple filters and conjunctions", () => {
-      expect(viewTemplate({
-        "name": "Test View",
-        "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
-        "filters": [{
-          "value": "Test",
-          "condition": "EQUALS",
-          "key": "Name"
-        }, {
-          "value": "Value",
-          "condition": "MT",
-          "key": "Yes",
-          "conjunction": "OR"
-        }]
-      })).toMatchSnapshot()
+      expect(
+        viewTemplate({
+          name: "Test View",
+          tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
+          filters: [
+            {
+              value: "Test",
+              condition: "EQUALS",
+              key: "Name",
+            },
+            {
+              value: "Value",
+              condition: "MT",
+              key: "Yes",
+              conjunction: "OR",
+            },
+          ],
+        })
+      ).toMatchSnapshot()
     })
   })
 
   describe("Calculate", () => {
     it("creates a view with the calculation statistics schema", () => {
-      expect(viewTemplate({
-        "name": "Calculate View",
-        "field": "myField",
-        "calculation": "stats",
-        "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
-        "filters": []
-      })).toMatchSnapshot()
+      expect(
+        viewTemplate({
+          name: "Calculate View",
+          field: "myField",
+          calculation: "stats",
+          tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
+          filters: [],
+        })
+      ).toMatchSnapshot()
     })
   })
 
   describe("Group By", () => {
     it("creates a view emitting the group by field", () => {
-      expect(viewTemplate({
-        "name": "Test Scores Grouped By Age",
-        "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
-        "groupBy": "age",
-        "field": "score",
-        "filters": [], 
-      })).toMatchSnapshot()
+      expect(
+        viewTemplate({
+          name: "Test Scores Grouped By Age",
+          tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
+          groupBy: "age",
+          field: "score",
+          filters: [],
+        })
+      ).toMatchSnapshot()
     })
   })
 
   describe("Calculate and filter", () => {
     it("creates a view with the calculation statistics and filter schema", () => {
-      expect(viewTemplate({
-        "name": "Calculate View",
-        "field": "myField",
-        "calculation": "stats",
-        "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
-        "filters": [
-          {
-            "value": 17,
-            "condition": "MT",
-            "key": "age",
-          }
-        ]
-      })).toMatchSnapshot()
+      expect(
+        viewTemplate({
+          name: "Calculate View",
+          field: "myField",
+          calculation: "stats",
+          tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
+          filters: [
+            {
+              value: 17,
+              condition: "MT",
+              key: "age",
+            },
+          ],
+        })
+      ).toMatchSnapshot()
     })
   })
-});
\ No newline at end of file
+})
diff --git a/packages/server/src/api/routes/public/tests/metrics.spec.js b/packages/server/src/api/routes/public/tests/metrics.spec.js
index 8e226ec958..8231596d59 100644
--- a/packages/server/src/api/routes/public/tests/metrics.spec.js
+++ b/packages/server/src/api/routes/public/tests/metrics.spec.js
@@ -25,10 +25,9 @@ describe("/metrics", () => {
 
     it("endpoint should not be publicly exposed", async () => {
       await request
-      .get(`/api/public/v1/metrics`)
-      .set(config.publicHeaders())
-      .expect(403)
+        .get(`/api/public/v1/metrics`)
+        .set(config.publicHeaders())
+        .expect(403)
     })
   })
-
 })
diff --git a/packages/server/src/api/routes/tests/analytics.spec.js b/packages/server/src/api/routes/tests/analytics.spec.js
index 5dfe2ce3e6..c69307656d 100644
--- a/packages/server/src/api/routes/tests/analytics.spec.js
+++ b/packages/server/src/api/routes/tests/analytics.spec.js
@@ -13,8 +13,8 @@ describe("/static", () => {
   beforeAll(async () => {
     app = await config.init()
   })
-  
-  beforeEach(()=>{
+
+  beforeEach(() => {
     jest.clearAllMocks()
   })
 
@@ -22,7 +22,7 @@ describe("/static", () => {
     it("should ping from builder", async () => {
       await request
         .post("/api/bbtel/ping")
-        .send({source: "builder", timezone})
+        .send({ source: "builder", timezone })
         .set(config.defaultHeaders())
         .expect(200)
 
@@ -35,12 +35,15 @@ describe("/static", () => {
     it("should ping from app preview", async () => {
       await request
         .post("/api/bbtel/ping")
-        .send({source: "app", timezone})
+        .send({ source: "app", timezone })
         .set(config.defaultHeaders())
         .expect(200)
 
       expect(events.serve.servedAppPreview).toBeCalledTimes(1)
-      expect(events.serve.servedAppPreview).toBeCalledWith(config.getApp(), timezone)
+      expect(events.serve.servedAppPreview).toBeCalledWith(
+        config.getApp(),
+        timezone
+      )
       expect(events.serve.servedApp).not.toBeCalled()
     })
 
@@ -50,12 +53,16 @@ describe("/static", () => {
 
       await request
         .post("/api/bbtel/ping")
-        .send({source: "app", timezone})
+        .send({ source: "app", timezone })
         .set(headers)
         .expect(200)
 
       expect(events.serve.servedApp).toBeCalledTimes(1)
-      expect(events.serve.servedApp).toBeCalledWith(config.getProdApp(), timezone, undefined)
+      expect(events.serve.servedApp).toBeCalledWith(
+        config.getProdApp(),
+        timezone,
+        undefined
+      )
       expect(events.serve.servedAppPreview).not.toBeCalled()
     })
 
@@ -65,12 +72,16 @@ describe("/static", () => {
 
       await request
         .post("/api/bbtel/ping")
-        .send({source: "app", timezone, embedded: true})
+        .send({ source: "app", timezone, embedded: true })
         .set(headers)
         .expect(200)
 
       expect(events.serve.servedApp).toBeCalledTimes(1)
-      expect(events.serve.servedApp).toBeCalledWith(config.getProdApp(), timezone, true)
+      expect(events.serve.servedApp).toBeCalledWith(
+        config.getProdApp(),
+        timezone,
+        true
+      )
       expect(events.serve.servedAppPreview).not.toBeCalled()
     })
   })
diff --git a/packages/server/src/api/routes/tests/apikeys.spec.js b/packages/server/src/api/routes/tests/apikeys.spec.js
index 4a52ea84b4..eb6933af7d 100644
--- a/packages/server/src/api/routes/tests/apikeys.spec.js
+++ b/packages/server/src/api/routes/tests/apikeys.spec.js
@@ -38,7 +38,7 @@ describe("/api/keys", () => {
         const res = await request
           .put(`/api/keys/TEST`)
           .send({
-            value: "test"
+            value: "test",
           })
           .set(config.defaultHeaders())
           .expect("Content-Type", /json/)
@@ -56,4 +56,4 @@ describe("/api/keys", () => {
       })
     })
   })
-})
\ No newline at end of file
+})
diff --git a/packages/server/src/api/routes/tests/auth.spec.js b/packages/server/src/api/routes/tests/auth.spec.js
index 9418d44046..5fbead99b7 100644
--- a/packages/server/src/api/routes/tests/auth.spec.js
+++ b/packages/server/src/api/routes/tests/auth.spec.js
@@ -21,4 +21,4 @@ describe("/authenticate", () => {
       expect(res.body._id).toEqual(generateUserMetadataID(config.user._id))
     })
   })
-})
\ No newline at end of file
+})
diff --git a/packages/server/src/api/routes/tests/component.spec.js b/packages/server/src/api/routes/tests/component.spec.js
index 160a1ca160..c621afae0a 100644
--- a/packages/server/src/api/routes/tests/component.spec.js
+++ b/packages/server/src/api/routes/tests/component.spec.js
@@ -29,4 +29,4 @@ describe("/component", () => {
       })
     })
   })
-})
\ No newline at end of file
+})
diff --git a/packages/server/src/api/routes/tests/dev.spec.js b/packages/server/src/api/routes/tests/dev.spec.js
index af1dc82a9d..73f506d17f 100644
--- a/packages/server/src/api/routes/tests/dev.spec.js
+++ b/packages/server/src/api/routes/tests/dev.spec.js
@@ -1,7 +1,6 @@
 const setup = require("./utilities")
 const { events } = require("@budibase/backend-core")
 
-
 describe("/dev", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
@@ -32,9 +31,9 @@ describe("/dev", () => {
         .expect("Content-Type", /json/)
         .expect(200)
 
-      expect(res.body.version).toBe('0.0.0+jest')
+      expect(res.body.version).toBe("0.0.0+jest")
       expect(events.installation.versionChecked).toBeCalledTimes(1)
-      expect(events.installation.versionChecked).toBeCalledWith('0.0.0+jest')
+      expect(events.installation.versionChecked).toBeCalledWith("0.0.0+jest")
     })
   })
-})
\ No newline at end of file
+})
diff --git a/packages/server/src/api/routes/tests/integration.spec.js b/packages/server/src/api/routes/tests/integration.spec.js
index dde780681b..798ddcd969 100644
--- a/packages/server/src/api/routes/tests/integration.spec.js
+++ b/packages/server/src/api/routes/tests/integration.spec.js
@@ -49,4 +49,4 @@ describe("/integrations", () => {
       })
     })
   })
-})
\ No newline at end of file
+})
diff --git a/packages/server/src/api/routes/tests/layout.spec.js b/packages/server/src/api/routes/tests/layout.spec.js
index ebd6ab7278..c3a3010e32 100644
--- a/packages/server/src/api/routes/tests/layout.spec.js
+++ b/packages/server/src/api/routes/tests/layout.spec.js
@@ -56,4 +56,4 @@ describe("/layouts", () => {
       })
     })
   })
-})
\ No newline at end of file
+})
diff --git a/packages/server/src/api/routes/tests/metadata.spec.js b/packages/server/src/api/routes/tests/metadata.spec.js
index bb28951ee3..d5563f417e 100644
--- a/packages/server/src/api/routes/tests/metadata.spec.js
+++ b/packages/server/src/api/routes/tests/metadata.spec.js
@@ -14,7 +14,10 @@ describe("/metadata", () => {
     automation = await config.createAutomation()
   })
 
-  async function createMetadata(data, type = MetadataTypes.AUTOMATION_TEST_INPUT) {
+  async function createMetadata(
+    data,
+    type = MetadataTypes.AUTOMATION_TEST_INPUT
+  ) {
     const res = await request
       .post(`/api/metadata/${type}/${automation._id}`)
       .send(data)
@@ -53,7 +56,9 @@ describe("/metadata", () => {
   describe("destroy", () => {
     it("should be able to delete some test inputs", async () => {
       const res = await request
-        .delete(`/api/metadata/${MetadataTypes.AUTOMATION_TEST_INPUT}/${automation._id}`)
+        .delete(
+          `/api/metadata/${MetadataTypes.AUTOMATION_TEST_INPUT}/${automation._id}`
+        )
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
diff --git a/packages/server/src/api/routes/tests/query.seq.spec.js b/packages/server/src/api/routes/tests/query.seq.spec.js
index d7481fd224..05ca5f7cbb 100644
--- a/packages/server/src/api/routes/tests/query.seq.spec.js
+++ b/packages/server/src/api/routes/tests/query.seq.spec.js
@@ -1,4 +1,4 @@
-const tk = require( "timekeeper")
+const tk = require("timekeeper")
 tk.freeze(Date.now())
 
 // Mock out postgres for this
@@ -14,7 +14,7 @@ jest.mock("@budibase/backend-core", () => {
     db: {
       ...core.db,
       isProdAppID: jest.fn(),
-    }
+    },
   }
 })
 const setup = require("./utilities")
@@ -30,8 +30,7 @@ describe("/queries", () => {
 
   afterAll(setup.afterAll)
 
-  const setupTest = async()=>{
-
+  const setupTest = async () => {
     await config.init()
     datasource = await config.createDatasource()
     query = await config.createQuery()
@@ -52,7 +51,7 @@ describe("/queries", () => {
     return { datasource, query }
   }
 
-  const createQuery = async (query) => {
+  const createQuery = async query => {
     return request
       .post(`/api/queries`)
       .send(query)
@@ -76,7 +75,7 @@ describe("/queries", () => {
         _id: res.body._id,
         ...query,
         createdAt: new Date().toISOString(),
-        updatedAt: new Date().toISOString()
+        updatedAt: new Date().toISOString(),
       })
       expect(events.query.created).toBeCalledTimes(1)
       expect(events.query.updated).not.toBeCalled()
@@ -101,7 +100,7 @@ describe("/queries", () => {
         _id: res.body._id,
         ...query,
         createdAt: new Date().toISOString(),
-        updatedAt: new Date().toISOString()
+        updatedAt: new Date().toISOString(),
       })
       expect(events.query.created).not.toBeCalled()
       expect(events.query.updated).toBeCalledTimes(1)
@@ -109,7 +108,7 @@ describe("/queries", () => {
   })
 
   describe("fetch", () => {
-    beforeEach(async() => {
+    beforeEach(async () => {
       await setupTest()
     })
 
@@ -190,7 +189,7 @@ describe("/queries", () => {
   })
 
   describe("destroy", () => {
-    beforeEach(async() => {
+    beforeEach(async () => {
       await setupTest()
     })
 
@@ -237,8 +236,8 @@ describe("/queries", () => {
         .expect(200)
       // these responses come from the mock
       expect(res.body.schemaFields).toEqual({
-        "a": "string",
-        "b": "number",
+        a: "string",
+        b: "number",
       })
       expect(res.body.rows.length).toEqual(1)
       expect(events.query.previewed).toBeCalledTimes(1)
@@ -256,7 +255,7 @@ describe("/queries", () => {
   })
 
   describe("execute", () => {
-    beforeEach(async() => {
+    beforeEach(async () => {
       await setupTest()
     })
 
@@ -302,9 +301,9 @@ describe("/queries", () => {
       })
       // these responses come from the mock
       expect(res.body.schemaFields).toEqual({
-        "opts": "json",
-        "url": "string",
-        "value": "string",
+        opts: "json",
+        url: "string",
+        value: "string",
       })
       expect(res.body.rows[0].url).toEqual("http://www.google.com?test=1")
     })
@@ -316,9 +315,9 @@ describe("/queries", () => {
         queryString: "test={{ variable3 }}",
       })
       expect(res.body.schemaFields).toEqual({
-        "opts": "json",
-        "url": "string",
-        "value": "string"
+        opts: "json",
+        url: "string",
+        value: "string",
       })
       expect(res.body.rows[0].url).toContain("doctype%20html")
     })
@@ -339,9 +338,9 @@ describe("/queries", () => {
         queryString: "test={{ variable3 }}",
       })
       expect(res.body.schemaFields).toEqual({
-        "fails": "number",
-        "opts": "json",
-        "url": "string"
+        fails: "number",
+        opts: "json",
+        url: "string",
       })
       expect(res.body.rows[0].fails).toEqual(1)
     })
@@ -371,13 +370,19 @@ describe("/queries", () => {
   })
 
   describe("Current User Request Mapping", () => {
-    
     async function previewGet(datasource, fields, params) {
       return config.previewQuery(request, config, datasource, fields, params)
     }
 
     async function previewPost(datasource, fields, params) {
-      return config.previewQuery(request, config, datasource, fields, params, "create")
+      return config.previewQuery(
+        request,
+        config,
+        datasource,
+        fields,
+        params,
+        "create"
+      )
     }
 
     it("should parse global and query level header mappings", async () => {
@@ -385,120 +390,160 @@ describe("/queries", () => {
 
       const datasource = await config.restDatasource({
         defaultHeaders: {
-          "test": "headerVal",
-          "emailHdr": "{{[user].[email]}}"
-        }
+          test: "headerVal",
+          emailHdr: "{{[user].[email]}}",
+        },
       })
       const res = await previewGet(datasource, {
         path: "www.google.com",
         queryString: "email={{[user].[email]}}",
         headers: {
-          queryHdr : "{{[user].[firstName]}}",
-          secondHdr : "1234"
-        }
+          queryHdr: "{{[user].[firstName]}}",
+          secondHdr: "1234",
+        },
       })
 
       const parsedRequest = JSON.parse(res.body.extra.raw)
       expect(parsedRequest.opts.headers).toEqual({
-        "test": "headerVal",
-        "emailHdr": userDetails.email,
-        "queryHdr": userDetails.firstName,
-        "secondHdr" : "1234"
+        test: "headerVal",
+        emailHdr: userDetails.email,
+        queryHdr: userDetails.firstName,
+        secondHdr: "1234",
       })
-      expect(res.body.rows[0].url).toEqual("http://www.google.com?email=" + userDetails.email.replace("@", "%40"))
+      expect(res.body.rows[0].url).toEqual(
+        "http://www.google.com?email=" + userDetails.email.replace("@", "%40")
+      )
     })
 
     it("should bind the current user to query parameters", async () => {
       const userDetails = config.getUserDetails()
-  
+
       const datasource = await config.restDatasource()
-  
-      const res = await previewGet(datasource, {
-        path: "www.google.com",
-        queryString: "test={{myEmail}}&testName={{myName}}&testParam={{testParam}}",
-      }, {
-        "myEmail" : "{{[user].[email]}}",
-        "myName" : "{{[user].[firstName]}}",
-        "testParam" : "1234"
-      })
-  
-      expect(res.body.rows[0].url).toEqual("http://www.google.com?test=" + userDetails.email.replace("@", "%40") +
-        "&testName=" + userDetails.firstName + "&testParam=1234")
+
+      const res = await previewGet(
+        datasource,
+        {
+          path: "www.google.com",
+          queryString:
+            "test={{myEmail}}&testName={{myName}}&testParam={{testParam}}",
+        },
+        {
+          myEmail: "{{[user].[email]}}",
+          myName: "{{[user].[firstName]}}",
+          testParam: "1234",
+        }
+      )
+
+      expect(res.body.rows[0].url).toEqual(
+        "http://www.google.com?test=" +
+          userDetails.email.replace("@", "%40") +
+          "&testName=" +
+          userDetails.firstName +
+          "&testParam=1234"
+      )
     })
 
     it("should bind the current user the request body - plain text", async () => {
       const userDetails = config.getUserDetails()
       const datasource = await config.restDatasource()
-  
-      const res = await previewPost(datasource, {
-        path: "www.google.com",
-        queryString: "testParam={{testParam}}",
-        requestBody: "This is plain text and this is my email: {{[user].[email]}}. This is a test param: {{testParam}}",
-        bodyType: "text"
-      }, {
-        "testParam" : "1234"
-      })
+
+      const res = await previewPost(
+        datasource,
+        {
+          path: "www.google.com",
+          queryString: "testParam={{testParam}}",
+          requestBody:
+            "This is plain text and this is my email: {{[user].[email]}}. This is a test param: {{testParam}}",
+          bodyType: "text",
+        },
+        {
+          testParam: "1234",
+        }
+      )
 
       const parsedRequest = JSON.parse(res.body.extra.raw)
-      expect(parsedRequest.opts.body).toEqual(`This is plain text and this is my email: ${userDetails.email}. This is a test param: 1234`)
-      expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234")
+      expect(parsedRequest.opts.body).toEqual(
+        `This is plain text and this is my email: ${userDetails.email}. This is a test param: 1234`
+      )
+      expect(res.body.rows[0].url).toEqual(
+        "http://www.google.com?testParam=1234"
+      )
     })
 
     it("should bind the current user the request body - json", async () => {
       const userDetails = config.getUserDetails()
       const datasource = await config.restDatasource()
-  
-      const res = await previewPost(datasource, {
-        path: "www.google.com",
-        queryString: "testParam={{testParam}}",
-        requestBody: "{\"email\":\"{{[user].[email]}}\",\"queryCode\":{{testParam}},\"userRef\":\"{{userRef}}\"}",
-        bodyType: "json"
-      }, {
-        "testParam" : "1234",
-        "userRef" : "{{[user].[firstName]}}"
-      })
+
+      const res = await previewPost(
+        datasource,
+        {
+          path: "www.google.com",
+          queryString: "testParam={{testParam}}",
+          requestBody:
+            '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}',
+          bodyType: "json",
+        },
+        {
+          testParam: "1234",
+          userRef: "{{[user].[firstName]}}",
+        }
+      )
 
       const parsedRequest = JSON.parse(res.body.extra.raw)
       const test = `{"email":"${userDetails.email}","queryCode":1234,"userRef":"${userDetails.firstName}"}`
       expect(parsedRequest.opts.body).toEqual(test)
-      expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234")
+      expect(res.body.rows[0].url).toEqual(
+        "http://www.google.com?testParam=1234"
+      )
     })
-   
+
     it("should bind the current user the request body - xml", async () => {
       const userDetails = config.getUserDetails()
       const datasource = await config.restDatasource()
-  
-      const res = await previewPost(datasource, {
-        path: "www.google.com",
-        queryString: "testParam={{testParam}}",
-        requestBody: "<note> <email>{{[user].[email]}}</email> <code>{{testParam}}</code> " + 
-          "<ref>{{userId}}</ref> <somestring>testing</somestring> </note>",
-        bodyType: "xml"
-      }, {
-        "testParam" : "1234",
-        "userId" : "{{[user].[firstName]}}"
-      })
+
+      const res = await previewPost(
+        datasource,
+        {
+          path: "www.google.com",
+          queryString: "testParam={{testParam}}",
+          requestBody:
+            "<note> <email>{{[user].[email]}}</email> <code>{{testParam}}</code> " +
+            "<ref>{{userId}}</ref> <somestring>testing</somestring> </note>",
+          bodyType: "xml",
+        },
+        {
+          testParam: "1234",
+          userId: "{{[user].[firstName]}}",
+        }
+      )
 
       const parsedRequest = JSON.parse(res.body.extra.raw)
       const test = `<note> <email>${userDetails.email}</email> <code>1234</code> <ref>${userDetails.firstName}</ref> <somestring>testing</somestring> </note>`
-        
+
       expect(parsedRequest.opts.body).toEqual(test)
-      expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234")
+      expect(res.body.rows[0].url).toEqual(
+        "http://www.google.com?testParam=1234"
+      )
     })
 
     it("should bind the current user the request body - form-data", async () => {
       const userDetails = config.getUserDetails()
       const datasource = await config.restDatasource()
-  
-      const res = await previewPost(datasource, {
-        path: "www.google.com",
-        queryString: "testParam={{testParam}}",
-        requestBody: "{\"email\":\"{{[user].[email]}}\",\"queryCode\":{{testParam}},\"userRef\":\"{{userRef}}\"}",
-        bodyType: "form"
-      }, {
-        "testParam" : "1234",
-        "userRef" : "{{[user].[firstName]}}"
-      })
+
+      const res = await previewPost(
+        datasource,
+        {
+          path: "www.google.com",
+          queryString: "testParam={{testParam}}",
+          requestBody:
+            '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}',
+          bodyType: "form",
+        },
+        {
+          testParam: "1234",
+          userRef: "{{[user].[firstName]}}",
+        }
+      )
 
       const parsedRequest = JSON.parse(res.body.extra.raw)
 
@@ -511,28 +556,34 @@ describe("/queries", () => {
       const userRef = parsedRequest.opts.body._streams[7]
       expect(userRef).toEqual(userDetails.firstName)
 
-      expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234")
+      expect(res.body.rows[0].url).toEqual(
+        "http://www.google.com?testParam=1234"
+      )
     })
 
     it("should bind the current user the request body - encoded", async () => {
       const userDetails = config.getUserDetails()
       const datasource = await config.restDatasource()
-  
-      const res = await previewPost(datasource, {
-        path: "www.google.com",
-        queryString: "testParam={{testParam}}",
-        requestBody: "{\"email\":\"{{[user].[email]}}\",\"queryCode\":{{testParam}},\"userRef\":\"{{userRef}}\"}",
-        bodyType: "encoded"
-      }, {
-        "testParam" : "1234",
-        "userRef" : "{{[user].[firstName]}}"
-      })
+
+      const res = await previewPost(
+        datasource,
+        {
+          path: "www.google.com",
+          queryString: "testParam={{testParam}}",
+          requestBody:
+            '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}',
+          bodyType: "encoded",
+        },
+        {
+          testParam: "1234",
+          userRef: "{{[user].[firstName]}}",
+        }
+      )
       const parsedRequest = JSON.parse(res.body.extra.raw)
-      
+
       expect(parsedRequest.opts.body.email).toEqual(userDetails.email)
       expect(parsedRequest.opts.body.queryCode).toEqual("1234")
       expect(parsedRequest.opts.body.userRef).toEqual(userDetails.firstName)
     })
-
-  });
+  })
 })
diff --git a/packages/server/src/api/routes/tests/role.spec.js b/packages/server/src/api/routes/tests/role.spec.js
index d133a69d64..a653b573b2 100644
--- a/packages/server/src/api/routes/tests/role.spec.js
+++ b/packages/server/src/api/routes/tests/role.spec.js
@@ -170,7 +170,7 @@ describe("/roles", () => {
         .get("/api/roles/accessible")
         .set({
           ...config.defaultHeaders(),
-          "x-budibase-role": "CUSTOM_ROLE"
+          "x-budibase-role": "CUSTOM_ROLE",
         })
         .expect(200)
       expect(res.body.length).toBe(3)
diff --git a/packages/server/src/api/routes/tests/routing.spec.js b/packages/server/src/api/routes/tests/routing.spec.js
index 4076f4879c..5bdd4ee852 100644
--- a/packages/server/src/api/routes/tests/routing.spec.js
+++ b/packages/server/src/api/routes/tests/routing.spec.js
@@ -37,19 +37,23 @@ describe("/routing", () => {
       await runInProd(async () => {
         return request
           .get(`/api/routing/client`)
-          .set(await config.roleHeaders({
-            roleId: BUILTIN_ROLE_IDS.BASIC,
-            prodApp: false
-          }))
+          .set(
+            await config.roleHeaders({
+              roleId: BUILTIN_ROLE_IDS.BASIC,
+              prodApp: false,
+            })
+          )
           .expect(302)
       })
     })
     it("returns the correct routing for basic user", async () => {
       const res = await request
         .get(`/api/routing/client`)
-        .set(await config.roleHeaders({
-          roleId: BUILTIN_ROLE_IDS.BASIC
-        }))
+        .set(
+          await config.roleHeaders({
+            roleId: BUILTIN_ROLE_IDS.BASIC,
+          })
+        )
         .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body.routes).toBeDefined()
@@ -57,18 +61,20 @@ describe("/routing", () => {
         subpaths: {
           [route]: {
             screenId: basic._id,
-            roleId: basic.routing.roleId
-          }
-        }
+            roleId: basic.routing.roleId,
+          },
+        },
       })
     })
 
     it("returns the correct routing for power user", async () => {
       const res = await request
         .get(`/api/routing/client`)
-        .set(await config.roleHeaders({
-          roleId: BUILTIN_ROLE_IDS.POWER
-        }))
+        .set(
+          await config.roleHeaders({
+            roleId: BUILTIN_ROLE_IDS.POWER,
+          })
+        )
         .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body.routes).toBeDefined()
@@ -76,9 +82,9 @@ describe("/routing", () => {
         subpaths: {
           [route]: {
             screenId: power._id,
-            roleId: power.routing.roleId
-          }
-        }
+            roleId: power.routing.roleId,
+          },
+        },
       })
     })
   })
@@ -87,10 +93,12 @@ describe("/routing", () => {
     it("should fetch all routes for builder", async () => {
       const res = await request
         .get(`/api/routing`)
-        .set(await config.roleHeaders({
-          roleId: BUILTIN_ROLE_IDS.POWER,
-          builder: true,
-        }))
+        .set(
+          await config.roleHeaders({
+            roleId: BUILTIN_ROLE_IDS.POWER,
+            builder: true,
+          })
+        )
         .expect(200)
         .expect("Content-Type", /json/)
       expect(res.body.routes).toBeDefined()
@@ -108,4 +116,4 @@ describe("/routing", () => {
       })
     })
   })
-})
\ No newline at end of file
+})
diff --git a/packages/server/src/api/routes/tests/screen.spec.js b/packages/server/src/api/routes/tests/screen.spec.js
index a3748d6a79..481baac703 100644
--- a/packages/server/src/api/routes/tests/screen.spec.js
+++ b/packages/server/src/api/routes/tests/screen.spec.js
@@ -36,7 +36,7 @@ describe("/screens", () => {
   })
 
   describe("save", () => {
-    const saveScreen = async (screen) => {
+    const saveScreen = async screen => {
       const res = await request
         .post(`/api/screens`)
         .send(screen)
@@ -100,4 +100,4 @@ describe("/screens", () => {
       })
     })
   })
-})
\ No newline at end of file
+})
diff --git a/packages/server/src/api/routes/tests/templates.spec.js b/packages/server/src/api/routes/tests/templates.spec.js
index 3cb22fd2d3..1406a75c59 100644
--- a/packages/server/src/api/routes/tests/templates.spec.js
+++ b/packages/server/src/api/routes/tests/templates.spec.js
@@ -21,4 +21,4 @@ describe("/templates", () => {
       expect(Array.isArray(res.body)).toEqual(true)
     })
   })
-})
\ No newline at end of file
+})
diff --git a/packages/server/src/api/routes/tests/view.spec.js b/packages/server/src/api/routes/tests/view.spec.js
index e633279058..92ff097899 100644
--- a/packages/server/src/api/routes/tests/view.spec.js
+++ b/packages/server/src/api/routes/tests/view.spec.js
@@ -31,18 +31,18 @@ describe("/views", () => {
   beforeAll(async () => {
     await config.init()
   })
-  
-  beforeEach(async() => {
+
+  beforeEach(async () => {
     table = await config.createTable(priceTable())
   })
 
-  const saveView = async (view) => {
+  const saveView = async view => {
     const viewToSave = {
       name: "TestView",
       field: "Price",
       calculation: "stats",
       tableId: table._id,
-      ...view
+      ...view,
     }
     return request
       .post(`/api/views`)
@@ -53,7 +53,6 @@ describe("/views", () => {
   }
 
   describe("create", () => {
-
     it("returns a success message when the view is successfully created", async () => {
       const res = await saveView()
       expect(res.body.tableId).toBe(table._id)
@@ -81,11 +80,13 @@ describe("/views", () => {
 
       const res = await saveView({
         calculation: null,
-        filters: [{
-          value: "1",
-          condition: "EQUALS",
-          key: "price"
-        }],
+        filters: [
+          {
+            value: "1",
+            condition: "EQUALS",
+            key: "price",
+          },
+        ],
       })
 
       expect(res.body.tableId).toBe(table._id)
@@ -199,18 +200,26 @@ describe("/views", () => {
     })
 
     it("updates a view filter", async () => {
-      await saveView({ filters: [{
-        value: "1",
-        condition: "EQUALS",
-        key: "price"
-      }] })
+      await saveView({
+        filters: [
+          {
+            value: "1",
+            condition: "EQUALS",
+            key: "price",
+          },
+        ],
+      })
       jest.clearAllMocks()
 
-      await saveView({ filters: [{
-        value: "2",
-        condition: "EQUALS",
-        key: "price"
-      }] })
+      await saveView({
+        filters: [
+          {
+            value: "2",
+            condition: "EQUALS",
+            key: "price",
+          },
+        ],
+      })
 
       expect(events.view.created).not.toBeCalled()
       expect(events.view.updated).toBeCalledTimes(1)
@@ -223,11 +232,15 @@ describe("/views", () => {
     })
 
     it("deletes a view filter", async () => {
-      await saveView({ filters: [{
-        value: "1",
-        condition: "EQUALS",
-        key: "price"
-      }] })
+      await saveView({
+        filters: [
+          {
+            value: "1",
+            condition: "EQUALS",
+            key: "price",
+          },
+        ],
+      })
       jest.clearAllMocks()
 
       await saveView({ filters: [] })
@@ -344,7 +357,6 @@ describe("/views", () => {
   })
 
   describe("exportView", () => {
-
     beforeEach(() => {
       jest.clearAllMocks()
     })
@@ -362,14 +374,14 @@ describe("/views", () => {
         .expect(200)
     }
 
-    const assertJsonExport = (res) => {
+    const assertJsonExport = res => {
       const rows = JSON.parse(res.text)
       expect(rows.length).toBe(1)
       expect(rows[0].name).toBe("test-name")
       expect(rows[0].description).toBe("ùúûü")
     }
 
-    const assertCSVExport = (res) => {
+    const assertCSVExport = res => {
       expect(res.text).toBe(`"name","description"\n"test-name","ùúûü"`)
     }
 
diff --git a/packages/server/src/automations/tests/bash.spec.js b/packages/server/src/automations/tests/bash.spec.js
index a8695c7460..d2f3bb8031 100644
--- a/packages/server/src/automations/tests/bash.spec.js
+++ b/packages/server/src/automations/tests/bash.spec.js
@@ -1,34 +1,33 @@
 const setup = require("./utilities")
 
 describe("test the bash action", () => {
-    let config = setup.getConfig()
+  let config = setup.getConfig()
 
-    beforeAll(async () => {
-        await config.init()
-    })
-    afterAll(setup.afterAll)
+  beforeAll(async () => {
+    await config.init()
+  })
+  afterAll(setup.afterAll)
 
-    it("should be able to execute a script", async () => {
+  it("should be able to execute a script", async () => {
+    let res = await setup.runStep(
+      "EXECUTE_BASH",
+      (inputs = {
+        code: "echo 'test'",
+      })
+    )
+    expect(res.stdout).toEqual("test\n")
+    expect(res.success).toEqual(true)
+  })
 
-        let res = await setup.runStep("EXECUTE_BASH",
-            inputs = {
-                code: "echo 'test'"
-            }
-
-        )
-        expect(res.stdout).toEqual("test\n")
-        expect(res.success).toEqual(true)
-    })
-
-    it("should handle a null value", async () => {
-
-        let res = await setup.runStep("EXECUTE_BASH",
-            inputs = {
-                code: null
-            }
-
-
-        )
-        expect(res.stdout).toEqual("Budibase bash automation failed: Invalid inputs")
-    })
+  it("should handle a null value", async () => {
+    let res = await setup.runStep(
+      "EXECUTE_BASH",
+      (inputs = {
+        code: null,
+      })
+    )
+    expect(res.stdout).toEqual(
+      "Budibase bash automation failed: Invalid inputs"
+    )
+  })
 })
diff --git a/packages/server/src/automations/tests/delay.spec.js b/packages/server/src/automations/tests/delay.spec.js
index ad67c74606..5f6e0fd164 100644
--- a/packages/server/src/automations/tests/delay.spec.js
+++ b/packages/server/src/automations/tests/delay.spec.js
@@ -1,7 +1,7 @@
 const setup = require("./utilities")
 
 // need real Date for this test
-const tk = require('timekeeper');
+const tk = require("timekeeper")
 tk.reset()
 
 describe("test the delay logic", () => {
diff --git a/packages/server/src/automations/tests/discord.spec.js b/packages/server/src/automations/tests/discord.spec.js
index d91c46094a..ccae42d073 100644
--- a/packages/server/src/automations/tests/discord.spec.js
+++ b/packages/server/src/automations/tests/discord.spec.js
@@ -4,24 +4,23 @@ const fetch = require("node-fetch")
 jest.mock("node-fetch")
 
 describe("test the outgoing webhook action", () => {
-    let inputs
-    let config = setup.getConfig()
+  let inputs
+  let config = setup.getConfig()
 
-    beforeAll(async () => {
-        await config.init()
-        inputs = {
-            username: "joe_bloggs",
-            url: "http://www.test.com",
-        }
-    })
+  beforeAll(async () => {
+    await config.init()
+    inputs = {
+      username: "joe_bloggs",
+      url: "http://www.test.com",
+    }
+  })
 
-    afterAll(setup.afterAll)
-
-    it("should be able to run the action", async () => {
-        const res = await setup.runStep(setup.actions.discord.stepId, inputs)
-        expect(res.response.url).toEqual("http://www.test.com")
-        expect(res.response.method).toEqual("post")
-        expect(res.success).toEqual(true)
-    })
+  afterAll(setup.afterAll)
 
+  it("should be able to run the action", async () => {
+    const res = await setup.runStep(setup.actions.discord.stepId, inputs)
+    expect(res.response.url).toEqual("http://www.test.com")
+    expect(res.response.method).toEqual("post")
+    expect(res.success).toEqual(true)
+  })
 })
diff --git a/packages/server/src/automations/tests/executeQuery.spec.js b/packages/server/src/automations/tests/executeQuery.spec.js
index ae2ca1dfd8..3b747f2ba3 100644
--- a/packages/server/src/automations/tests/executeQuery.spec.js
+++ b/packages/server/src/automations/tests/executeQuery.spec.js
@@ -1,48 +1,47 @@
 const setup = require("./utilities")
 
 describe("test the execute query action", () => {
-    let config = setup.getConfig()
+  let config = setup.getConfig()
 
-    beforeAll(async () => {
-        await config.init()
+  beforeAll(async () => {
+    await config.init()
 
-        await config.createDatasource()
-        query = await config.createQuery()
+    await config.createDatasource()
+    query = await config.createQuery()
+  })
 
-    })
+  afterAll(setup.afterAll)
 
-    afterAll(setup.afterAll)
-
-    it("should be able to execute a query", async () => {
-        let res = await setup.runStep(setup.actions.EXECUTE_QUERY.stepId,
-            inputs = {
-                query: { queryId: query._id }
-            }
-        )
-        expect(res.response).toEqual([{ a: 'string', b: 1 }])
-        expect(res.success).toEqual(true)
-    })
-
-    it("should handle a null query value", async () => {
-        let res = await setup.runStep(setup.actions.EXECUTE_QUERY.stepId,
-            inputs = {
-                query: null
-            }
-        )
-        expect(res.response.message).toEqual("Invalid inputs")
-        expect(res.success).toEqual(false)
-    })
-
-
-    it("should handle an error executing a query", async () => {
-        let res = await setup.runStep(setup.actions.EXECUTE_QUERY.stepId,
-            inputs = {
-                query: { queryId: "wrong_id" }
-            }
-        )
-        expect(res.response).toEqual('Error: missing')
-        expect(res.success).toEqual(false)
-    })
+  it("should be able to execute a query", async () => {
+    let res = await setup.runStep(
+      setup.actions.EXECUTE_QUERY.stepId,
+      (inputs = {
+        query: { queryId: query._id },
+      })
+    )
+    expect(res.response).toEqual([{ a: "string", b: 1 }])
+    expect(res.success).toEqual(true)
+  })
 
+  it("should handle a null query value", async () => {
+    let res = await setup.runStep(
+      setup.actions.EXECUTE_QUERY.stepId,
+      (inputs = {
+        query: null,
+      })
+    )
+    expect(res.response.message).toEqual("Invalid inputs")
+    expect(res.success).toEqual(false)
+  })
 
+  it("should handle an error executing a query", async () => {
+    let res = await setup.runStep(
+      setup.actions.EXECUTE_QUERY.stepId,
+      (inputs = {
+        query: { queryId: "wrong_id" },
+      })
+    )
+    expect(res.response).toEqual("Error: missing")
+    expect(res.success).toEqual(false)
+  })
 })
diff --git a/packages/server/src/automations/tests/executeScript.spec.js b/packages/server/src/automations/tests/executeScript.spec.js
index b9aedb8e31..5aac8400cd 100644
--- a/packages/server/src/automations/tests/executeScript.spec.js
+++ b/packages/server/src/automations/tests/executeScript.spec.js
@@ -1,48 +1,43 @@
 const setup = require("./utilities")
 
 describe("test the execute script action", () => {
-    let config = setup.getConfig()
+  let config = setup.getConfig()
 
-    beforeAll(async () => {
-        await config.init()
-    })
-    afterAll(setup.afterAll)
+  beforeAll(async () => {
+    await config.init()
+  })
+  afterAll(setup.afterAll)
 
-    it("should be able to execute a script", async () => {
+  it("should be able to execute a script", async () => {
+    let res = await setup.runStep(
+      setup.actions.EXECUTE_SCRIPT.stepId,
+      (inputs = {
+        code: "return 1 + 1",
+      })
+    )
+    expect(res.value).toEqual(2)
+    expect(res.success).toEqual(true)
+  })
 
-        let res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId,
-            inputs = {
-                code: "return 1 + 1"
-            }
-
-        )
-        expect(res.value).toEqual(2)
-        expect(res.success).toEqual(true)
-    })
-
-    it("should handle a null value", async () => {
-
-        let res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId,
-            inputs = {
-                code: null
-            }
-
-
-        )
-        expect(res.response.message).toEqual("Invalid inputs")
-        expect(res.success).toEqual(false)
-    })
-
-    it("should be able to handle an error gracefully", async () => {
-
-        let res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId,
-            inputs = {
-                code: "return something.map(x => x.name)"
-            }
-
-        )
-        expect(res.response).toEqual("ReferenceError: something is not defined")
-        expect(res.success).toEqual(false)
-    })
+  it("should handle a null value", async () => {
+    let res = await setup.runStep(
+      setup.actions.EXECUTE_SCRIPT.stepId,
+      (inputs = {
+        code: null,
+      })
+    )
+    expect(res.response.message).toEqual("Invalid inputs")
+    expect(res.success).toEqual(false)
+  })
 
+  it("should be able to handle an error gracefully", async () => {
+    let res = await setup.runStep(
+      setup.actions.EXECUTE_SCRIPT.stepId,
+      (inputs = {
+        code: "return something.map(x => x.name)",
+      })
+    )
+    expect(res.response).toEqual("ReferenceError: something is not defined")
+    expect(res.success).toEqual(false)
+  })
 })
diff --git a/packages/server/src/automations/tests/outgoingWebhook.spec.js b/packages/server/src/automations/tests/outgoingWebhook.spec.js
index c31ed850e7..4bc51d0989 100644
--- a/packages/server/src/automations/tests/outgoingWebhook.spec.js
+++ b/packages/server/src/automations/tests/outgoingWebhook.spec.js
@@ -21,7 +21,10 @@ describe("test the outgoing webhook action", () => {
   afterAll(setup.afterAll)
 
   it("should be able to run the action", async () => {
-    const res = await setup.runStep(setup.actions.OUTGOING_WEBHOOK.stepId, inputs)
+    const res = await setup.runStep(
+      setup.actions.OUTGOING_WEBHOOK.stepId,
+      inputs
+    )
     expect(res.success).toEqual(true)
     expect(res.response.url).toEqual("http://www.test.com")
     expect(res.response.method).toEqual("POST")
@@ -31,9 +34,8 @@ describe("test the outgoing webhook action", () => {
   it("should return an error if something goes wrong in fetch", async () => {
     const res = await setup.runStep(setup.actions.OUTGOING_WEBHOOK.stepId, {
       requestMethod: "GET",
-      url: "www.invalid.com"
+      url: "www.invalid.com",
     })
     expect(res.success).toEqual(false)
   })
-
 })
diff --git a/packages/server/src/automations/tests/serverLog.spec.js b/packages/server/src/automations/tests/serverLog.spec.js
index 8b5ce2e64f..8869c07a4b 100644
--- a/packages/server/src/automations/tests/serverLog.spec.js
+++ b/packages/server/src/automations/tests/serverLog.spec.js
@@ -1,22 +1,19 @@
 const setup = require("./utilities")
 
 describe("test the server log action", () => {
-    let config = setup.getConfig()
+  let config = setup.getConfig()
 
-    beforeAll(async () => {
-        await config.init()
-        inputs = {
-            text: "log message",
-        }
-    })
-    afterAll(setup.afterAll)
+  beforeAll(async () => {
+    await config.init()
+    inputs = {
+      text: "log message",
+    }
+  })
+  afterAll(setup.afterAll)
 
-    it("should be able to log the text", async () => {
-
-        let res = await setup.runStep(setup.actions.SERVER_LOG.stepId,
-            inputs
-        )
-        expect(res.message).toEqual(`App ${config.getAppId()} - ${inputs.text}`)
-        expect(res.success).toEqual(true)
-    })
+  it("should be able to log the text", async () => {
+    let res = await setup.runStep(setup.actions.SERVER_LOG.stepId, inputs)
+    expect(res.message).toEqual(`App ${config.getAppId()} - ${inputs.text}`)
+    expect(res.success).toEqual(true)
+  })
 })
diff --git a/packages/server/src/db/tests/linkController.spec.js b/packages/server/src/db/tests/linkController.spec.js
index 5caf35f61a..313cfee837 100644
--- a/packages/server/src/db/tests/linkController.spec.js
+++ b/packages/server/src/db/tests/linkController.spec.js
@@ -1,5 +1,9 @@
 const TestConfig = require("../../tests/utilities/TestConfiguration")
-const { basicRow, basicLinkedRow, basicTable } = require("../../tests/utilities/structures")
+const {
+  basicRow,
+  basicLinkedRow,
+  basicTable,
+} = require("../../tests/utilities/structures")
 const LinkController = require("../linkedRows/LinkController").default
 const { context } = require("@budibase/backend-core")
 const { RelationshipType } = require("../../constants")
@@ -16,7 +20,10 @@ describe("test the link controller", () => {
 
   beforeEach(async () => {
     const { _id } = await config.createTable()
-    table2 = await config.createLinkedTable(RelationshipType.MANY_TO_MANY, ["link", "link2"])
+    table2 = await config.createLinkedTable(RelationshipType.MANY_TO_MANY, [
+      "link",
+      "link2",
+    ])
     // update table after creating link
     table1 = await config.getTable(_id)
   })
@@ -41,15 +48,23 @@ describe("test the link controller", () => {
 
   async function createLinkedRow(linkField = "link", t1 = table1, t2 = table2) {
     const row = await config.createRow(basicRow(t2._id))
-    const { _id } = await config.createRow(basicLinkedRow(t1._id, row._id, linkField))
+    const { _id } = await config.createRow(
+      basicLinkedRow(t1._id, row._id, linkField)
+    )
     return config.getRow(t1._id, _id)
   }
 
   it("should be able to confirm if two table schemas are equal", async () => {
     const controller = await createLinkController(table1)
-    let equal = controller.areLinkSchemasEqual(table2.schema.link, table2.schema.link)
+    let equal = controller.areLinkSchemasEqual(
+      table2.schema.link,
+      table2.schema.link
+    )
     expect(equal).toEqual(true)
-    equal = controller.areLinkSchemasEqual(table1.schema.link, table2.schema.link)
+    equal = controller.areLinkSchemasEqual(
+      table1.schema.link,
+      table2.schema.link
+    )
     expect(equal).toEqual(false)
   })
 
@@ -57,17 +72,42 @@ describe("test the link controller", () => {
     const controller = await createLinkController(table1)
     // empty case
     let output = controller.handleRelationshipType({}, {})
-    expect(output.linkedField.relationshipType).toEqual(RelationshipType.MANY_TO_MANY)
-    expect(output.linkerField.relationshipType).toEqual(RelationshipType.MANY_TO_MANY)
-    output = controller.handleRelationshipType({ relationshipType: RelationshipType.MANY_TO_MANY }, {})
-    expect(output.linkedField.relationshipType).toEqual(RelationshipType.MANY_TO_MANY)
-    expect(output.linkerField.relationshipType).toEqual(RelationshipType.MANY_TO_MANY)
-    output = controller.handleRelationshipType({ relationshipType: RelationshipType.MANY_TO_ONE }, {})
-    expect(output.linkedField.relationshipType).toEqual(RelationshipType.ONE_TO_MANY)
-    expect(output.linkerField.relationshipType).toEqual(RelationshipType.MANY_TO_ONE)
-    output = controller.handleRelationshipType({ relationshipType: RelationshipType.ONE_TO_MANY }, {})
-    expect(output.linkedField.relationshipType).toEqual(RelationshipType.MANY_TO_ONE)
-    expect(output.linkerField.relationshipType).toEqual(RelationshipType.ONE_TO_MANY)
+    expect(output.linkedField.relationshipType).toEqual(
+      RelationshipType.MANY_TO_MANY
+    )
+    expect(output.linkerField.relationshipType).toEqual(
+      RelationshipType.MANY_TO_MANY
+    )
+    output = controller.handleRelationshipType(
+      { relationshipType: RelationshipType.MANY_TO_MANY },
+      {}
+    )
+    expect(output.linkedField.relationshipType).toEqual(
+      RelationshipType.MANY_TO_MANY
+    )
+    expect(output.linkerField.relationshipType).toEqual(
+      RelationshipType.MANY_TO_MANY
+    )
+    output = controller.handleRelationshipType(
+      { relationshipType: RelationshipType.MANY_TO_ONE },
+      {}
+    )
+    expect(output.linkedField.relationshipType).toEqual(
+      RelationshipType.ONE_TO_MANY
+    )
+    expect(output.linkerField.relationshipType).toEqual(
+      RelationshipType.MANY_TO_ONE
+    )
+    output = controller.handleRelationshipType(
+      { relationshipType: RelationshipType.ONE_TO_MANY },
+      {}
+    )
+    expect(output.linkedField.relationshipType).toEqual(
+      RelationshipType.MANY_TO_ONE
+    )
+    expect(output.linkerField.relationshipType).toEqual(
+      RelationshipType.ONE_TO_MANY
+    )
   })
 
   it("should be able to delete a row", async () => {
@@ -100,7 +140,7 @@ describe("test the link controller", () => {
   it("should throw an error when validating a table which is invalid", async () => {
     const controller = await createLinkController(table1)
     const copyTable = {
-      ...table1
+      ...table1,
     }
     copyTable.schema.otherTableLink = {
       type: "link",
@@ -114,7 +154,9 @@ describe("test the link controller", () => {
       error = err
     }
     expect(error).toBeDefined()
-    expect(error.message).toEqual("Cannot re-use the linked column name for a linked table.")
+    expect(error.message).toEqual(
+      "Cannot re-use the linked column name for a linked table."
+    )
   })
 
   it("should be able to remove a link when saving/updating the row", async () => {
@@ -183,7 +225,10 @@ describe("test the link controller", () => {
 
   it("shouldn't allow one to many having many relationships against it", async () => {
     const firstTable = await config.createTable()
-    const { _id } = await config.createLinkedTable(RelationshipType.MANY_TO_ONE, ["link"])
+    const { _id } = await config.createLinkedTable(
+      RelationshipType.MANY_TO_ONE,
+      ["link"]
+    )
     const linkTable = await config.getTable(_id)
     // an initial row to link around
     const row = await createLinkedRow("link", linkTable, firstTable)
diff --git a/packages/server/src/db/tests/linkTests.spec.js b/packages/server/src/db/tests/linkTests.spec.js
index 19a0eb88d3..7b3fedb5a9 100644
--- a/packages/server/src/db/tests/linkTests.spec.js
+++ b/packages/server/src/db/tests/linkTests.spec.js
@@ -45,7 +45,9 @@ describe("test link functionality", () => {
     })
 
     it("should get the field from the link", () => {
-      expect(linkUtils.getRelatedTableForField(link, "otherLink")).toBe("tableID")
+      expect(linkUtils.getRelatedTableForField(link, "otherLink")).toBe(
+        "tableID"
+      )
     })
   })
 
@@ -62,4 +64,4 @@ describe("test link functionality", () => {
       })
     })
   })
-})
\ No newline at end of file
+})
diff --git a/packages/server/src/middleware/tests/resourceId.spec.js b/packages/server/src/middleware/tests/resourceId.spec.js
index 56d89cfede..759aa46f00 100644
--- a/packages/server/src/middleware/tests/resourceId.spec.js
+++ b/packages/server/src/middleware/tests/resourceId.spec.js
@@ -3,7 +3,7 @@ const {
   paramSubResource,
   bodyResource,
   bodySubResource,
-  ResourceIdGetter
+  ResourceIdGetter,
 } = require("../resourceId")
 
 class TestConfiguration {
@@ -41,7 +41,7 @@ describe("resourceId middleware", () => {
   it("generates a resourceId middleware for context query parameters", () => {
     const config = new TestConfiguration(paramResource("main"))
     config.setParams({
-      main: "test"
+      main: "test",
     })
 
     config.executeMiddleware()
@@ -53,7 +53,7 @@ describe("resourceId middleware", () => {
     const config = new TestConfiguration(paramSubResource("main", "sub"))
     config.setParams({
       main: "main",
-      sub: "test"
+      sub: "test",
     })
 
     config.executeMiddleware()
@@ -65,7 +65,7 @@ describe("resourceId middleware", () => {
   it("generates a resourceId middleware for context request body", () => {
     const config = new TestConfiguration(bodyResource("main"))
     config.setBody({
-      main: "test"
+      main: "test",
     })
 
     config.executeMiddleware()
@@ -77,7 +77,7 @@ describe("resourceId middleware", () => {
     const config = new TestConfiguration(bodySubResource("main", "sub"))
     config.setBody({
       main: "main",
-      sub: "test"
+      sub: "test",
     })
 
     config.executeMiddleware()
@@ -94,7 +94,7 @@ describe("resourceId middleware", () => {
     let config = new TestConfiguration(middleware)
     config.setBody({
       custom: "test",
-      customSub: "subtest"
+      customSub: "subtest",
     })
 
     config.executeMiddleware()
@@ -102,4 +102,4 @@ describe("resourceId middleware", () => {
     expect(config.ctx.resourceId).toEqual("test")
     expect(config.ctx.subResourceId).toEqual("subtest")
   })
-})
\ No newline at end of file
+})
diff --git a/packages/server/src/middleware/tests/selfhost.spec.js b/packages/server/src/middleware/tests/selfhost.spec.js
index 70ed981ac3..03372e9783 100644
--- a/packages/server/src/middleware/tests/selfhost.spec.js
+++ b/packages/server/src/middleware/tests/selfhost.spec.js
@@ -10,7 +10,7 @@ class TestConfiguration {
 
     this.ctx = {
       next: this.next,
-      throw: this.throw
+      throw: this.throw,
     }
   }
 
@@ -35,7 +35,7 @@ describe("Self host middleware", () => {
   })
 
   it("calls next() when SELF_HOSTED env var is set", async () => {
-    env.SELF_HOSTED = 1 
+    env.SELF_HOSTED = 1
 
     await config.executeMiddleware()
     expect(config.next).toHaveBeenCalled()
diff --git a/packages/server/src/migrations/functions/tests/appUrls.spec.js b/packages/server/src/migrations/functions/tests/appUrls.spec.js
index f60dff855c..7a2f324552 100644
--- a/packages/server/src/migrations/functions/tests/appUrls.spec.js
+++ b/packages/server/src/migrations/functions/tests/appUrls.spec.js
@@ -10,7 +10,7 @@ describe("run", () => {
     await config.init()
   })
 
-  afterAll(config.end)  
+  afterAll(config.end)
 
   it("runs successfully", async () => {
     const app = await config.createApp("testApp")
diff --git a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js b/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js
index 15ddcfd1ef..f0da25893c 100644
--- a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js
+++ b/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js
@@ -4,8 +4,8 @@ jest.mock("@budibase/backend-core", () => {
     ...core,
     db: {
       ...core.db,
-      createNewUserEmailView: jest.fn()
-    }
+      createNewUserEmailView: jest.fn(),
+    },
   }
 })
 const { context, db: dbCore } = require("@budibase/backend-core")
@@ -16,19 +16,19 @@ const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const migration = require("../userEmailViewCasing")
 
 describe("run", () => {
-    let config = new TestConfig(false)
+  let config = new TestConfig(false)
 
-    beforeAll(async () => {
-      await config.init()
-    })
-
-    afterAll(config.end)
-
-    it("runs successfully", async () => {
-      await config.doInTenant(async () => {
-        const globalDb = context.getGlobalDB()
-        await migration.run(globalDb)
-        expect(dbCore.createNewUserEmailView).toHaveBeenCalledTimes(1)
-      })
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)
+
+  it("runs successfully", async () => {
+    await config.doInTenant(async () => {
+      const globalDb = context.getGlobalDB()
+      await migration.run(globalDb)
+      expect(dbCore.createNewUserEmailView).toHaveBeenCalledTimes(1)
     })
+  })
 })
diff --git a/packages/string-templates/test/basic.spec.js b/packages/string-templates/test/basic.spec.js
index 3346a81ddd..a5054bf489 100644
--- a/packages/string-templates/test/basic.spec.js
+++ b/packages/string-templates/test/basic.spec.js
@@ -86,11 +86,11 @@ describe("Test that the object processing works correctly", () => {
   })
 
   it("check objects get converted to string JSON automatically", async () => {
-    const row = {a: 1}
+    const row = { a: 1 }
     const output = await processString("{{ trigger.row }}", {
       trigger: {
         row,
-      }
+      },
     })
     expect(JSON.parse(output)).toEqual(row)
   })
@@ -108,9 +108,9 @@ describe("Test that the object processing works correctly", () => {
 
 describe("check returning objects", () => {
   it("should handle an array of objects", async () => {
-    const json = [{a: 1},{a: 2}]
+    const json = [{ a: 1 }, { a: 2 }]
     const output = await processString("{{ testing }}", {
-      testing: json
+      testing: json,
     })
     expect(output).toEqual(JSON.stringify(json))
   })
@@ -164,9 +164,12 @@ describe("check manifest", () => {
 describe("check full stops that are safe", () => {
   it("should allow using an escaped full stop", async () => {
     const data = {
-      "c53a4a604fa754d33baaafd5bca4d3658-YXuUBqt5vI": { "persons.firstname": "1" }
+      "c53a4a604fa754d33baaafd5bca4d3658-YXuUBqt5vI": {
+        "persons.firstname": "1",
+      },
     }
-    const template = "{{ [c53a4a604fa754d33baaafd5bca4d3658-YXuUBqt5vI].[persons.firstname] }}"
+    const template =
+      "{{ [c53a4a604fa754d33baaafd5bca4d3658-YXuUBqt5vI].[persons.firstname] }}"
     const output = await processString(template, data)
     expect(output).toEqual("1")
   })
@@ -195,7 +198,9 @@ describe("check that disabling escaping function works", () => {
   })
 
   it("should work for two statements", () => {
-    expect(disableEscaping("{{ name }} welcome to {{ platform }}")).toEqual("{{{ name }}} welcome to {{{ platform }}}")
+    expect(disableEscaping("{{ name }} welcome to {{ platform }}")).toEqual(
+      "{{{ name }}} welcome to {{{ platform }}}"
+    )
   })
 
   it("shouldn't convert triple braces", () => {
@@ -203,11 +208,15 @@ describe("check that disabling escaping function works", () => {
   })
 
   it("should work with a combination", () => {
-    expect(disableEscaping("{{ name }} welcome to {{{ platform }}}")).toEqual("{{{ name }}} welcome to {{{ platform }}}")
+    expect(disableEscaping("{{ name }} welcome to {{{ platform }}}")).toEqual(
+      "{{{ name }}} welcome to {{{ platform }}}"
+    )
   })
 
   it("should work with multiple escaped", () => {
-    expect(disableEscaping("{{ name }} welcome to {{ name }}")).toEqual("{{{ name }}} welcome to {{{ name }}}")
+    expect(disableEscaping("{{ name }} welcome to {{ name }}")).toEqual(
+      "{{{ name }}} welcome to {{{ name }}}"
+    )
   })
 })
 
@@ -217,13 +226,20 @@ describe("check find hbs blocks function", () => {
   })
 
   it("should find two", () => {
-    expect(findHBSBlocks("{{ hello }} there {{{ name }}}")).toEqual(["{{ hello }}", "{{{ name }}}"])
+    expect(findHBSBlocks("{{ hello }} there {{{ name }}}")).toEqual([
+      "{{ hello }}",
+      "{{{ name }}}",
+    ])
   })
 })
 
 describe("should leave HBS blocks if not found using option", () => {
   it("should replace one, leave one", async () => {
-    const output = await processString("{{ a }}, {{ b }}", { b: 1 }, { onlyFound: true })
+    const output = await processString(
+      "{{ a }}, {{ b }}",
+      { b: 1 },
+      { onlyFound: true }
+    )
     expect(output).toBe("{{ a }}, 1")
   })
 })
diff --git a/packages/string-templates/test/escapes.spec.js b/packages/string-templates/test/escapes.spec.js
index a14e78d2f2..d27ca64240 100644
--- a/packages/string-templates/test/escapes.spec.js
+++ b/packages/string-templates/test/escapes.spec.js
@@ -68,11 +68,11 @@ describe("attempt some complex problems", () => {
 describe("check behaviour with newlines", () => {
   const context = {
     binding: `Hello
-      there`
+      there`,
   }
   it("should escape new line to \\n with double brace", async () => {
     const hbs = JSON.stringify({
-      body: "{{ binding }}"
+      body: "{{ binding }}",
     })
     const output = await processString(hbs, context, { escapeNewlines: true })
     expect(JSON.parse(output).body).toBe(context.binding)
@@ -80,7 +80,7 @@ describe("check behaviour with newlines", () => {
 
   it("should work the same with triple brace", async () => {
     const hbs = JSON.stringify({
-      body: "{{{ binding }}}"
+      body: "{{{ binding }}}",
     })
     const output = await processString(hbs, context, { escapeNewlines: true })
     expect(JSON.parse(output).body).toBe(context.binding)
@@ -88,9 +88,12 @@ describe("check behaviour with newlines", () => {
 
   it("should still work with helpers disabled", async () => {
     const hbs = JSON.stringify({
-      body: "{{ binding }}"
+      body: "{{ binding }}",
+    })
+    const output = await processString(hbs, context, {
+      escapeNewlines: true,
+      noHelpers: true,
     })
-    const output = await processString(hbs, context, { escapeNewlines: true, noHelpers: true })
     expect(JSON.parse(output).body).toBe(context.binding)
   })
 })
diff --git a/packages/string-templates/test/hbsToJs.spec.js b/packages/string-templates/test/hbsToJs.spec.js
index 63bd80db81..8a74cae526 100644
--- a/packages/string-templates/test/hbsToJs.spec.js
+++ b/packages/string-templates/test/hbsToJs.spec.js
@@ -1,6 +1,4 @@
-const {
-  convertToJS
-} = require("../src/index.cjs")
+const { convertToJS } = require("../src/index.cjs")
 
 function checkLines(response, lines) {
   const toCheck = response.split("\n")
@@ -18,25 +16,19 @@ describe("Test that the string processing works correctly", () => {
 
   it("basic example with square brackets", () => {
     const response = convertToJS("{{ [query] }}")
-    checkLines(response, [
-      "const var1 = $(\"[query]\");",
-      "return `${var1}`;",
-    ])
+    checkLines(response, ['const var1 = $("[query]");', "return `${var1}`;"])
   })
 
   it("handle properties", () => {
     const response = convertToJS("{{ [query].id }}")
-    checkLines(response, [
-      "const var1 = $(\"[query].id\");",
-      "return `${var1}`;",
-    ])
+    checkLines(response, ['const var1 = $("[query].id");', "return `${var1}`;"])
   })
 
   it("should convert some basic HBS strings", () => {
     const response = convertToJS("Hello {{ name }}, welcome to {{ company }}!")
     checkLines(response, [
-      "const var1 = $(\"name\");",
-      "const var2 = $(\"company\");",
+      'const var1 = $("name");',
+      'const var2 = $("company");',
       "return `Hello ${var1}, welcome to ${var2}`;",
     ])
   })
@@ -44,7 +36,7 @@ describe("Test that the string processing works correctly", () => {
   it("should handle many square brackets in helpers", () => {
     const response = convertToJS("Hello {{ avg [user].[_id] [user].[_rev] }}")
     checkLines(response, [
-      "const var1 = helpers.avg($(\"[user].[_id]\"), $(\"[user].[_rev]\"));",
+      'const var1 = helpers.avg($("[user].[_id]"), $("[user].[_rev]"));',
       "return `Hello ${var1}`;",
     ])
   })
@@ -61,7 +53,7 @@ describe("Test that the string processing works correctly", () => {
     const response = convertToJS("{{equalsLength '[1,2,3]' 3}}")
     checkLines(response, [
       "const var1 = helpers.equalsLength('[1,2,3]', 3);",
-      "return `${var1}`;"
+      "return `${var1}`;",
     ])
   })
 
@@ -84,23 +76,27 @@ describe("Test that the string processing works correctly", () => {
   it("should handle a helper block", () => {
     const response = convertToJS("This is the average: {{ avg array }}")
     checkLines(response, [
-      "const var1 = helpers.avg($(\"array\"));",
+      'const var1 = helpers.avg($("array"));',
       "return `This is the average: ${var1}`;",
     ])
   })
 
   it("should handle multi-variable helper", () => {
-    const response = convertToJS("This is the average: {{ join ( avg val1 val2 val3 ) }}")
+    const response = convertToJS(
+      "This is the average: {{ join ( avg val1 val2 val3 ) }}"
+    )
     checkLines(response, [
-      "const var1 = helpers.join(helpers.avg($(\"val1\"), $(\"val2\"), $(\"val3\")));",
+      'const var1 = helpers.join(helpers.avg($("val1"), $("val2"), $("val3")));',
       "return `This is the average: ${var1}`;",
     ])
   })
 
   it("should handle a complex statement", () => {
-    const response = convertToJS("This is the average: {{ join ( avg val1 val2 val3 ) val4 }}")
+    const response = convertToJS(
+      "This is the average: {{ join ( avg val1 val2 val3 ) val4 }}"
+    )
     checkLines(response, [
-      "const var1 = helpers.join(helpers.avg($(\"val1\"), $(\"val2\"), $(\"val3\")), $(\"val4\"));",
+      'const var1 = helpers.join(helpers.avg($("val1"), $("val2"), $("val3")), $("val4"));',
       "return `This is the average: ${var1}`;",
     ])
   })
@@ -108,7 +104,7 @@ describe("Test that the string processing works correctly", () => {
   it("should handle square brackets", () => {
     const response = convertToJS("This is: {{ [val thing] }}")
     checkLines(response, [
-      "const var1 = $(\"[val thing]\");",
+      'const var1 = $("[val thing]");',
       "return `This is: ${var1}`;",
     ])
   })
@@ -116,17 +112,19 @@ describe("Test that the string processing works correctly", () => {
   it("should handle square brackets with properties", () => {
     const response = convertToJS("{{ [user].[_id] }}")
     checkLines(response, [
-      "const var1 = $(\"[user].[_id]\");",
+      'const var1 = $("[user].[_id]");',
       "return `${var1}`;",
     ])
   })
 
   it("should handle multiple complex statements", () => {
-    const response = convertToJS("average: {{ avg ( abs val1 ) val2 }} add: {{ add 1 2 }}")
+    const response = convertToJS(
+      "average: {{ avg ( abs val1 ) val2 }} add: {{ add 1 2 }}"
+    )
     checkLines(response, [
-      "const var1 = helpers.avg(helpers.abs($(\"val1\")), $(\"val2\"));",
+      'const var1 = helpers.avg(helpers.abs($("val1")), $("val2"));',
       "const var2 = helpers.add(1, 2);",
       "return `average: ${var1} add: ${var2}`;",
     ])
   })
-})
\ No newline at end of file
+})
diff --git a/packages/string-templates/test/helpers.spec.js b/packages/string-templates/test/helpers.spec.js
index a587ca60dd..c047947fbf 100644
--- a/packages/string-templates/test/helpers.spec.js
+++ b/packages/string-templates/test/helpers.spec.js
@@ -266,10 +266,7 @@ describe("test the string helpers", () => {
   })
 
   it("should allow use of the ellipsis helper", async () => {
-    const output = await processString(
-      "{{ ellipsis \"adfasdfasdfasf\" 7 }}",
-      {},
-    )
+    const output = await processString('{{ ellipsis "adfasdfasdfasf" 7 }}', {})
     expect(output).toBe("adfasdf…")
   })
 })
diff --git a/packages/string-templates/test/javascript.spec.js b/packages/string-templates/test/javascript.spec.js
index 4ec7c48b1e..bbc1cce246 100644
--- a/packages/string-templates/test/javascript.spec.js
+++ b/packages/string-templates/test/javascript.spec.js
@@ -140,4 +140,4 @@ describe("check JS helpers", () => {
     const output = processJS(`return helpers.toInt(4.3)`)
     expect(output).toBe(4)
   })
-})
\ No newline at end of file
+})

From 7e037099b4c88607de0830d161c572519d208410 Mon Sep 17 00:00:00 2001
From: Gerard Burns <Ghrehh@users.noreply.github.com>
Date: Mon, 20 Nov 2023 15:05:58 +0000
Subject: [PATCH 147/155] Field help text (#12383)

* wip

* client finished

* add to bbui input

* linting

* PR Feedback

* remove test code

* linting

* PR feedback

---------

Co-authored-by: Martin McKeaveney <martin@budibase.com>
---
 packages/bbui/src/Form/Checkbox.svelte        |  3 +-
 packages/bbui/src/Form/Combobox.svelte        |  3 +-
 packages/bbui/src/Form/Core/Checkbox.svelte   |  2 -
 .../bbui/src/Form/Core/CheckboxGroup.svelte   |  2 -
 packages/bbui/src/Form/Core/Combobox.svelte   |  3 -
 packages/bbui/src/Form/Core/DatePicker.svelte | 13 ----
 packages/bbui/src/Form/Core/Dropzone.svelte   |  5 --
 .../bbui/src/Form/Core/InputDropdown.svelte   | 18 +----
 .../bbui/src/Form/Core/Multiselect.svelte     |  2 -
 packages/bbui/src/Form/Core/Picker.svelte     | 12 ----
 .../bbui/src/Form/Core/PickerDropdown.svelte  |  8 +--
 packages/bbui/src/Form/Core/RadioGroup.svelte |  2 -
 .../bbui/src/Form/Core/RichTextField.svelte   | 17 +----
 packages/bbui/src/Form/Core/Select.svelte     |  2 -
 packages/bbui/src/Form/Core/Stepper.svelte    | 12 ----
 packages/bbui/src/Form/Core/TextArea.svelte   | 12 ----
 packages/bbui/src/Form/Core/TextField.svelte  | 11 ----
 packages/bbui/src/Form/DatePicker.svelte      |  3 +-
 packages/bbui/src/Form/Dropzone.svelte        |  3 +-
 packages/bbui/src/Form/EnvDropdown.svelte     |  3 +-
 packages/bbui/src/Form/Field.svelte           | 23 +++++++
 packages/bbui/src/Form/File.svelte            |  3 +-
 packages/bbui/src/Form/Input.svelte           |  3 +-
 packages/bbui/src/Form/InputDropdown.svelte   |  3 +-
 packages/bbui/src/Form/Multiselect.svelte     |  3 +-
 packages/bbui/src/Form/PickerDropdown.svelte  |  3 +-
 packages/bbui/src/Form/RadioGroup.svelte      |  3 +-
 packages/bbui/src/Form/RichTextField.svelte   |  3 +-
 packages/bbui/src/Form/Search.svelte          |  3 +-
 packages/bbui/src/Form/Select.svelte          |  3 +-
 packages/bbui/src/Form/Slider.svelte          |  3 +-
 packages/bbui/src/Form/Stepper.svelte         |  3 +-
 packages/bbui/src/Form/TextArea.svelte        |  3 +-
 packages/bbui/src/Form/Toggle.svelte          |  3 +-
 packages/client/manifest.json                 | 65 +++++++++++++++++++
 .../app/forms/AttachmentField.svelte          |  2 +
 .../components/app/forms/BooleanField.svelte  |  2 +
 .../app/forms/CodeScannerField.svelte         |  2 +
 .../components/app/forms/DateTimeField.svelte |  2 +
 .../src/components/app/forms/Field.svelte     | 43 +++++++++++-
 .../src/components/app/forms/JSONField.svelte |  2 +
 .../components/app/forms/LongFormField.svelte |  2 +
 .../app/forms/MultiFieldSelect.svelte         |  2 +
 .../components/app/forms/OptionsField.svelte  |  2 +
 .../app/forms/RelationshipField.svelte        |  2 +
 .../components/app/forms/StringField.svelte   |  3 +-
 46 files changed, 189 insertions(+), 138 deletions(-)

diff --git a/packages/bbui/src/Form/Checkbox.svelte b/packages/bbui/src/Form/Checkbox.svelte
index 6aa88f6dee..6e6cf0d8e8 100644
--- a/packages/bbui/src/Form/Checkbox.svelte
+++ b/packages/bbui/src/Form/Checkbox.svelte
@@ -10,6 +10,7 @@
   export let disabled = false
   export let error = null
   export let size = "M"
+  export let helpText = null
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -18,6 +19,6 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error}>
+<Field {helpText} {label} {labelPosition} {error}>
   <Checkbox {error} {disabled} {text} {value} {size} on:change={onChange} />
 </Field>
diff --git a/packages/bbui/src/Form/Combobox.svelte b/packages/bbui/src/Form/Combobox.svelte
index 343af559cb..44854d949e 100644
--- a/packages/bbui/src/Form/Combobox.svelte
+++ b/packages/bbui/src/Form/Combobox.svelte
@@ -11,6 +11,7 @@
   export let error = null
   export let placeholder = "Choose an option or type"
   export let options = []
+  export let helpText = null
   export let getOptionLabel = option => extractProperty(option, "label")
   export let getOptionValue = option => extractProperty(option, "value")
 
@@ -27,7 +28,7 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error}>
+<Field {helpText} {label} {labelPosition} {error}>
   <Combobox
     {error}
     {disabled}
diff --git a/packages/bbui/src/Form/Core/Checkbox.svelte b/packages/bbui/src/Form/Core/Checkbox.svelte
index e24f5669eb..21fe00e864 100644
--- a/packages/bbui/src/Form/Core/Checkbox.svelte
+++ b/packages/bbui/src/Form/Core/Checkbox.svelte
@@ -4,7 +4,6 @@
   import { createEventDispatcher } from "svelte"
 
   export let value = false
-  export let error = null
   export let id = null
   export let text = null
   export let disabled = false
@@ -22,7 +21,6 @@
 
 <label
   class="spectrum-Checkbox spectrum-Checkbox--emphasized {sizeClass}"
-  class:is-invalid={!!error}
   class:checked={value}
   class:is-indeterminate={indeterminate}
   class:readonly
diff --git a/packages/bbui/src/Form/Core/CheckboxGroup.svelte b/packages/bbui/src/Form/Core/CheckboxGroup.svelte
index 66ac55561b..d1a107fcc5 100644
--- a/packages/bbui/src/Form/Core/CheckboxGroup.svelte
+++ b/packages/bbui/src/Form/Core/CheckboxGroup.svelte
@@ -6,7 +6,6 @@
   export let direction = "vertical"
   export let value = []
   export let options = []
-  export let error = null
   export let disabled = false
   export let readonly = false
   export let getOptionLabel = option => option
@@ -34,7 +33,6 @@
       <div
         title={getOptionLabel(option)}
         class="spectrum-Checkbox spectrum-FieldGroup-item"
-        class:is-invalid={!!error}
         class:readonly
       >
         <label
diff --git a/packages/bbui/src/Form/Core/Combobox.svelte b/packages/bbui/src/Form/Core/Combobox.svelte
index b1b264a9b7..572d82ecf8 100644
--- a/packages/bbui/src/Form/Core/Combobox.svelte
+++ b/packages/bbui/src/Form/Core/Combobox.svelte
@@ -10,7 +10,6 @@
   export let placeholder = "Choose an option or type"
   export let disabled = false
   export let readonly = false
-  export let error = null
   export let options = []
   export let getOptionLabel = option => option
   export let getOptionValue = option => option
@@ -39,12 +38,10 @@
 <div
   class="spectrum-InputGroup"
   class:is-focused={open || focus}
-  class:is-invalid={!!error}
   class:is-disabled={disabled}
 >
   <div
     class="spectrum-Textfield spectrum-InputGroup-textfield"
-    class:is-invalid={!!error}
     class:is-disabled={disabled}
     class:is-focused={open || focus}
   >
diff --git a/packages/bbui/src/Form/Core/DatePicker.svelte b/packages/bbui/src/Form/Core/DatePicker.svelte
index 786aee40b6..607899683a 100644
--- a/packages/bbui/src/Form/Core/DatePicker.svelte
+++ b/packages/bbui/src/Form/Core/DatePicker.svelte
@@ -10,7 +10,6 @@
   export let id = null
   export let disabled = false
   export let readonly = false
-  export let error = null
   export let enableTime = true
   export let value = null
   export let placeholder = null
@@ -188,7 +187,6 @@
     <div
       id={flatpickrId}
       class:is-disabled={disabled || readonly}
-      class:is-invalid={!!error}
       class="flatpickr spectrum-InputGroup spectrum-Datepicker"
       class:is-focused={open}
       aria-readonly="false"
@@ -199,17 +197,7 @@
         on:click={flatpickr?.open}
         class="spectrum-Textfield spectrum-InputGroup-textfield"
         class:is-disabled={disabled}
-        class:is-invalid={!!error}
       >
-        {#if !!error}
-          <svg
-            class="spectrum-Icon spectrum-Icon--sizeM spectrum-Textfield-validationIcon"
-            focusable="false"
-            aria-hidden="true"
-          >
-            <use xlink:href="#spectrum-icon-18-Alert" />
-          </svg>
-        {/if}
         <input
           {disabled}
           {readonly}
@@ -227,7 +215,6 @@
         class="spectrum-Picker spectrum-Picker--sizeM spectrum-InputGroup-button"
         tabindex="-1"
         class:is-disabled={disabled}
-        class:is-invalid={!!error}
         on:click={flatpickr?.open}
       >
         <svg
diff --git a/packages/bbui/src/Form/Core/Dropzone.svelte b/packages/bbui/src/Form/Core/Dropzone.svelte
index bfa0285250..dc89476db2 100644
--- a/packages/bbui/src/Form/Core/Dropzone.svelte
+++ b/packages/bbui/src/Form/Core/Dropzone.svelte
@@ -22,7 +22,6 @@
   export let handleFileTooLarge = null
   export let handleTooManyFiles = null
   export let gallery = true
-  export let error = null
   export let fileTags = []
   export let maximum = null
   export let extensions = "*"
@@ -222,7 +221,6 @@
   {#if showDropzone}
     <div
       class="spectrum-Dropzone"
-      class:is-invalid={!!error}
       class:disabled
       role="region"
       tabindex="0"
@@ -351,9 +349,6 @@
   .spectrum-Dropzone {
     user-select: none;
   }
-  .spectrum-Dropzone.is-invalid {
-    border-color: var(--spectrum-global-color-red-400);
-  }
   input[type="file"] {
     display: none;
   }
diff --git a/packages/bbui/src/Form/Core/InputDropdown.svelte b/packages/bbui/src/Form/Core/InputDropdown.svelte
index 944d79655d..153501ce5a 100644
--- a/packages/bbui/src/Form/Core/InputDropdown.svelte
+++ b/packages/bbui/src/Form/Core/InputDropdown.svelte
@@ -14,7 +14,6 @@
   export let disabled = false
   export let readonly = false
   export let updateOnChange = true
-  export let error = null
   export let options = []
   export let getOptionLabel = option => extractProperty(option, "label")
   export let getOptionValue = option => extractProperty(option, "value")
@@ -111,27 +110,12 @@
   }
 </script>
 
-<div
-  class="spectrum-InputGroup"
-  class:is-invalid={!!error}
-  class:is-disabled={disabled}
->
+<div class="spectrum-InputGroup" class:is-disabled={disabled}>
   <div
     class="spectrum-Textfield spectrum-InputGroup-textfield"
-    class:is-invalid={!!error}
     class:is-disabled={disabled}
     class:is-focused={focus}
   >
-    {#if error}
-      <svg
-        class="spectrum-Icon spectrum-Icon--sizeM spectrum-Textfield-validationIcon"
-        focusable="false"
-        aria-hidden="true"
-      >
-        <use xlink:href="#spectrum-icon-18-Alert" />
-      </svg>
-    {/if}
-
     <input
       {id}
       on:click
diff --git a/packages/bbui/src/Form/Core/Multiselect.svelte b/packages/bbui/src/Form/Core/Multiselect.svelte
index 8d72dd0652..d5d6515d2d 100644
--- a/packages/bbui/src/Form/Core/Multiselect.svelte
+++ b/packages/bbui/src/Form/Core/Multiselect.svelte
@@ -6,7 +6,6 @@
   export let id = null
   export let placeholder = null
   export let disabled = false
-  export let error = null
   export let options = []
   export let getOptionLabel = option => option
   export let getOptionValue = option => option
@@ -84,7 +83,6 @@
 <Picker
   on:loadMore
   {id}
-  {error}
   {disabled}
   {readonly}
   {fieldText}
diff --git a/packages/bbui/src/Form/Core/Picker.svelte b/packages/bbui/src/Form/Core/Picker.svelte
index aa06d5f748..6361b345d1 100644
--- a/packages/bbui/src/Form/Core/Picker.svelte
+++ b/packages/bbui/src/Form/Core/Picker.svelte
@@ -14,7 +14,6 @@
 
   export let id = null
   export let disabled = false
-  export let error = null
   export let fieldText = ""
   export let fieldIcon = ""
   export let fieldColour = ""
@@ -113,7 +112,6 @@
   class="spectrum-Picker spectrum-Picker--sizeM"
   class:spectrum-Picker--quiet={quiet}
   {disabled}
-  class:is-invalid={!!error}
   class:is-open={open}
   aria-haspopup="listbox"
   on:click={onClick}
@@ -142,16 +140,6 @@
   >
     {fieldText}
   </span>
-  {#if error}
-    <svg
-      class="spectrum-Icon spectrum-Icon--sizeM spectrum-Picker-validationIcon"
-      focusable="false"
-      aria-hidden="true"
-      aria-label="Folder"
-    >
-      <use xlink:href="#spectrum-icon-18-Alert" />
-    </svg>
-  {/if}
   <svg
     class="spectrum-Icon spectrum-UIIcon-ChevronDown100 spectrum-Picker-menuIcon"
     focusable="false"
diff --git a/packages/bbui/src/Form/Core/PickerDropdown.svelte b/packages/bbui/src/Form/Core/PickerDropdown.svelte
index b0a884738f..e22a30a1d3 100644
--- a/packages/bbui/src/Form/Core/PickerDropdown.svelte
+++ b/packages/bbui/src/Form/Core/PickerDropdown.svelte
@@ -16,7 +16,6 @@
   export let id = null
   export let placeholder = "Choose an option or type"
   export let disabled = false
-  export let error = null
   export let secondaryOptions = []
   export let primaryOptions = []
   export let secondaryFieldText = ""
@@ -105,14 +104,9 @@
   }
 </script>
 
-<div
-  class="spectrum-InputGroup"
-  class:is-invalid={!!error}
-  class:is-disabled={disabled}
->
+<div class="spectrum-InputGroup" class:is-disabled={disabled}>
   <div
     class="spectrum-Textfield spectrum-InputGroup-textfield"
-    class:is-invalid={!!error}
     class:is-disabled={disabled}
     class:is-focused={focus}
     class:is-full-width={!secondaryOptions.length}
diff --git a/packages/bbui/src/Form/Core/RadioGroup.svelte b/packages/bbui/src/Form/Core/RadioGroup.svelte
index 648c46b5fc..04007b58b1 100644
--- a/packages/bbui/src/Form/Core/RadioGroup.svelte
+++ b/packages/bbui/src/Form/Core/RadioGroup.svelte
@@ -6,7 +6,6 @@
   export let direction = "vertical"
   export let value = null
   export let options = []
-  export let error = null
   export let disabled = false
   export let readonly = false
   export let getOptionLabel = option => option
@@ -40,7 +39,6 @@
       <div
         title={getOptionTitle(option)}
         class="spectrum-Radio spectrum-FieldGroup-item spectrum-Radio--emphasized"
-        class:is-invalid={!!error}
         class:readonly
       >
         <input
diff --git a/packages/bbui/src/Form/Core/RichTextField.svelte b/packages/bbui/src/Form/Core/RichTextField.svelte
index 3e0b0caf4d..08032059e7 100644
--- a/packages/bbui/src/Form/Core/RichTextField.svelte
+++ b/packages/bbui/src/Form/Core/RichTextField.svelte
@@ -5,14 +5,13 @@
   export let placeholder = null
   export let disabled = false
   export let readonly = false
-  export let error = null
   export let height = null
   export let id = null
   export let fullScreenOffset = null
   export let easyMDEOptions = null
 </script>
 
-<div class:error>
+<div>
   <MarkdownEditor
     {value}
     {placeholder}
@@ -27,18 +26,4 @@
 </div>
 
 <style>
-  .error :global(.EasyMDEContainer .editor-toolbar) {
-    border-top-color: var(--spectrum-semantic-negative-color-default);
-    border-left-color: var(--spectrum-semantic-negative-color-default);
-    border-right-color: var(--spectrum-semantic-negative-color-default);
-  }
-  .error :global(.EasyMDEContainer .CodeMirror) {
-    border-bottom-color: var(--spectrum-semantic-negative-color-default);
-    border-left-color: var(--spectrum-semantic-negative-color-default);
-    border-right-color: var(--spectrum-semantic-negative-color-default);
-  }
-  .error :global(.EasyMDEContainer .editor-preview-side) {
-    border-bottom-color: var(--spectrum-semantic-negative-color-default);
-    border-right-color: var(--spectrum-semantic-negative-color-default);
-  }
 </style>
diff --git a/packages/bbui/src/Form/Core/Select.svelte b/packages/bbui/src/Form/Core/Select.svelte
index a85d4094c5..e928ae689d 100644
--- a/packages/bbui/src/Form/Core/Select.svelte
+++ b/packages/bbui/src/Form/Core/Select.svelte
@@ -6,7 +6,6 @@
   export let id = null
   export let placeholder = "Choose an option"
   export let disabled = false
-  export let error = null
   export let options = []
   export let getOptionLabel = option => option
   export let getOptionValue = option => option
@@ -71,7 +70,6 @@
   on:loadMore
   {quiet}
   {id}
-  {error}
   {disabled}
   {readonly}
   {fieldText}
diff --git a/packages/bbui/src/Form/Core/Stepper.svelte b/packages/bbui/src/Form/Core/Stepper.svelte
index 895dc2d3de..816add249f 100644
--- a/packages/bbui/src/Form/Core/Stepper.svelte
+++ b/packages/bbui/src/Form/Core/Stepper.svelte
@@ -7,7 +7,6 @@
   export let value = null
   export let placeholder = null
   export let disabled = false
-  export let error = null
   export let id = null
   export let readonly = false
   export let updateOnChange = true
@@ -98,20 +97,9 @@
 <div
   class="spectrum-Stepper"
   class:spectrum-Stepper--quiet={quiet}
-  class:is-invalid={!!error}
   class:is-disabled={disabled}
   class:is-focused={focus}
 >
-  {#if error}
-    <svg
-      class="spectrum-Icon spectrum-Icon--sizeM spectrum-Textfield-validationIcon"
-      focusable="false"
-      aria-hidden="true"
-    >
-      <use xlink:href="#spectrum-icon-18-Alert" />
-    </svg>
-  {/if}
-
   <div class="spectrum-Textfield spectrum-Stepper-textfield">
     <input
       {disabled}
diff --git a/packages/bbui/src/Form/Core/TextArea.svelte b/packages/bbui/src/Form/Core/TextArea.svelte
index be7eed466d..9b49f54f58 100644
--- a/packages/bbui/src/Form/Core/TextArea.svelte
+++ b/packages/bbui/src/Form/Core/TextArea.svelte
@@ -6,7 +6,6 @@
   export let placeholder = null
   export let disabled = false
   export let readonly = false
-  export let error = null
   export let id = null
   export let height = null
   export let minHeight = null
@@ -41,20 +40,9 @@
 <div
   style={`${heightString}${minHeightString}`}
   class="spectrum-Textfield spectrum-Textfield--multiline"
-  class:is-invalid={!!error}
   class:is-disabled={disabled}
   class:is-focused={focus}
 >
-  {#if error}
-    <svg
-      class="spectrum-Icon spectrum-Icon--sizeM
-      spectrum-Textfield-validationIcon"
-      focusable="false"
-      aria-hidden="true"
-    >
-      <use xlink:href="#spectrum-icon-18-Alert" />
-    </svg>
-  {/if}
   <!-- prettier-ignore -->
   <textarea
     bind:this={textarea}
diff --git a/packages/bbui/src/Form/Core/TextField.svelte b/packages/bbui/src/Form/Core/TextField.svelte
index 7afd8f86c3..b10997c904 100644
--- a/packages/bbui/src/Form/Core/TextField.svelte
+++ b/packages/bbui/src/Form/Core/TextField.svelte
@@ -6,7 +6,6 @@
   export let placeholder = null
   export let type = "text"
   export let disabled = false
-  export let error = null
   export let id = null
   export let readonly = false
   export let updateOnChange = true
@@ -78,19 +77,9 @@
 <div
   class="spectrum-Textfield"
   class:spectrum-Textfield--quiet={quiet}
-  class:is-invalid={!!error}
   class:is-disabled={disabled}
   class:is-focused={focus}
 >
-  {#if error}
-    <svg
-      class="spectrum-Icon spectrum-Icon--sizeM spectrum-Textfield-validationIcon"
-      focusable="false"
-      aria-hidden="true"
-    >
-      <use xlink:href="#spectrum-icon-18-Alert" />
-    </svg>
-  {/if}
   <input
     bind:this={field}
     {disabled}
diff --git a/packages/bbui/src/Form/DatePicker.svelte b/packages/bbui/src/Form/DatePicker.svelte
index f17871a576..fe861f59ec 100644
--- a/packages/bbui/src/Form/DatePicker.svelte
+++ b/packages/bbui/src/Form/DatePicker.svelte
@@ -16,6 +16,7 @@
   export let appendTo = undefined
   export let ignoreTimezones = false
   export let range = false
+  export let helpText = null
   const dispatch = createEventDispatcher()
 
   const onChange = e => {
@@ -30,7 +31,7 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error}>
+<Field {helpText} {label} {labelPosition} {error}>
   <DatePicker
     {error}
     {disabled}
diff --git a/packages/bbui/src/Form/Dropzone.svelte b/packages/bbui/src/Form/Dropzone.svelte
index 2a6fa1c57b..5a624e523c 100644
--- a/packages/bbui/src/Form/Dropzone.svelte
+++ b/packages/bbui/src/Form/Dropzone.svelte
@@ -17,6 +17,7 @@
   export let fileTags = []
   export let maximum = undefined
   export let compact = false
+  export let helpText = null
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -25,7 +26,7 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error}>
+<Field {helpText} {label} {labelPosition} {error}>
   <CoreDropzone
     {error}
     {disabled}
diff --git a/packages/bbui/src/Form/EnvDropdown.svelte b/packages/bbui/src/Form/EnvDropdown.svelte
index b5db23a870..4045d9e6df 100644
--- a/packages/bbui/src/Form/EnvDropdown.svelte
+++ b/packages/bbui/src/Form/EnvDropdown.svelte
@@ -16,6 +16,7 @@
   export let autofocus
   export let variables
   export let showModal
+  export let helpText = null
   export let environmentVariablesEnabled
   export let handleUpgradePanel
   const dispatch = createEventDispatcher()
@@ -25,7 +26,7 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error}>
+<Field {helpText} {label} {labelPosition} {error}>
   <EnvDropdown
     {updateOnChange}
     {error}
diff --git a/packages/bbui/src/Form/Field.svelte b/packages/bbui/src/Form/Field.svelte
index 5033c28f7d..0c031b0235 100644
--- a/packages/bbui/src/Form/Field.svelte
+++ b/packages/bbui/src/Form/Field.svelte
@@ -1,11 +1,13 @@
 <script>
   import "@spectrum-css/fieldlabel/dist/index-vars.css"
   import FieldLabel from "./FieldLabel.svelte"
+  import Icon from "../Icon/Icon.svelte"
 
   export let id = null
   export let label = null
   export let labelPosition = "above"
   export let error = null
+  export let helpText = null
   export let tooltip = ""
 </script>
 
@@ -17,6 +19,10 @@
     <slot />
     {#if error}
       <div class="error">{error}</div>
+    {:else if helpText}
+      <div class="helpText">
+        <Icon name="HelpOutline" /> <span>{helpText}</span>
+      </div>
     {/if}
   </div>
 </div>
@@ -39,4 +45,21 @@
     font-size: var(--spectrum-global-dimension-font-size-75);
     margin-top: var(--spectrum-global-dimension-size-75);
   }
+
+  .helpText {
+    display: flex;
+    margin-top: var(--spectrum-global-dimension-size-75);
+    align-items: center;
+  }
+
+  .helpText :global(svg) {
+    width: 14px;
+    color: var(--grey-5);
+    margin-right: 6px;
+  }
+
+  .helpText span {
+    color: var(--grey-7);
+    font-size: var(--spectrum-global-dimension-font-size-75);
+  }
 </style>
diff --git a/packages/bbui/src/Form/File.svelte b/packages/bbui/src/Form/File.svelte
index 03cacea814..4c00cafc35 100644
--- a/packages/bbui/src/Form/File.svelte
+++ b/packages/bbui/src/Form/File.svelte
@@ -14,6 +14,7 @@
   export let title = null
   export let value = null
   export let tooltip = null
+  export let helpText = null
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -22,7 +23,7 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error} {tooltip}>
+<Field {helpText} {label} {labelPosition} {error} {tooltip}>
   <CoreFile
     {error}
     {disabled}
diff --git a/packages/bbui/src/Form/Input.svelte b/packages/bbui/src/Form/Input.svelte
index d3cb13e731..47962720af 100644
--- a/packages/bbui/src/Form/Input.svelte
+++ b/packages/bbui/src/Form/Input.svelte
@@ -15,6 +15,7 @@
   export let quiet = false
   export let autofocus
   export let autocomplete
+  export let helpText = null
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -23,7 +24,7 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error}>
+<Field {helpText} {label} {labelPosition} {error}>
   <TextField
     {updateOnChange}
     {error}
diff --git a/packages/bbui/src/Form/InputDropdown.svelte b/packages/bbui/src/Form/InputDropdown.svelte
index 12db30a43a..93766495b8 100644
--- a/packages/bbui/src/Form/InputDropdown.svelte
+++ b/packages/bbui/src/Form/InputDropdown.svelte
@@ -15,6 +15,7 @@
   export let updateOnChange = true
   export let quiet = false
   export let autofocus
+  export let helpText = null
   export let options = []
 
   const dispatch = createEventDispatcher()
@@ -29,7 +30,7 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error}>
+<Field {helpText} {label} {labelPosition} {error}>
   <InputDropdown
     {updateOnChange}
     {error}
diff --git a/packages/bbui/src/Form/Multiselect.svelte b/packages/bbui/src/Form/Multiselect.svelte
index 3ae56693c7..b0246c8530 100644
--- a/packages/bbui/src/Form/Multiselect.svelte
+++ b/packages/bbui/src/Form/Multiselect.svelte
@@ -18,6 +18,7 @@
   export let autocomplete = false
   export let searchTerm = null
   export let customPopoverHeight
+  export let helpText = null
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -26,7 +27,7 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error}>
+<Field {helpText} {label} {labelPosition} {error}>
   <Multiselect
     {error}
     {disabled}
diff --git a/packages/bbui/src/Form/PickerDropdown.svelte b/packages/bbui/src/Form/PickerDropdown.svelte
index 156ec9bc7e..bf490ff2d7 100644
--- a/packages/bbui/src/Form/PickerDropdown.svelte
+++ b/packages/bbui/src/Form/PickerDropdown.svelte
@@ -26,6 +26,7 @@
   export let secondaryOptions = []
   export let searchTerm
   export let showClearIcon = true
+  export let helpText = null
 
   let primaryLabel
   let secondaryLabel
@@ -93,7 +94,7 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error}>
+<Field {helpText} {label} {labelPosition} {error}>
   <PickerDropdown
     {searchTerm}
     {autocomplete}
diff --git a/packages/bbui/src/Form/RadioGroup.svelte b/packages/bbui/src/Form/RadioGroup.svelte
index 843a3657b4..f261973e1f 100644
--- a/packages/bbui/src/Form/RadioGroup.svelte
+++ b/packages/bbui/src/Form/RadioGroup.svelte
@@ -13,6 +13,7 @@
   export let getOptionLabel = option => extractProperty(option, "label")
   export let getOptionValue = option => extractProperty(option, "value")
   export let getOptionTitle = option => extractProperty(option, "label")
+  export let helpText = null
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -27,7 +28,7 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error}>
+<Field {helpText} {label} {labelPosition} {error}>
   <RadioGroup
     {error}
     {disabled}
diff --git a/packages/bbui/src/Form/RichTextField.svelte b/packages/bbui/src/Form/RichTextField.svelte
index 275242df49..527710e010 100644
--- a/packages/bbui/src/Form/RichTextField.svelte
+++ b/packages/bbui/src/Form/RichTextField.svelte
@@ -13,6 +13,7 @@
   export let id = null
   export let fullScreenOffset = null
   export let easyMDEOptions = null
+  export let helpText = null
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -21,7 +22,7 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error}>
+<Field {helpText} {label} {labelPosition} {error}>
   <RichTextField
     {error}
     {disabled}
diff --git a/packages/bbui/src/Form/Search.svelte b/packages/bbui/src/Form/Search.svelte
index 74ffeeb22a..2fbda275eb 100644
--- a/packages/bbui/src/Form/Search.svelte
+++ b/packages/bbui/src/Form/Search.svelte
@@ -11,6 +11,7 @@
   export let updateOnChange = true
   export let quiet = false
   export let inputRef
+  export let helpText = null
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -19,7 +20,7 @@
   }
 </script>
 
-<Field {label} {labelPosition}>
+<Field {helpText} {label} {labelPosition}>
   <Search
     {updateOnChange}
     {disabled}
diff --git a/packages/bbui/src/Form/Select.svelte b/packages/bbui/src/Form/Select.svelte
index a9214320f9..f9d9532ba5 100644
--- a/packages/bbui/src/Form/Select.svelte
+++ b/packages/bbui/src/Form/Select.svelte
@@ -26,6 +26,7 @@
   export let align
   export let footer = null
   export let tag = null
+  export let helpText = null
   const dispatch = createEventDispatcher()
   const onChange = e => {
     value = e.detail
@@ -40,7 +41,7 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error} {tooltip}>
+<Field {helpText} {label} {labelPosition} {error} {tooltip}>
   <Select
     {quiet}
     {error}
diff --git a/packages/bbui/src/Form/Slider.svelte b/packages/bbui/src/Form/Slider.svelte
index 34b2251b35..192f4bc9b1 100644
--- a/packages/bbui/src/Form/Slider.svelte
+++ b/packages/bbui/src/Form/Slider.svelte
@@ -11,6 +11,7 @@
   export let step = 1
   export let disabled = false
   export let error = null
+  export let helpText = null
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -19,6 +20,6 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error}>
+<Field {helpText} {label} {labelPosition} {error}>
   <Slider {disabled} {value} {min} {max} {step} on:change={onChange} />
 </Field>
diff --git a/packages/bbui/src/Form/Stepper.svelte b/packages/bbui/src/Form/Stepper.svelte
index 9eff2e368e..509aa2178a 100644
--- a/packages/bbui/src/Form/Stepper.svelte
+++ b/packages/bbui/src/Form/Stepper.svelte
@@ -15,6 +15,7 @@
   export let min = null
   export let max = null
   export let step = 1
+  export let helpText = null
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -23,7 +24,7 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error}>
+<Field {helpText} {label} {labelPosition} {error}>
   <Stepper
     {updateOnChange}
     {error}
diff --git a/packages/bbui/src/Form/TextArea.svelte b/packages/bbui/src/Form/TextArea.svelte
index c9451a40cd..15e91d72c1 100644
--- a/packages/bbui/src/Form/TextArea.svelte
+++ b/packages/bbui/src/Form/TextArea.svelte
@@ -12,6 +12,7 @@
   export let getCaretPosition = null
   export let height = null
   export let minHeight = null
+  export let helpText = null
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -20,7 +21,7 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error}>
+<Field {helpText} {label} {labelPosition} {error}>
   <TextArea
     bind:getCaretPosition
     {error}
diff --git a/packages/bbui/src/Form/Toggle.svelte b/packages/bbui/src/Form/Toggle.svelte
index 8cc6506651..d452aa8f80 100644
--- a/packages/bbui/src/Form/Toggle.svelte
+++ b/packages/bbui/src/Form/Toggle.svelte
@@ -9,6 +9,7 @@
   export let text = null
   export let disabled = false
   export let error = null
+  export let helpText = null
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -17,6 +18,6 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error}>
+<Field {helpText} {label} {labelPosition} {error}>
   <Switch {error} {disabled} {text} {value} on:change={onChange} />
 </Field>
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 5979d177f4..202dc8bd2f 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -2731,6 +2731,11 @@
         "key": "defaultValue",
         "supportsConditions": false
       },
+      {
+        "type": "text",
+        "label": "Help text",
+        "key": "helpText"
+      },
       {
         "type": "event",
         "label": "On change",
@@ -2862,6 +2867,11 @@
         "key": "defaultValue",
         "supportsConditions": false
       },
+      {
+        "type": "text",
+        "label": "Help text",
+        "key": "helpText"
+      },
       {
         "type": "event",
         "label": "On change",
@@ -2959,6 +2969,11 @@
         "key": "defaultValue",
         "supportsConditions": false
       },
+      {
+        "type": "text",
+        "label": "Help text",
+        "key": "helpText"
+      },
       {
         "type": "event",
         "label": "On change",
@@ -3175,6 +3190,11 @@
         "key": "defaultValue",
         "supportsConditions": false
       },
+      {
+        "type": "text",
+        "label": "Help text",
+        "key": "helpText"
+      },
       {
         "type": "boolean",
         "label": "Autocomplete",
@@ -3335,6 +3355,11 @@
         "key": "defaultValue",
         "supportsConditions": false
       },
+      {
+        "type": "text",
+        "label": "Help text",
+        "key": "helpText"
+      },
       {
         "type": "event",
         "label": "On change",
@@ -3559,6 +3584,11 @@
         "key": "defaultValue",
         "supportsConditions": false
       },
+      {
+        "type": "text",
+        "label": "Help text",
+        "key": "helpText"
+      },
       {
         "type": "event",
         "label": "On change",
@@ -3657,6 +3687,11 @@
         "key": "defaultValue",
         "supportsConditions": false
       },
+      {
+        "type": "text",
+        "label": "Help text",
+        "key": "helpText"
+      },
       {
         "type": "event",
         "label": "On change",
@@ -3799,6 +3834,11 @@
         "key": "defaultValue",
         "supportsConditions": false
       },
+      {
+        "type": "text",
+        "label": "Help text",
+        "key": "helpText"
+      },
       {
         "type": "event",
         "label": "On change",
@@ -3894,6 +3934,11 @@
         "key": "defaultValue",
         "supportsConditions": false
       },
+      {
+        "type": "text",
+        "label": "Help text",
+        "key": "helpText"
+      },
       {
         "type": "boolean",
         "label": "Disabled",
@@ -4144,6 +4189,11 @@
         "label": "Label",
         "key": "label"
       },
+      {
+        "type": "text",
+        "label": "Help text",
+        "key": "helpText"
+      },
       {
         "type": "text",
         "label": "Extensions",
@@ -4247,6 +4297,11 @@
         "key": "defaultValue",
         "supportsConditions": false
       },
+      {
+        "type": "text",
+        "label": "Help text",
+        "key": "helpText"
+      },
       {
         "type": "event",
         "label": "On change",
@@ -4355,6 +4410,11 @@
         "key": "defaultValue",
         "supportsConditions": false
       },
+      {
+        "type": "text",
+        "label": "Help text",
+        "key": "helpText"
+      },
       {
         "type": "event",
         "label": "On change",
@@ -6395,6 +6455,11 @@
         "label": "Default value",
         "key": "defaultValue"
       },
+      {
+        "type": "text",
+        "label": "Help text",
+        "key": "helpText"
+      },
       {
         "type": "event",
         "label": "On change",
diff --git a/packages/client/src/components/app/forms/AttachmentField.svelte b/packages/client/src/components/app/forms/AttachmentField.svelte
index cc0f7aaac6..b61f88daeb 100644
--- a/packages/client/src/components/app/forms/AttachmentField.svelte
+++ b/packages/client/src/components/app/forms/AttachmentField.svelte
@@ -13,6 +13,7 @@
   export let onChange
   export let maximum = undefined
   export let span
+  export let helpText = null
 
   let fieldState
   let fieldApi
@@ -76,6 +77,7 @@
   {readonly}
   {validation}
   {span}
+  {helpText}
   type="attachment"
   bind:fieldState
   bind:fieldApi
diff --git a/packages/client/src/components/app/forms/BooleanField.svelte b/packages/client/src/components/app/forms/BooleanField.svelte
index 1f59ddcfa6..28cef56aec 100644
--- a/packages/client/src/components/app/forms/BooleanField.svelte
+++ b/packages/client/src/components/app/forms/BooleanField.svelte
@@ -11,6 +11,7 @@
   export let validation
   export let defaultValue
   export let onChange
+  export let helpText = null
 
   let fieldState
   let fieldApi
@@ -42,6 +43,7 @@
   {disabled}
   {readonly}
   {validation}
+  {helpText}
   defaultValue={isTruthy(defaultValue)}
   type="boolean"
   bind:fieldState
diff --git a/packages/client/src/components/app/forms/CodeScannerField.svelte b/packages/client/src/components/app/forms/CodeScannerField.svelte
index 8e5237a27c..be590106c2 100644
--- a/packages/client/src/components/app/forms/CodeScannerField.svelte
+++ b/packages/client/src/components/app/forms/CodeScannerField.svelte
@@ -16,6 +16,7 @@
   export let beepFrequency
   export let customFrequency
   export let preferredCamera
+  export let helpText = null
 
   let fieldState
   let fieldApi
@@ -38,6 +39,7 @@
   {validation}
   {defaultValue}
   {type}
+  {helpText}
   bind:fieldState
   bind:fieldApi
 >
diff --git a/packages/client/src/components/app/forms/DateTimeField.svelte b/packages/client/src/components/app/forms/DateTimeField.svelte
index 7e2598be81..8ab8e65a15 100644
--- a/packages/client/src/components/app/forms/DateTimeField.svelte
+++ b/packages/client/src/components/app/forms/DateTimeField.svelte
@@ -15,6 +15,7 @@
   export let defaultValue
   export let onChange
   export let span
+  export let helpText = null
 
   let fieldState
   let fieldApi
@@ -35,6 +36,7 @@
   {validation}
   {defaultValue}
   {span}
+  {helpText}
   type="datetime"
   bind:fieldState
   bind:fieldApi
diff --git a/packages/client/src/components/app/forms/Field.svelte b/packages/client/src/components/app/forms/Field.svelte
index 8878b25989..22420b7430 100644
--- a/packages/client/src/components/app/forms/Field.svelte
+++ b/packages/client/src/components/app/forms/Field.svelte
@@ -1,6 +1,7 @@
 <script>
   import Placeholder from "../Placeholder.svelte"
   import { getContext, onDestroy } from "svelte"
+  import { Icon } from "@budibase/bbui"
 
   export let label
   export let field
@@ -13,6 +14,7 @@
   export let readonly = false
   export let validation
   export let span = 6
+  export let helpText = null
 
   // Get contexts
   const formContext = getContext("form")
@@ -97,7 +99,14 @@
     {:else}
       <slot />
       {#if fieldState.error}
-        <div class="error">{fieldState.error}</div>
+        <div class="error">
+          <Icon name="Alert" />
+          <span>{fieldState.error}</span>
+        </div>
+      {:else if helpText}
+        <div class="helpText">
+          <Icon name="HelpOutline" /> <span>{helpText}</span>
+        </div>
       {/if}
     {/if}
   </div>
@@ -127,13 +136,45 @@
     position: relative;
     width: 100%;
   }
+
   .error {
+    display: flex;
+    margin-top: var(--spectrum-global-dimension-size-75);
+    align-items: center;
+  }
+
+  .error :global(svg) {
+    width: 14px;
+    color: var(
+      --spectrum-semantic-negative-color-default,
+      var(--spectrum-global-color-red-500)
+    );
+    margin-right: 4px;
+  }
+
+  .error span {
     color: var(
       --spectrum-semantic-negative-color-default,
       var(--spectrum-global-color-red-500)
     );
     font-size: var(--spectrum-global-dimension-font-size-75);
+  }
+
+  .helpText {
+    display: flex;
     margin-top: var(--spectrum-global-dimension-size-75);
+    align-items: center;
+  }
+
+  .helpText :global(svg) {
+    width: 14px;
+    color: var(--grey-7);
+    margin-right: 6px;
+  }
+
+  .helpText span {
+    color: var(--grey-5);
+    font-size: var(--spectrum-global-dimension-font-size-75);
   }
   .spectrum-FieldLabel--right,
   .spectrum-FieldLabel--left {
diff --git a/packages/client/src/components/app/forms/JSONField.svelte b/packages/client/src/components/app/forms/JSONField.svelte
index cf96f54a23..244a76fcfe 100644
--- a/packages/client/src/components/app/forms/JSONField.svelte
+++ b/packages/client/src/components/app/forms/JSONField.svelte
@@ -10,6 +10,7 @@
   export let readonly = false
   export let defaultValue = ""
   export let onChange
+  export let helpText = null
 
   const component = getContext("component")
   const validation = [
@@ -52,6 +53,7 @@
   {readonly}
   {validation}
   {defaultValue}
+  {helpText}
   type="json"
   bind:fieldState
   bind:fieldApi
diff --git a/packages/client/src/components/app/forms/LongFormField.svelte b/packages/client/src/components/app/forms/LongFormField.svelte
index a9087a0a9c..9b13fca43e 100644
--- a/packages/client/src/components/app/forms/LongFormField.svelte
+++ b/packages/client/src/components/app/forms/LongFormField.svelte
@@ -13,6 +13,7 @@
   export let defaultValue = ""
   export let format = "auto"
   export let onChange
+  export let helpText = null
 
   let fieldState
   let fieldApi
@@ -62,6 +63,7 @@
   {readonly}
   {validation}
   {defaultValue}
+  {helpText}
   type="longform"
   bind:fieldState
   bind:fieldApi
diff --git a/packages/client/src/components/app/forms/MultiFieldSelect.svelte b/packages/client/src/components/app/forms/MultiFieldSelect.svelte
index 519bef4659..5504ce8f38 100644
--- a/packages/client/src/components/app/forms/MultiFieldSelect.svelte
+++ b/packages/client/src/components/app/forms/MultiFieldSelect.svelte
@@ -19,6 +19,7 @@
   export let optionsType = "select"
   export let direction = "vertical"
   export let span
+  export let helpText = null
 
   let fieldState
   let fieldApi
@@ -60,6 +61,7 @@
   {readonly}
   {validation}
   {span}
+  {helpText}
   defaultValue={expandedDefaultValue}
   type="array"
   bind:fieldState
diff --git a/packages/client/src/components/app/forms/OptionsField.svelte b/packages/client/src/components/app/forms/OptionsField.svelte
index f8080419a3..d82a8127af 100644
--- a/packages/client/src/components/app/forms/OptionsField.svelte
+++ b/packages/client/src/components/app/forms/OptionsField.svelte
@@ -20,6 +20,7 @@
   export let onChange
   export let sort = true
   export let span
+  export let helpText = null
 
   let fieldState
   let fieldApi
@@ -51,6 +52,7 @@
   {validation}
   {defaultValue}
   {span}
+  {helpText}
   type="options"
   bind:fieldState
   bind:fieldApi
diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte
index 99906fe831..41a071686e 100644
--- a/packages/client/src/components/app/forms/RelationshipField.svelte
+++ b/packages/client/src/components/app/forms/RelationshipField.svelte
@@ -20,6 +20,7 @@
   export let datasourceType = "table"
   export let primaryDisplay
   export let span
+  export let helpText = null
 
   let fieldState
   let fieldApi
@@ -192,6 +193,7 @@
   defaultValue={expandedDefaultValue}
   {type}
   {span}
+  {helpText}
   bind:fieldState
   bind:fieldApi
   bind:fieldSchema
diff --git a/packages/client/src/components/app/forms/StringField.svelte b/packages/client/src/components/app/forms/StringField.svelte
index a8dec6649e..c86a98aeed 100644
--- a/packages/client/src/components/app/forms/StringField.svelte
+++ b/packages/client/src/components/app/forms/StringField.svelte
@@ -13,6 +13,7 @@
   export let align
   export let onChange
   export let span
+  export let helpText = null
 
   let fieldState
   let fieldApi
@@ -33,6 +34,7 @@
   {validation}
   {defaultValue}
   {span}
+  {helpText}
   type={type === "number" ? "number" : "string"}
   bind:fieldState
   bind:fieldApi
@@ -44,7 +46,6 @@
       on:change={handleChange}
       disabled={fieldState.disabled}
       readonly={fieldState.readonly}
-      error={fieldState.error}
       id={fieldState.fieldId}
       {placeholder}
       {type}

From 8fdf378a5721cba253b4019c1ecd59ddb6a6ad13 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 20 Nov 2023 15:34:16 +0000
Subject: [PATCH 148/155] Fix an issue with account portal redis connection, if
 no Redis port is supplied in URL we should assume the default port number
 6379.

---
 packages/backend-core/src/redis/utils.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/backend-core/src/redis/utils.ts b/packages/backend-core/src/redis/utils.ts
index 5187fe13f8..4d8b1bb9a4 100644
--- a/packages/backend-core/src/redis/utils.ts
+++ b/packages/backend-core/src/redis/utils.ts
@@ -75,10 +75,12 @@ export function getRedisConnectionDetails() {
   }
   const [host, port] = url.split(":")
 
+  const portNumber = parseInt(port)
   return {
     host,
     password,
-    port: parseInt(port),
+    // assume default port for redis if invalid found
+    port: isNaN(portNumber) ? 6379 : portNumber,
   }
 }
 

From 1ca8edc285c0ee02bc665f452c7631800faa284b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 20 Nov 2023 15:59:17 +0000
Subject: [PATCH 149/155] Fixing a documentation issue, OpenAPI definition said
 it was an unpublish rather than publish endpoint.

---
 packages/server/src/api/routes/public/applications.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/public/applications.ts b/packages/server/src/api/routes/public/applications.ts
index 5410eb7dcf..3de3b80e26 100644
--- a/packages/server/src/api/routes/public/applications.ts
+++ b/packages/server/src/api/routes/public/applications.ts
@@ -117,7 +117,7 @@ write.push(
  * /applications/{appId}/publish:
  *   post:
  *     operationId: appPublish
- *     summary: Unpublish an application
+ *     summary: Publish an application
  *     tags:
  *       - applications
  *     parameters:

From db20645ae4e976a74f72fdcb96ceed96f0c4f0a1 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Mon, 20 Nov 2023 16:32:57 +0000
Subject: [PATCH 150/155] Bump version to 2.13.12

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 64d6b745e5..9b22d286b5 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.13.11",
+  "version": "2.13.12",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From b4cb97963c60a857c58e195ef47d43d668102af6 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 21 Nov 2023 10:40:25 +0000
Subject: [PATCH 151/155] Move from an allow list to a block list of file
 extensions.

---
 .../src/api/controllers/static/index.ts       |  7 +-
 packages/shared-core/src/constants.ts         | 92 ++++++++++---------
 2 files changed, 56 insertions(+), 43 deletions(-)

diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index 544bde1fbb..4c5415a6c6 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -1,4 +1,4 @@
-import { ValidFileExtensions } from "@budibase/shared-core"
+import { InvalidFileExtensions } from "@budibase/shared-core"
 
 require("svelte/register")
 
@@ -86,7 +86,10 @@ export const uploadFile = async function (
         )
       }
 
-      if (!env.SELF_HOSTED && !ValidFileExtensions.includes(extension)) {
+      if (
+        !env.SELF_HOSTED &&
+        InvalidFileExtensions.includes(extension.toLowerCase())
+      ) {
         throw new BadRequestError(
           `File "${file.name}" has an invalid extension: "${extension}"`
         )
diff --git a/packages/shared-core/src/constants.ts b/packages/shared-core/src/constants.ts
index e7c6feb20a..5b50d0eb3b 100644
--- a/packages/shared-core/src/constants.ts
+++ b/packages/shared-core/src/constants.ts
@@ -96,45 +96,55 @@ export enum BuilderSocketEvent {
 export const SocketSessionTTL = 60
 export const ValidQueryNameRegex = /^[^()]*$/
 export const ValidColumnNameRegex = /^[_a-zA-Z0-9\s]*$/g
-export const ValidFileExtensions = [
-  "avif",
-  "css",
-  "csv",
-  "docx",
-  "drawio",
-  "editorconfig",
-  "edl",
-  "enc",
-  "export",
-  "geojson",
-  "gif",
-  "htm",
-  "html",
-  "ics",
-  "iqy",
-  "jfif",
-  "jpeg",
-  "jpg",
-  "json",
-  "log",
-  "md",
-  "mid",
-  "odt",
-  "pdf",
-  "png",
-  "ris",
-  "rtf",
-  "svg",
-  "tex",
-  "toml",
-  "twig",
-  "txt",
-  "url",
-  "wav",
-  "webp",
-  "xls",
-  "xlsx",
-  "xml",
-  "yaml",
-  "yml",
+
+export const InvalidFileExtensions = [
+  "action",
+  "apk",
+  "app",
+  "bat",
+  "bin",
+  "cab",
+  "cmd",
+  "com",
+  "command",
+  "cpl",
+  "csh",
+  "ex_",
+  "exe",
+  "gadget",
+  "inf1",
+  "ins",
+  "inx",
+  "ipa",
+  "isu",
+  "job",
+  "jse",
+  "ksh",
+  "lnk",
+  "msc",
+  "msi",
+  "msp",
+  "mst",
+  "osx",
+  "out",
+  "paf",
+  "pif",
+  "prg",
+  "ps1",
+  "reg",
+  "rgs",
+  "run",
+  "scr",
+  "sct",
+  "shb",
+  "shs",
+  "u3p",
+  "vb",
+  "vbe",
+  "vbs",
+  "vbscript",
+  "workflow",
+  "ws",
+  "wsf",
+  "wsh",
 ]

From 79dcc468b899a9caf922f74ef17316315ea6592f Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 21 Nov 2023 10:42:44 +0000
Subject: [PATCH 152/155] Add a test for uppercase malicious extensions.

---
 .../server/src/api/routes/tests/attachment.spec.ts    | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/packages/server/src/api/routes/tests/attachment.spec.ts b/packages/server/src/api/routes/tests/attachment.spec.ts
index 14d2e845f6..e230b0688a 100644
--- a/packages/server/src/api/routes/tests/attachment.spec.ts
+++ b/packages/server/src/api/routes/tests/attachment.spec.ts
@@ -35,6 +35,17 @@ describe("/api/applications/:appId/sync", () => {
       })
     })
 
+    it("should reject an upload with a malicious uppercase file extension", async () => {
+      await config.withEnv({ SELF_HOSTED: undefined }, async () => {
+        let resp = (await config.api.attachment.process(
+          "OHNO.EXE",
+          Buffer.from([0]),
+          { expectStatus: 400 }
+        )) as unknown as APIError
+        expect(resp.message).toContain("invalid extension")
+      })
+    })
+
     it("should reject an upload with no file", async () => {
       let resp = (await config.api.attachment.process(
         undefined as any,

From 0df315c4780a24a1dc98864a5649bcf0e3d28c58 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 21 Nov 2023 10:52:50 +0000
Subject: [PATCH 153/155] There has been quite a few redlock errors the last
 while - the message does not provide much information about what is going
 wrong - trying to rectify this to see if this is an error that could be
 impacting performance.

---
 packages/backend-core/src/redis/redlockImpl.ts | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts
index 7fe61a409e..266f1fe989 100644
--- a/packages/backend-core/src/redis/redlockImpl.ts
+++ b/packages/backend-core/src/redis/redlockImpl.ts
@@ -3,6 +3,7 @@ import { getLockClient } from "./init"
 import { LockOptions, LockType } from "@budibase/types"
 import * as context from "../context"
 import env from "../environment"
+import { logWarn } from "../logging"
 
 async function getClient(
   type: LockType,
@@ -116,7 +117,7 @@ export async function doWithLock<T>(
     const result = await task()
     return { executed: true, result }
   } catch (e: any) {
-    console.warn("lock error")
+    logWarn(`lock type: ${opts.type} error`, e)
     // lock limit exceeded
     if (e.name === "LockError") {
       if (opts.type === LockType.TRY_ONCE) {
@@ -124,11 +125,9 @@ export async function doWithLock<T>(
         // due to retry count (0) exceeded
         return { executed: false }
       } else {
-        console.error(e)
         throw e
       }
     } else {
-      console.error(e)
       throw e
     }
   } finally {

From 71b8b300bf4af9a91a0ab4558dace7462b3134a9 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 21 Nov 2023 11:26:25 +0000
Subject: [PATCH 154/155] Add archive file formats to blocklist.

---
 packages/shared-core/src/constants.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/shared-core/src/constants.ts b/packages/shared-core/src/constants.ts
index 5b50d0eb3b..3838cb7636 100644
--- a/packages/shared-core/src/constants.ts
+++ b/packages/shared-core/src/constants.ts
@@ -98,6 +98,7 @@ export const ValidQueryNameRegex = /^[^()]*$/
 export const ValidColumnNameRegex = /^[_a-zA-Z0-9\s]*$/g
 
 export const InvalidFileExtensions = [
+  "7z",
   "action",
   "apk",
   "app",
@@ -138,6 +139,7 @@ export const InvalidFileExtensions = [
   "sct",
   "shb",
   "shs",
+  "tar",
   "u3p",
   "vb",
   "vbe",
@@ -147,4 +149,5 @@ export const InvalidFileExtensions = [
   "ws",
   "wsf",
   "wsh",
+  "zip",
 ]

From 303ae3c58b0f8e646b7ac40da547ec1a03db3552 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 21 Nov 2023 11:41:32 +0000
Subject: [PATCH 155/155] Block some code extensions that could be used
 maliciously.

---
 packages/shared-core/src/constants.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/shared-core/src/constants.ts b/packages/shared-core/src/constants.ts
index 3838cb7636..0787b8bed1 100644
--- a/packages/shared-core/src/constants.ts
+++ b/packages/shared-core/src/constants.ts
@@ -119,6 +119,7 @@ export const InvalidFileExtensions = [
   "ipa",
   "isu",
   "job",
+  "js",
   "jse",
   "ksh",
   "lnk",
@@ -129,6 +130,7 @@ export const InvalidFileExtensions = [
   "osx",
   "out",
   "paf",
+  "php",
   "pif",
   "prg",
   "ps1",
@@ -145,6 +147,7 @@ export const InvalidFileExtensions = [
   "vbe",
   "vbs",
   "vbscript",
+  "wasm",
   "workflow",
   "ws",
   "wsf",