From b7ec577efd3ec7f97164ebf1f5ca520075d5ed5d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Dec 2023 16:57:41 +0100
Subject: [PATCH 01/58] Add redirect

---
 packages/builder/src/api.js | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/packages/builder/src/api.js b/packages/builder/src/api.js
index 37894d9bbc..e93dae65f8 100644
--- a/packages/builder/src/api.js
+++ b/packages/builder/src/api.js
@@ -45,4 +45,9 @@ export const API = createAPIClient({
       }
     }
   },
+  onMigrationDetected: appId => {
+    window.location = `/builder/updating/${appId}?returnUrl=${encodeURI(
+      window.location
+    )}`
+  },
 })

From 9067d38dce059718577f7fee214f7a31495e14c4 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Dec 2023 18:38:29 +0100
Subject: [PATCH 02/58] Create updating page

---
 packages/builder/src/api.js                        | 10 +++++++---
 .../src/pages/builder/updating/[appId].svelte      | 14 ++++++++++++++
 2 files changed, 21 insertions(+), 3 deletions(-)
 create mode 100644 packages/builder/src/pages/builder/updating/[appId].svelte

diff --git a/packages/builder/src/api.js b/packages/builder/src/api.js
index e93dae65f8..2fa48273f0 100644
--- a/packages/builder/src/api.js
+++ b/packages/builder/src/api.js
@@ -46,8 +46,12 @@ export const API = createAPIClient({
     }
   },
   onMigrationDetected: appId => {
-    window.location = `/builder/updating/${appId}?returnUrl=${encodeURI(
-      window.location
-    )}`
+    const updatingUrl = `/builder/updating/${appId}`
+
+    if (window.location.pathname === updatingUrl) {
+      return
+    }
+
+    window.location = `${updatingUrl}?returnUrl=${encodeURI(window.location)}`
   },
 })
diff --git a/packages/builder/src/pages/builder/updating/[appId].svelte b/packages/builder/src/pages/builder/updating/[appId].svelte
new file mode 100644
index 0000000000..69a2d58eed
--- /dev/null
+++ b/packages/builder/src/pages/builder/updating/[appId].svelte
@@ -0,0 +1,14 @@
+<script>
+  import Spinner from "components/common/Spinner.svelte"
+</script>
+
+<div class="loading">
+  <Spinner size="20" />
+</div>
+
+<style>
+  .loading {
+    display: flex;
+    justify-content: center;
+  }
+</style>

From a2cb32225d35a02179ba41d2ab191485b88a0678 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Dec 2023 18:38:44 +0100
Subject: [PATCH 03/58] Checks

---
 .../src/pages/builder/updating/[appId].svelte   | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/packages/builder/src/pages/builder/updating/[appId].svelte b/packages/builder/src/pages/builder/updating/[appId].svelte
index 69a2d58eed..25092dd1a0 100644
--- a/packages/builder/src/pages/builder/updating/[appId].svelte
+++ b/packages/builder/src/pages/builder/updating/[appId].svelte
@@ -1,5 +1,22 @@
 <script>
   import Spinner from "components/common/Spinner.svelte"
+  import { API } from "api"
+
+  setInterval(async () => {
+    const response = await API.get({ url: "/api/migrations/status" })
+    if (!response.migrated) {
+      return
+    }
+
+    const urlParams = new URLSearchParams(window.location.search)
+    const returnUrl = urlParams.get("returnUrl")
+
+    window.location = returnUrl
+  }, 1000)
+
+  setTimeout(() => {
+    alert("Something went wrong 💀")
+  }, 60000)
 </script>
 
 <div class="loading">

From 0b8d7904dfba652c68751f709643b80f399866a1 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Dec 2023 12:44:32 +0100
Subject: [PATCH 04/58] Encode url properly

---
 packages/builder/src/api.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/api.js b/packages/builder/src/api.js
index 2fa48273f0..4817ac2716 100644
--- a/packages/builder/src/api.js
+++ b/packages/builder/src/api.js
@@ -52,6 +52,8 @@ export const API = createAPIClient({
       return
     }
 
-    window.location = `${updatingUrl}?returnUrl=${encodeURI(window.location)}`
+    window.location = `${updatingUrl}?returnUrl=${encodeURIComponent(
+      window.location
+    )}`
   },
 })

From 342a1b487bc17d4aea7ddfd6d0d53037d523960c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Dec 2023 16:15:11 +0100
Subject: [PATCH 05/58] Update url

---
 packages/builder/src/api.js                                     | 2 +-
 .../[appId].svelte => app/[application]/updating.svelte}        | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename packages/builder/src/pages/builder/{updating/[appId].svelte => app/[application]/updating.svelte} (100%)

diff --git a/packages/builder/src/api.js b/packages/builder/src/api.js
index 4817ac2716..a854137e2b 100644
--- a/packages/builder/src/api.js
+++ b/packages/builder/src/api.js
@@ -46,7 +46,7 @@ export const API = createAPIClient({
     }
   },
   onMigrationDetected: appId => {
-    const updatingUrl = `/builder/updating/${appId}`
+    const updatingUrl = `/builder/app/${appId}/updating`
 
     if (window.location.pathname === updatingUrl) {
       return
diff --git a/packages/builder/src/pages/builder/updating/[appId].svelte b/packages/builder/src/pages/builder/app/[application]/updating.svelte
similarity index 100%
rename from packages/builder/src/pages/builder/updating/[appId].svelte
rename to packages/builder/src/pages/builder/app/[application]/updating.svelte

From 3a2a2a242cb906cf714864b95fdf0101bd808f15 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Dec 2023 12:52:51 +0100
Subject: [PATCH 06/58] Change interval for timeout

---
 .../builder/app/[application]/updating.svelte | 26 ++++++++++++-------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/updating.svelte b/packages/builder/src/pages/builder/app/[application]/updating.svelte
index 25092dd1a0..22d6371c18 100644
--- a/packages/builder/src/pages/builder/app/[application]/updating.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/updating.svelte
@@ -2,19 +2,27 @@
   import Spinner from "components/common/Spinner.svelte"
   import { API } from "api"
 
-  setInterval(async () => {
-    const response = await API.get({ url: "/api/migrations/status" })
-    if (!response.migrated) {
-      return
-    }
+  let timeout
+  async function checkMigrationsFinished() {
+    timeout = setTimeout(async () => {
+      const response = await API.get({ url: "/api/migrations/status" })
+      if (!response.migrated) {
+        checkMigrationsFinished()
+        return
+      }
 
-    const urlParams = new URLSearchParams(window.location.search)
-    const returnUrl = urlParams.get("returnUrl")
+      const urlParams = new URLSearchParams(window.location.search)
+      const returnUrl = urlParams.get("returnUrl")
 
-    window.location = returnUrl
-  }, 1000)
+      window.location = returnUrl
+    }, 1000)
+  }
+
+  checkMigrationsFinished()
 
   setTimeout(() => {
+    clearTimeout(timeout)
+    // TODO
     alert("Something went wrong 💀")
   }, 60000)
 </script>

From 5f6210e9a06ae5c9b3e94f487314a22228bef841 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Dec 2023 16:01:49 +0100
Subject: [PATCH 07/58] Use routify

---
 .../src/pages/builder/app/[application]/updating.svelte       | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/updating.svelte b/packages/builder/src/pages/builder/app/[application]/updating.svelte
index 22d6371c18..a89a3a7165 100644
--- a/packages/builder/src/pages/builder/app/[application]/updating.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/updating.svelte
@@ -1,5 +1,7 @@
 <script>
   import Spinner from "components/common/Spinner.svelte"
+  import { goto } from "@roxi/routify"
+
   import { API } from "api"
 
   let timeout
@@ -14,7 +16,7 @@
       const urlParams = new URLSearchParams(window.location.search)
       const returnUrl = urlParams.get("returnUrl")
 
-      window.location = returnUrl
+      $goto(returnUrl)
     }, 1000)
   }
 

From bfd52b6c50427a49a731cfaea70fd456afd692f6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 15 Dec 2023 13:27:16 +0100
Subject: [PATCH 08/58] Use routify

---
 packages/builder/src/api.js                   |  8 ++---
 .../builder/src/pages/builder/_layout.svelte  |  5 ++-
 packages/builder/src/stores/portal/index.js   |  1 +
 .../builder/src/stores/portal/navigation.js   | 35 +++++++++++++++++++
 4 files changed, 44 insertions(+), 5 deletions(-)
 create mode 100644 packages/builder/src/stores/portal/navigation.js

diff --git a/packages/builder/src/api.js b/packages/builder/src/api.js
index a854137e2b..1cc4e2b3da 100644
--- a/packages/builder/src/api.js
+++ b/packages/builder/src/api.js
@@ -5,7 +5,7 @@ import {
 } from "@budibase/frontend-core"
 import { store } from "./builderStore"
 import { get } from "svelte/store"
-import { auth } from "./stores/portal"
+import { auth, navigation } from "./stores/portal"
 
 export const API = createAPIClient({
   attachHeaders: headers => {
@@ -52,8 +52,8 @@ export const API = createAPIClient({
       return
     }
 
-    window.location = `${updatingUrl}?returnUrl=${encodeURIComponent(
-      window.location
-    )}`
+    navigation.actions.goTo(
+      `${updatingUrl}?returnUrl=${encodeURIComponent(window.location)}`
+    )
   },
 })
diff --git a/packages/builder/src/pages/builder/_layout.svelte b/packages/builder/src/pages/builder/_layout.svelte
index b216958045..fb9fdd68a4 100644
--- a/packages/builder/src/pages/builder/_layout.svelte
+++ b/packages/builder/src/pages/builder/_layout.svelte
@@ -1,10 +1,11 @@
 <script>
   import { isActive, redirect, params } from "@roxi/routify"
-  import { admin, auth, licensing } from "stores/portal"
+  import { admin, auth, licensing, navigation } from "stores/portal"
   import { onMount } from "svelte"
   import { CookieUtils, Constants } from "@budibase/frontend-core"
   import { API } from "api"
   import Branding from "./Branding.svelte"
+  import { goto } from "@roxi/routify"
 
   let loaded = false
 
@@ -17,6 +18,8 @@
 
   $: useAccountPortal = cloud && !$admin.disableAccountPortal
 
+  navigation.actions.init($goto)
+
   const validateTenantId = async () => {
     const host = window.location.host
     if (host.includes("localhost:") || !baseUrl) {
diff --git a/packages/builder/src/stores/portal/index.js b/packages/builder/src/stores/portal/index.js
index e70df5c3ee..7f1b9e10f0 100644
--- a/packages/builder/src/stores/portal/index.js
+++ b/packages/builder/src/stores/portal/index.js
@@ -16,5 +16,6 @@ export { environment } from "./environment"
 export { menu } from "./menu"
 export { auditLogs } from "./auditLogs"
 export { features } from "./features"
+export { navigation } from "./navigation"
 
 export const sideBarCollapsed = writable(false)
diff --git a/packages/builder/src/stores/portal/navigation.js b/packages/builder/src/stores/portal/navigation.js
new file mode 100644
index 0000000000..78a2851315
--- /dev/null
+++ b/packages/builder/src/stores/portal/navigation.js
@@ -0,0 +1,35 @@
+import { writable } from "svelte/store"
+
+export function createNavigationStore() {
+  const { subscribe } = writable([])
+
+  let initialisated = false
+  let _goTo
+
+  const init = goToFunc => {
+    initialisated = true
+    if (typeof goToFunc !== "function") {
+      throw new Error('A valid "goToFunc" must be provided')
+    }
+
+    _goTo = goToFunc
+  }
+
+  const goTo = (...params) => {
+    if (!initialisated) {
+      throw new Error("You need to call navigation.init first")
+    }
+
+    _goTo(...params)
+  }
+
+  return {
+    subscribe,
+    actions: {
+      init,
+      goTo,
+    },
+  }
+}
+
+export const navigation = createNavigationStore()

From 8515221cdfc7794563ed2788267e08a8c65e53c3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 15 Dec 2023 13:37:22 +0100
Subject: [PATCH 09/58] Use store

---
 packages/builder/src/api.js                   |  2 +-
 .../builder/src/stores/portal/navigation.js   | 30 ++++++++-----------
 2 files changed, 13 insertions(+), 19 deletions(-)

diff --git a/packages/builder/src/api.js b/packages/builder/src/api.js
index 1cc4e2b3da..9130c60dea 100644
--- a/packages/builder/src/api.js
+++ b/packages/builder/src/api.js
@@ -52,7 +52,7 @@ export const API = createAPIClient({
       return
     }
 
-    navigation.actions.goTo(
+    get(navigation).goto(
       `${updatingUrl}?returnUrl=${encodeURIComponent(window.location)}`
     )
   },
diff --git a/packages/builder/src/stores/portal/navigation.js b/packages/builder/src/stores/portal/navigation.js
index 78a2851315..a6d0f0a98e 100644
--- a/packages/builder/src/stores/portal/navigation.js
+++ b/packages/builder/src/stores/portal/navigation.js
@@ -1,33 +1,27 @@
 import { writable } from "svelte/store"
 
 export function createNavigationStore() {
-  const { subscribe } = writable([])
+  const store = writable({
+    initialisated: false,
+    goto: undefined,
+  })
+  const { set, subscribe, get } = store
 
-  let initialisated = false
-  let _goTo
-
-  const init = goToFunc => {
-    initialisated = true
-    if (typeof goToFunc !== "function") {
-      throw new Error('A valid "goToFunc" must be provided')
+  const init = gotoFunc => {
+    if (typeof gotoFunc !== "function") {
+      throw new Error('A valid "gotoFunc" must be provided')
     }
 
-    _goTo = goToFunc
-  }
-
-  const goTo = (...params) => {
-    if (!initialisated) {
-      throw new Error("You need to call navigation.init first")
-    }
-
-    _goTo(...params)
+    set({
+      initialisated: true,
+      goto: gotoFunc,
+    })
   }
 
   return {
     subscribe,
     actions: {
       init,
-      goTo,
     },
   }
 }

From f8ce1ee3654756e1a5851eb854b5902ea254dca5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 15 Dec 2023 13:44:52 +0100
Subject: [PATCH 10/58] Use redirect instead of goto

---
 packages/builder/src/pages/builder/_layout.svelte | 3 +--
 packages/builder/src/stores/portal/navigation.js  | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/pages/builder/_layout.svelte b/packages/builder/src/pages/builder/_layout.svelte
index fb9fdd68a4..62d3951fb5 100644
--- a/packages/builder/src/pages/builder/_layout.svelte
+++ b/packages/builder/src/pages/builder/_layout.svelte
@@ -5,7 +5,6 @@
   import { CookieUtils, Constants } from "@budibase/frontend-core"
   import { API } from "api"
   import Branding from "./Branding.svelte"
-  import { goto } from "@roxi/routify"
 
   let loaded = false
 
@@ -18,7 +17,7 @@
 
   $: useAccountPortal = cloud && !$admin.disableAccountPortal
 
-  navigation.actions.init($goto)
+  navigation.actions.init($redirect)
 
   const validateTenantId = async () => {
     const host = window.location.host
diff --git a/packages/builder/src/stores/portal/navigation.js b/packages/builder/src/stores/portal/navigation.js
index a6d0f0a98e..3fde06f7aa 100644
--- a/packages/builder/src/stores/portal/navigation.js
+++ b/packages/builder/src/stores/portal/navigation.js
@@ -5,7 +5,7 @@ export function createNavigationStore() {
     initialisated: false,
     goto: undefined,
   })
-  const { set, subscribe, get } = store
+  const { set, subscribe } = store
 
   const init = gotoFunc => {
     if (typeof gotoFunc !== "function") {

From bce1b86059e84ff26b0eee99068476b775530de2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 15 Dec 2023 13:52:15 +0100
Subject: [PATCH 11/58] Use redirect instead of goto

---
 .../src/pages/builder/app/[application]/updating.svelte       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/updating.svelte b/packages/builder/src/pages/builder/app/[application]/updating.svelte
index a89a3a7165..4da91ed074 100644
--- a/packages/builder/src/pages/builder/app/[application]/updating.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/updating.svelte
@@ -1,6 +1,6 @@
 <script>
   import Spinner from "components/common/Spinner.svelte"
-  import { goto } from "@roxi/routify"
+  import { redirect } from "@roxi/routify"
 
   import { API } from "api"
 
@@ -16,7 +16,7 @@
       const urlParams = new URLSearchParams(window.location.search)
       const returnUrl = urlParams.get("returnUrl")
 
-      $goto(returnUrl)
+      $redirect(returnUrl)
     }, 1000)
   }
 

From 13f80ad7350082b5cac30eefab69578880c0dbad Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 15 Dec 2023 14:15:39 +0100
Subject: [PATCH 12/58] Tidy up

---
 .../builder/app/[application]/updating.svelte | 20 +++++++++++--------
 packages/frontend-core/src/api/index.js       |  2 ++
 packages/frontend-core/src/api/migrations.js  | 10 ++++++++++
 3 files changed, 24 insertions(+), 8 deletions(-)
 create mode 100644 packages/frontend-core/src/api/migrations.js

diff --git a/packages/builder/src/pages/builder/app/[application]/updating.svelte b/packages/builder/src/pages/builder/app/[application]/updating.svelte
index 4da91ed074..dca01e3495 100644
--- a/packages/builder/src/pages/builder/app/[application]/updating.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/updating.svelte
@@ -4,13 +4,18 @@
 
   import { API } from "api"
 
-  let timeout
+  const timeoutMs = 60000 // 1 minute
+  const loadTime = Date.now()
+
   async function checkMigrationsFinished() {
-    timeout = setTimeout(async () => {
-      const response = await API.get({ url: "/api/migrations/status" })
+    setTimeout(async () => {
+      const response = await API.getMigrationStatus()
       if (!response.migrated) {
-        checkMigrationsFinished()
-        return
+        if (loadTime + timeoutMs > Date.now()) {
+          return checkMigrationsFinished()
+        }
+
+        return migrationTimeout()
       }
 
       const urlParams = new URLSearchParams(window.location.search)
@@ -22,11 +27,10 @@
 
   checkMigrationsFinished()
 
-  setTimeout(() => {
-    clearTimeout(timeout)
+  function migrationTimeout() {
     // TODO
     alert("Something went wrong 💀")
-  }, 60000)
+  }
 </script>
 
 <div class="loading">
diff --git a/packages/frontend-core/src/api/index.js b/packages/frontend-core/src/api/index.js
index d4b4f3636e..066ab16f6e 100644
--- a/packages/frontend-core/src/api/index.js
+++ b/packages/frontend-core/src/api/index.js
@@ -33,6 +33,7 @@ import { buildEnvironmentVariableEndpoints } from "./environmentVariables"
 import { buildEventEndpoints } from "./events"
 import { buildAuditLogsEndpoints } from "./auditLogs"
 import { buildLogsEndpoints } from "./logs"
+import { buildMigrationEndpoints } from "./migrations"
 
 /**
  * Random identifier to uniquely identify a session in a tab. This is
@@ -298,6 +299,7 @@ export const createAPIClient = config => {
     ...buildEventEndpoints(API),
     ...buildAuditLogsEndpoints(API),
     ...buildLogsEndpoints(API),
+    ...buildMigrationEndpoints(API),
     viewV2: buildViewV2Endpoints(API),
   }
 }
diff --git a/packages/frontend-core/src/api/migrations.js b/packages/frontend-core/src/api/migrations.js
new file mode 100644
index 0000000000..2da70d6fcb
--- /dev/null
+++ b/packages/frontend-core/src/api/migrations.js
@@ -0,0 +1,10 @@
+export const buildMigrationEndpoints = API => ({
+  /**
+   * Gets the info about the current app migration
+   */
+  getMigrationStatus: async () => {
+    return await API.get({
+      url: "/api/migrations/status",
+    })
+  },
+})

From c80504ca93d9afd54bc91a78d2bb2575aceb7711 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 18 Dec 2023 10:59:33 +0100
Subject: [PATCH 13/58] Use ?params

---
 packages/builder/src/api.js                                | 2 +-
 .../src/pages/builder/app/[application]/updating.svelte    | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/packages/builder/src/api.js b/packages/builder/src/api.js
index 9130c60dea..febf6603bb 100644
--- a/packages/builder/src/api.js
+++ b/packages/builder/src/api.js
@@ -53,7 +53,7 @@ export const API = createAPIClient({
     }
 
     get(navigation).goto(
-      `${updatingUrl}?returnUrl=${encodeURIComponent(window.location)}`
+      `${updatingUrl}?returnUrl=${encodeURIComponent(window.location.pathname)}`
     )
   },
 })
diff --git a/packages/builder/src/pages/builder/app/[application]/updating.svelte b/packages/builder/src/pages/builder/app/[application]/updating.svelte
index dca01e3495..0ea9de2ee3 100644
--- a/packages/builder/src/pages/builder/app/[application]/updating.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/updating.svelte
@@ -1,6 +1,6 @@
 <script>
   import Spinner from "components/common/Spinner.svelte"
-  import { redirect } from "@roxi/routify"
+  import { redirect, params } from "@roxi/routify"
 
   import { API } from "api"
 
@@ -18,9 +18,8 @@
         return migrationTimeout()
       }
 
-      const urlParams = new URLSearchParams(window.location.search)
-      const returnUrl = urlParams.get("returnUrl")
-
+      // For some reason routify params is not stripping the ? properly, so we need to check both with and without ?
+      const returnUrl = $params.returnUrl || $params["?returnUrl"]
       $redirect(returnUrl)
     }, 1000)
   }

From e4f70d12e9b9792dd4add2a4eaadbeb198c32c8c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 4 Jan 2024 12:14:05 +0100
Subject: [PATCH 14/58] Change url

---
 packages/builder/src/api.js                                     | 2 +-
 .../updating.svelte => updating/[application].svelte}           | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename packages/builder/src/pages/builder/app/{[application]/updating.svelte => updating/[application].svelte} (100%)

diff --git a/packages/builder/src/api.js b/packages/builder/src/api.js
index febf6603bb..ac878bf82f 100644
--- a/packages/builder/src/api.js
+++ b/packages/builder/src/api.js
@@ -46,7 +46,7 @@ export const API = createAPIClient({
     }
   },
   onMigrationDetected: appId => {
-    const updatingUrl = `/builder/app/${appId}/updating`
+    const updatingUrl = `/builder/app/updating/${appId}`
 
     if (window.location.pathname === updatingUrl) {
       return
diff --git a/packages/builder/src/pages/builder/app/[application]/updating.svelte b/packages/builder/src/pages/builder/app/updating/[application].svelte
similarity index 100%
rename from packages/builder/src/pages/builder/app/[application]/updating.svelte
rename to packages/builder/src/pages/builder/app/updating/[application].svelte

From 401f057b0447cf88df121324e16af5cfad329ace Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 4 Jan 2024 12:23:46 +0100
Subject: [PATCH 15/58] Basic design

---
 .../builder/app/updating/[application].svelte     | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/packages/builder/src/pages/builder/app/updating/[application].svelte b/packages/builder/src/pages/builder/app/updating/[application].svelte
index 0ea9de2ee3..bbcdccf917 100644
--- a/packages/builder/src/pages/builder/app/updating/[application].svelte
+++ b/packages/builder/src/pages/builder/app/updating/[application].svelte
@@ -33,6 +33,11 @@
 </script>
 
 <div class="loading">
+  <p class="loading-message">
+    🛠️ We are updating the system. <br />Please wait, we will be back in a
+    second!
+  </p>
+
   <Spinner size="20" />
 </div>
 
@@ -40,5 +45,15 @@
   .loading {
     display: flex;
     justify-content: center;
+    align-items: center;
+    justify-content: center;
+    flex-direction: column;
+    gap: var(--spacing-xl);
+    height: 100vh;
+    margin: 0;
+  }
+  .loading-message {
+    text-align: center;
+    font-size: 18px;
   }
 </style>

From ec4324697427d58e3ab5a211813f5dc9baf6c240 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 4 Jan 2024 12:28:36 +0100
Subject: [PATCH 16/58] Error

---
 .../builder/app/updating/[application].svelte | 24 +++++++++++++------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/updating/[application].svelte b/packages/builder/src/pages/builder/app/updating/[application].svelte
index bbcdccf917..ff8a873700 100644
--- a/packages/builder/src/pages/builder/app/updating/[application].svelte
+++ b/packages/builder/src/pages/builder/app/updating/[application].svelte
@@ -7,6 +7,8 @@
   const timeoutMs = 60000 // 1 minute
   const loadTime = Date.now()
 
+  let timedOut = false
+
   async function checkMigrationsFinished() {
     setTimeout(async () => {
       const response = await API.getMigrationStatus()
@@ -27,18 +29,23 @@
   checkMigrationsFinished()
 
   function migrationTimeout() {
-    // TODO
-    alert("Something went wrong 💀")
+    timedOut = true
   }
 </script>
 
 <div class="loading">
-  <p class="loading-message">
-    🛠️ We are updating the system. <br />Please wait, we will be back in a
-    second!
-  </p>
+  {#if !timedOut}
+    <p class="loading-message">
+      🛠️ We are updating the system. <br />Please wait, we will be back in a
+      second!
+    </p>
 
-  <Spinner size="20" />
+    <Spinner size="20" />
+  {:else}
+    <p class="loading-message timeout">
+      An error occurred. Please try again later.
+    </p>
+  {/if}
 </div>
 
 <style>
@@ -56,4 +63,7 @@
     text-align: center;
     font-size: 18px;
   }
+  .timeout {
+    color: rgb(196, 46, 46);
+  }
 </style>

From 261e60d3bc087a8c7614341acfb70be8d8c677bc Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 5 Jan 2024 15:08:43 +0000
Subject: [PATCH 17/58] Enable dd-trace logging in QA to try and figure out why
 user info isn't being added to traces.

---
 packages/server/src/ddApm.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/ddApm.ts b/packages/server/src/ddApm.ts
index 6c9b8aa289..f0f3ec6055 100644
--- a/packages/server/src/ddApm.ts
+++ b/packages/server/src/ddApm.ts
@@ -3,5 +3,9 @@ import apm from "dd-trace"
 // enable APM if configured
 if (process.env.DD_APM_ENABLED) {
   console.log("Starting dd-trace")
-  apm.init()
+  apm.init({
+    // @ts-ignore for some reason dd-trace types don't include this options,
+    // even though it's spoken about in the docs.
+    debug: process.env.DD_ENV === "qa",
+  })
 }

From f73f8f443343bd26118a2d6b62439dcfe750c1e1 Mon Sep 17 00:00:00 2001
From: rg2011 <52279456+rg2011@users.noreply.github.com>
Date: Fri, 5 Jan 2024 15:29:29 +0100
Subject: [PATCH 18/58] Add support for full search path to postgres
 integration

---
 .../src/integration-test/postgres.spec.ts     | 72 +++++++++++++++++++
 packages/server/src/integrations/postgres.ts  | 11 ++-
 2 files changed, 80 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 67e4fee81c..600566c813 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -1118,4 +1118,76 @@ describe("postgres integrations", () => {
       })
     })
   })
+
+  describe("Integration compatibility with postgres search_path", () => {
+    let client: Client, pathDatasource: Datasource
+    const schema1 = "test1",
+      schema2 = "test-2"
+
+    beforeAll(async () => {
+      const dsConfig = await databaseTestProviders.postgres.getDsConfig()
+      const dbConfig = dsConfig.config!
+
+      client = new Client(dbConfig)
+      await client.connect()
+      await client.query(`CREATE SCHEMA "${schema1}";`)
+      await client.query(`CREATE SCHEMA "${schema2}";`)
+
+      const pathConfig: any = {
+        ...dsConfig,
+        config: {
+          ...dbConfig,
+          schema: `${schema1}, ${schema2}`,
+        },
+      }
+      pathDatasource = await config.api.datasource.create(pathConfig)
+    })
+
+    afterAll(async () => {
+      await client.query(`DROP SCHEMA "${schema1}" CASCADE;`)
+      await client.query(`DROP SCHEMA "${schema2}" CASCADE;`)
+      await client.end()
+    })
+
+    it("discovers tables from any schema in search path", async () => {
+      await client.query(
+        `CREATE TABLE "${schema1}".table1 (id1 SERIAL PRIMARY KEY);`
+      )
+      await client.query(
+        `CREATE TABLE "${schema2}".table2 (id2 SERIAL PRIMARY KEY);`
+      )
+      const response = await makeRequest("post", "/api/datasources/info", {
+        datasource: pathDatasource,
+      })
+      expect(response.status).toBe(200)
+      expect(response.body.tableNames).toBeDefined()
+      expect(response.body.tableNames).toEqual(
+        expect.arrayContaining(["table1", "table2"])
+      )
+    })
+
+    it("does not mix columns from different tables", async () => {
+      const repeated_table_name = "table_same_name"
+      await client.query(
+        `CREATE TABLE "${schema1}".${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);`
+      )
+      await client.query(
+        `CREATE TABLE "${schema2}".${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);`
+      )
+      const response = await makeRequest(
+        "post",
+        `/api/datasources/${pathDatasource._id}/schema`,
+        {
+          tablesFilter: [repeated_table_name],
+        }
+      )
+      expect(response.status).toBe(200)
+      expect(
+        response.body.datasource.entities[repeated_table_name].schema
+      ).toBeDefined()
+      const schema =
+        response.body.datasource.entities[repeated_table_name].schema
+      expect(Object.keys(schema).sort()).toEqual(["id", "val1"])
+    })
+  })
 })
diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts
index de3bf0e59e..e1494ece36 100644
--- a/packages/server/src/integrations/postgres.ts
+++ b/packages/server/src/integrations/postgres.ts
@@ -159,7 +159,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
   JOIN pg_index ON pg_class.oid = pg_index.indrelid AND pg_index.indisprimary
   JOIN pg_attribute ON pg_attribute.attrelid = pg_class.oid AND pg_attribute.attnum = ANY(pg_index.indkey)
   JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace
-  WHERE pg_namespace.nspname = '${this.config.schema}';
+  WHERE pg_namespace.nspname = ANY(current_schemas(false))
+  AND pg_table_is_visible(pg_class.oid);
   `
 
   ENUM_VALUES = () => `
@@ -219,8 +220,12 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
     if (!this.config.schema) {
       this.config.schema = "public"
     }
-    await this.client.query(`SET search_path TO "${this.config.schema}"`)
-    this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'`
+    const search_path = this.config.schema
+      .split(",")
+      .map(item => `"${item.trim()}"`)
+    await this.client.query(`SET search_path TO ${search_path.join(",")};`)
+    this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = ANY(current_schemas(false))
+    AND pg_table_is_visible(to_regclass(table_schema || '.' || table_name));`
     this.open = true
   }
 

From ccc91dcca502600c4f7057ed5909c801bb820e37 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Fri, 5 Jan 2024 17:22:37 +0000
Subject: [PATCH 19/58] Enable layouts in multistep form blocks

---
 .../app/blocks/MultiStepFormblock.svelte      | 41 +++++++++++++------
 1 file changed, 29 insertions(+), 12 deletions(-)

diff --git a/packages/client/src/components/app/blocks/MultiStepFormblock.svelte b/packages/client/src/components/app/blocks/MultiStepFormblock.svelte
index 0573ba15b5..b90d0d4c7b 100644
--- a/packages/client/src/components/app/blocks/MultiStepFormblock.svelte
+++ b/packages/client/src/components/app/blocks/MultiStepFormblock.svelte
@@ -14,6 +14,7 @@
 
   const { fetchDatasourceSchema } = getContext("sdk")
   const component = getContext("component")
+  const context = getContext("context")
 
   // Set current step context to force child form to use it
   const currentStep = writable(1)
@@ -157,18 +158,23 @@
             <BlockComponent type="heading" props={{ text: step.title }} />
           </BlockComponent>
           <BlockComponent type="text" props={{ text: step.desc }} order={1} />
-          <BlockComponent type="fieldgroup" order={2}>
-            {#each step.fields as field, fieldIdx (`${field.field || field.name}_${stepIdx}_${fieldIdx}`)}
-              {#if getComponentForField(field)}
-                <BlockComponent
-                  type={getComponentForField(field)}
-                  props={getPropsForField(field)}
-                  order={fieldIdx}
-                  interactive
-                  name={field.field}
-                />
-              {/if}
-            {/each}
+          <BlockComponent type="container" order={2}>
+            <div
+              class="form-block fields"
+              class:mobile={$context.device.mobile}
+            >
+              {#each step.fields as field, fieldIdx (`${field.field || field.name}_${stepIdx}_${fieldIdx}`)}
+                {#if getComponentForField(field)}
+                  <BlockComponent
+                    type={getComponentForField(field)}
+                    props={getPropsForField(field)}
+                    order={fieldIdx}
+                    interactive
+                    name={field.field}
+                  />
+                {/if}
+              {/each}
+            </div>
           </BlockComponent>
           <BlockComponent
             type="buttongroup"
@@ -185,3 +191,14 @@
     {/each}
   </BlockComponent>
 </FormBlockWrapper>
+
+<style>
+  .fields {
+    display: grid;
+    grid-template-columns: repeat(6, 1fr);
+    gap: 8px 16px;
+  }
+  .fields.mobile :global(.spectrum-Form-item) {
+    grid-column: span 6 !important;
+  }
+</style>

From 7ca748a3be474e3f90e7b4c1eb25017f8897a6eb Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 5 Jan 2024 18:10:23 +0000
Subject: [PATCH 20/58] Remove objects from setUser call. We can add them later
 if we need them.

---
 packages/backend-core/src/middleware/authenticated.ts | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts
index e8e16589de..d357dbdbdc 100644
--- a/packages/backend-core/src/middleware/authenticated.ts
+++ b/packages/backend-core/src/middleware/authenticated.ts
@@ -172,11 +172,8 @@ export default function (
         tracer.setUser({
           id: user?._id,
           tenantId: user?.tenantId,
-          admin: user?.admin,
-          builder: user?.builder,
           budibaseAccess: user?.budibaseAccess,
           status: user?.status,
-          roles: user?.roles,
         })
       }
 

From b115ead329e7da914bfceb035ade56543998a38e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 10:46:14 +0100
Subject: [PATCH 21/58] Format frontend

---
 .../builder/app/updating/[application].svelte | 40 +++++++++----------
 1 file changed, 18 insertions(+), 22 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/updating/[application].svelte b/packages/builder/src/pages/builder/app/updating/[application].svelte
index ff8a873700..6994331eff 100644
--- a/packages/builder/src/pages/builder/app/updating/[application].svelte
+++ b/packages/builder/src/pages/builder/app/updating/[application].svelte
@@ -1,10 +1,9 @@
 <script>
-  import Spinner from "components/common/Spinner.svelte"
   import { redirect, params } from "@roxi/routify"
 
   import { API } from "api"
 
-  const timeoutMs = 60000 // 1 minute
+  const timeoutMs = 180000 // 3 minutes
   const loadTime = Date.now()
 
   let timedOut = false
@@ -31,39 +30,36 @@
   function migrationTimeout() {
     timedOut = true
   }
+
+  $: text = !timedOut ? "System update" : "Something went wrong!"
+  $: subtext = !timedOut
+    ? "Please wait and we will be back in a second!"
+    : "An error occurred, please try again later"
 </script>
 
-<div class="loading">
-  {#if !timedOut}
-    <p class="loading-message">
-      🛠️ We are updating the system. <br />Please wait, we will be back in a
-      second!
-    </p>
-
-    <Spinner size="20" />
-  {:else}
-    <p class="loading-message timeout">
-      An error occurred. Please try again later.
-    </p>
-  {/if}
+<div class="loading" class:timeout={timedOut}>
+  <span class="header">{text}</span>
+  <span class="subtext">{subtext}</span>
 </div>
 
 <style>
   .loading {
     display: flex;
     justify-content: center;
-    align-items: center;
-    justify-content: center;
     flex-direction: column;
-    gap: var(--spacing-xl);
+    gap: var(--spacing-l);
     height: 100vh;
-    margin: 0;
-  }
-  .loading-message {
     text-align: center;
     font-size: 18px;
   }
-  .timeout {
+  .header {
+    font-weight: 700;
+  }
+  .timeout .header {
     color: rgb(196, 46, 46);
   }
+  .subtext {
+    font-size: 16px;
+    color: var(--grey-7);
+  }
 </style>

From 6c20799b92748813c39004c67cfdabd5b51e07fe Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 8 Jan 2024 09:49:32 +0000
Subject: [PATCH 22/58] Extra traces inside of runJS to find out where time is
 being used.

---
 packages/server/src/jsRunner.ts | 64 +++++++++++++++++++--------------
 1 file changed, 38 insertions(+), 26 deletions(-)

diff --git a/packages/server/src/jsRunner.ts b/packages/server/src/jsRunner.ts
index a9dcd506d7..f041f211fb 100644
--- a/packages/server/src/jsRunner.ts
+++ b/packages/server/src/jsRunner.ts
@@ -12,36 +12,48 @@ export function init() {
       const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
       let track: TrackerFn = f => f()
       if (perRequestLimit) {
-        const bbCtx = context.getCurrentContext()
-        if (bbCtx) {
-          if (!bbCtx.jsExecutionTracker) {
-            bbCtx.jsExecutionTracker =
-              timers.ExecutionTimeTracker.withLimit(perRequestLimit)
+        tracer.trace("runJS.setupTracker", {}, span => {
+          const bbCtx = context.getCurrentContext()
+          if (bbCtx) {
+            if (!bbCtx.jsExecutionTracker) {
+              bbCtx.jsExecutionTracker =
+                timers.ExecutionTimeTracker.withLimit(perRequestLimit)
+            }
+            span?.addTags({
+              js: {
+                limitMS: bbCtx.jsExecutionTracker.limitMs,
+                elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
+              },
+            })
+            // We call checkLimit() here to prevent paying the cost of creating
+            // a new VM context below when we don't need to.
+            bbCtx.jsExecutionTracker.checkLimit()
+            track = bbCtx.jsExecutionTracker.track.bind(
+              bbCtx.jsExecutionTracker
+            )
           }
-          span?.addTags({
-            js: {
-              limitMS: bbCtx.jsExecutionTracker.limitMs,
-              elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
-            },
-          })
-          // We call checkLimit() here to prevent paying the cost of creating
-          // a new VM context below when we don't need to.
-          bbCtx.jsExecutionTracker.checkLimit()
-          track = bbCtx.jsExecutionTracker.track.bind(bbCtx.jsExecutionTracker)
-        }
+        })
       }
 
-      ctx = {
-        ...ctx,
-        alert: undefined,
-        setInterval: undefined,
-        setTimeout: undefined,
-      }
-      vm.createContext(ctx)
+      ctx = tracer.trace("runJS.ctxClone", {}, span => {
+        return {
+          ...ctx,
+          alert: undefined,
+          setInterval: undefined,
+          setTimeout: undefined,
+        }
+      })
+
+      tracer.trace("runJS.vm.createContext", {}, span => {
+        vm.createContext(ctx)
+      })
+
       return track(() =>
-        vm.runInNewContext(js, ctx, {
-          timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
-        })
+        tracer.trace("runJS.vm.runInNewContext", {}, span =>
+          vm.runInNewContext(js, ctx, {
+            timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
+          })
+        )
       )
     })
   })

From d5850d59de4224843dd8444571a30a53186ea74a Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 8 Jan 2024 09:52:07 +0000
Subject: [PATCH 23/58] Add bool to show when execution tracker gets created.

---
 packages/server/src/jsRunner.ts | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/jsRunner.ts b/packages/server/src/jsRunner.ts
index f041f211fb..6fde3ab82a 100644
--- a/packages/server/src/jsRunner.ts
+++ b/packages/server/src/jsRunner.ts
@@ -12,10 +12,13 @@ export function init() {
       const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
       let track: TrackerFn = f => f()
       if (perRequestLimit) {
-        tracer.trace("runJS.setupTracker", {}, span => {
+        tracer.trace<any>("runJS.setupTracker", {}, span => {
           const bbCtx = context.getCurrentContext()
           if (bbCtx) {
             if (!bbCtx.jsExecutionTracker) {
+              span?.addTags({
+                createdExecutionTracker: true,
+              })
               bbCtx.jsExecutionTracker =
                 timers.ExecutionTimeTracker.withLimit(perRequestLimit)
             }

From 010ddc2c34b108a1232d29cb9cea15e638f367c0 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 8 Jan 2024 10:07:14 +0000
Subject: [PATCH 24/58] Set the CouchDB credentials in a safer way inside of
 runner.sh.

---
 hosting/couchdb/runner.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hosting/couchdb/runner.sh b/hosting/couchdb/runner.sh
index e56b8e0e7f..9f6a853ca7 100644
--- a/hosting/couchdb/runner.sh
+++ b/hosting/couchdb/runner.sh
@@ -76,6 +76,6 @@ done
 
 # CouchDB needs the `_users` and `_replicator` databases to exist before it will
 # function correctly, so we create them here.
-curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_users
-curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_replicator
+curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users
+curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator
 sleep infinity
\ No newline at end of file

From a427b322beaa5ab63bc9969f83cde54ac1bd7b3e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 11:11:46 +0100
Subject: [PATCH 25/58] Move ui to frontend-core

---
 .../builder/app/updating/[application].svelte | 64 +++----------------
 .../src/components/Updating.svelte            | 61 ++++++++++++++++++
 .../frontend-core/src/components/index.js     |  1 +
 3 files changed, 71 insertions(+), 55 deletions(-)
 create mode 100644 packages/frontend-core/src/components/Updating.svelte

diff --git a/packages/builder/src/pages/builder/app/updating/[application].svelte b/packages/builder/src/pages/builder/app/updating/[application].svelte
index 6994331eff..ec6196aaca 100644
--- a/packages/builder/src/pages/builder/app/updating/[application].svelte
+++ b/packages/builder/src/pages/builder/app/updating/[application].svelte
@@ -1,65 +1,19 @@
 <script>
+  import { Updating } from "@budibase/frontend-core"
   import { redirect, params } from "@roxi/routify"
 
   import { API } from "api"
 
-  const timeoutMs = 180000 // 3 minutes
-  const loadTime = Date.now()
-
-  let timedOut = false
-
-  async function checkMigrationsFinished() {
-    setTimeout(async () => {
-      const response = await API.getMigrationStatus()
-      if (!response.migrated) {
-        if (loadTime + timeoutMs > Date.now()) {
-          return checkMigrationsFinished()
-        }
-
-        return migrationTimeout()
-      }
-
-      // For some reason routify params is not stripping the ? properly, so we need to check both with and without ?
-      const returnUrl = $params.returnUrl || $params["?returnUrl"]
-      $redirect(returnUrl)
-    }, 1000)
+  async function isMigrationDone() {
+    const response = await API.getMigrationStatus()
+    return response.migrated
   }
 
-  checkMigrationsFinished()
-
-  function migrationTimeout() {
-    timedOut = true
+  async function onMigrationDone() {
+    // For some reason routify params is not stripping the ? properly, so we need to check both with and without ?
+    const returnUrl = $params.returnUrl || $params["?returnUrl"]
+    $redirect(returnUrl)
   }
-
-  $: text = !timedOut ? "System update" : "Something went wrong!"
-  $: subtext = !timedOut
-    ? "Please wait and we will be back in a second!"
-    : "An error occurred, please try again later"
 </script>
 
-<div class="loading" class:timeout={timedOut}>
-  <span class="header">{text}</span>
-  <span class="subtext">{subtext}</span>
-</div>
-
-<style>
-  .loading {
-    display: flex;
-    justify-content: center;
-    flex-direction: column;
-    gap: var(--spacing-l);
-    height: 100vh;
-    text-align: center;
-    font-size: 18px;
-  }
-  .header {
-    font-weight: 700;
-  }
-  .timeout .header {
-    color: rgb(196, 46, 46);
-  }
-  .subtext {
-    font-size: 16px;
-    color: var(--grey-7);
-  }
-</style>
+<Updating timeoutMs={180000} {isMigrationDone} {onMigrationDone} />
diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte
new file mode 100644
index 0000000000..b591c7e61c
--- /dev/null
+++ b/packages/frontend-core/src/components/Updating.svelte
@@ -0,0 +1,61 @@
+<script>
+  export let isMigrationDone
+  export let onMigrationDone
+  export let timeoutMs = 180000 // 3 minutes
+
+  const loadTime = Date.now()
+  let timedOut = false
+
+  async function checkMigrationsFinished() {
+    setTimeout(async () => {
+      const isMigrated = await isMigrationDone()
+      if (!isMigrated) {
+        if (loadTime + timeoutMs > Date.now()) {
+          return checkMigrationsFinished()
+        }
+
+        return migrationTimeout()
+      }
+
+      onMigrationDone()
+    }, 1000)
+  }
+
+  checkMigrationsFinished()
+
+  function migrationTimeout() {
+    timedOut = true
+  }
+
+  $: text = !timedOut ? "System update" : "Something went wrong!"
+  $: subtext = !timedOut
+    ? "Please wait and we will be back in a second!"
+    : "An error occurred, please try again later"
+</script>
+
+<div class="loading" class:timeout={timedOut}>
+  <span class="header">{text}</span>
+  <span class="subtext">{subtext}</span>
+</div>
+
+<style>
+  .loading {
+    display: flex;
+    justify-content: center;
+    flex-direction: column;
+    gap: var(--spacing-l);
+    height: 100vh;
+    text-align: center;
+    font-size: 18px;
+  }
+  .header {
+    font-weight: 700;
+  }
+  .timeout .header {
+    color: rgb(196, 46, 46);
+  }
+  .subtext {
+    font-size: 16px;
+    color: var(--grey-7);
+  }
+</style>
diff --git a/packages/frontend-core/src/components/index.js b/packages/frontend-core/src/components/index.js
index 01a7c78cb8..f724e1e4d9 100644
--- a/packages/frontend-core/src/components/index.js
+++ b/packages/frontend-core/src/components/index.js
@@ -3,4 +3,5 @@ export { default as TestimonialPage } from "./TestimonialPage.svelte"
 export { default as Testimonial } from "./Testimonial.svelte"
 export { default as UserAvatar } from "./UserAvatar.svelte"
 export { default as UserAvatars } from "./UserAvatars.svelte"
+export { default as Updating } from "./Updating.svelte"
 export { Grid } from "./grid"

From d46b22e4a6af1dccd8353d176c879b5a48643682 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 11:52:31 +0100
Subject: [PATCH 26/58] Fix initial checks

---
 packages/server/src/appMigrations/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts
index b382d8b533..0758b9f324 100644
--- a/packages/server/src/appMigrations/index.ts
+++ b/packages/server/src/appMigrations/index.ts
@@ -17,7 +17,7 @@ export const getLatestMigrationId = () =>
     .sort()
     .reverse()[0]
 
-const getTimestamp = (versionId: string) => versionId?.split("_")[0]
+const getTimestamp = (versionId: string) => versionId?.split("_")[0] || ""
 
 export async function checkMissingMigrations(
   ctx: UserCtx,

From a8019d86cca93376a5e00f1697a09e6d67e8c54a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 12:09:24 +0100
Subject: [PATCH 27/58] Move api checks

---
 .../src/pages/builder/app/updating/[application].svelte  | 9 +--------
 packages/frontend-core/src/components/Updating.svelte    | 8 +++++---
 2 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/updating/[application].svelte b/packages/builder/src/pages/builder/app/updating/[application].svelte
index ec6196aaca..ec851fa08d 100644
--- a/packages/builder/src/pages/builder/app/updating/[application].svelte
+++ b/packages/builder/src/pages/builder/app/updating/[application].svelte
@@ -2,13 +2,6 @@
   import { Updating } from "@budibase/frontend-core"
   import { redirect, params } from "@roxi/routify"
 
-  import { API } from "api"
-
-  async function isMigrationDone() {
-    const response = await API.getMigrationStatus()
-    return response.migrated
-  }
-
   async function onMigrationDone() {
     // For some reason routify params is not stripping the ? properly, so we need to check both with and without ?
     const returnUrl = $params.returnUrl || $params["?returnUrl"]
@@ -16,4 +9,4 @@
   }
 </script>
 
-<Updating timeoutMs={180000} {isMigrationDone} {onMigrationDone} />
+<Updating {onMigrationDone} />
diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte
index b591c7e61c..57433c1164 100644
--- a/packages/frontend-core/src/components/Updating.svelte
+++ b/packages/frontend-core/src/components/Updating.svelte
@@ -1,5 +1,6 @@
 <script>
-  export let isMigrationDone
+  import { API } from "api"
+
   export let onMigrationDone
   export let timeoutMs = 180000 // 3 minutes
 
@@ -8,8 +9,9 @@
 
   async function checkMigrationsFinished() {
     setTimeout(async () => {
-      const isMigrated = await isMigrationDone()
-      if (!isMigrated) {
+      const response = await API.getMigrationStatus()
+
+      if (!response.migrated) {
         if (loadTime + timeoutMs > Date.now()) {
           return checkMigrationsFinished()
         }

From e24cf34c2fba0aeaa505e1e6205125625824a7f0 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 8 Jan 2024 11:43:22 +0000
Subject: [PATCH 28/58] Updating account portal reference to latest master.g

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index b11e6b4737..a97fb83d49 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit b11e6b47370d9b77c63648b45929c86bfed6360c
+Subproject commit a97fb83d4992f28c855c1cc5186bbdf4d1eba929

From 83dce5de267a39dde5c38b29c1ff640ca198e641 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 12:48:04 +0100
Subject: [PATCH 29/58] Update error message

Co-authored-by: Sam Rose <hello@samwho.dev>
---
 packages/builder/src/stores/portal/navigation.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/stores/portal/navigation.js b/packages/builder/src/stores/portal/navigation.js
index 3fde06f7aa..7bcda68eee 100644
--- a/packages/builder/src/stores/portal/navigation.js
+++ b/packages/builder/src/stores/portal/navigation.js
@@ -9,7 +9,7 @@ export function createNavigationStore() {
 
   const init = gotoFunc => {
     if (typeof gotoFunc !== "function") {
-      throw new Error('A valid "gotoFunc" must be provided')
+      throw new Error(`gotoFunc must be a function, found a "${typeof gotoFunc}" instead`)
     }
 
     set({

From 13325b63a274701af96c6a0e6cc1f7ea18482500 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 12:51:29 +0100
Subject: [PATCH 30/58] Use seconds instead of ms

---
 packages/frontend-core/src/components/Updating.svelte | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte
index 57433c1164..fb5171a6ca 100644
--- a/packages/frontend-core/src/components/Updating.svelte
+++ b/packages/frontend-core/src/components/Updating.svelte
@@ -2,7 +2,7 @@
   import { API } from "api"
 
   export let onMigrationDone
-  export let timeoutMs = 180000 // 3 minutes
+  export let timeoutSeconds = 3 * 60 // 3 minutes
 
   const loadTime = Date.now()
   let timedOut = false
@@ -11,6 +11,7 @@
     setTimeout(async () => {
       const response = await API.getMigrationStatus()
 
+      const timeoutMs = timeoutSeconds * 1000
       if (!response.migrated) {
         if (loadTime + timeoutMs > Date.now()) {
           return checkMigrationsFinished()

From 6546f776de1f182e6364a2ec2e4d1ef7097bdbcd Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 8 Jan 2024 11:59:45 +0000
Subject: [PATCH 31/58] Updating ref after linting.

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index a97fb83d49..8ee2734e77 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit a97fb83d4992f28c855c1cc5186bbdf4d1eba929
+Subproject commit 8ee2734e77709438cbcaaabc024f677c7b24c883

From f7f51e790980f3793ec5bcc793d89b0f22be2d7f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 13:07:33 +0100
Subject: [PATCH 32/58] Change copy

---
 .../src/components/Updating.svelte            | 23 +++++++++++++------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte
index fb5171a6ca..6b8a86425e 100644
--- a/packages/frontend-core/src/components/Updating.svelte
+++ b/packages/frontend-core/src/components/Updating.svelte
@@ -29,16 +29,25 @@
   function migrationTimeout() {
     timedOut = true
   }
-
-  $: text = !timedOut ? "System update" : "Something went wrong!"
-  $: subtext = !timedOut
-    ? "Please wait and we will be back in a second!"
-    : "An error occurred, please try again later"
 </script>
 
 <div class="loading" class:timeout={timedOut}>
-  <span class="header">{text}</span>
-  <span class="subtext">{subtext}</span>
+  <span class="header">
+    {#if !timedOut}
+      System update
+    {:else}
+      Something went wrong!
+    {/if}
+  </span>
+  <span class="subtext">
+    {#if !timedOut}
+      Please wait and we will be back in a second!
+    {:else}
+      An error occurred, please try again later.
+      <br />
+      Contact support if the issue persists.
+    {/if}</span
+  >
 </div>
 
 <style>

From 72410d1f281a7f10a2db37142e68aba0f625ae51 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 13:10:40 +0100
Subject: [PATCH 33/58] Lint

---
 packages/builder/src/stores/portal/navigation.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/stores/portal/navigation.js b/packages/builder/src/stores/portal/navigation.js
index 7bcda68eee..67a06eff53 100644
--- a/packages/builder/src/stores/portal/navigation.js
+++ b/packages/builder/src/stores/portal/navigation.js
@@ -9,7 +9,9 @@ export function createNavigationStore() {
 
   const init = gotoFunc => {
     if (typeof gotoFunc !== "function") {
-      throw new Error(`gotoFunc must be a function, found a "${typeof gotoFunc}" instead`)
+      throw new Error(
+        `gotoFunc must be a function, found a "${typeof gotoFunc}" instead`
+      )
     }
 
     set({

From 82ce1f2e93e388149d8239b14034424be7c1b43d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 13:31:41 +0100
Subject: [PATCH 34/58] Add link

---
 packages/frontend-core/src/components/Updating.svelte | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte
index 6b8a86425e..7a13de3ca5 100644
--- a/packages/frontend-core/src/components/Updating.svelte
+++ b/packages/frontend-core/src/components/Updating.svelte
@@ -45,7 +45,9 @@
     {:else}
       An error occurred, please try again later.
       <br />
-      Contact support if the issue persists.
+      Contact
+      <a href="https://budibase.com/support/" target="_blank">support</a> if the
+      issue persists.
     {/if}</span
   >
 </div>
@@ -70,4 +72,9 @@
     font-size: 16px;
     color: var(--grey-7);
   }
+
+  .subtext a {
+    color: var(--grey-7);
+    font-weight: 700;
+  }
 </style>

From 43aca917d718d4b75f0280fa5f2087d79958b234 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 13:33:41 +0100
Subject: [PATCH 35/58] Fix import

---
 packages/frontend-core/src/components/Updating.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte
index 7a13de3ca5..0b80fd2391 100644
--- a/packages/frontend-core/src/components/Updating.svelte
+++ b/packages/frontend-core/src/components/Updating.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { API } from "api"
+  import { API } from "../api"
 
   export let onMigrationDone
   export let timeoutSeconds = 3 * 60 // 3 minutes

From 3dcb3dec3f5bca818cf6526212a9e41105287367 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 13:35:07 +0100
Subject: [PATCH 36/58] Fix import

---
 packages/frontend-core/src/components/Updating.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte
index 0b80fd2391..16e4b51de4 100644
--- a/packages/frontend-core/src/components/Updating.svelte
+++ b/packages/frontend-core/src/components/Updating.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { API } from "../api"
+  import * as API from "../api"
 
   export let onMigrationDone
   export let timeoutSeconds = 3 * 60 // 3 minutes

From 03a933005d7f3c4bb595b6ee6e4d2f95aca987d3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 13:42:38 +0100
Subject: [PATCH 37/58] Pass isMigrationDone

---
 .../src/pages/builder/app/updating/[application].svelte  | 9 ++++++++-
 packages/frontend-core/src/components/Updating.svelte    | 9 ++++-----
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/updating/[application].svelte b/packages/builder/src/pages/builder/app/updating/[application].svelte
index ec851fa08d..1f92861edf 100644
--- a/packages/builder/src/pages/builder/app/updating/[application].svelte
+++ b/packages/builder/src/pages/builder/app/updating/[application].svelte
@@ -2,6 +2,13 @@
   import { Updating } from "@budibase/frontend-core"
   import { redirect, params } from "@roxi/routify"
 
+  import { API } from "api"
+
+  async function isMigrationDone() {
+    const response = await API.getMigrationStatus()
+    return response.migrated
+  }
+
   async function onMigrationDone() {
     // For some reason routify params is not stripping the ? properly, so we need to check both with and without ?
     const returnUrl = $params.returnUrl || $params["?returnUrl"]
@@ -9,4 +16,4 @@
   }
 </script>
 
-<Updating {onMigrationDone} />
+<Updating {isMigrationDone} {onMigrationDone} />
diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte
index 16e4b51de4..7d4a101fee 100644
--- a/packages/frontend-core/src/components/Updating.svelte
+++ b/packages/frontend-core/src/components/Updating.svelte
@@ -1,18 +1,17 @@
 <script>
-  import * as API from "../api"
-
+  export let isMigrationDone
   export let onMigrationDone
-  export let timeoutSeconds = 3 * 60 // 3 minutes
+  export let timeoutSeconds = 10 // 3 minutes
 
   const loadTime = Date.now()
   let timedOut = false
 
   async function checkMigrationsFinished() {
     setTimeout(async () => {
-      const response = await API.getMigrationStatus()
+      const isMigrated = await isMigrationDone()
 
       const timeoutMs = timeoutSeconds * 1000
-      if (!response.migrated) {
+      if (!isMigrated) {
         if (loadTime + timeoutMs > Date.now()) {
           return checkMigrationsFinished()
         }

From d880302553762476e9c4be9c0f4beea6ad19447c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Dec 2023 10:04:10 +0100
Subject: [PATCH 38/58] Check if migrations are needed on app serve

---
 .../src/api/controllers/static/index.ts       | 21 +++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index 2963546e7f..4828181149 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -25,8 +25,12 @@ import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
 import { App, Ctx, ProcessAttachmentResponse } from "@budibase/types"
+import {
+  getAppMigrationVersion,
+  getLatestMigrationId,
+} from "../../../appMigrations"
 
-const send = require("koa-send")
+import send from "koa-send"
 
 export const toggleBetaUiFeature = async function (ctx: Ctx) {
   const cookieName = `beta:${ctx.params.feature}`
@@ -125,7 +129,21 @@ export const deleteObjects = async function (ctx: Ctx) {
   )
 }
 
+const requiresMigration = async (ctx: Ctx) => {
+  const appId = context.getAppId()
+  if (!appId) {
+    ctx.throw("AppId could not be found")
+  }
+
+  const latestAppliedMigration = await getAppMigrationVersion(appId)
+
+  const requiresMigrations = latestAppliedMigration !== getLatestMigrationId()
+  return requiresMigrations
+}
+
 export const serveApp = async function (ctx: Ctx) {
+  const needMigrations = await requiresMigration(ctx)
+
   const bbHeaderEmbed =
     ctx.request.get("x-budibase-embed")?.toLowerCase() === "true"
 
@@ -273,7 +291,6 @@ export const getSignedUploadURL = async function (ctx: Ctx) {
     const { bucket, key } = ctx.request.body || {}
     if (!bucket || !key) {
       ctx.throw(400, "bucket and key values are required")
-      return
     }
     try {
       const s3 = new AWS.S3({

From 2cb9f10327324f6d64a2c9a3b68b293e465ef339 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Dec 2023 10:04:22 +0100
Subject: [PATCH 39/58] Nodemon, watch svelte on server

---
 packages/server/nodemon.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/nodemon.json b/packages/server/nodemon.json
index 33d277dd64..5535e0772e 100644
--- a/packages/server/nodemon.json
+++ b/packages/server/nodemon.json
@@ -7,7 +7,7 @@
     "../shared-core",
     "../string-templates"
   ],
-  "ext": "js,ts,json",
+  "ext": "js,ts,json,svelte",
   "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../*/dist/**/*"],
   "exec": "yarn build && node ./dist/index.js"
 }

From 357522ec6e419636a3829eaad78326dca1527ce3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 18 Dec 2023 16:15:00 +0100
Subject: [PATCH 40/58] Server render updating page

---
 packages/server/src/api/controllers/static/index.ts       | 6 ++++--
 .../src/api/controllers/static/templates/Updating.svelte  | 8 ++++++++
 2 files changed, 12 insertions(+), 2 deletions(-)
 create mode 100644 packages/server/src/api/controllers/static/templates/Updating.svelte

diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index 4828181149..c23ea349d4 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -156,6 +156,10 @@ export const serveApp = async function (ctx: Ctx) {
     appHbsPath = join(__dirname, "templates", "app.hbs")
   }
 
+  const App = !needMigrations
+    ? require("./templates/BudibaseApp.svelte").default
+    : require("./templates/Updating.svelte").default
+
   let db
   try {
     db = context.getAppDB({ skip_setup: true })
@@ -163,7 +167,6 @@ export const serveApp = async function (ctx: Ctx) {
     let appId = context.getAppId()
 
     if (!env.isJest()) {
-      const App = require("./templates/BudibaseApp.svelte").default
       const plugins = objectStore.enrichPluginURLs(appInfo.usedPlugins)
       const { head, html, css } = App.render({
         metaImage:
@@ -200,7 +203,6 @@ export const serveApp = async function (ctx: Ctx) {
     }
   } catch (error) {
     if (!env.isJest()) {
-      const App = require("./templates/BudibaseApp.svelte").default
       const { head, html, css } = App.render({
         title: branding?.metaTitle,
         metaTitle: branding?.metaTitle,
diff --git a/packages/server/src/api/controllers/static/templates/Updating.svelte b/packages/server/src/api/controllers/static/templates/Updating.svelte
new file mode 100644
index 0000000000..5c7e23b5b3
--- /dev/null
+++ b/packages/server/src/api/controllers/static/templates/Updating.svelte
@@ -0,0 +1,8 @@
+<div class="loading">Updating the system...</div>
+
+<style>
+  .loading {
+    display: flex;
+    justify-content: center;
+  }
+</style>

From 0b46b6883df543b6b33206831d04a6817257638c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 19 Dec 2023 10:37:14 +0100
Subject: [PATCH 41/58] Remove unused regexparam

---
 packages/client/package.json | 1 -
 yarn.lock                    | 5 -----
 2 files changed, 6 deletions(-)

diff --git a/packages/client/package.json b/packages/client/package.json
index 39ddb4bd49..227c7b25d4 100644
--- a/packages/client/package.json
+++ b/packages/client/package.json
@@ -37,7 +37,6 @@
     "downloadjs": "1.4.7",
     "html5-qrcode": "^2.2.1",
     "leaflet": "^1.7.1",
-    "regexparam": "^1.3.0",
     "sanitize-html": "^2.7.0",
     "screenfull": "^6.0.1",
     "shortid": "^2.2.15",
diff --git a/yarn.lock b/yarn.lock
index 150585ce1a..85dd6ab251 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -18312,11 +18312,6 @@ regexparam@2.0.1:
   resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-2.0.1.tgz#c912f5dae371e3798100b3c9ce22b7414d0889fa"
   integrity sha512-zRgSaYemnNYxUv+/5SeoHI0eJIgTL/A2pUtXUPLHQxUldagouJ9p+K6IbIZ/JiQuCEv2E2B1O11SjVQy3aMCkw==
 
-regexparam@^1.3.0:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-1.3.0.tgz#2fe42c93e32a40eff6235d635e0ffa344b92965f"
-  integrity sha512-6IQpFBv6e5vz1QAqI+V4k8P2e/3gRrqfCJ9FI+O1FLQTO+Uz6RXZEZOPmTJ6hlGj7gkERzY5BRCv09whKP96/g==
-
 regexpu-core@^5.3.1:
   version "5.3.1"
   resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-5.3.1.tgz#66900860f88def39a5cb79ebd9490e84f17bcdfb"

From 75b8190c01898b3522ce5605191d6a9211cd09a1 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 19 Dec 2023 12:05:27 +0100
Subject: [PATCH 42/58] Move updating page to the client

---
 .../src/components/UpdatingApp.svelte}                    | 3 +++
 packages/client/src/index.js                              | 8 ++++++++
 packages/server/src/api/controllers/static/index.ts       | 7 +++----
 .../api/controllers/static/templates/BudibaseApp.svelte   | 6 ++++++
 4 files changed, 20 insertions(+), 4 deletions(-)
 rename packages/{server/src/api/controllers/static/templates/Updating.svelte => client/src/components/UpdatingApp.svelte} (86%)

diff --git a/packages/server/src/api/controllers/static/templates/Updating.svelte b/packages/client/src/components/UpdatingApp.svelte
similarity index 86%
rename from packages/server/src/api/controllers/static/templates/Updating.svelte
rename to packages/client/src/components/UpdatingApp.svelte
index 5c7e23b5b3..b24f591c5c 100644
--- a/packages/server/src/api/controllers/static/templates/Updating.svelte
+++ b/packages/client/src/components/UpdatingApp.svelte
@@ -1,3 +1,6 @@
+<script>
+</script>
+
 <div class="loading">Updating the system...</div>
 
 <style>
diff --git a/packages/client/src/index.js b/packages/client/src/index.js
index a3cb4206c3..f6ed23b2a9 100644
--- a/packages/client/src/index.js
+++ b/packages/client/src/index.js
@@ -1,4 +1,5 @@
 import ClientApp from "./components/ClientApp.svelte"
+import UpdatingApp from "./components/UpdatingApp.svelte"
 import {
   builderStore,
   appStore,
@@ -52,6 +53,13 @@ const loadBudibase = async () => {
     window["##BUDIBASE_APP_EMBEDDED##"] === "true"
   )
 
+  if (window.MIGRATING_APP) {
+    new UpdatingApp({
+      target: window.document.body,
+    })
+    return
+  }
+
   // Fetch environment info
   if (!get(environmentStore)?.loaded) {
     await environmentStore.actions.fetchEnvironment()
diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index c23ea349d4..0166f92478 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -156,10 +156,6 @@ export const serveApp = async function (ctx: Ctx) {
     appHbsPath = join(__dirname, "templates", "app.hbs")
   }
 
-  const App = !needMigrations
-    ? require("./templates/BudibaseApp.svelte").default
-    : require("./templates/Updating.svelte").default
-
   let db
   try {
     db = context.getAppDB({ skip_setup: true })
@@ -168,6 +164,7 @@ export const serveApp = async function (ctx: Ctx) {
 
     if (!env.isJest()) {
       const plugins = objectStore.enrichPluginURLs(appInfo.usedPlugins)
+      const App = require("./templates/BudibaseApp.svelte").default
       const { head, html, css } = App.render({
         metaImage:
           branding?.metaImageUrl ||
@@ -188,6 +185,7 @@ export const serveApp = async function (ctx: Ctx) {
           config?.logoUrl !== ""
             ? objectStore.getGlobalFileUrl("settings", "logoUrl")
             : "",
+        appMigrating: needMigrations,
       })
       const appHbs = loadHandlebarsFile(appHbsPath)
       ctx.body = await processString(appHbs, {
@@ -203,6 +201,7 @@ export const serveApp = async function (ctx: Ctx) {
     }
   } catch (error) {
     if (!env.isJest()) {
+      const App = require("./templates/BudibaseApp.svelte").default
       const { head, html, css } = App.render({
         title: branding?.metaTitle,
         metaTitle: branding?.metaTitle,
diff --git a/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte b/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte
index 32edb6dc7b..7819368fc0 100644
--- a/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte
+++ b/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte
@@ -8,6 +8,7 @@
 
   export let clientLibPath
   export let usedPlugins
+  export let appMigrating
 </script>
 
 <svelte:head>
@@ -110,6 +111,11 @@
   <script type="application/javascript">
     window.INIT_TIME = Date.now()
   </script>
+  {#if appMigrating}
+    <script type="application/javascript">
+      window.MIGRATING_APP = true
+    </script>
+  {/if}
   <script type="application/javascript" src={clientLibPath}>
   </script>
   <!-- Custom components need inserted after the core client library -->

From 94f0d38fe41d5a1dd5e4e3a1bc2ee940a257b188 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 19 Dec 2023 12:20:05 +0100
Subject: [PATCH 43/58] Implement refresh

---
 .../client/src/components/UpdatingApp.svelte  | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/packages/client/src/components/UpdatingApp.svelte b/packages/client/src/components/UpdatingApp.svelte
index b24f591c5c..2843133f2d 100644
--- a/packages/client/src/components/UpdatingApp.svelte
+++ b/packages/client/src/components/UpdatingApp.svelte
@@ -1,4 +1,30 @@
 <script>
+  import { API } from "../api"
+
+  const timeoutMs = 60000 // 1 minute
+  const loadTime = Date.now()
+
+  async function checkMigrationsFinished() {
+    setTimeout(async () => {
+      const response = await API.getMigrationStatus()
+      if (!response.migrated) {
+        if (loadTime + timeoutMs > Date.now()) {
+          return checkMigrationsFinished()
+        }
+
+        return migrationTimeout()
+      }
+
+      window.location.reload()
+    }, 1000)
+  }
+
+  checkMigrationsFinished()
+
+  function migrationTimeout() {
+    // TODO
+    alert("Something went wrong 💀")
+  }
 </script>
 
 <div class="loading">Updating the system...</div>

From ee404342231deedfd654f62aaadd20c3698f840f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 19 Dec 2023 12:24:10 +0100
Subject: [PATCH 44/58] Handle migration when app loaded

---
 packages/client/src/api/api.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/client/src/api/api.js b/packages/client/src/api/api.js
index 8488b702b6..ee33b1dbe0 100644
--- a/packages/client/src/api/api.js
+++ b/packages/client/src/api/api.js
@@ -77,4 +77,8 @@ export const API = createAPIClient({
     // Log all errors to console
     console.warn(`[Client] HTTP ${status} on ${method}:${url}\n\t${message}`)
   },
+  onMigrationDetected: appId => {
+    // We will force a reload, that will display the updating screen until the migration is running
+    window.location.reload()
+  },
 })

From 4e48d1d66eedc6f9ff4e41419ee27c3b14baf052 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 19 Dec 2023 12:27:26 +0100
Subject: [PATCH 45/58] Avoid multiple reloads

---
 packages/client/src/api/api.js | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/packages/client/src/api/api.js b/packages/client/src/api/api.js
index ee33b1dbe0..c5a1cd2775 100644
--- a/packages/client/src/api/api.js
+++ b/packages/client/src/api/api.js
@@ -78,7 +78,9 @@ export const API = createAPIClient({
     console.warn(`[Client] HTTP ${status} on ${method}:${url}\n\t${message}`)
   },
   onMigrationDetected: appId => {
-    // We will force a reload, that will display the updating screen until the migration is running
-    window.location.reload()
+    if (!window.MIGRATING_APP) {
+      // We will force a reload, that will display the updating screen until the migration is running
+      window.location.reload()
+    }
   },
 })

From 5dd3c7cada1bc0ef76ba8806f457b876aa5814e5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 19 Dec 2023 12:46:27 +0100
Subject: [PATCH 46/58] Lint

---
 packages/client/src/api/api.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/api/api.js b/packages/client/src/api/api.js
index c5a1cd2775..d4c8faa4d2 100644
--- a/packages/client/src/api/api.js
+++ b/packages/client/src/api/api.js
@@ -77,7 +77,7 @@ export const API = createAPIClient({
     // Log all errors to console
     console.warn(`[Client] HTTP ${status} on ${method}:${url}\n\t${message}`)
   },
-  onMigrationDetected: appId => {
+  onMigrationDetected: _appId => {
     if (!window.MIGRATING_APP) {
       // We will force a reload, that will display the updating screen until the migration is running
       window.location.reload()

From 746178f16950721c9b27d14c0cc8eb65a33237cb Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 4 Jan 2024 13:18:55 +0100
Subject: [PATCH 47/58] Update frontend

---
 .../client/src/components/UpdatingApp.svelte  | 32 ++++++++++++++++---
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/packages/client/src/components/UpdatingApp.svelte b/packages/client/src/components/UpdatingApp.svelte
index 2843133f2d..7d8d54df45 100644
--- a/packages/client/src/components/UpdatingApp.svelte
+++ b/packages/client/src/components/UpdatingApp.svelte
@@ -4,6 +4,8 @@
   const timeoutMs = 60000 // 1 minute
   const loadTime = Date.now()
 
+  let timedOut = false
+
   async function checkMigrationsFinished() {
     setTimeout(async () => {
       const response = await API.getMigrationStatus()
@@ -11,7 +13,6 @@
         if (loadTime + timeoutMs > Date.now()) {
           return checkMigrationsFinished()
         }
-
         return migrationTimeout()
       }
 
@@ -22,16 +23,39 @@
   checkMigrationsFinished()
 
   function migrationTimeout() {
-    // TODO
-    alert("Something went wrong 💀")
+    timedOut = true
   }
 </script>
 
-<div class="loading">Updating the system...</div>
+<div class="loading">
+  {#if !timedOut}
+    <p class="loading-message">
+      🛠️ We are updating the system. <br />Please wait, we will be back in a
+      second!
+    </p>
+  {:else}
+    <p class="loading-message timeout">
+      An error occurred. Please try again later.
+    </p>
+  {/if}
+</div>
 
 <style>
   .loading {
     display: flex;
     justify-content: center;
+    align-items: center;
+    justify-content: center;
+    flex-direction: column;
+    gap: var(--spacing-xl);
+    height: 100vh;
+    margin: 0;
+  }
+  .loading-message {
+    text-align: center;
+    font-size: 18px;
+  }
+  .timeout {
+    color: rgb(196, 46, 46);
   }
 </style>

From bfaab228d57429df634263d34c1017d3ecbf008a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 12:07:26 +0100
Subject: [PATCH 48/58] Use updating page from frontend

---
 .../client/src/components/UpdatingApp.svelte  | 60 +++----------------
 1 file changed, 7 insertions(+), 53 deletions(-)

diff --git a/packages/client/src/components/UpdatingApp.svelte b/packages/client/src/components/UpdatingApp.svelte
index 7d8d54df45..d93355394e 100644
--- a/packages/client/src/components/UpdatingApp.svelte
+++ b/packages/client/src/components/UpdatingApp.svelte
@@ -1,61 +1,15 @@
 <script>
+  import { Updating } from "@budibase/frontend-core"
   import { API } from "../api"
 
-  const timeoutMs = 60000 // 1 minute
-  const loadTime = Date.now()
-
-  let timedOut = false
-
-  async function checkMigrationsFinished() {
-    setTimeout(async () => {
-      const response = await API.getMigrationStatus()
-      if (!response.migrated) {
-        if (loadTime + timeoutMs > Date.now()) {
-          return checkMigrationsFinished()
-        }
-        return migrationTimeout()
-      }
-
-      window.location.reload()
-    }, 1000)
+  async function isMigrationDone() {
+    const response = await API.getMigrationStatus()
+    return response.migrated
   }
 
-  checkMigrationsFinished()
-
-  function migrationTimeout() {
-    timedOut = true
+  async function onMigrationDone() {
+    window.location.reload()
   }
 </script>
 
-<div class="loading">
-  {#if !timedOut}
-    <p class="loading-message">
-      🛠️ We are updating the system. <br />Please wait, we will be back in a
-      second!
-    </p>
-  {:else}
-    <p class="loading-message timeout">
-      An error occurred. Please try again later.
-    </p>
-  {/if}
-</div>
-
-<style>
-  .loading {
-    display: flex;
-    justify-content: center;
-    align-items: center;
-    justify-content: center;
-    flex-direction: column;
-    gap: var(--spacing-xl);
-    height: 100vh;
-    margin: 0;
-  }
-  .loading-message {
-    text-align: center;
-    font-size: 18px;
-  }
-  .timeout {
-    color: rgb(196, 46, 46);
-  }
-</style>
+<Updating {isMigrationDone} {onMigrationDone} />

From 9ebe93f71d3eade90062ca955292d6ef8f3368d6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 12:10:35 +0100
Subject: [PATCH 49/58] Clean

---
 packages/client/src/components/UpdatingApp.svelte | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/packages/client/src/components/UpdatingApp.svelte b/packages/client/src/components/UpdatingApp.svelte
index d93355394e..4177ea017f 100644
--- a/packages/client/src/components/UpdatingApp.svelte
+++ b/packages/client/src/components/UpdatingApp.svelte
@@ -1,15 +1,9 @@
 <script>
   import { Updating } from "@budibase/frontend-core"
-  import { API } from "../api"
-
-  async function isMigrationDone() {
-    const response = await API.getMigrationStatus()
-    return response.migrated
-  }
 
   async function onMigrationDone() {
     window.location.reload()
   }
 </script>
 
-<Updating {isMigrationDone} {onMigrationDone} />
+<Updating {onMigrationDone} />

From f77b81267933d2f868b4af188be73b2bbb70a879 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 12:15:12 +0100
Subject: [PATCH 50/58] Handle no migrations

---
 packages/server/src/api/controllers/static/index.ts | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index 0166f92478..5f383e837d 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -135,9 +135,14 @@ const requiresMigration = async (ctx: Ctx) => {
     ctx.throw("AppId could not be found")
   }
 
-  const latestAppliedMigration = await getAppMigrationVersion(appId)
+  const latestMigration = getLatestMigrationId()
+  if (!latestMigration) {
+    return false
+  }
 
-  const requiresMigrations = latestAppliedMigration !== getLatestMigrationId()
+  const latestMigrationApplied = await getAppMigrationVersion(appId)
+
+  const requiresMigrations = latestMigrationApplied !== latestMigration
   return requiresMigrations
 }
 

From f123a821bd95add40876b331aeae213e9152238d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 12:21:34 +0100
Subject: [PATCH 51/58] Styling

---
 packages/client/src/components/UpdatingApp.svelte | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/packages/client/src/components/UpdatingApp.svelte b/packages/client/src/components/UpdatingApp.svelte
index 4177ea017f..b493b7cd67 100644
--- a/packages/client/src/components/UpdatingApp.svelte
+++ b/packages/client/src/components/UpdatingApp.svelte
@@ -6,4 +6,12 @@
   }
 </script>
 
-<Updating {onMigrationDone} />
+<div class="updating">
+  <Updating {onMigrationDone} />
+</div>
+
+<style>
+  .updating {
+    font-family: var(--font-sans);
+  }
+</style>

From 2a07aa35f4da73e1447c45708eea967d516b33e5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 13:48:50 +0100
Subject: [PATCH 52/58] Pass api call

---
 packages/client/src/components/UpdatingApp.svelte | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/packages/client/src/components/UpdatingApp.svelte b/packages/client/src/components/UpdatingApp.svelte
index b493b7cd67..74e5500715 100644
--- a/packages/client/src/components/UpdatingApp.svelte
+++ b/packages/client/src/components/UpdatingApp.svelte
@@ -1,5 +1,11 @@
 <script>
   import { Updating } from "@budibase/frontend-core"
+  import { API } from "../api"
+
+  async function isMigrationDone() {
+    const response = await API.getMigrationStatus()
+    return response.migrated
+  }
 
   async function onMigrationDone() {
     window.location.reload()
@@ -7,7 +13,7 @@
 </script>
 
 <div class="updating">
-  <Updating {onMigrationDone} />
+  <Updating {isMigrationDone} {onMigrationDone} />
 </div>
 
 <style>

From 6df4bf84df1d166af6c2ce206d891b764cdf1aa2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 4 Jan 2024 15:42:16 +0100
Subject: [PATCH 53/58] Lint

---
 packages/server/src/api/controllers/datasource.ts | 7 ++-----
 packages/server/src/integrations/postgres.ts      | 4 ++--
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts
index 0a7430aa94..b794312fb6 100644
--- a/packages/server/src/api/controllers/datasource.ts
+++ b/packages/server/src/api/controllers/datasource.ts
@@ -57,17 +57,14 @@ async function getAndMergeDatasource(datasource: Datasource) {
 
 async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
   const connector = (await getConnector(datasource)) as DatasourcePlus
-  return await connector.buildSchema(
-    datasource._id!,
-    datasource.entities! as Record<string, Table>
-  )
+  return await connector.buildSchema(datasource._id!, datasource.entities!)
 }
 
 async function buildFilteredSchema(
   datasource: Datasource,
   filter?: string[]
 ): Promise<Schema> {
-  let schema = await buildSchemaHelper(datasource)
+  const schema = await buildSchemaHelper(datasource)
   if (!filter) {
     return schema
   }
diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts
index de3bf0e59e..2d61d05a01 100644
--- a/packages/server/src/integrations/postgres.ts
+++ b/packages/server/src/integrations/postgres.ts
@@ -362,8 +362,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
         })
       }
 
-      let finalizedTables = finaliseExternalTables(tables, entities)
-      let errors = checkExternalTables(finalizedTables)
+      const finalizedTables = finaliseExternalTables(tables, entities)
+      const errors = checkExternalTables(finalizedTables)
       return { tables: finalizedTables, errors }
     } catch (err) {
       // @ts-ignore

From dd77779bc16fb1e7794afbec89497ac0d4745f2b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 5 Jan 2024 15:14:31 +0100
Subject: [PATCH 54/58] Move code to sdk

---
 .../server/src/api/controllers/datasource.ts  | 80 +++----------------
 .../server/src/sdk/app/datasources/index.ts   |  2 +
 .../server/src/sdk/app/datasources/plus.ts    | 71 ++++++++++++++++
 3 files changed, 86 insertions(+), 67 deletions(-)
 create mode 100644 packages/server/src/sdk/app/datasources/plus.ts

diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts
index b794312fb6..364490e5f5 100644
--- a/packages/server/src/api/controllers/datasource.ts
+++ b/packages/server/src/api/controllers/datasource.ts
@@ -14,10 +14,7 @@ import {
   DatasourcePlus,
   FetchDatasourceInfoRequest,
   FetchDatasourceInfoResponse,
-  IntegrationBase,
-  Schema,
   SourceName,
-  Table,
   UpdateDatasourceResponse,
   UserCtx,
   VerifyDatasourceRequest,
@@ -28,62 +25,6 @@ import { builderSocket } from "../../websockets"
 import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets"
 import { isEqual } from "lodash"
 
-async function getConnector(
-  datasource: Datasource
-): Promise<IntegrationBase | DatasourcePlus> {
-  const Connector = await getIntegration(datasource.source)
-  // can't enrich if it doesn't have an ID yet
-  if (datasource._id) {
-    datasource = await sdk.datasources.enrich(datasource)
-  }
-  // Connect to the DB and build the schema
-  return new Connector(datasource.config)
-}
-
-async function getAndMergeDatasource(datasource: Datasource) {
-  let existingDatasource: undefined | Datasource
-  if (datasource._id) {
-    existingDatasource = await sdk.datasources.get(datasource._id)
-  }
-  let enrichedDatasource = datasource
-  if (existingDatasource) {
-    enrichedDatasource = sdk.datasources.mergeConfigs(
-      datasource,
-      existingDatasource
-    )
-  }
-  return await sdk.datasources.enrich(enrichedDatasource)
-}
-
-async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
-  const connector = (await getConnector(datasource)) as DatasourcePlus
-  return await connector.buildSchema(datasource._id!, datasource.entities!)
-}
-
-async function buildFilteredSchema(
-  datasource: Datasource,
-  filter?: string[]
-): Promise<Schema> {
-  const schema = await buildSchemaHelper(datasource)
-  if (!filter) {
-    return schema
-  }
-
-  let filteredSchema: Schema = { tables: {}, errors: {} }
-  for (let key in schema.tables) {
-    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
-      filteredSchema.tables[key] = schema.tables[key]
-    }
-  }
-
-  for (let key in schema.errors) {
-    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
-      filteredSchema.errors[key] = schema.errors[key]
-    }
-  }
-  return filteredSchema
-}
-
 export async function fetch(ctx: UserCtx) {
   ctx.body = await sdk.datasources.fetch()
 }
@@ -92,8 +33,7 @@ export async function verify(
   ctx: UserCtx<VerifyDatasourceRequest, VerifyDatasourceResponse>
 ) {
   const { datasource } = ctx.request.body
-  const enrichedDatasource = await getAndMergeDatasource(datasource)
-  const connector = await getConnector(enrichedDatasource)
+  const connector = await sdk.datasources.getConnector(datasource)
   if (!connector.testConnection) {
     ctx.throw(400, "Connection information verification not supported")
   }
@@ -109,8 +49,9 @@ export async function information(
   ctx: UserCtx<FetchDatasourceInfoRequest, FetchDatasourceInfoResponse>
 ) {
   const { datasource } = ctx.request.body
-  const enrichedDatasource = await getAndMergeDatasource(datasource)
-  const connector = (await getConnector(enrichedDatasource)) as DatasourcePlus
+  const connector = (await sdk.datasources.getConnector(
+    datasource
+  )) as DatasourcePlus
   if (!connector.getTableNames) {
     ctx.throw(400, "Table name fetching not supported by datasource")
   }
@@ -125,7 +66,10 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
   const tablesFilter = ctx.request.body.tablesFilter
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
 
-  const { tables, errors } = await buildFilteredSchema(datasource, tablesFilter)
+  const { tables, errors } = await sdk.datasources.buildFilteredSchema(
+    datasource,
+    tablesFilter
+  )
   datasource.entities = tables
 
   setDefaultDisplayColumns(datasource)
@@ -277,7 +221,10 @@ export async function save(
 
   let errors: Record<string, string> = {}
   if (fetchSchema) {
-    const schema = await buildFilteredSchema(datasource, tablesFilter)
+    const schema = await sdk.datasources.buildFilteredSchema(
+      datasource,
+      tablesFilter
+    )
     datasource.entities = schema.tables
     setDefaultDisplayColumns(datasource)
     errors = schema.errors
@@ -381,8 +328,7 @@ export async function query(ctx: UserCtx) {
 
 export async function getExternalSchema(ctx: UserCtx) {
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
-  const enrichedDatasource = await getAndMergeDatasource(datasource)
-  const connector = await getConnector(enrichedDatasource)
+  const connector = await sdk.datasources.getConnector(datasource)
 
   if (!connector.getExternalSchema) {
     ctx.throw(400, "Datasource does not support exporting external schema")
diff --git a/packages/server/src/sdk/app/datasources/index.ts b/packages/server/src/sdk/app/datasources/index.ts
index 1ce6b0e689..8f06e989d3 100644
--- a/packages/server/src/sdk/app/datasources/index.ts
+++ b/packages/server/src/sdk/app/datasources/index.ts
@@ -1,5 +1,7 @@
 import * as datasources from "./datasources"
+import * as plus from "./plus"
 
 export default {
   ...datasources,
+  ...plus,
 }
diff --git a/packages/server/src/sdk/app/datasources/plus.ts b/packages/server/src/sdk/app/datasources/plus.ts
new file mode 100644
index 0000000000..65b908526c
--- /dev/null
+++ b/packages/server/src/sdk/app/datasources/plus.ts
@@ -0,0 +1,71 @@
+import {
+  Datasource,
+  DatasourcePlus,
+  IntegrationBase,
+  Schema,
+} from "@budibase/types"
+import * as datasources from "./datasources"
+import { getIntegration } from "../../../integrations"
+
+export async function buildFilteredSchema(
+  datasource: Datasource,
+  filter?: string[]
+): Promise<Schema> {
+  const schema = await buildSchemaHelper(datasource)
+  if (!filter) {
+    return schema
+  }
+
+  let filteredSchema: Schema = { tables: {}, errors: {} }
+  for (let key in schema.tables) {
+    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
+      filteredSchema.tables[key] = schema.tables[key]
+    }
+  }
+
+  for (let key in schema.errors) {
+    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
+      filteredSchema.errors[key] = schema.errors[key]
+    }
+  }
+  return filteredSchema
+}
+
+async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
+  const connector = (await getConnector(datasource)) as DatasourcePlus
+  const externalSchema = await connector.buildSchema(
+    datasource._id!,
+    datasource.entities!
+  )
+  return externalSchema
+}
+
+export async function getConnector(
+  datasource: Datasource
+): Promise<IntegrationBase | DatasourcePlus> {
+  const Connector = await getIntegration(datasource.source)
+
+  datasource = await getAndMergeDatasource(datasource)
+
+  // can't enrich if it doesn't have an ID yet
+  if (datasource._id) {
+    datasource = await datasources.enrich(datasource)
+  }
+  // Connect to the DB and build the schema
+  return new Connector(datasource.config)
+}
+
+async function getAndMergeDatasource(datasource: Datasource) {
+  let existingDatasource: undefined | Datasource
+  if (datasource._id) {
+    existingDatasource = await datasources.get(datasource._id)
+  }
+  let enrichedDatasource = datasource
+  if (existingDatasource) {
+    enrichedDatasource = datasources.mergeConfigs(
+      datasource,
+      existingDatasource
+    )
+  }
+  return await datasources.enrich(enrichedDatasource)
+}

From 1f9ee5a2066e18331fcd5967b5f32b5e2ecfc37e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 5 Jan 2024 16:27:45 +0100
Subject: [PATCH 55/58] Clean code

---
 .../server/src/sdk/app/datasources/plus.ts     | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/packages/server/src/sdk/app/datasources/plus.ts b/packages/server/src/sdk/app/datasources/plus.ts
index 65b908526c..0049730119 100644
--- a/packages/server/src/sdk/app/datasources/plus.ts
+++ b/packages/server/src/sdk/app/datasources/plus.ts
@@ -45,7 +45,7 @@ export async function getConnector(
 ): Promise<IntegrationBase | DatasourcePlus> {
   const Connector = await getIntegration(datasource.source)
 
-  datasource = await getAndMergeDatasource(datasource)
+  datasource = await mergeAndEnrich(datasource)
 
   // can't enrich if it doesn't have an ID yet
   if (datasource._id) {
@@ -55,17 +55,11 @@ export async function getConnector(
   return new Connector(datasource.config)
 }
 
-async function getAndMergeDatasource(datasource: Datasource) {
-  let existingDatasource: undefined | Datasource
+async function mergeAndEnrich(datasource: Datasource) {
   if (datasource._id) {
-    existingDatasource = await datasources.get(datasource._id)
+    const existingDatasource = await datasources.get(datasource._id)
+
+    datasource = datasources.mergeConfigs(datasource, existingDatasource)
   }
-  let enrichedDatasource = datasource
-  if (existingDatasource) {
-    enrichedDatasource = datasources.mergeConfigs(
-      datasource,
-      existingDatasource
-    )
-  }
-  return await datasources.enrich(enrichedDatasource)
+  return await datasources.enrich(datasource)
 }

From 498137ba7c8a1a40d6382f2f81abc0ddeac43af5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 5 Jan 2024 16:39:55 +0100
Subject: [PATCH 56/58] Split back

---
 packages/server/src/api/controllers/datasource.ts | 15 ++++++++++++---
 packages/server/src/sdk/app/datasources/plus.ts   |  5 +----
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts
index 364490e5f5..8c177f1704 100644
--- a/packages/server/src/api/controllers/datasource.ts
+++ b/packages/server/src/api/controllers/datasource.ts
@@ -33,7 +33,10 @@ export async function verify(
   ctx: UserCtx<VerifyDatasourceRequest, VerifyDatasourceResponse>
 ) {
   const { datasource } = ctx.request.body
-  const connector = await sdk.datasources.getConnector(datasource)
+  const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(
+    datasource
+  )
+  const connector = await sdk.datasources.getConnector(enrichedDatasource)
   if (!connector.testConnection) {
     ctx.throw(400, "Connection information verification not supported")
   }
@@ -49,8 +52,11 @@ export async function information(
   ctx: UserCtx<FetchDatasourceInfoRequest, FetchDatasourceInfoResponse>
 ) {
   const { datasource } = ctx.request.body
-  const connector = (await sdk.datasources.getConnector(
+  const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(
     datasource
+  )
+  const connector = (await sdk.datasources.getConnector(
+    enrichedDatasource
   )) as DatasourcePlus
   if (!connector.getTableNames) {
     ctx.throw(400, "Table name fetching not supported by datasource")
@@ -328,7 +334,10 @@ export async function query(ctx: UserCtx) {
 
 export async function getExternalSchema(ctx: UserCtx) {
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
-  const connector = await sdk.datasources.getConnector(datasource)
+  const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(
+    datasource
+  )
+  const connector = await sdk.datasources.getConnector(enrichedDatasource)
 
   if (!connector.getExternalSchema) {
     ctx.throw(400, "Datasource does not support exporting external schema")
diff --git a/packages/server/src/sdk/app/datasources/plus.ts b/packages/server/src/sdk/app/datasources/plus.ts
index 0049730119..117d19a6a7 100644
--- a/packages/server/src/sdk/app/datasources/plus.ts
+++ b/packages/server/src/sdk/app/datasources/plus.ts
@@ -44,9 +44,6 @@ export async function getConnector(
   datasource: Datasource
 ): Promise<IntegrationBase | DatasourcePlus> {
   const Connector = await getIntegration(datasource.source)
-
-  datasource = await mergeAndEnrich(datasource)
-
   // can't enrich if it doesn't have an ID yet
   if (datasource._id) {
     datasource = await datasources.enrich(datasource)
@@ -55,7 +52,7 @@ export async function getConnector(
   return new Connector(datasource.config)
 }
 
-async function mergeAndEnrich(datasource: Datasource) {
+export async function getAndMergeDatasource(datasource: Datasource) {
   if (datasource._id) {
     const existingDatasource = await datasources.get(datasource._id)
 

From 59d57d94220c6ba7c940da9d02b54ed1d18b0443 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 16:20:31 +0100
Subject: [PATCH 57/58] Update account-portal ref

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index 8ee2734e77..bcd86d9034 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit 8ee2734e77709438cbcaaabc024f677c7b24c883
+Subproject commit bcd86d9034ba954f013da4c10171bf495ab88189

From 024c3ec3ae7fab74c519eb9474043bb9d267d502 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Mon, 8 Jan 2024 15:51:36 +0000
Subject: [PATCH 58/58] Bump version to 2.14.4

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index d6fa262685..386fbe425a 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.14.3",
+  "version": "2.14.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*",