From d880302553762476e9c4be9c0f4beea6ad19447c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Dec 2023 10:04:10 +0100
Subject: [PATCH 01/19] Check if migrations are needed on app serve

---
 .../src/api/controllers/static/index.ts       | 21 +++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index 2963546e7f..4828181149 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -25,8 +25,12 @@ import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
 import { App, Ctx, ProcessAttachmentResponse } from "@budibase/types"
+import {
+  getAppMigrationVersion,
+  getLatestMigrationId,
+} from "../../../appMigrations"
 
-const send = require("koa-send")
+import send from "koa-send"
 
 export const toggleBetaUiFeature = async function (ctx: Ctx) {
   const cookieName = `beta:${ctx.params.feature}`
@@ -125,7 +129,21 @@ export const deleteObjects = async function (ctx: Ctx) {
   )
 }
 
+const requiresMigration = async (ctx: Ctx) => {
+  const appId = context.getAppId()
+  if (!appId) {
+    ctx.throw("AppId could not be found")
+  }
+
+  const latestAppliedMigration = await getAppMigrationVersion(appId)
+
+  const requiresMigrations = latestAppliedMigration !== getLatestMigrationId()
+  return requiresMigrations
+}
+
 export const serveApp = async function (ctx: Ctx) {
+  const needMigrations = await requiresMigration(ctx)
+
   const bbHeaderEmbed =
     ctx.request.get("x-budibase-embed")?.toLowerCase() === "true"
 
@@ -273,7 +291,6 @@ export const getSignedUploadURL = async function (ctx: Ctx) {
     const { bucket, key } = ctx.request.body || {}
     if (!bucket || !key) {
       ctx.throw(400, "bucket and key values are required")
-      return
     }
     try {
       const s3 = new AWS.S3({

From 2cb9f10327324f6d64a2c9a3b68b293e465ef339 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 14 Dec 2023 10:04:22 +0100
Subject: [PATCH 02/19] Nodemon, watch svelte on server

---
 packages/server/nodemon.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/nodemon.json b/packages/server/nodemon.json
index 33d277dd64..5535e0772e 100644
--- a/packages/server/nodemon.json
+++ b/packages/server/nodemon.json
@@ -7,7 +7,7 @@
     "../shared-core",
     "../string-templates"
   ],
-  "ext": "js,ts,json",
+  "ext": "js,ts,json,svelte",
   "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../*/dist/**/*"],
   "exec": "yarn build && node ./dist/index.js"
 }

From 357522ec6e419636a3829eaad78326dca1527ce3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 18 Dec 2023 16:15:00 +0100
Subject: [PATCH 03/19] Server render updating page

---
 packages/server/src/api/controllers/static/index.ts       | 6 ++++--
 .../src/api/controllers/static/templates/Updating.svelte  | 8 ++++++++
 2 files changed, 12 insertions(+), 2 deletions(-)
 create mode 100644 packages/server/src/api/controllers/static/templates/Updating.svelte

diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index 4828181149..c23ea349d4 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -156,6 +156,10 @@ export const serveApp = async function (ctx: Ctx) {
     appHbsPath = join(__dirname, "templates", "app.hbs")
   }
 
+  const App = !needMigrations
+    ? require("./templates/BudibaseApp.svelte").default
+    : require("./templates/Updating.svelte").default
+
   let db
   try {
     db = context.getAppDB({ skip_setup: true })
@@ -163,7 +167,6 @@ export const serveApp = async function (ctx: Ctx) {
     let appId = context.getAppId()
 
     if (!env.isJest()) {
-      const App = require("./templates/BudibaseApp.svelte").default
       const plugins = objectStore.enrichPluginURLs(appInfo.usedPlugins)
       const { head, html, css } = App.render({
         metaImage:
@@ -200,7 +203,6 @@ export const serveApp = async function (ctx: Ctx) {
     }
   } catch (error) {
     if (!env.isJest()) {
-      const App = require("./templates/BudibaseApp.svelte").default
       const { head, html, css } = App.render({
         title: branding?.metaTitle,
         metaTitle: branding?.metaTitle,
diff --git a/packages/server/src/api/controllers/static/templates/Updating.svelte b/packages/server/src/api/controllers/static/templates/Updating.svelte
new file mode 100644
index 0000000000..5c7e23b5b3
--- /dev/null
+++ b/packages/server/src/api/controllers/static/templates/Updating.svelte
@@ -0,0 +1,8 @@
+<div class="loading">Updating the system...</div>
+
+<style>
+  .loading {
+    display: flex;
+    justify-content: center;
+  }
+</style>

From 0b46b6883df543b6b33206831d04a6817257638c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 19 Dec 2023 10:37:14 +0100
Subject: [PATCH 04/19] Remove unused regexparam

---
 packages/client/package.json | 1 -
 yarn.lock                    | 5 -----
 2 files changed, 6 deletions(-)

diff --git a/packages/client/package.json b/packages/client/package.json
index 39ddb4bd49..227c7b25d4 100644
--- a/packages/client/package.json
+++ b/packages/client/package.json
@@ -37,7 +37,6 @@
     "downloadjs": "1.4.7",
     "html5-qrcode": "^2.2.1",
     "leaflet": "^1.7.1",
-    "regexparam": "^1.3.0",
     "sanitize-html": "^2.7.0",
     "screenfull": "^6.0.1",
     "shortid": "^2.2.15",
diff --git a/yarn.lock b/yarn.lock
index 150585ce1a..85dd6ab251 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -18312,11 +18312,6 @@ regexparam@2.0.1:
   resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-2.0.1.tgz#c912f5dae371e3798100b3c9ce22b7414d0889fa"
   integrity sha512-zRgSaYemnNYxUv+/5SeoHI0eJIgTL/A2pUtXUPLHQxUldagouJ9p+K6IbIZ/JiQuCEv2E2B1O11SjVQy3aMCkw==
 
-regexparam@^1.3.0:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-1.3.0.tgz#2fe42c93e32a40eff6235d635e0ffa344b92965f"
-  integrity sha512-6IQpFBv6e5vz1QAqI+V4k8P2e/3gRrqfCJ9FI+O1FLQTO+Uz6RXZEZOPmTJ6hlGj7gkERzY5BRCv09whKP96/g==
-
 regexpu-core@^5.3.1:
   version "5.3.1"
   resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-5.3.1.tgz#66900860f88def39a5cb79ebd9490e84f17bcdfb"

From 75b8190c01898b3522ce5605191d6a9211cd09a1 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 19 Dec 2023 12:05:27 +0100
Subject: [PATCH 05/19] Move updating page to the client

---
 .../src/components/UpdatingApp.svelte}                    | 3 +++
 packages/client/src/index.js                              | 8 ++++++++
 packages/server/src/api/controllers/static/index.ts       | 7 +++----
 .../api/controllers/static/templates/BudibaseApp.svelte   | 6 ++++++
 4 files changed, 20 insertions(+), 4 deletions(-)
 rename packages/{server/src/api/controllers/static/templates/Updating.svelte => client/src/components/UpdatingApp.svelte} (86%)

diff --git a/packages/server/src/api/controllers/static/templates/Updating.svelte b/packages/client/src/components/UpdatingApp.svelte
similarity index 86%
rename from packages/server/src/api/controllers/static/templates/Updating.svelte
rename to packages/client/src/components/UpdatingApp.svelte
index 5c7e23b5b3..b24f591c5c 100644
--- a/packages/server/src/api/controllers/static/templates/Updating.svelte
+++ b/packages/client/src/components/UpdatingApp.svelte
@@ -1,3 +1,6 @@
+<script>
+</script>
+
 <div class="loading">Updating the system...</div>
 
 <style>
diff --git a/packages/client/src/index.js b/packages/client/src/index.js
index a3cb4206c3..f6ed23b2a9 100644
--- a/packages/client/src/index.js
+++ b/packages/client/src/index.js
@@ -1,4 +1,5 @@
 import ClientApp from "./components/ClientApp.svelte"
+import UpdatingApp from "./components/UpdatingApp.svelte"
 import {
   builderStore,
   appStore,
@@ -52,6 +53,13 @@ const loadBudibase = async () => {
     window["##BUDIBASE_APP_EMBEDDED##"] === "true"
   )
 
+  if (window.MIGRATING_APP) {
+    new UpdatingApp({
+      target: window.document.body,
+    })
+    return
+  }
+
   // Fetch environment info
   if (!get(environmentStore)?.loaded) {
     await environmentStore.actions.fetchEnvironment()
diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index c23ea349d4..0166f92478 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -156,10 +156,6 @@ export const serveApp = async function (ctx: Ctx) {
     appHbsPath = join(__dirname, "templates", "app.hbs")
   }
 
-  const App = !needMigrations
-    ? require("./templates/BudibaseApp.svelte").default
-    : require("./templates/Updating.svelte").default
-
   let db
   try {
     db = context.getAppDB({ skip_setup: true })
@@ -168,6 +164,7 @@ export const serveApp = async function (ctx: Ctx) {
 
     if (!env.isJest()) {
       const plugins = objectStore.enrichPluginURLs(appInfo.usedPlugins)
+      const App = require("./templates/BudibaseApp.svelte").default
       const { head, html, css } = App.render({
         metaImage:
           branding?.metaImageUrl ||
@@ -188,6 +185,7 @@ export const serveApp = async function (ctx: Ctx) {
           config?.logoUrl !== ""
             ? objectStore.getGlobalFileUrl("settings", "logoUrl")
             : "",
+        appMigrating: needMigrations,
       })
       const appHbs = loadHandlebarsFile(appHbsPath)
       ctx.body = await processString(appHbs, {
@@ -203,6 +201,7 @@ export const serveApp = async function (ctx: Ctx) {
     }
   } catch (error) {
     if (!env.isJest()) {
+      const App = require("./templates/BudibaseApp.svelte").default
       const { head, html, css } = App.render({
         title: branding?.metaTitle,
         metaTitle: branding?.metaTitle,
diff --git a/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte b/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte
index 32edb6dc7b..7819368fc0 100644
--- a/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte
+++ b/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte
@@ -8,6 +8,7 @@
 
   export let clientLibPath
   export let usedPlugins
+  export let appMigrating
 </script>
 
 <svelte:head>
@@ -110,6 +111,11 @@
   <script type="application/javascript">
     window.INIT_TIME = Date.now()
   </script>
+  {#if appMigrating}
+    <script type="application/javascript">
+      window.MIGRATING_APP = true
+    </script>
+  {/if}
   <script type="application/javascript" src={clientLibPath}>
   </script>
   <!-- Custom components need inserted after the core client library -->

From 94f0d38fe41d5a1dd5e4e3a1bc2ee940a257b188 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 19 Dec 2023 12:20:05 +0100
Subject: [PATCH 06/19] Implement refresh

---
 .../client/src/components/UpdatingApp.svelte  | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/packages/client/src/components/UpdatingApp.svelte b/packages/client/src/components/UpdatingApp.svelte
index b24f591c5c..2843133f2d 100644
--- a/packages/client/src/components/UpdatingApp.svelte
+++ b/packages/client/src/components/UpdatingApp.svelte
@@ -1,4 +1,30 @@
 <script>
+  import { API } from "../api"
+
+  const timeoutMs = 60000 // 1 minute
+  const loadTime = Date.now()
+
+  async function checkMigrationsFinished() {
+    setTimeout(async () => {
+      const response = await API.getMigrationStatus()
+      if (!response.migrated) {
+        if (loadTime + timeoutMs > Date.now()) {
+          return checkMigrationsFinished()
+        }
+
+        return migrationTimeout()
+      }
+
+      window.location.reload()
+    }, 1000)
+  }
+
+  checkMigrationsFinished()
+
+  function migrationTimeout() {
+    // TODO
+    alert("Something went wrong 💀")
+  }
 </script>
 
 <div class="loading">Updating the system...</div>

From ee404342231deedfd654f62aaadd20c3698f840f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 19 Dec 2023 12:24:10 +0100
Subject: [PATCH 07/19] Handle migration when app loaded

---
 packages/client/src/api/api.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/client/src/api/api.js b/packages/client/src/api/api.js
index 8488b702b6..ee33b1dbe0 100644
--- a/packages/client/src/api/api.js
+++ b/packages/client/src/api/api.js
@@ -77,4 +77,8 @@ export const API = createAPIClient({
     // Log all errors to console
     console.warn(`[Client] HTTP ${status} on ${method}:${url}\n\t${message}`)
   },
+  onMigrationDetected: appId => {
+    // We will force a reload, that will display the updating screen until the migration is running
+    window.location.reload()
+  },
 })

From 4e48d1d66eedc6f9ff4e41419ee27c3b14baf052 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 19 Dec 2023 12:27:26 +0100
Subject: [PATCH 08/19] Avoid multiple reloads

---
 packages/client/src/api/api.js | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/packages/client/src/api/api.js b/packages/client/src/api/api.js
index ee33b1dbe0..c5a1cd2775 100644
--- a/packages/client/src/api/api.js
+++ b/packages/client/src/api/api.js
@@ -78,7 +78,9 @@ export const API = createAPIClient({
     console.warn(`[Client] HTTP ${status} on ${method}:${url}\n\t${message}`)
   },
   onMigrationDetected: appId => {
-    // We will force a reload, that will display the updating screen until the migration is running
-    window.location.reload()
+    if (!window.MIGRATING_APP) {
+      // We will force a reload, that will display the updating screen until the migration is running
+      window.location.reload()
+    }
   },
 })

From 5dd3c7cada1bc0ef76ba8806f457b876aa5814e5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 19 Dec 2023 12:46:27 +0100
Subject: [PATCH 09/19] Lint

---
 packages/client/src/api/api.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/api/api.js b/packages/client/src/api/api.js
index c5a1cd2775..d4c8faa4d2 100644
--- a/packages/client/src/api/api.js
+++ b/packages/client/src/api/api.js
@@ -77,7 +77,7 @@ export const API = createAPIClient({
     // Log all errors to console
     console.warn(`[Client] HTTP ${status} on ${method}:${url}\n\t${message}`)
   },
-  onMigrationDetected: appId => {
+  onMigrationDetected: _appId => {
     if (!window.MIGRATING_APP) {
       // We will force a reload, that will display the updating screen until the migration is running
       window.location.reload()

From 746178f16950721c9b27d14c0cc8eb65a33237cb Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 4 Jan 2024 13:18:55 +0100
Subject: [PATCH 10/19] Update frontend

---
 .../client/src/components/UpdatingApp.svelte  | 32 ++++++++++++++++---
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/packages/client/src/components/UpdatingApp.svelte b/packages/client/src/components/UpdatingApp.svelte
index 2843133f2d..7d8d54df45 100644
--- a/packages/client/src/components/UpdatingApp.svelte
+++ b/packages/client/src/components/UpdatingApp.svelte
@@ -4,6 +4,8 @@
   const timeoutMs = 60000 // 1 minute
   const loadTime = Date.now()
 
+  let timedOut = false
+
   async function checkMigrationsFinished() {
     setTimeout(async () => {
       const response = await API.getMigrationStatus()
@@ -11,7 +13,6 @@
         if (loadTime + timeoutMs > Date.now()) {
           return checkMigrationsFinished()
         }
-
         return migrationTimeout()
       }
 
@@ -22,16 +23,39 @@
   checkMigrationsFinished()
 
   function migrationTimeout() {
-    // TODO
-    alert("Something went wrong 💀")
+    timedOut = true
   }
 </script>
 
-<div class="loading">Updating the system...</div>
+<div class="loading">
+  {#if !timedOut}
+    <p class="loading-message">
+      🛠️ We are updating the system. <br />Please wait, we will be back in a
+      second!
+    </p>
+  {:else}
+    <p class="loading-message timeout">
+      An error occurred. Please try again later.
+    </p>
+  {/if}
+</div>
 
 <style>
   .loading {
     display: flex;
     justify-content: center;
+    align-items: center;
+    justify-content: center;
+    flex-direction: column;
+    gap: var(--spacing-xl);
+    height: 100vh;
+    margin: 0;
+  }
+  .loading-message {
+    text-align: center;
+    font-size: 18px;
+  }
+  .timeout {
+    color: rgb(196, 46, 46);
   }
 </style>

From bfaab228d57429df634263d34c1017d3ecbf008a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 12:07:26 +0100
Subject: [PATCH 11/19] Use updating page from frontend

---
 .../client/src/components/UpdatingApp.svelte  | 60 +++----------------
 1 file changed, 7 insertions(+), 53 deletions(-)

diff --git a/packages/client/src/components/UpdatingApp.svelte b/packages/client/src/components/UpdatingApp.svelte
index 7d8d54df45..d93355394e 100644
--- a/packages/client/src/components/UpdatingApp.svelte
+++ b/packages/client/src/components/UpdatingApp.svelte
@@ -1,61 +1,15 @@
 <script>
+  import { Updating } from "@budibase/frontend-core"
   import { API } from "../api"
 
-  const timeoutMs = 60000 // 1 minute
-  const loadTime = Date.now()
-
-  let timedOut = false
-
-  async function checkMigrationsFinished() {
-    setTimeout(async () => {
-      const response = await API.getMigrationStatus()
-      if (!response.migrated) {
-        if (loadTime + timeoutMs > Date.now()) {
-          return checkMigrationsFinished()
-        }
-        return migrationTimeout()
-      }
-
-      window.location.reload()
-    }, 1000)
+  async function isMigrationDone() {
+    const response = await API.getMigrationStatus()
+    return response.migrated
   }
 
-  checkMigrationsFinished()
-
-  function migrationTimeout() {
-    timedOut = true
+  async function onMigrationDone() {
+    window.location.reload()
   }
 </script>
 
-<div class="loading">
-  {#if !timedOut}
-    <p class="loading-message">
-      🛠️ We are updating the system. <br />Please wait, we will be back in a
-      second!
-    </p>
-  {:else}
-    <p class="loading-message timeout">
-      An error occurred. Please try again later.
-    </p>
-  {/if}
-</div>
-
-<style>
-  .loading {
-    display: flex;
-    justify-content: center;
-    align-items: center;
-    justify-content: center;
-    flex-direction: column;
-    gap: var(--spacing-xl);
-    height: 100vh;
-    margin: 0;
-  }
-  .loading-message {
-    text-align: center;
-    font-size: 18px;
-  }
-  .timeout {
-    color: rgb(196, 46, 46);
-  }
-</style>
+<Updating {isMigrationDone} {onMigrationDone} />

From 9ebe93f71d3eade90062ca955292d6ef8f3368d6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 12:10:35 +0100
Subject: [PATCH 12/19] Clean

---
 packages/client/src/components/UpdatingApp.svelte | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/packages/client/src/components/UpdatingApp.svelte b/packages/client/src/components/UpdatingApp.svelte
index d93355394e..4177ea017f 100644
--- a/packages/client/src/components/UpdatingApp.svelte
+++ b/packages/client/src/components/UpdatingApp.svelte
@@ -1,15 +1,9 @@
 <script>
   import { Updating } from "@budibase/frontend-core"
-  import { API } from "../api"
-
-  async function isMigrationDone() {
-    const response = await API.getMigrationStatus()
-    return response.migrated
-  }
 
   async function onMigrationDone() {
     window.location.reload()
   }
 </script>
 
-<Updating {isMigrationDone} {onMigrationDone} />
+<Updating {onMigrationDone} />

From f77b81267933d2f868b4af188be73b2bbb70a879 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 12:15:12 +0100
Subject: [PATCH 13/19] Handle no migrations

---
 packages/server/src/api/controllers/static/index.ts | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index 0166f92478..5f383e837d 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -135,9 +135,14 @@ const requiresMigration = async (ctx: Ctx) => {
     ctx.throw("AppId could not be found")
   }
 
-  const latestAppliedMigration = await getAppMigrationVersion(appId)
+  const latestMigration = getLatestMigrationId()
+  if (!latestMigration) {
+    return false
+  }
 
-  const requiresMigrations = latestAppliedMigration !== getLatestMigrationId()
+  const latestMigrationApplied = await getAppMigrationVersion(appId)
+
+  const requiresMigrations = latestMigrationApplied !== latestMigration
   return requiresMigrations
 }
 

From f123a821bd95add40876b331aeae213e9152238d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 12:21:34 +0100
Subject: [PATCH 14/19] Styling

---
 packages/client/src/components/UpdatingApp.svelte | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/packages/client/src/components/UpdatingApp.svelte b/packages/client/src/components/UpdatingApp.svelte
index 4177ea017f..b493b7cd67 100644
--- a/packages/client/src/components/UpdatingApp.svelte
+++ b/packages/client/src/components/UpdatingApp.svelte
@@ -6,4 +6,12 @@
   }
 </script>
 
-<Updating {onMigrationDone} />
+<div class="updating">
+  <Updating {onMigrationDone} />
+</div>
+
+<style>
+  .updating {
+    font-family: var(--font-sans);
+  }
+</style>

From 2a07aa35f4da73e1447c45708eea967d516b33e5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 8 Jan 2024 13:48:50 +0100
Subject: [PATCH 15/19] Pass api call

---
 packages/client/src/components/UpdatingApp.svelte | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/packages/client/src/components/UpdatingApp.svelte b/packages/client/src/components/UpdatingApp.svelte
index b493b7cd67..74e5500715 100644
--- a/packages/client/src/components/UpdatingApp.svelte
+++ b/packages/client/src/components/UpdatingApp.svelte
@@ -1,5 +1,11 @@
 <script>
   import { Updating } from "@budibase/frontend-core"
+  import { API } from "../api"
+
+  async function isMigrationDone() {
+    const response = await API.getMigrationStatus()
+    return response.migrated
+  }
 
   async function onMigrationDone() {
     window.location.reload()
@@ -7,7 +13,7 @@
 </script>
 
 <div class="updating">
-  <Updating {onMigrationDone} />
+  <Updating {isMigrationDone} {onMigrationDone} />
 </div>
 
 <style>

From 6df4bf84df1d166af6c2ce206d891b764cdf1aa2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 4 Jan 2024 15:42:16 +0100
Subject: [PATCH 16/19] Lint

---
 packages/server/src/api/controllers/datasource.ts | 7 ++-----
 packages/server/src/integrations/postgres.ts      | 4 ++--
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts
index 0a7430aa94..b794312fb6 100644
--- a/packages/server/src/api/controllers/datasource.ts
+++ b/packages/server/src/api/controllers/datasource.ts
@@ -57,17 +57,14 @@ async function getAndMergeDatasource(datasource: Datasource) {
 
 async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
   const connector = (await getConnector(datasource)) as DatasourcePlus
-  return await connector.buildSchema(
-    datasource._id!,
-    datasource.entities! as Record<string, Table>
-  )
+  return await connector.buildSchema(datasource._id!, datasource.entities!)
 }
 
 async function buildFilteredSchema(
   datasource: Datasource,
   filter?: string[]
 ): Promise<Schema> {
-  let schema = await buildSchemaHelper(datasource)
+  const schema = await buildSchemaHelper(datasource)
   if (!filter) {
     return schema
   }
diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts
index de3bf0e59e..2d61d05a01 100644
--- a/packages/server/src/integrations/postgres.ts
+++ b/packages/server/src/integrations/postgres.ts
@@ -362,8 +362,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
         })
       }
 
-      let finalizedTables = finaliseExternalTables(tables, entities)
-      let errors = checkExternalTables(finalizedTables)
+      const finalizedTables = finaliseExternalTables(tables, entities)
+      const errors = checkExternalTables(finalizedTables)
       return { tables: finalizedTables, errors }
     } catch (err) {
       // @ts-ignore

From dd77779bc16fb1e7794afbec89497ac0d4745f2b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 5 Jan 2024 15:14:31 +0100
Subject: [PATCH 17/19] Move code to sdk

---
 .../server/src/api/controllers/datasource.ts  | 80 +++----------------
 .../server/src/sdk/app/datasources/index.ts   |  2 +
 .../server/src/sdk/app/datasources/plus.ts    | 71 ++++++++++++++++
 3 files changed, 86 insertions(+), 67 deletions(-)
 create mode 100644 packages/server/src/sdk/app/datasources/plus.ts

diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts
index b794312fb6..364490e5f5 100644
--- a/packages/server/src/api/controllers/datasource.ts
+++ b/packages/server/src/api/controllers/datasource.ts
@@ -14,10 +14,7 @@ import {
   DatasourcePlus,
   FetchDatasourceInfoRequest,
   FetchDatasourceInfoResponse,
-  IntegrationBase,
-  Schema,
   SourceName,
-  Table,
   UpdateDatasourceResponse,
   UserCtx,
   VerifyDatasourceRequest,
@@ -28,62 +25,6 @@ import { builderSocket } from "../../websockets"
 import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets"
 import { isEqual } from "lodash"
 
-async function getConnector(
-  datasource: Datasource
-): Promise<IntegrationBase | DatasourcePlus> {
-  const Connector = await getIntegration(datasource.source)
-  // can't enrich if it doesn't have an ID yet
-  if (datasource._id) {
-    datasource = await sdk.datasources.enrich(datasource)
-  }
-  // Connect to the DB and build the schema
-  return new Connector(datasource.config)
-}
-
-async function getAndMergeDatasource(datasource: Datasource) {
-  let existingDatasource: undefined | Datasource
-  if (datasource._id) {
-    existingDatasource = await sdk.datasources.get(datasource._id)
-  }
-  let enrichedDatasource = datasource
-  if (existingDatasource) {
-    enrichedDatasource = sdk.datasources.mergeConfigs(
-      datasource,
-      existingDatasource
-    )
-  }
-  return await sdk.datasources.enrich(enrichedDatasource)
-}
-
-async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
-  const connector = (await getConnector(datasource)) as DatasourcePlus
-  return await connector.buildSchema(datasource._id!, datasource.entities!)
-}
-
-async function buildFilteredSchema(
-  datasource: Datasource,
-  filter?: string[]
-): Promise<Schema> {
-  const schema = await buildSchemaHelper(datasource)
-  if (!filter) {
-    return schema
-  }
-
-  let filteredSchema: Schema = { tables: {}, errors: {} }
-  for (let key in schema.tables) {
-    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
-      filteredSchema.tables[key] = schema.tables[key]
-    }
-  }
-
-  for (let key in schema.errors) {
-    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
-      filteredSchema.errors[key] = schema.errors[key]
-    }
-  }
-  return filteredSchema
-}
-
 export async function fetch(ctx: UserCtx) {
   ctx.body = await sdk.datasources.fetch()
 }
@@ -92,8 +33,7 @@ export async function verify(
   ctx: UserCtx<VerifyDatasourceRequest, VerifyDatasourceResponse>
 ) {
   const { datasource } = ctx.request.body
-  const enrichedDatasource = await getAndMergeDatasource(datasource)
-  const connector = await getConnector(enrichedDatasource)
+  const connector = await sdk.datasources.getConnector(datasource)
   if (!connector.testConnection) {
     ctx.throw(400, "Connection information verification not supported")
   }
@@ -109,8 +49,9 @@ export async function information(
   ctx: UserCtx<FetchDatasourceInfoRequest, FetchDatasourceInfoResponse>
 ) {
   const { datasource } = ctx.request.body
-  const enrichedDatasource = await getAndMergeDatasource(datasource)
-  const connector = (await getConnector(enrichedDatasource)) as DatasourcePlus
+  const connector = (await sdk.datasources.getConnector(
+    datasource
+  )) as DatasourcePlus
   if (!connector.getTableNames) {
     ctx.throw(400, "Table name fetching not supported by datasource")
   }
@@ -125,7 +66,10 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
   const tablesFilter = ctx.request.body.tablesFilter
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
 
-  const { tables, errors } = await buildFilteredSchema(datasource, tablesFilter)
+  const { tables, errors } = await sdk.datasources.buildFilteredSchema(
+    datasource,
+    tablesFilter
+  )
   datasource.entities = tables
 
   setDefaultDisplayColumns(datasource)
@@ -277,7 +221,10 @@ export async function save(
 
   let errors: Record<string, string> = {}
   if (fetchSchema) {
-    const schema = await buildFilteredSchema(datasource, tablesFilter)
+    const schema = await sdk.datasources.buildFilteredSchema(
+      datasource,
+      tablesFilter
+    )
     datasource.entities = schema.tables
     setDefaultDisplayColumns(datasource)
     errors = schema.errors
@@ -381,8 +328,7 @@ export async function query(ctx: UserCtx) {
 
 export async function getExternalSchema(ctx: UserCtx) {
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
-  const enrichedDatasource = await getAndMergeDatasource(datasource)
-  const connector = await getConnector(enrichedDatasource)
+  const connector = await sdk.datasources.getConnector(datasource)
 
   if (!connector.getExternalSchema) {
     ctx.throw(400, "Datasource does not support exporting external schema")
diff --git a/packages/server/src/sdk/app/datasources/index.ts b/packages/server/src/sdk/app/datasources/index.ts
index 1ce6b0e689..8f06e989d3 100644
--- a/packages/server/src/sdk/app/datasources/index.ts
+++ b/packages/server/src/sdk/app/datasources/index.ts
@@ -1,5 +1,7 @@
 import * as datasources from "./datasources"
+import * as plus from "./plus"
 
 export default {
   ...datasources,
+  ...plus,
 }
diff --git a/packages/server/src/sdk/app/datasources/plus.ts b/packages/server/src/sdk/app/datasources/plus.ts
new file mode 100644
index 0000000000..65b908526c
--- /dev/null
+++ b/packages/server/src/sdk/app/datasources/plus.ts
@@ -0,0 +1,71 @@
+import {
+  Datasource,
+  DatasourcePlus,
+  IntegrationBase,
+  Schema,
+} from "@budibase/types"
+import * as datasources from "./datasources"
+import { getIntegration } from "../../../integrations"
+
+export async function buildFilteredSchema(
+  datasource: Datasource,
+  filter?: string[]
+): Promise<Schema> {
+  const schema = await buildSchemaHelper(datasource)
+  if (!filter) {
+    return schema
+  }
+
+  let filteredSchema: Schema = { tables: {}, errors: {} }
+  for (let key in schema.tables) {
+    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
+      filteredSchema.tables[key] = schema.tables[key]
+    }
+  }
+
+  for (let key in schema.errors) {
+    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
+      filteredSchema.errors[key] = schema.errors[key]
+    }
+  }
+  return filteredSchema
+}
+
+async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
+  const connector = (await getConnector(datasource)) as DatasourcePlus
+  const externalSchema = await connector.buildSchema(
+    datasource._id!,
+    datasource.entities!
+  )
+  return externalSchema
+}
+
+export async function getConnector(
+  datasource: Datasource
+): Promise<IntegrationBase | DatasourcePlus> {
+  const Connector = await getIntegration(datasource.source)
+
+  datasource = await getAndMergeDatasource(datasource)
+
+  // can't enrich if it doesn't have an ID yet
+  if (datasource._id) {
+    datasource = await datasources.enrich(datasource)
+  }
+  // Connect to the DB and build the schema
+  return new Connector(datasource.config)
+}
+
+async function getAndMergeDatasource(datasource: Datasource) {
+  let existingDatasource: undefined | Datasource
+  if (datasource._id) {
+    existingDatasource = await datasources.get(datasource._id)
+  }
+  let enrichedDatasource = datasource
+  if (existingDatasource) {
+    enrichedDatasource = datasources.mergeConfigs(
+      datasource,
+      existingDatasource
+    )
+  }
+  return await datasources.enrich(enrichedDatasource)
+}

From 1f9ee5a2066e18331fcd5967b5f32b5e2ecfc37e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 5 Jan 2024 16:27:45 +0100
Subject: [PATCH 18/19] Clean code

---
 .../server/src/sdk/app/datasources/plus.ts     | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/packages/server/src/sdk/app/datasources/plus.ts b/packages/server/src/sdk/app/datasources/plus.ts
index 65b908526c..0049730119 100644
--- a/packages/server/src/sdk/app/datasources/plus.ts
+++ b/packages/server/src/sdk/app/datasources/plus.ts
@@ -45,7 +45,7 @@ export async function getConnector(
 ): Promise<IntegrationBase | DatasourcePlus> {
   const Connector = await getIntegration(datasource.source)
 
-  datasource = await getAndMergeDatasource(datasource)
+  datasource = await mergeAndEnrich(datasource)
 
   // can't enrich if it doesn't have an ID yet
   if (datasource._id) {
@@ -55,17 +55,11 @@ export async function getConnector(
   return new Connector(datasource.config)
 }
 
-async function getAndMergeDatasource(datasource: Datasource) {
-  let existingDatasource: undefined | Datasource
+async function mergeAndEnrich(datasource: Datasource) {
   if (datasource._id) {
-    existingDatasource = await datasources.get(datasource._id)
+    const existingDatasource = await datasources.get(datasource._id)
+
+    datasource = datasources.mergeConfigs(datasource, existingDatasource)
   }
-  let enrichedDatasource = datasource
-  if (existingDatasource) {
-    enrichedDatasource = datasources.mergeConfigs(
-      datasource,
-      existingDatasource
-    )
-  }
-  return await datasources.enrich(enrichedDatasource)
+  return await datasources.enrich(datasource)
 }

From 498137ba7c8a1a40d6382f2f81abc0ddeac43af5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 5 Jan 2024 16:39:55 +0100
Subject: [PATCH 19/19] Split back

---
 packages/server/src/api/controllers/datasource.ts | 15 ++++++++++++---
 packages/server/src/sdk/app/datasources/plus.ts   |  5 +----
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts
index 364490e5f5..8c177f1704 100644
--- a/packages/server/src/api/controllers/datasource.ts
+++ b/packages/server/src/api/controllers/datasource.ts
@@ -33,7 +33,10 @@ export async function verify(
   ctx: UserCtx<VerifyDatasourceRequest, VerifyDatasourceResponse>
 ) {
   const { datasource } = ctx.request.body
-  const connector = await sdk.datasources.getConnector(datasource)
+  const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(
+    datasource
+  )
+  const connector = await sdk.datasources.getConnector(enrichedDatasource)
   if (!connector.testConnection) {
     ctx.throw(400, "Connection information verification not supported")
   }
@@ -49,8 +52,11 @@ export async function information(
   ctx: UserCtx<FetchDatasourceInfoRequest, FetchDatasourceInfoResponse>
 ) {
   const { datasource } = ctx.request.body
-  const connector = (await sdk.datasources.getConnector(
+  const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(
     datasource
+  )
+  const connector = (await sdk.datasources.getConnector(
+    enrichedDatasource
   )) as DatasourcePlus
   if (!connector.getTableNames) {
     ctx.throw(400, "Table name fetching not supported by datasource")
@@ -328,7 +334,10 @@ export async function query(ctx: UserCtx) {
 
 export async function getExternalSchema(ctx: UserCtx) {
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
-  const connector = await sdk.datasources.getConnector(datasource)
+  const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(
+    datasource
+  )
+  const connector = await sdk.datasources.getConnector(enrichedDatasource)
 
   if (!connector.getExternalSchema) {
     ctx.throw(400, "Datasource does not support exporting external schema")
diff --git a/packages/server/src/sdk/app/datasources/plus.ts b/packages/server/src/sdk/app/datasources/plus.ts
index 0049730119..117d19a6a7 100644
--- a/packages/server/src/sdk/app/datasources/plus.ts
+++ b/packages/server/src/sdk/app/datasources/plus.ts
@@ -44,9 +44,6 @@ export async function getConnector(
   datasource: Datasource
 ): Promise<IntegrationBase | DatasourcePlus> {
   const Connector = await getIntegration(datasource.source)
-
-  datasource = await mergeAndEnrich(datasource)
-
   // can't enrich if it doesn't have an ID yet
   if (datasource._id) {
     datasource = await datasources.enrich(datasource)
@@ -55,7 +52,7 @@ export async function getConnector(
   return new Connector(datasource.config)
 }
 
-async function mergeAndEnrich(datasource: Datasource) {
+export async function getAndMergeDatasource(datasource: Datasource) {
   if (datasource._id) {
     const existingDatasource = await datasources.get(datasource._id)