Merge branch 'master' of github.com:budibase/budibase into remove-jest-testcontainers

.eslintignore

@@ -13,3 +13,4 @@ packages/account-portal/packages/server/build
 packages/account-portal/packages/ui/.routify
 packages/account-portal/packages/ui/build
 **/*.ivm.bundle.js
+packages/server/build/oldClientVersions/**/**

.github/workflows/budibase_ci.yml

@@ -138,6 +138,8 @@ jobs:
 
   test-server:
     runs-on: ubuntu-latest
+    env:
+      DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -151,7 +153,19 @@ jobs:
         with:
           node-version: 20.x
           cache: yarn
+
+      - name: Pull testcontainers images
+        run: |
+          docker pull mcr.microsoft.com/mssql/server:2022-latest
+          docker pull mysql:8.3
+          docker pull postgres:16.1-bullseye
+          docker pull mongo:7.0-jammy
+          docker pull mariadb:lts
+          docker pull testcontainers/ryuk:0.3.0
+          docker pull budibase/couchdb
+
       - run: yarn --frozen-lockfile
+
       - name: Test server
         run: |
           if ${{ env.USE_NX_AFFECTED }}; then

.gitignore

@@ -5,6 +5,9 @@ packages/server/runtime_apps/
 bb-airgapped.tar.gz
 *.iml
 
+packages/server/build/oldClientVersions/**/*
+packages/builder/src/components/deploy/clientVersions.json
+
 # Logs
 logs
 *.log

package.json

@@ -31,6 +31,7 @@
   },
   "scripts": {
     "preinstall": "node scripts/syncProPackage.js",
+    "get-past-client-version": "node scripts/getPastClientVersion.js",
     "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
     "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream",
     "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",

packages/builder/src/components/deploy/RevertModalVersionSelect.svelte

@@ -0,0 +1,33 @@
+<script>
+  import { API } from "api"
+  import clientVersions from "./clientVersions.json"
+  import { appStore } from "stores/builder"
+  import { Select } from "@budibase/bbui"
+
+  export let revertableVersion
+  $: appId = $appStore.appId
+
+  const handleChange = e => {
+    const value = e.detail
+    if (value == null) return
+
+    API.setRevertableVersion(appId, value)
+  }
+</script>
+
+<div class="select">
+  <Select
+    autoWidth
+    value={revertableVersion}
+    options={clientVersions}
+    on:change={handleChange}
+    footer={"Older versions of the Budibase client can be acquired using `yarn get-past-client-version x.x.x`. This toggle is only available in dev mode."}
+  />
+</div>
+
+<style>
+  .select {
+    width: 120px;
+    display: inline-block;
+  }
+</style>

packages/builder/src/components/deploy/VersionModal.svelte

@@ -1,4 +1,5 @@
 <script>
+  import { admin } from "stores/portal"
   import {
     Modal,
     notifications,
@@ -9,6 +10,7 @@
   } from "@budibase/bbui"
   import { appStore, initialise } from "stores/builder"
   import { API } from "api"
+  import RevertModalVersionSelect from "./RevertModalVersionSelect.svelte"
 
   export function show() {
     updateModal.show()
@@ -28,7 +30,9 @@
     $appStore.upgradableVersion &&
     $appStore.version &&
     $appStore.upgradableVersion !== $appStore.version
-  $: revertAvailable = $appStore.revertableVersion != null
+  $: revertAvailable =
+    $appStore.revertableVersion != null ||
+    ($admin.isDev && $appStore.version === "0.0.0")
 
   const refreshAppPackage = async () => {
     try {
@@ -62,7 +66,9 @@
       // Don't wait for the async refresh, since this causes modal flashing
       refreshAppPackage()
       notifications.success(
-        `App reverted successfully to version ${$appStore.revertableVersion}`
+        $appStore.revertableVersion
+          ? `App reverted successfully to version ${$appStore.revertableVersion}`
+          : "App reverted successfully"
       )
     } catch (err) {
       notifications.error(`Error reverting app: ${err}`)
@@ -103,7 +109,13 @@
     {#if revertAvailable}
       <Body size="S">
         You can revert this app to version
+        {#if $admin.isDev}
+          <RevertModalVersionSelect
+            revertableVersion={$appStore.revertableVersion}
+          />
+        {:else}
           <b>{$appStore.revertableVersion}</b>
+        {/if}
         if you're experiencing issues with the current version.
       </Body>
     {/if}

packages/builder/src/components/deploy/clientVersions.json

@@ -0,0 +1 @@
+[]

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte

@@ -14,17 +14,11 @@
     snippets,
   } from "stores/builder"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
-  import {
-    ProgressCircle,
-    Layout,
-    Heading,
-    Body,
-    Icon,
-    notifications,
-  } from "@budibase/bbui"
+  import { Layout, Heading, Body, Icon, notifications } from "@budibase/bbui"
   import ErrorSVG from "@budibase/frontend-core/assets/error.svg?raw"
   import { findComponent, findComponentPath } from "helpers/components"
   import { isActive, goto } from "@roxi/routify"
+  import { ClientAppSkeleton } from "@budibase/frontend-core"
 
   let iframe
   let layout
@@ -254,8 +248,16 @@
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <div class="component-container">
   {#if loading}
-    <div class="center">
-      <ProgressCircle />
+    <div
+      class={`loading ${$themeStore.theme}`}
+      class:tablet={$previewStore.previewDevice === "tablet"}
+      class:mobile={$previewStore.previewDevice === "mobile"}
+    >
+      <ClientAppSkeleton
+        sideNav={$navigationStore?.navigation === "Left"}
+        hideFooter
+        hideDevTools
+      />
     </div>
   {:else if error}
     <div class="center error">
@@ -272,8 +274,6 @@
     bind:this={iframe}
     src="/app/preview"
     class:hidden={loading || error}
-    class:tablet={$previewStore.previewDevice === "tablet"}
-    class:mobile={$previewStore.previewDevice === "mobile"}
   />
   <div
     class="add-component"
@@ -293,6 +293,25 @@
 />
 
 <style>
+  .loading {
+    position: absolute;
+    container-type: inline-size;
+    width: 100%;
+    height: 100%;
+    border: 2px solid transparent;
+    box-sizing: border-box;
+  }
+
+  .loading.tablet {
+    width: calc(1024px + 6px);
+    max-height: calc(768px + 6px);
+  }
+
+  .loading.mobile {
+    width: calc(390px + 6px);
+    max-height: calc(844px + 6px);
+  }
+
   .component-container {
     grid-row-start: middle;
     grid-column-start: middle;

packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte

@@ -1,6 +1,12 @@
 <script>
+  import { onMount, onDestroy } from "svelte"
   import { params, goto } from "@roxi/routify"
-  import { auth, sideBarCollapsed, enrichedApps } from "stores/portal"
+  import {
+    licensing,
+    auth,
+    sideBarCollapsed,
+    enrichedApps,
+  } from "stores/portal"
   import AppRowContext from "components/start/AppRowContext.svelte"
   import FavouriteAppButton from "../FavouriteAppButton.svelte"
   import {
@@ -14,12 +20,17 @@
   import { sdk } from "@budibase/shared-core"
   import { API } from "api"
   import ErrorSVG from "./ErrorSVG.svelte"
+  import { ClientAppSkeleton } from "@budibase/frontend-core"
 
   $: app = $enrichedApps.find(app => app.appId === $params.appId)
   $: iframeUrl = getIframeURL(app)
   $: isBuilder = sdk.users.isBuilder($auth.user, app?.devId)
 
+  let loading = true
+
   const getIframeURL = app => {
+    loading = true
+
     if (app.status === "published") {
       return `/app${app.url}`
     }
@@ -37,6 +48,20 @@
   }
 
   $: fetchScreens(app?.devId)
+
+  const receiveMessage = async message => {
+    if (message.data.type === "docLoaded") {
+      loading = false
+    }
+  }
+
+  onMount(() => {
+    window.addEventListener("message", receiveMessage)
+  })
+
+  onDestroy(() => {
+    window.removeEventListener("message", receiveMessage)
+  })
 </script>
 
 <!-- svelte-ignore a11y-click-events-have-key-events -->
@@ -108,7 +133,24 @@
       </Body>
     </div>
   {:else}
-    <iframe src={iframeUrl} title={app.name} />
+    <div
+      class:hide={!loading || !app?.features?.skeletonLoader}
+      class="loading"
+    >
+      <div class={`loadingThemeWrapper ${app.theme}`}>
+        <ClientAppSkeleton
+          noAnimation
+          hideDevTools={app?.status === "published"}
+          sideNav={app?.navigation.navigation === "Left"}
+          hideFooter={$licensing.brandingEnabled}
+        />
+      </div>
+    </div>
+    <iframe
+      class:hide={loading && app?.features?.skeletonLoader}
+      src={iframeUrl}
+      title={app.name}
+    />
   {/if}
 </div>
 
@@ -139,6 +181,23 @@
     flex: 0 0 50px;
   }
 
+  .loading {
+    height: 100%;
+    border: 1px solid var(--spectrum-global-color-gray-300);
+    border-radius: var(--spacing-s);
+    overflow: hidden;
+  }
+  .loadingThemeWrapper {
+    height: 100%;
+    container-type: inline-size;
+  }
+
+  .hide {
+    visibility: hidden;
+    height: 0;
+    border: none;
+  }
+
   iframe {
     flex: 1 1 auto;
     border-radius: var(--spacing-s);

packages/client/manifest.json

@@ -10,7 +10,8 @@
     "rowSelection": true,
     "continueIfAction": true,
     "showNotificationAction": true,
-    "sidePanel": true
+    "sidePanel": true,
+    "skeletonLoader": true
   },
   "layout": {
     "name": "Layout",

packages/client/src/components/ClientApp.svelte

@@ -7,6 +7,7 @@
   import Component from "./Component.svelte"
   import SDK from "sdk"
   import {
+    featuresStore,
     createContextStore,
     initialise,
     screenStore,
@@ -38,7 +39,6 @@
   import DevTools from "components/devtools/DevTools.svelte"
   import FreeFooter from "components/FreeFooter.svelte"
   import MaintenanceScreen from "components/MaintenanceScreen.svelte"
-  import licensing from "../licensing"
   import SnippetsProvider from "./context/SnippetsProvider.svelte"
 
   // Provide contexts
@@ -83,11 +83,18 @@
     }
   }
 
+  let fontsLoaded = false
+
   // Load app config
   onMount(async () => {
+    document.fonts.ready.then(() => {
+      fontsLoaded = true
+    })
+
     await initialise()
     await authStore.actions.fetchUser()
     dataLoaded = true
+
     if (get(builderStore).inBuilder) {
       builderStore.actions.notifyLoaded()
     } else {
@@ -96,6 +103,12 @@
       })
     }
   })
+
+  $: {
+    if (dataLoaded && fontsLoaded) {
+      document.getElementById("clientAppSkeletonLoader")?.remove()
+    }
+  }
 </script>
 
 <svelte:head>
@@ -106,13 +119,13 @@
   {/if}
 </svelte:head>
 
-{#if dataLoaded}
 <div
   id="spectrum-root"
   lang="en"
   dir="ltr"
   class="spectrum spectrum--medium {$themeStore.baseTheme} {$themeStore.theme}"
   class:builder={$builderStore.inBuilder}
+  class:show={fontsLoaded && dataLoaded}
 >
   {#if $environmentStore.maintenance.length > 0}
     <MaintenanceScreen maintenanceList={$environmentStore.maintenance} />
@@ -214,7 +227,7 @@
                       {/if}
                     </div>
 
-                      {#if !$builderStore.inBuilder && licensing.logoEnabled()}
+                    {#if !$builderStore.inBuilder && $featuresStore.logoEnabled}
                       <FreeFooter />
                     {/if}
                   </div>
@@ -240,14 +253,14 @@
   {/if}
 </div>
 <KeyboardManager />
-{/if}
 
 <style>
   #spectrum-root {
+    height: 0;
+    visibility: hidden;
     padding: 0;
     margin: 0;
     overflow: hidden;
-    height: 100%;
     width: 100%;
     display: flex;
     flex-direction: row;
@@ -268,6 +281,11 @@
     background-color: transparent;
   }
 
+  #spectrum-root.show {
+    height: 100%;
+    visibility: visible;
+  }
+
   #app-root {
     overflow: hidden;
     height: 100%;

packages/client/src/components/FreeFooter.svelte

@@ -13,6 +13,7 @@
 
 <style>
   .free-footer {
+    min-height: 51px;
     flex: 0 0 auto;
     padding: 16px 20px;
     border-top: 1px solid var(--spectrum-global-color-gray-300);

packages/client/src/licensing/features.js

@@ -1,5 +0,0 @@
-import { isFreePlan } from "./utils.js"
-
-export const logoEnabled = () => {
-  return isFreePlan()
-}

packages/client/src/licensing/index.js

@@ -1,7 +0,0 @@
-import * as features from "./features"
-
-const licensing = {
-  ...features,
-}
-
-export default licensing

packages/client/src/licensing/utils.js

@@ -1,32 +0,0 @@
-import { authStore } from "../stores/auth.js"
-import { appStore } from "../stores/app.js"
-import { get } from "svelte/store"
-import { Constants } from "@budibase/frontend-core"
-
-const getUserLicense = () => {
-  const user = get(authStore)
-  if (user) {
-    return user.license
-  }
-}
-
-const getAppLicenseType = () => {
-  const appDef = get(appStore)
-  if (appDef?.licenseType) {
-    return appDef.licenseType
-  }
-}
-
-export const isFreePlan = () => {
-  let licenseType = getAppLicenseType()
-  if (!licenseType) {
-    const license = getUserLicense()
-    licenseType = license?.plan?.type
-  }
-  if (licenseType) {
-    return licenseType === Constants.PlanType.FREE
-  } else {
-    // safety net - no license means free plan
-    return true
-  }
-}

packages/client/src/stores/features.js

@@ -0,0 +1,42 @@
+import { derived } from "svelte/store"
+import { appStore } from "./app"
+import { authStore } from "./auth"
+import { Constants } from "@budibase/frontend-core"
+
+const createFeaturesStore = () => {
+  return derived([authStore, appStore], ([$authStore, $appStore]) => {
+    const getUserLicense = () => {
+      const user = $authStore
+      if (user) {
+        return user.license
+      }
+    }
+
+    const getAppLicenseType = () => {
+      const appDef = $appStore
+      if (appDef?.licenseType) {
+        return appDef.licenseType
+      }
+    }
+
+    const isFreePlan = () => {
+      let licenseType = getAppLicenseType()
+      if (!licenseType) {
+        const license = getUserLicense()
+        licenseType = license?.plan?.type
+      }
+      if (licenseType) {
+        return licenseType === Constants.PlanType.FREE
+      } else {
+        // safety net - no license means free plan
+        return true
+      }
+    }
+
+    return {
+      logoEnabled: isFreePlan(),
+    }
+  })
+}
+
+export const featuresStore = createFeaturesStore()

packages/client/src/stores/index.js

@@ -31,6 +31,7 @@ export { hoverStore } from "./hover"
 
 // Context stores are layered and duplicated, so it is not a singleton
 export { createContextStore } from "./context"
+export { featuresStore } from "./features"
 
 // Initialises an app by loading screens and routes
 export { initialise } from "./initialise"

packages/frontend-core/src/api/app.js

@@ -197,4 +197,13 @@ export const buildAppEndpoints = API => ({
       url: `/api/applications/${appId}/sample`,
     })
   },
+
+  setRevertableVersion: async (appId, revertableVersion) => {
+    return await API.post({
+      url: `/api/applications/${appId}/setRevertableVersion`,
+      body: {
+        revertableVersion,
+      },
+    })
+  },
 })

packages/frontend-core/src/components/ClientAppSkeleton.svelte

@@ -0,0 +1,244 @@
+<script>
+  export let sideNav = false
+  export let hideDevTools = false
+  export let hideFooter = false
+  export let noAnimation = false
+</script>
+
+<div class:sideNav id="clientAppSkeletonLoader" class="skeleton">
+  <div class="animation" class:noAnimation />
+
+  {#if !hideDevTools}
+    <div class="devTools" />
+  {/if}
+  <div class="main">
+    <div class="nav" />
+    <div class="body">
+      <div class="bodyVerticalPadding" />
+      <div class="bodyHorizontal">
+        <div class="bodyHorizontalPadding" />
+        <svg
+          class="svg"
+          xmlns="http://www.w3.org/2000/svg"
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          width="240"
+          height="256"
+        >
+          <mask id="mask">
+            <rect x="0" y="0" width="240" height="256" fill="white" />
+            <rect x="0" y="0" width="240" height="32" rx="6" fill="black" />
+            <rect x="0" y="56" width="240" height="32" rx="6" fill="black" />
+            <rect x="0" y="112" width="240" height="32" rx="6" fill="black" />
+            <rect x="0" y="168" width="240" height="32" rx="6" fill="black" />
+            <rect x="71" y="224" width="98" height="32" rx="6" fill="black" />
+          </mask>
+
+          <rect
+            x="0"
+            y="0"
+            width="240"
+            height="256"
+            fill="black"
+            mask="url(#mask)"
+          />
+        </svg>
+        <div class="bodyHorizontalPadding" />
+      </div>
+      <div class="bodyVerticalPadding" />
+    </div>
+  </div>
+  {#if !hideFooter}
+    <div class="footer" />
+  {/if}
+</div>
+
+<style>
+  .skeleton {
+    position: relative;
+    height: 100%;
+    display: flex;
+    flex-direction: column;
+    border-radius: 4px;
+    overflow: hidden;
+    background-color: var(--spectrum-global-color-gray-200);
+  }
+
+  .animation {
+    position: absolute;
+    height: 100%;
+    width: 100%;
+    background: linear-gradient(
+      to right,
+      transparent 0%,
+      var(--spectrum-global-color-gray-300) 20%,
+      transparent 40%,
+      transparent 100%
+    );
+    animation-duration: 1.3s;
+    animation-fill-mode: forwards;
+    animation-iteration-count: infinite;
+    animation-name: shimmer;
+    animation-timing-function: linear;
+  }
+
+  .noAnimation {
+    animation-name: none;
+    background: transparent;
+  }
+
+  .devTools {
+    display: flex;
+    box-sizing: border-box;
+    background-color: black;
+    height: 60px;
+    padding: 1px 24px 1px 20px;
+    display: flex;
+    align-items: center;
+    z-index: 1;
+    flex-shrink: 0;
+
+    color: white;
+    mix-blend-mode: multiply;
+    background: rgb(0 0 0);
+    font-size: 30px;
+    font-family: Source Sans Pro;
+    -webkit-font-smoothing: antialiased;
+  }
+
+  .main {
+    height: 100%;
+    display: flex;
+    flex-direction: column;
+  }
+
+  @media (max-width: 720px) {
+    #clientAppSkeletonLoader .main {
+      flex-direction: column;
+      width: initial;
+    }
+  }
+
+  @container (max-width: 720px) {
+    #clientAppSkeletonLoader .main {
+      flex-direction: column;
+      width: initial;
+    }
+  }
+
+  .sideNav .main {
+    flex-direction: row;
+    width: 100%;
+  }
+
+  .nav {
+    flex-shrink: 0;
+    width: 100%;
+    height: 141px;
+    background-color: transparent;
+  }
+
+  @media (max-width: 720px) {
+    #clientAppSkeletonLoader .nav {
+      height: 61px;
+      width: initial;
+    }
+  }
+
+  @container (max-width: 720px) {
+    #clientAppSkeletonLoader .nav {
+      height: 61px;
+      width: initial;
+    }
+  }
+
+  .sideNav .nav {
+    height: 100%;
+    width: 251px;
+  }
+
+  .body {
+    z-index: 2;
+    display: flex;
+    flex-direction: column;
+    height: 100%;
+    position: relative;
+  }
+
+  @media (max-width: 720px) {
+    #clientAppSkeletonLoader .body {
+      width: initial;
+      height: 100%;
+    }
+  }
+
+  @container (max-width: 720px) {
+    #clientAppSkeletonLoader .body {
+      width: initial;
+      height: 100%;
+    }
+  }
+
+  .sideNav .body {
+    width: 100%;
+    height: initial;
+  }
+
+  .body :global(svg > rect) {
+    fill: var(--spectrum-alias-background-color-primary);
+  }
+
+  .body :global(svg) {
+    flex-shrink: 0;
+  }
+
+  .bodyHorizontal {
+    display: flex;
+    flex-shrink: 0;
+  }
+
+  .bodyHorizontalPadding {
+    height: 100%;
+    flex-grow: 1;
+    background-color: var(--spectrum-alias-background-color-primary);
+  }
+
+  .bodyVerticalPadding {
+    width: 100%;
+    flex-grow: 1;
+    background-color: var(--spectrum-alias-background-color-primary);
+  }
+
+  .footer {
+    flex-shrink: 0;
+    box-sizing: border-box;
+    z-index: 1;
+    height: 52px;
+    width: 100%;
+  }
+
+  @media (max-width: 720px) {
+    #clientAppSkeletonLoader .footer {
+      border-top: none;
+    }
+  }
+
+  @container (max-width: 720px) {
+    #clientAppSkeletonLoader .footer {
+      border-top: none;
+    }
+  }
+
+  .sideNav .footer {
+    border-top: 3px solid var(--spectrum-alias-background-color-primary);
+  }
+
+  @keyframes shimmer {
+    0% {
+      left: -170%;
+    }
+
+    100% {
+      left: 170%;
+    }
+  }
+</style>

packages/frontend-core/src/components/index.js

@@ -5,3 +5,4 @@ export { default as UserAvatar } from "./UserAvatar.svelte"
 export { default as UserAvatars } from "./UserAvatars.svelte"
 export { default as Updating } from "./Updating.svelte"
 export { Grid } from "./grid"
+export { default as ClientAppSkeleton } from "./ClientAppSkeleton.svelte"

packages/frontend-core/src/themes/midnight.css

@@ -18,4 +18,3 @@
     --drop-shadow: rgba(0, 0, 0, 0.25) !important;
     --spectrum-global-color-blue-100: rgba(35, 40, 50) !important;
 }
-

packages/server/package.json

@@ -53,6 +53,7 @@
     "@budibase/pro": "0.0.0",
     "@budibase/shared-core": "0.0.0",
     "@budibase/string-templates": "0.0.0",
+    "@budibase/frontend-core": "0.0.0",
     "@budibase/types": "0.0.0",
     "@bull-board/api": "5.10.2",
     "@bull-board/koa": "5.10.2",

packages/server/src/api/controllers/application.ts

@@ -306,6 +306,7 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
       features: {
         componentValidation: true,
         disableUserMetadata: true,
+        skeletonLoader: true,
       },
     }
 
@@ -486,10 +487,11 @@ export async function updateClient(ctx: UserCtx) {
   const application = await db.get<App>(DocumentType.APP_METADATA)
   const currentVersion = application.version
 
+  let manifest
   // Update client library and manifest
   if (!env.isTest()) {
     await backupClientLibrary(ctx.params.appId)
-    await updateClientLibrary(ctx.params.appId)
+    manifest = await updateClientLibrary(ctx.params.appId)
   }
 
   // Update versions in app package
@@ -497,6 +499,10 @@ export async function updateClient(ctx: UserCtx) {
   const appPackageUpdates = {
     version: updatedToVersion,
     revertableVersion: currentVersion,
+    features: {
+      ...(application.features ?? {}),
+      skeletonLoader: manifest?.features?.skeletonLoader ?? false,
+    },
   }
   const app = await updateAppPackage(appPackageUpdates, ctx.params.appId)
   await events.app.versionUpdated(app, currentVersion, updatedToVersion)
@@ -512,9 +518,10 @@ export async function revertClient(ctx: UserCtx) {
     ctx.throw(400, "There is no version to revert to")
   }
 
+  let manifest
   // Update client library and manifest
   if (!env.isTest()) {
-    await revertClientLibrary(ctx.params.appId)
+    manifest = await revertClientLibrary(ctx.params.appId)
   }
 
   // Update versions in app package
@@ -523,6 +530,10 @@ export async function revertClient(ctx: UserCtx) {
   const appPackageUpdates = {
     version: revertedToVersion,
     revertableVersion: undefined,
+    features: {
+      ...(application.features ?? {}),
+      skeletonLoader: manifest?.features?.skeletonLoader ?? false,
+    },
   }
   const app = await updateAppPackage(appPackageUpdates, ctx.params.appId)
   await events.app.versionReverted(app, currentVersion, revertedToVersion)
@@ -729,6 +740,21 @@ export async function updateAppPackage(
   })
 }
 
+export async function setRevertableVersion(
+  ctx: UserCtx<{ revertableVersion: string }, App>
+) {
+  if (!env.isDev()) {
+    ctx.status = 403
+    return
+  }
+  const db = context.getAppDB()
+  const app = await db.get<App>(DocumentType.APP_METADATA)
+  app.revertableVersion = ctx.request.body.revertableVersion
+  await db.put(app)
+
+  ctx.status = 200
+}
+
 async function migrateAppNavigation() {
   const db = context.getAppDB()
   const existing: App = await db.get(DocumentType.APP_METADATA)

packages/server/src/api/controllers/query/index.ts

@@ -6,7 +6,7 @@ import { invalidateDynamicVariables } from "../../../threads/utils"
 import env from "../../../environment"
 import { events, context, utils, constants } from "@budibase/backend-core"
 import sdk from "../../../sdk"
-import { QueryEvent } from "../../../threads/definitions"
+import { QueryEvent, QueryEventParameters } from "../../../threads/definitions"
 import {
   ConfigType,
   Query,
@@ -18,7 +18,6 @@ import {
   FieldType,
   ExecuteQueryRequest,
   ExecuteQueryResponse,
-  QueryParameter,
   PreviewQueryRequest,
   PreviewQueryResponse,
 } from "@budibase/types"
@@ -29,7 +28,7 @@ const Runner = new Thread(ThreadType.QUERY, {
   timeoutMs: env.QUERY_THREAD_TIMEOUT,
 })
 
-function validateQueryInputs(parameters: Record<string, string>) {
+function validateQueryInputs(parameters: QueryEventParameters) {
   for (let entry of Object.entries(parameters)) {
     const [key, value] = entry
     if (typeof value !== "string") {
@@ -100,10 +99,18 @@ export async function save(ctx: UserCtx<Query, Query>) {
   const datasource = await sdk.datasources.get(query.datasourceId)
 
   let eventFn
-  if (!query._id) {
+  if (!query._id && !query._rev) {
     query._id = generateQueryID(query.datasourceId)
+    // flag to state whether the default bindings are empty strings (old behaviour) or null
+    query.nullDefaultSupport = true
     eventFn = () => events.query.created(datasource, query)
   } else {
+    // check if flag has previously been set, don't let it change
+    // allow it to be explicitly set to false via API incase this is ever needed
+    const existingQuery = await db.get<Query>(query._id)
+    if (existingQuery.nullDefaultSupport && query.nullDefaultSupport == null) {
+      query.nullDefaultSupport = true
+    }
     eventFn = () => events.query.updated(datasource, query)
   }
   const response = await db.put(query)
@@ -135,16 +142,20 @@ function getAuthConfig(ctx: UserCtx) {
 }
 
 function enrichParameters(
-  queryParameters: QueryParameter[],
-  requestParameters: Record<string, string> = {}
-): Record<string, string> {
+  query: Query,
+  requestParameters: QueryEventParameters = {}
+): QueryEventParameters {
+  const paramNotSet = (val: unknown) => val === "" || val == undefined
   // first check parameters are all valid
   validateQueryInputs(requestParameters)
   // make sure parameters are fully enriched with defaults
-  for (let parameter of queryParameters) {
-    if (!requestParameters[parameter.name]) {
-      requestParameters[parameter.name] = parameter.default
+  for (const parameter of query.parameters) {
+    let value: string | null =
+      requestParameters[parameter.name] || parameter.default
+    if (query.nullDefaultSupport && paramNotSet(value)) {
+      value = null
     }
+    requestParameters[parameter.name] = value
   }
   return requestParameters
 }
@@ -157,10 +168,15 @@ export async function preview(
   )
   // preview may not have a queryId as it hasn't been saved, but if it does
   // this stops dynamic variables from calling the same query
-  const { fields, parameters, queryVerb, transformer, queryId, schema } =
-    ctx.request.body
+  const queryId = ctx.request.body.queryId
+  // the body contains the makings of a query, which has not been saved yet
+  const query: Query = ctx.request.body
+  // hasn't been saved, new query
+  if (!queryId && !query._id) {
+    query.nullDefaultSupport = true
+  }
 
-  let existingSchema = schema
+  let existingSchema = query.schema
   if (queryId && !existingSchema) {
     try {
       const db = context.getAppDB()
@@ -268,13 +284,14 @@ export async function preview(
   try {
     const inputs: QueryEvent = {
       appId: ctx.appId,
-      datasource,
-      queryVerb,
-      fields,
-      parameters: enrichParameters(parameters),
-      transformer,
+      queryVerb: query.queryVerb,
+      fields: query.fields,
+      parameters: enrichParameters(query),
+      transformer: query.transformer,
+      schema: query.schema,
+      nullDefaultSupport: query.nullDefaultSupport,
       queryId,
-      schema,
+      datasource,
       // have to pass down to the thread runner - can't put into context now
       environmentVariables: envVars,
       ctx: {
@@ -336,14 +353,12 @@ async function execute(
       queryVerb: query.queryVerb,
       fields: query.fields,
       pagination: ctx.request.body.pagination,
-      parameters: enrichParameters(
-        query.parameters,
-        ctx.request.body.parameters
-      ),
+      parameters: enrichParameters(query, ctx.request.body.parameters),
       transformer: query.transformer,
       queryId: ctx.params.queryId,
       // have to pass down to the thread runner - can't put into context now
       environmentVariables: envVars,
+      nullDefaultSupport: query.nullDefaultSupport,
       ctx: {
         user: ctx.user,
         auth: { ...authConfigCtx },

packages/server/src/api/controllers/static/index.ts

@@ -1,10 +1,8 @@
 import { InvalidFileExtensions } from "@budibase/shared-core"
-
 import AppComponent from "./templates/BudibaseApp.svelte"
-
 import { join } from "../../../utilities/centralPath"
 import * as uuid from "uuid"
-import { ObjectStoreBuckets } from "../../../constants"
+import { ObjectStoreBuckets, devClientVersion } from "../../../constants"
 import { processString } from "@budibase/string-templates"
 import {
   loadHandlebarsFile,
@@ -24,13 +22,20 @@ import AWS from "aws-sdk"
 import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
-import { App, Ctx, ProcessAttachmentResponse } from "@budibase/types"
+import {
+  UserCtx,
+  App,
+  Ctx,
+  ProcessAttachmentResponse,
+  Feature,
+} from "@budibase/types"
 import {
   getAppMigrationVersion,
   getLatestMigrationId,
 } from "../../../appMigrations"
 
 import send from "koa-send"
+import { getThemeVariables } from "../../../constants/themes"
 
 export const toggleBetaUiFeature = async function (ctx: Ctx) {
   const cookieName = `beta:${ctx.params.feature}`
@@ -146,7 +151,7 @@ const requiresMigration = async (ctx: Ctx) => {
   return requiresMigrations
 }
 
-export const serveApp = async function (ctx: Ctx) {
+export const serveApp = async function (ctx: UserCtx) {
   const needMigrations = await requiresMigration(ctx)
 
   const bbHeaderEmbed =
@@ -165,12 +170,23 @@ export const serveApp = async function (ctx: Ctx) {
   try {
     db = context.getAppDB({ skip_setup: true })
     const appInfo = await db.get<any>(DocumentType.APP_METADATA)
+
     let appId = context.getAppId()
+    const hideDevTools = !!ctx.params.appUrl
+    const sideNav = appInfo.navigation.navigation === "Left"
+    const hideFooter =
+      ctx?.user?.license?.features?.includes(Feature.BRANDING) || false
+    const themeVariables = getThemeVariables(appInfo?.theme)
 
     if (!env.isJest()) {
       const plugins = objectStore.enrichPluginURLs(appInfo.usedPlugins)
+
       const { head, html, css } = AppComponent.render({
         title: branding?.platformTitle || `${appInfo.name}`,
+        showSkeletonLoader: appInfo.features?.skeletonLoader ?? false,
+        hideDevTools,
+        sideNav,
+        hideFooter,
         metaImage:
           branding?.metaImageUrl ||
           "https://res.cloudinary.com/daog6scxm/image/upload/v1698759482/meta-images/plain-branded-meta-image-coral_ocxmgu.png",
@@ -195,7 +211,7 @@ export const serveApp = async function (ctx: Ctx) {
       ctx.body = await processString(appHbs, {
         head,
         body: html,
-        style: css.code,
+        css: `:root{${themeVariables}} ${css.code}`,
         appId,
         embedded: bbHeaderEmbed,
       })
@@ -247,18 +263,20 @@ export const serveBuilderPreview = async function (ctx: Ctx) {
 }
 
 export const serveClientLibrary = async function (ctx: Ctx) {
+  const version = ctx.request.query.version
+
   const appId = context.getAppId() || (ctx.request.query.appId as string)
   let rootPath = join(NODE_MODULES_PATH, "@budibase", "client", "dist")
   if (!appId) {
     ctx.throw(400, "No app ID provided - cannot fetch client library.")
   }
-  if (env.isProd()) {
+  if (env.isProd() || (env.isDev() && version !== devClientVersion)) {
     ctx.body = await objectStore.getReadStream(
       ObjectStoreBuckets.APPS,
       objectStore.clientLibraryPath(appId!)
     )
     ctx.set("Content-Type", "application/javascript")
-  } else if (env.isDev()) {
+  } else if (env.isDev() && version === devClientVersion) {
     // incase running from TS directly
     const tsPath = join(require.resolve("@budibase/client"), "..")
     return send(ctx, "budibase-client.js", {

packages/server/src/api/controllers/static/templates/BudibaseApp.svelte

@@ -1,4 +1,6 @@
 <script>
+  import ClientAppSkeleton from "@budibase/frontend-core/src/components/ClientAppSkeleton.svelte"
+
   export let title = ""
   export let favicon = ""
 
@@ -9,6 +11,11 @@
   export let clientLibPath
   export let usedPlugins
   export let appMigrating
+
+  export let showSkeletonLoader = false
+  export let hideDevTools
+  export let sideNav
+  export let hideFooter
 </script>
 
 <svelte:head>
@@ -96,6 +103,9 @@
 </svelte:head>
 
 <body id="app">
+  {#if showSkeletonLoader}
+    <ClientAppSkeleton {hideDevTools} {sideNav} {hideFooter} />
+  {/if}
   <div id="error">
     {#if clientLibPath}
       <h1>There was an error loading your app</h1>

packages/server/src/api/controllers/static/templates/app.hbs

@@ -1,8 +1,12 @@
 <html>
-
+  <script>
+    document.fonts.ready.then(() => {
+      window.parent.postMessage({ type: "docLoaded" });
+    })
+  </script>
   <head>
     {{{head}}}
-    <style>{{{style}}}</style>
+    <style>{{{css}}}</style>
   </head>
 
   <script>

packages/server/src/api/routes/application.ts

@@ -68,5 +68,10 @@ router
     authorized(permissions.BUILDER),
     controller.importToApp
   )
+  .post(
+    "/api/applications/:appId/setRevertableVersion",
+    authorized(permissions.BUILDER),
+    controller.setRevertableVersion
+  )
 
 export default router

packages/server/src/api/routes/static.ts

@@ -51,8 +51,8 @@ router
     controller.deleteObjects
   )
   .get("/app/preview", authorized(BUILDER), controller.serveBuilderPreview)
-  .get("/:appId/:path*", controller.serveApp)
   .get("/app/:appUrl/:path*", controller.serveApp)
+  .get("/:appId/:path*", controller.serveApp)
   .post(
     "/api/attachments/:datasourceId/url",
     authorized(PermissionType.TABLE, PermissionLevel.READ),

packages/server/src/api/routes/tests/environmentVariables.spec.ts

@@ -143,7 +143,10 @@ describe("/api/env/variables", () => {
     delete response.body.datasource.config
     expect(events.query.previewed).toHaveBeenCalledWith(
       response.body.datasource,
-      queryPreview
+      {
+        ...queryPreview,
+        nullDefaultSupport: true,
+      }
     )
     expect(pg.Client).toHaveBeenCalledWith({ password: "test", ssl: undefined })
   })

packages/server/src/api/routes/tests/queries/generic-sql.spec.ts

@@ -12,19 +12,22 @@ const createTableSQL: Record<string, string> = {
     CREATE TABLE test_table (
         id serial PRIMARY KEY,
         name VARCHAR ( 50 ) NOT NULL,
-        birthday TIMESTAMP
+        birthday TIMESTAMP,
+        number INT
     );`,
   [SourceName.MYSQL]: `
     CREATE TABLE test_table (
         id INT AUTO_INCREMENT PRIMARY KEY,
         name VARCHAR(50) NOT NULL,
-        birthday TIMESTAMP
+        birthday TIMESTAMP,
+        number INT
     );`,
   [SourceName.SQL_SERVER]: `
     CREATE TABLE test_table (
         id INT IDENTITY(1,1) PRIMARY KEY,
         name NVARCHAR(50) NOT NULL,
-        birthday DATETIME
+        birthday DATETIME,
+        number INT
     );`,
 }
 
@@ -36,7 +39,7 @@ describe.each([
   ["mysql", databaseTestProviders.mysql],
   ["mssql", databaseTestProviders.mssql],
   ["mariadb", databaseTestProviders.mariadb],
-])("queries (%s)", (__, dsProvider) => {
+])("queries (%s)", (dbName, dsProvider) => {
   const config = setup.getConfig()
   let datasource: Datasource
 
@@ -51,7 +54,7 @@ describe.each([
       transformer: "return data",
       readable: true,
     }
-    return await config.api.query.create({ ...defaultQuery, ...query })
+    return await config.api.query.save({ ...defaultQuery, ...query })
   }
 
   async function rawQuery(sql: string): Promise<any> {
@@ -221,26 +224,31 @@ describe.each([
           id: 1,
           name: "one",
           birthday: null,
+          number: null,
         },
         {
           id: 2,
           name: "two",
           birthday: null,
+          number: null,
         },
         {
           id: 3,
           name: "three",
           birthday: null,
+          number: null,
         },
         {
           id: 4,
           name: "four",
           birthday: null,
+          number: null,
         },
         {
           id: 5,
           name: "five",
           birthday: null,
+          number: null,
         },
       ])
     })
@@ -263,6 +271,7 @@ describe.each([
           id: 2,
           name: "one",
           birthday: null,
+          number: null,
         },
       ])
     })
@@ -291,6 +300,7 @@ describe.each([
           id: 1,
           name: "one",
           birthday: null,
+          number: null,
         },
       ])
     })
@@ -329,7 +339,9 @@ describe.each([
       ])
 
       const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1")
-      expect(rows).toEqual([{ id: 1, name: "foo", birthday: null }])
+      expect(rows).toEqual([
+        { id: 1, name: "foo", birthday: null, number: null },
+      ])
     })
 
     it("should be able to execute an update that updates no rows", async () => {
@@ -398,4 +410,55 @@ describe.each([
       expect(rows).toHaveLength(0)
     })
   })
+
+  // this parameter really only impacts SQL queries
+  describe("confirm nullDefaultSupport", () => {
+    const queryParams = {
+      fields: {
+        sql: "INSERT INTO test_table (name, number) VALUES ({{ bindingName }}, {{ bindingNumber }})",
+      },
+      parameters: [
+        {
+          name: "bindingName",
+          default: "",
+        },
+        {
+          name: "bindingNumber",
+          default: "",
+        },
+      ],
+      queryVerb: "create",
+    }
+
+    it("should error for old queries", async () => {
+      const query = await createQuery(queryParams)
+      await config.api.query.save({ ...query, nullDefaultSupport: false })
+      let error: string | undefined
+      try {
+        await config.api.query.execute(query._id!, {
+          parameters: {
+            bindingName: "testing",
+          },
+        })
+      } catch (err: any) {
+        error = err.message
+      }
+      if (dbName === "mssql") {
+        expect(error).toBeUndefined()
+      } else {
+        expect(error).toBeDefined()
+        expect(error).toContain("integer")
+      }
+    })
+
+    it("should not error for new queries", async () => {
+      const query = await createQuery(queryParams)
+      const results = await config.api.query.execute(query._id!, {
+        parameters: {
+          bindingName: "testing",
+        },
+      })
+      expect(results).toEqual({ data: [{ created: true }] })
+    })
+  })
 })

packages/server/src/api/routes/tests/queries/mongodb.spec.ts

@@ -31,7 +31,7 @@ describe("/queries", () => {
     ) {
       combinedQuery.fields.extra.collection = collection
     }
-    return await config.api.query.create(combinedQuery)
+    return await config.api.query.save(combinedQuery)
   }
 
   async function withClient<T>(
@@ -464,7 +464,7 @@ describe("/queries", () => {
     })
   })
 
-  it("should ignore be able to save deeply nested data", async () => {
+  it("should be able to save deeply nested data", async () => {
     const data = {
       foo: "bar",
       data: [

packages/server/src/api/routes/tests/queries/query.seq.spec.ts

@@ -78,6 +78,7 @@ describe("/queries", () => {
         _rev: res.body._rev,
         _id: res.body._id,
         ...query,
+        nullDefaultSupport: true,
         createdAt: new Date().toISOString(),
         updatedAt: new Date().toISOString(),
       })
@@ -103,6 +104,7 @@ describe("/queries", () => {
         _rev: res.body._rev,
         _id: res.body._id,
         ...query,
+        nullDefaultSupport: true,
         createdAt: new Date().toISOString(),
         updatedAt: new Date().toISOString(),
       })
@@ -130,6 +132,7 @@ describe("/queries", () => {
           _id: query._id,
           createdAt: new Date().toISOString(),
           ...basicQuery(datasource._id),
+          nullDefaultSupport: true,
           updatedAt: new Date().toISOString(),
           readable: true,
         },
@@ -245,10 +248,10 @@ describe("/queries", () => {
       expect(responseBody.rows.length).toEqual(1)
       expect(events.query.previewed).toHaveBeenCalledTimes(1)
       delete datasource.config
-      expect(events.query.previewed).toHaveBeenCalledWith(
-        datasource,
-        queryPreview
-      )
+      expect(events.query.previewed).toHaveBeenCalledWith(datasource, {
+        ...queryPreview,
+        nullDefaultSupport: true,
+      })
     })
 
     it("should apply authorization to endpoint", async () => {

packages/server/src/constants/index.ts

@@ -165,6 +165,8 @@ export enum AutomationErrors {
   FAILURE_CONDITION = "FAILURE_CONDITION_MET",
 }
 
+export const devClientVersion = "0.0.0"
+
 // pass through the list from the auth/core lib
 export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets
 export const MAX_AUTOMATION_RECURRING_ERRORS = 5

packages/server/src/constants/themes.ts

@@ -0,0 +1,54 @@
+export const getThemeVariables = (theme: string) => {
+  if (theme === "spectrum--lightest") {
+    return `
+      --spectrum-global-color-gray-50: rgb(255, 255, 255);
+      --spectrum-global-color-gray-200: rgb(244, 244, 244);
+      --spectrum-global-color-gray-300: rgb(234, 234, 234);
+      --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-50);
+    `
+  }
+  if (theme === "spectrum--light") {
+    return `
+    --spectrum-global-color-gray-50: rgb(255, 255, 255);
+    --spectrum-global-color-gray-200: rgb(234, 234, 234);
+    --spectrum-global-color-gray-300: rgb(225, 225, 225);
+    --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-50);
+
+    `
+  }
+  if (theme === "spectrum--dark") {
+    return `
+    --spectrum-global-color-gray-100: rgb(50, 50, 50);
+    --spectrum-global-color-gray-200: rgb(62, 62, 62);
+    --spectrum-global-color-gray-300: rgb(74, 74, 74);
+    --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-100);
+    `
+  }
+  if (theme === "spectrum--darkest") {
+    return `
+  --spectrum-global-color-gray-100: rgb(30, 30, 30);
+  --spectrum-global-color-gray-200: rgb(44, 44, 44);
+  --spectrum-global-color-gray-300: rgb(57, 57, 57);
+  --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-100);
+    `
+  }
+  if (theme === "spectrum--nord") {
+    return `
+    --spectrum-global-color-gray-100: #3b4252;
+
+  --spectrum-global-color-gray-200: #424a5c;
+  --spectrum-global-color-gray-300: #4c566a;
+  --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-100);
+    `
+  }
+  if (theme === "spectrum--midnight") {
+    return `
+    --hue: 220;
+    --sat: 10%;
+    --spectrum-global-color-gray-100: hsl(var(--hue), var(--sat), 17%);
+    --spectrum-global-color-gray-200: hsl(var(--hue), var(--sat), 20%);
+    --spectrum-global-color-gray-300: hsl(var(--hue), var(--sat), 24%);
+    --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-100);
+    `
+  }
+}

packages/server/src/integrations/queries/sql.ts

@@ -5,9 +5,10 @@ import sdk from "../../sdk"
 const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g")
 
 export async function interpolateSQL(
-  fields: { [key: string]: any },
+  fields: { sql: string; bindings: any[] },
   parameters: { [key: string]: any },
-  integration: DatasourcePlus
+  integration: DatasourcePlus,
+  opts: { nullDefaultSupport: boolean }
 ) {
   let sql = fields.sql
   if (!sql || typeof sql !== "string") {
@@ -64,7 +65,14 @@ export async function interpolateSQL(
   }
   // replicate the knex structure
   fields.sql = sql
-  fields.bindings = await sdk.queries.enrichContext(variables, parameters)
+  fields.bindings = await sdk.queries.enrichArrayContext(variables, parameters)
+  if (opts.nullDefaultSupport) {
+    for (let index in fields.bindings) {
+      if (fields.bindings[index] === "") {
+        fields.bindings[index] = null
+      }
+    }
+  }
   // check for arrays in the data
   let updated: string[] = []
   for (let i = 0; i < variables.length; i++) {

packages/server/src/sdk/app/queries/queries.ts

@@ -65,14 +65,33 @@ export async function fetch(opts: { enrich: boolean } = { enrich: true }) {
   return updateSchemas(queries)
 }
 
+export async function enrichArrayContext(
+  fields: any[],
+  inputs = {}
+): Promise<any[]> {
+  const map: Record<string, any> = {}
+  for (let index in fields) {
+    map[index] = fields[index]
+  }
+  const output = await enrichContext(map, inputs)
+  const outputArray: any[] = []
+  for (let [key, value] of Object.entries(output)) {
+    outputArray[parseInt(key)] = value
+  }
+  return outputArray
+}
+
 export async function enrichContext(
   fields: Record<string, any>,
   inputs = {}
 ): Promise<Record<string, any>> {
-  const enrichedQuery: Record<string, any> = Array.isArray(fields) ? [] : {}
+  const enrichedQuery: Record<string, any> = {}
   if (!fields || !inputs) {
     return enrichedQuery
   }
+  if (Array.isArray(fields)) {
+    return enrichArrayContext(fields, inputs)
+  }
   const env = await getEnvironmentVariables()
   const parameters = { ...inputs, env }
   // enrich the fields with dynamic parameters

packages/server/src/sdk/app/rows/search/tests/external.spec.ts

@@ -26,7 +26,7 @@ describe("external search", () => {
   const rows: Row[] = []
 
   beforeAll(async () => {
-    const container = await new GenericContainer("mysql")
+    const container = await new GenericContainer("mysql:8.3")
       .withExposedPorts(3306)
       .withEnvironment({
         MYSQL_ROOT_PASSWORD: "admin",

packages/server/src/tests/utilities/api/query.ts

@@ -8,7 +8,7 @@ import {
 import { Expectations, TestAPI } from "./base"
 
 export class QueryAPI extends TestAPI {
-  create = async (body: Query): Promise<Query> => {
+  save = async (body: Query): Promise<Query> => {
     return await this._post<Query>(`/api/queries`, { body })
   }
 

packages/server/src/threads/definitions.ts

@@ -1,21 +1,20 @@
-import { Datasource, QuerySchema, Row } from "@budibase/types"
+import { Datasource, Row, Query } from "@budibase/types"
 
 export type WorkerCallback = (error: any, response?: any) => void
 
-export interface QueryEvent {
+export interface QueryEvent
+  extends Omit<Query, "datasourceId" | "name" | "parameters" | "readable"> {
   appId?: string
   datasource: Datasource
-  queryVerb: string
-  fields: { [key: string]: any }
-  parameters: { [key: string]: unknown }
   pagination?: any
-  transformer: any
   queryId?: string
   environmentVariables?: Record<string, string>
+  parameters: QueryEventParameters
   ctx?: any
-  schema?: Record<string, QuerySchema | string>
 }
 
+export type QueryEventParameters = Record<string, string | null>
+
 export interface QueryResponse {
   rows: Row[]
   keys: string[]

packages/server/src/threads/query.ts

@@ -26,10 +26,11 @@ class QueryRunner {
   fields: any
   parameters: any
   pagination: any
-  transformer: string
+  transformer: string | null
   cachedVariables: any[]
   ctx: any
   queryResponse: any
+  nullDefaultSupport: boolean
   noRecursiveQuery: boolean
   hasRerun: boolean
   hasRefreshedOAuth: boolean
@@ -45,6 +46,7 @@ class QueryRunner {
     this.transformer = input.transformer
     this.queryId = input.queryId!
     this.schema = input.schema
+    this.nullDefaultSupport = !!input.nullDefaultSupport
     this.noRecursiveQuery = flags.noRecursiveQuery
     this.cachedVariables = []
     // Additional context items for enrichment
@@ -59,7 +61,14 @@ class QueryRunner {
   }
 
   async execute(): Promise<QueryResponse> {
-    let { datasource, fields, queryVerb, transformer, schema } = this
+    let {
+      datasource,
+      fields,
+      queryVerb,
+      transformer,
+      schema,
+      nullDefaultSupport,
+    } = this
     let datasourceClone = cloneDeep(datasource)
     let fieldsClone = cloneDeep(fields)
 
@@ -100,10 +109,12 @@ class QueryRunner {
       )
     }
 
-    let query
+    let query: Record<string, any>
     // handle SQL injections by interpolating the variables
     if (isSQL(datasourceClone)) {
-      query = await interpolateSQL(fieldsClone, enrichedContext, integration)
+      query = await interpolateSQL(fieldsClone, enrichedContext, integration, {
+        nullDefaultSupport,
+      })
     } else {
       query = await sdk.queries.enrichContext(fieldsClone, enrichedContext)
     }
@@ -137,7 +148,9 @@ class QueryRunner {
         data: rows,
         params: enrichedParameters,
       }
-      rows = vm.withContext(ctx, () => vm.execute(transformer))
+      if (transformer != null) {
+        rows = vm.withContext(ctx, () => vm.execute(transformer!))
+      }
     }
 
     // if the request fails we retry once, invalidating the cached value
@@ -191,13 +204,15 @@ class QueryRunner {
     })
     return new QueryRunner(
       {
-        datasource,
+        schema: query.schema,
         queryVerb: query.queryVerb,
         fields: query.fields,
-        parameters,
         transformer: query.transformer,
-        queryId,
+        nullDefaultSupport: query.nullDefaultSupport,
         ctx: this.ctx,
+        parameters,
+        datasource,
+        queryId,
       },
       { noRecursiveQuery: true }
     ).execute()

packages/server/src/utilities/fileSystem/app.ts

@@ -1,11 +1,13 @@
 import { budibaseTempDir } from "../budibaseDir"
 import fs from "fs"
 import { join } from "path"
-import { ObjectStoreBuckets } from "../../constants"
+import { ObjectStoreBuckets, devClientVersion } from "../../constants"
 import { updateClientLibrary } from "./clientLibrary"
 import env from "../../environment"
 import { objectStore, context } from "@budibase/backend-core"
 import { TOP_LEVEL_PATH } from "./filesystem"
+import { DocumentType } from "../../db/utils"
+import { App } from "@budibase/types"
 
 export const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
 
@@ -35,6 +37,10 @@ export const getComponentLibraryManifest = async (library: string) => {
   const filename = "manifest.json"
 
   if (env.isDev() || env.isTest()) {
+    const db = context.getAppDB()
+    const app = await db.get<App>(DocumentType.APP_METADATA)
+
+    if (app.version === devClientVersion || env.isTest()) {
       const paths = [
         join(TOP_LEVEL_PATH, "packages/client", filename),
         join(process.cwd(), "client", filename),
@@ -50,6 +56,7 @@ export const getComponentLibraryManifest = async (library: string) => {
         `Unable to find ${filename} in development environment (may need to build).`
       )
     }
+  }
 
   if (!appId) {
     throw new Error("No app ID found - cannot get component libraries")

packages/server/src/utilities/fileSystem/clientLibrary.ts

@@ -1,10 +1,12 @@
 import path, { join } from "path"
 import { ObjectStoreBuckets } from "../../constants"
 import fs from "fs"
-import { objectStore } from "@budibase/backend-core"
+import { context, objectStore } from "@budibase/backend-core"
 import { resolve } from "../centralPath"
 import env from "../../environment"
 import { TOP_LEVEL_PATH } from "./filesystem"
+import { DocumentType } from "../../db/utils"
+import { App } from "@budibase/types"
 
 export function devClientLibPath() {
   return require.resolve("@budibase/client")
@@ -120,7 +122,12 @@ export async function updateClientLibrary(appId: string) {
       ContentType: "application/javascript",
     }
   )
-  await Promise.all([manifestUpload, clientUpload])
+
+  const manifestSrc = fs.promises.readFile(manifest, "utf8")
+
+  await Promise.all([manifestUpload, clientUpload, manifestSrc])
+
+  return JSON.parse(await manifestSrc)
 }
 
 /**
@@ -130,30 +137,49 @@ export async function updateClientLibrary(appId: string) {
  * @returns {Promise<void>}
  */
 export async function revertClientLibrary(appId: string) {
+  let manifestPath, clientPath
+
+  if (env.isDev()) {
+    const db = context.getAppDB()
+    const app = await db.get<App>(DocumentType.APP_METADATA)
+    clientPath = join(
+      __dirname,
+      `/oldClientVersions/${app.revertableVersion}/app.js`
+    )
+    manifestPath = join(
+      __dirname,
+      `/oldClientVersions/${app.revertableVersion}/manifest.json`
+    )
+  } else {
     // Copy backups manifest to tmp directory
-  const tmpManifestPath = await objectStore.retrieveToTmp(
+    manifestPath = await objectStore.retrieveToTmp(
       ObjectStoreBuckets.APPS,
       join(appId, "manifest.json.bak")
     )
 
     // Copy backup client lib to tmp
-  const tmpClientPath = await objectStore.retrieveToTmp(
+    clientPath = await objectStore.retrieveToTmp(
       ObjectStoreBuckets.APPS,
       join(appId, "budibase-client.js.bak")
     )
+  }
+
+  const manifestSrc = fs.promises.readFile(manifestPath, "utf8")
 
   // Upload backups as new versions
   const manifestUpload = objectStore.upload({
     bucket: ObjectStoreBuckets.APPS,
     filename: join(appId, "manifest.json"),
-    path: tmpManifestPath,
+    path: manifestPath,
     type: "application/json",
   })
   const clientUpload = objectStore.upload({
     bucket: ObjectStoreBuckets.APPS,
     filename: join(appId, "budibase-client.js"),
-    path: tmpClientPath,
+    path: clientPath,
     type: "application/javascript",
   })
-  await Promise.all([manifestUpload, clientUpload])
+  await Promise.all([manifestSrc, manifestUpload, clientUpload])
+
+  return JSON.parse(await manifestSrc)
 }

packages/types/src/documents/app/app.ts

@@ -71,6 +71,7 @@ export interface AppIcon {
 export interface AppFeatures {
   componentValidation?: boolean
   disableUserMetadata?: boolean
+  skeletonLoader?: boolean
 }
 
 export interface AutomationSettings {

packages/types/src/documents/app/query.ts

@@ -15,6 +15,8 @@ export interface Query extends Document {
   schema: Record<string, QuerySchema | string>
   readable: boolean
   queryVerb: string
+  // flag to state whether the default bindings are empty strings (old behaviour) or null
+  nullDefaultSupport?: boolean
 }
 
 export interface QueryPreview extends Omit<Query, "_id"> {

qa-core/src/integrations/external-schema/mysql.integration.spec.ts

@@ -6,7 +6,7 @@ describe("datasource validators", () => {
     let config: any
 
     beforeAll(async () => {
-      const container = await new GenericContainer("mysql")
+      const container = await new GenericContainer("mysql:8.3")
         .withExposedPorts(3306)
         .withEnv("MYSQL_ROOT_PASSWORD", "admin")
         .withEnv("MYSQL_DATABASE", "db")

qa-core/src/integrations/external-schema/postgres.integration.spec.ts

@@ -17,7 +17,7 @@ describe("getExternalSchema", () => {
     }
 
     beforeAll(async () => {
-      const container = await new GenericContainer("postgres:13.12")
+      const container = await new GenericContainer("postgres:16.1-bullseye")
         .withExposedPorts(5432)
         .withEnv("POSTGRES_PASSWORD", "password")
         .start()

qa-core/src/integrations/validators/mongo.integration.spec.ts

@@ -26,7 +26,7 @@ describe("datasource validators", () => {
     beforeAll(async () => {
       const user = generator.name()
       const password = generator.hash()
-      const container = await new GenericContainer("mongo")
+      const container = await new GenericContainer("mongo:7.0-jammy")
         .withExposedPorts(27017)
         .withEnv("MONGO_INITDB_ROOT_USERNAME", user)
         .withEnv("MONGO_INITDB_ROOT_PASSWORD", password)

qa-core/src/integrations/validators/mssql.integration.spec.ts

@@ -13,7 +13,7 @@ describe("datasource validators", () => {
 
     beforeAll(async () => {
       const container = await new GenericContainer(
-        "mcr.microsoft.com/mssql/server"
+        "mcr.microsoft.com/mssql/server:2022-latest"
       )
         .withExposedPorts(1433)
         .withEnv("ACCEPT_EULA", "Y")

qa-core/src/integrations/validators/mysql.integration.spec.ts

@@ -7,7 +7,7 @@ describe("datasource validators", () => {
     let port: number
 
     beforeAll(async () => {
-      const container = await new GenericContainer("mysql")
+      const container = await new GenericContainer("mysql:8.3")
         .withExposedPorts(3306)
         .withEnv("MYSQL_ROOT_PASSWORD", "admin")
         .withEnv("MYSQL_DATABASE", "db")

qa-core/src/integrations/validators/postgres.integration.spec.ts

@@ -9,7 +9,7 @@ describe("datasource validators", () => {
     let port: number
 
     beforeAll(async () => {
-      const container = await new GenericContainer("postgres")
+      const container = await new GenericContainer("postgres:16.1-bullseye")
         .withExposedPorts(5432)
         .withEnv("POSTGRES_PASSWORD", "password")
         .start()

scripts/build.js

@@ -3,7 +3,7 @@
 const start = Date.now()
 
 const fs = require("fs")
-const { readdir, copyFile, mkdir } = require('node:fs/promises');
+const { cp, readdir, copyFile, mkdir } = require('node:fs/promises');
 const path = require("path")
 
 const { build } = require("esbuild")
@@ -17,12 +17,6 @@ const { nodeExternalsPlugin } = require("esbuild-node-externals")
 const svelteCompilePlugin = {
   name: 'svelteCompile',
   setup(build) {
-    // This resolve handler is necessary to bundle the Svelte runtime into the the final output,
-    // otherwise the bundled script will attempt to resolve it at runtime
-    build.onResolve({ filter: /svelte\/internal/ }, async () => {
-      return { path: `${process.cwd()}/../../node_modules/svelte/src/runtime/internal/ssr.js` }
-    })
-
     // Compiles `.svelte` files into JS classes so that they can be directly imported into our
     // Typescript packages
     build.onLoad({ filter: /\.svelte$/ }, async (args) => {
@@ -80,11 +74,11 @@ async function runBuild(entry, outfile) {
     plugins: [
       svelteCompilePlugin,
       TsconfigPathsPlugin({ tsconfig: tsconfigPathPluginContent }),
-      nodeExternalsPlugin(),
+      nodeExternalsPlugin({
+        allowList: ["@budibase/frontend-core", "svelte"]
+      }),
     ],
     preserveSymlinks: true,
-    loader: {
-    },
     metafile: true,
     external: [
       "deasync",
@@ -109,13 +103,23 @@ async function runBuild(entry, outfile) {
     await Promise.all(fileCopyPromises)
   })()
 
+  const oldClientVersions = (async () => {
+    try {
+      await cp('./build/oldClientVersions', './dist/oldClientVersions', { recursive: true });
+    } catch (e) {
+      if (e.code !== "EEXIST" && e.code !== "ENOENT") {
+        throw e;
+      }
+    }
+  })()
+
   const mainBuild = build({
     ...sharedConfig,
     platform: "node",
     outfile,
   })
 
-  await Promise.all([hbsFiles, mainBuild])
+  await Promise.all([hbsFiles, mainBuild, oldClientVersions])
 
   fs.writeFileSync(
     `dist/${path.basename(outfile)}.meta.json`,

scripts/getPastClientVersion.js

@@ -0,0 +1,45 @@
+const fs = require('node:fs/promises');
+const util = require('node:util');
+const { argv } = require('node:process');
+const exec = util.promisify(require('node:child_process').exec);
+
+const version = argv[2]
+
+const getPastClientVersion = async () => {
+  const manifestRaw = await fetch(`https://api.github.com/repos/budibase/budibase/contents/packages/client/manifest.json?ref=v${version}`).then(r => r.json());
+
+  // GitHub response will contain a message field containing the following string if the version can't be found.
+  if (manifestRaw?.message?.includes("No commit found")) {
+    throw `Can't find a GitHub tag with version "${version}"`
+  }
+
+  const manifest = Buffer.from(manifestRaw.content, 'base64').toString('utf8')
+
+  await fs.mkdir(`packages/server/build/oldClientVersions/${version}`, { recursive: true });
+  await fs.writeFile(`packages/server/build/oldClientVersions/${version}/manifest.json`, manifest)
+
+  const npmRegistry = await fetch(`https://registry.npmjs.org/@budibase/client/${version}`).then(r => r.json());
+
+  // The json response from npm is just a string starting with the following if the version can't be found
+  if (typeof npmRegistry === "string" &&  npmRegistry.startsWith("version not found")) {
+    throw `Can't find @budibase/client with version "${version}" in npm registry`
+  }
+
+  // Create a temp directory to store the @budibase/client library in
+  await fs.mkdir("clientVersionTmp", { recursive: true });
+
+  // Get the tarball of the @budibase/client library and pipe it into tar to extract it
+  await exec(`curl -L ${npmRegistry.dist.tarball} --output - | tar -xvzf - -C clientVersionTmp`);
+
+  // Copy the compiled client version we want to the oldClientVersions dir and delete the temp directory
+  await fs.copyFile('./clientVersionTmp/package/dist/budibase-client.js', `./packages/server/build/oldClientVersions/${version}/app.js`);
+  await fs.rm("clientVersionTmp", { recursive: true });
+
+  // Check what client versions the user has locally and update the `clientVersions.json` file in the builder so that they can be selected
+  const rootDir = await fs.readdir('packages/server/build/oldClientVersions/', { withFileTypes: true });
+  const dirs = rootDir.filter(entry => entry.isDirectory()).map(dir => dir.name);
+
+  await fs.writeFile("packages/builder/src/components/deploy/clientVersions.json", JSON.stringify(dirs))
+}
+
+getPastClientVersion().catch(e => console.error(e));