Merge pull request #1953 from Budibase/client-feature-flags

Client library updating and client library feature awareness

packages/auth/src/objectStore/index.js

@@ -22,11 +22,13 @@ const CONTENT_TYPE_MAP = {
   html: "text/html",
   css: "text/css",
   js: "application/javascript",
+  json: "application/json",
 }
 const STRING_CONTENT_TYPES = [
   CONTENT_TYPE_MAP.html,
   CONTENT_TYPE_MAP.css,
   CONTENT_TYPE_MAP.js,
+  CONTENT_TYPE_MAP.json,
 ]
 
 // does normal sanitization and then swaps dev apps to apps

packages/builder/src/builderStore/store/frontend.js

@@ -32,6 +32,10 @@ const INITIAL_FRONTEND_STATE = {
   layouts: [],
   screens: [],
   components: [],
+  clientFeatures: {
+    spectrumThemes: false,
+    intelligentLoading: false,
+  },
   currentFrontEndType: "none",
   selectedScreenId: "",
   selectedLayoutId: "",
@@ -56,6 +60,10 @@ export const getFrontendStore = () => {
         ...state,
         libraries: application.componentLibraries,
         components,
+        clientFeatures: {
+          ...state.clientFeatures,
+          ...components.features,
+        },
         name: application.name,
         description: application.description,
         appId: application.appId,
@@ -67,6 +75,8 @@ export const getFrontendStore = () => {
         appInstance: application.instance,
         clientLibPath,
         previousTopNavPath: {},
+        version: application.version,
+        revertableVersion: application.revertableVersion,
       }))
       await hostingStore.actions.fetch()
 

packages/builder/src/components/deploy/VersionModal.svelte

@@ -0,0 +1,120 @@
+<script>
+  import {
+    Icon,
+    Modal,
+    notifications,
+    ModalContent,
+    Body,
+    Button,
+  } from "@budibase/bbui"
+  import { store } from "builderStore"
+  import api from "builderStore/api"
+  import clientPackage from "@budibase/client/package.json"
+
+  let updateModal
+
+  $: appId = $store.appId
+  $: updateAvailable = clientPackage.version !== $store.version
+  $: revertAvailable = $store.revertableVersion != null
+
+  const refreshAppPackage = async () => {
+    const applicationPkg = await api.get(
+      `/api/applications/${appId}/appPackage`
+    )
+    const pkg = await applicationPkg.json()
+    if (applicationPkg.ok) {
+      await store.actions.initialise(pkg)
+    } else {
+      throw new Error(pkg)
+    }
+  }
+
+  const update = async () => {
+    try {
+      const response = await api.post(
+        `/api/applications/${appId}/client/update`
+      )
+      const json = await response.json()
+      if (response.status !== 200) {
+        throw json.message
+      }
+
+      // Don't wait for the async refresh, since this causes modal flashing
+      refreshAppPackage()
+      notifications.success(
+        `App updated successfully to version ${clientPackage.version}`
+      )
+    } catch (err) {
+      notifications.error(`Error updating app: ${err}`)
+    }
+  }
+
+  const revert = async () => {
+    try {
+      const revertableVersion = $store.revertableVersion
+      const response = await api.post(
+        `/api/applications/${appId}/client/revert`
+      )
+      const json = await response.json()
+      if (response.status !== 200) {
+        throw json.message
+      }
+
+      // Don't wait for the async refresh, since this causes modal flashing
+      refreshAppPackage()
+      notifications.success(
+        `App reverted successfully to version ${revertableVersion}`
+      )
+    } catch (err) {
+      notifications.error(`Error reverting app: ${err}`)
+    }
+    updateModal.hide()
+  }
+</script>
+
+<div class="icon-wrapper" class:highlight={updateAvailable}>
+  <Icon name="Refresh" hoverable on:click={updateModal.show} />
+</div>
+<Modal bind:this={updateModal}>
+  <ModalContent
+    title="App version"
+    confirmText="Update"
+    cancelText={updateAvailable ? "Cancel" : "Close"}
+    onConfirm={update}
+    showConfirmButton={updateAvailable}
+  >
+    <div slot="footer">
+      {#if revertAvailable}
+        <Button quiet secondary on:click={revert}>Revert</Button>
+      {/if}
+    </div>
+    {#if updateAvailable}
+      <Body size="S">
+        This app is currently using version <b>{$store.version}</b>, but version
+        <b>{clientPackage.version}</b> is available. Updates can contain new features,
+        performance improvements and bug fixes.
+      </Body>
+    {:else}
+      <Body size="S">
+        This app is currently using version <b>{$store.version}</b> which is the
+        latest version available.
+      </Body>
+    {/if}
+    {#if revertAvailable}
+      <Body size="S">
+        You can revert this app to version
+        <b>{$store.revertableVersion}</b>
+        if you're experiencing issues with the current version.
+      </Body>
+    {/if}
+  </ModalContent>
+</Modal>
+
+<style>
+  .icon-wrapper {
+    display: contents;
+  }
+  .icon-wrapper.highlight :global(svg) {
+    color: var(--spectrum-global-color-blue-600);
+  }
+</style>

packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte

@@ -74,14 +74,14 @@
     // Initialise the app when mounted
     iframe.contentWindow.addEventListener(
       "ready",
-      () => refreshContent(strippedJson),
+      () => {
-      { once: true }
+        // Display preview immediately if the intelligent loading feature
-    )
+        // is not supported
-
+        if (!$store.clientFeatures.intelligentLoading) {
-    // Display the client app once the iframe is initialised
+          loading = false
-    iframe.contentWindow.addEventListener(
+        }
-      "iframe-loaded",
+        refreshContent(strippedJson)
-      () => (loading = false),
+      },
       { once: true }
     )
 
@@ -106,11 +106,9 @@
         idToDelete = data.id
         confirmDeleteDialog.show()
       } else if (type === "preview-loaded") {
-        // We can use this in future to delay displaying the preview
+        // Wait for this event to show the client library if intelligent
-        // until the client app has actually initialised.
+        // loading is supported
-        // This makes a smoother loading experience, but is not backwards
+        loading = false
-        // compatible with old client library versions.
-        // So do nothing with this for now.
       } else {
         console.warning(`Client sent unknown event type: ${type}`)
       }

packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte

@@ -65,7 +65,7 @@
   }
 </script>
 
-{#if definition.editable !== false}
+{#if definition?.editable !== false}
   <ActionMenu>
     <div slot="control" class="icon">
       <Icon size="S" hoverable name="MoreSmallList" />

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -4,6 +4,7 @@
   import { Icon, ActionGroup, Tabs, Tab } from "@budibase/bbui"
   import DeployModal from "components/deploy/DeployModal.svelte"
   import RevertModal from "components/deploy/RevertModal.svelte"
+  import VersionModal from "components/deploy/VersionModal.svelte"
   import { get } from "builderStore/api"
   import { isActive, goto, layout } from "@roxi/routify"
   import Logo from "assets/bb-emblem.svg"
@@ -80,6 +81,7 @@
         <ActionGroup />
       </div>
       <div class="toprightnav">
+        <VersionModal />
         <RevertModal />
         <Icon
           name="Play"

packages/builder/src/pages/builder/app/[application]/design/[assetType]/_layout.svelte

@@ -150,10 +150,14 @@
     {#if $currentAsset}
       <div class="preview-header">
         <ComponentSelectionList />
-        <AppThemeSelect />
+        {#if $store.clientFeatures.spectrumThemes}
+          <AppThemeSelect />
+        {/if}
       </div>
       <div class="preview-content">
-        <CurrentItemPreview />
+        {#key $store.version}
+          <CurrentItemPreview />
+        {/key}
       </div>
     {/if}
   </div>

packages/server/.gitignore

@@ -2,6 +2,7 @@ node_modules/
 myapps/
 .env
 builder/*
+client/*
 public/
 db/dev.db/
 dist

packages/server/package.json

@@ -13,7 +13,8 @@
     "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
     "test": "jest --coverage --maxWorkers=2",
     "test:watch": "jest --watch",
-    "build:docker": "docker build . -t app-service",
+    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../standard-components/manifest.json client",
+    "build:docker": "yarn run predocker && docker build . -t app-service",
     "run:docker": "node dist/index.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
     "dev:stack:down": "node scripts/dev/manage.js down",
@@ -61,6 +62,7 @@
   "dependencies": {
     "@budibase/auth": "^0.9.71",
     "@budibase/client": "^0.9.71",
+    "@budibase/standard-components": "^0.9.71",
     "@budibase/string-templates": "^0.9.71",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
@@ -114,7 +116,6 @@
   "devDependencies": {
     "@babel/core": "^7.14.3",
     "@babel/preset-env": "^7.14.4",
-    "@budibase/standard-components": "^0.9.71",
     "@jest/test-sequencer": "^24.8.0",
     "@types/bull": "^3.15.1",
     "@types/jest": "^26.0.23",

packages/server/src/api/controllers/application.js

@@ -33,6 +33,11 @@ const {
 } = require("../../utilities/workerRequests")
 const { clientLibraryPath } = require("../../utilities")
 const { getAllLocks } = require("../../utilities/redis")
+const {
+  updateClientLibrary,
+  backupClientLibrary,
+  revertClientLibrary,
+} = require("../../utilities/fileSystem/clientLibrary")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -231,27 +236,54 @@ exports.create = async function (ctx) {
 }
 
 exports.update = async function (ctx) {
-  const url = await getAppUrlIfNotInUse(ctx)
+  const data = await updateAppPackage(ctx, ctx.request.body, ctx.params.appId)
+  ctx.status = 200
+  ctx.body = data
+}
+
+exports.updateClient = async function (ctx) {
+  // Get current app version
   const db = new CouchDB(ctx.params.appId)
   const application = await db.get(DocumentTypes.APP_METADATA)
+  const currentVersion = application.version
 
-  const data = ctx.request.body
+  // Update client library and manifest
-  const newData = { ...application, ...data, url }
+  if (!env.isTest()) {
-  if (ctx.request.body._rev !== application._rev) {
+    await backupClientLibrary(ctx.params.appId)
-    newData._rev = application._rev
+    await updateClientLibrary(ctx.params.appId)
   }
 
-  // the locked by property is attached by server but generated from
+  // Update versions in app package
-  // Redis, shouldn't ever store it
+  const appPackageUpdates = {
-  if (newData.lockedBy) {
+    version: packageJson.version,
-    delete newData.lockedBy
+    revertableVersion: currentVersion,
   }
-
+  const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId)
-  const response = await db.put(newData)
-  data._rev = response.rev
-
   ctx.status = 200
-  ctx.body = response
+  ctx.body = data
+}
+
+exports.revertClient = async function (ctx) {
+  // Check app can be reverted
+  const db = new CouchDB(ctx.params.appId)
+  const application = await db.get(DocumentTypes.APP_METADATA)
+  if (!application.revertableVersion) {
+    ctx.throw(400, "There is no version to revert to")
+  }
+
+  // Update client library and manifest
+  if (!env.isTest()) {
+    await revertClientLibrary(ctx.params.appId)
+  }
+
+  // Update versions in app package
+  const appPackageUpdates = {
+    version: application.revertableVersion,
+    revertableVersion: null,
+  }
+  const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId)
+  ctx.status = 200
+  ctx.body = data
 }
 
 exports.delete = async function (ctx) {
@@ -269,6 +301,23 @@ exports.delete = async function (ctx) {
   ctx.body = result
 }
 
+const updateAppPackage = async (ctx, appPackage, appId) => {
+  const url = await getAppUrlIfNotInUse(ctx)
+  const db = new CouchDB(appId)
+  const application = await db.get(DocumentTypes.APP_METADATA)
+
+  const newAppPackage = { ...application, ...appPackage, url }
+  if (appPackage._rev !== application._rev) {
+    newAppPackage._rev = application._rev
+  }
+
+  // the locked by property is attached by server but generated from
+  // Redis, shouldn't ever store it
+  delete newAppPackage.lockedBy
+
+  return await db.put(newAppPackage)
+}
+
 const createEmptyAppPackage = async (ctx, app) => {
   const db = new CouchDB(app.appId)
 

packages/server/src/api/controllers/component.js

@@ -20,10 +20,14 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
   const definitions = {}
   for (let { manifest, library } of componentManifests) {
     for (let key of Object.keys(manifest)) {
-      const fullComponentName = `${library}/${key}`.toLowerCase()
+      if (key === "features") {
-      definitions[fullComponentName] = {
+        definitions[key] = manifest[key]
-        component: fullComponentName,
+      } else {
-        ...manifest[key],
+        const fullComponentName = `${library}/${key}`.toLowerCase()
+        definitions[fullComponentName] = {
+          component: fullComponentName,
+          ...manifest[key],
+        }
       }
     }
   }

packages/server/src/api/routes/application.js

@@ -11,6 +11,16 @@ router
   .get("/api/applications/:appId/appPackage", controller.fetchAppPackage)
   .put("/api/applications/:appId", authorized(BUILDER), controller.update)
   .post("/api/applications", authorized(BUILDER), controller.create)
+  .post(
+    "/api/applications/:appId/client/update",
+    authorized(BUILDER),
+    controller.updateClient
+  )
+  .post(
+    "/api/applications/:appId/client/revert",
+    authorized(BUILDER),
+    controller.revertClient
+  )
   .delete("/api/applications/:appId", authorized(BUILDER), controller.delete)
 
 module.exports = router

packages/server/src/api/routes/tests/application.spec.js

@@ -94,7 +94,7 @@ describe("/applications", () => {
   })
 
   describe("update", () => {
-    it("should be able to fetch the app package", async () => {
+    it("should be able to update the app package", async () => {
       const res = await request
         .put(`/api/applications/${config.getAppId()}`)
         .send({
@@ -107,6 +107,30 @@ describe("/applications", () => {
     })
   })
 
+  describe("manage client library version", () => {
+    it("should be able to update the app client library version", async () => {
+      console.log(config.getAppId())
+      await request
+        .post(`/api/applications/${config.getAppId()}/client/update`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+    })
+    it("should be able to revert the app client library version", async () => {
+      // We need to first update the version so that we can then revert
+      await request
+        .post(`/api/applications/${config.getAppId()}/client/update`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      await request
+        .post(`/api/applications/${config.getAppId()}/client/revert`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+    })
+  })
+
   describe("edited at", () => {
     it("middleware should set edited at", async () => {
       const headers = config.defaultHeaders()

packages/server/src/utilities/fileSystem/clientLibrary.js

@@ -0,0 +1,154 @@
+const { join } = require("path")
+const { ObjectStoreBuckets } = require("../../constants")
+const fs = require("fs")
+const { upload, retrieveToTmp, streamUpload } = require("./utilities")
+const { resolve } = require("../centralPath")
+const env = require("../../environment")
+const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+
+/**
+ * Client library paths in the object store:
+ * Previously, the entire standard-components package was downloaded from NPM
+ * as a tarball and extracted to the object store, even though only the manifest
+ * was ever needed. Therefore we need to support old apps which may still have
+ * the manifest at this location for the first update.
+ *
+ * The new paths for the in-use version are:
+ * {appId}/manifest.json
+ * {appId}/budibase-client.js
+ *
+ * The paths for the backups are:
+ * {appId}/manifest.json.bak
+ * {appId}/budibase-client.js.bak
+ *
+ * We don't rely on NPM at all any more, as when updating to the latest version
+ * we pull both the manifest and client bundle from the server's dependencies
+ * in the local file system.
+ */
+
+/**
+ * Backs up the current client library version by copying both the manifest
+ * and client bundle to .bak extensions in the object store. Only the one
+ * previous version is stored as a backup, which can be reverted to.
+ * @param appId The app ID to backup
+ * @returns {Promise<void>}
+ */
+exports.backupClientLibrary = async appId => {
+  // Copy existing manifest to tmp
+  let tmpManifestPath
+  try {
+    // Try to load the manifest from the new file location
+    tmpManifestPath = await retrieveToTmp(
+      ObjectStoreBuckets.APPS,
+      join(appId, "manifest.json")
+    )
+  } catch (error) {
+    // Fallback to loading it from the old location for old apps
+    tmpManifestPath = await retrieveToTmp(
+      ObjectStoreBuckets.APPS,
+      join(
+        appId,
+        "node_modules",
+        "budibase",
+        "standard-components",
+        "package",
+        "manifest.json"
+      )
+    )
+  }
+
+  // Copy existing client lib to tmp
+  const tmpClientPath = await retrieveToTmp(
+    ObjectStoreBuckets.APPS,
+    join(appId, "budibase-client.js")
+  )
+
+  // Upload manifest and client library as backups
+  const manifestUpload = upload({
+    bucket: ObjectStoreBuckets.APPS,
+    filename: join(appId, "manifest.json.bak"),
+    path: tmpManifestPath,
+    type: "application/json",
+  })
+  const clientUpload = upload({
+    bucket: ObjectStoreBuckets.APPS,
+    filename: join(appId, "budibase-client.js.bak"),
+    path: tmpClientPath,
+    type: "application/javascript",
+  })
+  await Promise.all([manifestUpload, clientUpload])
+}
+
+/**
+ * Uploads the latest version of the component manifest and the client library
+ * to the object store, overwriting the existing version.
+ * @param appId The app ID to update
+ * @returns {Promise<void>}
+ */
+exports.updateClientLibrary = async appId => {
+  let manifest, client
+
+  if (env.isDev()) {
+    // Load the symlinked version in dev which is always the newest
+    manifest = require.resolve("@budibase/standard-components/manifest.json")
+    client = require.resolve("@budibase/client")
+  } else {
+    // Load the bundled version in prod
+    manifest = resolve(TOP_LEVEL_PATH, "client", "manifest.json")
+    client = resolve(TOP_LEVEL_PATH, "client", "budibase-client.js")
+  }
+
+  // Upload latest manifest and client library
+  const manifestUpload = streamUpload(
+    ObjectStoreBuckets.APPS,
+    join(appId, "manifest.json"),
+    fs.createReadStream(manifest),
+    {
+      ContentType: "application/json",
+    }
+  )
+  const clientUpload = streamUpload(
+    ObjectStoreBuckets.APPS,
+    join(appId, "budibase-client.js"),
+    fs.createReadStream(client),
+    {
+      ContentType: "application/javascript",
+    }
+  )
+  await Promise.all([manifestUpload, clientUpload])
+}
+
+/**
+ * Reverts the version of the client library and manifest to the previously
+ * used version for an app.
+ * @param appId The app ID to revert
+ * @returns {Promise<void>}
+ */
+exports.revertClientLibrary = async appId => {
+  // Copy backups manifest to tmp directory
+  const tmpManifestPath = await retrieveToTmp(
+    ObjectStoreBuckets.APPS,
+    join(appId, "manifest.json.bak")
+  )
+
+  // Copy backup client lib to tmp
+  const tmpClientPath = await retrieveToTmp(
+    ObjectStoreBuckets.APPS,
+    join(appId, "budibase-client.js.bak")
+  )
+
+  // Upload backups as new versions
+  const manifestUpload = upload({
+    bucket: ObjectStoreBuckets.APPS,
+    filename: join(appId, "manifest.json"),
+    path: tmpManifestPath,
+    type: "application/json",
+  })
+  const clientUpload = upload({
+    bucket: ObjectStoreBuckets.APPS,
+    filename: join(appId, "budibase-client.js"),
+    path: tmpClientPath,
+    type: "application/javascript",
+  })
+  await Promise.all([manifestUpload, clientUpload])
+}

packages/server/src/utilities/fileSystem/index.js

@@ -13,7 +13,7 @@ const {
   deleteFolder,
   downloadTarball,
 } = require("./utilities")
-const { downloadLibraries, uploadClientLibrary } = require("./newApp")
+const { updateClientLibrary } = require("./clientLibrary")
 const download = require("download")
 const env = require("../../environment")
 const { homedir } = require("os")
@@ -139,13 +139,12 @@ exports.performBackup = async (appId, backupName) => {
 }
 
 /**
- * Downloads required libraries and creates a new path in the object store.
+ * Uploads the latest client library to the object store.
  * @param {string} appId The ID of the app which is being created.
  * @return {Promise<void>} once promise completes app resources should be ready in object store.
  */
 exports.createApp = async appId => {
-  await downloadLibraries(appId)
+  await updateClientLibrary(appId)
-  await uploadClientLibrary(appId)
 }
 
 /**
@@ -193,8 +192,17 @@ exports.getComponentLibraryManifest = async (appId, library) => {
     delete require.cache[require.resolve(path)]
     return require(path)
   }
-  const path = join(appId, "node_modules", library, "package", filename)
+
-  let resp = await retrieve(ObjectStoreBuckets.APPS, path)
+  let resp
+  try {
+    // Try to load the manifest from the new file location
+    const path = join(appId, filename)
+    resp = await retrieve(ObjectStoreBuckets.APPS, path)
+  } catch (error) {
+    // Fallback to loading it from the old location for old apps
+    const path = join(appId, "node_modules", library, "package", filename)
+    resp = await retrieve(ObjectStoreBuckets.APPS, path)
+  }
   if (typeof resp !== "string") {
     resp = resp.toString("utf8")
   }

packages/server/src/utilities/fileSystem/newApp.js

@@ -1,36 +0,0 @@
-const packageJson = require("../../../package.json")
-const { join } = require("path")
-const { ObjectStoreBuckets } = require("../../constants")
-const { streamUpload, downloadTarball } = require("./utilities")
-const fs = require("fs")
-
-const BUCKET_NAME = ObjectStoreBuckets.APPS
-
-// can't really test this due to the downloading nature of it, wouldn't be a great test case
-/* istanbul ignore next */
-exports.downloadLibraries = async appId => {
-  const LIBRARIES = ["standard-components"]
-
-  const paths = {}
-  // Need to download tarballs directly from NPM as our users may not have node on their machine
-  for (let lib of LIBRARIES) {
-    // download tarball
-    const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz`
-    const path = join(appId, "node_modules", "@budibase", lib)
-    paths[`@budibase/${lib}`] = await downloadTarball(
-      registryUrl,
-      BUCKET_NAME,
-      path
-    )
-  }
-  return paths
-}
-
-exports.uploadClientLibrary = async appId => {
-  const sourcepath = require.resolve("@budibase/client")
-  const destPath = join(appId, "budibase-client.js")
-
-  await streamUpload(BUCKET_NAME, destPath, fs.createReadStream(sourcepath), {
-    ContentType: "application/javascript",
-  })
-}

packages/standard-components/manifest.json

@@ -1,4 +1,8 @@
 {
+  "features": {
+    "spectrumThemes": true,
+    "intelligentLoading": true
+  },
   "layout": {
     "name": "Layout",
     "description": "This component is specific only to layouts",