From 9bf6600ccd93dbfdc312cd62ac27e759fcdc7041 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 7 Jul 2021 13:53:59 +0100 Subject: [PATCH 01/16] Add feature flags to manifest --- packages/server/src/api/controllers/component.js | 12 ++++++++---- packages/standard-components/manifest.json | 4 ++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/component.js b/packages/server/src/api/controllers/component.js index c678d8e587..06cb2cd211 100644 --- a/packages/server/src/api/controllers/component.js +++ b/packages/server/src/api/controllers/component.js @@ -20,10 +20,14 @@ exports.fetchAppComponentDefinitions = async function (ctx) { const definitions = {} for (let { manifest, library } of componentManifests) { for (let key of Object.keys(manifest)) { - const fullComponentName = `${library}/${key}`.toLowerCase() - definitions[fullComponentName] = { - component: fullComponentName, - ...manifest[key], + if (key === "features") { + definitions[key] = manifest[key] + } else { + const fullComponentName = `${library}/${key}`.toLowerCase() + definitions[fullComponentName] = { + component: fullComponentName, + ...manifest[key], + } } } } diff --git a/packages/standard-components/manifest.json b/packages/standard-components/manifest.json index be66a4ce87..ec32a5879d 100644 --- a/packages/standard-components/manifest.json +++ b/packages/standard-components/manifest.json @@ -1,4 +1,8 @@ { + "features": { + "spectrumThemes": true, + "intelligentLoading": true + }, "layout": { "name": "Layout", "description": "This component is specific only to layouts", From 24a2c547ebef662490b21b6b660908cf966ab8ab Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 7 Jul 2021 13:54:21 +0100 Subject: [PATCH 02/16] Store client features in frontend store --- packages/builder/src/builderStore/store/frontend.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index ac837978a9..c33a1d653a 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -32,6 +32,10 @@ const INITIAL_FRONTEND_STATE = { layouts: [], screens: [], components: [], + clientFeatures: { + spectrumThemes: false, + intelligentLoading: false, + }, currentFrontEndType: "none", selectedScreenId: "", selectedLayoutId: "", @@ -56,6 +60,10 @@ export const getFrontendStore = () => { ...state, libraries: application.componentLibraries, components, + clientFeatures: { + ...state.clientFeatures, + ...components.features, + }, name: application.name, description: application.description, appId: application.appId, From cc7b07a02021aeebfd2ad36d577324dbffc4ad7f Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 7 Jul 2021 13:54:44 +0100 Subject: [PATCH 03/16] Use client features to determine when to show client preview on initial load --- .../AppPreview/CurrentItemPreview.svelte | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte b/packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte index 3e4789a31d..607061013d 100644 --- a/packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte +++ b/packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte @@ -74,14 +74,14 @@ // Initialise the app when mounted iframe.contentWindow.addEventListener( "ready", - () => refreshContent(strippedJson), - { once: true } - ) - - // Display the client app once the iframe is initialised - iframe.contentWindow.addEventListener( - "iframe-loaded", - () => (loading = false), + () => { + // Display preview immediately if the intelligent loading feature + // is not supported + if (!$store.clientFeatures.intelligentLoading) { + loading = false + } + refreshContent(strippedJson) + }, { once: true } ) @@ -106,11 +106,9 @@ idToDelete = data.id confirmDeleteDialog.show() } else if (type === "preview-loaded") { - // We can use this in future to delay displaying the preview - // until the client app has actually initialised. - // This makes a smoother loading experience, but is not backwards - // compatible with old client library versions. - // So do nothing with this for now. + // Wait for this event to show the client library if intelligent + // loading is supported + loading = false } else { console.warning(`Client sent unknown event type: ${type}`) } From fb6c115adf46e0f06e82de7a67870e1b83c0eac8 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 7 Jul 2021 13:55:05 +0100 Subject: [PATCH 04/16] Use client feature flags to conditionally show theme picker --- .../app/[application]/design/[assetType]/_layout.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/app/[application]/design/[assetType]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/design/[assetType]/_layout.svelte index 533522af3c..241b0d5200 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[assetType]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[assetType]/_layout.svelte @@ -150,7 +150,9 @@ {#if $currentAsset}
- + {#if $store.clientFeatures.spectrumThemes} + + {/if}
From 0ff1f0fbe949b8fba080205abec1188132408024 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 7 Jul 2021 17:07:16 +0100 Subject: [PATCH 05/16] Fix crash when referencing a component definition that doesn't exist --- .../design/NavigationPanel/ComponentDropdownMenu.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte b/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte index df30a17640..24050e1088 100644 --- a/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte +++ b/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte @@ -65,7 +65,7 @@ } -{#if definition.editable !== false} +{#if definition?.editable !== false}
From 0a44b1e3d83dbae2f0ebe8c08b626cdb34bbd766 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 7 Jul 2021 17:07:42 +0100 Subject: [PATCH 06/16] Add client libary update management from inside the builder --- .../src/builderStore/store/frontend.js | 1 + .../src/components/deploy/VersionModal.svelte | 83 +++++++++++++++++++ .../builder/app/[application]/_layout.svelte | 2 + .../server/src/api/controllers/application.js | 56 ++++++++----- packages/server/src/api/routes/application.js | 5 ++ .../server/src/utilities/fileSystem/index.js | 16 +++- .../server/src/utilities/fileSystem/newApp.js | 47 +++++------ 7 files changed, 158 insertions(+), 52 deletions(-) create mode 100644 packages/builder/src/components/deploy/VersionModal.svelte diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index c33a1d653a..cea4e98dfe 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -75,6 +75,7 @@ export const getFrontendStore = () => { appInstance: application.instance, clientLibPath, previousTopNavPath: {}, + version: application.version, })) await hostingStore.actions.fetch() diff --git a/packages/builder/src/components/deploy/VersionModal.svelte b/packages/builder/src/components/deploy/VersionModal.svelte new file mode 100644 index 0000000000..4a42fac0c1 --- /dev/null +++ b/packages/builder/src/components/deploy/VersionModal.svelte @@ -0,0 +1,83 @@ + + +
+ +
+ + + {#if updateAvailable} + + This app is currently using version {$store.version}, but version + {clientPackage.version} is available. Updates can contain new + features, performance improvements and bug fixes. +

+ Would you like to update this app? + + {:else} + + This app is currently using version {$store.version} which is the + latest version available. + + {/if} + + + + diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index eab765b502..f3ae6e7c8e 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -4,6 +4,7 @@ import { Icon, ActionGroup, Tabs, Tab } from "@budibase/bbui" import DeployModal from "components/deploy/DeployModal.svelte" import RevertModal from "components/deploy/RevertModal.svelte" + import VersionModal from "components/deploy/VersionModal.svelte" import { get } from "builderStore/api" import { isActive, goto, layout } from "@roxi/routify" import Logo from "assets/bb-emblem.svg" @@ -80,6 +81,7 @@
+ { + const url = await getAppUrlIfNotInUse(ctx) + const db = new CouchDB(appId) + const application = await db.get(DocumentTypes.APP_METADATA) + + const newAppPackage = { ...application, ...appPackage, url } + if (appPackage._rev !== application._rev) { + newAppPackage._rev = application._rev + } + + // the locked by property is attached by server but generated from + // Redis, shouldn't ever store it + if (newAppPackage.lockedBy) { + delete newAppPackage.lockedBy + } + + const response = await db.put(newAppPackage) + console.log(response) + + return response +} + const createEmptyAppPackage = async (ctx, app) => { const db = new CouchDB(app.appId) diff --git a/packages/server/src/api/routes/application.js b/packages/server/src/api/routes/application.js index a7209df3e9..6ffd3a4b5b 100644 --- a/packages/server/src/api/routes/application.js +++ b/packages/server/src/api/routes/application.js @@ -11,6 +11,11 @@ router .get("/api/applications/:appId/appPackage", controller.fetchAppPackage) .put("/api/applications/:appId", authorized(BUILDER), controller.update) .post("/api/applications", authorized(BUILDER), controller.create) + .post( + "/api/applications/:appId/client/update", + authorized(BUILDER), + controller.updateClient + ) .delete("/api/applications/:appId", authorized(BUILDER), controller.delete) module.exports = router diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index afacbf8cdf..c64d83dd67 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -13,7 +13,7 @@ const { deleteFolder, downloadTarball, } = require("./utilities") -const { downloadLibraries, uploadClientLibrary } = require("./newApp") +const { uploadClientLibrary } = require("./newApp") const download = require("download") const env = require("../../environment") const { homedir } = require("os") @@ -144,7 +144,6 @@ exports.performBackup = async (appId, backupName) => { * @return {Promise} once promise completes app resources should be ready in object store. */ exports.createApp = async appId => { - await downloadLibraries(appId) await uploadClientLibrary(appId) } @@ -193,8 +192,17 @@ exports.getComponentLibraryManifest = async (appId, library) => { delete require.cache[require.resolve(path)] return require(path) } - const path = join(appId, "node_modules", library, "package", filename) - let resp = await retrieve(ObjectStoreBuckets.APPS, path) + + let resp + try { + // Try to load the manifest from the new file location + const path = join(appId, filename) + resp = await retrieve(ObjectStoreBuckets.APPS, path) + } catch (error) { + // Fallback to loading it from the old location for old apps + const path = join(appId, "node_modules", library, "package", filename) + resp = await retrieve(ObjectStoreBuckets.APPS, path) + } if (typeof resp !== "string") { resp = resp.toString("utf8") } diff --git a/packages/server/src/utilities/fileSystem/newApp.js b/packages/server/src/utilities/fileSystem/newApp.js index 735f0d523e..749e7a278d 100644 --- a/packages/server/src/utilities/fileSystem/newApp.js +++ b/packages/server/src/utilities/fileSystem/newApp.js @@ -1,36 +1,27 @@ -const packageJson = require("../../../package.json") const { join } = require("path") const { ObjectStoreBuckets } = require("../../constants") -const { streamUpload, downloadTarball } = require("./utilities") +const { streamUpload } = require("./utilities") const fs = require("fs") const BUCKET_NAME = ObjectStoreBuckets.APPS -// can't really test this due to the downloading nature of it, wouldn't be a great test case -/* istanbul ignore next */ -exports.downloadLibraries = async appId => { - const LIBRARIES = ["standard-components"] - - const paths = {} - // Need to download tarballs directly from NPM as our users may not have node on their machine - for (let lib of LIBRARIES) { - // download tarball - const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz` - const path = join(appId, "node_modules", "@budibase", lib) - paths[`@budibase/${lib}`] = await downloadTarball( - registryUrl, - BUCKET_NAME, - path - ) - } - return paths -} - exports.uploadClientLibrary = async appId => { - const sourcepath = require.resolve("@budibase/client") - const destPath = join(appId, "budibase-client.js") - - await streamUpload(BUCKET_NAME, destPath, fs.createReadStream(sourcepath), { - ContentType: "application/javascript", - }) + await streamUpload( + BUCKET_NAME, + join(appId, "budibase-client.js"), + fs.createReadStream(require.resolve("@budibase/client")), + { + ContentType: "application/javascript", + } + ) + await streamUpload( + BUCKET_NAME, + join(appId, "manifest.json"), + fs.createReadStream( + require.resolve("@budibase/standard-components/manifest.json") + ), + { + ContentType: "application/javascript", + } + ) } From 9440e4a5bf2f927636d3ddef21da72665e8746fd Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 7 Jul 2021 17:35:28 +0100 Subject: [PATCH 07/16] Add UI to revert app version and initial work on app revert backend --- .../src/builderStore/store/frontend.js | 1 + .../src/components/deploy/VersionModal.svelte | 64 ++++++++++++++----- .../server/src/api/controllers/application.js | 13 +++- 3 files changed, 61 insertions(+), 17 deletions(-) diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index cea4e98dfe..b9cd134067 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -76,6 +76,7 @@ export const getFrontendStore = () => { clientLibPath, previousTopNavPath: {}, version: application.version, + revertableVersion: application.revertableVersion, })) await hostingStore.actions.fetch() diff --git a/packages/builder/src/components/deploy/VersionModal.svelte b/packages/builder/src/components/deploy/VersionModal.svelte index 4a42fac0c1..541f37ad50 100644 --- a/packages/builder/src/components/deploy/VersionModal.svelte +++ b/packages/builder/src/components/deploy/VersionModal.svelte @@ -5,6 +5,7 @@ notifications, ModalContent, Body, + Button, } from "@budibase/bbui" import { store } from "builderStore" import api from "builderStore/api" @@ -14,6 +15,19 @@ $: appId = $store.appId $: updateAvailable = clientPackage.version !== $store.version + $: revertAvailable = $store.revertableVersion != null + + const refreshAppPackage = async () => { + const applicationPkg = await api.get( + `/api/applications/${appId}/appPackage` + ) + const pkg = await applicationPkg.json() + if (applicationPkg.ok) { + await store.actions.initialise(pkg) + } else { + throw new Error(pkg) + } + } const update = async () => { try { @@ -24,18 +38,7 @@ if (response.status !== 200) { throw json.message } - - // Reset frontend state after revert - const applicationPkg = await api.get( - `/api/applications/${appId}/appPackage` - ) - const pkg = await applicationPkg.json() - if (applicationPkg.ok) { - await store.actions.initialise(pkg) - } else { - throw new Error(pkg) - } - + await refreshAppPackage() notifications.success( `App updated successfully to version ${clientPackage.version}` ) @@ -43,6 +46,25 @@ notifications.error(`Error updating app: ${err}`) } } + + const revert = async () => { + try { + const response = await api.post( + `/api/applications/${appId}/client/revert` + ) + const json = await response.json() + if (response.status !== 200) { + throw json.message + } + await refreshAppPackage() + notifications.success( + `App reverted successfully to version ${$store.revertableVersion}` + ) + } catch (err) { + notifications.error(`Error reverting app: ${err}`) + } + updateModal.hide() + }
@@ -56,13 +78,16 @@ onConfirm={update} showConfirmButton={updateAvailable} > +
+ {#if revertAvailable} + + {/if} +
{#if updateAvailable} This app is currently using version {$store.version}, but version - {clientPackage.version} is available. Updates can contain new - features, performance improvements and bug fixes. -

- Would you like to update this app? + {clientPackage.version} is available. Updates can contain new features, + performance improvements and bug fixes. {:else} @@ -70,6 +95,13 @@ latest version available. {/if} + {#if revertAvailable} + + You can revert this app to version + {$store.revertableVersion} + if you're experiencing issues with the current version. + + {/if} diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index 6719a95211..e8ace42bcd 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -241,8 +241,19 @@ exports.update = async function (ctx) { } exports.updateClient = async function (ctx) { + // Get current app version + const db = new CouchDB(ctx.params.appId) + const application = await db.get(DocumentTypes.APP_METADATA) + const currentVersion = application.version + + // Update client library and manifest await uploadClientLibrary(ctx.params.appId) - const appPackageUpdates = { version: packageJson.version } + + // Update versions in app package + const appPackageUpdates = { + version: packageJson.version, + revertableVersion: currentVersion, + } const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId) ctx.status = 200 ctx.body = data From a7a081da8c3ae7953ed70587a20cbd2baa30d6e5 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 8 Jul 2021 12:55:51 +0100 Subject: [PATCH 08/16] Add application/json as a known file type --- packages/auth/src/objectStore/index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/auth/src/objectStore/index.js b/packages/auth/src/objectStore/index.js index 80875fdfee..81bdd06b62 100644 --- a/packages/auth/src/objectStore/index.js +++ b/packages/auth/src/objectStore/index.js @@ -22,11 +22,13 @@ const CONTENT_TYPE_MAP = { html: "text/html", css: "text/css", js: "application/javascript", + json: "application/json", } const STRING_CONTENT_TYPES = [ CONTENT_TYPE_MAP.html, CONTENT_TYPE_MAP.css, CONTENT_TYPE_MAP.js, + CONTENT_TYPE_MAP.json, ] // does normal sanitization and then swaps dev apps to apps From 4eb6d1e624edf10dc2c53fc9f1794196ca01732c Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 8 Jul 2021 12:56:35 +0100 Subject: [PATCH 09/16] Fix modal flashing when reverting app version and fix version number being incorrect --- .../builder/src/components/deploy/VersionModal.svelte | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/components/deploy/VersionModal.svelte b/packages/builder/src/components/deploy/VersionModal.svelte index 541f37ad50..0fb061face 100644 --- a/packages/builder/src/components/deploy/VersionModal.svelte +++ b/packages/builder/src/components/deploy/VersionModal.svelte @@ -38,7 +38,9 @@ if (response.status !== 200) { throw json.message } - await refreshAppPackage() + + // Don't wait for the async refresh, since this causes modal flashing + refreshAppPackage() notifications.success( `App updated successfully to version ${clientPackage.version}` ) @@ -49,6 +51,7 @@ const revert = async () => { try { + const revertableVersion = $store.revertableVersion const response = await api.post( `/api/applications/${appId}/client/revert` ) @@ -56,9 +59,11 @@ if (response.status !== 200) { throw json.message } - await refreshAppPackage() + + // Don't wait for the async refresh, since this causes modal flashing + refreshAppPackage() notifications.success( - `App reverted successfully to version ${$store.revertableVersion}` + `App reverted successfully to version ${revertableVersion}` ) } catch (err) { notifications.error(`Error reverting app: ${err}`) From 0ce553eaf2852993980575ec5d4a845d0f6d265a Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 8 Jul 2021 12:56:54 +0100 Subject: [PATCH 10/16] Add endpoint to revert client app version --- .../server/src/api/controllers/application.js | 36 ++++- packages/server/src/api/routes/application.js | 5 + .../src/utilities/fileSystem/clientLibrary.js | 149 ++++++++++++++++++ .../server/src/utilities/fileSystem/index.js | 6 +- .../server/src/utilities/fileSystem/newApp.js | 27 ---- 5 files changed, 185 insertions(+), 38 deletions(-) create mode 100644 packages/server/src/utilities/fileSystem/clientLibrary.js delete mode 100644 packages/server/src/utilities/fileSystem/newApp.js diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index e8ace42bcd..d8e1e232d3 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -34,9 +34,10 @@ const { const { clientLibraryPath } = require("../../utilities") const { getAllLocks } = require("../../utilities/redis") const { - uploadClientLibrary, - downloadLibraries, -} = require("../../utilities/fileSystem/newApp") + updateClientLibrary, + backupClientLibrary, + revertClientLibrary, +} = require("../../utilities/fileSystem/clientLibrary") const URL_REGEX_SLASH = /\/|\\/g @@ -247,7 +248,8 @@ exports.updateClient = async function (ctx) { const currentVersion = application.version // Update client library and manifest - await uploadClientLibrary(ctx.params.appId) + await backupClientLibrary(ctx.params.appId) + await updateClientLibrary(ctx.params.appId) // Update versions in app package const appPackageUpdates = { @@ -259,6 +261,27 @@ exports.updateClient = async function (ctx) { ctx.body = data } +exports.revertClient = async function (ctx) { + // Check app can be reverted + const db = new CouchDB(ctx.params.appId) + const application = await db.get(DocumentTypes.APP_METADATA) + if (!application.revertableVersion) { + ctx.throw(400, "There is no version to revert to") + } + + // Update client library and manifest + await revertClientLibrary(ctx.params.appId) + + // Update versions in app package + const appPackageUpdates = { + version: application.revertableVersion, + revertableVersion: null, + } + const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId) + ctx.status = 200 + ctx.body = data +} + exports.delete = async function (ctx) { const db = new CouchDB(ctx.params.appId) @@ -290,10 +313,7 @@ const updateAppPackage = async (ctx, appPackage, appId) => { delete newAppPackage.lockedBy } - const response = await db.put(newAppPackage) - console.log(response) - - return response + return await db.put(newAppPackage) } const createEmptyAppPackage = async (ctx, app) => { diff --git a/packages/server/src/api/routes/application.js b/packages/server/src/api/routes/application.js index 6ffd3a4b5b..c2eb19e101 100644 --- a/packages/server/src/api/routes/application.js +++ b/packages/server/src/api/routes/application.js @@ -16,6 +16,11 @@ router authorized(BUILDER), controller.updateClient ) + .post( + "/api/applications/:appId/client/revert", + authorized(BUILDER), + controller.revertClient + ) .delete("/api/applications/:appId", authorized(BUILDER), controller.delete) module.exports = router diff --git a/packages/server/src/utilities/fileSystem/clientLibrary.js b/packages/server/src/utilities/fileSystem/clientLibrary.js new file mode 100644 index 0000000000..6b7e8d837d --- /dev/null +++ b/packages/server/src/utilities/fileSystem/clientLibrary.js @@ -0,0 +1,149 @@ +const { join } = require("path") +const { ObjectStoreBuckets } = require("../../constants") +const fs = require("fs") +const { upload, retrieveToTmp, streamUpload } = require("./utilities") + +/** + * Client library paths in the object store: + * Previously, the entire standard-components package was downloaded from NPM + * as a tarball and extracted to the object store, even though only the manifest + * was ever needed. Therefore we need to support old apps which may still have + * the manifest at this location for the first update. + * + * The new paths for the in-use version are: + * {appId}/manifest.json + * {appId}/budibase-client.js + * + * The paths for the backups are: + * {appId}/manifest.json.bak + * {appId}/budibase-client.js.bak + * + * We don't rely on NPM at all any more, as when updating to the latest version + * we pull both the manifest and client bundle from the server's dependencies + * in the local file system. + */ + +/** + * Backs up the current client library version by copying both the manifest + * and client bundle to .bak extensions in the object store. Only the one + * previous version is stored as a backup, which can be reverted to. + * @param appId The app ID to backup + * @returns {Promise} + */ +exports.backupClientLibrary = async appId => { + let tmpManifestPath + let tmpClientPath + + // Copy existing manifest to tmp + try { + // Try to load the manifest from the new file location + tmpManifestPath = await retrieveToTmp( + ObjectStoreBuckets.APPS, + join(appId, "manifest.json") + ) + } catch (error) { + // Fallback to loading it from the old location for old apps + tmpManifestPath = await retrieveToTmp( + ObjectStoreBuckets.APPS, + join( + appId, + "node_modules", + "budibase", + "standard-components", + "package", + "manifest.json" + ) + ) + } + + // Copy existing client lib to tmp + tmpClientPath = await retrieveToTmp( + ObjectStoreBuckets.APPS, + join(appId, "budibase-client.js") + ) + + // Upload manifest as backup + await upload({ + bucket: ObjectStoreBuckets.APPS, + filename: join(appId, "manifest.json.bak"), + path: tmpManifestPath, + type: "application/json", + }) + + // Upload client library as backup + await upload({ + bucket: ObjectStoreBuckets.APPS, + filename: join(appId, "budibase-client.js.bak"), + path: tmpClientPath, + type: "application/javascript", + }) +} + +/** + * Uploads the latest version of the component manifest and the client library + * to the object store, overwriting the existing version. + * @param appId The app ID to update + * @returns {Promise} + */ +exports.updateClientLibrary = async appId => { + // Upload latest component manifest + await streamUpload( + ObjectStoreBuckets.APPS, + join(appId, "manifest.json"), + fs.createReadStream( + require.resolve("@budibase/standard-components/manifest.json") + ), + { + ContentType: "application/json", + } + ) + + // Upload latest component library + await streamUpload( + ObjectStoreBuckets.APPS, + join(appId, "budibase-client.js"), + fs.createReadStream(require.resolve("@budibase/client")), + { + ContentType: "application/javascript", + } + ) +} + +/** + * Reverts the version of the client library and manifest to the previously + * used version for an app. + * @param appId The app ID to revert + * @returns {Promise} + */ +exports.revertClientLibrary = async appId => { + let tmpManifestPath + let tmpClientPath + + // Copy backup manifest to tmp + tmpManifestPath = await retrieveToTmp( + ObjectStoreBuckets.APPS, + join(appId, "manifest.json.bak") + ) + + // Copy backup client lib to tmp + tmpClientPath = await retrieveToTmp( + ObjectStoreBuckets.APPS, + join(appId, "budibase-client.js.bak") + ) + + // Upload manifest backup + await upload({ + bucket: ObjectStoreBuckets.APPS, + filename: join(appId, "manifest.json"), + path: tmpManifestPath, + type: "application/json", + }) + + // Upload client library backup + await upload({ + bucket: ObjectStoreBuckets.APPS, + filename: join(appId, "budibase-client.js"), + path: tmpClientPath, + type: "application/javascript", + }) +} diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index c64d83dd67..b83ff03854 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -13,7 +13,7 @@ const { deleteFolder, downloadTarball, } = require("./utilities") -const { uploadClientLibrary } = require("./newApp") +const { updateClientLibrary } = require("./clientLibrary") const download = require("download") const env = require("../../environment") const { homedir } = require("os") @@ -139,12 +139,12 @@ exports.performBackup = async (appId, backupName) => { } /** - * Downloads required libraries and creates a new path in the object store. + * Uploads the latest client library to the object store. * @param {string} appId The ID of the app which is being created. * @return {Promise} once promise completes app resources should be ready in object store. */ exports.createApp = async appId => { - await uploadClientLibrary(appId) + await updateClientLibrary(appId) } /** diff --git a/packages/server/src/utilities/fileSystem/newApp.js b/packages/server/src/utilities/fileSystem/newApp.js deleted file mode 100644 index 749e7a278d..0000000000 --- a/packages/server/src/utilities/fileSystem/newApp.js +++ /dev/null @@ -1,27 +0,0 @@ -const { join } = require("path") -const { ObjectStoreBuckets } = require("../../constants") -const { streamUpload } = require("./utilities") -const fs = require("fs") - -const BUCKET_NAME = ObjectStoreBuckets.APPS - -exports.uploadClientLibrary = async appId => { - await streamUpload( - BUCKET_NAME, - join(appId, "budibase-client.js"), - fs.createReadStream(require.resolve("@budibase/client")), - { - ContentType: "application/javascript", - } - ) - await streamUpload( - BUCKET_NAME, - join(appId, "manifest.json"), - fs.createReadStream( - require.resolve("@budibase/standard-components/manifest.json") - ), - { - ContentType: "application/javascript", - } - ) -} From 959405332d6a3a674bb28aff377d3caf7695cc8d Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 8 Jul 2021 13:20:52 +0100 Subject: [PATCH 11/16] Add tests for updating and reverting client library version --- .../server/src/api/controllers/application.js | 10 ++++--- .../src/api/routes/tests/application.spec.js | 26 ++++++++++++++++++- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index d8e1e232d3..1a1b8d51c3 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -248,8 +248,10 @@ exports.updateClient = async function (ctx) { const currentVersion = application.version // Update client library and manifest - await backupClientLibrary(ctx.params.appId) - await updateClientLibrary(ctx.params.appId) + if (!env.isTest()) { + await backupClientLibrary(ctx.params.appId) + await updateClientLibrary(ctx.params.appId) + } // Update versions in app package const appPackageUpdates = { @@ -270,7 +272,9 @@ exports.revertClient = async function (ctx) { } // Update client library and manifest - await revertClientLibrary(ctx.params.appId) + if (!env.isTest()) { + await revertClientLibrary(ctx.params.appId) + } // Update versions in app package const appPackageUpdates = { diff --git a/packages/server/src/api/routes/tests/application.spec.js b/packages/server/src/api/routes/tests/application.spec.js index 2333787e6e..05e0bc231b 100644 --- a/packages/server/src/api/routes/tests/application.spec.js +++ b/packages/server/src/api/routes/tests/application.spec.js @@ -94,7 +94,7 @@ describe("/applications", () => { }) describe("update", () => { - it("should be able to fetch the app package", async () => { + it("should be able to update the app package", async () => { const res = await request .put(`/api/applications/${config.getAppId()}`) .send({ @@ -107,6 +107,30 @@ describe("/applications", () => { }) }) + describe("manage client library version", () => { + it("should be able to update the app client library version", async () => { + console.log(config.getAppId()) + await request + .post(`/api/applications/${config.getAppId()}/client/update`) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + }) + it("should be able to revert the app client library version", async () => { + // We need to first update the version so that we can then revert + await request + .post(`/api/applications/${config.getAppId()}/client/update`) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + await request + .post(`/api/applications/${config.getAppId()}/client/revert`) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + }) + }) + describe("edited at", () => { it("middleware should set edited at", async () => { const headers = config.defaultHeaders() From bd27aa0caf69e8e853f24de1d33eedeb1c314e7b Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 8 Jul 2021 14:00:14 +0100 Subject: [PATCH 12/16] Move standard components from server dev dependencies to real dependencies --- packages/server/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/package.json b/packages/server/package.json index 26ba52e656..ace5fe693f 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -61,6 +61,7 @@ "dependencies": { "@budibase/auth": "^0.9.71", "@budibase/client": "^0.9.71", + "@budibase/standard-components": "^0.9.71", "@budibase/string-templates": "^0.9.71", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", @@ -114,7 +115,6 @@ "devDependencies": { "@babel/core": "^7.14.3", "@babel/preset-env": "^7.14.4", - "@budibase/standard-components": "^0.9.71", "@jest/test-sequencer": "^24.8.0", "@types/bull": "^3.15.1", "@types/jest": "^26.0.23", From ef4a0f8c79076b3a01ee08aac6549e837f8e587f Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 9 Jul 2021 12:33:09 +0100 Subject: [PATCH 13/16] Bundle latest client library with the server and use it when updating apps in production --- packages/server/.gitignore | 1 + packages/server/package.json | 3 ++- .../src/utilities/fileSystem/clientLibrary.js | 24 +++++++++++++++---- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/packages/server/.gitignore b/packages/server/.gitignore index c737d00466..e8589f631d 100644 --- a/packages/server/.gitignore +++ b/packages/server/.gitignore @@ -2,6 +2,7 @@ node_modules/ myapps/ .env builder/* +client/* public/ db/dev.db/ dist diff --git a/packages/server/package.json b/packages/server/package.json index ace5fe693f..d3d99a5b16 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -13,7 +13,8 @@ "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/", "test": "jest --coverage --maxWorkers=2", "test:watch": "jest --watch", - "build:docker": "docker build . -t app-service", + "predocker": "copyfiles -f ../client/dist/budibase-client.js ../standard-components/manifest.json client", + "build:docker": "yarn run predocker && docker build . -t app-service", "run:docker": "node dist/index.js", "dev:stack:up": "node scripts/dev/manage.js up", "dev:stack:down": "node scripts/dev/manage.js down", diff --git a/packages/server/src/utilities/fileSystem/clientLibrary.js b/packages/server/src/utilities/fileSystem/clientLibrary.js index 6b7e8d837d..0b57bbc3b2 100644 --- a/packages/server/src/utilities/fileSystem/clientLibrary.js +++ b/packages/server/src/utilities/fileSystem/clientLibrary.js @@ -2,6 +2,9 @@ const { join } = require("path") const { ObjectStoreBuckets } = require("../../constants") const fs = require("fs") const { upload, retrieveToTmp, streamUpload } = require("./utilities") +const { resolve } = require("../centralPath") +const env = require("../../environment") +const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..") /** * Client library paths in the object store: @@ -86,13 +89,26 @@ exports.backupClientLibrary = async appId => { * @returns {Promise} */ exports.updateClientLibrary = async appId => { + let manifest, client + + if (false && env.isDev()) { + // Load the symlinked version in dev which is always the newest + manifest = require.resolve("@budibase/standard-components/manifest.json") + client = require.resolve("@budibase/client") + } else { + // Load the bundled version in prod + manifest = resolve(TOP_LEVEL_PATH, "client", "manifest.json") + client = resolve(TOP_LEVEL_PATH, "client", "budibase-client.js") + + console.log(manifest) + console.log(client) + } + // Upload latest component manifest await streamUpload( ObjectStoreBuckets.APPS, join(appId, "manifest.json"), - fs.createReadStream( - require.resolve("@budibase/standard-components/manifest.json") - ), + fs.createReadStream(manifest), { ContentType: "application/json", } @@ -102,7 +118,7 @@ exports.updateClientLibrary = async appId => { await streamUpload( ObjectStoreBuckets.APPS, join(appId, "budibase-client.js"), - fs.createReadStream(require.resolve("@budibase/client")), + fs.createReadStream(client), { ContentType: "application/javascript", } From 60987e3d5cab87426b721d690b8e54ff0ebb59a5 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 9 Jul 2021 12:42:48 +0100 Subject: [PATCH 14/16] Remove leftover code from testing --- packages/server/src/utilities/fileSystem/clientLibrary.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utilities/fileSystem/clientLibrary.js b/packages/server/src/utilities/fileSystem/clientLibrary.js index 0b57bbc3b2..a56e681832 100644 --- a/packages/server/src/utilities/fileSystem/clientLibrary.js +++ b/packages/server/src/utilities/fileSystem/clientLibrary.js @@ -91,7 +91,7 @@ exports.backupClientLibrary = async appId => { exports.updateClientLibrary = async appId => { let manifest, client - if (false && env.isDev()) { + if (env.isDev()) { // Load the symlinked version in dev which is always the newest manifest = require.resolve("@budibase/standard-components/manifest.json") client = require.resolve("@budibase/client") From 8cc67d815c8c853f31ba9acb92bb7916181e3792 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 9 Jul 2021 13:23:49 +0100 Subject: [PATCH 15/16] Reload the client preview whenever client library version changes --- .../app/[application]/design/[assetType]/_layout.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/app/[application]/design/[assetType]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/design/[assetType]/_layout.svelte index 241b0d5200..207daae84a 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[assetType]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[assetType]/_layout.svelte @@ -155,7 +155,9 @@ {/if}
- + {#key $store.version} + + {/key}
{/if}
From 4112049a17f6310d1573f983834319162bcb6aee Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 9 Jul 2021 16:42:09 +0100 Subject: [PATCH 16/16] Remove log statement, batch uploads and remove unecessary conditional --- .../server/src/api/controllers/application.js | 4 +- .../src/utilities/fileSystem/clientLibrary.js | 45 +++++++------------ 2 files changed, 18 insertions(+), 31 deletions(-) diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index 1a1b8d51c3..a2e254461a 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -313,9 +313,7 @@ const updateAppPackage = async (ctx, appPackage, appId) => { // the locked by property is attached by server but generated from // Redis, shouldn't ever store it - if (newAppPackage.lockedBy) { - delete newAppPackage.lockedBy - } + delete newAppPackage.lockedBy return await db.put(newAppPackage) } diff --git a/packages/server/src/utilities/fileSystem/clientLibrary.js b/packages/server/src/utilities/fileSystem/clientLibrary.js index a56e681832..3f0d31f257 100644 --- a/packages/server/src/utilities/fileSystem/clientLibrary.js +++ b/packages/server/src/utilities/fileSystem/clientLibrary.js @@ -34,10 +34,8 @@ const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..") * @returns {Promise} */ exports.backupClientLibrary = async appId => { - let tmpManifestPath - let tmpClientPath - // Copy existing manifest to tmp + let tmpManifestPath try { // Try to load the manifest from the new file location tmpManifestPath = await retrieveToTmp( @@ -60,26 +58,25 @@ exports.backupClientLibrary = async appId => { } // Copy existing client lib to tmp - tmpClientPath = await retrieveToTmp( + const tmpClientPath = await retrieveToTmp( ObjectStoreBuckets.APPS, join(appId, "budibase-client.js") ) - // Upload manifest as backup - await upload({ + // Upload manifest and client library as backups + const manifestUpload = upload({ bucket: ObjectStoreBuckets.APPS, filename: join(appId, "manifest.json.bak"), path: tmpManifestPath, type: "application/json", }) - - // Upload client library as backup - await upload({ + const clientUpload = upload({ bucket: ObjectStoreBuckets.APPS, filename: join(appId, "budibase-client.js.bak"), path: tmpClientPath, type: "application/javascript", }) + await Promise.all([manifestUpload, clientUpload]) } /** @@ -99,13 +96,10 @@ exports.updateClientLibrary = async appId => { // Load the bundled version in prod manifest = resolve(TOP_LEVEL_PATH, "client", "manifest.json") client = resolve(TOP_LEVEL_PATH, "client", "budibase-client.js") - - console.log(manifest) - console.log(client) } - // Upload latest component manifest - await streamUpload( + // Upload latest manifest and client library + const manifestUpload = streamUpload( ObjectStoreBuckets.APPS, join(appId, "manifest.json"), fs.createReadStream(manifest), @@ -113,9 +107,7 @@ exports.updateClientLibrary = async appId => { ContentType: "application/json", } ) - - // Upload latest component library - await streamUpload( + const clientUpload = streamUpload( ObjectStoreBuckets.APPS, join(appId, "budibase-client.js"), fs.createReadStream(client), @@ -123,6 +115,7 @@ exports.updateClientLibrary = async appId => { ContentType: "application/javascript", } ) + await Promise.all([manifestUpload, clientUpload]) } /** @@ -132,34 +125,30 @@ exports.updateClientLibrary = async appId => { * @returns {Promise} */ exports.revertClientLibrary = async appId => { - let tmpManifestPath - let tmpClientPath - - // Copy backup manifest to tmp - tmpManifestPath = await retrieveToTmp( + // Copy backups manifest to tmp directory + const tmpManifestPath = await retrieveToTmp( ObjectStoreBuckets.APPS, join(appId, "manifest.json.bak") ) // Copy backup client lib to tmp - tmpClientPath = await retrieveToTmp( + const tmpClientPath = await retrieveToTmp( ObjectStoreBuckets.APPS, join(appId, "budibase-client.js.bak") ) - // Upload manifest backup - await upload({ + // Upload backups as new versions + const manifestUpload = upload({ bucket: ObjectStoreBuckets.APPS, filename: join(appId, "manifest.json"), path: tmpManifestPath, type: "application/json", }) - - // Upload client library backup - await upload({ + const clientUpload = upload({ bucket: ObjectStoreBuckets.APPS, filename: join(appId, "budibase-client.js"), path: tmpClientPath, type: "application/javascript", }) + await Promise.all([manifestUpload, clientUpload]) }