diff --git a/packages/auth/src/objectStore/index.js b/packages/auth/src/objectStore/index.js index 80875fdfee..81bdd06b62 100644 --- a/packages/auth/src/objectStore/index.js +++ b/packages/auth/src/objectStore/index.js @@ -22,11 +22,13 @@ const CONTENT_TYPE_MAP = { html: "text/html", css: "text/css", js: "application/javascript", + json: "application/json", } const STRING_CONTENT_TYPES = [ CONTENT_TYPE_MAP.html, CONTENT_TYPE_MAP.css, CONTENT_TYPE_MAP.js, + CONTENT_TYPE_MAP.json, ] // does normal sanitization and then swaps dev apps to apps diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index ac837978a9..b9cd134067 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -32,6 +32,10 @@ const INITIAL_FRONTEND_STATE = { layouts: [], screens: [], components: [], + clientFeatures: { + spectrumThemes: false, + intelligentLoading: false, + }, currentFrontEndType: "none", selectedScreenId: "", selectedLayoutId: "", @@ -56,6 +60,10 @@ export const getFrontendStore = () => { ...state, libraries: application.componentLibraries, components, + clientFeatures: { + ...state.clientFeatures, + ...components.features, + }, name: application.name, description: application.description, appId: application.appId, @@ -67,6 +75,8 @@ export const getFrontendStore = () => { appInstance: application.instance, clientLibPath, previousTopNavPath: {}, + version: application.version, + revertableVersion: application.revertableVersion, })) await hostingStore.actions.fetch() diff --git a/packages/builder/src/components/deploy/VersionModal.svelte b/packages/builder/src/components/deploy/VersionModal.svelte new file mode 100644 index 0000000000..0fb061face --- /dev/null +++ b/packages/builder/src/components/deploy/VersionModal.svelte @@ -0,0 +1,120 @@ + + +
+ +
+ + +
+ {#if revertAvailable} + + {/if} +
+ {#if updateAvailable} + + This app is currently using version {$store.version}, but version + {clientPackage.version} is available. Updates can contain new features, + performance improvements and bug fixes. + + {:else} + + This app is currently using version {$store.version} which is the + latest version available. + + {/if} + {#if revertAvailable} + + You can revert this app to version + {$store.revertableVersion} + if you're experiencing issues with the current version. + + {/if} + + + + diff --git a/packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte b/packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte index 3e4789a31d..607061013d 100644 --- a/packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte +++ b/packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte @@ -74,14 +74,14 @@ // Initialise the app when mounted iframe.contentWindow.addEventListener( "ready", - () => refreshContent(strippedJson), - { once: true } - ) - - // Display the client app once the iframe is initialised - iframe.contentWindow.addEventListener( - "iframe-loaded", - () => (loading = false), + () => { + // Display preview immediately if the intelligent loading feature + // is not supported + if (!$store.clientFeatures.intelligentLoading) { + loading = false + } + refreshContent(strippedJson) + }, { once: true } ) @@ -106,11 +106,9 @@ idToDelete = data.id confirmDeleteDialog.show() } else if (type === "preview-loaded") { - // We can use this in future to delay displaying the preview - // until the client app has actually initialised. - // This makes a smoother loading experience, but is not backwards - // compatible with old client library versions. - // So do nothing with this for now. + // Wait for this event to show the client library if intelligent + // loading is supported + loading = false } else { console.warning(`Client sent unknown event type: ${type}`) } diff --git a/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte b/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte index df30a17640..24050e1088 100644 --- a/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte +++ b/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte @@ -65,7 +65,7 @@ } -{#if definition.editable !== false} +{#if definition?.editable !== false}
diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index eab765b502..f3ae6e7c8e 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -4,6 +4,7 @@ import { Icon, ActionGroup, Tabs, Tab } from "@budibase/bbui" import DeployModal from "components/deploy/DeployModal.svelte" import RevertModal from "components/deploy/RevertModal.svelte" + import VersionModal from "components/deploy/VersionModal.svelte" import { get } from "builderStore/api" import { isActive, goto, layout } from "@roxi/routify" import Logo from "assets/bb-emblem.svg" @@ -80,6 +81,7 @@
+ - + {#if $store.clientFeatures.spectrumThemes} + + {/if}
- + {#key $store.version} + + {/key}
{/if} diff --git a/packages/server/.gitignore b/packages/server/.gitignore index c737d00466..e8589f631d 100644 --- a/packages/server/.gitignore +++ b/packages/server/.gitignore @@ -2,6 +2,7 @@ node_modules/ myapps/ .env builder/* +client/* public/ db/dev.db/ dist diff --git a/packages/server/package.json b/packages/server/package.json index 26ba52e656..d3d99a5b16 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -13,7 +13,8 @@ "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/", "test": "jest --coverage --maxWorkers=2", "test:watch": "jest --watch", - "build:docker": "docker build . -t app-service", + "predocker": "copyfiles -f ../client/dist/budibase-client.js ../standard-components/manifest.json client", + "build:docker": "yarn run predocker && docker build . -t app-service", "run:docker": "node dist/index.js", "dev:stack:up": "node scripts/dev/manage.js up", "dev:stack:down": "node scripts/dev/manage.js down", @@ -61,6 +62,7 @@ "dependencies": { "@budibase/auth": "^0.9.71", "@budibase/client": "^0.9.71", + "@budibase/standard-components": "^0.9.71", "@budibase/string-templates": "^0.9.71", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", @@ -114,7 +116,6 @@ "devDependencies": { "@babel/core": "^7.14.3", "@babel/preset-env": "^7.14.4", - "@budibase/standard-components": "^0.9.71", "@jest/test-sequencer": "^24.8.0", "@types/bull": "^3.15.1", "@types/jest": "^26.0.23", diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index 9881a07f43..a2e254461a 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -33,6 +33,11 @@ const { } = require("../../utilities/workerRequests") const { clientLibraryPath } = require("../../utilities") const { getAllLocks } = require("../../utilities/redis") +const { + updateClientLibrary, + backupClientLibrary, + revertClientLibrary, +} = require("../../utilities/fileSystem/clientLibrary") const URL_REGEX_SLASH = /\/|\\/g @@ -231,27 +236,54 @@ exports.create = async function (ctx) { } exports.update = async function (ctx) { - const url = await getAppUrlIfNotInUse(ctx) + const data = await updateAppPackage(ctx, ctx.request.body, ctx.params.appId) + ctx.status = 200 + ctx.body = data +} + +exports.updateClient = async function (ctx) { + // Get current app version const db = new CouchDB(ctx.params.appId) const application = await db.get(DocumentTypes.APP_METADATA) + const currentVersion = application.version - const data = ctx.request.body - const newData = { ...application, ...data, url } - if (ctx.request.body._rev !== application._rev) { - newData._rev = application._rev + // Update client library and manifest + if (!env.isTest()) { + await backupClientLibrary(ctx.params.appId) + await updateClientLibrary(ctx.params.appId) } - // the locked by property is attached by server but generated from - // Redis, shouldn't ever store it - if (newData.lockedBy) { - delete newData.lockedBy + // Update versions in app package + const appPackageUpdates = { + version: packageJson.version, + revertableVersion: currentVersion, } - - const response = await db.put(newData) - data._rev = response.rev - + const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId) ctx.status = 200 - ctx.body = response + ctx.body = data +} + +exports.revertClient = async function (ctx) { + // Check app can be reverted + const db = new CouchDB(ctx.params.appId) + const application = await db.get(DocumentTypes.APP_METADATA) + if (!application.revertableVersion) { + ctx.throw(400, "There is no version to revert to") + } + + // Update client library and manifest + if (!env.isTest()) { + await revertClientLibrary(ctx.params.appId) + } + + // Update versions in app package + const appPackageUpdates = { + version: application.revertableVersion, + revertableVersion: null, + } + const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId) + ctx.status = 200 + ctx.body = data } exports.delete = async function (ctx) { @@ -269,6 +301,23 @@ exports.delete = async function (ctx) { ctx.body = result } +const updateAppPackage = async (ctx, appPackage, appId) => { + const url = await getAppUrlIfNotInUse(ctx) + const db = new CouchDB(appId) + const application = await db.get(DocumentTypes.APP_METADATA) + + const newAppPackage = { ...application, ...appPackage, url } + if (appPackage._rev !== application._rev) { + newAppPackage._rev = application._rev + } + + // the locked by property is attached by server but generated from + // Redis, shouldn't ever store it + delete newAppPackage.lockedBy + + return await db.put(newAppPackage) +} + const createEmptyAppPackage = async (ctx, app) => { const db = new CouchDB(app.appId) diff --git a/packages/server/src/api/controllers/component.js b/packages/server/src/api/controllers/component.js index c678d8e587..06cb2cd211 100644 --- a/packages/server/src/api/controllers/component.js +++ b/packages/server/src/api/controllers/component.js @@ -20,10 +20,14 @@ exports.fetchAppComponentDefinitions = async function (ctx) { const definitions = {} for (let { manifest, library } of componentManifests) { for (let key of Object.keys(manifest)) { - const fullComponentName = `${library}/${key}`.toLowerCase() - definitions[fullComponentName] = { - component: fullComponentName, - ...manifest[key], + if (key === "features") { + definitions[key] = manifest[key] + } else { + const fullComponentName = `${library}/${key}`.toLowerCase() + definitions[fullComponentName] = { + component: fullComponentName, + ...manifest[key], + } } } } diff --git a/packages/server/src/api/routes/application.js b/packages/server/src/api/routes/application.js index a7209df3e9..c2eb19e101 100644 --- a/packages/server/src/api/routes/application.js +++ b/packages/server/src/api/routes/application.js @@ -11,6 +11,16 @@ router .get("/api/applications/:appId/appPackage", controller.fetchAppPackage) .put("/api/applications/:appId", authorized(BUILDER), controller.update) .post("/api/applications", authorized(BUILDER), controller.create) + .post( + "/api/applications/:appId/client/update", + authorized(BUILDER), + controller.updateClient + ) + .post( + "/api/applications/:appId/client/revert", + authorized(BUILDER), + controller.revertClient + ) .delete("/api/applications/:appId", authorized(BUILDER), controller.delete) module.exports = router diff --git a/packages/server/src/api/routes/tests/application.spec.js b/packages/server/src/api/routes/tests/application.spec.js index 2333787e6e..05e0bc231b 100644 --- a/packages/server/src/api/routes/tests/application.spec.js +++ b/packages/server/src/api/routes/tests/application.spec.js @@ -94,7 +94,7 @@ describe("/applications", () => { }) describe("update", () => { - it("should be able to fetch the app package", async () => { + it("should be able to update the app package", async () => { const res = await request .put(`/api/applications/${config.getAppId()}`) .send({ @@ -107,6 +107,30 @@ describe("/applications", () => { }) }) + describe("manage client library version", () => { + it("should be able to update the app client library version", async () => { + console.log(config.getAppId()) + await request + .post(`/api/applications/${config.getAppId()}/client/update`) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + }) + it("should be able to revert the app client library version", async () => { + // We need to first update the version so that we can then revert + await request + .post(`/api/applications/${config.getAppId()}/client/update`) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + await request + .post(`/api/applications/${config.getAppId()}/client/revert`) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + }) + }) + describe("edited at", () => { it("middleware should set edited at", async () => { const headers = config.defaultHeaders() diff --git a/packages/server/src/utilities/fileSystem/clientLibrary.js b/packages/server/src/utilities/fileSystem/clientLibrary.js new file mode 100644 index 0000000000..3f0d31f257 --- /dev/null +++ b/packages/server/src/utilities/fileSystem/clientLibrary.js @@ -0,0 +1,154 @@ +const { join } = require("path") +const { ObjectStoreBuckets } = require("../../constants") +const fs = require("fs") +const { upload, retrieveToTmp, streamUpload } = require("./utilities") +const { resolve } = require("../centralPath") +const env = require("../../environment") +const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..") + +/** + * Client library paths in the object store: + * Previously, the entire standard-components package was downloaded from NPM + * as a tarball and extracted to the object store, even though only the manifest + * was ever needed. Therefore we need to support old apps which may still have + * the manifest at this location for the first update. + * + * The new paths for the in-use version are: + * {appId}/manifest.json + * {appId}/budibase-client.js + * + * The paths for the backups are: + * {appId}/manifest.json.bak + * {appId}/budibase-client.js.bak + * + * We don't rely on NPM at all any more, as when updating to the latest version + * we pull both the manifest and client bundle from the server's dependencies + * in the local file system. + */ + +/** + * Backs up the current client library version by copying both the manifest + * and client bundle to .bak extensions in the object store. Only the one + * previous version is stored as a backup, which can be reverted to. + * @param appId The app ID to backup + * @returns {Promise} + */ +exports.backupClientLibrary = async appId => { + // Copy existing manifest to tmp + let tmpManifestPath + try { + // Try to load the manifest from the new file location + tmpManifestPath = await retrieveToTmp( + ObjectStoreBuckets.APPS, + join(appId, "manifest.json") + ) + } catch (error) { + // Fallback to loading it from the old location for old apps + tmpManifestPath = await retrieveToTmp( + ObjectStoreBuckets.APPS, + join( + appId, + "node_modules", + "budibase", + "standard-components", + "package", + "manifest.json" + ) + ) + } + + // Copy existing client lib to tmp + const tmpClientPath = await retrieveToTmp( + ObjectStoreBuckets.APPS, + join(appId, "budibase-client.js") + ) + + // Upload manifest and client library as backups + const manifestUpload = upload({ + bucket: ObjectStoreBuckets.APPS, + filename: join(appId, "manifest.json.bak"), + path: tmpManifestPath, + type: "application/json", + }) + const clientUpload = upload({ + bucket: ObjectStoreBuckets.APPS, + filename: join(appId, "budibase-client.js.bak"), + path: tmpClientPath, + type: "application/javascript", + }) + await Promise.all([manifestUpload, clientUpload]) +} + +/** + * Uploads the latest version of the component manifest and the client library + * to the object store, overwriting the existing version. + * @param appId The app ID to update + * @returns {Promise} + */ +exports.updateClientLibrary = async appId => { + let manifest, client + + if (env.isDev()) { + // Load the symlinked version in dev which is always the newest + manifest = require.resolve("@budibase/standard-components/manifest.json") + client = require.resolve("@budibase/client") + } else { + // Load the bundled version in prod + manifest = resolve(TOP_LEVEL_PATH, "client", "manifest.json") + client = resolve(TOP_LEVEL_PATH, "client", "budibase-client.js") + } + + // Upload latest manifest and client library + const manifestUpload = streamUpload( + ObjectStoreBuckets.APPS, + join(appId, "manifest.json"), + fs.createReadStream(manifest), + { + ContentType: "application/json", + } + ) + const clientUpload = streamUpload( + ObjectStoreBuckets.APPS, + join(appId, "budibase-client.js"), + fs.createReadStream(client), + { + ContentType: "application/javascript", + } + ) + await Promise.all([manifestUpload, clientUpload]) +} + +/** + * Reverts the version of the client library and manifest to the previously + * used version for an app. + * @param appId The app ID to revert + * @returns {Promise} + */ +exports.revertClientLibrary = async appId => { + // Copy backups manifest to tmp directory + const tmpManifestPath = await retrieveToTmp( + ObjectStoreBuckets.APPS, + join(appId, "manifest.json.bak") + ) + + // Copy backup client lib to tmp + const tmpClientPath = await retrieveToTmp( + ObjectStoreBuckets.APPS, + join(appId, "budibase-client.js.bak") + ) + + // Upload backups as new versions + const manifestUpload = upload({ + bucket: ObjectStoreBuckets.APPS, + filename: join(appId, "manifest.json"), + path: tmpManifestPath, + type: "application/json", + }) + const clientUpload = upload({ + bucket: ObjectStoreBuckets.APPS, + filename: join(appId, "budibase-client.js"), + path: tmpClientPath, + type: "application/javascript", + }) + await Promise.all([manifestUpload, clientUpload]) +} diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index afacbf8cdf..b83ff03854 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -13,7 +13,7 @@ const { deleteFolder, downloadTarball, } = require("./utilities") -const { downloadLibraries, uploadClientLibrary } = require("./newApp") +const { updateClientLibrary } = require("./clientLibrary") const download = require("download") const env = require("../../environment") const { homedir } = require("os") @@ -139,13 +139,12 @@ exports.performBackup = async (appId, backupName) => { } /** - * Downloads required libraries and creates a new path in the object store. + * Uploads the latest client library to the object store. * @param {string} appId The ID of the app which is being created. * @return {Promise} once promise completes app resources should be ready in object store. */ exports.createApp = async appId => { - await downloadLibraries(appId) - await uploadClientLibrary(appId) + await updateClientLibrary(appId) } /** @@ -193,8 +192,17 @@ exports.getComponentLibraryManifest = async (appId, library) => { delete require.cache[require.resolve(path)] return require(path) } - const path = join(appId, "node_modules", library, "package", filename) - let resp = await retrieve(ObjectStoreBuckets.APPS, path) + + let resp + try { + // Try to load the manifest from the new file location + const path = join(appId, filename) + resp = await retrieve(ObjectStoreBuckets.APPS, path) + } catch (error) { + // Fallback to loading it from the old location for old apps + const path = join(appId, "node_modules", library, "package", filename) + resp = await retrieve(ObjectStoreBuckets.APPS, path) + } if (typeof resp !== "string") { resp = resp.toString("utf8") } diff --git a/packages/server/src/utilities/fileSystem/newApp.js b/packages/server/src/utilities/fileSystem/newApp.js deleted file mode 100644 index 735f0d523e..0000000000 --- a/packages/server/src/utilities/fileSystem/newApp.js +++ /dev/null @@ -1,36 +0,0 @@ -const packageJson = require("../../../package.json") -const { join } = require("path") -const { ObjectStoreBuckets } = require("../../constants") -const { streamUpload, downloadTarball } = require("./utilities") -const fs = require("fs") - -const BUCKET_NAME = ObjectStoreBuckets.APPS - -// can't really test this due to the downloading nature of it, wouldn't be a great test case -/* istanbul ignore next */ -exports.downloadLibraries = async appId => { - const LIBRARIES = ["standard-components"] - - const paths = {} - // Need to download tarballs directly from NPM as our users may not have node on their machine - for (let lib of LIBRARIES) { - // download tarball - const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz` - const path = join(appId, "node_modules", "@budibase", lib) - paths[`@budibase/${lib}`] = await downloadTarball( - registryUrl, - BUCKET_NAME, - path - ) - } - return paths -} - -exports.uploadClientLibrary = async appId => { - const sourcepath = require.resolve("@budibase/client") - const destPath = join(appId, "budibase-client.js") - - await streamUpload(BUCKET_NAME, destPath, fs.createReadStream(sourcepath), { - ContentType: "application/javascript", - }) -} diff --git a/packages/standard-components/manifest.json b/packages/standard-components/manifest.json index be66a4ce87..ec32a5879d 100644 --- a/packages/standard-components/manifest.json +++ b/packages/standard-components/manifest.json @@ -1,4 +1,8 @@ { + "features": { + "spectrumThemes": true, + "intelligentLoading": true + }, "layout": { "name": "Layout", "description": "This component is specific only to layouts",