From 484070e0e08ef49ae9874dd781de9a4ed1f585e7 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 20 Oct 2020 15:06:34 +0100 Subject: [PATCH] Changing how invalidation is checked for Cloudfront cache, making sure that we don't lose state of the invalidation and can check it fully in the background. --- .../deploy/DeploymentHistory.svelte | 2 +- .../server/src/api/controllers/deploy/aws.js | 66 ++++++------ .../src/api/controllers/deploy/index.js | 100 +++++++++++++----- 3 files changed, 105 insertions(+), 63 deletions(-) diff --git a/packages/builder/src/components/deploy/DeploymentHistory.svelte b/packages/builder/src/components/deploy/DeploymentHistory.svelte index de31554b24..ca918c148f 100644 --- a/packages/builder/src/components/deploy/DeploymentHistory.svelte +++ b/packages/builder/src/components/deploy/DeploymentHistory.svelte @@ -19,7 +19,7 @@ hour12: true, }, } - const POLL_INTERVAL = 1000 + const POLL_INTERVAL = 5000 export let appId diff --git a/packages/server/src/api/controllers/deploy/aws.js b/packages/server/src/api/controllers/deploy/aws.js index 1dee1926cd..ac8b62df54 100644 --- a/packages/server/src/api/controllers/deploy/aws.js +++ b/packages/server/src/api/controllers/deploy/aws.js @@ -1,6 +1,5 @@ const fs = require("fs") const { join } = require("../../../utilities/centralPath") -let { wait } = require("../../../utilities") const AWS = require("aws-sdk") const fetch = require("node-fetch") const uuid = require("uuid") @@ -8,12 +7,6 @@ const { budibaseAppsDir } = require("../../../utilities/budibaseDir") const PouchDB = require("../../../db") const environment = require("../../../environment") -const MAX_INVALIDATE_WAIT_MS = 120000 -const INVALIDATE_WAIT_PERIODS_MS = 5000 - -// export so main deploy functions can use too -exports.MAX_INVALIDATE_WAIT_MS = MAX_INVALIDATE_WAIT_MS - async function invalidateCDN(cfDistribution, appId) { const cf = new AWS.CloudFront({}) const resp = await cf @@ -28,28 +21,24 @@ async function invalidateCDN(cfDistribution, appId) { }, }) .promise() - let totalWaitTimeMs = 0 - let complete = false - do { - try { - const state = await cf - .getInvalidation({ - DistributionId: cfDistribution, - Id: resp.Invalidation.Id, - }) - .promise() - if (state.Invalidation.Status === "Completed") { - complete = true - } - } catch (err) { - console.log() - } - await wait(INVALIDATE_WAIT_PERIODS_MS) - totalWaitTimeMs += INVALIDATE_WAIT_PERIODS_MS - } while (totalWaitTimeMs <= MAX_INVALIDATE_WAIT_MS && !complete) - if (!complete) { - throw "Unable to invalidate old app version" + return resp.Invalidation.Id +} + +exports.isInvalidationComplete = async function( + distributionId, + invalidationId +) { + if (distributionId == null || invalidationId == null) { + return false } + const cf = new AWS.CloudFront({}) + const resp = await cf + .getInvalidation({ + DistributionId: distributionId, + Id: invalidationId, + }) + .promise() + return resp.Invalidation.Status === "Completed" } exports.updateDeploymentQuota = async function(quota) { @@ -102,6 +91,18 @@ exports.verifyDeployment = async function({ instanceId, appId, quota }) { } const json = await response.json() + if (json.errors) { + throw new Error(json.errors) + } + + // set credentials here, means any time we're verified we're ready to go + if (json.credentials) { + AWS.config.update({ + accessKeyId: json.credentials.AccessKeyId, + secretAccessKey: json.credentials.SecretAccessKey, + sessionToken: json.credentials.SessionToken, + }) + } return json } @@ -157,17 +158,10 @@ exports.prepareUploadForS3 = prepareUploadForS3 exports.uploadAppAssets = async function({ appId, instanceId, - credentials, bucket, cfDistribution, accountId, }) { - AWS.config.update({ - accessKeyId: credentials.AccessKeyId, - secretAccessKey: credentials.SecretAccessKey, - sessionToken: credentials.SessionToken, - }) - const s3 = new AWS.S3({ params: { Bucket: bucket, @@ -225,7 +219,7 @@ exports.uploadAppAssets = async function({ try { await Promise.all(uploads) - await invalidateCDN(cfDistribution, appId) + return await invalidateCDN(cfDistribution, appId) } catch (err) { console.error("Error uploading budibase app assets to s3", err) throw err diff --git a/packages/server/src/api/controllers/deploy/index.js b/packages/server/src/api/controllers/deploy/index.js index 912a1dafed..d80eb779ad 100644 --- a/packages/server/src/api/controllers/deploy/index.js +++ b/packages/server/src/api/controllers/deploy/index.js @@ -4,17 +4,67 @@ const { uploadAppAssets, verifyDeployment, updateDeploymentQuota, - MAX_INVALIDATE_WAIT_MS, + isInvalidationComplete, } = require("./aws") const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils") const newid = require("../../../db/newid") +// the max time we can wait for an invalidation to complete before considering it failed +const MAX_PENDING_TIME_MS = 30 * 60000 + const DeploymentStatus = { SUCCESS: "SUCCESS", PENDING: "PENDING", FAILURE: "FAILURE", } +// checks that deployments are in a good state, any pending will be updated +async function checkAllDeployments(deployments, user) { + let updated = false + function update(deployment, status) { + deployment.status = status + delete deployment.invalidationId + delete deployment.cfDistribution + updated = true + } + + for (let deployment of Object.values(deployments.history)) { + // check that no deployments have crashed etc and are now stuck + if ( + deployment.status === DeploymentStatus.PENDING && + Date.now() - deployment.updatedAt > MAX_PENDING_TIME_MS + ) { + update(deployment, DeploymentStatus.FAILURE) + } + // if pending but not past failure point need to update them + else if (deployment.status === DeploymentStatus.PENDING) { + let complete = false + try { + complete = await isInvalidationComplete( + deployment.cfDistribution, + deployment.invalidationId + ) + } catch (err) { + // system may have restarted, need to re-verify + if (err != null && err.code === "InvalidClientTokenId") { + await verifyDeployment({ + ...user, + quota: deployment.quota, + }) + complete = await isInvalidationComplete( + deployment.cfDistribution, + deployment.invalidationId + ) + } + } + if (complete) { + update(deployment, DeploymentStatus.SUCCESS) + } + } + } + return { updated, deployments } +} + function replicate(local, remote) { return new Promise((resolve, reject) => { const replication = local.sync(remote) @@ -102,7 +152,7 @@ async function storeLocalDeploymentHistory(deployment) { async function deployApp({ instanceId, appId, clientId, deploymentId }) { try { const instanceQuota = await getCurrentInstanceQuota(instanceId) - const credentials = await verifyDeployment({ + const verification = await verifyDeployment({ instanceId, appId, quota: instanceQuota, @@ -110,31 +160,36 @@ async function deployApp({ instanceId, appId, clientId, deploymentId }) { console.log(`Uploading assets for appID ${appId} assets to s3..`) - if (credentials.errors) throw new Error(credentials.errors) - - await uploadAppAssets({ clientId, appId, instanceId, ...credentials }) + const invalidationId = await uploadAppAssets({ + clientId, + appId, + instanceId, + ...verification, + }) // replicate the DB to the couchDB cluster in prod console.log("Replicating local PouchDB to remote..") await replicateCouch({ instanceId, clientId, - session: credentials.couchDbSession, + session: verification.couchDbSession, }) - await updateDeploymentQuota(credentials.quota) + await updateDeploymentQuota(verification.quota) await storeLocalDeploymentHistory({ _id: deploymentId, instanceId, - quota: credentials.quota, - status: "SUCCESS", + invalidationId, + cfDistribution: verification.cfDistribution, + quota: verification.quota, + status: DeploymentStatus.PENDING, }) } catch (err) { await storeLocalDeploymentHistory({ _id: deploymentId, instanceId, - status: "FAILURE", + status: DeploymentStatus.FAILURE, err: err.message, }) throw new Error(`Deployment Failed: ${err.message}`) @@ -145,21 +200,14 @@ exports.fetchDeployments = async function(ctx) { try { const db = new PouchDB(ctx.user.instanceId) const deploymentDoc = await db.get("_local/deployments") - // check that no deployments have crashed etc and are now stuck - let changed = false - for (let deployment of Object.values(deploymentDoc.history)) { - if ( - deployment.status === DeploymentStatus.PENDING && - Date.now() - deployment.updatedAt > MAX_INVALIDATE_WAIT_MS - ) { - deployment.status = DeploymentStatus.FAILURE - changed = true - } + const { updated, deployments } = await checkAllDeployments( + deploymentDoc, + ctx.user + ) + if (updated) { + await db.put(deployments) } - if (changed) { - await db.put(deploymentDoc) - } - ctx.body = Object.values(deploymentDoc.history).reverse() + ctx.body = Object.values(deployments.history).reverse() } catch (err) { ctx.body = [] } @@ -185,10 +233,10 @@ exports.deployApp = async function(ctx) { const deployment = await storeLocalDeploymentHistory({ instanceId: ctx.user.instanceId, appId: ctx.user.appId, - status: "PENDING", + status: DeploymentStatus.PENDING, }) - deployApp({ + await deployApp({ ...ctx.user, clientId, deploymentId: deployment._id,