Changing how invalidation is checked for Cloudfront cache, making sure that we don't lose state of the invalidation and can check it fully in the background.
This commit is contained in:
parent
7884ab2c1f
commit
484070e0e0
|
@ -19,7 +19,7 @@
|
||||||
hour12: true,
|
hour12: true,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
const POLL_INTERVAL = 1000
|
const POLL_INTERVAL = 5000
|
||||||
|
|
||||||
export let appId
|
export let appId
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
const fs = require("fs")
|
const fs = require("fs")
|
||||||
const { join } = require("../../../utilities/centralPath")
|
const { join } = require("../../../utilities/centralPath")
|
||||||
let { wait } = require("../../../utilities")
|
|
||||||
const AWS = require("aws-sdk")
|
const AWS = require("aws-sdk")
|
||||||
const fetch = require("node-fetch")
|
const fetch = require("node-fetch")
|
||||||
const uuid = require("uuid")
|
const uuid = require("uuid")
|
||||||
|
@ -8,12 +7,6 @@ const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
|
||||||
const PouchDB = require("../../../db")
|
const PouchDB = require("../../../db")
|
||||||
const environment = require("../../../environment")
|
const environment = require("../../../environment")
|
||||||
|
|
||||||
const MAX_INVALIDATE_WAIT_MS = 120000
|
|
||||||
const INVALIDATE_WAIT_PERIODS_MS = 5000
|
|
||||||
|
|
||||||
// export so main deploy functions can use too
|
|
||||||
exports.MAX_INVALIDATE_WAIT_MS = MAX_INVALIDATE_WAIT_MS
|
|
||||||
|
|
||||||
async function invalidateCDN(cfDistribution, appId) {
|
async function invalidateCDN(cfDistribution, appId) {
|
||||||
const cf = new AWS.CloudFront({})
|
const cf = new AWS.CloudFront({})
|
||||||
const resp = await cf
|
const resp = await cf
|
||||||
|
@ -28,28 +21,24 @@ async function invalidateCDN(cfDistribution, appId) {
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
.promise()
|
.promise()
|
||||||
let totalWaitTimeMs = 0
|
return resp.Invalidation.Id
|
||||||
let complete = false
|
}
|
||||||
do {
|
|
||||||
try {
|
exports.isInvalidationComplete = async function(
|
||||||
const state = await cf
|
distributionId,
|
||||||
|
invalidationId
|
||||||
|
) {
|
||||||
|
if (distributionId == null || invalidationId == null) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
const cf = new AWS.CloudFront({})
|
||||||
|
const resp = await cf
|
||||||
.getInvalidation({
|
.getInvalidation({
|
||||||
DistributionId: cfDistribution,
|
DistributionId: distributionId,
|
||||||
Id: resp.Invalidation.Id,
|
Id: invalidationId,
|
||||||
})
|
})
|
||||||
.promise()
|
.promise()
|
||||||
if (state.Invalidation.Status === "Completed") {
|
return resp.Invalidation.Status === "Completed"
|
||||||
complete = true
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
console.log()
|
|
||||||
}
|
|
||||||
await wait(INVALIDATE_WAIT_PERIODS_MS)
|
|
||||||
totalWaitTimeMs += INVALIDATE_WAIT_PERIODS_MS
|
|
||||||
} while (totalWaitTimeMs <= MAX_INVALIDATE_WAIT_MS && !complete)
|
|
||||||
if (!complete) {
|
|
||||||
throw "Unable to invalidate old app version"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
exports.updateDeploymentQuota = async function(quota) {
|
exports.updateDeploymentQuota = async function(quota) {
|
||||||
|
@ -102,6 +91,18 @@ exports.verifyDeployment = async function({ instanceId, appId, quota }) {
|
||||||
}
|
}
|
||||||
|
|
||||||
const json = await response.json()
|
const json = await response.json()
|
||||||
|
if (json.errors) {
|
||||||
|
throw new Error(json.errors)
|
||||||
|
}
|
||||||
|
|
||||||
|
// set credentials here, means any time we're verified we're ready to go
|
||||||
|
if (json.credentials) {
|
||||||
|
AWS.config.update({
|
||||||
|
accessKeyId: json.credentials.AccessKeyId,
|
||||||
|
secretAccessKey: json.credentials.SecretAccessKey,
|
||||||
|
sessionToken: json.credentials.SessionToken,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
return json
|
return json
|
||||||
}
|
}
|
||||||
|
@ -157,17 +158,10 @@ exports.prepareUploadForS3 = prepareUploadForS3
|
||||||
exports.uploadAppAssets = async function({
|
exports.uploadAppAssets = async function({
|
||||||
appId,
|
appId,
|
||||||
instanceId,
|
instanceId,
|
||||||
credentials,
|
|
||||||
bucket,
|
bucket,
|
||||||
cfDistribution,
|
cfDistribution,
|
||||||
accountId,
|
accountId,
|
||||||
}) {
|
}) {
|
||||||
AWS.config.update({
|
|
||||||
accessKeyId: credentials.AccessKeyId,
|
|
||||||
secretAccessKey: credentials.SecretAccessKey,
|
|
||||||
sessionToken: credentials.SessionToken,
|
|
||||||
})
|
|
||||||
|
|
||||||
const s3 = new AWS.S3({
|
const s3 = new AWS.S3({
|
||||||
params: {
|
params: {
|
||||||
Bucket: bucket,
|
Bucket: bucket,
|
||||||
|
@ -225,7 +219,7 @@ exports.uploadAppAssets = async function({
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await Promise.all(uploads)
|
await Promise.all(uploads)
|
||||||
await invalidateCDN(cfDistribution, appId)
|
return await invalidateCDN(cfDistribution, appId)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error("Error uploading budibase app assets to s3", err)
|
console.error("Error uploading budibase app assets to s3", err)
|
||||||
throw err
|
throw err
|
||||||
|
|
|
@ -4,17 +4,67 @@ const {
|
||||||
uploadAppAssets,
|
uploadAppAssets,
|
||||||
verifyDeployment,
|
verifyDeployment,
|
||||||
updateDeploymentQuota,
|
updateDeploymentQuota,
|
||||||
MAX_INVALIDATE_WAIT_MS,
|
isInvalidationComplete,
|
||||||
} = require("./aws")
|
} = require("./aws")
|
||||||
const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils")
|
const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils")
|
||||||
const newid = require("../../../db/newid")
|
const newid = require("../../../db/newid")
|
||||||
|
|
||||||
|
// the max time we can wait for an invalidation to complete before considering it failed
|
||||||
|
const MAX_PENDING_TIME_MS = 30 * 60000
|
||||||
|
|
||||||
const DeploymentStatus = {
|
const DeploymentStatus = {
|
||||||
SUCCESS: "SUCCESS",
|
SUCCESS: "SUCCESS",
|
||||||
PENDING: "PENDING",
|
PENDING: "PENDING",
|
||||||
FAILURE: "FAILURE",
|
FAILURE: "FAILURE",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// checks that deployments are in a good state, any pending will be updated
|
||||||
|
async function checkAllDeployments(deployments, user) {
|
||||||
|
let updated = false
|
||||||
|
function update(deployment, status) {
|
||||||
|
deployment.status = status
|
||||||
|
delete deployment.invalidationId
|
||||||
|
delete deployment.cfDistribution
|
||||||
|
updated = true
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let deployment of Object.values(deployments.history)) {
|
||||||
|
// check that no deployments have crashed etc and are now stuck
|
||||||
|
if (
|
||||||
|
deployment.status === DeploymentStatus.PENDING &&
|
||||||
|
Date.now() - deployment.updatedAt > MAX_PENDING_TIME_MS
|
||||||
|
) {
|
||||||
|
update(deployment, DeploymentStatus.FAILURE)
|
||||||
|
}
|
||||||
|
// if pending but not past failure point need to update them
|
||||||
|
else if (deployment.status === DeploymentStatus.PENDING) {
|
||||||
|
let complete = false
|
||||||
|
try {
|
||||||
|
complete = await isInvalidationComplete(
|
||||||
|
deployment.cfDistribution,
|
||||||
|
deployment.invalidationId
|
||||||
|
)
|
||||||
|
} catch (err) {
|
||||||
|
// system may have restarted, need to re-verify
|
||||||
|
if (err != null && err.code === "InvalidClientTokenId") {
|
||||||
|
await verifyDeployment({
|
||||||
|
...user,
|
||||||
|
quota: deployment.quota,
|
||||||
|
})
|
||||||
|
complete = await isInvalidationComplete(
|
||||||
|
deployment.cfDistribution,
|
||||||
|
deployment.invalidationId
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (complete) {
|
||||||
|
update(deployment, DeploymentStatus.SUCCESS)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { updated, deployments }
|
||||||
|
}
|
||||||
|
|
||||||
function replicate(local, remote) {
|
function replicate(local, remote) {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
const replication = local.sync(remote)
|
const replication = local.sync(remote)
|
||||||
|
@ -102,7 +152,7 @@ async function storeLocalDeploymentHistory(deployment) {
|
||||||
async function deployApp({ instanceId, appId, clientId, deploymentId }) {
|
async function deployApp({ instanceId, appId, clientId, deploymentId }) {
|
||||||
try {
|
try {
|
||||||
const instanceQuota = await getCurrentInstanceQuota(instanceId)
|
const instanceQuota = await getCurrentInstanceQuota(instanceId)
|
||||||
const credentials = await verifyDeployment({
|
const verification = await verifyDeployment({
|
||||||
instanceId,
|
instanceId,
|
||||||
appId,
|
appId,
|
||||||
quota: instanceQuota,
|
quota: instanceQuota,
|
||||||
|
@ -110,31 +160,36 @@ async function deployApp({ instanceId, appId, clientId, deploymentId }) {
|
||||||
|
|
||||||
console.log(`Uploading assets for appID ${appId} assets to s3..`)
|
console.log(`Uploading assets for appID ${appId} assets to s3..`)
|
||||||
|
|
||||||
if (credentials.errors) throw new Error(credentials.errors)
|
const invalidationId = await uploadAppAssets({
|
||||||
|
clientId,
|
||||||
await uploadAppAssets({ clientId, appId, instanceId, ...credentials })
|
appId,
|
||||||
|
instanceId,
|
||||||
|
...verification,
|
||||||
|
})
|
||||||
|
|
||||||
// replicate the DB to the couchDB cluster in prod
|
// replicate the DB to the couchDB cluster in prod
|
||||||
console.log("Replicating local PouchDB to remote..")
|
console.log("Replicating local PouchDB to remote..")
|
||||||
await replicateCouch({
|
await replicateCouch({
|
||||||
instanceId,
|
instanceId,
|
||||||
clientId,
|
clientId,
|
||||||
session: credentials.couchDbSession,
|
session: verification.couchDbSession,
|
||||||
})
|
})
|
||||||
|
|
||||||
await updateDeploymentQuota(credentials.quota)
|
await updateDeploymentQuota(verification.quota)
|
||||||
|
|
||||||
await storeLocalDeploymentHistory({
|
await storeLocalDeploymentHistory({
|
||||||
_id: deploymentId,
|
_id: deploymentId,
|
||||||
instanceId,
|
instanceId,
|
||||||
quota: credentials.quota,
|
invalidationId,
|
||||||
status: "SUCCESS",
|
cfDistribution: verification.cfDistribution,
|
||||||
|
quota: verification.quota,
|
||||||
|
status: DeploymentStatus.PENDING,
|
||||||
})
|
})
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
await storeLocalDeploymentHistory({
|
await storeLocalDeploymentHistory({
|
||||||
_id: deploymentId,
|
_id: deploymentId,
|
||||||
instanceId,
|
instanceId,
|
||||||
status: "FAILURE",
|
status: DeploymentStatus.FAILURE,
|
||||||
err: err.message,
|
err: err.message,
|
||||||
})
|
})
|
||||||
throw new Error(`Deployment Failed: ${err.message}`)
|
throw new Error(`Deployment Failed: ${err.message}`)
|
||||||
|
@ -145,21 +200,14 @@ exports.fetchDeployments = async function(ctx) {
|
||||||
try {
|
try {
|
||||||
const db = new PouchDB(ctx.user.instanceId)
|
const db = new PouchDB(ctx.user.instanceId)
|
||||||
const deploymentDoc = await db.get("_local/deployments")
|
const deploymentDoc = await db.get("_local/deployments")
|
||||||
// check that no deployments have crashed etc and are now stuck
|
const { updated, deployments } = await checkAllDeployments(
|
||||||
let changed = false
|
deploymentDoc,
|
||||||
for (let deployment of Object.values(deploymentDoc.history)) {
|
ctx.user
|
||||||
if (
|
)
|
||||||
deployment.status === DeploymentStatus.PENDING &&
|
if (updated) {
|
||||||
Date.now() - deployment.updatedAt > MAX_INVALIDATE_WAIT_MS
|
await db.put(deployments)
|
||||||
) {
|
|
||||||
deployment.status = DeploymentStatus.FAILURE
|
|
||||||
changed = true
|
|
||||||
}
|
}
|
||||||
}
|
ctx.body = Object.values(deployments.history).reverse()
|
||||||
if (changed) {
|
|
||||||
await db.put(deploymentDoc)
|
|
||||||
}
|
|
||||||
ctx.body = Object.values(deploymentDoc.history).reverse()
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
ctx.body = []
|
ctx.body = []
|
||||||
}
|
}
|
||||||
|
@ -185,10 +233,10 @@ exports.deployApp = async function(ctx) {
|
||||||
const deployment = await storeLocalDeploymentHistory({
|
const deployment = await storeLocalDeploymentHistory({
|
||||||
instanceId: ctx.user.instanceId,
|
instanceId: ctx.user.instanceId,
|
||||||
appId: ctx.user.appId,
|
appId: ctx.user.appId,
|
||||||
status: "PENDING",
|
status: DeploymentStatus.PENDING,
|
||||||
})
|
})
|
||||||
|
|
||||||
deployApp({
|
await deployApp({
|
||||||
...ctx.user,
|
...ctx.user,
|
||||||
clientId,
|
clientId,
|
||||||
deploymentId: deployment._id,
|
deploymentId: deployment._id,
|
||||||
|
|
Loading…
Reference in New Issue