Merge branch 'master' of github.com:Budibase/budibase into dropdown-components

packages/builder/src/components/deploy/DeploymentHistory.svelte

@@ -19,7 +19,7 @@
       hour12: true,
     },
   }
-  const POLL_INTERVAL = 1000
+  const POLL_INTERVAL = 5000
 
   export let appId
 
@@ -68,7 +68,7 @@
             </span>
           </div>
           <div class="deployment-right">
-            {#if deployment.status.toLowerCase() === "pending"}
+            {#if deployment.status.toLowerCase() === 'pending'}
               <Spinner size="10" />
             {/if}
             <div class={`deployment-status ${deployment.status}`}>

packages/builder/src/pages/[application]/_reset.svelte

@@ -1,5 +1,6 @@
 <script>
   import { store, automationStore, backendUiStore } from "builderStore"
+  import { Button } from "@budibase/bbui"
   import SettingsLink from "components/settings/Link.svelte"
   import { get } from "builderStore/api"
   import { isActive, goto, layout } from "@sveltech/routify"
@@ -83,6 +84,13 @@
       </span>
     </div>
   </div>
+  <div class="beta">
+    <Button
+      secondary
+      href="https://www.budibase.com/blog/budibase-public-beta/">
+      Budibase is in Beta
+    </Button>
+  </div>
 
   {#await promise}
     <!-- This should probably be some kind of loading state? -->
@@ -201,4 +209,10 @@
     font-size: 24px;
     color: var(--grey-7);
   }
+
+  .beta {
+    position: absolute;
+    bottom: var(--spacing-m);
+    left: var(--spacing-m);
+  }
 </style>

packages/server/package.json

@@ -57,6 +57,7 @@
     "electron-updater": "^4.3.1",
     "fix-path": "^3.0.0",
     "fs-extra": "^8.1.0",
+    "jimp": "^0.16.1",
     "joi": "^17.2.1",
     "jsonwebtoken": "^8.5.1",
     "koa": "^2.7.0",
@@ -73,7 +74,7 @@
     "pouchdb": "^7.2.1",
     "pouchdb-all-dbs": "^1.0.2",
     "pouchdb-replication-stream": "^1.2.9",
-    "sharp": "^0.26.0",
+    "sanitize-s3-objectkey": "^0.0.1",
     "squirrelly": "^7.5.0",
     "tar-fs": "^2.1.0",
     "uuid": "^3.3.2",

packages/server/src/api/controllers/deploy/aws.js

@@ -1,19 +1,13 @@
 const fs = require("fs")
 const { join } = require("../../../utilities/centralPath")
-let { wait } = require("../../../utilities")
 const AWS = require("aws-sdk")
 const fetch = require("node-fetch")
 const uuid = require("uuid")
+const sanitize = require("sanitize-s3-objectkey")
 const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
 const PouchDB = require("../../../db")
 const environment = require("../../../environment")
 
-const MAX_INVALIDATE_WAIT_MS = 120000
-const INVALIDATE_WAIT_PERIODS_MS = 5000
-
-// export so main deploy functions can use too
-exports.MAX_INVALIDATE_WAIT_MS = MAX_INVALIDATE_WAIT_MS
-
 async function invalidateCDN(cfDistribution, appId) {
   const cf = new AWS.CloudFront({})
   const resp = await cf
@@ -28,28 +22,24 @@ async function invalidateCDN(cfDistribution, appId) {
       },
     })
     .promise()
-  let totalWaitTimeMs = 0
+  return resp.Invalidation.Id
-  let complete = false
+}
-  do {
+
-    try {
+exports.isInvalidationComplete = async function(
-      const state = await cf
+  distributionId,
+  invalidationId
+) {
+  if (distributionId == null || invalidationId == null) {
+    return false
+  }
+  const cf = new AWS.CloudFront({})
+  const resp = await cf
     .getInvalidation({
-          DistributionId: cfDistribution,
+      DistributionId: distributionId,
-          Id: resp.Invalidation.Id,
+      Id: invalidationId,
     })
     .promise()
-      if (state.Invalidation.Status === "Completed") {
+  return resp.Invalidation.Status === "Completed"
-        complete = true
-      }
-    } catch (err) {
-      console.log()
-    }
-    await wait(INVALIDATE_WAIT_PERIODS_MS)
-    totalWaitTimeMs += INVALIDATE_WAIT_PERIODS_MS
-  } while (totalWaitTimeMs <= MAX_INVALIDATE_WAIT_MS && !complete)
-  if (!complete) {
-    throw "Unable to invalidate old app version"
-  }
 }
 
 exports.updateDeploymentQuota = async function(quota) {
@@ -102,6 +92,18 @@ exports.verifyDeployment = async function({ instanceId, appId, quota }) {
   }
 
   const json = await response.json()
+  if (json.errors) {
+    throw new Error(json.errors)
+  }
+
+  // set credentials here, means any time we're verified we're ready to go
+  if (json.credentials) {
+    AWS.config.update({
+      accessKeyId: json.credentials.AccessKeyId,
+      secretAccessKey: json.credentials.SecretAccessKey,
+      sessionToken: json.credentials.SessionToken,
+    })
+  }
 
   return json
 }
@@ -136,7 +138,8 @@ async function prepareUploadForS3({ s3Key, metadata, s3, file }) {
 
   const upload = await s3
     .upload({
-      Key: s3Key,
+      // windows filepaths need to be converted to forward slashes for s3
+      Key: sanitize(s3Key).replace(/\\/g, "/"),
       Body: fileBytes,
       ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()],
       Metadata: metadata,
@@ -157,17 +160,10 @@ exports.prepareUploadForS3 = prepareUploadForS3
 exports.uploadAppAssets = async function({
   appId,
   instanceId,
-  credentials,
   bucket,
   cfDistribution,
   accountId,
 }) {
-  AWS.config.update({
-    accessKeyId: credentials.AccessKeyId,
-    secretAccessKey: credentials.SecretAccessKey,
-    sessionToken: credentials.SessionToken,
-  })
-
   const s3 = new AWS.S3({
     params: {
       Bucket: bucket,
@@ -225,7 +221,7 @@ exports.uploadAppAssets = async function({
 
   try {
     await Promise.all(uploads)
-    await invalidateCDN(cfDistribution, appId)
+    return await invalidateCDN(cfDistribution, appId)
   } catch (err) {
     console.error("Error uploading budibase app assets to s3", err)
     throw err

packages/server/src/api/controllers/deploy/index.js

@@ -4,17 +4,73 @@ const {
   uploadAppAssets,
   verifyDeployment,
   updateDeploymentQuota,
-  MAX_INVALIDATE_WAIT_MS,
+  isInvalidationComplete,
 } = require("./aws")
 const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils")
 const newid = require("../../../db/newid")
 
+// the max time we can wait for an invalidation to complete before considering it failed
+const MAX_PENDING_TIME_MS = 30 * 60000
+
 const DeploymentStatus = {
   SUCCESS: "SUCCESS",
   PENDING: "PENDING",
   FAILURE: "FAILURE",
 }
 
+// checks that deployments are in a good state, any pending will be updated
+async function checkAllDeployments(deployments, user) {
+  let updated = false
+  function update(deployment, status) {
+    deployment.status = status
+    delete deployment.invalidationId
+    delete deployment.cfDistribution
+    updated = true
+  }
+
+  for (let deployment of Object.values(deployments.history)) {
+    // check that no deployments have crashed etc and are now stuck
+    if (
+      deployment.status === DeploymentStatus.PENDING &&
+      Date.now() - deployment.updatedAt > MAX_PENDING_TIME_MS
+    ) {
+      update(deployment, DeploymentStatus.FAILURE)
+    }
+    // if pending but not past failure point need to update them
+    else if (deployment.status === DeploymentStatus.PENDING) {
+      let complete = false
+      try {
+        complete = await isInvalidationComplete(
+          deployment.cfDistribution,
+          deployment.invalidationId
+        )
+      } catch (err) {
+        // system may have restarted, need to re-verify
+        if (
+          err !== undefined &&
+          err.code === "InvalidClientTokenId" &&
+          deployment.quota
+        ) {
+          await verifyDeployment({
+            ...user,
+            quota: deployment.quota,
+          })
+          complete = await isInvalidationComplete(
+            deployment.cfDistribution,
+            deployment.invalidationId
+          )
+        } else {
+          throw err
+        }
+      }
+      if (complete) {
+        update(deployment, DeploymentStatus.SUCCESS)
+      }
+    }
+  }
+  return { updated, deployments }
+}
+
 function replicate(local, remote) {
   return new Promise((resolve, reject) => {
     const replication = local.sync(remote)
@@ -102,7 +158,7 @@ async function storeLocalDeploymentHistory(deployment) {
 async function deployApp({ instanceId, appId, clientId, deploymentId }) {
   try {
     const instanceQuota = await getCurrentInstanceQuota(instanceId)
-    const credentials = await verifyDeployment({
+    const verification = await verifyDeployment({
       instanceId,
       appId,
       quota: instanceQuota,
@@ -110,31 +166,36 @@ async function deployApp({ instanceId, appId, clientId, deploymentId }) {
 
     console.log(`Uploading assets for appID ${appId} assets to s3..`)
 
-    if (credentials.errors) throw new Error(credentials.errors)
+    const invalidationId = await uploadAppAssets({
-
+      clientId,
-    await uploadAppAssets({ clientId, appId, instanceId, ...credentials })
+      appId,
+      instanceId,
+      ...verification,
+    })
 
     // replicate the DB to the couchDB cluster in prod
     console.log("Replicating local PouchDB to remote..")
     await replicateCouch({
       instanceId,
       clientId,
-      session: credentials.couchDbSession,
+      session: verification.couchDbSession,
     })
 
-    await updateDeploymentQuota(credentials.quota)
+    await updateDeploymentQuota(verification.quota)
 
     await storeLocalDeploymentHistory({
       _id: deploymentId,
       instanceId,
-      quota: credentials.quota,
+      invalidationId,
-      status: "SUCCESS",
+      cfDistribution: verification.cfDistribution,
+      quota: verification.quota,
+      status: DeploymentStatus.PENDING,
     })
   } catch (err) {
     await storeLocalDeploymentHistory({
       _id: deploymentId,
       instanceId,
-      status: "FAILURE",
+      status: DeploymentStatus.FAILURE,
       err: err.message,
     })
     throw new Error(`Deployment Failed: ${err.message}`)
@@ -145,21 +206,14 @@ exports.fetchDeployments = async function(ctx) {
   try {
     const db = new PouchDB(ctx.user.instanceId)
     const deploymentDoc = await db.get("_local/deployments")
-    // check that no deployments have crashed etc and are now stuck
+    const { updated, deployments } = await checkAllDeployments(
-    let changed = false
+      deploymentDoc,
-    for (let deployment of Object.values(deploymentDoc.history)) {
+      ctx.user
-      if (
+    )
-        deployment.status === DeploymentStatus.PENDING &&
+    if (updated) {
-        Date.now() - deployment.updatedAt > MAX_INVALIDATE_WAIT_MS
+      await db.put(deployments)
-      ) {
-        deployment.status = DeploymentStatus.FAILURE
-        changed = true
     }
-    }
+    ctx.body = Object.values(deployments.history).reverse()
-    if (changed) {
-      await db.put(deploymentDoc)
-    }
-    ctx.body = Object.values(deploymentDoc.history).reverse()
   } catch (err) {
     ctx.body = []
   }
@@ -185,10 +239,10 @@ exports.deployApp = async function(ctx) {
   const deployment = await storeLocalDeploymentHistory({
     instanceId: ctx.user.instanceId,
     appId: ctx.user.appId,
-    status: "PENDING",
+    status: DeploymentStatus.PENDING,
   })
 
-  deployApp({
+  await deployApp({
     ...ctx.user,
     clientId,
     deploymentId: deployment._id,

packages/server/src/api/controllers/static.js

@@ -163,15 +163,14 @@ exports.serveApp = async function(ctx) {
 
 exports.serveAttachment = async function(ctx) {
   const appId = ctx.user.appId
-
   const attachmentsPath = resolve(budibaseAppsDir(), appId, "attachments")
 
   // Serve from CloudFront
   if (process.env.CLOUD) {
     const S3_URL = `https://cdn.app.budi.live/assets/${appId}/attachments/${ctx.file}`
-
     const response = await fetch(S3_URL)
     const body = await response.text()
+    ctx.set("Content-Type", response.headers.get("Content-Type"))
     ctx.body = body
     return
   }

packages/server/src/utilities/fileProcessor.js

@@ -1,24 +1,21 @@
 const fs = require("fs")
-const sharp = require("sharp")
+const jimp = require("jimp")
 const fsPromises = fs.promises
 
 const FORMATS = {
-  IMAGES: ["png", "jpg", "jpeg", "gif", "svg", "tiff", "raw"],
+  IMAGES: ["png", "jpg", "jpeg", "gif", "bmp", "tiff"],
 }
 
-async function processImage(file) {
+function processImage(file) {
-  const imgMeta = await sharp(file.path)
+  return jimp.read(file.path).then(img => {
-    .resize(300)
+    return img.resize(300, jimp.AUTO).write(file.outputPath)
-    .toFile(file.outputPath)
+  })
-  return {
-    ...file,
-    ...imgMeta,
-  }
 }
 
 async function process(file) {
   if (FORMATS.IMAGES.includes(file.extension.toLowerCase())) {
-    return await processImage(file)
+    await processImage(file)
+    return file
   }
 
   // No processing required

packages/standard-components/package.json

@@ -36,7 +36,7 @@
   "gitHead": "284cceb9b703c38566c6e6363c022f79a08d5691",
   "dependencies": {
     "@beyonk/svelte-googlemaps": "^2.2.0",
-    "@budibase/bbui": "^1.44.0",
+    "@budibase/bbui": "^1.44.1",
     "@budibase/svelte-ag-grid": "^0.0.13",
     "@fortawesome/fontawesome-free": "^5.14.0",
     "@svelteschool/svelte-forms": "^0.7.0",