budibase complete deployment

.github/workflows/budibase_ci.yml

@@ -33,35 +33,3 @@ jobs:
       env:
         CI: true
         name: Budibase CI
-
-    - name: Login to Amazon ECR
-      id: login-ecr
-      uses: aws-actions/amazon-ecr-login@v1
-    
-    - name: Build, tag, and push image to Amazon ECR
-      id: build-image
-      env:
-        ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-        ECR_REPOSITORY: my-ecr-repo
-        IMAGE_TAG: ${{ github.sha }}
-      run: |
-        docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
-        docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
-        echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG"
-
-    - name: Fill in the new image ID in the Amazon ECS task definition
-      id: task-def
-      uses: aws-actions/amazon-ecs-render-task-definition@v1
-      with:
-        task-definition: task-definition.json
-        container-name: my-container
-        image: ${{ steps.build-image.outputs.image }}
-
-  - name: Deploy Amazon ECS task definition
-    uses: aws-actions/amazon-ecs-deploy-task-definition@v1
-    with:
-      task-definition: ${{ steps.task-def.outputs.task-definition }}
-      service: my-service
-      cluster: my-cluster
-      wait-for-service-stability: true
-  

packages/builder/src/components/userInterface/ComponentDropdownMenu.svelte

@@ -27,8 +27,7 @@
     animation: false,
   })
   $: dropdown && UIkit.util.on(dropdown, "shown", () => (hidden = false))
-  $: noChildrenAllowed = false
-    !component ||
+  $: noChildrenAllowed = !component ||
     getComponentDefinition($store, component._component).children === false
   $: noPaste = !$store.componentToPaste
 

packages/client/src/index.js

@@ -23,7 +23,6 @@ export const loadBudibase = async opts => {
     componentLibraryModules[library] = await import(
       `/componentlibrary?library=${encodeURI(library)}`
     )
-    // componentLibraryModules[library] = await import(`/assets/componentlibrary/${library}/dist/index.js`)
   }
 
   componentLibraryModules[builtinLibName] = builtins(_window)

packages/server/.env.template

@@ -14,6 +14,4 @@ PORT=4001
 # error level for koa-pino
 LOG_LEVEL=error
 
-DEPLOYMENT_CF_DISTRIBUTION_ID=
-DEPLOYMENT_APP_ASSETS_BUCKET=g
 DEPLOYMENT_CREDENTIALS_URL="https://dt4mpwwap8.execute-api.eu-west-1.amazonaws.com/prod/"

packages/server/envfile

@@ -1,17 +0,0 @@
-# url of couch db, including username and password
-# http://admin:password@localhost:5984
-COUCH_DB_URL=
-# identifies a client database - i.e. group of apps
-CLIENT_ID=1
-# used to create cookie hashes
-JWT_SECRET=4715888e-144f-4802-b8d8-862a1b8365dd
-# port to run http server on
-PORT=4001 
-
-# error level for koa-pino
-LOG_LEVEL=error
-
-COUCH_DB_REMOTE=https://admin:afasdgafgF342G@couchdb.budi.live:5984
-BUDIBASE_APP_ASSETS_BUCKET=prod-budi-app-assets
-BUDIBASE_API_KEY=d498278c-4ab4-144b-c212-b8f9e6da5c2b
-

packages/server/src/api/controllers/deploy/aws.js

@@ -5,11 +5,11 @@ const {
   budibaseAppsDir,
 } = require("../../../utilities/budibaseDir")
 
-async function invalidateCDN(appId) {
+async function invalidateCDN(cfDistribution, appId) {
   const cf = new AWS.CloudFront({})
 
   return cf.createInvalidation({ 
-    DistributionId: process.env.DEPLOYMENT_CF_DISTRIBUTION_ID,
+    DistributionId: cfDistribution,
     InvalidationBatch: {
       CallerReference: appId,
       Paths: {
@@ -47,8 +47,8 @@ const CONTENT_TYPE_MAP = {
 
 /**
  * Recursively walk a directory tree and execute a callback on all files.
- * @param {Re} dirPath - Directory to traverse
- * @param {*} callback - callback to execute on files
+ * @param {String} dirPath - Directory to traverse
+ * @param {Function} callback - callback to execute on files
  */
 function walkDir(dirPath, callback) {
   for (let filename of fs.readdirSync(dirPath)) {
@@ -56,10 +56,7 @@ function walkDir(dirPath, callback) {
     const stat = fs.lstatSync(filePath)
     
     if (stat.isFile()) {
-      callback({ 
-        bytes: fs.readFileSync(filePath), 
-        filename 
-      })
+      callback(filePath)
     } else {
       walkDir(filePath, callback)
     }
@@ -67,7 +64,12 @@ function walkDir(dirPath, callback) {
 }
 
 exports.uploadAppAssets = async function ({ appId }) {
-  const { credentials, accountId } = await fetchTemporaryCredentials()
+  const { 
+    credentials, 
+    accountId, 
+    bucket,
+    cfDistribution,
+  } = await fetchTemporaryCredentials()
 
   AWS.config.update({
     accessKeyId: credentials.AccessKeyId, 
@@ -77,7 +79,7 @@ exports.uploadAppAssets = async function ({ appId }) {
 
   const s3 = new AWS.S3({
     params: {
-      Bucket: process.env.DEPLOYMENT_APP_ASSETS_BUCKET
+      Bucket: bucket 
     }
   })
 
@@ -88,12 +90,13 @@ exports.uploadAppAssets = async function ({ appId }) {
   const uploads = []
 
   for (let page of appPages) {
-    walkDir(`${appAssetsPath}/${page}`, function prepareUploadsForS3({ bytes, filename }) {
-      const fileExtension = [...filename.split(".")].pop()
+    walkDir(`${appAssetsPath}/${page}`, function prepareUploadsForS3(filePath) {
+      const fileExtension = [...filePath.split(".")].pop()
+      const fileBytes = fs.readFileSync(filePath)
 
       const upload = s3.upload({
-        Key: `assets/${appId}/${page}/${filename}`,
-        Body: bytes,
+        Key: filePath.replace(appAssetsPath, `assets/${appId}`),
+        Body: fileBytes,
         ContentType: CONTENT_TYPE_MAP[fileExtension],
         Metadata: {
           accountId
@@ -105,10 +108,8 @@ exports.uploadAppAssets = async function ({ appId }) {
   }
 
   try {
-    const uploadAllFiles = Promise.all(uploads)
-    const invalidateCloudfront = invalidateCDN(appId) 
-    await uploadAllFiles
-    await invalidateCloudfront
+    await Promise.all(uploads)
+    await invalidateCDN(cfDistribution, appId) 
   } catch (err) {
     console.error("Error uploading budibase app assets to s3", err)
     throw err

packages/server/src/api/controllers/deploy/index.js

@@ -7,7 +7,7 @@ const {
 function replicate(local, remote) {
   return new Promise((resolve, reject) => {
 
-    const replication = local.replicate.to(remote);
+    const replication = local.sync(remote);
 
     replication.on("complete", () => resolve())
     replication.on("error", err => reject(err))
@@ -15,24 +15,24 @@ function replicate(local, remote) {
 }
 
 async function replicateCouch(instanceId, clientId) {
-    const clientDb = new PouchDB(`client_${clientId}`)
-    const remoteClientDb = new CouchDB(`${process.env.COUCH_DB_REMOTE}/client_${clientId}`)
 
-    const clientAppLookupDb = new PouchDB("client_app_lookup")
-    const remoteClientAppLookupDb = new CouchDB(`${process.env.COUCH_DB_REMOTE}/client_app_lookup`)
+    const databases = [
+      `client_${clientId}`,
+      "client_app_lookup",
+      instanceId
+    ];
 
-    const localDb = new PouchDB(instanceId)
-    const remoteDb = new CouchDB(`${process.env.COUCH_DB_REMOTE}/${instanceId}`)
+    const replications = databases.map(local => {
+      const localDb = new PouchDB(local);
+      const remoteDb = new CouchDB(`${process.env.DEPLOYMENT_COUCH_DB_URL}/${local}`)
 
-    await Promise.all([
-      replicate(clientDb, remoteClientDb),
-      replicate(clientAppLookupDb, remoteClientAppLookupDb),
-      replicate(localDb, remoteDb)
-    ])
+      return replicate(localDb, remoteDb);
+    });
+
+    await Promise.all(replications)
 }
 
 exports.deployApp = async function(ctx) {
-  // TODO: This should probably be async - it could take a while  
   try {
     const clientAppLookupDB = new PouchDB("client_app_lookup")
     const { clientId } = await clientAppLookupDB.get(ctx.user.appId)
@@ -51,8 +51,7 @@ exports.deployApp = async function(ctx) {
       status: "SUCCESS", 
       completed: Date.now()
     }
-
-    } catch (err) {
-      ctx.throw(err.status || 500, `Deployment Failed: ${err.message}`);
-    }
+  } catch (err) {
+    ctx.throw(err.status || 500, `Deployment Failed: ${err.message}`);
+  }
 }

packages/server/src/api/controllers/static.js

@@ -8,7 +8,6 @@ const setBuilderToken = require("../../utilities/builder/setBuilderToken")
 const { ANON_LEVEL_ID } = require("../../utilities/accessLevels")
 const jwt = require("jsonwebtoken")
 const fetch = require("node-fetch")
-const { S3 } = require("aws-sdk")
 
 exports.serveBuilder = async function(ctx) {
   let builderPath = resolve(__dirname, "../../../builder")
@@ -28,14 +27,20 @@ exports.serveApp = async function(ctx) {
     "public",
     mainOrAuth
   )
+
+  let appId = ctx.params.appId
+  if (process.env.CLOUD) {
+    appId = ctx.subdomains[1]
+  }
+  
   // only set the appId cookie for /appId .. we COULD check for valid appIds
   // but would like to avoid that DB hit
-  const looksLikeAppId = /^[0-9a-f]{32}$/.test(ctx.params.appId)
+  const looksLikeAppId = /^[0-9a-f]{32}$/.test(appId)
   if (looksLikeAppId && !ctx.isAuthenticated) {
     const anonUser = {
       userId: "ANON",
       accessLevelId: ANON_LEVEL_ID,
-      appId: ctx.params.appId,
+      appId,
     }
     const anonToken = jwt.sign(anonUser, ctx.config.jwtSecret)
     ctx.cookies.set("budibase:token", anonToken, {
@@ -45,7 +50,7 @@ exports.serveApp = async function(ctx) {
   }
 
   if (process.env.CLOUD) {
-    const S3_URL = `https://${ctx.params.appId}.app.budi.live/assets/${ctx.params.appId}/${mainOrAuth}/${ctx.file || "index.production.html"}`
+    const S3_URL = `https://${appId}.app.budi.live/assets/${appId}/${mainOrAuth}/${ctx.file || "index.production.html"}`
     const response = await fetch(S3_URL)
     const body = await response.text()
     ctx.body = body 

packages/server/src/app.js

@@ -16,7 +16,7 @@ app.use(
     prettyPrint: {
       levelFirst: true,
     },
-    level: "info" || "info",
+    level: env.LOG_LEVEL || "error",
   })
 )