Merge branch 'master' of github.com:Budibase/budibase into endpoint-renaming

packages/builder/src/pages/[application]/deploy/index.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { Button } from "@budibase/bbui"
+  import { Button, Spacer } from "@budibase/bbui"
   import { store } from "builderStore"
   import { notifier } from "builderStore/store/notifications"
   import api from "builderStore/api"
@@ -51,6 +51,7 @@
       <Button secondary medium on:click={deployApp}>
         Deploy App
         {#if loading}
+          <Spacer extraSmall />
           <Spinner size="10" />
         {/if}
       </Button>

packages/server/.dockerignore

@@ -0,0 +1,6 @@
+node_modules
+npm-debug.log
+Dockerfile
+.dockerignore
+.git
+.gitignore

packages/server/src/api/controllers/auth.js

@@ -55,7 +55,8 @@ exports.authenticate = async ctx => {
     }
     // if in cloud add the user api key
     if (environment.CLOUD) {
-      payload.apiKey = getAPIKey(ctx.user.appId)
+      const { apiKey } = await getAPIKey(ctx.user.appId)
+      payload.apiKey = apiKey
     }
 
     const token = jwt.sign(payload, ctx.config.jwtSecret, {

packages/server/src/api/controllers/deploy/aws.js

@@ -4,6 +4,7 @@ const AWS = require("aws-sdk")
 const fetch = require("node-fetch")
 const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
 const PouchDB = require("../../../db")
+const environment = require("../../../environment")
 
 async function invalidateCDN(cfDistribution, appId) {
   const cf = new AWS.CloudFront({})
@@ -22,11 +23,46 @@ async function invalidateCDN(cfDistribution, appId) {
     .promise()
 }
 
-exports.fetchTemporaryCredentials = async function() {
+exports.updateDeploymentQuota = async function(quota) {
+  const DEPLOYMENT_SUCCESS_URL =
+    environment.DEPLOYMENT_CREDENTIALS_URL + "deploy/success"
+
+  const response = await fetch(DEPLOYMENT_SUCCESS_URL, {
+    method: "POST",
+    body: JSON.stringify({
+      apiKey: process.env.BUDIBASE_API_KEY,
+      quota,
+    }),
+    headers: {
+      "Content-Type": "application/json",
+      Accept: "application/json",
+    },
+  })
+
+  if (response.status !== 200) {
+    throw new Error(`Error updating deployment quota for API Key`)
+  }
+
+  const json = await response.json()
+
+  return json
+}
+
+/**
+ * Verifies the users API key and
+ * Verifies that the deployment fits within the quota of the user,
+ * @param {String} instanceId - instanceId being deployed
+ * @param {String} appId - appId being deployed
+ * @param {quota} quota - current quota being changed with this application
+ */
+exports.verifyDeployment = async function({ instanceId, appId, quota }) {
   const response = await fetch(process.env.DEPLOYMENT_CREDENTIALS_URL, {
     method: "POST",
     body: JSON.stringify({
       apiKey: process.env.BUDIBASE_API_KEY,
+      instanceId,
+      appId,
+      quota,
     }),
   })
 
@@ -133,30 +169,33 @@ exports.uploadAppAssets = async function({
 
   // Upload file attachments
   const db = new PouchDB(instanceId)
-  const fileUploads = await db.get("_local/fileuploads")
+  let fileUploads
-  if (fileUploads) {
+  try {
-    for (let file of fileUploads.uploads) {
+    fileUploads = await db.get("_local/fileuploads")
-      if (file.uploaded) continue
+  } catch (err) {
-
+    fileUploads = { _id: "_local/fileuploads", uploads: [] }
-      const attachmentUpload = prepareUploadForS3({
-        file,
-        s3Key: `assets/${appId}/attachments/${file.processedFileName}`,
-        s3,
-        metadata: { accountId },
-      })
-
-      uploads.push(attachmentUpload)
-
-      // mark file as uploaded
-      file.uploaded = true
-    }
-
-    db.put(fileUploads)
   }
 
+  for (let file of fileUploads.uploads) {
+    if (file.uploaded) continue
+
+    const attachmentUpload = prepareUploadForS3({
+      file,
+      s3Key: `assets/${appId}/attachments/${file.processedFileName}`,
+      s3,
+      metadata: { accountId },
+    })
+
+    uploads.push(attachmentUpload)
+
+    // mark file as uploaded
+    file.uploaded = true
+  }
+
+  db.put(fileUploads)
+
   try {
     await Promise.all(uploads)
-    // TODO: update dynamoDB with a synopsis of the app deployment for historical purposes
     await invalidateCDN(cfDistribution, appId)
   } catch (err) {
     console.error("Error uploading budibase app assets to s3", err)

packages/server/src/api/controllers/deploy/index.js

@@ -1,6 +1,11 @@
 const CouchDB = require("pouchdb")
 const PouchDB = require("../../../db")
-const { uploadAppAssets, fetchTemporaryCredentials } = require("./aws")
+const {
+  uploadAppAssets,
+  verifyDeployment,
+  updateDeploymentQuota,
+} = require("./aws")
+const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils")
 
 function replicate(local, remote) {
   return new Promise((resolve, reject) => {
@@ -31,13 +36,49 @@ async function replicateCouch({ instanceId, clientId, credentials }) {
   await Promise.all(replications)
 }
 
+async function getCurrentInstanceQuota(instanceId) {
+  const db = new PouchDB(instanceId)
+
+  const rows = await db.allDocs({
+    startkey: DocumentTypes.ROW + SEPARATOR,
+    endkey: DocumentTypes.ROW + SEPARATOR + UNICODE_MAX,
+  })
+
+  const users = await db.allDocs({
+    startkey: DocumentTypes.USER + SEPARATOR,
+    endkey: DocumentTypes.USER + SEPARATOR + UNICODE_MAX,
+  })
+
+  const existingRows = rows.rows.length
+  const existingUsers = users.rows.length
+
+  const designDoc = await db.get("_design/database")
+
+  return {
+    rows: existingRows,
+    users: existingUsers,
+    views: Object.keys(designDoc.views).length,
+  }
+}
+
 exports.deployApp = async function(ctx) {
   try {
     const clientAppLookupDB = new PouchDB("client_app_lookup")
     const { clientId } = await clientAppLookupDB.get(ctx.user.appId)
 
+    const instanceQuota = await getCurrentInstanceQuota(ctx.user.instanceId)
+    const credentials = await verifyDeployment({
+      instanceId: ctx.user.instanceId,
+      appId: ctx.user.appId,
+      quota: instanceQuota,
+    })
+
     ctx.log.info(`Uploading assets for appID ${ctx.user.appId} assets to s3..`)
-    const credentials = await fetchTemporaryCredentials()
+
+    if (credentials.errors) {
+      ctx.throw(500, credentials.errors)
+      return
+    }
 
     await uploadAppAssets({
       clientId,
@@ -54,6 +95,8 @@ exports.deployApp = async function(ctx) {
       credentials: credentials.couchDbCreds,
     })
 
+    await updateDeploymentQuota(credentials.quota)
+
     ctx.body = {
       status: "SUCCESS",
       completed: Date.now(),

packages/server/src/api/controllers/static.js

@@ -153,7 +153,7 @@ exports.serveApp = async function(ctx) {
 
   // only set the appId cookie for /appId .. we COULD check for valid appIds
   // but would like to avoid that DB hit
-  const looksLikeAppId = /^[0-9a-f]{32}$/.test(appId)
+  const looksLikeAppId = /^(app_)?[0-9a-f]{32}$/.test(appId)
   if (looksLikeAppId && !ctx.isAuthenticated) {
     const anonUser = {
       userId: "ANON",

packages/server/src/db/linkedRows/index.js

@@ -82,7 +82,7 @@ exports.updateLinks = async function({
  * then an array will be output, object input -> object output.
  */
 exports.attachLinkInfo = async (instanceId, rows) => {
-  // handle a single record as well as multiple
+  // handle a single row as well as multiple
   let wasArray = true
   if (!(rows instanceof Array)) {
     rows = [rows]

packages/server/src/db/utils.js

@@ -15,6 +15,7 @@ const DocumentTypes = {
 
 exports.DocumentTypes = DocumentTypes
 exports.SEPARATOR = SEPARATOR
+exports.UNICODE_MAX = UNICODE_MAX
 
 /**
  * If creating DB allDocs/query params with only a single top level ID this can be used, this

packages/server/src/environment.js

@@ -14,4 +14,5 @@ module.exports = {
   CLOUD: process.env.CLOUD,
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
   AWS_REGION: process.env.AWS_REGION,
+  DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
 }

packages/server/src/middleware/authenticated.js

@@ -45,7 +45,7 @@ module.exports = async (ctx, next) => {
 
   try {
     const jwtPayload = jwt.verify(appToken, ctx.config.jwtSecret)
-
+    ctx.apiKey = jwtPayload.apiKey
     ctx.user = {
       ...jwtPayload,
       accessLevel: await getAccessLevel(

packages/server/src/utilities/usageQuota.js

@@ -22,7 +22,7 @@ function buildUpdateParams(key, property, usage) {
   return {
     primary: key,
     condition:
-      "#quota.#prop < #limits.#prop AND #quotaReset > :now AND attribute_exists(#quota) AND attribute_exists(#limits)",
+      "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now",
     expression: "ADD #quota.#prop :usage",
     names: {
       "#quota": "usageQuota",
@@ -69,13 +69,16 @@ exports.update = async (apiKey, property, usage) => {
   try {
     await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
   } catch (err) {
+    // conditional check means the condition failed, need to check why
     if (err.code === "ConditionalCheckFailedException") {
       // get the API key so we can check it
       const keyObj = await apiKeyTable.get({ primary: apiKey })
       // the usage quota or usage limits didn't exist
       if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) {
-        keyObj.usageQuota = DEFAULT_USAGE
+        keyObj.usageQuota =
-        keyObj.usageLimits = DEFAULT_PLAN
+          keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota
+        keyObj.usageLimits =
+          keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits
         keyObj.quotaReset = getNewQuotaReset()
         await apiKeyTable.put({ item: keyObj })
         return
@@ -94,7 +97,7 @@ exports.update = async (apiKey, property, usage) => {
         await apiKeyTable.put({ item: keyObj })
         return
       }
-      throw "Resource limits have been reached"
     }
+    throw err
   }
 }