merge master

packages/builder/src/pages/[application]/deploy/index.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { Button } from "@budibase/bbui"
+  import { Button, Spacer } from "@budibase/bbui"
   import { store } from "builderStore"
   import { notifier } from "builderStore/store/notifications"
   import api from "builderStore/api"
@@ -51,6 +51,7 @@
       <Button secondary medium on:click={deployApp}>
         Deploy App
         {#if loading}
+          <Spacer extraSmall />
           <Spinner size="10" />
         {/if}
       </Button>

packages/server/.dockerignore

@@ -0,0 +1,6 @@
+node_modules
+npm-debug.log
+Dockerfile
+.dockerignore
+.git
+.gitignore

packages/server/package.json

@@ -46,7 +46,7 @@
     "@koa/router": "^8.0.0",
     "@sendgrid/mail": "^7.1.1",
     "@sentry/node": "^5.19.2",
-    "aws-sdk": "^2.706.0",
+    "aws-sdk": "^2.767.0",
     "bcryptjs": "^2.4.3",
     "chmodr": "^1.2.0",
     "csvtojson": "^2.0.10",

packages/server/scripts/createApiKeyAndAppId.js

@@ -0,0 +1,55 @@
+// THIS will create API Keys and App Ids input in a local Dynamo instance if it is running
+const dynamoClient = require("../src/db/dynamoClient")
+
+if (process.argv[2] == null || process.argv[3] == null) {
+  console.error(
+    "Inputs incorrect format, was expecting: node createApiKeyAndAppId.js <API_KEY> <APP_ID>"
+  )
+  process.exit(-1)
+}
+
+const FAKE_STRING = "fakestring"
+
+// set fake credentials for local dynamo to actually work
+process.env.AWS_ACCESS_KEY_ID = "KEY_ID"
+process.env.AWS_SECRET_ACCESS_KEY = "SECRET_KEY"
+dynamoClient.init("http://localhost:8333")
+
+async function run() {
+  await dynamoClient.apiKeyTable.put({
+    item: {
+      pk: process.argv[2],
+      accountId: FAKE_STRING,
+      trackingId: FAKE_STRING,
+      quotaReset: Date.now() + 2592000000,
+      usageQuota: {
+        automationRuns: 0,
+        records: 0,
+        storage: 0,
+        users: 0,
+        views: 0,
+      },
+      usageLimits: {
+        automationRuns: 10,
+        records: 10,
+        storage: 1000,
+        users: 10,
+        views: 10,
+      },
+    },
+  })
+  await dynamoClient.apiKeyTable.put({
+    item: {
+      pk: process.argv[3],
+      apiKey: process.argv[2],
+    },
+  })
+}
+
+run()
+  .then(() => {
+    console.log("Records should have been created.")
+  })
+  .catch(err => {
+    console.error("Cannot create records - " + err)
+  })

packages/server/src/api/controllers/auth.js

@@ -2,6 +2,8 @@ const jwt = require("jsonwebtoken")
 const CouchDB = require("../../db")
 const ClientDb = require("../../db/clientDb")
 const bcrypt = require("../../utilities/bcrypt")
+const environment = require("../../environment")
+const { getAPIKey } = require("../../utilities/usageQuota")
 const { generateUserID } = require("../../db/utils")
 
 exports.authenticate = async ctx => {
@@ -51,6 +53,11 @@ exports.authenticate = async ctx => {
       appId: ctx.user.appId,
       instanceId,
     }
+    // if in cloud add the user api key
+    if (environment.CLOUD) {
+      const { apiKey } = await getAPIKey(ctx.user.appId)
+      payload.apiKey = apiKey
+    }
 
     const token = jwt.sign(payload, ctx.config.jwtSecret, {
       expiresIn: "1 day",

packages/server/src/api/controllers/automation.js

@@ -33,6 +33,7 @@ function cleanAutomationInputs(automation) {
 exports.create = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
   let automation = ctx.request.body
+  automation.appId = ctx.user.appId
 
   automation._id = generateAutomationID()
 
@@ -54,6 +55,7 @@ exports.create = async function(ctx) {
 exports.update = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
   let automation = ctx.request.body
+  automation.appId = ctx.user.appId
 
   automation = cleanAutomationInputs(automation)
   const response = await db.put(automation)

packages/server/src/api/controllers/deploy/aws.js

@@ -4,6 +4,7 @@ const AWS = require("aws-sdk")
 const fetch = require("node-fetch")
 const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
 const PouchDB = require("../../../db")
+const environment = require("../../../environment")
 
 async function invalidateCDN(cfDistribution, appId) {
   const cf = new AWS.CloudFront({})
@@ -22,11 +23,46 @@ async function invalidateCDN(cfDistribution, appId) {
     .promise()
 }
 
-exports.fetchTemporaryCredentials = async function() {
+exports.updateDeploymentQuota = async function(quota) {
+  const DEPLOYMENT_SUCCESS_URL =
+    environment.DEPLOYMENT_CREDENTIALS_URL + "deploy/success"
+
+  const response = await fetch(DEPLOYMENT_SUCCESS_URL, {
+    method: "POST",
+    body: JSON.stringify({
+      apiKey: process.env.BUDIBASE_API_KEY,
+      quota,
+    }),
+    headers: {
+      "Content-Type": "application/json",
+      Accept: "application/json",
+    },
+  })
+
+  if (response.status !== 200) {
+    throw new Error(`Error updating deployment quota for API Key`)
+  }
+
+  const json = await response.json()
+
+  return json
+}
+
+/**
+ * Verifies the users API key and
+ * Verifies that the deployment fits within the quota of the user,
+ * @param {String} instanceId - instanceId being deployed
+ * @param {String} appId - appId being deployed
+ * @param {quota} quota - current quota being changed with this application
+ */
+exports.verifyDeployment = async function({ instanceId, appId, quota }) {
   const response = await fetch(process.env.DEPLOYMENT_CREDENTIALS_URL, {
     method: "POST",
     body: JSON.stringify({
       apiKey: process.env.BUDIBASE_API_KEY,
+      instanceId,
+      appId,
+      quota,
     }),
   })
 
@@ -133,30 +169,33 @@ exports.uploadAppAssets = async function({
 
   // Upload file attachments
   const db = new PouchDB(instanceId)
-  const fileUploads = await db.get("_local/fileuploads")
+  let fileUploads
-  if (fileUploads) {
+  try {
-    for (let file of fileUploads.uploads) {
+    fileUploads = await db.get("_local/fileuploads")
-      if (file.uploaded) continue
+  } catch (err) {
-
+    fileUploads = { _id: "_local/fileuploads", uploads: [] }
-      const attachmentUpload = prepareUploadForS3({
-        file,
-        s3Key: `assets/${appId}/attachments/${file.processedFileName}`,
-        s3,
-        metadata: { accountId },
-      })
-
-      uploads.push(attachmentUpload)
-
-      // mark file as uploaded
-      file.uploaded = true
-    }
-
-    db.put(fileUploads)
   }
 
+  for (let file of fileUploads.uploads) {
+    if (file.uploaded) continue
+
+    const attachmentUpload = prepareUploadForS3({
+      file,
+      s3Key: `assets/${appId}/attachments/${file.processedFileName}`,
+      s3,
+      metadata: { accountId },
+    })
+
+    uploads.push(attachmentUpload)
+
+    // mark file as uploaded
+    file.uploaded = true
+  }
+
+  db.put(fileUploads)
+
   try {
     await Promise.all(uploads)
-    // TODO: update dynamoDB with a synopsis of the app deployment for historical purposes
     await invalidateCDN(cfDistribution, appId)
   } catch (err) {
     console.error("Error uploading budibase app assets to s3", err)

packages/server/src/api/controllers/deploy/index.js

@@ -1,6 +1,11 @@
 const CouchDB = require("pouchdb")
 const PouchDB = require("../../../db")
-const { uploadAppAssets, fetchTemporaryCredentials } = require("./aws")
+const {
+  uploadAppAssets,
+  verifyDeployment,
+  updateDeploymentQuota,
+} = require("./aws")
+const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils")
 
 function replicate(local, remote) {
   return new Promise((resolve, reject) => {
@@ -31,13 +36,49 @@ async function replicateCouch({ instanceId, clientId, credentials }) {
   await Promise.all(replications)
 }
 
+async function getCurrentInstanceQuota(instanceId) {
+  const db = new PouchDB(instanceId)
+
+  const records = await db.allDocs({
+    startkey: DocumentTypes.RECORD + SEPARATOR,
+    endkey: DocumentTypes.RECORD + SEPARATOR + UNICODE_MAX,
+  })
+
+  const users = await db.allDocs({
+    startkey: DocumentTypes.USER + SEPARATOR,
+    endkey: DocumentTypes.USER + SEPARATOR + UNICODE_MAX,
+  })
+
+  const existingRecords = records.rows.length
+  const existingUsers = users.rows.length
+
+  const designDoc = await db.get("_design/database")
+
+  return {
+    records: existingRecords,
+    users: existingUsers,
+    views: Object.keys(designDoc.views).length,
+  }
+}
+
 exports.deployApp = async function(ctx) {
   try {
     const clientAppLookupDB = new PouchDB("client_app_lookup")
     const { clientId } = await clientAppLookupDB.get(ctx.user.appId)
 
+    const instanceQuota = await getCurrentInstanceQuota(ctx.user.instanceId)
+    const credentials = await verifyDeployment({
+      instanceId: ctx.user.instanceId,
+      appId: ctx.user.appId,
+      quota: instanceQuota,
+    })
+
     ctx.log.info(`Uploading assets for appID ${ctx.user.appId} assets to s3..`)
-    const credentials = await fetchTemporaryCredentials()
+
+    if (credentials.errors) {
+      ctx.throw(500, credentials.errors)
+      return
+    }
 
     await uploadAppAssets({
       clientId,
@@ -54,6 +95,8 @@ exports.deployApp = async function(ctx) {
       credentials: credentials.couchDbCreds,
     })
 
+    await updateDeploymentQuota(credentials.quota)
+
     ctx.body = {
       status: "SUCCESS",
       completed: Date.now(),

packages/server/src/api/controllers/model.js

@@ -33,17 +33,9 @@ exports.save = async function(ctx) {
     views: {},
     ...rest,
   }
-  // get the model in its previous state for differencing
+
-  let oldModel
+  // if the model obj had an _id then it will have been retrieved
-  let oldModelId = ctx.request.body._id
+  const oldModel = ctx.preExisting
-  if (oldModelId) {
-    // if it errors then the oldModelId is invalid - can't diff it
-    try {
-      oldModel = await db.get(oldModelId)
-    } catch (err) {
-      oldModel = null
-    }
-  }
 
   // rename record fields when table column is renamed
   const { _rename } = modelToSave

packages/server/src/api/controllers/record.js

@@ -12,16 +12,16 @@ const { cloneDeep } = require("lodash")
 const MODEL_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.MODEL}${SEPARATOR}`
 
 validateJs.extend(validateJs.validators.datetime, {
-  parse: function (value) {
+  parse: function(value) {
     return new Date(value).getTime()
   },
   // Input is a unix timestamp
-  format: function (value) {
+  format: function(value) {
     return new Date(value).toISOString()
   },
 })
 
-exports.patch = async function (ctx) {
+exports.patch = async function(ctx) {
   const instanceId = ctx.user.instanceId
   const db = new CouchDB(instanceId)
   let record = await db.get(ctx.params.id)
@@ -67,15 +67,15 @@ exports.patch = async function (ctx) {
   ctx.message = `${model.name} updated successfully.`
 }
 
-exports.save = async function (ctx) {
+exports.save = async function(ctx) {
   if (ctx.request.body.type === 'delete') {
     await bulkDelete(ctx)
   } else {
-    await saveRecords(ctx)
+    await saveRecord(ctx)
   }
 }
 
-exports.fetchView = async function (ctx) {
+exports.fetchView = async function(ctx) {
   const instanceId = ctx.user.instanceId
   const db = new CouchDB(instanceId)
   const { stats, group, field } = ctx.query
@@ -107,7 +107,7 @@ exports.fetchView = async function (ctx) {
   ctx.body = await linkRecords.attachLinkInfo(instanceId, response.rows)
 }
 
-exports.fetchModelRecords = async function (ctx) {
+exports.fetchModelRecords = async function(ctx) {
   const instanceId = ctx.user.instanceId
   const db = new CouchDB(instanceId)
   const response = await db.allDocs(
@@ -122,7 +122,7 @@ exports.fetchModelRecords = async function (ctx) {
   )
 }
 
-exports.search = async function (ctx) {
+exports.search = async function(ctx) {
   const instanceId = ctx.user.instanceId
   const db = new CouchDB(instanceId)
   const response = await db.allDocs({
@@ -135,7 +135,7 @@ exports.search = async function (ctx) {
   )
 }
 
-exports.find = async function (ctx) {
+exports.find = async function(ctx) {
   const instanceId = ctx.user.instanceId
   const db = new CouchDB(instanceId)
   const record = await db.get(ctx.params.recordId)
@@ -146,7 +146,7 @@ exports.find = async function (ctx) {
   ctx.body = await linkRecords.attachLinkInfo(instanceId, record)
 }
 
-exports.destroy = async function (ctx) {
+exports.destroy = async function(ctx) {
   const instanceId = ctx.user.instanceId
   const db = new CouchDB(instanceId)
   const record = await db.get(ctx.params.recordId)
@@ -169,7 +169,7 @@ exports.destroy = async function (ctx) {
     ctx.eventEmitter.emitRecord(`record:delete`, instanceId, record)
 }
 
-exports.validate = async function (ctx) {
+exports.validate = async function(ctx) {
   const errors = await validate({
     instanceId: ctx.user.instanceId,
     modelId: ctx.params.modelId,
@@ -195,7 +195,7 @@ async function validate({ instanceId, modelId, record, model }) {
   return { valid: Object.keys(errors).length === 0, errors }
 }
 
-exports.fetchEnrichedRecord = async function (ctx) {
+exports.fetchEnrichedRecord = async function(ctx) {
   const instanceId = ctx.user.instanceId
   const db = new CouchDB(instanceId)
   const modelId = ctx.params.modelId
@@ -256,83 +256,6 @@ function coerceRecordValues(rec, model) {
   return record
 }
 
-
-async function bulkDelete(ctx) {
-  const { records } = ctx.request.body
-  const db = new CouchDB(ctx.user.instanceId)
-
-  await db.bulkDocs(
-    records.map(record => ({ ...record, _deleted: true }), (err, res) => {
-      if (err) {
-        ctx.status = 500
-      } else {
-        records.forEach(record => {
-          emitEvent(`record:delete`, ctx, record)
-        })
-        ctx.status = 200
-      }
-    }))
-}
-async function saveRecords(ctx) {
-  const instanceId = ctx.user.instanceId
-  const db = new CouchDB(instanceId)
-  let record = ctx.request.body
-  record.modelId = ctx.params.modelId
-
-  if (!record._rev && !record._id) {
-    record._id = generateRecordID(record.modelId)
-  }
-
-  const model = await db.get(record.modelId)
-
-  record = coerceRecordValues(record, model)
-
-  const validateResult = await validate({
-    record,
-    model,
-  })
-
-  if (!validateResult.valid) {
-    ctx.status = 400
-    ctx.body = {
-      status: 400,
-      errors: validateResult.errors,
-    }
-    return
-  }
-
-  const existingRecord = record._rev && (await db.get(record._id))
-
-  // make sure link records are up to date
-  record = await linkRecords.updateLinks({
-    instanceId,
-    eventType: linkRecords.EventType.RECORD_SAVE,
-    record,
-    modelId: record.modelId,
-    model,
-  })
-
-  if (existingRecord) {
-    const response = await db.put(record)
-    record._rev = response.rev
-    record.type = "record"
-    ctx.body = record
-    ctx.status = 200
-    ctx.message = `${model.name} updated successfully.`
-    return
-  }
-
-  record.type = "record"
-  const response = await db.post(record)
-  record._rev = response.rev
-
-  ctx.eventEmitter &&
-    ctx.eventEmitter.emitRecord(`record:save`, instanceId, record, model)
-  ctx.body = record
-  ctx.status = 200
-  ctx.message = `${model.name} created successfully`
-}
-
 const TYPE_TRANSFORM_MAP = {
   link: {
     "": [],
@@ -373,3 +296,81 @@ const TYPE_TRANSFORM_MAP = {
     false: false,
   },
 }
+
+async function bulkDelete(ctx) {
+  const { records } = ctx.request.body
+  const db = new CouchDB(ctx.user.instanceId)
+
+  await db.bulkDocs(
+    records.map(record => ({ ...record, _deleted: true }), (err, res) => {
+      if (err) {
+        ctx.status = 500
+      } else {
+        records.forEach(record => {
+          emitEvent(`record:delete`, ctx, record)
+        })
+        ctx.status = 200
+      }
+    }))
+}
+
+async function saveRecord(ctx) {
+  const instanceId = ctx.user.instanceId
+  const db = new CouchDB(instanceId)
+  let record = ctx.request.body
+  record.modelId = ctx.params.modelId
+
+  if (!record._rev && !record._id) {
+    record._id = generateRecordID(record.modelId)
+  }
+
+  // if the record obj had an _id then it will have been retrieved
+  const existingRecord = ctx.preExisting
+
+  const model = await db.get(record.modelId)
+
+  record = coerceRecordValues(record, model)
+
+  const validateResult = await validate({
+    record,
+    model,
+  })
+
+  if (!validateResult.valid) {
+    ctx.status = 400
+    ctx.body = {
+      status: 400,
+      errors: validateResult.errors,
+    }
+    return
+  }
+
+  // make sure link records are up to date
+  record = await linkRecords.updateLinks({
+    instanceId,
+    eventType: linkRecords.EventType.RECORD_SAVE,
+    record,
+    modelId: record.modelId,
+    model,
+  })
+
+  if (existingRecord) {
+    const response = await db.put(record)
+    record._rev = response.rev
+    record.type = "record"
+    ctx.body = record
+    ctx.status = 200
+    ctx.message = `${model.name} updated successfully.`
+    return
+  }
+
+  record.type = "record"
+  const response = await db.post(record)
+  record._rev = response.rev
+
+  ctx.eventEmitter &&
+    ctx.eventEmitter.emitRecord(`record:save`, instanceId, record, model)
+  ctx.body = record
+  ctx.status = 200
+  ctx.message = `${model.name} created successfully`
+}

packages/server/src/api/controllers/static.js

@@ -153,7 +153,7 @@ exports.serveApp = async function(ctx) {
 
   // only set the appId cookie for /appId .. we COULD check for valid appIds
   // but would like to avoid that DB hit
-  const looksLikeAppId = /^[0-9a-f]{32}$/.test(appId)
+  const looksLikeAppId = /^(app_)?[0-9a-f]{32}$/.test(appId)
   if (looksLikeAppId && !ctx.isAuthenticated) {
     const anonUser = {
       userId: "ANON",

packages/server/src/api/routes/record.js

@@ -1,6 +1,7 @@
 const Router = require("@koa/router")
 const recordController = require("../controllers/record")
 const authorized = require("../../middleware/authorized")
+const usage = require("../../middleware/usageQuota")
 const { READ_MODEL, WRITE_MODEL } = require("../../utilities/accessLevels")
 
 const router = Router()
@@ -25,6 +26,7 @@ router
   .post(
     "/api/:modelId/records",
     authorized(WRITE_MODEL, ctx => ctx.params.modelId),
+    usage,
     recordController.save
   )
   .patch(
@@ -40,6 +42,7 @@ router
   .delete(
     "/api/:modelId/records/:recordId/:revId",
     authorized(WRITE_MODEL, ctx => ctx.params.modelId),
+    usage,
     recordController.destroy
   )
 

packages/server/src/api/routes/static.js

@@ -4,6 +4,7 @@ const { budibaseTempDir } = require("../../utilities/budibaseDir")
 const env = require("../../environment")
 const authorized = require("../../middleware/authorized")
 const { BUILDER } = require("../../utilities/accessLevels")
+const usage = require("../../middleware/usageQuota")
 
 const router = Router()
 
@@ -28,7 +29,7 @@ router
     authorized(BUILDER),
     controller.performLocalFileProcessing
   )
-  .post("/api/attachments/upload", controller.uploadFile)
+  .post("/api/attachments/upload", usage, controller.uploadFile)
   .get("/componentlibrary", controller.serveComponentLibrary)
   .get("/assets/:file*", controller.serveAppAsset)
   .get("/attachments/:file*", controller.serveAttachment)

packages/server/src/api/routes/tests/couchTestUtils.js

@@ -40,6 +40,9 @@ exports.defaultHeaders = (appId, instanceId) => {
 }
 
 exports.createModel = async (request, appId, instanceId, model) => {
+  if (model != null && model._id) {
+    delete model._id
+  }
   model = model || {
     name: "TestModel",
     type: "model",

packages/server/src/api/routes/user.js

@@ -2,6 +2,7 @@ const Router = require("@koa/router")
 const controller = require("../controllers/user")
 const authorized = require("../../middleware/authorized")
 const { USER_MANAGEMENT, LIST_USERS } = require("../../utilities/accessLevels")
+const usage = require("../../middleware/usageQuota")
 
 const router = Router()
 
@@ -9,10 +10,11 @@ router
   .get("/api/users", authorized(LIST_USERS), controller.fetch)
   .get("/api/users/:username", authorized(USER_MANAGEMENT), controller.find)
   .put("/api/users/", authorized(USER_MANAGEMENT), controller.update)
-  .post("/api/users", authorized(USER_MANAGEMENT), controller.create)
+  .post("/api/users", authorized(USER_MANAGEMENT), usage, controller.create)
   .delete(
     "/api/users/:username",
     authorized(USER_MANAGEMENT),
+    usage,
     controller.destroy
   )
 

packages/server/src/api/routes/view.js

@@ -3,6 +3,7 @@ const viewController = require("../controllers/view")
 const recordController = require("../controllers/record")
 const authorized = require("../../middleware/authorized")
 const { BUILDER, READ_VIEW } = require("../../utilities/accessLevels")
+const usage = require("../../middleware/usageQuota")
 
 const router = Router()
 
@@ -13,8 +14,13 @@ router
     recordController.fetchView
   )
   .get("/api/views", authorized(BUILDER), viewController.fetch)
-  .delete("/api/views/:viewName", authorized(BUILDER), viewController.destroy)
+  .delete(
-  .post("/api/views", authorized(BUILDER), viewController.save)
+    "/api/views/:viewName",
+    authorized(BUILDER),
+    usage,
+    viewController.destroy
+  )
+  .post("/api/views", authorized(BUILDER), usage, viewController.save)
   .post("/api/views/export", authorized(BUILDER), viewController.exportView)
   .get(
     "/api/views/export/download/:fileName",

packages/server/src/automations/index.js

@@ -3,6 +3,7 @@ const actions = require("./actions")
 const environment = require("../environment")
 const workerFarm = require("worker-farm")
 const singleThread = require("./thread")
+const { getAPIKey, update, Properties } = require("../utilities/usageQuota")
 
 let workers = workerFarm(require.resolve("./thread"))
 
@@ -18,16 +19,33 @@ function runWorker(job) {
   })
 }
 
+async function updateQuota(automation) {
+  const appId = automation.appId
+  const apiObj = await getAPIKey(appId)
+  // this will fail, causing automation to escape if limits reached
+  await update(apiObj.apiKey, Properties.AUTOMATION, 1)
+  return apiObj.apiKey
+}
+
 /**
  * This module is built purely to kick off the worker farm and manage the inputs/outputs
  */
 module.exports.init = function() {
   actions.init().then(() => {
     triggers.automationQueue.process(async job => {
-      if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
+      try {
-        await runWorker(job)
+        if (environment.CLOUD && job.data.automation) {
-      } else {
+          job.data.automation.apiKey = await updateQuota(job.data.automation)
-        await singleThread(job)
+        }
+        if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
+          await runWorker(job)
+        } else {
+          await singleThread(job)
+        }
+      } catch (err) {
+        console.error(
+          `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}`
+        )
       }
     })
   })

packages/server/src/automations/steps/createRecord.js

@@ -1,5 +1,7 @@
 const recordController = require("../../api/controllers/record")
 const automationUtils = require("../automationUtils")
+const environment = require("../../environment")
+const usage = require("../../utilities/usageQuota")
 
 module.exports.definition = {
   name: "Create Row",
@@ -56,7 +58,7 @@ module.exports.definition = {
   },
 }
 
-module.exports.run = async function({ inputs, instanceId }) {
+module.exports.run = async function({ inputs, instanceId, apiKey }) {
   // TODO: better logging of when actions are missed due to missing parameters
   if (inputs.record == null || inputs.record.modelId == null) {
     return
@@ -78,6 +80,9 @@ module.exports.run = async function({ inputs, instanceId }) {
   }
 
   try {
+    if (environment.CLOUD) {
+      await usage.update(apiKey, usage.Properties.RECORD, 1)
+    }
     await recordController.save(ctx)
     return {
       record: inputs.record,

packages/server/src/automations/steps/createUser.js

@@ -1,5 +1,7 @@
 const accessLevels = require("../../utilities/accessLevels")
 const userController = require("../../api/controllers/user")
+const environment = require("../../environment")
+const usage = require("../../utilities/usageQuota")
 
 module.exports.definition = {
   description: "Create a new user",
@@ -56,7 +58,7 @@ module.exports.definition = {
   },
 }
 
-module.exports.run = async function({ inputs, instanceId }) {
+module.exports.run = async function({ inputs, instanceId, apiKey }) {
   const { username, password, accessLevelId } = inputs
   const ctx = {
     user: {
@@ -68,6 +70,9 @@ module.exports.run = async function({ inputs, instanceId }) {
   }
 
   try {
+    if (environment.CLOUD) {
+      await usage.update(apiKey, usage.Properties.USER, 1)
+    }
     await userController.create(ctx)
     return {
       response: ctx.body,

packages/server/src/automations/steps/deleteRecord.js

@@ -1,4 +1,6 @@
 const recordController = require("../../api/controllers/record")
+const environment = require("../../environment")
+const usage = require("../../utilities/usageQuota")
 
 module.exports.definition = {
   description: "Delete a row from your database",
@@ -48,7 +50,7 @@ module.exports.definition = {
   },
 }
 
-module.exports.run = async function({ inputs, instanceId }) {
+module.exports.run = async function({ inputs, instanceId, apiKey }) {
   // TODO: better logging of when actions are missed due to missing parameters
   if (inputs.id == null || inputs.revision == null) {
     return
@@ -63,6 +65,9 @@ module.exports.run = async function({ inputs, instanceId }) {
   }
 
   try {
+    if (environment.CLOUD) {
+      await usage.update(apiKey, usage.Properties.RECORD, -1)
+    }
     await recordController.destroy(ctx)
     return {
       response: ctx.body,

packages/server/src/automations/thread.js

@@ -62,6 +62,7 @@ class Orchestrator {
         const outputs = await stepFn({
           inputs: step.inputs,
           instanceId: this._instanceId,
+          apiKey: automation.apiKey,
         })
         if (step.stepId === FILTER_STEP_ID && !outputs.success) {
           break

packages/server/src/db/dynamoClient.js

@@ -0,0 +1,128 @@
+let _ = require("lodash")
+let environment = require("../environment")
+
+const AWS_REGION = environment.AWS_REGION ? environment.AWS_REGION : "eu-west-1"
+
+const TableInfo = {
+  API_KEYS: {
+    name: "beta-api-key-table",
+    primary: "pk",
+  },
+  USERS: {
+    name: "prod-budi-table",
+    primary: "pk",
+    sort: "sk",
+  },
+}
+
+let docClient = null
+
+class Table {
+  constructor(tableInfo) {
+    if (!tableInfo.name || !tableInfo.primary) {
+      throw "Table info must specify a name and a primary key"
+    }
+    this._name = tableInfo.name
+    this._primary = tableInfo.primary
+    this._sort = tableInfo.sort
+  }
+
+  async get({ primary, sort, otherProps }) {
+    let params = {
+      TableName: this._name,
+      Key: {
+        [this._primary]: primary,
+      },
+    }
+    if (this._sort && sort) {
+      params.Key[this._sort] = sort
+    }
+    if (otherProps) {
+      params = _.merge(params, otherProps)
+    }
+    let response = await docClient.get(params).promise()
+    return response.Item
+  }
+
+  async update({
+    primary,
+    sort,
+    expression,
+    condition,
+    names,
+    values,
+    exists,
+    otherProps,
+  }) {
+    let params = {
+      TableName: this._name,
+      Key: {
+        [this._primary]: primary,
+      },
+      ExpressionAttributeNames: names,
+      ExpressionAttributeValues: values,
+      UpdateExpression: expression,
+    }
+    if (condition) {
+      params.ConditionExpression = condition
+    }
+    if (this._sort && sort) {
+      params.Key[this._sort] = sort
+    }
+    if (exists) {
+      params.ExpressionAttributeNames["#PRIMARY"] = this._primary
+      if (params.ConditionExpression) {
+        params.ConditionExpression += " AND "
+      }
+      params.ConditionExpression += "attribute_exists(#PRIMARY)"
+    }
+    if (otherProps) {
+      params = _.merge(params, otherProps)
+    }
+    return docClient.update(params).promise()
+  }
+
+  async put({ item, otherProps }) {
+    if (
+      item[this._primary] == null ||
+      (this._sort && item[this._sort] == null)
+    ) {
+      throw "Cannot put item without primary and sort key (if required)"
+    }
+    let params = {
+      TableName: this._name,
+      Item: item,
+    }
+    if (otherProps) {
+      params = _.merge(params, otherProps)
+    }
+    return docClient.put(params).promise()
+  }
+}
+
+exports.init = endpoint => {
+  let AWS = require("aws-sdk")
+  AWS.config.update({
+    region: AWS_REGION,
+  })
+  let docClientParams = {
+    correctClockSkew: true,
+  }
+  if (endpoint) {
+    docClientParams.endpoint = endpoint
+  } else if (environment.DYNAMO_ENDPOINT) {
+    docClientParams.endpoint = environment.DYNAMO_ENDPOINT
+  }
+  docClient = new AWS.DynamoDB.DocumentClient(docClientParams)
+}
+
+exports.apiKeyTable = new Table(TableInfo.API_KEYS)
+exports.userTable = new Table(TableInfo.USERS)
+
+if (environment.CLOUD) {
+  exports.init(`https://dynamodb.${AWS_REGION}.amazonaws.com`)
+} else {
+  process.env.AWS_ACCESS_KEY_ID = "KEY_ID"
+  process.env.AWS_SECRET_ACCESS_KEY = "SECRET_KEY"
+  exports.init("http://localhost:8333")
+}

packages/server/src/db/linkedRecords/index.js

@@ -1,5 +1,6 @@
 const LinkController = require("./LinkController")
 const { IncludeDocs, getLinkDocuments, createLinkView } = require("./linkUtils")
+const _ = require("lodash")
 
 /**
  * This functionality makes sure that when records with links are created, updated or deleted they are processed
@@ -88,23 +89,23 @@ exports.attachLinkInfo = async (instanceId, records) => {
     records = [records]
     wasArray = false
   }
+  let modelIds = [...new Set(records.map(el => el.modelId))]
   // start by getting all the link values for performance reasons
-  let responses = await Promise.all(
+  let responses = _.flatten(
-    records.map(record =>
+    await Promise.all(
-      getLinkDocuments({
+      modelIds.map(modelId =>
-        instanceId,
+        getLinkDocuments({
-        modelId: record.modelId,
+          instanceId,
-        recordId: record._id,
+          modelId: modelId,
-        includeDocs: IncludeDocs.EXCLUDE,
+          includeDocs: IncludeDocs.EXCLUDE,
-      })
+        })
+      )
     )
   )
-  // can just use an index to access responses, order maintained
-  let index = 0
   // now iterate through the records and all field information
   for (let record of records) {
     // get all links for record, ignore fieldName for now
-    const linkVals = responses[index++]
+    const linkVals = responses.filter(el => el.thisId === record._id)
     for (let linkVal of linkVals) {
       // work out which link pertains to this record
       if (!(record[linkVal.fieldName] instanceof Array)) {

packages/server/src/db/linkedRecords/linkUtils.js

@@ -27,10 +27,12 @@ exports.createLinkView = async instanceId => {
         let doc2 = doc.doc2
         emit([doc1.modelId, doc1.recordId], {
           id: doc2.recordId,
+          thisId: doc1.recordId,
           fieldName: doc1.fieldName,
         })
         emit([doc2.modelId, doc2.recordId], {
           id: doc1.recordId,
+          thisId: doc2.recordId,
           fieldName: doc2.fieldName,
         })
       }

packages/server/src/db/utils.js

@@ -15,6 +15,7 @@ const DocumentTypes = {
 
 exports.DocumentTypes = DocumentTypes
 exports.SEPARATOR = SEPARATOR
+exports.UNICODE_MAX = UNICODE_MAX
 
 /**
  * If creating DB allDocs/query params with only a single top level ID this can be used, this

packages/server/src/environment.js

@@ -11,4 +11,8 @@ module.exports = {
   AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET,
   BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
+  CLOUD: process.env.CLOUD,
+  DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
+  AWS_REGION: process.env.AWS_REGION,
+  DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
 }

packages/server/src/middleware/authenticated.js

@@ -20,6 +20,7 @@ module.exports = async (ctx, next) => {
   if (builderToken) {
     try {
       const jwtPayload = jwt.verify(builderToken, ctx.config.jwtSecret)
+      ctx.apiKey = jwtPayload.apiKey
       ctx.isAuthenticated = jwtPayload.accessLevelId === BUILDER_LEVEL_ID
       ctx.user = {
         ...jwtPayload,
@@ -44,7 +45,7 @@ module.exports = async (ctx, next) => {
 
   try {
     const jwtPayload = jwt.verify(appToken, ctx.config.jwtSecret)
-
+    ctx.apiKey = jwtPayload.apiKey
     ctx.user = {
       ...jwtPayload,
       accessLevel: await getAccessLevel(

packages/server/src/middleware/authorized.js

@@ -16,8 +16,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
   }
 
   if (ctx.user.accessLevel._id === BUILDER_LEVEL_ID) {
-    await next()
+    return next()
-    return
   }
 
   if (permName === BUILDER) {
@@ -28,8 +27,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
   const permissionId = ({ name, itemId }) => name + (itemId ? `-${itemId}` : "")
 
   if (ctx.user.accessLevel._id === ADMIN_LEVEL_ID) {
-    await next()
+    return next()
-    return
   }
 
   const thisPermissionId = permissionId({
@@ -42,8 +40,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
     ctx.user.accessLevel._id === POWERUSER_LEVEL_ID &&
     !adminPermissions.map(permissionId).includes(thisPermissionId)
   ) {
-    await next()
+    return next()
-    return
   }
 
   if (
@@ -51,8 +48,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
       .map(permissionId)
       .includes(thisPermissionId)
   ) {
-    await next()
+    return next()
-    return
   }
 
   ctx.throw(403, "Not Authorized")

packages/server/src/middleware/usageQuota.js

@@ -0,0 +1,63 @@
+const CouchDB = require("../db")
+const usageQuota = require("../utilities/usageQuota")
+const environment = require("../environment")
+
+// currently only counting new writes and deletes
+const METHOD_MAP = {
+  POST: 1,
+  DELETE: -1,
+}
+
+const DOMAIN_MAP = {
+  records: usageQuota.Properties.RECORD,
+  upload: usageQuota.Properties.UPLOAD,
+  views: usageQuota.Properties.VIEW,
+  users: usageQuota.Properties.USER,
+  // this will not be updated by endpoint calls
+  // instead it will be updated by triggers
+  automationRuns: usageQuota.Properties.AUTOMATION,
+}
+
+function getProperty(url) {
+  for (let domain of Object.keys(DOMAIN_MAP)) {
+    if (url.indexOf(domain) !== -1) {
+      return DOMAIN_MAP[domain]
+    }
+  }
+}
+
+module.exports = async (ctx, next) => {
+  const db = new CouchDB(ctx.user.instanceId)
+  let usage = METHOD_MAP[ctx.req.method]
+  const property = getProperty(ctx.req.url)
+  if (usage == null || property == null) {
+    return next()
+  }
+  // post request could be a save of a pre-existing entry
+  if (ctx.request.body && ctx.request.body._id) {
+    try {
+      ctx.preExisting = await db.get(ctx.request.body._id)
+      return next()
+    } catch (err) {
+      ctx.throw(404, `${ctx.request.body._id} does not exist`)
+      return
+    }
+  }
+  // update usage for uploads to be the total size
+  if (property === usageQuota.Properties.UPLOAD) {
+    const files =
+      ctx.request.files.file.length > 1
+        ? Array.from(ctx.request.files.file)
+        : [ctx.request.files.file]
+    usage = files.map(file => file.size).reduce((total, size) => total + size)
+  }
+  if (!environment.CLOUD) {
+    return next()
+  }
+  try {
+    await usageQuota.update(ctx.apiKey, property, usage)
+    return next()
+  } catch (err) {
+    ctx.throw(403, err)
+  }
+}

packages/server/src/utilities/builder/setBuilderToken.js

@@ -8,7 +8,9 @@ module.exports = (ctx, appId, instanceId) => {
     instanceId,
     appId,
   }
-
+  if (process.env.BUDIBASE_API_KEY) {
+    builderUser.apiKey = process.env.BUDIBASE_API_KEY
+  }
   const token = jwt.sign(builderUser, ctx.config.jwtSecret, {
     expiresIn: "30 days",
   })

packages/server/src/utilities/usageQuota.js

@@ -0,0 +1,103 @@
+const environment = require("../environment")
+const { apiKeyTable } = require("../db/dynamoClient")
+
+const DEFAULT_USAGE = {
+  records: 0,
+  storage: 0,
+  views: 0,
+  automationRuns: 0,
+  users: 0,
+}
+
+const DEFAULT_PLAN = {
+  records: 1000,
+  // 1 GB
+  storage: 8589934592,
+  views: 10,
+  automationRuns: 100,
+  users: 10000,
+}
+
+function buildUpdateParams(key, property, usage) {
+  return {
+    primary: key,
+    condition:
+      "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now",
+    expression: "ADD #quota.#prop :usage",
+    names: {
+      "#quota": "usageQuota",
+      "#prop": property,
+      "#limits": "usageLimits",
+      "#quotaReset": "quotaReset",
+    },
+    values: {
+      ":usage": usage,
+      ":now": Date.now(),
+    },
+  }
+}
+
+function getNewQuotaReset() {
+  return Date.now() + 2592000000
+}
+
+exports.Properties = {
+  RECORD: "records",
+  UPLOAD: "storage",
+  VIEW: "views",
+  USER: "users",
+  AUTOMATION: "automationRuns",
+}
+
+exports.getAPIKey = async appId => {
+  return apiKeyTable.get({ primary: appId })
+}
+
+/**
+ * Given a specified API key this will add to the usage object for the specified property.
+ * @param {string} apiKey The API key which is to be updated.
+ * @param {string} property The property which is to be added to (within the nested usageQuota object).
+ * @param {number} usage The amount (this can be negative) to adjust the number by.
+ * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
+ * also been reset after this call.
+ */
+exports.update = async (apiKey, property, usage) => {
+  // don't try validate in builder
+  if (!environment.CLOUD) {
+    return
+  }
+  try {
+    await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
+  } catch (err) {
+    // conditional check means the condition failed, need to check why
+    if (err.code === "ConditionalCheckFailedException") {
+      // get the API key so we can check it
+      const keyObj = await apiKeyTable.get({ primary: apiKey })
+      // the usage quota or usage limits didn't exist
+      if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) {
+        keyObj.usageQuota =
+          keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota
+        keyObj.usageLimits =
+          keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits
+        keyObj.quotaReset = getNewQuotaReset()
+        await apiKeyTable.put({ item: keyObj })
+        return
+      }
+      // we have infact breached the reset period
+      else if (keyObj && keyObj.quotaReset <= Date.now()) {
+        // update the quota reset period and reset the values for all properties
+        keyObj.quotaReset = getNewQuotaReset()
+        for (let prop of Object.keys(keyObj.usageQuota)) {
+          if (prop === property) {
+            keyObj.usageQuota[prop] = usage > 0 ? usage : 0
+          } else {
+            keyObj.usageQuota[prop] = 0
+          }
+        }
+        await apiKeyTable.put({ item: keyObj })
+        return
+      }
+    }
+    throw err
+  }
+}

packages/server/yarn.lock

@@ -925,9 +925,10 @@ atomic-sleep@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/atomic-sleep/-/atomic-sleep-1.0.0.tgz#eb85b77a601fc932cfe432c5acd364a9e2c9075b"
 
-aws-sdk@^2.706.0:
+aws-sdk@^2.767.0:
-  version "2.706.0"
+  version "2.767.0"
-  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.706.0.tgz#09f65e9a91ecac5a635daf934082abae30eca953"
+  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.767.0.tgz#9863c8bfd5990106b95f38e9345a547fee782470"
+  integrity sha512-soPZxjNpat0CtuIqm54GO/FDT4SZTlQG0icSptWYfMFYdkXe8b0tJqaPssNn9TzlgoWDCNTdaoepM6TN0rNHkQ==
   dependencies:
     buffer "4.9.2"
     events "1.1.1"
@@ -1060,6 +1061,7 @@ bluebird-lst@^1.0.9:
 bluebird@^3.5.1, bluebird@^3.5.5:
   version "3.7.2"
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
+  integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
 
 boolean@^3.0.0, boolean@^3.0.1:
   version "3.0.1"

packages/standard-components/src/DataTable.svelte

@@ -99,7 +99,7 @@
             {#if schema[header].type === 'attachment'}
               <AttachmentList files={row[header]} />
             {:else if schema[header].type === 'link'}
-              <td>{row[header] ? row[header].length : 0} related row(s)</td>
+              <td>{row[header]} related row(s)</td>
             {:else if row[header]}
               <td>{row[header]}</td>
             {/if}