Merge pull request #1496 from Budibase/feature/draft-apps

Feature/draft apps

packages/auth/README.md

@@ -1 +1,12 @@
-# Budibase Authentication Library
+# Budibase Core backend library
+
+This library contains core functionality, like auth and security features
+which are shared between backend services.
+
+#### Note about top level JS files
+For the purposes of being able to do say `require("@budibase/auth/permissions")` we need to
+specify the exports at the top-level of the module.
+
+For these files they should be limited to a single `require` of the file that should
+be exported and then a single `module.exports = ...` to export the file in
+commonJS.

packages/auth/db.js

@@ -0,0 +1 @@
+module.exports = require("./src/db/utils")

packages/auth/package.json

@@ -11,6 +11,7 @@
     "ioredis": "^4.27.1",
     "jsonwebtoken": "^8.5.1",
     "koa-passport": "^4.1.4",
+    "lodash": "^4.17.21",
     "node-fetch": "^2.6.1",
     "passport-google-auth": "^1.0.2",
     "passport-google-oauth": "^2.0.0",

packages/auth/permissions.js

@@ -0,0 +1 @@
+module.exports = require("./src/security/permissions")

packages/auth/redis.js

@@ -0,0 +1,4 @@
+module.exports = {
+  Client: require("./src/redis"),
+  utils: require("./src/redis/utils"),
+}

packages/auth/roles.js

@@ -0,0 +1 @@
+module.exports = require("./src/security/roles")

packages/auth/src/constants.js

@@ -14,3 +14,10 @@ exports.GlobalRoles = {
   BUILDER: "builder",
   GROUP_MANAGER: "group_manager",
 }
+
+exports.Configs = {
+  SETTINGS: "settings",
+  ACCOUNT: "account",
+  SMTP: "smtp",
+  GOOGLE: "google",
+}

packages/auth/src/db/Replication.js

@@ -0,0 +1,79 @@
+const { getDB } = require(".")
+
+class Replication {
+  /**
+   *
+   * @param {String} source - the DB you want to replicate or rollback to
+   * @param {String} target - the DB you want to replicate to, or rollback from
+   */
+  constructor({ source, target }) {
+    this.source = getDB(source)
+    this.target = getDB(target)
+  }
+
+  promisify(operation, opts = {}) {
+    return new Promise(resolve => {
+      operation(this.target, opts)
+        .on("denied", function (err) {
+          // a document failed to replicate (e.g. due to permissions)
+          throw new Error(`Denied: Document failed to replicate ${err}`)
+        })
+        .on("complete", function (info) {
+          return resolve(info)
+        })
+        .on("error", function (err) {
+          throw new Error(`Replication Error: ${err}`)
+        })
+    })
+  }
+
+  /**
+   * Two way replication operation, intended to be promise based.
+   * @param {Object} opts - PouchDB replication options
+   */
+  sync(opts = {}) {
+    this.replication = this.promisify(this.source.sync, opts)
+    return this.replication
+  }
+
+  /**
+   * One way replication operation, intended to be promise based.
+   * @param {Object} opts - PouchDB replication options
+   */
+  replicate(opts = {}) {
+    this.replication = this.promisify(this.source.replicate.to, opts)
+    return this.replication
+  }
+
+  /**
+   * Set up an ongoing live sync between 2 CouchDB databases.
+   * @param {Object} opts - PouchDB replication options
+   */
+  subscribe(opts = {}) {
+    this.replication = this.source.replicate
+      .to(this.target, {
+        live: true,
+        retry: true,
+        ...opts,
+      })
+      .on("error", function (err) {
+        throw new Error(`Replication Error: ${err}`)
+      })
+  }
+
+  /**
+   * Rollback the target DB back to the state of the source DB
+   */
+  async rollback() {
+    await this.target.destroy()
+    // Recreate the DB again
+    this.target = getDB(this.target.name)
+    await this.replicate()
+  }
+
+  cancel() {
+    this.replication.cancel()
+  }
+}
+
+module.exports = Replication

packages/auth/src/db/index.js

@@ -7,3 +7,7 @@ module.exports.setDB = pouch => {
 module.exports.getDB = dbName => {
   return new Pouch(dbName)
 }
+
+module.exports.getCouch = () => {
+  return Pouch
+}

packages/auth/src/db/utils.js

@@ -1,4 +1,9 @@
 const { newid } = require("../hashing")
+const Replication = require("./Replication")
+const { getCouch } = require("./index")
+
+const UNICODE_MAX = "\ufff0"
+const SEPARATOR = "_"
 
 exports.ViewNames = {
   USER_BY_EMAIL: "by_email",
@@ -8,23 +13,50 @@ exports.StaticDatabases = {
   GLOBAL: {
     name: "global-db",
   },
+  DEPLOYMENTS: {
+    name: "deployments",
+  },
 }
 
 const DocumentTypes = {
   USER: "us",
-  APP: "app",
   GROUP: "group",
   CONFIG: "config",
   TEMPLATE: "template",
+  APP: "app",
+  APP_DEV: "app_dev",
+  APP_METADATA: "app_metadata",
+  ROLE: "role",
 }
 
 exports.DocumentTypes = DocumentTypes
-
+exports.APP_PREFIX = DocumentTypes.APP + SEPARATOR
-const UNICODE_MAX = "\ufff0"
+exports.APP_DEV_PREFIX = DocumentTypes.APP_DEV + SEPARATOR
-const SEPARATOR = "_"
-
 exports.SEPARATOR = SEPARATOR
 
+/**
+ * If creating DB allDocs/query params with only a single top level ID this can be used, this
+ * is usually the case as most of our docs are top level e.g. tables, automations, users and so on.
+ * More complex cases such as link docs and rows which have multiple levels of IDs that their
+ * ID consists of need their own functions to build the allDocs parameters.
+ * @param {string} docType The type of document which input params are being built for, e.g. user,
+ * link, app, table and so on.
+ * @param {string|null} docId The ID of the document minus its type - this is only needed if looking
+ * for a singular document.
+ * @param {object} otherProps Add any other properties onto the request, e.g. include_docs.
+ * @returns {object} Parameters which can then be used with an allDocs request.
+ */
+function getDocParams(docType, docId = null, otherProps = {}) {
+  if (docId == null) {
+    docId = ""
+  }
+  return {
+    ...otherProps,
+    startkey: `${docType}${SEPARATOR}${docId}`,
+    endkey: `${docType}${SEPARATOR}${docId}${UNICODE_MAX}`,
+  }
+}
+
 /**
  * Generates a new group ID.
  * @returns {string} The new group ID which the group doc can be stored under.
@@ -94,6 +126,65 @@ exports.getTemplateParams = (ownerId, templateId, otherProps = {}) => {
   }
 }
 
+/**
+ * Generates a new role ID.
+ * @returns {string} The new role ID which the role doc can be stored under.
+ */
+exports.generateRoleID = id => {
+  return `${DocumentTypes.ROLE}${SEPARATOR}${id || newid()}`
+}
+
+/**
+ * Gets parameters for retrieving a role, this is a utility function for the getDocParams function.
+ */
+exports.getRoleParams = (roleId = null, otherProps = {}) => {
+  return getDocParams(DocumentTypes.ROLE, roleId, otherProps)
+}
+
+/**
+ * Convert a development app ID to a deployed app ID.
+ */
+exports.getDeployedAppID = appId => {
+  // if dev, convert it
+  if (appId.startsWith(exports.APP_DEV_PREFIX)) {
+    const id = appId.split(exports.APP_DEV_PREFIX)[1]
+    return `${exports.APP_PREFIX}${id}`
+  }
+  return appId
+}
+
+/**
+ * Lots of different points in the system need to find the full list of apps, this will
+ * enumerate the entire CouchDB cluster and get the list of databases (every app).
+ * NOTE: this operation is fine in self hosting, but cannot be used when hosting many
+ * different users/companies apps as there is no security around it - all apps are returned.
+ * @return {Promise<object[]>} returns the app information document stored in each app database.
+ */
+exports.getAllApps = async (devApps = false) => {
+  const CouchDB = getCouch()
+  let allDbs = await CouchDB.allDbs()
+  const appDbNames = allDbs.filter(dbName =>
+    dbName.startsWith(exports.APP_PREFIX)
+  )
+  const appPromises = appDbNames.map(db =>
+    new CouchDB(db).get(DocumentTypes.APP_METADATA)
+  )
+  if (appPromises.length === 0) {
+    return []
+  } else {
+    const response = await Promise.allSettled(appPromises)
+    const apps = response
+      .filter(result => result.status === "fulfilled")
+      .map(({ value }) => value)
+    return apps.filter(app => {
+      if (devApps) {
+        return app.appId.startsWith(exports.APP_DEV_PREFIX)
+      }
+      return !app.appId.startsWith(exports.APP_DEV_PREFIX)
+    })
+  }
+}
+
 /**
  * Generates a new configuration ID.
  * @returns {string} The new configuration ID which the config doc can be stored under.
@@ -165,6 +256,7 @@ async function getScopedConfig(db, params) {
   return configDoc && configDoc.config ? configDoc.config : configDoc
 }
 
+exports.Replication = Replication
 exports.getScopedConfig = getScopedConfig
 exports.generateConfigID = generateConfigID
 exports.getConfigParams = getConfigParams

packages/auth/src/objectStore/index.js

@@ -10,6 +10,7 @@ const fs = require("fs")
 const env = require("../environment")
 const { budibaseTempDir, ObjectStoreBuckets } = require("./utils")
 const { v4 } = require("uuid")
+const { APP_PREFIX, APP_DEV_PREFIX } = require("../db/utils")
 
 const streamPipeline = promisify(stream.pipeline)
 // use this as a temporary store of buckets that are being created
@@ -28,6 +29,16 @@ const STRING_CONTENT_TYPES = [
   CONTENT_TYPE_MAP.js,
 ]
 
+// does normal sanitization and then swaps dev apps to apps
+function sanitizeKey(input) {
+  return sanitize(sanitizeBucket(input)).replace(/\\/g, "/")
+}
+
+// simply handles the dev app to app conversion
+function sanitizeBucket(input) {
+  return input.replace(new RegExp(APP_DEV_PREFIX, "g"), APP_PREFIX)
+}
+
 function publicPolicy(bucketName) {
   return {
     Version: "2012-10-17",
@@ -61,7 +72,7 @@ exports.ObjectStore = bucket => {
     s3ForcePathStyle: true,
     signatureVersion: "v4",
     params: {
-      Bucket: bucket,
+      Bucket: sanitizeBucket(bucket),
     },
   }
   if (env.MINIO_URL) {
@@ -75,6 +86,7 @@ exports.ObjectStore = bucket => {
  * if it does not exist then it will create it.
  */
 exports.makeSureBucketExists = async (client, bucketName) => {
+  bucketName = sanitizeBucket(bucketName)
   try {
     await client
       .headBucket({
@@ -114,16 +126,22 @@ exports.makeSureBucketExists = async (client, bucketName) => {
  * Uploads the contents of a file given the required parameters, useful when
  * temp files in use (for example file uploaded as an attachment).
  */
-exports.upload = async ({ bucket, filename, path, type, metadata }) => {
+exports.upload = async ({
+  bucket: bucketName,
+  filename,
+  path,
+  type,
+  metadata,
+}) => {
   const extension = [...filename.split(".")].pop()
   const fileBytes = fs.readFileSync(path)
 
-  const objectStore = exports.ObjectStore(bucket)
+  const objectStore = exports.ObjectStore(bucketName)
-  await exports.makeSureBucketExists(objectStore, bucket)
+  await exports.makeSureBucketExists(objectStore, bucketName)
 
   const config = {
     // windows file paths need to be converted to forward slashes for s3
-    Key: sanitize(filename).replace(/\\/g, "/"),
+    Key: sanitizeKey(filename),
     Body: fileBytes,
     ContentType: type || CONTENT_TYPE_MAP[extension.toLowerCase()],
   }
@@ -137,13 +155,13 @@ exports.upload = async ({ bucket, filename, path, type, metadata }) => {
  * Similar to the upload function but can be used to send a file stream
  * through to the object store.
  */
-exports.streamUpload = async (bucket, filename, stream) => {
+exports.streamUpload = async (bucketName, filename, stream) => {
-  const objectStore = exports.ObjectStore(bucket)
+  const objectStore = exports.ObjectStore(bucketName)
-  await exports.makeSureBucketExists(objectStore, bucket)
+  await exports.makeSureBucketExists(objectStore, bucketName)
 
   const params = {
-    Bucket: bucket,
+    Bucket: sanitizeBucket(bucketName),
-    Key: sanitize(filename).replace(/\\/g, "/"),
+    Key: sanitizeKey(filename),
     Body: stream,
   }
   return objectStore.upload(params).promise()
@@ -153,11 +171,11 @@ exports.streamUpload = async (bucket, filename, stream) => {
  * retrieves the contents of a file from the object store, if it is a known content type it
  * will be converted, otherwise it will be returned as a buffer stream.
  */
-exports.retrieve = async (bucket, filepath) => {
+exports.retrieve = async (bucketName, filepath) => {
-  const objectStore = exports.ObjectStore(bucket)
+  const objectStore = exports.ObjectStore(bucketName)
   const params = {
-    Bucket: bucket,
+    Bucket: sanitizeBucket(bucketName),
-    Key: sanitize(filepath).replace(/\\/g, "/"),
+    Key: sanitizeKey(filepath),
   }
   const response = await objectStore.getObject(params).promise()
   // currently these are all strings
@@ -171,17 +189,21 @@ exports.retrieve = async (bucket, filepath) => {
 /**
  * Same as retrieval function but puts to a temporary file.
  */
-exports.retrieveToTmp = async (bucket, filepath) => {
+exports.retrieveToTmp = async (bucketName, filepath) => {
-  const data = await exports.retrieve(bucket, filepath)
+  bucketName = sanitizeBucket(bucketName)
+  filepath = sanitizeKey(filepath)
+  const data = await exports.retrieve(bucketName, filepath)
   const outputPath = join(budibaseTempDir(), v4())
   fs.writeFileSync(outputPath, data)
   return outputPath
 }
 
-exports.deleteFolder = async (bucket, folder) => {
+exports.deleteFolder = async (bucketName, folder) => {
-  const client = exports.ObjectStore(bucket)
+  bucketName = sanitizeBucket(bucketName)
+  folder = sanitizeKey(folder)
+  const client = exports.ObjectStore(bucketName)
   const listParams = {
-    Bucket: bucket,
+    Bucket: bucketName,
     Prefix: folder,
   }
 
@@ -190,7 +212,7 @@ exports.deleteFolder = async (bucket, folder) => {
     return
   }
   const deleteParams = {
-    Bucket: bucket,
+    Bucket: bucketName,
     Delete: {
       Objects: [],
     },
@@ -203,28 +225,31 @@ exports.deleteFolder = async (bucket, folder) => {
   response = await client.deleteObjects(deleteParams).promise()
   // can only empty 1000 items at once
   if (response.Deleted.length === 1000) {
-    return exports.deleteFolder(bucket, folder)
+    return exports.deleteFolder(bucketName, folder)
   }
 }
 
-exports.uploadDirectory = async (bucket, localPath, bucketPath) => {
+exports.uploadDirectory = async (bucketName, localPath, bucketPath) => {
+  bucketName = sanitizeBucket(bucketName)
   let uploads = []
   const files = fs.readdirSync(localPath, { withFileTypes: true })
   for (let file of files) {
-    const path = join(bucketPath, file.name)
+    const path = sanitizeKey(join(bucketPath, file.name))
     const local = join(localPath, file.name)
     if (file.isDirectory()) {
-      uploads.push(exports.uploadDirectory(bucket, local, path))
+      uploads.push(exports.uploadDirectory(bucketName, local, path))
     } else {
       uploads.push(
-        exports.streamUpload(bucket, path, fs.createReadStream(local))
+        exports.streamUpload(bucketName, path, fs.createReadStream(local))
       )
     }
   }
   await Promise.all(uploads)
 }
 
-exports.downloadTarball = async (url, bucket, path) => {
+exports.downloadTarball = async (url, bucketName, path) => {
+  bucketName = sanitizeBucket(bucketName)
+  path = sanitizeKey(path)
   const response = await fetch(url)
   if (!response.ok) {
     throw new Error(`unexpected response ${response.statusText}`)
@@ -233,7 +258,7 @@ exports.downloadTarball = async (url, bucket, path) => {
   const tmpPath = join(budibaseTempDir(), path)
   await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath))
   if (!env.isTest()) {
-    await exports.uploadDirectory(bucket, tmpPath, path)
+    await exports.uploadDirectory(bucketName, tmpPath, path)
   }
   // return the temporary path incase there is a use for it
   return tmpPath

packages/auth/src/redis/index.js

@@ -143,9 +143,8 @@ class RedisWrapper {
   }
 
   async clear() {
-    const db = this._db
+    let items = await this.scan()
-    let items = await this.scan(db)
+    await Promise.all(items.map(obj => this.delete(obj.key)))
-    await Promise.all(items.map(obj => this.delete(db, obj.key)))
   }
 }
 

packages/auth/src/redis/utils.js

@@ -9,6 +9,7 @@ const REDIS_PASSWORD = !env.REDIS_PASSWORD ? "budibase" : env.REDIS_PASSWORD
 exports.Databases = {
   PW_RESETS: "pwReset",
   INVITATIONS: "invitation",
+  DEV_LOCKS: "devLocks",
 }
 
 exports.getRedisOptions = (clustered = false) => {
@@ -31,6 +32,9 @@ exports.getRedisOptions = (clustered = false) => {
 }
 
 exports.addDbPrefix = (db, key) => {
+  if (key.includes(db)) {
+    return key
+  }
   return `${db}${SEPARATOR}${key}`
 }
 

packages/auth/src/security/roles.js

@@ -1,7 +1,12 @@
-const CouchDB = require("../../db")
+const { getDB } = require("../db")
 const { cloneDeep } = require("lodash/fp")
 const { BUILTIN_PERMISSION_IDS, higherPermission } = require("./permissions")
-const { generateRoleID, DocumentTypes, SEPARATOR } = require("../../db/utils")
+const {
+  generateRoleID,
+  getRoleParams,
+  DocumentTypes,
+  SEPARATOR,
+} = require("../db/utils")
 
 const BUILTIN_IDS = {
   ADMIN: "ADMIN",
@@ -11,6 +16,14 @@ const BUILTIN_IDS = {
   BUILDER: "BUILDER",
 }
 
+// exclude internal roles like builder
+const EXTERNAL_BUILTIN_ROLE_IDS = [
+  BUILTIN_IDS.ADMIN,
+  BUILTIN_IDS.POWER,
+  BUILTIN_IDS.BASIC,
+  BUILTIN_IDS.PUBLIC,
+]
+
 function Role(id, name) {
   this._id = id
   this.name = name
@@ -116,7 +129,7 @@ exports.getRole = async (appId, roleId) => {
     )
   }
   try {
-    const db = new CouchDB(appId)
+    const db = getDB(appId)
     const dbRole = await db.get(exports.getDBRoleID(roleId))
     role = Object.assign(role, dbRole)
     // finalise the ID
@@ -192,6 +205,39 @@ exports.getUserPermissions = async (appId, userRoleId) => {
   }
 }
 
+/**
+ * Given an app ID this will retrieve all of the roles that are currently within that app.
+ * @param {string} appId The ID of the app to retrieve the roles from.
+ * @return {Promise<object[]>} An array of the role objects that were found.
+ */
+exports.getAllRoles = async appId => {
+  const db = getDB(appId)
+  const body = await db.allDocs(
+    getRoleParams(null, {
+      include_docs: true,
+    })
+  )
+  let roles = body.rows.map(row => row.doc)
+  const builtinRoles = exports.getBuiltinRoles()
+
+  // need to combine builtin with any DB record of them (for sake of permissions)
+  for (let builtinRoleId of EXTERNAL_BUILTIN_ROLE_IDS) {
+    const builtinRole = builtinRoles[builtinRoleId]
+    const dbBuiltin = roles.filter(
+      dbRole => exports.getExternalRoleID(dbRole._id) === builtinRoleId
+    )[0]
+    if (dbBuiltin == null) {
+      roles.push(builtinRole)
+    } else {
+      // remove role and all back after combining with the builtin
+      roles = roles.filter(role => role._id !== dbBuiltin._id)
+      dbBuiltin._id = exports.getExternalRoleID(dbBuiltin._id)
+      roles.push(Object.assign(builtinRole, dbBuiltin))
+    }
+  }
+  return roles
+}
+
 class AccessController {
   constructor(appId) {
     this.appId = appId

packages/bbui/src/Link/Link.svelte

@@ -11,6 +11,7 @@
 </script>
 
 <a
+  on:click
   {href}
   {target}
   class:spectrum-Link--primary={primary}

packages/builder/src/builderStore/store/frontend.js

@@ -51,14 +51,14 @@ export const getFrontendStore = () => {
   store.actions = {
     initialise: async pkg => {
       const { layouts, screens, application, clientLibPath } = pkg
-      const components = await fetchComponentLibDefinitions(application._id)
+      const components = await fetchComponentLibDefinitions(application.appId)
       store.update(state => ({
         ...state,
         libraries: application.componentLibraries,
         components,
         name: application.name,
         description: application.description,
-        appId: application._id,
+        appId: application.appId,
         url: application.url,
         layouts,
         screens,

packages/builder/src/builderStore/store/hosting.js

@@ -2,7 +2,6 @@ import { writable } from "svelte/store"
 import api, { get } from "../api"
 
 const INITIAL_HOSTING_UI_STATE = {
-  hostingInfo: {},
   appUrl: "",
   deployedApps: {},
   deployedAppNames: [],
@@ -13,28 +12,12 @@ export const getHostingStore = () => {
   const store = writable({ ...INITIAL_HOSTING_UI_STATE })
   store.actions = {
     fetch: async () => {
-      const responses = await Promise.all([
+      const response = await api.get("/api/hosting/urls")
-        api.get("/api/hosting/"),
+      const urls = await response.json()
-        api.get("/api/hosting/urls"),
-      ])
-      const [info, urls] = await Promise.all(responses.map(resp => resp.json()))
       store.update(state => {
-        state.hostingInfo = info
         state.appUrl = urls.app
         return state
       })
-      return info
-    },
-    save: async hostingInfo => {
-      const response = await api.post("/api/hosting", hostingInfo)
-      const revision = (await response.json()).rev
-      store.update(state => {
-        state.hostingInfo = {
-          ...hostingInfo,
-          _rev: revision,
-        }
-        return state
-      })
     },
     fetchDeployedApps: async () => {
       let deployments = await (await get("/api/hosting/apps")).json()

packages/builder/src/components/common/bindings/ServerBindingPanel.svelte

@@ -50,7 +50,7 @@
       <div class="section">
         {#each categories as [categoryName, bindings]}
           <Heading size="XS">{categoryName}</Heading>
-          {#each bindableProperties.filter(binding =>
+          {#each bindings.filter(binding =>
             binding.label.match(searchRgx)
           ) as binding}
             <div

packages/builder/src/components/deploy/DeployModal.svelte

@@ -0,0 +1,113 @@
+<script>
+  import { onMount, onDestroy } from "svelte"
+  import { Button, Modal, notifications, ModalContent } from "@budibase/bbui"
+  import { store } from "builderStore"
+  import api from "builderStore/api"
+  import analytics from "analytics"
+  import FeedbackIframe from "components/feedback/FeedbackIframe.svelte"
+
+  const DeploymentStatus = {
+    SUCCESS: "SUCCESS",
+    PENDING: "PENDING",
+    FAILURE: "FAILURE",
+  }
+
+  const POLL_INTERVAL = 1000
+
+  let loading = false
+  let feedbackModal
+  let deployments = []
+  let poll
+  let publishModal
+
+  $: appId = $store.appId
+
+  async function deployApp() {
+    try {
+      notifications.info(`Deployment started. Please wait.`)
+      const response = await api.post("/api/deploy")
+      const json = await response.json()
+      if (response.status !== 200) {
+        throw new Error()
+      }
+
+      if (analytics.requestFeedbackOnDeploy()) {
+        feedbackModal.show()
+      }
+    } catch (err) {
+      analytics.captureException(err)
+      notifications.error("Deployment unsuccessful. Please try again later.")
+    }
+  }
+
+  async function fetchDeployments() {
+    try {
+      const response = await api.get(`/api/deployments`)
+      const json = await response.json()
+
+      if (deployments.length > 0) {
+        checkIncomingDeploymentStatus(deployments, json)
+      }
+
+      deployments = json
+    } catch (err) {
+      console.error(err)
+      clearInterval(poll)
+      notifications.error(
+        "Error fetching deployment history. Please try again."
+      )
+    }
+  }
+
+  // Required to check any updated deployment statuses between polls
+  function checkIncomingDeploymentStatus(current, incoming) {
+    console.log(current, incoming)
+    for (let incomingDeployment of incoming) {
+      if (
+        incomingDeployment.status === DeploymentStatus.FAILURE ||
+        incomingDeployment.status === DeploymentStatus.SUCCESS
+      ) {
+        const currentDeployment = current.find(
+          deployment => deployment._id === incomingDeployment._id
+        )
+
+        // We have just been notified of an ongoing deployments status change
+        if (
+          !currentDeployment ||
+          currentDeployment.status === DeploymentStatus.PENDING
+        ) {
+          if (incomingDeployment.status === DeploymentStatus.FAILURE) {
+            notifications.error(incomingDeployment.err)
+          } else {
+            notifications.send(
+              "Published to Production.",
+              "success",
+              "CheckmarkCircle"
+            )
+          }
+        }
+      }
+    }
+  }
+
+  onMount(() => {
+    fetchDeployments()
+    poll = setInterval(fetchDeployments, POLL_INTERVAL)
+  })
+
+  onDestroy(() => clearInterval(poll))
+</script>
+
+<Button secondary on:click={publishModal.show}>Publish</Button>
+<Modal bind:this={publishModal}>
+  <ModalContent
+    title="Publish to Production"
+    confirmText="Publish"
+    onConfirm={deployApp}
+  >
+    <span
+      >The changes you have made will be published to the production version of
+      the application.</span
+    >
+  </ModalContent>
+</Modal>

packages/builder/src/components/deploy/DeploymentHistory.svelte

@@ -36,8 +36,7 @@
   let errorReason
   let poll
   let deployments = []
-  let urlComponent =
+  let urlComponent = $store.url || `/${appId}`
-    $hostingStore.hostingInfo.type === "self" ? $store.url : `/${appId}`
   let deploymentUrl = `${$hostingStore.appUrl}${urlComponent}`
 
   const formatDate = (date, format) =>

packages/builder/src/components/deploy/RevertModal.svelte

@@ -0,0 +1,50 @@
+<script>
+  import { onMount, onDestroy } from "svelte"
+  import {
+    Button,
+    Icon,
+    Modal,
+    notifications,
+    ModalContent,
+  } from "@budibase/bbui"
+  import { store } from "builderStore"
+  import { apps } from "stores/portal"
+  import api from "builderStore/api"
+
+  let revertModal
+
+  $: appId = $store.appId
+
+  const revert = async () => {
+    try {
+      const response = await api.post(`/api/dev/${appId}/revert`)
+      const json = await response.json()
+      if (response.status !== 200) throw json.message
+
+      // Reset frontend state after revert
+      const applicationPkg = await api.get(
+        `/api/applications/${appId}/appPackage`
+      )
+      const pkg = await applicationPkg.json()
+      if (applicationPkg.ok) {
+        await store.actions.initialise(pkg)
+      } else {
+        throw new Error(pkg)
+      }
+
+      notifications.info("Changes reverted.")
+    } catch (err) {
+      notifications.error(`Error reverting changes: ${err}`)
+    }
+  }
+</script>
+
+<Icon name="Revert" hoverable on:click={revertModal.show} />
+<Modal bind:this={revertModal}>
+  <ModalContent title="Revert Changes" confirmText="Revert" onConfirm={revert}>
+    <span
+      >The changes you have made will be deleted and the application reverted
+      back to its production state.</span
+    >
+  </ModalContent>
+</Modal>

packages/builder/src/components/settings/SettingsModal.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { General, DangerZone, APIKeys } from "./tabs"
+  import { General, DangerZone } from "./tabs"
   import { ModalContent, Tab, Tabs } from "@budibase/bbui"
 </script>
 
@@ -13,9 +13,9 @@
       <Tab title="General">
         <General />
       </Tab>
-      <Tab title="API Keys">
+      <!-- <Tab title="API Keys">
         <APIKeys />
-      </Tab>
+      </Tab> -->
       <Tab title="Danger Zone">
         <DangerZone />
       </Tab>

packages/builder/src/components/settings/tabs/General.svelte

@@ -49,8 +49,6 @@
   onMount(async () => {
     const nameError = "Your application must have a name.",
       urlError = "Your application must have a URL."
-    let hostingInfo = await hostingStore.actions.fetch()
-    if (hostingInfo.type === "self") {
     await hostingStore.actions.fetchDeployedApps()
     const existingAppNames = get(hostingStore).deployedAppNames
     const existingAppUrls = get(hostingStore).deployedAppUrls
@@ -68,9 +66,6 @@
     urlValidation = {
       url: string().required(urlError).notOneOf(existingAppUrls),
     }
-    } else {
-      nameValidation = { name: string().required(nameError) }
-    }
   })
 </script>
 
@@ -81,14 +76,12 @@
     error={nameError}
     label="App Name"
   />
-  {#if $hostingStore.hostingInfo.type === "self"}
   <Input
     on:change={e => updateApplication({ url: e.detail })}
     value={$store.url}
     error={urlError}
     label="App URL"
   />
-  {/if}
   <TextArea
     on:change={e => updateApplication({ description: e.detail })}
     value={$store.description}

packages/builder/src/components/start/AppCard.svelte

@@ -9,18 +9,23 @@
     Link,
   } from "@budibase/bbui"
   import { gradient } from "actions"
+  import { AppStatus } from "constants"
   import { url } from "@roxi/routify"
+  import { auth } from "stores/backend"
 
   export let app
   export let exportApp
+  export let openApp
   export let deleteApp
+  export let releaseLock
+  export let deletable
 </script>
 
 <div class="wrapper">
   <Layout noPadding gap="XS" alignContent="start">
     <div class="preview" use:gradient={{ seed: app.name }} />
     <div class="title">
-      <Link href={$url(`../../app/${app._id}`)}>
+      <Link on:click={() => openApp(app)}>
         <Heading size="XS">
           {app.name}
         </Heading>
@@ -30,16 +35,23 @@
         <MenuItem on:click={() => exportApp(app)} icon="Download">
           Export
         </MenuItem>
+        {#if deletable}
           <MenuItem on:click={() => deleteApp(app)} icon="Delete">
             Delete
           </MenuItem>
+        {/if}
+        {#if app.lockedBy && app.lockedBy?.email === $auth.user?.email}
+          <MenuItem on:click={() => releaseLock(app.appId)} icon="LockOpen">
+            Release Lock
+          </MenuItem>
+        {/if}
       </ActionMenu>
     </div>
     <div class="status">
       <Body noPadding size="S">
         Edited {Math.floor(1 + Math.random() * 10)} months ago
       </Body>
-      {#if Math.random() > 0.5}
+      {#if app.lockedBy}
         <Icon name="LockClosed" />
       {/if}
     </div>

packages/builder/src/components/start/AppRow.svelte

@@ -8,18 +8,22 @@
     MenuItem,
     Link,
   } from "@budibase/bbui"
+  import { AppStatus } from "constants"
   import { url } from "@roxi/routify"
+  import { auth } from "stores/backend"
 
   export let app
   export let openApp
   export let exportApp
   export let deleteApp
+  export let releaseLock
   export let last
+  export let deletable
 </script>
 
 <div class="title" class:last>
   <div class="preview" use:gradient={{ seed: app.name }} />
-  <Link href={$url(`../../app/${app._id}`)}>
+  <Link on:click={() => openApp(app)}>
     <Heading size="XS">
       {app.name}
     </Heading>
@@ -29,15 +33,12 @@
   Edited {Math.round(Math.random() * 10 + 1)} months ago
 </div>
 <div class:last>
-  {#if Math.random() < 0.33}
+  {#if app.lockedBy}
+    <div class="status status--locked-you" />
+    Locked by {app.lockedBy.email}
+  {:else}
     <div class="status status--open" />
     Open
-  {:else if Math.random() < 0.33}
-    <div class="status status--locked-other" />
-    Locked by Will Wheaton
-  {:else}
-    <div class="status status--locked-you" />
-    Locked by you
   {/if}
 </div>
 <div class:last>
@@ -45,7 +46,14 @@
   <ActionMenu align="right">
     <Icon hoverable slot="control" name="More" />
     <MenuItem on:click={() => exportApp(app)} icon="Download">Export</MenuItem>
+    {#if deletable}
       <MenuItem on:click={() => deleteApp(app)} icon="Delete">Delete</MenuItem>
+    {/if}
+    {#if app.lockedBy && app.lockedBy?.email === $auth.user?.email}
+      <MenuItem on:click={() => releaseLock(app.appId)} icon="LockOpen">
+        Release Lock
+      </MenuItem>
+    {/if}
   </ActionMenu>
 </div>
 

packages/builder/src/components/start/BuilderSettingsModal.svelte

@@ -1,47 +1,19 @@
 <script>
-  import { hostingStore } from "builderStore"
-  import { HostingTypes } from "constants/backend"
   import {
     Heading,
     Divider,
     notifications,
-    Input,
     ModalContent,
     Toggle,
     Body,
   } from "@budibase/bbui"
   import ThemeEditor from "components/settings/ThemeEditor.svelte"
   import analytics from "analytics"
-  import { onMount } from "svelte"
-
-  let hostingInfo
-  let selfhosted = false
 
   $: analyticsDisabled = analytics.disabled()
 
   async function save() {
-    hostingInfo.type = selfhosted ? HostingTypes.SELF : HostingTypes.CLOUD
-    if (!selfhosted && hostingInfo._rev) {
-      hostingInfo = {
-        type: hostingInfo.type,
-        _id: hostingInfo._id,
-        _rev: hostingInfo._rev,
-      }
-    }
-    try {
-      await hostingStore.actions.save(hostingInfo)
     notifications.success(`Settings saved.`)
-    } catch (err) {
-      notifications.error(`Failed to update builder settings.`)
-    }
-  }
-
-  function updateSelfHosting(event) {
-    if (hostingInfo.type === HostingTypes.CLOUD && event.detail) {
-      hostingInfo.hostingUrl = "localhost:10000"
-      hostingInfo.useHttps = false
-      hostingInfo.selfHostKey = "budibase"
-    }
   }
 
   function toggleAnalytics() {
@@ -51,33 +23,12 @@
       analytics.optOut()
     }
   }
-
-  onMount(async () => {
-    hostingInfo = await hostingStore.actions.fetch()
-    selfhosted = hostingInfo.type === "self"
-  })
 </script>
 
 <ModalContent title="Builder settings" confirmText="Save" onConfirm={save}>
   <Heading size="XS">Theme</Heading>
   <ThemeEditor />
   <Divider noMargin noGrid />
-  <Heading size="XS">Hosting</Heading>
-  <Body size="S">
-    This section contains settings that relate to the deployment and hosting of
-    apps made in this builder.
-  </Body>
-  <Toggle
-    text="Self hosted"
-    on:change={updateSelfHosting}
-    bind:value={selfhosted}
-  />
-  {#if selfhosted}
-    <Input bind:value={hostingInfo.hostingUrl} label="Hosting URL" />
-    <Input bind:value={hostingInfo.selfHostKey} label="Hosting Key" />
-    <Toggle text="HTTPS" bind:value={hostingInfo.useHttps} />
-  {/if}
-  <Divider noMargin noGrid />
   <Heading size="XS">Analytics</Heading>
   <Body size="S">
     If you would like to send analytics that help us make budibase better,

packages/builder/src/components/start/CreateAppModal.svelte

@@ -10,8 +10,7 @@
   } from "@budibase/bbui"
   import { store, automationStore, hostingStore } from "builderStore"
   import { string, mixed, object } from "yup"
-  import api, { get } from "builderStore/api"
+  import api, { get, post } from "builderStore/api"
-  import { post } from "builderStore/api"
   import analytics from "analytics"
   import { onMount } from "svelte"
   import { capitalise } from "helpers"
@@ -97,7 +96,7 @@
 
       // Select Correct Application/DB in prep for creating user
       const applicationPkg = await get(
-        `/api/applications/${appJson._id}/appPackage`
+        `/api/applications/${appJson.instance._id}/appPackage`
       )
       const pkg = await applicationPkg.json()
       if (applicationPkg.ok) {
@@ -113,7 +112,7 @@
       }
       const userResp = await api.post(`/api/users/metadata/self`, user)
       await userResp.json()
-      $goto(`/builder/app/${appJson._id}`)
+      $goto(`/builder/app/${appJson.instance._id}`)
     } catch (error) {
       console.error(error)
       notifications.error(error)

packages/builder/src/constants/index.js

@@ -9,6 +9,11 @@ export const FrontendTypes = {
   NONE: "none",
 }
 
+export const AppStatus = {
+  DEV: "dev",
+  PUBLISHED: "published",
+}
+
 // fields on the user table that cannot be edited
 export const UNEDITABLE_USER_FIELDS = ["email", "password", "roleId", "status"]
 

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -1,10 +1,21 @@
 <script>
   import { store, automationStore } from "builderStore"
   import { roles } from "stores/backend"
-  import { Button, ActionGroup, ActionButton, Tabs, Tab } from "@budibase/bbui"
+  import {
+    Button,
+    Icon,
+    Modal,
+    ModalContent,
+    ActionGroup,
+    ActionButton,
+    Tabs,
+    Tab,
+  } from "@budibase/bbui"
   import SettingsLink from "components/settings/Link.svelte"
   import ThemeEditorDropdown from "components/settings/ThemeEditorDropdown.svelte"
   import FeedbackNavLink from "components/feedback/FeedbackNavLink.svelte"
+  import DeployModal from "components/deploy/DeployModal.svelte"
+  import RevertModal from "components/deploy/RevertModal.svelte"
   import { get } from "builderStore/api"
   import { isActive, goto, layout } from "@roxi/routify"
   import Logo from "/assets/bb-logo.svg"
@@ -81,25 +92,15 @@
         <ActionGroup />
       </div>
       <div class="toprightnav">
-        <ThemeEditorDropdown />
+        <RevertModal />
-        <FeedbackNavLink />
+        <Icon
-        <div class="topnavitemright">
+          name="Play"
-          <a
+          hoverable
-            target="_blank"
-            href="https://github.com/Budibase/budibase/discussions"
-          >
-            <i class="ri-github-fill" />
-          </a>
-        </div>
-        <SettingsLink />
-        <Button
-          secondary
           on:click={() => {
             window.open(`/${application}`)
           }}
-        >
+        />
-          Preview
+        <DeployModal />
-        </Button>
       </div>
     </div>
     <div class="beta">
@@ -153,6 +154,7 @@
     flex-direction: row;
     justify-content: flex-end;
     align-items: center;
+    gap: var(--spacing-xl);
   }
 
   .topleftnav {

packages/builder/src/pages/builder/app/[application]/deploy/_layout.svelte

@@ -1,2 +0,0 @@
-<!-- routify:options index=4 -->
-<slot />

packages/builder/src/pages/builder/app/[application]/deploy/index.svelte

@@ -1,102 +0,0 @@
-<script>
-  import { Button, Modal, notifications, Heading } from "@budibase/bbui"
-  import { store, hostingStore } from "builderStore"
-  import api from "builderStore/api"
-  import DeploymentHistory from "components/deploy/DeploymentHistory.svelte"
-  import analytics from "analytics"
-  import FeedbackIframe from "components/feedback/FeedbackIframe.svelte"
-  import Rocket from "/assets/deploy-rocket.jpg"
-
-  let loading = false
-  let deployments = []
-  let poll
-  let feedbackModal
-
-  $: appId = $store.appId
-
-  async function deployApp() {
-    // Must have cloud or self host API key to deploy
-    if (!$hostingStore.hostingInfo?.selfHostKey) {
-      const response = await api.get(`/api/keys/`)
-      const userKeys = await response.json()
-      if (!userKeys.budibase) {
-        notifications.error(
-          "No budibase API Keys configured. You must set either a self hosted or cloud API key to deploy your budibase app."
-        )
-        return
-      }
-    }
-
-    const DEPLOY_URL = `/api/deploy`
-
-    try {
-      notifications.info(`Deployment started. Please wait.`)
-      const response = await api.post(DEPLOY_URL)
-      const json = await response.json()
-      if (response.status !== 200) {
-        throw new Error()
-      }
-
-      analytics.captureEvent("Deployed App", {
-        appId,
-        hostingType: $hostingStore.hostingInfo?.type,
-      })
-
-      if (analytics.requestFeedbackOnDeploy()) {
-        feedbackModal.show()
-      }
-    } catch (err) {
-      analytics.captureEvent("Deploy App Failed", {
-        appId,
-        hostingType: $hostingStore.hostingInfo?.type,
-      })
-      analytics.captureException(err)
-      notifications.error("Deployment unsuccessful. Please try again later.")
-    }
-  }
-</script>
-
-<section>
-  <img src={Rocket} alt="Rocket flying through sky" />
-  <div>
-    <Heading size="M">It's time to shine!</Heading>
-    <Button size="XL" cta medium on:click={deployApp}>Deploy App</Button>
-  </div>
-</section>
-<Modal bind:this={feedbackModal}>
-  <FeedbackIframe on:finished={() => feedbackModal.hide()} />
-</Modal>
-<DeploymentHistory {appId} />
-
-<style>
-  img {
-    width: 100%;
-    height: 100%;
-    object-fit: cover;
-    filter: brightness(80%);
-  }
-
-  section {
-    position: relative;
-    min-height: 100%;
-  }
-
-  div {
-    position: absolute;
-    display: flex;
-    text-align: center;
-    flex-direction: column;
-    align-items: center;
-    justify-content: center;
-    left: 0;
-    right: 0;
-    top: 20%;
-    margin-left: auto;
-    margin-right: auto;
-    width: 50%;
-    gap: var(--spacing-xl);
-  }
-  div :global(h1) {
-    color: white;
-  }
-</style>

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -18,13 +18,16 @@
   import analytics from "analytics"
   import { onMount } from "svelte"
   import { apps } from "stores/portal"
+  import { auth } from "stores/backend"
   import download from "downloadjs"
   import { goto } from "@roxi/routify"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import AppCard from "components/start/AppCard.svelte"
   import AppRow from "components/start/AppRow.svelte"
+  import { AppStatus } from "constants"
 
   let layout = "grid"
+  let appStatus = AppStatus.PUBLISHED
   let template
   let appToDelete
   let creationModal
@@ -32,6 +35,8 @@
   let creatingApp = false
   let loaded = false
 
+  $: appStatus && apps.load(appStatus)
+
   const checkKeys = async () => {
     const response = await api.get(`/api/keys/`)
     const keys = await response.json()
@@ -57,13 +62,24 @@
   }
 
   const openApp = app => {
-    $goto(`../../app/${app._id}`)
+    if (app.lockedBy && app.lockedBy?.email !== $auth.user?.email) {
+      notifications.error(
+        `App locked by ${app.lockedBy.email}. Please allow lock to expire or have them unlock this app.`
+      )
+      return
+    }
+
+    if (appStatus === AppStatus.DEV) {
+      $goto(`../../app/${app.appId}`)
+    } else {
+      window.open(`/${app.appId}`, "_blank")
+    }
   }
 
   const exportApp = app => {
     try {
       download(
-        `/api/backups/export?appId=${app._id}&appname=${encodeURIComponent(
+        `/api/backups/export?appId=${app.appId}&appname=${encodeURIComponent(
           app.name
         )}`
       )
@@ -83,20 +99,34 @@
     if (!appToDelete) {
       return
     }
+
     await del(`/api/applications/${appToDelete?._id}`)
     await apps.load()
     appToDelete = null
+    notifications.success("App deleted successfully.")
+  }
+
+  const releaseLock = async appId => {
+    try {
+      const response = await del(`/api/dev/${appId}/lock`)
+      const json = await response.json()
+      if (response.status !== 200) throw json.message
+
+      notifications.success("Lock released")
+      await apps.load(appStatus)
+    } catch (err) {
+      notifications.error(`Error releasing lock: ${err}`)
+    }
   }
 
   onMount(async () => {
     checkKeys()
-    await apps.load()
+    await apps.load(appStatus)
     loaded = true
   })
 </script>
 
 <Page wide>
-  {#if $apps.length}
   <Layout noPadding>
     <div class="title">
       <Heading>Apps</Heading>
@@ -107,7 +137,13 @@
     </div>
     <div class="filter">
       <div class="select">
-          <Select quiet placeholder="Filter by groups" />
+        <Select
+          bind:value={appStatus}
+          options={[
+            { label: "Published", value: AppStatus.PUBLISHED },
+            { label: "In Development", value: AppStatus.DEV },
+          ]}
+        />
       </div>
       <ActionGroup>
         <ActionButton
@@ -124,13 +160,16 @@
         />
       </ActionGroup>
     </div>
+    {#if $apps.length}
       <div
         class:appGrid={layout === "grid"}
         class:appTable={layout === "table"}
       >
-        {#each $apps as app, idx (app._id)}
+        {#each $apps as app, idx (app.appId)}
           <svelte:component
             this={layout === "grid" ? AppCard : AppRow}
+            deletable={appStatus === AppStatus.PUBLISHED}
+            {releaseLock}
             {app}
             {openApp}
             {exportApp}
@@ -139,8 +178,8 @@
           />
         {/each}
       </div>
-    </Layout>
     {/if}
+  </Layout>
   {#if !$apps.length && !creatingApp && loaded}
     <div class="empty-wrapper">
       <Modal inline>

packages/builder/src/stores/portal/apps.js

@@ -4,15 +4,16 @@ import { get } from "builderStore/api"
 export function createAppStore() {
   const store = writable([])
 
-  async function load() {
+  async function load(status = "") {
     try {
-      const res = await get("/api/applications")
+      const res = await get(`/api/applications?status=${status}`)
       const json = await res.json()
       if (res.ok && Array.isArray(json)) {
         store.set(json)
       } else {
         store.set([])
       }
+      return json
     } catch (error) {
       store.set([])
     }

packages/client/yarn.lock

@@ -2873,7 +2873,7 @@ svelte-spa-router@^3.0.5:
   dependencies:
     regexparam "1.3.0"
 
-svelte@^3.37.0:
+svelte@^3.38.2:
   version "3.38.2"
   resolved "https://registry.yarnpkg.com/svelte/-/svelte-3.38.2.tgz#55e5c681f793ae349b5cc2fe58e5782af4275ef5"
   integrity sha512-q5Dq0/QHh4BLJyEVWGe7Cej5NWs040LWjMbicBGZ+3qpFWJ1YObRmUDZKbbovddLC9WW7THTj3kYbTOFmU9fbg==

packages/server/scripts/replicateApp.js

@@ -6,6 +6,7 @@
  */
 
 const CouchDB = require("../src/db")
+const { DocumentTypes } = require("../src/db/utils")
 
 const appName = process.argv[2].toLowerCase()
 const remoteUrl = process.argv[3]
@@ -18,7 +19,7 @@ const run = async () => {
   let apps = []
   for (let dbName of appDbNames) {
     const db = new CouchDB(dbName)
-    apps.push(db.get(dbName))
+    apps.push(db.get(DocumentTypes.APP_METADATA))
   }
   apps = await Promise.all(apps)
   const app = apps.find(
@@ -32,7 +33,7 @@ const run = async () => {
     return
   }
 
-  const instanceDb = new CouchDB(app._id)
+  const instanceDb = new CouchDB(app.appId)
   const remoteDb = new CouchDB(`${remoteUrl}/${appName}`)
 
   instanceDb.replicate

packages/server/src/api/controllers/application.js

@@ -16,11 +16,11 @@ const {
   getLayoutParams,
   getScreenParams,
   generateScreenID,
+  generateDevAppID,
+  DocumentTypes,
+  AppStatus,
 } = require("../../db/utils")
-const {
+const { BUILTIN_ROLE_IDS, AccessController } = require("@budibase/auth/roles")
-  BUILTIN_ROLE_IDS,
-  AccessController,
-} = require("../../utilities/security/roles")
 const { BASE_LAYOUTS } = require("../../constants/layouts")
 const {
   createHomeScreen,
@@ -30,8 +30,9 @@ const { cloneDeep } = require("lodash/fp")
 const { processObject } = require("@budibase/string-templates")
 const { getAllApps } = require("../../utilities")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
-const { getDeployedApps } = require("../../utilities/builder/hosting")
+const { getDeployedApps } = require("../../utilities/workerRequests")
 const { clientLibraryPath } = require("../../utilities")
+const { getAllLocks } = require("../../utilities/redis")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -84,7 +85,9 @@ async function getAppUrlIfNotInUse(ctx) {
 }
 
 async function createInstance(template) {
-  const appId = generateAppID()
+  const baseAppId = generateAppID()
+  const appId = generateDevAppID(baseAppId)
+
   const db = new CouchDB(appId)
   await db.put({
     _id: "_design/database",
@@ -114,7 +117,24 @@ async function createInstance(template) {
 }
 
 exports.fetch = async function (ctx) {
-  ctx.body = await getAllApps()
+  const isDev = ctx.query && ctx.query.status === AppStatus.DEV
+  const apps = await getAllApps(isDev)
+
+  // get the locks for all the dev apps
+  if (isDev) {
+    const locks = await getAllLocks()
+    for (let app of apps) {
+      const lock = locks.find(lock => lock.appId === app.appId)
+      if (lock) {
+        app.lockedBy = lock.user
+      } else {
+        // make sure its definitely not present
+        delete app.lockedBy
+      }
+    }
+  }
+
+  ctx.body = apps
 }
 
 exports.fetchAppDefinition = async function (ctx) {
@@ -135,7 +155,7 @@ exports.fetchAppDefinition = async function (ctx) {
 
 exports.fetchAppPackage = async function (ctx) {
   const db = new CouchDB(ctx.params.appId)
-  const application = await db.get(ctx.params.appId)
+  const application = await db.get(DocumentTypes.APP_METADATA)
   const [layouts, screens] = await Promise.all([getLayouts(db), getScreens(db)])
 
   ctx.body = {
@@ -160,7 +180,8 @@ exports.create = async function (ctx) {
   const url = await getAppUrlIfNotInUse(ctx)
   const appId = instance._id
   const newApplication = {
-    _id: appId,
+    _id: DocumentTypes.APP_METADATA,
+    appId: instance._id,
     type: "app",
     version: packageJson.version,
     componentLibraries: ["@budibase/standard-components"],
@@ -194,6 +215,12 @@ exports.update = async function (ctx) {
   const data = ctx.request.body
   const newData = { ...application, ...data, url }
 
+  // the locked by property is attached by server but generated from
+  // Redis, shouldn't ever store it
+  if (newData.lockedBy) {
+    delete newData.lockedBy
+  }
+
   const response = await db.put(newData)
   data._rev = response.rev
 
@@ -217,7 +244,7 @@ exports.delete = async function (ctx) {
 }
 
 const createEmptyAppPackage = async (ctx, app) => {
-  const db = new CouchDB(app._id)
+  const db = new CouchDB(app.appId)
 
   let screensAndLayouts = []
   for (let layout of BASE_LAYOUTS) {

packages/server/src/api/controllers/component.js

@@ -1,10 +1,11 @@
 const CouchDB = require("../../db")
+const { DocumentTypes } = require("../../db/utils")
 const { getComponentLibraryManifest } = require("../../utilities/fileSystem")
 
 exports.fetchAppComponentDefinitions = async function (ctx) {
   const appId = ctx.params.appId || ctx.appId
   const db = new CouchDB(appId)
-  const app = await db.get(appId)
+  const app = await db.get(DocumentTypes.APP_METADATA)
 
   let componentManifests = await Promise.all(
     app.componentLibraries.map(async library => {

packages/server/src/api/controllers/deploy/Deployment.js

@@ -1,5 +1,3 @@
-const { getAppQuota } = require("./quota")
-const env = require("../../../environment")
 const newid = require("../../../db/newid")
 
 /**
@@ -11,24 +9,6 @@ class Deployment {
     this._id = id || newid()
   }
 
-  // purely so that we can do quota stuff outside the main deployment context
-  async init() {
-    if (!env.SELF_HOSTED) {
-      this.setQuota(await getAppQuota(this.appId))
-    }
-  }
-
-  setQuota(quota) {
-    if (!quota) {
-      return
-    }
-    this.quota = quota
-  }
-
-  getQuota() {
-    return this.quota
-  }
-
   getAppId() {
     return this.appId
   }
@@ -38,9 +18,6 @@ class Deployment {
       return
     }
     this.verification = verification
-    if (this.verification.quota) {
-      this.quota = this.verification.quota
-    }
   }
 
   getVerification() {
@@ -58,9 +35,6 @@ class Deployment {
     if (json.verification) {
       this.setVerification(json.verification)
     }
-    if (json.quota) {
-      this.setQuota(json.quota)
-    }
     if (json.status) {
       this.setStatus(json.status, json.err)
     }
@@ -78,9 +52,6 @@ class Deployment {
     if (this.verification && this.verification.cfDistribution) {
       obj.cfDistribution = this.verification.cfDistribution
     }
-    if (this.quota) {
-      obj.quota = this.quota
-    }
     return obj
   }
 }

packages/server/src/api/controllers/deploy/awsDeploy.js

@@ -1,80 +0,0 @@
-const AWS = require("aws-sdk")
-const fetch = require("node-fetch")
-const env = require("../../../environment")
-const {
-  deployToObjectStore,
-  performReplication,
-  fetchCredentials,
-} = require("./utils")
-
-/**
- * Verifies the users API key and
- * Verifies that the deployment fits within the quota of the user
- * Links to the "check-api-key" lambda.
- * @param {object} deployment - information about the active deployment, including the appId and quota.
- */
-exports.preDeployment = async function (deployment) {
-  const json = await fetchCredentials(env.DEPLOYMENT_CREDENTIALS_URL, {
-    apiKey: env.BUDIBASE_API_KEY,
-    appId: deployment.getAppId(),
-    quota: deployment.getQuota(),
-  })
-
-  // set credentials here, means any time we're verified we're ready to go
-  if (json.credentials) {
-    AWS.config.update({
-      accessKeyId: json.credentials.AccessKeyId,
-      secretAccessKey: json.credentials.SecretAccessKey,
-      sessionToken: json.credentials.SessionToken,
-    })
-  }
-
-  return json
-}
-
-/**
- * Finalises the deployment, updating the quota for the user API key
- * The verification process returns the levels to update to.
- * Calls the "deployment-success" lambda.
- * @param {object} deployment information about the active deployment, including the quota info.
- * @returns {Promise<object>} The usage has been updated against the user API key.
- */
-exports.postDeployment = async function (deployment) {
-  const DEPLOYMENT_SUCCESS_URL =
-    env.DEPLOYMENT_CREDENTIALS_URL + "deploy/success"
-
-  const response = await fetch(DEPLOYMENT_SUCCESS_URL, {
-    method: "POST",
-    body: JSON.stringify({
-      apiKey: env.BUDIBASE_API_KEY,
-      quota: deployment.getQuota(),
-    }),
-    headers: {
-      "Content-Type": "application/json",
-      Accept: "application/json",
-    },
-  })
-
-  if (response.status !== 200) {
-    throw new Error(`Error updating deployment quota for API Key`)
-  }
-
-  return await response.json()
-}
-
-exports.deploy = async function (deployment) {
-  const appId = deployment.getAppId()
-  const { bucket, accountId } = deployment.getVerification()
-  const metadata = { accountId }
-  await deployToObjectStore(appId, bucket, metadata)
-}
-
-exports.replicateDb = async function (deployment) {
-  const appId = deployment.getAppId()
-  const verification = deployment.getVerification()
-  return performReplication(
-    appId,
-    verification.couchDbSession,
-    env.DEPLOYMENT_DB_URL
-  )
-}

packages/server/src/api/controllers/deploy/index.js

@@ -1,9 +1,8 @@
 const PouchDB = require("../../../db")
 const Deployment = require("./Deployment")
-const {
+const { Replication, StaticDatabases } = require("@budibase/auth/db")
-  getHostingInfo,
+const { DocumentTypes } = require("../../../db/utils")
-  HostingTypes,
+
-} = require("../../../utilities/builder/hosting")
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000
 const DeploymentStatus = {
@@ -12,9 +11,6 @@ const DeploymentStatus = {
   FAILURE: "FAILURE",
 }
 
-// default to AWS deployment, this will be updated before use (if required)
-let deploymentService = require("./awsDeploy")
-
 // checks that deployments are in a good state, any pending will be updated
 async function checkAllDeployments(deployments) {
   let updated = false
@@ -32,16 +28,16 @@ async function checkAllDeployments(deployments) {
   return { updated, deployments }
 }
 
-async function storeLocalDeploymentHistory(deployment) {
+async function storeDeploymentHistory(deployment) {
   const appId = deployment.getAppId()
   const deploymentJSON = deployment.getJSON()
-  const db = new PouchDB(appId)
+  const db = new PouchDB(StaticDatabases.DEPLOYMENTS.name)
 
   let deploymentDoc
   try {
-    deploymentDoc = await db.get("_local/deployments")
+    deploymentDoc = await db.get(appId)
   } catch (err) {
-    deploymentDoc = { _id: "_local/deployments", history: {} }
+    deploymentDoc = { _id: appId, history: {} }
   }
 
   const deploymentId = deploymentJSON._id
@@ -62,28 +58,37 @@ async function storeLocalDeploymentHistory(deployment) {
 }
 
 async function deployApp(deployment) {
-  const appId = deployment.getAppId()
   try {
-    await deployment.init()
+    const productionAppId = deployment.appId.replace("_dev", "")
-    deployment.setVerification(
-      await deploymentService.preDeployment(deployment)
-    )
 
-    console.log(`Uploading assets for appID ${appId}..`)
+    const replication = new Replication({
+      source: deployment.appId,
+      target: productionAppId,
+    })
 
-    await deploymentService.deploy(deployment)
+    await replication.replicate()
+    const db = new PouchDB(productionAppId)
+    const appDoc = await db.get(DocumentTypes.APP_METADATA)
+    appDoc.appId = productionAppId
+    appDoc.instance._id = productionAppId
+    await db.put(appDoc)
 
-    // replicate the DB to the main couchDB cluster
+    // Set up live sync between the live and dev instances
-    console.log("Replicating local PouchDB to CouchDB..")
+    const liveReplication = new Replication({
-    await deploymentService.replicateDb(deployment)
+      source: productionAppId,
-
+      target: deployment.appId,
-    await deploymentService.postDeployment(deployment)
+    })
+    liveReplication.subscribe({
+      filter: function (doc) {
+        return doc._id !== DocumentTypes.APP_METADATA
+      },
+    })
 
     deployment.setStatus(DeploymentStatus.SUCCESS)
-    await storeLocalDeploymentHistory(deployment)
+    await storeDeploymentHistory(deployment)
   } catch (err) {
     deployment.setStatus(DeploymentStatus.FAILURE, err.message)
-    await storeLocalDeploymentHistory(deployment)
+    await storeDeploymentHistory(deployment)
     throw {
       ...err,
       message: `Deployment Failed: ${err.message}`,
@@ -93,8 +98,8 @@ async function deployApp(deployment) {
 
 exports.fetchDeployments = async function (ctx) {
   try {
-    const db = new PouchDB(ctx.appId)
+    const db = new PouchDB(StaticDatabases.DEPLOYMENTS.name)
-    const deploymentDoc = await db.get("_local/deployments")
+    const deploymentDoc = await db.get(ctx.appId)
     const { updated, deployments } = await checkAllDeployments(
       deploymentDoc,
       ctx.user
@@ -110,8 +115,8 @@ exports.fetchDeployments = async function (ctx) {
 
 exports.deploymentProgress = async function (ctx) {
   try {
-    const db = new PouchDB(ctx.appId)
+    const db = new PouchDB(StaticDatabases.DEPLOYMENTS.name)
-    const deploymentDoc = await db.get("_local/deployments")
+    const deploymentDoc = await db.get(ctx.appId)
     ctx.body = deploymentDoc[ctx.params.deploymentId]
   } catch (err) {
     ctx.throw(
@@ -122,15 +127,9 @@ exports.deploymentProgress = async function (ctx) {
 }
 
 exports.deployApp = async function (ctx) {
-  // start by checking whether to deploy local or to cloud
-  const hostingInfo = await getHostingInfo()
-  deploymentService =
-    hostingInfo.type === HostingTypes.CLOUD
-      ? require("./awsDeploy")
-      : require("./selfDeploy")
   let deployment = new Deployment(ctx.appId)
   deployment.setStatus(DeploymentStatus.PENDING)
-  deployment = await storeLocalDeploymentHistory(deployment)
+  deployment = await storeDeploymentHistory(deployment)
 
   await deployApp(deployment)
 

packages/server/src/api/controllers/deploy/quota.js

@@ -1,39 +0,0 @@
-const PouchDB = require("../../../db")
-const {
-  DocumentTypes,
-  SEPARATOR,
-  UNICODE_MAX,
-  ViewNames,
-} = require("../../../db/utils")
-
-exports.getAppQuota = async function (appId) {
-  const db = new PouchDB(appId)
-
-  const rows = await db.allDocs({
-    startkey: DocumentTypes.ROW + SEPARATOR,
-    endkey: DocumentTypes.ROW + SEPARATOR + UNICODE_MAX,
-  })
-
-  const users = await db.allDocs({
-    startkey: DocumentTypes.USER + SEPARATOR,
-    endkey: DocumentTypes.USER + SEPARATOR + UNICODE_MAX,
-  })
-
-  const existingRows = rows.rows.length
-  const existingUsers = users.rows.length
-
-  const designDoc = await db.get("_design/database")
-
-  let views = 0
-  for (let viewName of Object.keys(designDoc.views)) {
-    if (Object.values(ViewNames).indexOf(viewName) === -1) {
-      views++
-    }
-  }
-
-  return {
-    rows: existingRows,
-    users: existingUsers,
-    views: views,
-  }
-}

packages/server/src/api/controllers/deploy/selfDeploy.js

@@ -1,60 +0,0 @@
-const AWS = require("aws-sdk")
-const {
-  deployToObjectStore,
-  performReplication,
-  fetchCredentials,
-} = require("./utils")
-const {
-  getWorkerUrl,
-  getCouchUrl,
-  getSelfHostKey,
-} = require("../../../utilities/builder/hosting")
-
-exports.preDeployment = async function () {
-  const url = `${await getWorkerUrl()}/api/deploy`
-  try {
-    const json = await fetchCredentials(url, {
-      selfHostKey: await getSelfHostKey(),
-    })
-
-    // response contains:
-    // couchDbSession, bucket, objectStoreSession
-
-    // set credentials here, means any time we're verified we're ready to go
-    if (json.objectStoreSession) {
-      AWS.config.update({
-        accessKeyId: json.objectStoreSession.accessKeyId,
-        secretAccessKey: json.objectStoreSession.secretAccessKey,
-      })
-    }
-
-    return json
-  } catch (err) {
-    throw {
-      message: "Unauthorised to deploy, check self hosting key",
-      status: 401,
-    }
-  }
-}
-
-exports.postDeployment = async function () {
-  // we don't actively need to do anything after deployment in self hosting
-}
-
-exports.deploy = async function (deployment) {
-  const appId = deployment.getAppId()
-  const verification = deployment.getVerification()
-  // no metadata, aws has account ID in metadata
-  const metadata = {}
-  await deployToObjectStore(appId, verification.bucket, metadata)
-}
-
-exports.replicateDb = async function (deployment) {
-  const appId = deployment.getAppId()
-  const verification = deployment.getVerification()
-  return performReplication(
-    appId,
-    verification.couchDbSession,
-    await getCouchUrl()
-  )
-}

packages/server/src/api/controllers/deploy/utils.js

@@ -1,136 +0,0 @@
-const { join } = require("../../../utilities/centralPath")
-const fs = require("fs")
-const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
-const fetch = require("node-fetch")
-const PouchDB = require("../../../db")
-const CouchDB = require("pouchdb")
-const { upload } = require("../../../utilities/fileSystem")
-const { attachmentsRelativeURL } = require("../../../utilities")
-
-// TODO: everything in this file is to be removed
-
-function walkDir(dirPath, callback) {
-  for (let filename of fs.readdirSync(dirPath)) {
-    const filePath = `${dirPath}/${filename}`
-    const stat = fs.lstatSync(filePath)
-
-    if (stat.isFile()) {
-      callback(filePath)
-    } else {
-      walkDir(filePath, callback)
-    }
-  }
-}
-
-exports.fetchCredentials = async function (url, body) {
-  const response = await fetch(url, {
-    method: "POST",
-    body: JSON.stringify(body),
-    headers: { "Content-Type": "application/json" },
-  })
-
-  const json = await response.json()
-  if (json.errors) {
-    throw new Error(json.errors)
-  }
-
-  if (response.status !== 200) {
-    throw new Error(
-      `Error fetching temporary credentials: ${JSON.stringify(json)}`
-    )
-  }
-
-  return json
-}
-
-exports.prepareUpload = async function ({ s3Key, bucket, metadata, file }) {
-  const response = await upload({
-    bucket,
-    metadata,
-    filename: s3Key,
-    path: file.path,
-    type: file.type,
-  })
-
-  // don't store a URL, work this out on the way out as the URL could change
-  return {
-    size: file.size,
-    name: file.name,
-    url: attachmentsRelativeURL(response.Key),
-    extension: [...file.name.split(".")].pop(),
-    key: response.Key,
-  }
-}
-
-exports.deployToObjectStore = async function (appId, bucket, metadata) {
-  const appAssetsPath = join(budibaseAppsDir(), appId, "public")
-
-  let uploads = []
-
-  // Upload HTML, CSS and JS for each page of the web app
-  walkDir(appAssetsPath, function (filePath) {
-    const filePathParts = filePath.split("/")
-    const appAssetUpload = exports.prepareUpload({
-      bucket,
-      file: {
-        path: filePath,
-        name: filePathParts.pop(),
-      },
-      s3Key: filePath.replace(appAssetsPath, `assets/${appId}`),
-      metadata,
-    })
-    uploads.push(appAssetUpload)
-  })
-
-  // Upload file attachments
-  const db = new PouchDB(appId)
-  let fileUploads
-  try {
-    fileUploads = await db.get("_local/fileuploads")
-  } catch (err) {
-    fileUploads = { _id: "_local/fileuploads", uploads: [] }
-  }
-
-  for (let file of fileUploads.uploads) {
-    if (file.uploaded) continue
-
-    const attachmentUpload = exports.prepareUpload({
-      file,
-      s3Key: `assets/${appId}/attachments/${file.processedFileName}`,
-      bucket,
-      metadata,
-    })
-
-    uploads.push(attachmentUpload)
-
-    // mark file as uploaded
-    file.uploaded = true
-  }
-
-  db.put(fileUploads)
-
-  try {
-    return await Promise.all(uploads)
-  } catch (err) {
-    console.error("Error uploading budibase app assets to s3", err)
-    throw err
-  }
-}
-
-exports.performReplication = (appId, session, dbUrl) => {
-  return new Promise((resolve, reject) => {
-    const local = new PouchDB(appId)
-
-    const remote = new CouchDB(`${dbUrl}/${appId}`, {
-      fetch: function (url, opts) {
-        opts.headers.set("Cookie", `${session};`)
-        return CouchDB.fetch(url, opts)
-      },
-    })
-
-    const replication = local.sync(remote)
-
-    replication.on("complete", () => resolve())
-    replication.on("error", err => reject(err))
-  })
-}

packages/server/src/api/controllers/dev.js

@@ -1,7 +1,11 @@
 const fetch = require("node-fetch")
+const CouchDB = require("../../db")
 const env = require("../../environment")
 const { checkSlashesInUrl } = require("../../utilities")
 const { request } = require("../../utilities/workerRequests")
+const { clearLock } = require("../../utilities/redis")
+const { Replication } = require("@budibase/auth").db
+const { DocumentTypes } = require("../../db/utils")
 
 async function redirect(ctx, method) {
   const { devPath } = ctx.params
@@ -32,3 +36,49 @@ exports.redirectPost = async ctx => {
 exports.redirectDelete = async ctx => {
   await redirect(ctx, "DELETE")
 }
+
+exports.clearLock = async ctx => {
+  const { appId } = ctx.params
+  try {
+    await clearLock(appId, ctx.user)
+  } catch (err) {
+    ctx.throw(400, `Unable to remove lock. ${err}`)
+  }
+  ctx.body = {
+    message: "Lock released successfully.",
+  }
+}
+
+exports.revert = async ctx => {
+  const { appId } = ctx.params
+  const productionAppId = appId.replace("_dev", "")
+
+  // App must have been deployed first
+  try {
+    const db = new CouchDB(productionAppId, { skip_setup: true })
+    const info = await db.info()
+    if (info.error) throw info.error
+  } catch (err) {
+    return ctx.throw(400, "App has not yet been deployed")
+  }
+
+  try {
+    const replication = new Replication({
+      source: productionAppId,
+      target: appId,
+    })
+
+    await replication.rollback()
+    // update appID in reverted app to be dev version again
+    const db = new CouchDB(appId)
+    const appDoc = await db.get(DocumentTypes.APP_METADATA)
+    appDoc.appId = appId
+    appDoc.instance._id = appId
+    await db.put(appDoc)
+    ctx.body = {
+      message: "Reverted changes successfully.",
+    }
+  } catch (err) {
+    ctx.throw(400, `Unable to revert. ${err}`)
+  }
+}

packages/server/src/api/controllers/hosting.js

@@ -1,41 +1,19 @@
 const CouchDB = require("../../db")
-const {
+const { getDeployedApps } = require("../../utilities/workerRequests")
-  getHostingInfo,
+const { getScopedConfig } = require("@budibase/auth/db")
-  getDeployedApps,
+const { Configs } = require("@budibase/auth").constants
-  HostingTypes,
+const { checkSlashesInUrl } = require("../../utilities")
-  getAppUrl,
-} = require("../../utilities/builder/hosting")
-const { StaticDatabases } = require("../../db/utils")
-
-exports.fetchInfo = async ctx => {
-  ctx.body = {
-    types: Object.values(HostingTypes),
-  }
-}
-
-exports.save = async ctx => {
-  const db = new CouchDB(StaticDatabases.BUILDER_HOSTING.name)
-  const { type } = ctx.request.body
-  if (type === HostingTypes.CLOUD && ctx.request.body._rev) {
-    ctx.body = await db.remove({
-      ...ctx.request.body,
-      _id: StaticDatabases.BUILDER_HOSTING.baseDoc,
-    })
-  } else {
-    ctx.body = await db.put({
-      ...ctx.request.body,
-      _id: StaticDatabases.BUILDER_HOSTING.baseDoc,
-    })
-  }
-}
-
-exports.fetch = async ctx => {
-  ctx.body = await getHostingInfo()
-}
 
 exports.fetchUrls = async ctx => {
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
+  const settings = await getScopedConfig(db, { type: Configs.SETTINGS })
+  let appUrl = "http://localhost:10000/app"
+  if (settings && settings["platformUrl"]) {
+    appUrl = checkSlashesInUrl(`${settings["platformUrl"]}/app`)
+  }
   ctx.body = {
-    app: await getAppUrl(ctx.appId),
+    app: appUrl,
   }
 }
 

packages/server/src/api/controllers/permission.js

@@ -3,19 +3,19 @@ const {
   PermissionLevels,
   isPermissionLevelHigherThanRead,
   higherPermission,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const {
   isBuiltin,
   getDBRoleID,
   getExternalRoleID,
   getBuiltinRoles,
-} = require("../../utilities/security/roles")
+} = require("@budibase/auth/roles")
 const { getRoleParams } = require("../../db/utils")
 const CouchDB = require("../../db")
 const {
   CURRENTLY_SUPPORTED_LEVELS,
   getBasePermissions,
-} = require("../../utilities/security/utilities")
+} = require("../../utilities/security")
 
 const PermissionUpdateType = {
   REMOVE: "remove",

packages/server/src/api/controllers/query.js

@@ -58,6 +58,9 @@ async function enrichQueryFields(fields, parameters) {
 
   // enrich the fields with dynamic parameters
   for (let key of Object.keys(fields)) {
+    if (fields[key] == null) {
+      continue
+    }
     if (typeof fields[key] === "object") {
       // enrich nested fields object
       enrichedQuery[key] = await enrichQueryFields(fields[key], parameters)

packages/server/src/api/controllers/role.js

@@ -1,15 +1,12 @@
 const CouchDB = require("../../db")
 const {
-  getBuiltinRoles,
-  BUILTIN_ROLE_IDS,
   Role,
   getRole,
   isBuiltin,
-  getExternalRoleID,
+  getAllRoles,
-} = require("../../utilities/security/roles")
+} = require("@budibase/auth/roles")
 const {
   generateRoleID,
-  getRoleParams,
   getUserMetadataParams,
   InternalTables,
 } = require("../../db/utils")
@@ -19,14 +16,6 @@ const UpdateRolesOptions = {
   REMOVED: "removed",
 }
 
-// exclude internal roles like builder
-const EXTERNAL_BUILTIN_ROLE_IDS = [
-  BUILTIN_ROLE_IDS.ADMIN,
-  BUILTIN_ROLE_IDS.POWER,
-  BUILTIN_ROLE_IDS.BASIC,
-  BUILTIN_ROLE_IDS.PUBLIC,
-]
-
 async function updateRolesOnUserTable(db, roleId, updateOption) {
   const table = await db.get(InternalTables.USER_METADATA)
   const schema = table.schema
@@ -51,31 +40,7 @@ async function updateRolesOnUserTable(db, roleId, updateOption) {
 }
 
 exports.fetch = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  ctx.body = await getAllRoles(ctx.appId)
-  const body = await db.allDocs(
-    getRoleParams(null, {
-      include_docs: true,
-    })
-  )
-  let roles = body.rows.map(row => row.doc)
-  const builtinRoles = getBuiltinRoles()
-
-  // need to combine builtin with any DB record of them (for sake of permissions)
-  for (let builtinRoleId of EXTERNAL_BUILTIN_ROLE_IDS) {
-    const builtinRole = builtinRoles[builtinRoleId]
-    const dbBuiltin = roles.filter(
-      dbRole => getExternalRoleID(dbRole._id) === builtinRoleId
-    )[0]
-    if (dbBuiltin == null) {
-      roles.push(builtinRole)
-    } else {
-      // remove role and all back after combining with the builtin
-      roles = roles.filter(role => role._id !== dbBuiltin._id)
-      dbBuiltin._id = getExternalRoleID(dbBuiltin._id)
-      roles.push(Object.assign(builtinRole, dbBuiltin))
-    }
-  }
-  ctx.body = roles
 }
 
 exports.find = async function (ctx) {

packages/server/src/api/controllers/routing.js

@@ -2,7 +2,7 @@ const { getRoutingInfo } = require("../../utilities/routing")
 const {
   getUserRoleHierarchy,
   BUILTIN_ROLE_IDS,
-} = require("../../utilities/security/roles")
+} = require("@budibase/auth/roles")
 
 const URL_SEPARATOR = "/"
 

packages/server/src/api/controllers/screen.js

@@ -1,6 +1,6 @@
 const CouchDB = require("../../db")
 const { getScreenParams, generateScreenID } = require("../../db/utils")
-const { AccessController } = require("../../utilities/security/roles")
+const { AccessController } = require("@budibase/auth/roles")
 
 exports.fetch = async ctx => {
   const appId = ctx.appId

packages/server/src/api/controllers/script.js

@@ -9,14 +9,16 @@ class ScriptExecutor {
   }
 
   execute() {
-    const returnValue = this.script.runInContext(this.context)
+    return this.script.runInContext(this.context)
-    return returnValue
   }
 }
 
 exports.execute = async function (ctx) {
   const executor = new ScriptExecutor(ctx.request.body)
 
-  const result = executor.execute()
+  ctx.body = executor.execute()
-  ctx.body = result
+}
+
+exports.save = async function (ctx) {
+  ctx.throw(501, "Not currently implemented")
 }

packages/server/src/api/controllers/static/index.js

@@ -5,10 +5,9 @@ const { resolve, join } = require("../../../utilities/centralPath")
 const fetch = require("node-fetch")
 const uuid = require("uuid")
 const { ObjectStoreBuckets } = require("../../../constants")
-const { prepareUpload } = require("../deploy/utils")
 const { processString } = require("@budibase/string-templates")
 const { budibaseTempDir } = require("../../../utilities/budibaseDir")
-const { getDeployedApps } = require("../../../utilities/builder/hosting")
+const { getDeployedApps } = require("../../../utilities/workerRequests")
 const CouchDB = require("../../../db")
 const {
   loadHandlebarsFile,
@@ -17,6 +16,28 @@ const {
 } = require("../../../utilities/fileSystem")
 const env = require("../../../environment")
 const { objectStoreUrl, clientLibraryPath } = require("../../../utilities")
+const { upload } = require("../../../utilities/fileSystem")
+const { attachmentsRelativeURL } = require("../../../utilities")
+const { DocumentTypes } = require("../../../db/utils")
+
+async function prepareUpload({ s3Key, bucket, metadata, file }) {
+  const response = await upload({
+    bucket,
+    metadata,
+    filename: s3Key,
+    path: file.path,
+    type: file.type,
+  })
+
+  // don't store a URL, work this out on the way out as the URL could change
+  return {
+    size: file.size,
+    name: file.name,
+    url: attachmentsRelativeURL(response.Key),
+    extension: [...file.name.split(".")].pop(),
+    key: response.Key,
+  }
+}
 
 async function checkForSelfHostedURL(ctx) {
   // the "appId" component of the URL may actually be a specific self hosted URL
@@ -65,7 +86,7 @@ exports.serveApp = async function (ctx) {
   }
   const App = require("./templates/BudibaseApp.svelte").default
   const db = new CouchDB(appId, { skip_setup: true })
-  const appInfo = await db.get(appId)
+  const appInfo = await db.get(DocumentTypes.APP_METADATA)
 
   const { head, html, css } = App.render({
     title: appInfo.name,
@@ -95,13 +116,17 @@ exports.serveComponentLibrary = async function (ctx) {
   if (env.isDev() || env.isTest()) {
     const componentLibraryPath = join(
       budibaseTempDir(),
-      decodeURI(ctx.query.library),
+      appId,
+      "node_modules",
+      "@budibase",
+      "standard-components",
+      "package",
       "dist"
     )
-    return send(ctx, "/awsDeploy.js", { root: componentLibraryPath })
+    return send(ctx, "/index.js", { root: componentLibraryPath })
   }
   const db = new CouchDB(appId)
-  const appInfo = await db.get(appId)
+  const appInfo = await db.get(DocumentTypes.APP_METADATA)
 
   let componentLib = "componentlibrary"
   if (appInfo && appInfo.version) {

packages/server/src/api/controllers/user.js

@@ -5,7 +5,7 @@ const {
   getGlobalIDFromUserMetadataID,
 } = require("../../db/utils")
 const { InternalTables } = require("../../db/utils")
-const { getRole } = require("../../utilities/security/roles")
+const { getRole, BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const {
   getGlobalUsers,
   saveGlobalUser,
@@ -73,6 +73,9 @@ exports.createMetadata = async function (ctx) {
 exports.updateSelfMetadata = async function (ctx) {
   // overwrite the ID with current users
   ctx.request.body._id = ctx.user._id
+  if (ctx.user.builder && ctx.user.builder.global) {
+    ctx.request.body.roleId = BUILTIN_ROLE_IDS.ADMIN
+  }
   // make sure no stale rev
   delete ctx.request.body._rev
   await exports.updateMetadata(ctx)

packages/server/src/api/routes/analytics.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const authorized = require("../../middleware/authorized")
 const controller = require("../controllers/analytics")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/apikeys.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/apikeys")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/application.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/application")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/automation.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const Joi = require("joi")
 const { bodyResource, paramResource } = require("../../middleware/resourceId")
 

packages/server/src/api/routes/backup.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/backup")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/component.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/component")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/datasource.js

@@ -5,7 +5,7 @@ const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/deploy.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/deploy")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/dev.js

@@ -1,6 +1,8 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/dev")
 const env = require("../../environment")
+const authorized = require("../../middleware/authorized")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 
@@ -11,4 +13,8 @@ if (env.isDev() || env.isTest()) {
     .delete("/api/admin/:devPath(.*)", controller.redirectDelete)
 }
 
+router
+  .delete("/api/dev/:appId/lock", authorized(BUILDER), controller.clearLock)
+  .post("/api/dev/:appId/revert", authorized(BUILDER), controller.revert)
+
 module.exports = router

packages/server/src/api/routes/hosting.js

@@ -2,15 +2,12 @@ const Router = require("@koa/router")
 const controller = require("../controllers/hosting")
 const authorized = require("../../middleware/authorized")
 const selfhost = require("../../middleware/selfhost")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 
 router
-  .get("/api/hosting/info", authorized(BUILDER), controller.fetchInfo)
   .get("/api/hosting/urls", authorized(BUILDER), controller.fetchUrls)
-  .get("/api/hosting", authorized(BUILDER), controller.fetch)
-  .post("/api/hosting", authorized(BUILDER), controller.save)
   // this isn't risky, doesn't return anything about apps other than names and URLs
   .get("/api/hosting/apps", selfhost, controller.getDeployedApps)
 

packages/server/src/api/routes/integration.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/integration")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/layout.js

@@ -1,6 +1,6 @@
 const Router = require("@koa/router")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 const controller = require("../controllers/layout")
 
 const router = Router()

packages/server/src/api/routes/permission.js

@@ -1,10 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/permission")
 const authorized = require("../../middleware/authorized")
-const {
+const { BUILDER, PermissionLevels } = require("@budibase/auth/permissions")
-  BUILDER,
-  PermissionLevels,
-} = require("../../utilities/security/permissions")
 const Joi = require("joi")
 const joiValidator = require("../../middleware/joi-validator")
 

packages/server/src/api/routes/query.js

@@ -1,12 +1,12 @@
 const Router = require("@koa/router")
 const queryController = require("../controllers/query")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
 const Joi = require("joi")
 const {
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+  BUILDER,
+} = require("@budibase/auth/permissions")
 const joiValidator = require("../../middleware/joi-validator")
 const {
   bodyResource,

packages/server/src/api/routes/role.js

@@ -1,15 +1,13 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/role")
 const authorized = require("../../middleware/authorized")
-const {
-  BUILDER,
-  PermissionLevels,
-} = require("../../utilities/security/permissions")
 const Joi = require("joi")
 const joiValidator = require("../../middleware/joi-validator")
 const {
   BUILTIN_PERMISSION_IDS,
-} = require("../../utilities/security/permissions")
+  BUILDER,
+  PermissionLevels,
+} = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/routing.js

@@ -1,6 +1,6 @@
 const Router = require("@koa/router")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 const controller = require("../controllers/routing")
 
 const router = Router()

packages/server/src/api/routes/row.js

@@ -9,7 +9,7 @@ const {
 const {
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/screen.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/screen")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 const joiValidator = require("../../middleware/joi-validator")
 const Joi = require("joi")
 

packages/server/src/api/routes/script.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
-const controller = require("../controllers/hosting")
+const controller = require("../controllers/script")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/search.js

@@ -3,7 +3,7 @@ const controller = require("../controllers/search")
 const {
   PermissionTypes,
   PermissionLevels,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const authorized = require("../../middleware/authorized")
 const { paramResource } = require("../../middleware/resourceId")
 

packages/server/src/api/routes/static.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionTypes,
   PermissionLevels,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const usage = require("../../middleware/usageQuota")
 const env = require("../../environment")
 

packages/server/src/api/routes/table.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const joiValidator = require("../../middleware/joi-validator")
 const Joi = require("joi")
 

packages/server/src/api/routes/templates.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/templates")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/tests/application.spec.js

@@ -1,6 +1,14 @@
 const { clearAllApps, checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
 
+jest.mock("../../../utilities/redis", () => ({
+  init: jest.fn(),
+  getAllLocks: () => {
+    return []
+  },
+  updateLock: jest.fn(),
+}))
+
 describe("/applications", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
@@ -40,7 +48,7 @@ describe("/applications", () => {
       await config.createApp(request, "app2")
 
       const res = await request
-        .get("/api/applications")
+        .get("/api/applications?status=dev")
         .set(config.defaultHeaders())
         .expect('Content-Type', /json/)
         .expect(200)

packages/server/src/api/routes/tests/hosting.spec.js

@@ -15,25 +15,6 @@ describe("/hosting", () => {
     app = await config.init()
   })
 
-  describe("fetchInfo", () => {
-    it("should be able to fetch hosting information", async () => {
-      const res = await request
-        .get(`/api/hosting/info`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body).toEqual({ types: ["cloud", "self"]})
-    })
-
-    it("should apply authorization to endpoint", async () => {
-      await checkBuilderEndpoint({
-        config,
-        method: "GET",
-        url: `/api/hosting/info`,
-      })
-    })
-  })
-
   describe("fetchUrls", () => {
     it("should be able to fetch current app URLs", async () => {
       const res = await request
@@ -41,7 +22,7 @@ describe("/hosting", () => {
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
-      expect(res.body.app).toEqual(`https://${config.getAppId()}.app.budi.live`)
+      expect(res.body.app).toEqual(`http://localhost:10000/app`)
     })
 
     it("should apply authorization to endpoint", async () => {
@@ -52,78 +33,4 @@ describe("/hosting", () => {
       })
     })
   })
-
-  describe("fetch", () => {
-    it("should be able to fetch the current hosting information", async () => {
-      const res = await request
-        .get(`/api/hosting`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body._id).toBeDefined()
-      expect(res.body.hostingUrl).toBeDefined()
-      expect(res.body.type).toEqual("cloud")
-    })
-
-    it("should apply authorization to endpoint", async () => {
-      await checkBuilderEndpoint({
-        config,
-        method: "GET",
-        url: `/api/hosting`,
-      })
-    })
-  })
-
-  describe("save", () => {
-    it("should be able to update the hosting information", async () => {
-      const res = await request
-        .post(`/api/hosting`)
-        .send({
-          type: "self",
-          selfHostKey: "budibase",
-          hostingUrl: "localhost:10000",
-          useHttps: false,
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.ok).toEqual(true)
-      // make sure URL updated
-      const urlRes = await request
-        .get(`/api/hosting/urls`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(urlRes.body.app).toEqual(`http://localhost:10000/app`)
-    })
-
-    it("should apply authorization to endpoint", async () => {
-      await checkBuilderEndpoint({
-        config,
-        method: "POST",
-        url: `/api/hosting`,
-      })
-    })
-  })
-
-  describe("getDeployedApps", () => {
-    it("should fail when not self hosted", async () => {
-      await request
-        .get(`/api/hosting/apps`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(400)
-    })
-
-    it("should get apps when in cloud", async () => {
-      await setup.switchToSelfHosted(async () => {
-        const res = await request
-          .get(`/api/hosting/apps`)
-          .set(config.defaultHeaders())
-          .expect("Content-Type", /json/)
-          .expect(200)
-        expect(res.body.app1).toEqual({url: "/app1"})
-      })
-    })
-  })
 })

packages/server/src/api/routes/tests/permissions.spec.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("../../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const setup = require("./utilities")
 const { basicRow } = setup.structures
 

packages/server/src/api/routes/tests/role.spec.js

@@ -1,7 +1,7 @@
-const { BUILTIN_ROLE_IDS } = require("../../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const {
   BUILTIN_PERMISSION_IDS,
-} = require("../../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const setup = require("./utilities")
 const { basicRole } = setup.structures
 

packages/server/src/api/routes/tests/routing.spec.js

@@ -1,7 +1,7 @@
 const setup = require("./utilities")
 const { basicScreen } = setup.structures
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
-const { BUILTIN_ROLE_IDS } = require("../../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const workerRequests = require("../../../utilities/workerRequests")
 
 const route = "/test"

packages/server/src/api/routes/tests/user.spec.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("../../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { checkPermissionsEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
 const { basicUser } = setup.structures

packages/server/src/api/routes/tests/utilities/TestFunctions.js

@@ -14,14 +14,14 @@ exports.getAllTableRows = async config => {
 }
 
 exports.clearAllApps = async () => {
-  const req = {}
+  const req = { query: { status: "dev" } }
   await appController.fetch(req)
   const apps = req.body
   if (!apps || apps.length <= 0) {
     return
   }
   for (let app of apps) {
-    const appId = app._id
+    const { appId } = app
     await appController.delete(new Request(null, { appId }))
   }
 }

packages/server/src/api/routes/user.js

@@ -4,7 +4,7 @@ const authorized = require("../../middleware/authorized")
 const {
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const usage = require("../../middleware/usageQuota")
 
 const router = Router()

packages/server/src/api/routes/view.js

@@ -7,7 +7,7 @@ const {
   BUILDER,
   PermissionTypes,
   PermissionLevels,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const usage = require("../../middleware/usageQuota")
 
 const router = Router()

packages/server/src/api/routes/webhook.js

@@ -2,7 +2,7 @@ const Router = require("@koa/router")
 const controller = require("../controllers/webhook")
 const authorized = require("../../middleware/authorized")
 const joiValidator = require("../../middleware/joi-validator")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 const Joi = require("joi")
 
 const router = Router()

packages/server/src/app.js

@@ -13,6 +13,7 @@ const automations = require("./automations/index")
 const Sentry = require("@sentry/node")
 const fileSystem = require("./utilities/fileSystem")
 const bullboard = require("./automations/bullboard")
+const redis = require("./utilities/redis")
 
 const app = new Koa()
 
@@ -84,6 +85,7 @@ module.exports = server.listen(env.PORT || 0, async () => {
   eventEmitter.emitPort(env.PORT)
   fileSystem.init()
   await automations.init()
+  await redis.init()
 })
 
 process.on("uncaughtException", err => {

packages/server/src/automations/steps/createUser.js

@@ -1,4 +1,4 @@
-const roles = require("../../utilities/security/roles")
+const roles = require("@budibase/auth/roles")
 const userController = require("../../api/controllers/user")
 const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")

packages/server/src/automations/tests/createUser.spec.js

@@ -1,6 +1,6 @@
 const usageQuota = require("../../utilities/usageQuota")
 const setup = require("./utilities")
-const { BUILTIN_ROLE_IDS } = require("../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { InternalTables } = require("../../db/utils")
 
 jest.mock("../../utilities/usageQuota")

packages/server/src/automations/triggers.js

@@ -6,7 +6,7 @@ const Queue = env.isTest()
   : require("bull")
 const { getAutomationParams } = require("../db/utils")
 const { coerce } = require("../utilities/rowProcessor")
-const { utils } = require("@budibase/auth").redis
+const { utils } = require("@budibase/auth/redis")
 
 const { opts } = utils.getRedisOptions()
 let automationQueue = new Queue("automationQueue", { redis: opts })

packages/server/src/constants/index.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { UserStatus } = require("@budibase/auth").constants
 const { ObjectStoreBuckets } = require("@budibase/auth").objectStore
 

packages/server/src/constants/screens.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { BASE_LAYOUT_PROP_IDS } = require("./layouts")
 const { LOGO_URL } = require("../constants")
 

packages/server/src/db/utils.js

@@ -1,28 +1,37 @@
 const newid = require("./newid")
+const {
+  DocumentTypes: CoreDocTypes,
+  getRoleParams,
+  generateRoleID,
+  APP_DEV_PREFIX,
+  APP_PREFIX,
+  SEPARATOR,
+} = require("@budibase/auth/db")
 
 const UNICODE_MAX = "\ufff0"
-const SEPARATOR = "_"
 
 const StaticDatabases = {
   BUILDER: {
     name: "builder-db",
     baseDoc: "builder-doc",
   },
-  // TODO: needs removed
+}
-  BUILDER_HOSTING: {
+
-    name: "builder-config-db",
+const AppStatus = {
-    baseDoc: "hosting-doc",
+  DEV: "dev",
-  },
+  DEPLOYED: "PUBLISHED",
 }
 
 const DocumentTypes = {
+  APP: CoreDocTypes.APP,
+  APP_DEV: CoreDocTypes.APP_DEV,
+  APP_METADATA: CoreDocTypes.APP_METADATA,
+  ROLE: CoreDocTypes.ROLE,
   TABLE: "ta",
   ROW: "ro",
   USER: "us",
   AUTOMATION: "au",
   LINK: "li",
-  APP: "app",
-  ROLE: "role",
   WEBHOOK: "wh",
   INSTANCE: "inst",
   LAYOUT: "layout",
@@ -44,6 +53,8 @@ const SearchIndexes = {
   ROWS: "rows",
 }
 
+exports.APP_PREFIX = APP_PREFIX
+exports.APP_DEV_PREFIX = APP_DEV_PREFIX
 exports.StaticDatabases = StaticDatabases
 exports.ViewNames = ViewNames
 exports.InternalTables = InternalTables
@@ -51,6 +62,10 @@ exports.DocumentTypes = DocumentTypes
 exports.SEPARATOR = SEPARATOR
 exports.UNICODE_MAX = UNICODE_MAX
 exports.SearchIndexes = SearchIndexes
+exports.AppStatus = AppStatus
+
+exports.generateRoleID = generateRoleID
+exports.getRoleParams = getRoleParams
 
 exports.getQueryIndex = viewName => {
   return `database/${viewName}`
@@ -143,9 +158,11 @@ exports.generateUserMetadataID = globalId => {
  * Breaks up the ID to get the global ID.
  */
 exports.getGlobalIDFromUserMetadataID = id => {
-  return id.split(
+  const prefix = `${DocumentTypes.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
-    `${DocumentTypes.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
+  if (!id.includes(prefix)) {
-  )[1]
+    return id
+  }
+  return id.split(prefix)[1]
 }
 
 /**
@@ -204,25 +221,13 @@ exports.generateAppID = () => {
 }
 
 /**
- * Gets parameters for retrieving apps, this is a utility function for the getDocParams function.
+ * Generates a development app ID from a real app ID.
+ * @returns {string} the dev app ID which can be used for dev database.
  */
-exports.getAppParams = (appId = null, otherProps = {}) => {
+exports.generateDevAppID = appId => {
-  return getDocParams(DocumentTypes.APP, appId, otherProps)
+  const prefix = `${DocumentTypes.APP}${SEPARATOR}`
-}
+  const uuid = appId.split(prefix)[1]
-
+  return `${DocumentTypes.APP_DEV}${SEPARATOR}${uuid}`
-/**
- * Generates a new role ID.
- * @returns {string} The new role ID which the role doc can be stored under.
- */
-exports.generateRoleID = id => {
-  return `${DocumentTypes.ROLE}${SEPARATOR}${id || newid()}`
-}
-
-/**
- * Gets parameters for retrieving a role, this is a utility function for the getDocParams function.
- */
-exports.getRoleParams = (roleId = null, otherProps = {}) => {
-  return getDocParams(DocumentTypes.ROLE, roleId, otherProps)
 }
 
 /**

packages/server/src/middleware/authorized.js

@@ -1,9 +1,11 @@
-const { getUserPermissions } = require("../utilities/security/roles")
+const { getUserPermissions } = require("@budibase/auth/roles")
 const {
   PermissionTypes,
   doesHaveResourcePermission,
   doesHaveBasePermission,
-} = require("../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
+const { APP_DEV_PREFIX } = require("../db/utils")
+const { doesUserHaveLock, updateLock } = require("../utilities/redis")
 
 function hasResource(ctx) {
   return ctx.resourceId != null
@@ -13,6 +15,21 @@ const WEBHOOK_ENDPOINTS = new RegExp(
   ["webhooks/trigger", "webhooks/schema"].join("|")
 )
 
+async function checkDevAppLocks(ctx) {
+  const appId = ctx.appId
+
+  // not a development app, don't need to do anything
+  if (!appId || !appId.startsWith(APP_DEV_PREFIX)) {
+    return
+  }
+  if (!(await doesUserHaveLock(appId, ctx.user))) {
+    ctx.throw(403, "User does not hold app lock.")
+  }
+
+  // they do have lock, update it
+  await updateLock(appId, ctx.user)
+}
+
 module.exports = (permType, permLevel = null) => async (ctx, next) => {
   // webhooks don't need authentication, each webhook unique
   if (WEBHOOK_ENDPOINTS.test(ctx.request.url)) {
@@ -23,8 +40,15 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => {
     return ctx.throw(403, "No user info found")
   }
 
-  const isAuthed = ctx.isAuthenticated
+  const builderCall = permType === PermissionTypes.BUILDER
+  const referer = ctx.headers["referer"]
+  const editingApp = referer ? referer.includes(ctx.appId) : false
+  // this makes sure that builder calls abide by dev locks
+  if (builderCall && editingApp) {
+    await checkDevAppLocks(ctx)
+  }
 
+  const isAuthed = ctx.isAuthenticated
   const { basePermissions, permissions } = await getUserPermissions(
     ctx.appId,
     ctx.roleId
@@ -35,7 +59,7 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => {
   let isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
   if (isBuilder) {
     return next()
-  } else if (permType === PermissionTypes.BUILDER && !isBuilder) {
+  } else if (builderCall && !isBuilder) {
     return ctx.throw(403, "Not Authorized")
   }
 

packages/server/src/middleware/currentapp.js

@@ -1,8 +1,8 @@
 const { getAppId, setCookie, getCookie } = require("@budibase/auth").utils
 const { Cookies } = require("@budibase/auth").constants
-const { getRole } = require("../utilities/security/roles")
+const { getRole } = require("@budibase/auth/roles")
 const { getGlobalUsers } = require("../utilities/workerRequests")
-const { BUILTIN_ROLE_IDS } = require("../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { generateUserMetadataID } = require("../db/utils")
 
 module.exports = async (ctx, next) => {
@@ -31,9 +31,8 @@ module.exports = async (ctx, next) => {
     const globalUser = await getGlobalUsers(ctx, requestAppId, ctx.user._id)
     updateCookie = true
     appId = requestAppId
-    if (globalUser.roles && globalUser.roles[requestAppId]) {
+    // retrieving global user gets the right role
-      roleId = globalUser.roles[requestAppId]
+    roleId = globalUser.roleId
-    }
   } else if (appCookie != null) {
     appId = appCookie.appId
     roleId = appCookie.roleId || BUILTIN_ROLE_IDS.PUBLIC

packages/server/src/middleware/tests/authorized.spec.js

@@ -1,6 +1,6 @@
 const authorizedMiddleware = require("../authorized")
 const env = require("../../environment")
-const { PermissionTypes, PermissionLevels } = require("../../utilities/security/permissions")
+const { PermissionTypes, PermissionLevels } = require("@budibase/auth/permissions")
 jest.mock("../../environment", () => ({
     prod: false,
     isTest: () => true,

packages/server/src/middleware/tests/currentapp.spec.js

@@ -8,7 +8,8 @@ function mockWorker() {
         _id: "us_uuid1",
         roles: {
           "app_test": "BASIC",
-        }
+        },
+        roleId: "BASIC",
       }
     }
   }))

packages/server/src/middleware/tests/selfhost.spec.js

@@ -1,7 +1,6 @@
 const selfHostMiddleware = require("../selfhost")
 const env = require("../../environment")
 jest.mock("../../environment")
-jest.mock("../../utilities/builder/hosting") 
 
 class TestConfiguration {
   constructor() {

packages/server/src/tests/utilities/TestConfiguration.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const env = require("../../environment")
 const {
   basicTable,
@@ -16,7 +16,7 @@ const supertest = require("supertest")
 const { cleanup } = require("../../utilities/fileSystem")
 const { Cookies } = require("@budibase/auth").constants
 const { jwt } = require("@budibase/auth").auth
-const { StaticDatabases } = require("@budibase/auth").db
+const { StaticDatabases } = require("@budibase/auth/db")
 const CouchDB = require("../../db")
 
 const GLOBAL_USER_ID = "us_uuid1"
@@ -89,7 +89,7 @@ class TestConfiguration {
     if (this.server) {
       this.server.close()
     }
-    cleanup(this.allApps.map(app => app._id))
+    cleanup(this.allApps.map(app => app.appId))
   }
 
   defaultHeaders() {
@@ -141,7 +141,7 @@ class TestConfiguration {
 
   async createApp(appName) {
     this.app = await this._req({ name: appName }, null, controllers.app.create)
-    this.appId = this.app._id
+    this.appId = this.app.appId
     this.allApps.push(this.app)
     return this.app
   }