Merge branch 'next' into admin/user-management-ui

hosting/docker-compose.dev.yaml

@@ -59,6 +59,7 @@ services:
     container_name: budi-redis-dev
     restart: always
     image: redis
+    command: redis-server --requirepass ${REDIS_PASSWORD}
     ports:
       - "${REDIS_PORT}:6379"
     volumes:

hosting/docker-compose.yaml

@@ -23,6 +23,8 @@ services:
       LOG_LEVEL: info
       SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
       ENABLE_ANALYTICS: "true"
+      REDIS_URL: redis-service:6379
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
     depends_on:
       - worker-service
 
@@ -43,6 +45,8 @@ services:
       COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
       COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
       SELF_HOST_KEY: ${HOSTING_KEY}
+      REDIS_URL: redis-service:6379
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
     depends_on:
       - minio-service
       - couch-init
@@ -100,8 +104,7 @@ services:
   redis-service:
     restart: always
     image: redis
-    ports:
-      - "${REDIS_PORT}:6379"
+    command: redis-server --requirepass ${REDIS_PASSWORD}
     volumes:
       - redis_data:/data
 

hosting/envoy.dev.yaml.hbs

@@ -21,11 +21,6 @@ static_resources:
                   cluster: couchdb-service
                   prefix_rewrite: "/"
 
-              - match: { prefix: "/cache/" }
-                route:
-                  cluster: redis-service
-                  prefix_rewrite: "/"
-
               - match: { prefix: "/api/admin/" }
                 route:
                   cluster: worker-dev
@@ -85,20 +80,6 @@ static_resources:
                 address: couchdb-service
                 port_value: 5984
 
-  - name: redis-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: redis-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: redis-service
-                port_value: 6379
-
   - name: server-dev
     connect_timeout: 0.25s
     type: strict_dns

hosting/envoy.yaml

@@ -41,11 +41,6 @@ static_resources:
                   cluster: worker-service
                   prefix_rewrite: "/"
 
-              - match: { prefix: "/cache/" }
-                route:
-                  cluster: redis-service
-                  prefix_rewrite: "/"
-
               - match: { prefix: "/db/" }
                 route:
                   cluster: couchdb-service
@@ -117,18 +112,3 @@ static_resources:
                 address: couchdb-service
                 port_value: 5984
 
-  - name: redis-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: redis-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: redis-service
-                port_value: 6379
-
-

hosting/hosting.properties

@@ -12,6 +12,7 @@ MINIO_ACCESS_KEY=budibase
 MINIO_SECRET_KEY=budibase
 COUCH_DB_PASSWORD=budibase
 COUCH_DB_USER=budibase
+REDIS_PASSWORD=budibase
 
 # This section contains variables that do not need to be altered under normal circumstances
 APP_PORT=4002

packages/auth/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@budibase/auth",
-  "version": "0.0.1",
+  "version": "0.18.6",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",
   "license": "AGPL-3.0",
   "dependencies": {
     "bcryptjs": "^2.4.3",
+    "ioredis": "^4.27.1",
     "jsonwebtoken": "^8.5.1",
     "koa-passport": "^4.1.4",
     "passport-google-auth": "^1.0.2",
@@ -14,5 +15,8 @@
     "passport-jwt": "^4.0.0",
     "passport-local": "^1.0.0",
     "uuid": "^8.3.2"
+  },
+  "devDependencies": {
+    "ioredis-mock": "^5.5.5"
   }
 }

packages/auth/src/db/utils.js

@@ -123,7 +123,7 @@ const getConfigParams = ({ type, group, user }, otherProps = {}) => {
  * @param {Object} scopes - the type, group and userID scopes of the configuration.
  * @returns The most granular configuration document based on the scope.
  */
-const determineScopedConfig = async function (db, { type, user, group }) {
+const getScopedFullConfig = async function (db, { type, user, group }) {
   const response = await db.allDocs(
     getConfigParams(
       { type, user, group },
@@ -160,6 +160,12 @@ const determineScopedConfig = async function (db, { type, user, group }) {
   return scopedConfig.doc
 }
 
+async function getScopedConfig(db, params) {
+  const configDoc = await getScopedFullConfig(db, params)
+  return configDoc && configDoc.config ? configDoc.config : configDoc
+}
+
+exports.getScopedConfig = getScopedConfig
 exports.generateConfigID = generateConfigID
 exports.getConfigParams = getConfigParams
-exports.determineScopedConfig = determineScopedConfig
+exports.getScopedFullConfig = getScopedFullConfig

packages/auth/src/environment.js

@@ -1,5 +1,16 @@
+function isTest() {
+  return (
+    process.env.NODE_ENV === "jest" ||
+    process.env.NODE_ENV === "cypress" ||
+    process.env.JEST_WORKER_ID != null
+  )
+}
+
 module.exports = {
   JWT_SECRET: process.env.JWT_SECRET,
   COUCH_DB_URL: process.env.COUCH_DB_URL,
   SALT_ROUNDS: process.env.SALT_ROUNDS,
+  REDIS_URL: process.env.REDIS_URL,
+  REDIS_PASSWORD: process.env.REDIS_PASSWORD,
+  isTest,
 }

packages/auth/src/index.js

@@ -28,6 +28,10 @@ module.exports = {
     setDB(pouch)
   },
   db: require("./db/utils"),
+  redis: {
+    Client: require("./redis"),
+    utils: require("./redis/utils"),
+  },
   utils: {
     ...require("./utils"),
     ...require("./hashing"),

packages/auth/src/middleware/authenticated.js

@@ -3,11 +3,35 @@ const database = require("../db")
 const { getCookie, clearCookie } = require("../utils")
 const { StaticDatabases } = require("../db/utils")
 
-module.exports = (noAuthPatterns = []) => {
-  const regex = new RegExp(noAuthPatterns.join("|"))
+const PARAM_REGEX = /\/:(.*?)\//g
+
+function buildNoAuthRegex(patterns) {
+  return patterns.map(pattern => {
+    const isObj = typeof pattern === "object" && pattern.route
+    const method = isObj ? pattern.method : "GET"
+    let route = isObj ? pattern.route : pattern
+
+    const matches = route.match(PARAM_REGEX)
+    if (matches) {
+      for (let match of matches) {
+        route = route.replace(match, "/.*/")
+      }
+    }
+    return { regex: new RegExp(route), method }
+  })
+}
+
+module.exports = (noAuthPatterns = [], opts) => {
+  const noAuthOptions = noAuthPatterns ? buildNoAuthRegex(noAuthPatterns) : []
   return async (ctx, next) => {
     // the path is not authenticated
-    if (regex.test(ctx.request.url)) {
+    const found = noAuthOptions.find(({ regex, method }) => {
+      return (
+        regex.test(ctx.request.url) &&
+        ctx.request.method.toLowerCase() === method.toLowerCase()
+      )
+    })
+    if (found != null) {
       return next()
     }
     try {
@@ -30,10 +54,14 @@ module.exports = (noAuthPatterns = []) => {
       if (ctx.isAuthenticated !== true) {
         ctx.isAuthenticated = false
       }
-
       return next()
     } catch (err) {
-      ctx.throw(err.status || 403, err)
+      // allow configuring for public access
+      if (opts && opts.publicAllowed) {
+        ctx.isAuthenticated = false
+      } else {
+        ctx.throw(err.status || 403, err)
+      }
     }
   }
 }

packages/auth/src/redis/index.js

@@ -0,0 +1,152 @@
+const env = require("../environment")
+// ioredis mock is all in memory
+const Redis = env.isTest() ? require("ioredis-mock") : require("ioredis")
+const { addDbPrefix, removeDbPrefix, getRedisOptions } = require("./utils")
+
+const CLUSTERED = false
+
+// for testing just generate the client once
+let CLIENT = env.isTest() ? new Redis(getRedisOptions()) : null
+
+/**
+ * Inits the system, will error if unable to connect to redis cluster (may take up to 10 seconds) otherwise
+ * will return the ioredis client which will be ready to use.
+ * @return {Promise<object>} The ioredis client.
+ */
+function init() {
+  return new Promise((resolve, reject) => {
+    // testing uses a single in memory client
+    if (env.isTest()) {
+      return resolve(CLIENT)
+    }
+    // if a connection existed, close it and re-create it
+    if (CLIENT) {
+      CLIENT.disconnect()
+      CLIENT = null
+    }
+    const { opts, host, port } = getRedisOptions(CLUSTERED)
+    if (CLUSTERED) {
+      CLIENT = new Redis.Cluster([{ host, port }], opts)
+    } else {
+      CLIENT = new Redis(opts)
+    }
+    CLIENT.on("end", err => {
+      reject(err)
+    })
+    CLIENT.on("error", err => {
+      reject(err)
+    })
+    CLIENT.on("connect", () => {
+      resolve(CLIENT)
+    })
+  })
+}
+
+/**
+ * Utility function, takes a redis stream and converts it to a promisified response -
+ * this can only be done with redis streams because they will have an end.
+ * @param stream A redis stream, specifically as this type of stream will have an end.
+ * @return {Promise<object>} The final output of the stream
+ */
+function promisifyStream(stream) {
+  return new Promise((resolve, reject) => {
+    const outputKeys = new Set()
+    stream.on("data", keys => {
+      keys.forEach(key => {
+        outputKeys.add(key)
+      })
+    })
+    stream.on("error", err => {
+      reject(err)
+    })
+    stream.on("end", async () => {
+      const keysArray = Array.from(outputKeys)
+      try {
+        let getPromises = []
+        for (let key of keysArray) {
+          getPromises.push(CLIENT.get(key))
+        }
+        const jsonArray = await Promise.all(getPromises)
+        resolve(
+          keysArray.map(key => ({
+            key: removeDbPrefix(key),
+            value: JSON.parse(jsonArray.shift()),
+          }))
+        )
+      } catch (err) {
+        reject(err)
+      }
+    })
+  })
+}
+
+class RedisWrapper {
+  constructor(db) {
+    this._db = db
+  }
+
+  async init() {
+    this._client = await init()
+    return this
+  }
+
+  async finish() {
+    this._client.disconnect()
+  }
+
+  async scan() {
+    const db = this._db,
+      client = this._client
+    let stream
+    if (CLUSTERED) {
+      let node = client.nodes("master")
+      stream = node[0].scanStream({ match: db + "-*", count: 100 })
+    } else {
+      stream = client.scanStream({ match: db + "-*", count: 100 })
+    }
+    return promisifyStream(stream)
+  }
+
+  async get(key) {
+    const db = this._db,
+      client = this._client
+    let response = await client.get(addDbPrefix(db, key))
+    // overwrite the prefixed key
+    if (response != null && response.key) {
+      response.key = key
+    }
+    // if its not an object just return the response
+    try {
+      return JSON.parse(response)
+    } catch (err) {
+      return response
+    }
+  }
+
+  async store(key, value, expirySeconds = null) {
+    const db = this._db,
+      client = this._client
+    if (typeof value === "object") {
+      value = JSON.stringify(value)
+    }
+    const prefixedKey = addDbPrefix(db, key)
+    await client.set(prefixedKey, value)
+    if (expirySeconds) {
+      await client.expire(prefixedKey, expirySeconds)
+    }
+  }
+
+  async delete(key) {
+    const db = this._db,
+      client = this._client
+    await client.del(addDbPrefix(db, key))
+  }
+
+  async clear() {
+    const db = this._db
+    let items = await this.scan(db)
+    await Promise.all(items.map(obj => this.delete(db, obj.key)))
+  }
+}
+
+module.exports = RedisWrapper

packages/auth/src/redis/utils.js

@@ -0,0 +1,46 @@
+const env = require("../environment")
+
+const SLOT_REFRESH_MS = 2000
+const CONNECT_TIMEOUT_MS = 10000
+const SEPARATOR = "-"
+const REDIS_URL = !env.REDIS_URL ? "localhost:6379" : env.REDIS_URL
+const REDIS_PASSWORD = !env.REDIS_PASSWORD ? "budibase" : env.REDIS_PASSWORD
+
+exports.Databases = {
+  PW_RESETS: "pwReset",
+  INVITATIONS: "invitation",
+}
+
+exports.getRedisOptions = (clustered = false) => {
+  const [host, port] = REDIS_URL.split(":")
+  const opts = {
+    connectTimeout: CONNECT_TIMEOUT_MS,
+  }
+  if (clustered) {
+    opts.redisOptions = {}
+    opts.redisOptions.tls = {}
+    opts.redisOptions.password = REDIS_PASSWORD
+    opts.slotsRefreshTimeout = SLOT_REFRESH_MS
+    opts.dnsLookup = (address, callback) => callback(null, address)
+  } else {
+    opts.host = host
+    opts.port = port
+    opts.password = REDIS_PASSWORD
+  }
+  return { opts, host, port }
+}
+
+exports.addDbPrefix = (db, key) => {
+  return `${db}${SEPARATOR}${key}`
+}
+
+exports.removeDbPrefix = key => {
+  let parts = key.split(SEPARATOR)
+  if (parts.length >= 2) {
+    parts.shift()
+    return parts.join(SEPARATOR)
+  } else {
+    // return the only part
+    return parts[0]
+  }
+}

packages/auth/src/utils.js

@@ -105,6 +105,12 @@ exports.isClient = ctx => {
   return ctx.headers["x-budibase-type"] === "client"
 }
 
+/**
+ * Given an email address this will use a view to search through
+ * all the users to find one with this email address.
+ * @param {string} email the email to lookup the user by.
+ * @return {Promise<object|null>}
+ */
 exports.getGlobalUserByEmail = async email => {
   const db = getDB(StaticDatabases.GLOBAL.name)
   try {

packages/auth/yarn.lock

@@ -46,6 +46,11 @@ aws4@^1.8.0:
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59"
   integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==
 
+balanced-match@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
+  integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
+
 base64url@3.x.x:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/base64url/-/base64url-3.0.1.tgz#6399d572e2bc3f90a9a8b22d5dbb0a32d33f788d"
@@ -63,6 +68,14 @@ bcryptjs@^2.4.3:
   resolved "https://registry.yarnpkg.com/bcryptjs/-/bcryptjs-2.4.3.tgz#9ab5627b93e60621ff7cdac5da9733027df1d0cb"
   integrity sha1-mrVie5PmBiH/fNrF2pczAn3x0Ms=
 
+brace-expansion@^1.1.7:
+  version "1.1.11"
+  resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
+  integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==
+  dependencies:
+    balanced-match "^1.0.0"
+    concat-map "0.0.1"
+
 buffer-equal-constant-time@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819"
@@ -73,6 +86,11 @@ caseless@~0.12.0:
   resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
   integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=
 
+cluster-key-slot@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
+  integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==
+
 combined-stream@^1.0.6, combined-stream@~1.0.6:
   version "1.0.8"
   resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f"
@@ -80,6 +98,11 @@ combined-stream@^1.0.6, combined-stream@~1.0.6:
   dependencies:
     delayed-stream "~1.0.0"
 
+concat-map@0.0.1:
+  version "0.0.1"
+  resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
+  integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=
+
 core-util-is@1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
@@ -92,11 +115,23 @@ dashdash@^1.12.0:
   dependencies:
     assert-plus "^1.0.0"
 
+debug@^4.3.1:
+  version "4.3.1"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee"
+  integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==
+  dependencies:
+    ms "2.1.2"
+
 delayed-stream@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
   integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk=
 
+denque@^1.1.0:
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.0.tgz#773de0686ff2d8ec2ff92914316a47b73b1c73de"
+  integrity sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==
+
 ecc-jsbn@~0.1.1:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9"
@@ -137,6 +172,20 @@ fast-json-stable-stringify@^2.0.0:
   resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633"
   integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==
 
+fengari-interop@^0.1.2:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/fengari-interop/-/fengari-interop-0.1.2.tgz#f7731dcdd2ff4449073fb7ac3c451a8841ce1e87"
+  integrity sha512-8iTvaByZVoi+lQJhHH9vC+c/Yaok9CwOqNQZN6JrVpjmWwW4dDkeblBXhnHC+BoI6eF4Cy5NKW3z6ICEjvgywQ==
+
+fengari@^0.1.4:
+  version "0.1.4"
+  resolved "https://registry.yarnpkg.com/fengari/-/fengari-0.1.4.tgz#72416693cd9e43bd7d809d7829ddc0578b78b0bb"
+  integrity sha512-6ujqUuiIYmcgkGz8MGAdERU57EIluGGPSUgGPTsco657EHa+srq0S3/YUl/r9kx1+D+d4rGfYObd+m8K22gB1g==
+  dependencies:
+    readline-sync "^1.4.9"
+    sprintf-js "^1.1.1"
+    tmp "^0.0.33"
+
 forever-agent@~0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
@@ -216,6 +265,33 @@ http-signature@~1.2.0:
     jsprim "^1.2.2"
     sshpk "^1.7.0"
 
+ioredis-mock@^5.5.5:
+  version "5.5.5"
+  resolved "https://registry.yarnpkg.com/ioredis-mock/-/ioredis-mock-5.5.5.tgz#dec9fedd238c6ab9f56c026fc366533144f8a256"
+  integrity sha512-7SxCAwNtDLC8IFDptqIhOC7ajp3fciVtCrXOEOkpyjPboAGRQkJbnpNPy1NYORoWi+0/iOtUPUQckSKtSQj4DA==
+  dependencies:
+    fengari "^0.1.4"
+    fengari-interop "^0.1.2"
+    lodash "^4.17.21"
+    minimatch "^3.0.4"
+    standard-as-callback "^2.1.0"
+
+ioredis@^4.27.1:
+  version "4.27.1"
+  resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.27.1.tgz#4ef947b455a1b995baa4b0d7e2c4e4f75f746421"
+  integrity sha512-PaFNFeBbOcEYHXAdrJuy7uesJcyvzStTM1aYMchTuky+VgKqDbXhnTJHaDsjAwcTwPx8Asatx+l2DW8zZ2xlsQ==
+  dependencies:
+    cluster-key-slot "^1.1.0"
+    debug "^4.3.1"
+    denque "^1.1.0"
+    lodash.defaults "^4.2.0"
+    lodash.flatten "^4.4.0"
+    p-map "^2.1.0"
+    redis-commands "1.7.0"
+    redis-errors "^1.2.0"
+    redis-parser "^3.0.0"
+    standard-as-callback "^2.1.0"
+
 is-typedarray@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a"
@@ -296,6 +372,16 @@ koa-passport@^4.1.4:
   dependencies:
     passport "^0.4.0"
 
+lodash.defaults@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
+  integrity sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=
+
+lodash.flatten@^4.4.0:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f"
+  integrity sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=
+
 lodash.includes@^4.3.0:
   version "4.3.0"
   resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f"
@@ -336,7 +422,7 @@ lodash.once@^4.0.0:
   resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac"
   integrity sha1-DdOXEhPHxW34gJd9UEyI+0cal6w=
 
-lodash@^4.14.0:
+lodash@^4.14.0, lodash@^4.17.21:
   version "4.17.21"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
   integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
@@ -358,6 +444,18 @@ mime@^1.4.1:
   resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
   integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==
 
+minimatch@^3.0.4:
+  version "3.0.4"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
+  integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==
+  dependencies:
+    brace-expansion "^1.1.7"
+
+ms@2.1.2:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
+  integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
+
 ms@^2.1.1:
   version "2.1.3"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
@@ -378,6 +476,16 @@ oauth@0.9.x:
   resolved "https://registry.yarnpkg.com/oauth/-/oauth-0.9.15.tgz#bd1fefaf686c96b75475aed5196412ff60cfb9c1"
   integrity sha1-vR/vr2hslrdUda7VGWQS/2DPucE=
 
+os-tmpdir@~1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"
+  integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=
+
+p-map@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175"
+  integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==
+
 passport-google-auth@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/passport-google-auth/-/passport-google-auth-1.0.2.tgz#8b300b5aa442ef433de1d832ed3112877d0b2938"
@@ -481,6 +589,28 @@ qs@~6.5.2:
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
   integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==
 
+readline-sync@^1.4.9:
+  version "1.4.10"
+  resolved "https://registry.yarnpkg.com/readline-sync/-/readline-sync-1.4.10.tgz#41df7fbb4b6312d673011594145705bf56d8873b"
+  integrity sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==
+
+redis-commands@1.7.0:
+  version "1.7.0"
+  resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89"
+  integrity sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==
+
+redis-errors@^1.0.0, redis-errors@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad"
+  integrity sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=
+
+redis-parser@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4"
+  integrity sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=
+  dependencies:
+    redis-errors "^1.0.0"
+
 request@^2.72.0, request@^2.74.0:
   version "2.88.2"
   resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3"
@@ -522,6 +652,11 @@ semver@^5.6.0:
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"
   integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==
 
+sprintf-js@^1.1.1:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.2.tgz#da1765262bf8c0f571749f2ad6c26300207ae673"
+  integrity sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==
+
 sshpk@^1.7.0:
   version "1.16.1"
   resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877"
@@ -537,11 +672,23 @@ sshpk@^1.7.0:
     safer-buffer "^2.0.2"
     tweetnacl "~0.14.0"
 
+standard-as-callback@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45"
+  integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==
+
 string-template@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/string-template/-/string-template-1.0.0.tgz#9e9f2233dc00f218718ec379a28a5673ecca8b96"
   integrity sha1-np8iM9wA8hhxjsN5oopWc+zKi5Y=
 
+tmp@^0.0.33:
+  version "0.0.33"
+  resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9"
+  integrity sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==
+  dependencies:
+    os-tmpdir "~1.0.2"
+
 tough-cookie@~2.5.0:
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2"

packages/bbui/src/Avatar/Avatar.svelte

@@ -45,12 +45,7 @@
     display: grid;
     place-items: center;
     font-weight: 500;
-    background: rgb(63, 94, 251);
-    background: linear-gradient(
-      155deg,
-      rgba(63, 94, 251, 1) 0%,
-      rgba(53, 199, 86, 1) 47%
-    );
+    background: #3aab87;
     border-radius: 50%;
     overflow: hidden;
     user-select: none;

packages/bbui/src/Layout/Page.svelte

@@ -0,0 +1,23 @@
+<script>
+  export let wide = false
+</script>
+
+<div class:wide>
+  <slot />
+</div>
+
+<style>
+  div {
+    display: grid;
+    grid-template-columns: 1fr;
+    max-width: 80ch;
+    margin: 0 auto;
+    padding: calc(var(--spacing-xl) * 2);
+  }
+
+  .wide {
+    max-width: none;
+    margin: 0;
+    padding: var(--spacing-xl) calc(var(--spacing-xl) * 2);
+  }
+</style>

packages/bbui/src/index.js

@@ -28,6 +28,7 @@ export { default as ProgressBar } from "./ProgressBar/ProgressBar.svelte"
 export { default as ProgressCircle } from "./ProgressCircle/ProgressCircle.svelte"
 export { default as Label } from "./Label/Label.svelte"
 export { default as Layout } from "./Layout/Layout.svelte"
+export { default as Page } from "./Layout/Page.svelte"
 export { default as Link } from "./Link/Link.svelte"
 export { default as Menu } from "./Menu/Menu.svelte"
 export { default as MenuSection } from "./Menu/Section.svelte"

packages/builder/src/global.css

@@ -13,8 +13,8 @@ html, body {
 body {
     --background: var(--spectrum-alias-background-color-primary);
     --background-alt: var(--spectrum-alias-background-color-secondary);
-    --border-light: 1px solid var(--spectrum-global-color-gray-200);
-    --border-dark: 1px solid var(--spectrum-global-color-gray-300);
+    --border-light: 1px solid var(--spectrum-global-color-gray-300);
+    --border-dark: 1px solid var(--spectrum-global-color-gray-400);
     --ink: var(--spectrum-alias-text-color);
     --grey-1: var(--spectrum-global-color-gray-100);
     --grey-2: var(--spectrum-global-color-gray-200);

packages/builder/src/pages/portal/_layout.svelte

@@ -23,7 +23,7 @@
   onMount(getInfo)
 
   let menu = [
-    { title: "Apps", href: "/portal/" },
+    { title: "Apps", href: "/portal/apps" },
     { title: "Drafts", href: "/portal/drafts" },
     { title: "Users", href: "/portal/manage/users", heading: "Manage" },
     { title: "Groups", href: "/portal/manage/groups" },
@@ -36,15 +36,15 @@
 </script>
 
 <div class="container">
-  <Layout>
-    <div class="nav">
+  <div class="nav">
+    <Layout paddingX="L" paddingY="L">
       <div class="branding">
         <div class="name">
           <img
-            src={$organisation?.logoUrl || "https://via.placeholder.com/50"}
+            src={$organisation?.logoUrl || "https://i.imgur.com/ZKyklgF.png"}
             alt="Logotype"
           />
-          <span>{$organisation?.company}</span>
+          <span>{$organisation?.company || "Budibase"}</span>
         </div>
         <div class="onboarding">
           <ProgressCircle size="S" value={onBoardingProgress} />
@@ -59,17 +59,17 @@
           {/each}
         </Navigation>
       </div>
-    </div>
-  </Layout>
+    </Layout>
+  </div>
   <div class="main">
     <div class="toolbar">
-      <Search />
+      <Search placeholder="Global search" />
       <div class="avatar">
         <Avatar size="M" name="John Doe" />
         <Icon size="XL" name="ChevronDown" />
       </div>
     </div>
-    <div class="content">
+    <div>
       <slot />
     </div>
   </div>
@@ -81,10 +81,13 @@
     display: grid;
     grid-template-columns: 250px 1fr;
   }
+  .nav {
+    background: var(--background);
+    border-right: var(--border-light);
+  }
   .main {
     display: grid;
     grid-template-rows: auto 1fr;
-    border-left: 2px solid var(--spectrum-alias-background-color-primary);
   }
   .branding {
     display: grid;
@@ -92,7 +95,6 @@
     grid-template-columns: auto auto;
     justify-content: space-between;
     align-items: center;
-    margin-bottom: var(--spacing-xl);
   }
   .name {
     display: grid;
@@ -100,13 +102,6 @@
     grid-gap: var(--spacing-m);
     align-items: center;
   }
-  .content {
-    display: grid;
-    padding: calc(var(--spacing-xl) * 2) var(--spacing-m) var(--spacing-m)
-      var(--spacing-m);
-    max-width: 80ch;
-    margin: 0 auto;
-  }
   .avatar {
     display: grid;
     grid-template-columns: auto auto;
@@ -118,18 +113,20 @@
     filter: brightness(110%);
   }
   .toolbar {
-    border-bottom: 2px solid var(--spectrum-alias-background-color-primary);
+    background: var(--background);
+    border-bottom: var(--border-light);
     display: grid;
     grid-template-columns: 250px auto;
     justify-content: space-between;
     padding: var(--spacing-m) calc(var(--spacing-xl) * 2);
   }
   img {
-    width: 32px;
-    height: 32px;
+    width: 28px;
+    height: 28px;
   }
   span {
     overflow: hidden;
     text-overflow: ellipsis;
+    font-weight: 500;
   }
 </style>

packages/builder/src/pages/portal/apps/_layout.svelte

@@ -0,0 +1,7 @@
+<script>
+  import { Page } from "@budibase/bbui"
+</script>
+
+<Page wide>
+  <slot />
+</Page>

packages/builder/src/pages/portal/apps/index.svelte

@@ -0,0 +1,27 @@
+<script>
+  import { Heading, Layout } from "@budibase/bbui"
+</script>
+
+<Layout noPadding>
+  <div>
+    <Heading>Apps</Heading>
+  </div>
+  <div class="appList">
+    {#each new Array(10) as _}
+      <div class="app" />
+    {/each}
+  </div>
+</Layout>
+
+<style>
+  .appList {
+    display: grid;
+    grid-gap: 50px;
+    grid-template-columns: repeat(auto-fill, 300px);
+  }
+  .app {
+    height: 130px;
+    border-radius: 4px;
+    background-color: var(--spectrum-global-color-gray-200);
+  }
+</style>

packages/builder/src/pages/portal/index.svelte

@@ -0,0 +1,4 @@
+<script>
+  import { goto } from "@roxi/routify"
+  $goto("./apps")
+</script>

packages/builder/src/pages/portal/manage/oauth/index.svelte

@@ -9,6 +9,7 @@
     Layout,
     Input,
     Body,
+    Page,
   } from "@budibase/bbui"
   import { onMount } from "svelte"
   import api from "builderStore/api"
@@ -58,7 +59,7 @@
   })
 </script>
 
-<section>
+<Page>
   <header>
     <Heading size="M">OAuth</Heading>
     <Body size="S">
@@ -88,7 +89,7 @@
     </div>
     <Divider />
   {/if}
-</section>
+</Page>
 
 <style>
   .config-form {

packages/builder/src/pages/portal/settings/_layout.svelte

@@ -0,0 +1,7 @@
+<script>
+  import { Page } from "@budibase/bbui"
+</script>
+
+<Page>
+  <slot />
+</Page>

packages/cli/src/hosting/makeEnv.js

@@ -20,6 +20,7 @@ MINIO_ACCESS_KEY=${randomString.generate()}
 MINIO_SECRET_KEY=${randomString.generate()}
 COUCH_DB_PASSWORD=${randomString.generate()}
 COUCH_DB_USER=${randomString.generate()}
+REDIS_PASSWORD=${randomString.generate()}
 
 # This section contains variables that do not need to be altered under normal circumstances
 APP_PORT=4002

packages/server/package.json

@@ -79,7 +79,7 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.0.1",
+    "@budibase/auth": "^0.18.6",
     "@budibase/client": "^0.8.16",
     "@budibase/string-templates": "^0.8.16",
     "@elastic/elasticsearch": "7.10.0",

packages/server/scripts/dev/manage.js

@@ -37,9 +37,10 @@ async function init() {
     PORT: 4001,
     MINIO_URL: "http://localhost:10000/",
     COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/",
-    REDIS_URL: "http://localhost:10000/cache/",
+    REDIS_URL: "localhost:6379",
     WORKER_URL: "http://localhost:4002",
     JWT_SECRET: "testsecret",
+    REDIS_PASSWORD: "budibase",
     MINIO_ACCESS_KEY: "budibase",
     MINIO_SECRET_KEY: "budibase",
     COUCH_DB_PASSWORD: "budibase",

packages/server/src/api/index.js

@@ -5,20 +5,16 @@ const compress = require("koa-compress")
 const zlib = require("zlib")
 const { mainRoutes, staticRoutes } = require("./routes")
 const pkg = require("../../package.json")
-const bullboard = require("bull-board")
-const expressApp = require("express")()
-
-expressApp.use("/bulladmin", bullboard.router)
-
-const router = new Router()
 const env = require("../environment")
 
-const NO_AUTH_ENDPOINTS = [
-  "/health",
-  "/version",
-  "webhooks/trigger",
-  "webhooks/schema",
-]
+if (!env.isTest()) {
+  const bullboard = require("bull-board")
+  const expressApp = require("express")()
+
+  expressApp.use("/bulladmin", bullboard.router)
+}
+
+const router = new Router()
 
 router
   .use(
@@ -42,7 +38,11 @@ router
   })
   .use("/health", ctx => (ctx.status = 200))
   .use("/version", ctx => (ctx.body = pkg.version))
-  .use(buildAuthMiddleware(NO_AUTH_ENDPOINTS))
+  .use(
+    buildAuthMiddleware(null, {
+      publicAllowed: true,
+    })
+  )
   .use(currentApp)
 
 // error handling middleware

packages/server/src/api/routes/tests/row.spec.js

@@ -402,14 +402,16 @@ describe("/rows", () => {
         name: "test",
         description: "test",
         attachment: [{
-          url: "/test/thing",
+          key: `/assets/${config.getAppId()}/attachment/test/thing.csv`,
         }],
         tableId: table._id,
       })
       // the environment needs configured for this
       await setup.switchToSelfHosted(async () => {
         const enriched = await outputProcessing(config.getAppId(), table, [row])
-        expect(enriched[0].attachment[0].url).toBe(`/app-assets/assets/${config.getAppId()}/test/thing`)
+        expect(enriched[0].attachment[0].url).toBe(
+          `/prod-budi-app-assets/assets/${config.getAppId()}/attachment/test/thing.csv`
+        )
       })
     })
   })

packages/server/src/automations/triggers.js

@@ -1,11 +1,16 @@
 const CouchDB = require("../db")
 const emitter = require("../events/index")
-const Queue = require("bull")
+const env = require("../environment")
+const Queue = env.isTest()
+  ? require("../utilities/queue/inMemoryQueue")
+  : require("bull")
 const { setQueues, BullAdapter } = require("bull-board")
 const { getAutomationParams } = require("../db/utils")
 const { coerce } = require("../utilities/rowProcessor")
+const { utils } = require("@budibase/auth").redis
 
-let automationQueue = new Queue("automationQueue")
+const { opts } = utils.getRedisOptions()
+let automationQueue = new Queue("automationQueue", { redis: opts })
 
 // Set up queues for bull board admin
 setQueues([new BullAdapter(automationQueue)])

packages/server/src/environment.js

@@ -32,6 +32,8 @@ module.exports = {
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   USE_QUOTAS: process.env.USE_QUOTAS,
+  REDIS_URL: process.env.REDIS_URL,
+  REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   // environment
   NODE_ENV: process.env.NODE_ENV,
   JEST_WORKER_ID: process.env.JEST_WORKER_ID,

packages/worker/package.json

@@ -14,12 +14,13 @@
   "scripts": {
     "run:docker": "node src/index.js",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
-    "dev:builder": "npm run dev:stack:init && nodemon src/index.js"
+    "dev:builder": "npm run dev:stack:init && nodemon src/index.js",
+    "test": "jest --runInBand"
   },
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "0.0.1",
+    "@budibase/auth": "^0.18.6",
     "@budibase/string-templates": "^0.8.16",
     "@koa/router": "^8.0.0",
     "aws-sdk": "^2.811.0",
@@ -50,5 +51,11 @@
     "nodemon": "^2.0.7",
     "pouchdb-adapter-memory": "^7.2.2",
     "supertest": "^6.1.3"
+  },
+  "jest": {
+    "testEnvironment": "node",
+    "setupFiles": [
+      "./scripts/jestSetup.js"
+    ]
   }
 }

packages/worker/scripts/dev/manage.js

@@ -12,6 +12,8 @@ async function init() {
     MINIO_SECRET_KEY: "budibase",
     COUCH_DB_USER: "budibase",
     COUCH_DB_PASSWORD: "budibase",
+    REDIS_URL: "localhost:6379",
+    REDIS_PASSWORD: "budibase",
     MINIO_URL: "http://localhost:10000/",
     COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/",
   }

packages/worker/scripts/jestSetup.js

@@ -0,0 +1,5 @@
+const env = require("../src/environment")
+
+env._set("NODE_ENV", "jest")
+env._set("JWT_SECRET", "test-jwtsecret")
+env._set("LOG_LEVEL", "silent")

packages/worker/src/api/controllers/admin/auth.js

@@ -0,0 +1,125 @@
+const authPkg = require("@budibase/auth")
+const { google } = require("@budibase/auth/src/middleware")
+const { Configs, EmailTemplatePurpose } = require("../../../constants")
+const CouchDB = require("../../../db")
+const { sendEmail, isEmailConfigured } = require("../../../utilities/email")
+const { clearCookie, getGlobalUserByEmail, hash } = authPkg.utils
+const { Cookies } = authPkg.constants
+const { passport } = authPkg.auth
+const { checkResetPasswordCode } = require("../../../utilities/redis")
+
+const GLOBAL_DB = authPkg.StaticDatabases.GLOBAL.name
+
+function authInternal(ctx, user, err = null) {
+  if (err) {
+    return ctx.throw(403, "Unauthorized")
+  }
+
+  const expires = new Date()
+  expires.setDate(expires.getDate() + 1)
+
+  if (!user) {
+    return ctx.throw(403, "Unauthorized")
+  }
+
+  ctx.cookies.set(Cookies.Auth, user.token, {
+    expires,
+    path: "/",
+    httpOnly: false,
+    overwrite: true,
+  })
+}
+
+exports.authenticate = async (ctx, next) => {
+  return passport.authenticate("local", async (err, user) => {
+    authInternal(ctx, user, err)
+
+    delete user.token
+
+    ctx.body = { user }
+  })(ctx, next)
+}
+
+/**
+ * Reset the user password, used as part of a forgotten password flow.
+ */
+exports.reset = async ctx => {
+  const { email } = ctx.request.body
+  const configured = await isEmailConfigured()
+  if (!configured) {
+    ctx.throw(
+      400,
+      "Please contact your platform administrator, SMTP is not configured."
+    )
+  }
+  try {
+    const user = await getGlobalUserByEmail(email)
+    await sendEmail(email, EmailTemplatePurpose.PASSWORD_RECOVERY, { user })
+  } catch (err) {
+    // don't throw any kind of error to the user, this might give away something
+  }
+  ctx.body = {
+    message: "Please check your email for a reset link.",
+  }
+}
+
+/**
+ * Perform the user password update if the provided reset code is valid.
+ */
+exports.resetUpdate = async ctx => {
+  const { resetCode, password } = ctx.request.body
+  try {
+    const userId = await checkResetPasswordCode(resetCode)
+    const db = new CouchDB(GLOBAL_DB)
+    const user = await db.get(userId)
+    user.password = await hash(password)
+    await db.put(user)
+    ctx.body = {
+      message: "password reset successfully.",
+    }
+  } catch (err) {
+    ctx.throw(400, "Cannot reset password.")
+  }
+}
+
+exports.logout = async ctx => {
+  clearCookie(ctx, Cookies.Auth)
+  ctx.body = { message: "User logged out." }
+}
+
+/**
+ * The initial call that google authentication makes to take you to the google login screen.
+ * On a successful login, you will be redirected to the googleAuth callback route.
+ */
+exports.googlePreAuth = async (ctx, next) => {
+  const db = new CouchDB(GLOBAL_DB)
+  const config = await authPkg.db.getScopedConfig(db, {
+    type: Configs.GOOGLE,
+    group: ctx.query.group,
+  })
+  const strategy = await google.strategyFactory(config)
+
+  return passport.authenticate(strategy, {
+    scope: ["profile", "email"],
+  })(ctx, next)
+}
+
+exports.googleAuth = async (ctx, next) => {
+  const db = new CouchDB(GLOBAL_DB)
+
+  const config = await authPkg.db.getScopedConfig(db, {
+    type: Configs.GOOGLE,
+    group: ctx.query.group,
+  })
+  const strategy = await google.strategyFactory(config)
+
+  return passport.authenticate(
+    strategy,
+    { successRedirect: "/", failureRedirect: "/error" },
+    async (err, user) => {
+      authInternal(ctx, user, err)
+
+      ctx.redirect("/")
+    }
+  )(ctx, next)
+}

packages/worker/src/api/controllers/admin/configs.js

@@ -3,7 +3,7 @@ const {
   generateConfigID,
   StaticDatabases,
   getConfigParams,
-  determineScopedConfig,
+  getScopedFullConfig,
 } = require("@budibase/auth").db
 const { Configs } = require("../../../constants")
 const email = require("../../../utilities/email")
@@ -45,9 +45,12 @@ exports.save = async function (ctx) {
 exports.fetch = async function (ctx) {
   const db = new CouchDB(GLOBAL_DB)
   const response = await db.allDocs(
-    getConfigParams(undefined, {
-      include_docs: true,
-    })
+    getConfigParams(
+      { type: ctx.params.type },
+      {
+        include_docs: true,
+      }
+    )
   )
   ctx.body = response.rows.map(row => row.doc)
 }
@@ -58,11 +61,10 @@ exports.fetch = async function (ctx) {
  */
 exports.find = async function (ctx) {
   const db = new CouchDB(GLOBAL_DB)
-  const userId = ctx.params.user && ctx.params.user._id
 
-  const { group } = ctx.query
-  if (group) {
-    const group = await db.get(group)
+  const { userId, groupId } = ctx.query
+  if (groupId && userId) {
+    const group = await db.get(groupId)
     const userInGroup = group.users.some(groupUser => groupUser === userId)
     if (!ctx.user.admin && !userInGroup) {
       ctx.throw(400, `User is not in specified group: ${group}.`)
@@ -71,10 +73,10 @@ exports.find = async function (ctx) {
 
   try {
     // Find the config with the most granular scope based on context
-    const scopedConfig = await determineScopedConfig(db, {
+    const scopedConfig = await getScopedFullConfig(db, {
       type: ctx.params.type,
       user: userId,
-      group,
+      group: groupId,
     })
 
     if (scopedConfig) {

packages/worker/src/api/controllers/admin/email.js

@@ -1,82 +1,17 @@
+const { sendEmail } = require("../../../utilities/email")
 const CouchDB = require("../../../db")
-const { StaticDatabases, determineScopedConfig } = require("@budibase/auth").db
-const {
-  EmailTemplatePurpose,
-  TemplateTypes,
-  Configs,
-} = require("../../../constants")
-const { getTemplateByPurpose } = require("../../../constants/templates")
-const { getSettingsTemplateContext } = require("../../../utilities/templates")
-const { processString } = require("@budibase/string-templates")
-const { createSMTPTransport } = require("../../../utilities/email")
+const authPkg = require("@budibase/auth")
 
-const GLOBAL_DB = StaticDatabases.GLOBAL.name
-const TYPE = TemplateTypes.EMAIL
-
-const FULL_EMAIL_PURPOSES = [
-  EmailTemplatePurpose.INVITATION,
-  EmailTemplatePurpose.PASSWORD_RECOVERY,
-  EmailTemplatePurpose.WELCOME,
-]
-
-async function buildEmail(purpose, email, user) {
-  // this isn't a full email
-  if (FULL_EMAIL_PURPOSES.indexOf(purpose) === -1) {
-    throw `Unable to build an email of type ${purpose}`
-  }
-  let [base, styles, body] = await Promise.all([
-    getTemplateByPurpose(TYPE, EmailTemplatePurpose.BASE),
-    getTemplateByPurpose(TYPE, EmailTemplatePurpose.STYLES),
-    getTemplateByPurpose(TYPE, purpose),
-  ])
-  if (!base || !styles || !body) {
-    throw "Unable to build email, missing base components"
-  }
-  base = base.contents
-  styles = styles.contents
-  body = body.contents
-
-  // TODO: need to extend the context as much as possible
-  const context = {
-    ...(await getSettingsTemplateContext()),
-    email,
-    user: user || {},
-  }
-
-  body = await processString(body, context)
-  styles = await processString(styles, context)
-  // this should now be the complete email HTML
-  return processString(base, {
-    ...context,
-    styles,
-    body,
-  })
-}
+const GLOBAL_DB = authPkg.StaticDatabases.GLOBAL.name
 
 exports.sendEmail = async ctx => {
   const { groupId, email, userId, purpose } = ctx.request.body
-  const db = new CouchDB(GLOBAL_DB)
-  const params = {}
-  if (groupId) {
-    params.group = groupId
-  }
-  params.type = Configs.SMTP
-  let user = {}
+  let user
   if (userId) {
-    user = db.get(userId)
+    const db = new CouchDB(GLOBAL_DB)
+    user = await db.get(userId)
   }
-  const { config } = await determineScopedConfig(db, params)
-  if (!config) {
-    ctx.throw(400, "Unable to find SMTP configuration")
-  }
-  const transport = createSMTPTransport(config)
-  const message = {
-    from: config.from,
-    subject: config.subject,
-    to: email,
-    html: await buildEmail(purpose, email, user),
-  }
-  const response = await transport.sendMail(message)
+  const response = await sendEmail(email, purpose, { groupId, user })
   ctx.body = {
     ...response,
     message: `Email sent to ${email}.`,

packages/worker/src/api/controllers/admin/users.js

@@ -5,7 +5,9 @@ const {
   StaticDatabases,
 } = require("@budibase/auth").db
 const { hash, getGlobalUserByEmail } = require("@budibase/auth").utils
-const { UserStatus } = require("../../../constants")
+const { UserStatus, EmailTemplatePurpose } = require("../../../constants")
+const { checkInviteCode } = require("../../../utilities/redis")
+const { sendEmail } = require("../../../utilities/email")
 
 const FIRST_USER_EMAIL = "test@test.com"
 const FIRST_USER_PASSWORD = "test"
@@ -42,7 +44,7 @@ exports.save = async ctx => {
     user.status = UserStatus.ACTIVE
   }
   try {
-    const response = await db.post({
+    const response = await db.put({
       password: hashedPassword,
       ...user,
     })
@@ -61,7 +63,14 @@ exports.save = async ctx => {
 }
 
 exports.firstUser = async ctx => {
+  const existing = await getGlobalUserByEmail(FIRST_USER_EMAIL)
+  const params = {}
+  if (existing) {
+    params._id = existing._id
+    params._rev = existing._rev
+  }
   ctx.request.body = {
+    ...params,
     email: FIRST_USER_EMAIL,
     password: FIRST_USER_PASSWORD,
     roles: {},
@@ -114,3 +123,29 @@ exports.find = async ctx => {
   }
   ctx.body = user
 }
+
+exports.invite = async ctx => {
+  const { email } = ctx.request.body
+  const existing = await getGlobalUserByEmail(email)
+  if (existing) {
+    ctx.throw(400, "Email address already in use.")
+  }
+  await sendEmail(email, EmailTemplatePurpose.INVITATION)
+  ctx.body = {
+    message: "Invitation has been sent.",
+  }
+}
+
+exports.inviteAccept = async ctx => {
+  const { inviteCode } = ctx.request.body
+  try {
+    const email = await checkInviteCode(inviteCode)
+    // redirect the request
+    delete ctx.request.body.inviteCode
+    ctx.request.body.email = email
+    // this will flesh out the body response
+    await exports.save(ctx)
+  } catch (err) {
+    ctx.throw(400, "Unable to create new user, invitation invalid.")
+  }
+}

packages/worker/src/api/controllers/auth.js

@@ -1,93 +0,0 @@
-const authPkg = require("@budibase/auth")
-const { google } = require("@budibase/auth/src/middleware")
-const { Configs } = require("../../constants")
-const CouchDB = require("../../db")
-const { clearCookie } = authPkg.utils
-const { Cookies } = authPkg.constants
-const { passport } = authPkg.auth
-
-const GLOBAL_DB = authPkg.StaticDatabases.GLOBAL.name
-
-exports.authenticate = async (ctx, next) => {
-  return passport.authenticate("local", async (err, user) => {
-    if (err) {
-      return ctx.throw(403, "Unauthorized")
-    }
-
-    const expires = new Date()
-    expires.setDate(expires.getDate() + 1)
-
-    if (!user) {
-      return ctx.throw(403, "Unauthorized")
-    }
-
-    ctx.cookies.set(Cookies.Auth, user.token, {
-      expires,
-      path: "/",
-      httpOnly: false,
-      overwrite: true,
-    })
-
-    delete user.token
-
-    ctx.body = { user }
-  })(ctx, next)
-}
-
-exports.logout = async ctx => {
-  clearCookie(ctx, Cookies.Auth)
-  ctx.body = { message: "User logged out" }
-}
-
-/**
- * The initial call that google authentication makes to take you to the google login screen.
- * On a successful login, you will be redirected to the googleAuth callback route.
- */
-exports.googlePreAuth = async (ctx, next) => {
-  const db = new CouchDB(GLOBAL_DB)
-  const config = await authPkg.db.determineScopedConfig(db, {
-    type: Configs.GOOGLE,
-    group: ctx.query.group,
-  })
-  const strategy = await google.strategyFactory(config)
-
-  return passport.authenticate(strategy, {
-    scope: ["profile", "email"],
-  })(ctx, next)
-}
-
-exports.googleAuth = async (ctx, next) => {
-  const db = new CouchDB(GLOBAL_DB)
-
-  const config = await authPkg.db.determineScopedConfig(db, {
-    type: Configs.GOOGLE,
-    group: ctx.query.group,
-  })
-  const strategy = await google.strategyFactory(config)
-
-  return passport.authenticate(
-    strategy,
-    { successRedirect: "/", failureRedirect: "/error" },
-    async (err, user) => {
-      if (err) {
-        return ctx.throw(403, "Unauthorized")
-      }
-
-      const expires = new Date()
-      expires.setDate(expires.getDate() + 1)
-
-      if (!user) {
-        return ctx.throw(403, "Unauthorized")
-      }
-
-      ctx.cookies.set(Cookies.Auth, user.token, {
-        expires,
-        path: "/",
-        httpOnly: false,
-        overwrite: true,
-      })
-
-      ctx.redirect("/")
-    }
-  )(ctx, next)
-}

packages/worker/src/api/index.js

@@ -4,15 +4,34 @@ const zlib = require("zlib")
 const { routes } = require("./routes")
 const { buildAuthMiddleware } = require("@budibase/auth").auth
 
-const NO_AUTH_ENDPOINTS = [
-  "/api/admin/users/first",
-  "/api/admin/auth",
-  "/api/admin/auth/google",
-  "/api/admin/auth/google/callback",
+const PUBLIC_ENDPOINTS = [
+  {
+    route: "/api/admin/users/first",
+    method: "POST",
+  },
+  {
+    route: "/api/admin/users/invite/accept",
+    method: "POST",
+  },
+  {
+    route: "/api/admin/auth",
+    method: "POST",
+  },
+  {
+    route: "/api/admin/auth/google",
+    method: "GET",
+  },
+  {
+    route: "/api/admin/auth/google/callback",
+    method: "GET",
+  },
+  {
+    route: "/api/admin/auth/reset",
+    method: "POST",
+  },
 ]
 
 const router = new Router()
-
 router
   .use(
     compress({
@@ -27,7 +46,7 @@ router
     })
   )
   .use("/health", ctx => (ctx.status = 200))
-  .use(buildAuthMiddleware(NO_AUTH_ENDPOINTS))
+  .use(buildAuthMiddleware(PUBLIC_ENDPOINTS))
   // for now no public access is allowed to worker (bar health check)
   .use((ctx, next) => {
     if (!ctx.isAuthenticated) {

packages/worker/src/api/routes/admin/auth.js

@@ -0,0 +1,43 @@
+const Router = require("@koa/router")
+const authController = require("../../controllers/admin/auth")
+const joiValidator = require("../../../middleware/joi-validator")
+const Joi = require("joi")
+
+const router = Router()
+
+function buildAuthValidation() {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    username: Joi.string().required(),
+    password: Joi.string().required(),
+  }).required().unknown(false))
+}
+
+function buildResetValidation() {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    email: Joi.string().required(),
+  }).required().unknown(false))
+}
+
+function buildResetUpdateValidation() {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    resetCode: Joi.string().required(),
+    password: Joi.string().required(),
+  }).required().unknown(false))
+}
+
+router
+  .post("/api/admin/auth", buildAuthValidation(), authController.authenticate)
+  .post("/api/admin/auth/reset", buildResetValidation(), authController.reset)
+  .post(
+    "/api/admin/auth/reset/update",
+    buildResetUpdateValidation(),
+    authController.resetUpdate
+  )
+  .post("/api/admin/auth/logout", authController.logout)
+  .get("/api/admin/auth/google", authController.googlePreAuth)
+  .get("/api/admin/auth/google/callback", authController.googleAuth)
+
+module.exports = router

packages/worker/src/api/routes/admin/configs.js

@@ -57,14 +57,25 @@ function buildConfigSaveValidation() {
           { is: Configs.GOOGLE, then: googleValidation() }
         ],
       }),
-    }),
+    }).required(),
   )
 }
 
+function buildConfigGetValidation() {
+  // prettier-ignore
+  return joiValidator.params(Joi.object({
+    type: Joi.string().valid(...Object.values(Configs)).required()
+  }).unknown(true).required())
+}
+
 router
   .post("/api/admin/configs", buildConfigSaveValidation(), controller.save)
   .delete("/api/admin/configs/:id", controller.destroy)
-  .get("/api/admin/configs", controller.fetch)
-  .get("/api/admin/configs/:type", controller.find)
+  .get(
+    "/api/admin/configs/all/:type",
+    buildConfigGetValidation(),
+    controller.fetch
+  )
+  .get("/api/admin/configs/:type", buildConfigGetValidation(), controller.find)
 
 module.exports = router

packages/worker/src/api/routes/admin/users.js

@@ -21,7 +21,22 @@ function buildUserSaveValidation() {
       .pattern(/.*/, Joi.string())
       .required()
       .unknown(true)
-  }).required().unknown(true).optional())
+  }).required().unknown(true))
+}
+
+function buildInviteValidation() {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    email: Joi.string().required(),
+  }).required())
+}
+
+function buildInviteAcceptValidation() {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    inviteCode: Joi.string().required(),
+    password: Joi.string().required(),
+  }).required().unknown(true))
 }
 
 router
@@ -30,5 +45,11 @@ router
   .post("/api/admin/users/first", controller.firstUser)
   .delete("/api/admin/users/:id", controller.destroy)
   .get("/api/admin/users/:id", controller.find)
+  .post("/api/admin/users/invite", buildInviteValidation(), controller.invite)
+  .post(
+    "/api/admin/users/invite/accept",
+    buildInviteAcceptValidation(),
+    controller.inviteAccept
+  )
 
 module.exports = router

packages/worker/src/api/routes/auth.js

@@ -1,12 +0,0 @@
-const Router = require("@koa/router")
-const authController = require("../controllers/auth")
-
-const router = Router()
-
-router
-  .post("/api/admin/auth", authController.authenticate)
-  .get("/api/admin/auth/google", authController.googlePreAuth)
-  .get("/api/admin/auth/google/callback", authController.googleAuth)
-  .post("/api/admin/auth/logout", authController.logout)
-
-module.exports = router

packages/worker/src/api/routes/index.js

@@ -3,7 +3,7 @@ const configRoutes = require("./admin/configs")
 const groupRoutes = require("./admin/groups")
 const templateRoutes = require("./admin/templates")
 const emailRoutes = require("./admin/email")
-const authRoutes = require("./auth")
+const authRoutes = require("./admin/auth")
 const appRoutes = require("./app")
 
 exports.routes = [

packages/worker/src/api/routes/tests/auth.spec.js

@@ -0,0 +1,49 @@
+const setup = require("./utilities")
+
+jest.mock("nodemailer")
+const sendMailMock = setup.emailMock()
+
+describe("/api/admin/auth", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+  let code
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  afterAll(setup.afterAll)
+
+  it("should be able to generate password reset email", async () => {
+    // initially configure settings
+    await config.saveSmtpConfig()
+    await config.saveSettingsConfig()
+    await config.createUser("test@test.com")
+    const res = await request
+      .post(`/api/admin/auth/reset`)
+      .send({
+        email: "test@test.com",
+      })
+      .expect("Content-Type", /json/)
+      .expect(200)
+    expect(res.body).toEqual({ message: "Please check your email for a reset link." })
+    expect(sendMailMock).toHaveBeenCalled()
+    const emailCall = sendMailMock.mock.calls[0][0]
+    // after this URL there should be a code
+    const parts = emailCall.html.split("http://localhost:10000/reset?code=")
+    code = parts[1].split("\"")[0]
+    expect(code).toBeDefined()
+  })
+
+  it("should allow resetting user password with code", async () => {
+    const res = await request
+      .post(`/api/admin/auth/reset/update`)
+      .send({
+        password: "newpassword",
+        resetCode: code,
+      })
+      .expect("Content-Type", /json/)
+      .expect(200)
+    expect(res.body).toEqual({ message: "password reset successfully." })
+  })
+})

packages/worker/src/api/routes/tests/email.spec.js

@@ -2,13 +2,8 @@ const setup = require("./utilities")
 const { EmailTemplatePurpose } = require("../../../constants")
 
 // mock the email system
-const sendMailMock = jest.fn()
 jest.mock("nodemailer")
-const nodemailer = require("nodemailer")
-nodemailer.createTransport.mockReturnValue({
-  sendMail: sendMailMock,
-  verify: jest.fn()
-})
+const sendMailMock = setup.emailMock()
 
 describe("/api/admin/email", () => {
   let request = setup.getRequest()

packages/worker/src/api/routes/tests/realEmail.spec.js

@@ -16,11 +16,13 @@ describe("/api/admin/email", () => {
   async function sendRealEmail(purpose) {
     await config.saveEtherealSmtpConfig()
     await config.saveSettingsConfig()
+    const user = await config.getUser("test@test.com")
     const res = await request
       .post(`/api/admin/email/send`)
       .send({
         email: "test@test.com",
         purpose,
+        userId: user._id,
       })
       .set(config.defaultHeaders())
       .expect("Content-Type", /json/)
@@ -55,6 +57,6 @@ describe("/api/admin/email", () => {
   })
 
   it("should be able to send a password recovery email", async () => {
-    const res = await sendRealEmail(EmailTemplatePurpose.PASSWORD_RECOVERY)
+    await sendRealEmail(EmailTemplatePurpose.PASSWORD_RECOVERY)
   })
 })

packages/worker/src/api/routes/tests/users.spec.js

@@ -0,0 +1,52 @@
+const setup = require("./utilities")
+
+jest.mock("nodemailer")
+const sendMailMock = setup.emailMock()
+
+describe("/api/admin/users", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+  let code
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  afterAll(setup.afterAll)
+
+  it("should be able to generate an invitation", async () => {
+    // initially configure settings
+    await config.saveSmtpConfig()
+    await config.saveSettingsConfig()
+    const res = await request
+      .post(`/api/admin/users/invite`)
+      .send({
+        email: "invite@test.com",
+      })
+      .set(config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+    expect(res.body).toEqual({ message: "Invitation has been sent." })
+    expect(sendMailMock).toHaveBeenCalled()
+    const emailCall = sendMailMock.mock.calls[0][0]
+    // after this URL there should be a code
+    const parts = emailCall.html.split("http://localhost:10000/invite?code=")
+    code = parts[1].split("\"")[0]
+    expect(code).toBeDefined()
+  })
+
+  it("should be able to create new user from invite", async () => {
+    const res = await request
+      .post(`/api/admin/users/invite/accept`)
+      .send({
+        password: "newpassword",
+        inviteCode: code,
+      })
+      .expect("Content-Type", /json/)
+      .expect(200)
+    expect(res.body._id).toBeDefined()
+    const user = await config.getUser("invite@test.com")
+    expect(user).toBeDefined()
+    expect(user._id).toEqual(res.body._id)
+  })
+})

packages/worker/src/api/routes/tests/utilities/TestConfiguration.js

@@ -4,6 +4,7 @@ const supertest = require("supertest")
 const { jwt } = require("@budibase/auth").auth
 const { Cookies } = require("@budibase/auth").constants
 const { Configs, LOGO_URL } = require("../../../../constants")
+const { getGlobalUserByEmail } = require("@budibase/auth").utils
 
 class TestConfiguration {
   constructor(openServer = true) {
@@ -53,6 +54,12 @@ class TestConfiguration {
     )
   }
 
+  async end() {
+    if (this.server) {
+      await this.server.close()
+    }
+  }
+
   defaultHeaders() {
     const user = {
       _id: "us_uuid1",
@@ -65,6 +72,25 @@ class TestConfiguration {
     }
   }
 
+  async getUser(email) {
+    return getGlobalUserByEmail(email)
+  }
+
+  async createUser(email = "test@test.com", password = "test") {
+    const user = await this.getUser(email)
+    if (user) {
+      return user
+    }
+    await this._req(
+      {
+        email,
+        password,
+      },
+      null,
+      controllers.users.save
+    )
+  }
+
   async deleteConfig(type) {
     try {
       const cfg = await this._req(

packages/worker/src/api/routes/tests/utilities/index.js

@@ -7,9 +7,9 @@ exports.beforeAll = () => {
   request = config.getRequest()
 }
 
-exports.afterAll = () => {
+exports.afterAll = async () => {
   if (config) {
-    config.end()
+    await config.end()
   }
   request = null
   config = null
@@ -28,3 +28,14 @@ exports.getConfig = () => {
   }
   return config
 }
+
+exports.emailMock = () => {
+  // mock the email system
+  const sendMailMock = jest.fn()
+  const nodemailer = require("nodemailer")
+  nodemailer.createTransport.mockReturnValue({
+    sendMail: sendMailMock,
+    verify: jest.fn(),
+  })
+  return sendMailMock
+}

packages/worker/src/constants/index.js

@@ -45,6 +45,8 @@ const TemplateBindings = {
   LOGIN_URL: "loginUrl",
   CURRENT_YEAR: "currentYear",
   CURRENT_DATE: "currentDate",
+  RESET_CODE: "resetCode",
+  INVITE_CODE: "inviteCode",
 }
 
 const TemplateMetadata = {

packages/worker/src/environment.js

@@ -11,7 +11,7 @@ function isTest() {
 }
 
 let LOADED = false
-if (!LOADED && isDev()) {
+if (!LOADED && isDev() && !isTest()) {
   require("dotenv").config()
   LOADED = true
 }
@@ -26,6 +26,8 @@ module.exports = {
   LOG_LEVEL: process.env.LOG_LEVEL,
   JWT_SECRET: process.env.JWT_SECRET,
   SALT_ROUNDS: process.env.SALT_ROUNDS,
+  REDIS_URL: process.env.REDIS_URL,
+  REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   /* TODO: to remove - once deployment removed */
   SELF_HOST_KEY: process.env.SELF_HOST_KEY,
   COUCH_DB_USERNAME: process.env.COUCH_DB_USERNAME,

packages/worker/src/index.js

@@ -12,10 +12,6 @@ const api = require("./api")
 
 const app = new Koa()
 
-if (!env.SELF_HOSTED) {
-  throw "Currently this service only supports use in self hosting"
-}
-
 // set up top level koa middleware
 app.use(koaBody({ multipart: true }))
 

packages/worker/src/utilities/email.js

@@ -1,6 +1,22 @@
 const nodemailer = require("nodemailer")
+const CouchDB = require("../db")
+const { StaticDatabases, getScopedConfig } = require("@budibase/auth").db
+const { EmailTemplatePurpose, TemplateTypes, Configs } = require("../constants")
+const { getTemplateByPurpose } = require("../constants/templates")
+const { getSettingsTemplateContext } = require("./templates")
+const { processString } = require("@budibase/string-templates")
+const { getResetPasswordCode, getInviteCode } = require("../utilities/redis")
 
-exports.createSMTPTransport = config => {
+const GLOBAL_DB = StaticDatabases.GLOBAL.name
+const TYPE = TemplateTypes.EMAIL
+
+const FULL_EMAIL_PURPOSES = [
+  EmailTemplatePurpose.INVITATION,
+  EmailTemplatePurpose.PASSWORD_RECOVERY,
+  EmailTemplatePurpose.WELCOME,
+]
+
+function createSMTPTransport(config) {
   const options = {
     port: config.port,
     host: config.host,
@@ -15,7 +31,118 @@ exports.createSMTPTransport = config => {
   return nodemailer.createTransport(options)
 }
 
+async function getLinkCode(purpose, email, user) {
+  switch (purpose) {
+    case EmailTemplatePurpose.PASSWORD_RECOVERY:
+      return getResetPasswordCode(user._id)
+    case EmailTemplatePurpose.INVITATION:
+      return getInviteCode(email)
+    default:
+      return null
+  }
+}
+
+/**
+ * Builds an email using handlebars and the templates found in the system (default or otherwise).
+ * @param {string} purpose the purpose of the email being built, e.g. invitation, password reset.
+ * @param {string} email the address which it is being sent to for contextual purposes.
+ * @param {object|null} user If being sent to an existing user then the object can be provided for context.
+ * @return {Promise<string>} returns the built email HTML if all provided parameters were valid.
+ */
+async function buildEmail(purpose, email, user) {
+  // this isn't a full email
+  if (FULL_EMAIL_PURPOSES.indexOf(purpose) === -1) {
+    throw `Unable to build an email of type ${purpose}`
+  }
+  let [base, styles, body] = await Promise.all([
+    getTemplateByPurpose(TYPE, EmailTemplatePurpose.BASE),
+    getTemplateByPurpose(TYPE, EmailTemplatePurpose.STYLES),
+    getTemplateByPurpose(TYPE, purpose),
+  ])
+  if (!base || !styles || !body) {
+    throw "Unable to build email, missing base components"
+  }
+  base = base.contents
+  styles = styles.contents
+  body = body.contents
+
+  // if there is a link code needed this will retrieve it
+  const code = await getLinkCode(purpose, email, user)
+  const context = {
+    ...(await getSettingsTemplateContext(purpose, code)),
+    email,
+    user: user || {},
+  }
+
+  body = await processString(body, context)
+  styles = await processString(styles, context)
+  // this should now be the complete email HTML
+  return processString(base, {
+    ...context,
+    styles,
+    body,
+  })
+}
+
+/**
+ * Utility function for finding most valid SMTP configuration.
+ * @param {object} db The CouchDB database which is to be looked up within.
+ * @param {string|null} groupId If using finer grain control of configs a group can be used.
+ * @return {Promise<object|null>} returns the SMTP configuration if it exists
+ */
+async function getSmtpConfiguration(db, groupId = null) {
+  const params = {
+    type: Configs.SMTP,
+  }
+  if (groupId) {
+    params.group = groupId
+  }
+  return getScopedConfig(db, params)
+}
+
+/**
+ * Checks if a SMTP config exists based on passed in parameters.
+ * @param groupId
+ * @return {Promise<boolean>} returns true if there is a configuration that can be used.
+ */
+exports.isEmailConfigured = async (groupId = null) => {
+  const db = new CouchDB(GLOBAL_DB)
+  const config = await getSmtpConfiguration(db, groupId)
+  return config != null
+}
+
+/**
+ * Given an email address and an email purpose this will retrieve the SMTP configuration and
+ * send an email using it.
+ * @param {string} email The email address to send to.
+ * @param {string} purpose The purpose of the email being sent (e.g. reset password).
+ * @param {string|undefined} groupId If finer grain controls being used then this will lookup config for group.
+ * @param {object|undefined} user if sending to an existing user the object can be provided, this is used in the context.
+ * @return {Promise<object>} returns details about the attempt to send email, e.g. if it is successful; based on
+ * nodemailer response.
+ */
+exports.sendEmail = async (email, purpose, { groupId, user } = {}) => {
+  const db = new CouchDB(GLOBAL_DB)
+  const config = await getSmtpConfiguration(db, groupId)
+  if (!config) {
+    throw "Unable to find SMTP configuration."
+  }
+  const transport = createSMTPTransport(config)
+  const message = {
+    from: config.from,
+    subject: config.subject,
+    to: email,
+    html: await buildEmail(purpose, email, user),
+  }
+  return transport.sendMail(message)
+}
+
+/**
+ * Given an SMTP configuration this runs it through nodemailer to see if it is infact functional.
+ * @param {object} config an SMTP configuration - this is based on the nodemailer API.
+ * @return {Promise<boolean>} returns true if the configuration is valid.
+ */
 exports.verifyConfig = async config => {
-  const transport = exports.createSMTPTransport(config)
+  const transport = createSMTPTransport(config)
   await transport.verify()
 }

packages/worker/src/utilities/redis.js

@@ -0,0 +1,85 @@
+const { Client, utils } = require("@budibase/auth").redis
+const { newid } = require("@budibase/auth").utils
+
+function getExpirySecondsForDB(db) {
+  switch (db) {
+    case utils.Databases.PW_RESETS:
+      // a hour
+      return 3600
+    case utils.Databases.INVITATIONS:
+      // a day
+      return 86400
+  }
+}
+
+async function getClient(db) {
+  return await new Client(db).init()
+}
+
+async function writeACode(db, value) {
+  const client = await getClient(db)
+  const code = newid()
+  await client.store(code, value, getExpirySecondsForDB(db))
+  client.finish()
+  return code
+}
+
+async function getACode(db, code, deleteCode = true) {
+  const client = await getClient(db)
+  const value = await client.get(code)
+  if (!value) {
+    throw "Invalid code."
+  }
+  if (deleteCode) {
+    await client.delete(code)
+  }
+  client.finish()
+  return value
+}
+
+/**
+ * Given a user ID this will store a code (that is returned) for an hour in redis.
+ * The user can then return this code for resetting their password (through their reset link).
+ * @param {string} userId the ID of the user which is to be reset.
+ * @return {Promise<string>} returns the code that was stored to redis.
+ */
+exports.getResetPasswordCode = async userId => {
+  return writeACode(utils.Databases.PW_RESETS, userId)
+}
+
+/**
+ * Given a reset code this will lookup to redis, check if the code is valid and delete if required.
+ * @param {string} resetCode The code provided via the email link.
+ * @param {boolean} deleteCode If the code is used/finished with this will delete it - defaults to true.
+ * @return {Promise<string>} returns the user ID if it is found
+ */
+exports.checkResetPasswordCode = async (resetCode, deleteCode = true) => {
+  try {
+    return getACode(utils.Databases.PW_RESETS, resetCode, deleteCode)
+  } catch (err) {
+    throw "Provided information is not valid, cannot reset password - please try again."
+  }
+}
+
+/**
+ * Generates an invitation code and writes it to redis - which can later be checked for user creation.
+ * @param {string} email the email address which the code is being sent to (for use later).
+ * @return {Promise<string>} returns the code that was stored to redis.
+ */
+exports.getInviteCode = async email => {
+  return writeACode(utils.Databases.INVITATIONS, email)
+}
+
+/**
+ * Checks that the provided invite code is valid - will return the email address of user that was invited.
+ * @param {string} inviteCode the invite code that was provided as part of the link.
+ * @param {boolean} deleteCode whether or not the code should be deleted after retrieval - defaults to true.
+ * @return {Promise<string>} If the code is valid then an email address will be returned.
+ */
+exports.checkInviteCode = async (inviteCode, deleteCode = true) => {
+  try {
+    return getACode(utils.Databases.INVITATIONS, inviteCode, deleteCode)
+  } catch (err) {
+    throw "Invitation is not valid or has expired, please request a new one."
+  }
+}

packages/worker/src/utilities/templates.js

@@ -1,32 +1,28 @@
 const CouchDB = require("../db")
-const { getConfigParams, StaticDatabases } = require("@budibase/auth").db
-const { Configs, TemplateBindings, LOGO_URL } = require("../constants")
+const { getScopedConfig, StaticDatabases } = require("@budibase/auth").db
+const {
+  Configs,
+  TemplateBindings,
+  LOGO_URL,
+  EmailTemplatePurpose,
+} = require("../constants")
 const { checkSlashesInUrl } = require("./index")
 const env = require("../environment")
 
 const LOCAL_URL = `http://localhost:${env.PORT}`
 const BASE_COMPANY = "Budibase"
 
-exports.getSettingsTemplateContext = async () => {
+exports.getSettingsTemplateContext = async (purpose, code = null) => {
   const db = new CouchDB(StaticDatabases.GLOBAL.name)
-  const response = await db.allDocs(
-    getConfigParams(Configs.SETTINGS, {
-      include_docs: true,
-    })
-  )
-  let settings = response.rows.map(row => row.doc)[0] || {}
+  // TODO: use more granular settings in the future if required
+  const settings = await getScopedConfig(db, { type: Configs.SETTINGS })
   if (!settings.platformUrl) {
     settings.platformUrl = LOCAL_URL
   }
-  // TODO: need to fully spec out the context
   const URL = settings.platformUrl
-  return {
+  const context = {
     [TemplateBindings.LOGO_URL]: settings.logoUrl || LOGO_URL,
     [TemplateBindings.PLATFORM_URL]: URL,
-    [TemplateBindings.REGISTRATION_URL]: checkSlashesInUrl(
-      `${URL}/registration`
-    ),
-    [TemplateBindings.RESET_URL]: checkSlashesInUrl(`${URL}/reset`),
     [TemplateBindings.COMPANY]: settings.company || BASE_COMPANY,
     [TemplateBindings.DOCS_URL]:
       settings.docsUrl || "https://docs.budibase.com/",
@@ -34,4 +30,20 @@ exports.getSettingsTemplateContext = async () => {
     [TemplateBindings.CURRENT_DATE]: new Date().toISOString(),
     [TemplateBindings.CURRENT_YEAR]: new Date().getFullYear(),
   }
+  // attach purpose specific context
+  switch (purpose) {
+    case EmailTemplatePurpose.PASSWORD_RECOVERY:
+      context[TemplateBindings.RESET_CODE] = code
+      context[TemplateBindings.RESET_URL] = checkSlashesInUrl(
+        `${URL}/reset?code=${code}`
+      )
+      break
+    case EmailTemplatePurpose.INVITATION:
+      context[TemplateBindings.INVITE_CODE] = code
+      context[TemplateBindings.REGISTRATION_URL] = checkSlashesInUrl(
+        `${URL}/invite?code=${code}`
+      )
+      break
+  }
+  return context
 }