Merge branch 'develop' into scope-or-view

docs/index.yaml

@@ -2,8 +2,8 @@ apiVersion: v1
 entries:
   budibase:
   - apiVersion: v2
-    appVersion: 0.9.163
+    appVersion: 0.9.169
-    created: "2021-10-18T16:25:04.374924613+01:00"
+    created: "2021-10-20T14:27:23.521358+01:00"
     dependencies:
     - condition: services.couchdb.enabled
       name: couchdb
@@ -13,8 +13,33 @@ entries:
       name: ingress-nginx
       repository: https://github.com/kubernetes/ingress-nginx
       version: 3.35.0
-    description: Budibase is an open source low-code platform, helping thousands of
+    description: Budibase is an open source low-code platform, helping thousands of teams build apps for their workplace in minutes.
-      teams build apps for their workplace in minutes.
+    digest: 57f365d799fcaace4658883cb8ec961a7905383a68acf065af4f6e57f9878ff8
+    keywords:
+    - low-code
+    - database
+    - cluster
+    name: budibase
+    sources:
+    - https://github.com/Budibase/budibase
+    - https://budibase.com
+    type: application
+    urls:
+    - https://budibase.github.io/budibase/budibase-0.2.2.tgz
+    version: 0.2.2
+  - apiVersion: v2
+    appVersion: 0.9.163
+    created: "2021-10-20T14:27:23.5153+01:00"
+    dependencies:
+    - condition: services.couchdb.enabled
+      name: couchdb
+      repository: https://apache.github.io/couchdb-helm
+      version: 3.3.4
+    - condition: ingress.nginx
+      name: ingress-nginx
+      repository: https://github.com/kubernetes/ingress-nginx
+      version: 3.35.0
+    description: Budibase is an open source low-code platform, helping thousands of teams build apps for their workplace in minutes.
     digest: ebac6d8631cc38b266c3689508b5123f5afc395f23bdb02738be26c7cae0b0b5
     keywords:
     - low-code
@@ -30,7 +55,7 @@ entries:
     version: 0.2.1
   - apiVersion: v2
     appVersion: 0.9.163
-    created: "2021-10-18T16:25:04.36936805+01:00"
+    created: "2021-10-20T14:27:23.510041+01:00"
     dependencies:
     - condition: services.couchdb.enabled
       name: couchdb
@@ -40,8 +65,7 @@ entries:
       name: ingress-nginx
       repository: https://github.com/kubernetes/ingress-nginx
       version: 3.35.0
-    description: Budibase is an open source low-code platform, helping thousands of
+    description: Budibase is an open source low-code platform, helping thousands of teams build apps for their workplace in minutes.
-      teams build apps for their workplace in minutes.
     digest: f369536c0eac1f6959d51e8ce6d74a87a7a9df29ae84fb9cbed0a273ab77429b
     keywords:
     - low-code
@@ -57,7 +81,7 @@ entries:
     version: 0.2.0
   - apiVersion: v2
     appVersion: 0.9.56
-    created: "2021-10-18T16:25:04.36360616+01:00"
+    created: "2021-10-20T14:27:23.504543+01:00"
     dependencies:
     - condition: services.couchdb.enabled
       name: couchdb
@@ -66,8 +90,7 @@ entries:
     - name: ingress-nginx
       repository: https://github.com/kubernetes/ingress-nginx
       version: 3.35.0
-    description: Budibase is an open source low-code platform, helping thousands of
+    description: Budibase is an open source low-code platform, helping thousands of teams build apps for their workplace in minutes.
-      teams build apps for their workplace in minutes.
     digest: 8dc4f2ed4d98cad5adf25936aefea680042d3e4e17832f846b961fd8708ad192
     keywords:
     - low-code
@@ -83,7 +106,7 @@ entries:
     version: 0.1.1
   - apiVersion: v2
     appVersion: 0.9.56
-    created: "2021-10-18T16:25:04.354504201+01:00"
+    created: "2021-10-20T14:27:23.496847+01:00"
     dependencies:
     - condition: services.couchdb.enabled
       name: couchdb
@@ -92,8 +115,7 @@ entries:
     - name: ingress-nginx
       repository: https://github.com/kubernetes/ingress-nginx
       version: 3.35.0
-    description: Budibase is an open source low-code platform, helping thousands of
+    description: Budibase is an open source low-code platform, helping thousands of teams build apps for their workplace in minutes.
-      teams build apps for their workplace in minutes.
     digest: 08031b0803cce0eff64472e569d454d9176119c8207aa9873a9c95ee66cc7d3f
     keywords:
     - low-code
@@ -107,4 +129,4 @@ entries:
     urls:
     - https://budibase.github.io/budibase/budibase-0.1.0.tgz
     version: 0.1.0
-generated: "2021-10-18T16:25:04.346266269+01:00"
+generated: "2021-10-20T14:27:23.491132+01:00"

hosting/kubernetes/budibase/Chart.yaml

@@ -22,13 +22,13 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.1
+version: 0.2.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.9.163"
+appVersion: "0.9.169"
 
 dependencies:
   - name: couchdb

hosting/kubernetes/budibase/templates/app-service-deployment.yaml

@@ -82,6 +82,8 @@ spec:
           value: {{ .Values.services.apps.port | quote }}
         - name: MULTI_TENANCY
           value: {{ .Values.globals.multiTenancy | quote }}
+        - name: LOG_LEVEL
+          value: {{ .Values.services.apps.logLevel | quote }}
         - name: REDIS_PASSWORD
           value: {{ .Values.services.redis.password }}
         - name: REDIS_URL

hosting/kubernetes/budibase/templates/worker-service-deployment.yaml

@@ -79,6 +79,8 @@ spec:
           value: {{ .Values.services.worker.port | quote }}
         - name: MULTI_TENANCY
           value: {{ .Values.globals.multiTenancy | quote }}
+        - name: LOG_LEVEL
+          value: {{ .Values.services.worker.logLevel | quote }}
         - name: REDIS_PASSWORD
           value: {{ .Values.services.redis.password | quote }}
         - name: REDIS_URL

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.169-alpha.17",
+  "version": "0.9.172",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.169-alpha.17",
+  "version": "0.9.172",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/auth/src/db/Replication.js

@@ -45,22 +45,6 @@ class Replication {
     return this.replication
   }
 
-  /**
-   * Set up an ongoing live sync between 2 CouchDB databases.
-   * @param {Object} opts - PouchDB replication options
-   */
-  subscribe(opts = {}) {
-    this.replication = this.source.replicate
-      .to(this.target, {
-        live: true,
-        retry: true,
-        ...opts,
-      })
-      .on("error", function (err) {
-        throw new Error(`Replication Error: ${err}`)
-      })
-  }
-
   /**
    * Rollback the target DB back to the state of the source DB
    */

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.169-alpha.17",
+  "version": "0.9.172",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.169-alpha.17",
+  "version": "0.9.172",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,11 +65,11 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.169-alpha.17",
+    "@budibase/bbui": "^0.9.172",
-    "@budibase/client": "^0.9.169-alpha.17",
+    "@budibase/client": "^0.9.172",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.169-alpha.17",
+    "@budibase/string-templates": "^0.9.172",
-    "@sentry/browser": "6.0.0",
+    "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
     "codemirror": "^5.59.0",

packages/builder/src/components/settings/UpdateUserInfoModal.svelte

@@ -26,6 +26,7 @@
   <Body size="S">
     Personalise the platform by adding your first name and last name.
   </Body>
+  <Input disabled bind:value={$auth.user.email} label="Email" />
   <Input bind:value={$values.firstName} label="First name" />
   <Input bind:value={$values.lastName} label="Last name" />
 </ModalContent>

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -1,21 +1,24 @@
 <script>
   import { store, automationStore } from "builderStore"
   import { roles } from "stores/backend"
-  import { Icon, ActionGroup, Tabs, Tab } from "@budibase/bbui"
+  import { Icon, ActionGroup, Tabs, Tab, notifications } from "@budibase/bbui"
   import DeployModal from "components/deploy/DeployModal.svelte"
   import RevertModal from "components/deploy/RevertModal.svelte"
   import VersionModal from "components/deploy/VersionModal.svelte"
   import NPSFeedbackForm from "components/feedback/NPSFeedbackForm.svelte"
-  import { get } from "builderStore/api"
+  import { get, post } from "builderStore/api"
   import { auth, admin } from "stores/portal"
   import { isActive, goto, layout, redirect } from "@roxi/routify"
   import Logo from "assets/bb-emblem.svg"
   import { capitalise } from "helpers"
   import UpgradeModal from "../../../../components/upgrade/UpgradeModal.svelte"
+  import { onMount } from "svelte"
 
   // Get Package and set store
   export let application
   let promise = getPackage()
+  // sync once when you load the app
+  let hasSynced = false
   $: selected = capitalise(
     $layout.children.find(layout => $isActive(layout.path))?.title ?? "data"
   )
@@ -67,6 +70,16 @@
       return state
     })
   }
+
+  onMount(async () => {
+    if (!hasSynced && application) {
+      const res = await post(`/api/applications/${application}/sync`)
+      if (res.status !== 200) {
+        notifications.error("Failed to sync with production database")
+      }
+      hasSynced = true
+    }
+  })
 </script>
 
 {#await promise}

packages/builder/src/pages/builder/portal/manage/users/_components/BasicOnboardingModal.svelte

@@ -16,7 +16,13 @@
     admin = false
 
   async function createUser() {
-    const res = await users.create({ email: $email, password, builder, admin })
+    const res = await users.create({
+      email: $email,
+      password,
+      builder,
+      admin,
+      forceResetPassword: true,
+    })
     if (res.status) {
       notifications.error(res.message)
     } else {

packages/builder/src/stores/portal/auth.js

@@ -60,6 +60,8 @@ export function createAuthStore() {
           name: user.name,
           user_id: user._id,
           tenant: user.tenantId,
+          "Company size": user.size,
+          "Job role": user.profession,
         })
       })
     }

packages/builder/src/stores/portal/users.js

@@ -35,12 +35,21 @@ export function createUsersStore() {
     return await response.json()
   }
 
-  async function create({ email, password, admin, builder }) {
+  async function create({
+    email,
+    password,
+    admin,
+    builder,
+    forceResetPassword,
+  }) {
     const body = {
       email,
       password,
       roles: {},
     }
+    if (forceResetPassword) {
+      body.forceResetPassword = forceResetPassword
+    }
     if (builder) {
       body.builder = { global: true }
     }

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.169-alpha.17",
+  "version": "0.9.172",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.169-alpha.17",
+  "version": "0.9.172",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.169-alpha.17",
+    "@budibase/bbui": "^0.9.172",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^0.9.169-alpha.17",
+    "@budibase/string-templates": "^0.9.172",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.169-alpha.17",
+  "version": "0.9.172",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -68,9 +68,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.169-alpha.17",
+    "@budibase/auth": "^0.9.172",
-    "@budibase/client": "^0.9.169-alpha.17",
+    "@budibase/client": "^0.9.172",
-    "@budibase/string-templates": "^0.9.169-alpha.17",
+    "@budibase/string-templates": "^0.9.172",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",

packages/server/src/api/controllers/application.js

@@ -25,7 +25,12 @@ const { BASE_LAYOUTS } = require("../../constants/layouts")
 const { createHomeScreen } = require("../../constants/screens")
 const { cloneDeep } = require("lodash/fp")
 const { processObject } = require("@budibase/string-templates")
-const { getAllApps } = require("@budibase/auth/db")
+const {
+  getAllApps,
+  isDevAppID,
+  getDeployedAppID,
+  Replication,
+} = require("@budibase/auth/db")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
 const {
   getDeployedApps,
@@ -134,7 +139,7 @@ async function createInstance(template) {
   return { _id: appId }
 }
 
-exports.fetch = async function (ctx) {
+exports.fetch = async ctx => {
   const dev = ctx.query && ctx.query.status === AppStatus.DEV
   const all = ctx.query && ctx.query.status === AppStatus.ALL
   const apps = await getAllApps(CouchDB, { dev, all })
@@ -159,7 +164,7 @@ exports.fetch = async function (ctx) {
   ctx.body = apps
 }
 
-exports.fetchAppDefinition = async function (ctx) {
+exports.fetchAppDefinition = async ctx => {
   const db = new CouchDB(ctx.params.appId)
   const layouts = await getLayouts(db)
   const userRoleId = getUserRoleId(ctx)
@@ -175,7 +180,7 @@ exports.fetchAppDefinition = async function (ctx) {
   }
 }
 
-exports.fetchAppPackage = async function (ctx) {
+exports.fetchAppPackage = async ctx => {
   const db = new CouchDB(ctx.params.appId)
   const application = await db.get(DocumentTypes.APP_METADATA)
   const layouts = await getLayouts(db)
@@ -196,7 +201,7 @@ exports.fetchAppPackage = async function (ctx) {
   }
 }
 
-exports.create = async function (ctx) {
+exports.create = async ctx => {
   const { useTemplate, templateKey, templateString } = ctx.request.body
   const instanceConfig = {
     useTemplate,
@@ -252,13 +257,13 @@ exports.create = async function (ctx) {
   ctx.body = newApplication
 }
 
-exports.update = async function (ctx) {
+exports.update = async ctx => {
   const data = await updateAppPackage(ctx, ctx.request.body, ctx.params.appId)
   ctx.status = 200
   ctx.body = data
 }
 
-exports.updateClient = async function (ctx) {
+exports.updateClient = async ctx => {
   // Get current app version
   const db = new CouchDB(ctx.params.appId)
   const application = await db.get(DocumentTypes.APP_METADATA)
@@ -280,7 +285,7 @@ exports.updateClient = async function (ctx) {
   ctx.body = data
 }
 
-exports.revertClient = async function (ctx) {
+exports.revertClient = async ctx => {
   // Check app can be reverted
   const db = new CouchDB(ctx.params.appId)
   const application = await db.get(DocumentTypes.APP_METADATA)
@@ -303,7 +308,7 @@ exports.revertClient = async function (ctx) {
   ctx.body = data
 }
 
-exports.delete = async function (ctx) {
+exports.delete = async ctx => {
   const db = new CouchDB(ctx.params.appId)
 
   const result = await db.destroy()
@@ -318,6 +323,35 @@ exports.delete = async function (ctx) {
   ctx.body = result
 }
 
+exports.sync = async ctx => {
+  const appId = ctx.params.appId
+  if (!isDevAppID(appId)) {
+    ctx.throw(400, "This action cannot be performed for production apps")
+  }
+  const prodAppId = getDeployedAppID(appId)
+  const replication = new Replication({
+    source: prodAppId,
+    target: appId,
+  })
+  let error
+  try {
+    await replication.replicate({
+      filter: function (doc) {
+        return doc._id !== DocumentTypes.APP_METADATA
+      },
+    })
+  } catch (err) {
+    error = err
+  }
+  if (error) {
+    ctx.throw(400, error)
+  } else {
+    ctx.body = {
+      message: "App sync completed successfully.",
+    }
+  }
+}
+
 const updateAppPackage = async (ctx, appPackage, appId) => {
   const url = await getAppUrlIfNotInUse(ctx)
   const db = new CouchDB(appId)

packages/server/src/api/controllers/auth.js

@@ -28,14 +28,23 @@ exports.fetchSelf = async ctx => {
         ...metadata,
       })
     } catch (err) {
+      let response
       // user didn't exist in app, don't pretend they do
       if (user.roleId === BUILTIN_ROLE_IDS.PUBLIC) {
-        ctx.body = {}
+        response = {}
       }
       // user has a role of some sort, return them
-      else {
+      else if (err.status === 404) {
-        ctx.body = user
+        const metadata = {
+          _id: userId,
+        }
+        const dbResp = await db.put(metadata)
+        user._rev = dbResp.rev
+        response = user
+      } else {
+        response = user
       }
+      ctx.body = response
     }
   } else {
     ctx.body = user

packages/server/src/api/controllers/deploy/index.js

@@ -1,6 +1,6 @@
 const CouchDB = require("../../../db")
 const Deployment = require("./Deployment")
-const { Replication } = require("@budibase/auth/db")
+const { Replication, getDeployedAppID } = require("@budibase/auth/db")
 const { DocumentTypes, getAutomationParams } = require("../../../db/utils")
 const {
   disableAllCrons,
@@ -87,7 +87,7 @@ async function initDeployedApp(prodAppId) {
 
 async function deployApp(deployment) {
   try {
-    const productionAppId = deployment.appId.replace("_dev", "")
+    const productionAppId = getDeployedAppID(deployment.appId)
 
     const replication = new Replication({
       source: deployment.appId,
@@ -104,23 +104,8 @@ async function deployApp(deployment) {
     appDoc.instance._id = productionAppId
     await db.put(appDoc)
     console.log("New app doc written successfully.")
-
-    console.log("Setting up live repl between dev and prod")
-    // Set up live sync between the live and dev instances
-    const liveReplication = new Replication({
-      source: productionAppId,
-      target: deployment.appId,
-    })
-    liveReplication.subscribe({
-      filter: function (doc) {
-        return doc._id !== DocumentTypes.APP_METADATA
-      },
-    })
-    console.log("Set up live repl between dev and prod")
-
-    console.log("Initialising deployed app")
     await initDeployedApp(productionAppId)
-    console.log("Init complete, setting deployment to successful")
+    console.log("Deployed app initialised, setting deployment to successful")
     deployment.setStatus(DeploymentStatus.SUCCESS)
     await storeDeploymentHistory(deployment)
   } catch (err) {

packages/server/src/api/routes/application.js

@@ -7,6 +7,7 @@ const usage = require("../../middleware/usageQuota")
 const router = Router()
 
 router
+  .post("/api/applications/:appId/sync", authorized(BUILDER), controller.sync)
   .post("/api/applications", authorized(BUILDER), usage, controller.create)
   .get("/api/applications/:appId/definition", controller.fetchAppDefinition)
   .get("/api/applications", controller.fetch)

packages/server/src/db/linkedRows/index.js

@@ -81,7 +81,9 @@ async function getFullLinkedDocs(ctx, appId, links) {
     row => row.doc
   )
   // convert the unique db rows back to a full list of linked rows
-  const linked = linkedRowIds.map(id => dbRows.find(row => row._id === id))
+  const linked = linkedRowIds
+    .map(id => dbRows.find(row => row && row._id === id))
+    .filter(row => row != null)
   // need to handle users as specific cases
   let [users, other] = partition(linked, linkRow =>
     linkRow._id.startsWith(USER_METDATA_PREFIX)
@@ -172,13 +174,18 @@ exports.attachFullLinkedDocs = async (ctx, table, rows) => {
         row[link.fieldName] = []
       }
       const linkedRow = linked.find(row => row._id === link.id)
-      const linkedTableId =
+      if (linkedRow) {
-        linkedRow.tableId || getRelatedTableForField(table, link.fieldName)
+        const linkedTableId =
-      const linkedTable = await getLinkedTable(db, linkedTableId, linkedTables)
+          linkedRow.tableId || getRelatedTableForField(table, link.fieldName)
-      if (!linkedRow || !linkedTable) {
+        const linkedTable = await getLinkedTable(
-        continue
+          db,
+          linkedTableId,
+          linkedTables
+        )
+        if (linkedTable) {
+          row[link.fieldName].push(processFormulas(linkedTable, linkedRow))
+        }
       }
-      row[link.fieldName].push(processFormulas(linkedTable, linkedRow))
     }
   }
   return rows

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "0.9.169-alpha.17",
+  "version": "0.9.172",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "0.9.169-alpha.17",
+  "version": "0.9.172",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -29,8 +29,8 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.169-alpha.17",
+    "@budibase/auth": "^0.9.172",
-    "@budibase/string-templates": "^0.9.169-alpha.17",
+    "@budibase/string-templates": "^0.9.172",
     "@koa/router": "^8.0.0",
     "@sentry/node": "^6.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",

packages/worker/src/api/routes/tests/realEmail.spec.js

@@ -3,9 +3,6 @@ const { EmailTemplatePurpose } = require("../../../constants")
 const nodemailer = require("nodemailer")
 const fetch = require("node-fetch")
 
-// need a longer timeout for getting these
-jest.setTimeout(30000)
-
 describe("/api/global/email", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()

packages/worker/src/api/routes/tests/utilities/TestConfiguration.js

@@ -234,6 +234,7 @@ class TestConfiguration {
             user: "don.bahringer@ethereal.email",
             pass: "yCKSH8rWyUPbnhGYk9",
           },
+          connectionTimeout: 1000, // must be less than the jest default of 5000
         },
       },
       null,

packages/worker/src/utilities/email.js

@@ -35,6 +35,9 @@ function createSMTPTransport(config) {
     options.tls = {
       rejectUnauthorized: false,
     }
+    if (config.connectionTimeout) {
+      options.connectionTimeout = config.connectionTimeout
+    }
   } else {
     options = {
       port: 587,