Merge branch 'master' into feature/layout-poc

.github/workflows/budibase_ci.yml

@@ -45,9 +45,10 @@ jobs:
 
     - name: Build and Push Staging Docker Image
       # Only run on push
-      if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+      if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' }}
       run: |
         docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+        yarn build
         yarn build:docker:staging
       env:
         DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}

.github/workflows/release.yml

@@ -4,6 +4,11 @@ on:
  push:
     branches:
       - master
+
+env:
+  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
+  POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
+  SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
       
 jobs:
   release:
@@ -18,10 +23,6 @@ jobs:
       - run: yarn lint 
       - run: yarn bootstrap 
       - run: yarn build 
-        env:
-          POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
-          POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
-          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
       - run: yarn test
 
       - name: Configure AWS Credentials

hosting/docker-compose.yaml

@@ -6,6 +6,7 @@ services:
   app-service:
     restart: always
     image: budibase/apps
+    container_name: bbapps
     ports:
       - "${APP_PORT}:4002"
     environment:
@@ -32,6 +33,7 @@ services:
   worker-service:
     restart: always
     image: budibase/worker
+    container_name: bbworker
     ports:
       - "${WORKER_PORT}:4003"
     environment:
@@ -118,7 +120,7 @@ services:
     image: containrrr/watchtower
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
-    command: --debug --http-api-update budibase/apps budibase/worker
+    command: --debug --http-api-update bbapps bbworker
     environment:
       - WATCHTOWER_HTTP_API=true
       - WATCHTOWER_HTTP_API_TOKEN=budibase

hosting/scripts/linux/release-to-docker-hub.sh

@@ -3,7 +3,7 @@
 tag=$1
 tag=${tag:-latest}
 
-echo "Tagging images with SHA: $GITHUB_SHA and version: $BUDIBASE_VERSION"
+echo "Tagging images with SHA: $GITHUB_SHA and tag: $tag"
 
 docker tag app-service budibase/apps:$tag
 docker tag worker-service budibase/worker:$tag

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.24",
+  "version": "0.9.34",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -39,6 +39,6 @@
     "test:e2e": "lerna run cy:test",
     "test:e2e:ci": "lerna run cy:ci",
     "build:docker": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh && cd -",
-    "build:docker:staging": "cd hosting/scripts/linux/ && ./release-to-docker-hub.sh staging && cd -"
+    "build:docker:staging": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh staging && cd -"
   }
 }

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.24",
+  "version": "0.9.34",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/auth/src/db/utils.js

@@ -1,6 +1,5 @@
 const { newid } = require("../hashing")
 const Replication = require("./Replication")
-const { getCouch } = require("./index")
 
 const UNICODE_MAX = "\ufff0"
 const SEPARATOR = "_"
@@ -164,14 +163,14 @@ exports.getDeployedAppID = appId => {
  * different users/companies apps as there is no security around it - all apps are returned.
  * @return {Promise<object[]>} returns the app information document stored in each app database.
  */
-exports.getAllApps = async ({ dev, all } = {}) => {
+exports.getAllApps = async ({ CouchDB, dev, all } = {}) => {
-  const CouchDB = getCouch()
   let allDbs = await CouchDB.allDbs()
   const appDbNames = allDbs.filter(dbName =>
     dbName.startsWith(exports.APP_PREFIX)
   )
   const appPromises = appDbNames.map(db =>
-    new CouchDB(db).get(DocumentTypes.APP_METADATA)
+    // skip setup otherwise databases could be re-created
+    new CouchDB(db, { skip_setup: true }).get(DocumentTypes.APP_METADATA)
   )
   if (appPromises.length === 0) {
     return []
@@ -196,6 +195,21 @@ exports.getAllApps = async ({ dev, all } = {}) => {
   }
 }
 
+exports.dbExists = async (CouchDB, dbName) => {
+  let exists = false
+  try {
+    const db = CouchDB(dbName, { skip_setup: true })
+    // check if database exists
+    const info = await db.info()
+    if (info && !info.error) {
+      exists = true
+    }
+  } catch (err) {
+    exists = false
+  }
+  return exists
+}
+
 /**
  * Generates a new configuration ID.
  * @returns {string} The new configuration ID which the config doc can be stored under.

packages/auth/src/middleware/passport/google.js

@@ -20,16 +20,10 @@ async function authenticate(token, tokenSecret, profile, done) {
     // use the google profile id
     dbUser = await db.get(userId)
   } catch (err) {
-    console.log("Google user not found. Creating..")
-
-    // create the user
     const user = {
       _id: userId,
       provider: profile.provider,
       roles: {},
-      builder: {
-        global: true,
-      },
       ...profile._json,
     }
 
@@ -50,12 +44,18 @@ async function authenticate(token, tokenSecret, profile, done) {
       user.roles = existing.roles
       user.builder = existing.builder
       user.admin = existing.admin
+
+      const response = await db.post(user)
+      dbUser = user
+      dbUser._rev = response.rev
+    } else {
+      return done(
+        new Error(
+          "email does not yet exist. You must set up your local budibase account first."
+        ),
+        false
+      )
     }
-
-    const response = await db.post(user)
-
-    dbUser = user
-    dbUser._rev = response.rev
   }
 
   // authenticate

packages/auth/src/objectStore/index.js

@@ -159,7 +159,7 @@ exports.upload = async ({
  * Similar to the upload function but can be used to send a file stream
  * through to the object store.
  */
-exports.streamUpload = async (bucketName, filename, stream) => {
+exports.streamUpload = async (bucketName, filename, stream, extra = {}) => {
   const objectStore = exports.ObjectStore(bucketName)
   await exports.makeSureBucketExists(objectStore, bucketName)
 
@@ -167,6 +167,7 @@ exports.streamUpload = async (bucketName, filename, stream) => {
     Bucket: sanitizeBucket(bucketName),
     Key: sanitizeKey(filename),
     Body: stream,
+    ...extra,
   }
   return objectStore.upload(params).promise()
 }

packages/auth/src/security/roles.js

@@ -13,7 +13,6 @@ const BUILTIN_IDS = {
   POWER: "POWER",
   BASIC: "BASIC",
   PUBLIC: "PUBLIC",
-  BUILDER: "BUILDER",
 }
 
 // exclude internal roles like builder

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.24",
+  "version": "0.9.34",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/bbui/src/Form/Core/RadioGroup.svelte

@@ -37,3 +37,9 @@
     {/each}
   {/if}
 </div>
+
+<style>
+  .spectrum-Radio-input {
+    opacity: 0;
+  }
+</style>

packages/builder/cypress.json

@@ -1,9 +1,9 @@
 {
-  "baseUrl": "http://localhost:10000/builder/",
+  "baseUrl": "http://localhost:10001/builder/",
   "video": true,
   "projectId": "bmbemn",
   "env": {
-    "PORT": "10000",
+    "PORT": "10001",
     "JWT_SECRET": "test"
   }
 }

packages/builder/cypress/integration/createAutomation.spec.js

@@ -8,31 +8,27 @@ context("Create a automation", () => {
   it("should create a automation", () => {
     cy.createTestTableWithData()
 
-    cy.contains("automate").click()
+    cy.contains("Automate").click()
-    cy.get("[data-cy=new-automation]").click()
+    cy.get("[data-cy='new-screen'] > .spectrum-Icon").click()
-    cy.get(".modal").within(() => {
+    cy.get(".spectrum-Dialog-grid").within(() => {
       cy.get("input").type("Add Row")
-      cy.get(".buttons")
+      cy.get(".spectrum-Button--cta").click()
-        .contains("Create")
-        .click()
     })
 
     // Add trigger
     cy.contains("Trigger").click()
     cy.contains("Row Created").click()
     cy.get(".setup").within(() => {
-      cy.get("select")
+      cy.get(".spectrum-Picker-label").click()
-        .first()
+      cy.contains("dog").click()
-        .select("dog")
     })
 
     // Create action
     cy.contains("Action").click()
     cy.contains("Create Row").click()
     cy.get(".setup").within(() => {
-      cy.get("select")
+      cy.get(".spectrum-Picker-label").click()
-        .first()
+      cy.contains("dog").click()
-        .select("dog")
       cy.get("input")
         .first()
         .type("goodboy")
@@ -45,12 +41,11 @@ context("Create a automation", () => {
     cy.contains("Save Automation").click()
 
     // Activate Automation
-    cy.get("[data-cy=activate-automation]").click()
+    cy.get("[aria-label=PlayCircle]").click()
-    cy.get(".ri-stop-circle-fill.highlighted").should("be.visible")
   })
 
   it("should add row when a new row is added", () => {
-    cy.contains("data").click()
+    cy.contains("Data").click()
     cy.addRow(["Rover", 15])
     cy.reload()
     cy.contains("goodboy").should("have.text", "goodboy")

packages/builder/cypress/integration/createBinding.spec.js

@@ -36,7 +36,9 @@ context("Create Bindings", () => {
   it("should add a binding with a handlebars helper", () => {
     cy.addComponent("Elements", "Paragraph").then(componentId => {
       // Cypress needs to escape curly brackets
-      addSettingBinding("text", "{{}{{} add 1 2 {}}{}}", false)
+      cy.get("[data-cy=setting-text] input")
+        .type("{{}{{} add 1 2 {}}{}}")
+        .blur()
       cy.getComponent(componentId).should("have.text", "3")
     })
   })
@@ -51,6 +53,6 @@ const addSettingBinding = (setting, bindingText, clickOption = true) => {
     } else {
       cy.get("textarea").type(bindingText)
     }
-    cy.get("button").click()
+    cy.contains("Save").click()
   })
 }

packages/builder/cypress/integration/createComponents.spec.js

@@ -1,13 +1,14 @@
-context("Create Components", () => {
+// TODO for now components are skipped, might not be good to keep doing this
+xcontext("Create Components", () => {
   let headlineId
 
   before(() => {
     cy.login()
     cy.createTestApp()
     cy.createTable("dog")
-    cy.addColumn("dog", "name", "string")
+    cy.addColumn("dog", "name", "Text")
-    cy.addColumn("dog", "age", "number")
+    cy.addColumn("dog", "age", "Number")
-    cy.addColumn("dog", "type", "options")
+    cy.addColumn("dog", "type", "Options")
     cy.navigateToFrontend()
   })
 

packages/builder/cypress/integration/createTable.spec.js

@@ -22,54 +22,49 @@ context("Create a Table", () => {
   })
 
   it("updates a column on the table", () => {
-    cy.contains("header", "name")
+    cy.get(".title").click()
-      .trigger("mouseover")
+    cy.get(".spectrum-Table-editIcon > use").click()
-      .find(".ri-pencil-line")
+    cy.get("input")
-      .click({ force: true })
+      .eq(1)
-    cy.get(".actions input")
+      .type("updated", { force: true })
-      .first()
-      .type("updated")
     // Unset table display column
-    cy.contains("display column").click()
+    cy.get(".spectrum-Switch-input").eq(1).click()
     cy.contains("Save Column").click()
-    cy.contains("nameupdated ").should("have.text", "nameupdated ")
+    cy.contains("nameupdated ").should("contain", "nameupdated")
   })
 
   it("edits a row", () => {
     cy.contains("button", "Edit").click({ force: true })
     cy.wait(1000)
-    cy.get(".modal input").type("Updated")
+    cy.get(".spectrum-Modal input").type("Updated")
     cy.contains("Save").click()
     cy.contains("RoverUpdated").should("have.text", "RoverUpdated")
   })
 
   it("deletes a row", () => {
-    cy.get(".ag-checkbox-input").check({ force: true })
+    cy.get(".spectrum-Checkbox-input").check({ force: true })
     cy.contains("Delete 1 row(s)").click()
-    cy.get(".modal")
+    cy.get(".spectrum-Modal")
       .contains("Delete")
       .click()
     cy.contains("RoverUpdated").should("not.exist")
   })
 
   it("deletes a column", () => {
-    cy.contains("header", "name")
+    cy.get(".title").click()
-      .trigger("mouseover")
+    cy.get(".spectrum-Table-editIcon > use").click()
-      .find(".ri-pencil-line")
-      .click({ force: true })
     cy.contains("Delete").click()
     cy.wait(50)
-    cy.get(".buttons")
+    cy.contains("Delete Column")
-      .contains("Delete")
       .click()
     cy.contains("nameupdated").should("not.exist")
   })
 
   it("deletes a table", () => {
-    cy.get(".ri-more-line")
+    cy.get(".actions > :nth-child(1) > .icon > .spectrum-Icon > use")
       .first()
       .click({ force: true })
-    cy.get("[data-cy=delete-table]").click()
+    cy.get(".spectrum-Menu > :nth-child(2)").click()
     cy.contains("Delete Table").click()
     cy.contains("dog").should("not.exist")
   })

packages/builder/cypress/integration/createUser.spec.js

@@ -1,11 +1,10 @@
 context("Create a User", () => {
   before(() => {
     cy.login()
-    cy.createTestApp()
   })
 
   it("should create a user", () => {
-    cy.createUser("bbuser@test.com", "test", "ADMIN")
+    cy.createUser("bbuser@test.com")
     cy.contains("bbuser").should("be.visible")
   })
 })

packages/builder/cypress/integration/createView.spec.js

@@ -17,21 +17,21 @@ context("Create a View", () => {
   })
 
   it("creates a view", () => {
-    cy.contains("Create New View").click()
+    cy.contains("Create view").click()
-    cy.get(".menu-container").within(() => {
+    cy.get(".modal-inner-wrapper").within(() => {
       cy.get("input").type("Test View")
-      cy.contains("Save View").click()
+      cy.get("button").contains("Create View").click({ force: true })
     })
     cy.get(".table-title h1").contains("Test View")
-    cy.get("[data-cy=table-header]").then($headers => {
+    cy.get(".title").then($headers => {
       expect($headers).to.have.length(3)
       const headers = Array.from($headers).map(header =>
         header.textContent.trim()
       )
       expect(removeSpacing(headers)).to.deep.eq([
-        "rating Number",
+        "group",
-        "age Number",
+        "age",
-        "group Text",
+        "rating",
       ])
     })
   })
@@ -39,97 +39,95 @@ context("Create a View", () => {
   it("filters the view by age over 10", () => {
     cy.contains("Filter").click()
     cy.contains("Add Filter").click()
-    cy.get(".menu-container")
+
-      .find("select")
+    cy.get(".modal-inner-wrapper").within(() => {
-      .first()
+      cy.get(".spectrum-Picker-label").eq(0).click()
-      .select("age")
+      cy.contains("age").click({ force: true })
-    cy.get(".menu-container")
+
-      .find("select")
+      cy.get(".spectrum-Picker-label").eq(1).click()
-      .eq(1)
+      cy.contains("More Than").click({ force: true })
-      .select("More Than")
+
-    cy.get(".menu-container")
+      cy.get("input").type(18)
-      .find("input")
+      cy.contains("Save").click()
-      .type(18)
+    })
-    cy.contains("Save").click()
+
-    cy.get("[role=rowgroup] .ag-row").get($values => {
+    cy.get(".spectrum-Table-row").get($values => {
       expect($values).to.have.length(5)
     })
   })
 
   it("creates a stats calculation view based on age", () => {
-    // Required due to responsive bug with ag grid in cypress
-    cy.viewport("macbook-15")
-
     cy.contains("Calculate").click()
-    cy.get(".menu-container")
+    cy.get(".modal-inner-wrapper").within(() => {
-      .find("select")
+      cy.get(".spectrum-Picker-label").eq(0).click()
-      .eq(0)
+      cy.contains("Statistics").click()
-      .select("Statistics")
+      
-    cy.wait(50)
+      cy.get(".spectrum-Picker-label").eq(1).click()
-    cy.get(".menu-container")
+      cy.contains("age").click({ force: true })
-      .find("select")
+
-      .eq(1)
+      cy.contains("Save").click()
-      .select("age")
+    })
-    cy.contains("Save").click()
     cy.wait(1000)
-    cy.get(".ag-center-cols-viewport").scrollTo("100%")
+
-    cy.get("[data-cy=table-header]").then($headers => {
+    cy.get(".title").then($headers => {
       expect($headers).to.have.length(7)
       const headers = Array.from($headers).map(header =>
         header.textContent.trim()
       )
       expect(removeSpacing(headers)).to.deep.eq([
-        "avg Number",
+        "field",
-        "sumsqr Number",
+        "sum",
-        "count Number",
+        "min",
-        "max Number",
+        "max",
-        "min Number",
+        "count",
-        "sum Number",
+        "sumsqr",
-        "field Text",
+        "avg",
       ])
     })
-    cy.get(".ag-cell").then($values => {
+    cy.get(".spectrum-Table-cell").then($values => {
       let values = Array.from($values).map(header => header.textContent.trim())
-      expect(values).to.deep.eq(["31", "5347", "5", "49", "20", "155", "age"])
+      expect(values).to.deep.eq(["age", "155", "20", "49", "5", "5347", "31"])
     })
   })
 
   it("groups the view by group", () => {
-    // Required due to responsive bug with ag grid in cypress
+    cy.contains("Group by").click()
-    cy.viewport("macbook-15")
+    cy.get(".modal-inner-wrapper").within(() => {
-
+      cy.get(".spectrum-Picker-label").eq(0).click()
-    cy.contains("Group By").click()
+      cy.contains("group").click()
-    cy.get("select").select("group")
+      cy.contains("Save").click()
-    cy.contains("Save").click()
+    })
     cy.wait(1000)
-    cy.get(".ag-center-cols-viewport").scrollTo("100%")
     cy.contains("Students").should("be.visible")
     cy.contains("Teachers").should("be.visible")
 
-    cy.get(".ag-row[row-index=0]")
+    cy.get(".spectrum-Table-cell").then($values => {
-      .find(".ag-cell")
+      let values = Array.from($values).map(header => header.textContent.trim())
-      .then($values => {
+      expect(values).to.deep.eq([
-        const values = Array.from($values).map(value => value.textContent)
-        expect(values.sort()).to.deep.eq(
-          [
             "Students",
-            "23.333333333333332",
-            "1650",
-            "3",
-            "25",
-            "20",
             "70",
-          ].sort()
+            "20",
-        )
+            "25",
-      })
+            "3",
+            "1650",
+            "23.333333333333332",
+            "Teachers",
+            "85",
+            "36",
+            "49",
+            "2",
+            "3697",
+            "42.5",
+      ])
+    })
   })
 
   it("renames a view", () => {
     cy.contains(".nav-item", "Test View")
-      .find(".ri-more-line")
+      .find(".actions .icon")
       .click({ force: true })
-    cy.get("[data-cy=edit-view]").click()
+    cy.contains("Edit").click()
-    cy.get(".menu-container").within(() => {
+    cy.get(".modal-inner-wrapper").within(() => {
       cy.get("input").type(" Updated")
       cy.contains("Save").click()
     })
@@ -139,9 +137,9 @@ context("Create a View", () => {
 
   it("deletes a view", () => {
     cy.contains(".nav-item", "Test View Updated")
-      .find(".ri-more-line")
+      .find(".actions .icon")
       .click({ force: true })
-    cy.get("[data-cy=delete-view]").click()
+    cy.contains("Delete").click()
     cy.contains("Delete View").click()
     cy.wait(1000)
     cy.contains("TestView Updated").should("not.be.visible")

packages/builder/cypress/setup.js

@@ -3,7 +3,9 @@ const path = require("path")
 
 const tmpdir = path.join(require("os").tmpdir(), ".budibase")
 
-const WORKER_PORT = "4002"
+// these run on ports we don't normally use so that they can run alongside the
+// normal development system
+const WORKER_PORT = "10002"
 const MAIN_PORT = cypressConfig.env.PORT
 process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE"
 process.env.NODE_ENV = "cypress"
@@ -12,8 +14,8 @@ process.env.PORT = MAIN_PORT
 process.env.JWT_SECRET = cypressConfig.env.JWT_SECRET
 process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
 process.env.SELF_HOSTED = 1
-process.env.WORKER_URL = "http://localhost:4002/"
+process.env.WORKER_URL = "http://localhost:10002/"
-process.env.MINIO_URL = "http://localhost:10000/"
+process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/`
 process.env.MINIO_ACCESS_KEY = "budibase"
 process.env.MINIO_SECRET_KEY = "budibase"
 process.env.COUCH_DB_USER = "budibase"

packages/builder/cypress/support/commands.js

@@ -6,80 +6,61 @@
 //
 
 Cypress.Commands.add("login", () => {
-  cy.getCookie("budibase:auth").then(cookie => {
-    // Already logged in
-    if (cookie) return
-
-    cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
-
-    // cy.get("button").then(btn => {
-    //   const adminUserButton = "Create super admin user"
-    //   console.log(btn.first().first())
-    //   if (!btn.first().contains(adminUserButton)) {
-    //     // create admin user
-    //     cy.get("input").first().type("test@test.com")
-    //     cy.get('input[type="password"]').first().type("test")
-    //     cy.get('input[type="password"]').eq(1).type("test")
-    //     cy.contains(adminUserButton).click()
-    //   }
-
-    // login
-    cy.get("input").first().type("test@test.com")
-    cy.get('input[type="password"]').type("test")
-    cy.contains("Login").click()
-    // })
-  })
-})
-
-Cypress.Commands.add("createApp", name => {
   cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
-  // wait for init API calls on visit
+  cy.wait(500)
-  cy.wait(100)
+  cy.url().then(url => {
-  cy.contains("Create New Web App").click()
+    if (url.includes("builder/admin")) {
-  cy.get("body")
+      // create admin user
-    .then($body => {
+      cy.get("input").first().type("test@test.com")
-      if ($body.find("input[name=apiKey]").length) {
+      cy.get('input[type="password"]').first().type("test")
-        // input was found, do something else here
+      cy.get('input[type="password"]').eq(1).type("test")
-        cy.get("input[name=apiKey]").type(name).should("have.value", name)
+      cy.contains("Create super admin user").click()
-        cy.contains("Next").click()
+    }
-      }
+    if (url.includes("builder/auth/login") || url.includes("builder/admin")) {
-    })
+      // login
-    .then(() => {
+      cy.contains("Sign in to Budibase").then(() => {
-      cy.get(".spectrum-Modal")
+        cy.get("input").first().type("test@test.com")
-        .within(() => {
+        cy.get('input[type="password"]').type("test")
-          cy.get("input").eq(0).type(name).should("have.value", name).blur()
+        cy.get("button").first().click()
-          cy.contains("Next").click()
-          cy.get("input").eq(1).type("test@test.com").blur()
-          cy.get("input").eq(2).type("test").blur()
-          cy.contains("Submit").click()
-        })
-        .then(() => {
-          cy.get("[data-cy=new-table]", {
-            timeout: 20000,
-          }).should("be.visible")
-        })
-    })
-})
-
-Cypress.Commands.add("deleteApp", name => {
-  cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
-  cy.get(".apps").then($apps => {
-    cy.wait(1000)
-    if ($apps.find(`[data-cy="app-${name}"]`).length) {
-      cy.get(`[data-cy="app-${name}"]`).contains("Open").click()
-      cy.get("[data-cy=settings-icon]").click()
-      cy.get(".spectrum-Dialog").within(() => {
-        cy.contains("Danger Zone").click()
-        cy.get("input").type("DELETE").blur()
-        cy.contains("Delete Entire App").click()
       })
     }
   })
 })
 
+Cypress.Commands.add("createApp", name => {
+  cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
+  cy.wait(500)
+  cy.contains(/Create (new )?app/).click()
+  cy.get(".spectrum-Modal")
+    .within(() => {
+      cy.get("input").eq(0).type(name).should("have.value", name).blur()
+      cy.contains("Create app").click()
+    })
+    .then(() => {
+      cy.get("[data-cy=new-table]", {
+        timeout: 20000,
+      }).should("be.visible")
+    })
+})
+
+Cypress.Commands.add("deleteApp", () => {
+  cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
+  cy.wait(1000)
+  cy.request(`localhost:${Cypress.env("PORT")}/api/applications?status=all`)
+    .its("body")
+    .then(val => {
+      console.log(val)
+      if (val.length > 0) {
+        cy.get(".hoverable > use").click()
+        cy.contains("Delete").click()
+        cy.get(".spectrum-Button--warning").click()
+      }
+    })
+})
+
 Cypress.Commands.add("createTestApp", () => {
   const appName = "Cypress Tests"
-  cy.deleteApp(appName)
+  cy.deleteApp()
   cy.createApp(appName, "This app is used for Cypress testing.")
 })
 
@@ -110,8 +91,10 @@ Cypress.Commands.add("addColumn", (tableName, columnName, type) => {
 
     // Unset table display column
     cy.contains("display column").click({ force: true })
-    cy.get("select").select(type)
+    cy.get(".spectrum-Picker-label").click()
-    cy.contains("Save").click()
+    cy.contains(type).click()
+
+    cy.contains("Save Column").click()
   })
 })
 
@@ -125,18 +108,18 @@ Cypress.Commands.add("addRow", values => {
   })
 })
 
-Cypress.Commands.add("createUser", (email, password, role) => {
+Cypress.Commands.add("createUser", email => {
-  // Create User
+  // quick hacky recorded way to create a user
   cy.contains("Users").click()
-  cy.contains("Create user").click()
+  cy.get(".spectrum-Button--primary").click()
-  cy.get(".spectrum-Modal").within(() => {
+  cy.get(".spectrum-Picker-label").click()
-    cy.get("input").first().type(email).blur()
+  cy.get(".spectrum-Menu-item:nth-child(2) > .spectrum-Menu-itemLabel").click()
-    cy.get("input").eq(1).type(password).blur()
+  cy.get(
-    cy.get("select").first().select(role)
+    ":nth-child(2) > .spectrum-Form-itemField > .spectrum-Textfield > .spectrum-Textfield-input"
-
+  )
-    // Save
+    .first()
-    cy.get(".spectrum-ButtonGroup").contains("Create User").click()
+    .type(email, { force: true })
-  })
+  cy.get(".spectrum-Button--cta").click({ force: true })
 })
 
 Cypress.Commands.add("addComponent", (category, component) => {
@@ -165,17 +148,16 @@ Cypress.Commands.add("getComponent", componentId => {
 })
 
 Cypress.Commands.add("navigateToFrontend", () => {
-  cy.contains("design").click()
+  cy.wait(1000)
+  cy.contains("Design").click()
 })
 
 Cypress.Commands.add("createScreen", (screenName, route) => {
-  cy.get("[data-cy=new-screen]").click()
+  cy.get("[aria-label=AddCircle]").click()
   cy.get(".spectrum-Modal").within(() => {
-    cy.get("input").eq(0).type(screenName).blur()
+    cy.get("input").first().type(screenName)
-    if (route) {
+    cy.get("input").eq(1).type(route)
-      cy.get("input").eq(1).type(route).blur()
+    cy.get(".spectrum-Button--cta").click()
-    }
-    cy.contains("Create Screen").click()
   })
   cy.get(".nav-items-container").within(() => {
     cy.contains(route).should("exist")

packages/builder/cypress/support/index.js

@@ -14,8 +14,8 @@
 // ***********************************************************
 
 // Import commands.js using ES2015 syntax:
-import "./cookies"
 import "./commands"
+import "./cookies"
 
 // Alternatively you can use CommonJS syntax:
 // require('./commands')

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.24",
+  "version": "0.9.34",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -15,9 +15,9 @@
     "cy:run": "cypress run",
     "cy:open": "cypress open",
     "cy:run:ci": "cypress run --record --key f308590b-6070-41af-b970-794a3823d451",
-    "cy:test": "start-server-and-test cy:setup http://localhost:10000/builder cy:run",
+    "cy:test": "start-server-and-test cy:setup http://localhost:10001/builder cy:run",
-    "cy:ci": "start-server-and-test cy:setup http://localhost:10000/builder cy:run:ci",
+    "cy:ci": "start-server-and-test cy:setup http://localhost:10001/builder cy:run:ci",
-    "cy:debug": "start-server-and-test cy:setup http://localhost:10000/builder cy:open"
+    "cy:debug": "start-server-and-test cy:setup http://localhost:10001/builder cy:open"
   },
   "jest": {
     "globals": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.24",
+    "@budibase/bbui": "^0.9.34",
-    "@budibase/client": "^0.9.24",
+    "@budibase/client": "^0.9.34",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.24",
+    "@budibase/string-templates": "^0.9.34",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/builderStore/dataBinding.js

@@ -136,7 +136,7 @@ const getContextBindings = (asset, componentId) => {
       if (!datasource) {
         return
       }
-      const info = getSchemaForDatasource(datasource)
+      const info = getSchemaForDatasource(asset, datasource)
       schema = info.schema
       readablePrefix = info.table?.name
     }
@@ -191,7 +191,7 @@ const getContextBindings = (asset, componentId) => {
  */
 const getUserBindings = () => {
   let bindings = []
-  const { schema } = getSchemaForDatasource({
+  const { schema } = getSchemaForDatasource(null, {
     type: "table",
     tableId: TableNames.USERS,
   })
@@ -244,11 +244,15 @@ const getUrlBindings = asset => {
 /**
  * Gets a schema for a datasource object.
  */
-export const getSchemaForDatasource = (datasource, isForm = false) => {
+export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
   let schema, table
   if (datasource) {
     const { type } = datasource
-    if (type === "query") {
+    if (type === "provider") {
+      const component = findComponent(asset.props, datasource.providerId)
+      const source = getDatasourceForProvider(asset, component)
+      return getSchemaForDatasource(asset, source, isForm)
+    } else if (type === "query") {
       const queries = get(queriesStores).list
       table = queries.find(query => query._id === datasource._id)
     } else {

packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js

@@ -174,7 +174,7 @@ const fieldTypeToComponentMap = {
 }
 
 export function makeDatasourceFormComponents(datasource) {
-  const { schema } = getSchemaForDatasource(datasource, true)
+  const { schema } = getSchemaForDatasource(null, datasource, true)
   let components = []
   let fields = Object.keys(schema || {})
   fields.forEach(field => {

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -43,6 +43,7 @@
   }
 
   let originalName = field.name
+  const linkEditDisabled = originalName != null
   let primaryDisplay =
     $tables.selected.primaryDisplay == null ||
     $tables.selected.primaryDisplay === field.name
@@ -99,6 +100,7 @@
       tables.deleteField(field)
       notifications.success(`Column ${field.name} deleted.`)
       confirmDeleteDialog.hide()
+      hide()
       deletion = false
     }
   }
@@ -197,7 +199,7 @@
   <Input
     label="Name"
     bind:value={field.name}
-    disabled={uneditable || field.type === LINK_TYPE}
+    disabled={uneditable || (linkEditDisabled && field.type === LINK_TYPE)}
   />
 
   <Select
@@ -284,6 +286,7 @@
   {:else if field.type === "link"}
     <Select
       label="Table"
+      disabled={linkEditDisabled}
       bind:value={field.tableId}
       options={tableOptions}
       getOptionLabel={table => table.name}
@@ -291,7 +294,7 @@
     />
     {#if relationshipOptions && relationshipOptions.length > 0}
       <RadioGroup
-        disabled={originalName != null}
+        disabled={linkEditDisabled}
         label="Define the relationship"
         bind:value={field.relationshipType}
         options={relationshipOptions}
@@ -299,7 +302,11 @@
         getOptionValue={option => option.value}
       />
     {/if}
-    <Input label={`Column name in other table`} bind:value={field.fieldName} />
+    <Input
+      disabled={linkEditDisabled}
+      label={`Column name in other table`}
+      bind:value={field.fieldName}
+    />
   {:else if field.type === FORMULA_TYPE}
     <ModalBindableInput
       title="Handlebars Formula"

packages/builder/src/components/design/AppPreview/componentStructure.json

@@ -19,7 +19,8 @@
       "longformfield",
       "datetimefield",
       "attachmentfield",
-      "relationshipfield"
+      "relationshipfield",
+      "daterangepicker"
     ]
   },
   {

packages/builder/src/components/design/NavigationPanel/NewScreenModal.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { store, allScreens } from "builderStore"
+  import { store, allScreens, selectedAccessRole } from "builderStore"
   import { tables } from "stores/backend"
   import { roles } from "stores/backend"
   import { Input, Select, ModalContent, Toggle } from "@budibase/bbui"
@@ -14,7 +14,7 @@
   let templateIndex
   let draftScreen
   let createLink = true
-  let roleId = "BASIC"
+  let roleId = $selectedAccessRole || "BASIC"
 
   $: templates = getTemplates($store, $tables.list)
   $: route = !route && $allScreens.length === 0 ? "*" : route

packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataSourceSelect.svelte

@@ -1,5 +1,8 @@
 <script>
-  import { getBindableProperties } from "builderStore/dataBinding"
+  import {
+    getBindableProperties,
+    getDataProviderComponents,
+  } from "builderStore/dataBinding"
   import {
     Button,
     Popover,
@@ -61,6 +64,17 @@
     $currentAsset,
     $store.selectedComponentId
   )
+  $: dataProviders = getDataProviderComponents(
+    $currentAsset,
+    $store.selectedComponentId
+  ).map(provider => ({
+    label: provider._instanceName,
+    name: provider._instanceName,
+    providerId: provider._id,
+    value: `{{ literal [${provider._id}] }}`,
+    type: "provider",
+    schema: provider.schema,
+  }))
   $: queryBindableProperties = bindableProperties.map(property => ({
     ...property,
     category: property.type === "instance" ? "Component" : "Table",
@@ -182,7 +196,20 @@
         </li>
       {/each}
     </ul>
-
+    <Divider size="S" />
+    <div class="title">
+      <Heading size="XS">Data Providers</Heading>
+    </div>
+    <ul>
+      {#each dataProviders as provider}
+        <li
+          class:selected={value === provider}
+          on:click={() => handleSelected(provider)}
+        >
+          {provider.label}
+        </li>
+      {/each}
+    </ul>
     {#if otherSources?.length}
       <Divider size="S" />
       <div class="title">

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/SaveRow.svelte

@@ -14,11 +14,11 @@
     $currentAsset,
     $store.selectedComponentId
   )
-  $: schemaFields = getSchemaFields(parameters?.tableId)
+  $: schemaFields = getSchemaFields($currentAsset, parameters?.tableId)
   $: tableOptions = $tables.list || []
 
-  const getSchemaFields = tableId => {
+  const getSchemaFields = (asset, tableId) => {
-    const { schema } = getSchemaForDatasource({ type: "table", tableId })
+    const { schema } = getSchemaForDatasource(asset, { type: "table", tableId })
     return Object.values(schema || {})
   }
 

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/index.js

@@ -5,11 +5,13 @@ import ExecuteQuery from "./ExecuteQuery.svelte"
 import TriggerAutomation from "./TriggerAutomation.svelte"
 import ValidateForm from "./ValidateForm.svelte"
 
-// defines what actions are available, when adding a new one
+// Defines which actions are available to configure in the front end.
-// the component is the setup panel for the action
+// Unfortunately the "name" property is used as the identifier so please don't
-// NOTE that the "name" is used by the client library,
+// change them.
-// so if you want to change it, you must change it client lib too
+// The client library removes any spaces when processing actions, so they can
-
+// be considered as camel case too.
+// There is technical debt here to sanitize all these and standardise them
+// across the packages but it's a breaking change to existing apps.
 export default [
   {
     name: "Save Row",

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FieldSelect.svelte

@@ -14,7 +14,7 @@
 
   const dispatch = createEventDispatcher()
   $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource(datasource).schema
+  $: schema = getSchemaForDatasource($currentAsset, datasource).schema
   $: options = Object.keys(schema || {})
   $: boundValue = getValidValue(value, options)
 

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte

@@ -27,19 +27,16 @@
     ? tempValue.length
     : Object.keys(tempValue || {}).length
   $: dataSource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource(dataSource)?.schema
+  $: schema = getSchemaForDatasource($currentAsset, dataSource)?.schema
   $: schemaFields = Object.values(schema || {})
   $: internalTable = dataSource?.type === "table"
 
   // Reset value if value is wrong type for the datasource.
   // Lucene editor needs an array, and simple editor needs an object.
   $: {
-    if (internalTable && !Array.isArray(value)) {
+    if (!Array.isArray(value)) {
       tempValue = []
       dispatch("change", [])
-    } else if (!internalTable && Array.isArray(value)) {
-      tempValue = {}
-      dispatch("change", {})
     }
   }
 
@@ -63,28 +60,7 @@
           constaints.
         {/if}
       </Body>
-      {#if internalTable}
+      <LuceneFilterBuilder bind:value={tempValue} {schemaFields} />
-        <LuceneFilterBuilder bind:value={tempValue} {schemaFields} />
-      {:else}
-        <div class="fields">
-          <SaveFields
-            parameterFields={Array.isArray(value) ? {} : value}
-            {schemaFields}
-            valueLabel="Equals"
-            on:change={e => (tempValue = e.detail)}
-          />
-        </div>
-      {/if}
     </Layout>
   </DrawerContent>
 </Drawer>
-
-<style>
-  .fields {
-    display: grid;
-    column-gap: var(--spacing-l);
-    row-gap: var(--spacing-s);
-    align-items: center;
-    grid-template-columns: auto 1fr auto 1fr auto;
-  }
-</style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FormFieldSelect.svelte

@@ -18,7 +18,7 @@
     component => component._component === "@budibase/standard-components/form"
   )
   $: datasource = getDatasourceForProvider($currentAsset, form)
-  $: schema = getSchemaForDatasource(datasource, true).schema
+  $: schema = getSchemaForDatasource($currentAsset, datasource, true).schema
   $: options = getOptions(schema, type)
 
   const getOptions = (schema, fieldType) => {

packages/builder/src/components/design/PropertiesPanel/PropertyControls/MultiFieldSelect.svelte

@@ -14,7 +14,7 @@
 
   const dispatch = createEventDispatcher()
   $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource(datasource).schema
+  $: schema = getSchemaForDatasource($currentAsset, datasource).schema
   $: options = Object.keys(schema || {})
   $: boundValue = getValidOptions(value, options)
 

packages/builder/src/components/integration/QueryViewer.svelte

@@ -48,6 +48,8 @@
   $: integrationInfo = $integrations[datasourceType]
   $: queryConfig = integrationInfo?.query
   $: shouldShowQueryConfig = queryConfig && query.queryVerb
+  $: readQuery = query.queryVerb === "read" || query.readable
+  $: queryInvalid = !query.name || (readQuery && data.length === 0)
 
   function newField() {
     fields = [...fields, {}]
@@ -150,11 +152,7 @@
     <div class="viewer-controls">
       <Heading size="S">Results</Heading>
       <ButtonGroup>
-        <Button
+        <Button cta disabled={queryInvalid} on:click={saveQuery}>
-          cta
-          disabled={data.length === 0 || !query.name}
-          on:click={saveQuery}
-        >
           Save Query
         </Button>
         <Button secondary on:click={previewQuery}>Run Query</Button>

packages/builder/src/stores/backend/permissions.js

@@ -6,6 +6,13 @@ export function createPermissionStore() {
 
   return {
     subscribe,
+    save: async ({ level, role, resource }) => {
+      const response = await api.post(
+        `/api/permission/${role}/${resource}/${level}`
+      )
+      const json = await response.json()
+      return json
+    },
     forResource: async resourceId => {
       const response = await api.get(`/api/permission/${resourceId}`)
       const json = await response.json()

packages/builder/src/stores/backend/tables.js

@@ -101,6 +101,12 @@ export function createTablesStore() {
         // Optionally set display column
         if (primaryDisplay) {
           state.draft.primaryDisplay = field.name
+        } else if (state.draft.primaryDisplay === originalName) {
+          const fields = Object.keys(state.draft.schema)
+          // pick another display column randomly if unselecting
+          state.draft.primaryDisplay = fields.filter(
+            name => name !== originalName || name !== field
+          )[0]
         }
 
         if (indexes) {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.24",
+  "version": "0.9.34",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -25,7 +25,7 @@
     "inquirer": "^8.0.0",
     "lookpath": "^1.1.0",
     "pkg": "^4.4.9",
-    "posthog-node": "^1.0.7",
+    "posthog-node": "1.0.7",
     "randomstring": "^1.1.5"
   },
   "devDependencies": {

packages/cli/yarn.lock

@@ -1223,7 +1223,7 @@ pkg@^4.4.9:
     resolve "^1.15.1"
     stream-meter "^1.0.4"
 
-posthog-node@^1.0.7:
+posthog-node@1.0.7:
   version "1.0.7"
   resolved "https://registry.yarnpkg.com/posthog-node/-/posthog-node-1.0.7.tgz#a7a9525eebff23312117e57cff3ddac82afb2262"
   integrity sha512-KTCwyU+PU1eAQtjy5ZSJ47mrxv2d/mMkxo+vvV5c+YqfE4mBAY1UPEPMv1nElb5Vq0UnxvyQXaUnOn8d8Xr6Eg==

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.24",
+  "version": "0.9.34",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -18,13 +18,13 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/string-templates": "^0.9.24",
+    "@budibase/string-templates": "^0.9.34",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"
   },
   "devDependencies": {
-    "@budibase/standard-components": "^0.9.24",
+    "@budibase/standard-components": "^0.9.34",
     "@rollup/plugin-commonjs": "^18.0.0",
     "@rollup/plugin-node-resolve": "^11.2.1",
     "fs-extra": "^8.1.0",

packages/client/src/api/attachments.js

@@ -3,9 +3,9 @@ import API from "./api"
 /**
  * Uploads an attachment to the server.
  */
-export const uploadAttachment = async data => {
+export const uploadAttachment = async (data, tableId = "") => {
   return await API.post({
-    url: "/api/attachments/upload",
+    url: `/api/attachments/${tableId}/upload`,
     body: data,
     json: false,
   })

packages/client/src/components/Component.svelte

@@ -55,13 +55,13 @@
   }
 
   // Enriches any string component props using handlebars
-  const updateComponentProps = async (definition, context) => {
+  const updateComponentProps = (definition, context) => {
     // Record the timestamp so we can reference it after enrichment
     latestUpdateTime = Date.now()
     const enrichmentTime = latestUpdateTime
 
     // Enrich props with context
-    const enrichedProps = await enrichProps(definition, context)
+    const enrichedProps = enrichProps(definition, context)
 
     // Abandon this update if a newer update has started
     if (enrichmentTime !== latestUpdateTime) {

packages/client/src/constants.js

@@ -5,4 +5,5 @@ export const TableNames = {
 export const ActionTypes = {
   ValidateForm: "ValidateForm",
   RefreshDatasource: "RefreshDatasource",
+  SetDataProviderQuery: "SetDataProviderQuery",
 }

packages/client/src/sdk.js

@@ -9,6 +9,7 @@ import {
 import { styleable } from "./utils/styleable"
 import transition from "./utils/transition"
 import { linkable } from "./utils/linkable"
+import { getAction } from "./utils/getAction"
 import Provider from "./components/Provider.svelte"
 import { ActionTypes } from "./constants"
 
@@ -22,6 +23,7 @@ export default {
   styleable,
   transition,
   linkable,
+  getAction,
   Provider,
   ActionTypes,
 }

packages/client/src/utils/componentProps.js

@@ -21,7 +21,7 @@ export const propsAreSame = (a, b) => {
  * Enriches component props.
  * Data bindings are enriched, and button actions are enriched.
  */
-export const enrichProps = async (props, context) => {
+export const enrichProps = (props, context) => {
   // Exclude all private props that start with an underscore
   let validProps = {}
   Object.entries(props)
@@ -41,7 +41,7 @@ export const enrichProps = async (props, context) => {
   }
 
   // Enrich all data bindings in top level props
-  let enrichedProps = await enrichDataBindings(validProps, totalContext)
+  let enrichedProps = enrichDataBindings(validProps, totalContext)
 
   // Enrich click actions if they exist
   if (enrichedProps.onClick) {

packages/client/src/utils/enrichDataBinding.js

@@ -1,5 +1,5 @@
 import { cloneDeep } from "lodash/fp"
-import { processString, processObject } from "@budibase/string-templates"
+import { processString, processObjectSync } from "@budibase/string-templates"
 
 // Regex to test inputs with to see if they are likely candidates for template strings
 const looksLikeTemplate = /{{.*}}/
@@ -23,6 +23,6 @@ export const enrichDataBinding = async (input, context) => {
  * Recursively enriches all props in a props object and returns the new props.
  * Props are deeply cloned so that no mutation is done to the source object.
  */
-export const enrichDataBindings = async (props, context) => {
+export const enrichDataBindings = (props, context) => {
-  return await processObject(cloneDeep(props), context)
+  return processObjectSync(cloneDeep(props), context)
 }

packages/client/src/utils/getAction.js

@@ -0,0 +1,19 @@
+import { get } from "svelte/store"
+import { getContext } from "svelte"
+
+/**
+ * Gets a component action.
+ * @param id The component ID that provides the action
+ * @param type The action type to get
+ * @returns {null|*} The action function
+ */
+export const getAction = (id, type) => {
+  if (!id || !type) {
+    return null
+  }
+  const context = getContext("context")
+  if (!context) {
+    return null
+  }
+  return get(context)?.[`${id}_${type}`]
+}

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.24",
+  "version": "0.9.34",
   "description": "Budibase Web Server",
   "main": "src/electron.js",
   "repository": {
@@ -55,9 +55,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.24",
+    "@budibase/auth": "^0.9.34",
-    "@budibase/client": "^0.9.24",
+    "@budibase/client": "^0.9.34",
-    "@budibase/string-templates": "^0.9.24",
+    "@budibase/string-templates": "^0.9.34",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -109,7 +109,7 @@
   "devDependencies": {
     "@babel/core": "^7.14.3",
     "@babel/preset-env": "^7.14.4",
-    "@budibase/standard-components": "^0.9.24",
+    "@budibase/standard-components": "^0.9.34",
     "@jest/test-sequencer": "^24.8.0",
     "babel-jest": "^27.0.2",
     "docker-compose": "^0.23.6",

packages/server/scripts/exportAppTemplate.js

@@ -5,6 +5,10 @@ const { join } = require("path")
 const CouchDB = require("../src/db")
 // load environment
 const env = require("../src/environment")
+const {
+  USER_METDATA_PREFIX,
+  LINK_USER_METADATA_PREFIX,
+} = require("../src/db/utils")
 
 // Script to export a chosen budibase app into a package
 // Usage: ./scripts/exportAppTemplate.js export --name=Funky --appId=appId
@@ -44,7 +48,13 @@ yargs
       // perform couch dump
 
       const instanceDb = new CouchDB(appId)
-      await instanceDb.dump(writeStream, {})
+      await instanceDb.dump(writeStream, {
+        filter: doc =>
+          !(
+            doc._id.includes(USER_METDATA_PREFIX) ||
+            doc.includes(LINK_USER_METADATA_PREFIX)
+          ),
+      })
       console.log(`Template ${name} exported to ${exportPath}`)
     }
   )

packages/server/src/api/controllers/application.js

@@ -104,7 +104,6 @@ async function createInstance(template) {
     if (!ok) {
       throw "Error loading database dump from template."
     }
-    var { _rev } = await db.get(DocumentTypes.APP_METADATA)
   } else {
     // create the users table
     await db.put(USERS_TABLE_SCHEMA)
@@ -115,13 +114,13 @@ async function createInstance(template) {
   await createRoutingView(appId)
   await createAllSearchIndex(appId)
 
-  return { _id: appId, _rev }
+  return { _id: appId }
 }
 
 exports.fetch = async function (ctx) {
   const dev = ctx.query && ctx.query.status === AppStatus.DEV
   const all = ctx.query && ctx.query.status === AppStatus.ALL
-  const apps = await getAllApps({ dev, all })
+  const apps = await getAllApps({ CouchDB, dev, all })
 
   // get the locks for all the dev apps
   if (dev || all) {
@@ -182,11 +181,21 @@ exports.create = async function (ctx) {
     instanceConfig.file = ctx.request.files.templateFile
   }
   const instance = await createInstance(instanceConfig)
+  const appId = instance._id
 
   const url = await getAppUrlIfNotInUse(ctx)
-  const appId = instance._id
+  const db = new CouchDB(appId)
+  let _rev
+  try {
+    // if template there will be an existing doc
+    const existing = await db.get(DocumentTypes.APP_METADATA)
+    _rev = existing._rev
+  } catch (err) {
+    // nothing to do
+  }
   const newApplication = {
     _id: DocumentTypes.APP_METADATA,
+    _rev,
     appId: instance._id,
     type: "app",
     version: packageJson.version,
@@ -197,15 +206,8 @@ exports.create = async function (ctx) {
     instance: instance,
     updatedAt: new Date().toISOString(),
     createdAt: new Date().toISOString(),
-    deployment: {
-      type: "cloud",
-    },
   }
-  if (instance._rev) {
+  await db.put(newApplication, { force: true })
-    newApplication._rev = instance._rev
-  }
-  const instanceDb = new CouchDB(appId)
-  await instanceDb.put(newApplication)
 
   await createEmptyAppPackage(ctx, newApplication)
   /* istanbul ignore next */

packages/server/src/api/controllers/auth.js

@@ -5,7 +5,7 @@ const { getFullUser } = require("../../utilities/users")
 
 exports.fetchSelf = async ctx => {
   const appId = ctx.appId
-  const { userId } = ctx.user
+  let userId = ctx.user.userId || ctx.user._id
   /* istanbul ignore next */
   if (!userId) {
     ctx.body = {}

packages/server/src/api/controllers/query.js

@@ -60,7 +60,7 @@ exports.save = async function (ctx) {
   ctx.message = `Query ${query.name} saved successfully.`
 }
 
-async function enrichQueryFields(fields, parameters) {
+async function enrichQueryFields(fields, parameters = {}) {
   const enrichedQuery = {}
 
   // enrich the fields with dynamic parameters

packages/server/src/api/controllers/routing.js

@@ -63,10 +63,6 @@ exports.fetch = async ctx => {
 exports.clientFetch = async ctx => {
   const routing = await getRoutingStructure(ctx.appId)
   let roleId = ctx.user.role._id
-  // builder is a special case, always return the full routing structure
-  if (roleId === BUILTIN_ROLE_IDS.BUILDER) {
-    roleId = BUILTIN_ROLE_IDS.ADMIN
-  }
   const roleIds = await getUserRoleHierarchy(ctx.appId, roleId)
   for (let topLevel of Object.values(routing.routes)) {
     for (let subpathKey of Object.keys(topLevel.subpaths)) {

packages/server/src/api/controllers/static/templates/BudibaseApp.svelte

@@ -31,7 +31,6 @@
       margin: 0;
       padding: 0;
     }
-
     *,
     *:before,
     *:after {
@@ -41,9 +40,9 @@
 </svelte:head>
 
 <body id="app">
-  <script src={clientLibPath}>
+  <script type="application/javascript" src={clientLibPath}>
   </script>
-  <script>
+  <script type="application/javascript">
     loadBudibase()
   </script>
 </body>

packages/server/src/api/controllers/table/index.js

@@ -75,8 +75,6 @@ exports.save = async function (ctx) {
   /* istanbul ignore next */
   if (_rename && tableToSave.schema[_rename.updated].type === FieldTypes.LINK) {
     ctx.throw(400, "Cannot rename a linked column.")
-  } else if (_rename && tableToSave.primaryDisplay === _rename.old) {
-    ctx.throw(400, "Cannot rename the display column.")
   }
 
   tableToSave = await tableSaveFunctions.mid(tableToSave)

packages/server/src/api/controllers/user.js

@@ -4,11 +4,8 @@ const {
   getUserMetadataParams,
 } = require("../../db/utils")
 const { InternalTables } = require("../../db/utils")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { addAppRoleToUser } = require("../../utilities/workerRequests")
-const {
+const { getGlobalUsers } = require("../../utilities/global")
-  getGlobalUsers,
-  addAppRoleToUser,
-} = require("../../utilities/workerRequests")
 const { getFullUser } = require("../../utilities/users")
 
 function removeGlobalProps(user) {
@@ -21,7 +18,7 @@ function removeGlobalProps(user) {
 
 exports.fetchMetadata = async function (ctx) {
   const database = new CouchDB(ctx.appId)
-  const global = await getGlobalUsers(ctx, ctx.appId)
+  const global = await getGlobalUsers(ctx.appId)
   const metadata = (
     await database.allDocs(
       getUserMetadataParams(null, {
@@ -47,10 +44,6 @@ exports.fetchMetadata = async function (ctx) {
 exports.updateSelfMetadata = async function (ctx) {
   // overwrite the ID with current users
   ctx.request.body._id = ctx.user._id
-  if (ctx.user.builder && ctx.user.builder.global) {
-    // specific case, update self role in global user
-    await addAppRoleToUser(ctx, ctx.appId, BUILTIN_ROLE_IDS.ADMIN)
-  }
   // make sure no stale rev
   delete ctx.request.body._rev
   await exports.updateMetadata(ctx)

packages/server/src/api/routes/static.js

@@ -7,8 +7,8 @@ const {
   PermissionTypes,
   PermissionLevels,
 } = require("@budibase/auth/permissions")
-const usage = require("../../middleware/usageQuota")
 const env = require("../../environment")
+const { paramResource } = require("../../middleware/resourceId")
 
 const router = Router()
 
@@ -39,9 +39,9 @@ router
   .get("/builder/:file*", controller.serveBuilder)
   .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
   .post(
-    "/api/attachments/upload",
+    "/api/attachments/:tableId/upload",
+    paramResource("tableId"),
     authorized(PermissionTypes.TABLE, PermissionLevels.WRITE),
-    usage,
     controller.uploadFile
   )
   .get("/componentlibrary", controller.serveComponentLibrary)

packages/server/src/api/routes/tests/misc.spec.js

@@ -37,7 +37,37 @@ describe("run misc tests", () => {
 
   describe("test table utilities", () => {
     it("should be able to import a CSV", async () => {
-      const table = await config.createTable()
+      const table = await config.createTable({
+        name: "table",
+        type: "table",
+        key: "name",
+        schema: {
+          a: {
+            type: "string",
+            constraints: {
+              type: "string",
+            },
+          },
+          b: {
+            type: "string",
+            constraints: {
+              type: "string",
+            },
+          },
+          c: {
+            type: "string",
+            constraints: {
+              type: "string",
+            },
+          },
+          d: {
+            type: "string",
+            constraints: {
+              type: "string",
+            },
+          },
+        },
+      })
       const dataImport = {
         csvString: "a,b,c,d\n1,2,3,4"
       }

packages/server/src/api/routes/tests/routing.spec.js

@@ -26,13 +26,6 @@ describe("/routing", () => {
 
   describe("fetch", () => {
     it("returns the correct routing for basic user", async () => {
-      workerRequests.getGlobalUsers.mockImplementationOnce((ctx, appId) => {
-        return {
-          roles: {
-            [appId]: BUILTIN_ROLE_IDS.BASIC,
-          }
-        }
-      })
       const res = await request
         .get(`/api/routing/client`)
         .set(await config.roleHeaders({
@@ -54,13 +47,6 @@ describe("/routing", () => {
     })
 
     it("returns the correct routing for power user", async () => {
-      workerRequests.getGlobalUsers.mockImplementationOnce((ctx, appId) => {
-        return {
-          roles: {
-            [appId]: BUILTIN_ROLE_IDS.POWER,
-          }
-        }
-      })
       const res = await request
         .get(`/api/routing/client`)
         .set(await config.roleHeaders({

packages/server/src/api/routes/tests/row.spec.js

@@ -125,6 +125,7 @@ describe("/rows", () => {
           numberNull: number,
           numberUndefined: number,
           numberString: number,
+          numberNumber: number,
           datetimeEmptyString: datetime,
           datetimeNull: datetime,
           datetimeUndefined: datetime,

packages/server/src/api/routes/tests/user.spec.js

@@ -1,7 +1,6 @@
 const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { checkPermissionsEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
-const workerRequests = require("../../../utilities/workerRequests")
 
 jest.mock("../../../utilities/workerRequests", () => ({
   getGlobalUsers: jest.fn(() => {
@@ -25,30 +24,18 @@ describe("/users", () => {
   })
 
   describe("fetch", () => {
-    beforeEach(() => {
-      workerRequests.getGlobalUsers.mockImplementationOnce(() => ([
-          {
-            _id: "us_uuid1",
-          },
-          {
-            _id: "us_uuid2",
-          }
-        ]
-      ))
-    })
-
     it("returns a list of users from an instance db", async () => {
-      await config.createUser("brenda@brenda.com", "brendas_password")
+      await config.createUser("uuidx")
-      await config.createUser("pam@pam.com", "pam_password")
+      await config.createUser("uuidy")
       const res = await request
         .get(`/api/users/metadata`)
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
 
-      expect(res.body.length).toBe(2)
+      expect(res.body.length).toBe(3)
-      expect(res.body.find(u => u._id === `ro_ta_users_us_uuid1`)).toBeDefined()
+      expect(res.body.find(u => u._id === `ro_ta_users_us_uuidx`)).toBeDefined()
-      expect(res.body.find(u => u._id === `ro_ta_users_us_uuid2`)).toBeDefined()
+      expect(res.body.find(u => u._id === `ro_ta_users_us_uuidy`)).toBeDefined()
     })
 
     it("should apply authorization to endpoint", async () => {
@@ -65,9 +52,6 @@ describe("/users", () => {
   })
 
   describe("update", () => {
-    beforeEach(() => {
-    })
-
     it("should be able to update the user", async () => {
       const user = await config.createUser()
       user.roleId = BUILTIN_ROLE_IDS.BASIC
@@ -94,14 +78,6 @@ describe("/users", () => {
   })
 
   describe("find", () => {
-    beforeEach(() => {
-      jest.resetAllMocks()
-      workerRequests.getGlobalUsers.mockImplementationOnce(() => ({
-        _id: "us_uuid1",
-        roleId: BUILTIN_ROLE_IDS.POWER,
-      }))
-    })
-
     it("should be able to find the user", async () => {
       const user = await config.createUser()
       const res = await request
@@ -110,7 +86,7 @@ describe("/users", () => {
         .expect(200)
         .expect("Content-Type", /json/)
       expect(res.body._id).toEqual(user._id)
-      expect(res.body.roleId).toEqual(BUILTIN_ROLE_IDS.POWER)
+      expect(res.body.roleId).toEqual(BUILTIN_ROLE_IDS.ADMIN)
       expect(res.body.tableId).toBeDefined()
     })
   })

packages/server/src/api/routes/tests/utilities/TestFunctions.js

@@ -2,6 +2,7 @@ const rowController = require("../../../controllers/row")
 const appController = require("../../../controllers/application")
 const CouchDB = require("../../../../db")
 const { AppStatus } = require("../../../../db/utils")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 
 function Request(appId, params) {
   this.appId = appId
@@ -77,11 +78,17 @@ exports.checkPermissionsEndpoint = async ({
     .set(passHeader)
     .expect(200)
 
-  user = await config.createUser("fail@budibase.com", password, failRole)
+  let failHeader
-  const failHeader = await config.login("fail@budibase.com", password, {
+  if (failRole === BUILTIN_ROLE_IDS.PUBLIC) {
-    roleId: failRole,
+    failHeader = config.publicHeaders()
-    userId: user.globalId,
+  } else {
-  })
+    user = await config.createUser("fail@budibase.com", password, failRole)
+    failHeader = await config.login("fail@budibase.com", password, {
+      roleId: failRole,
+      userId: user.globalId,
+      builder: false,
+    })
+  }
 
   await exports
     .createRequest(config.request, method, url, body)

packages/server/src/api/routes/tests/view.spec.js

@@ -1,5 +1,25 @@
 const setup = require("./utilities")
 
+function priceTable() {
+  return {
+    name: "table",
+    type: "table",
+    key: "name",
+    schema: {
+      Price: {
+        type: "number",
+        constraints: {},
+      },
+      Category: {
+        type: "string",
+        constraints: {
+          type: "string",
+        },
+      },
+    },
+  }
+}
+
 describe("/views", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
@@ -13,7 +33,7 @@ describe("/views", () => {
 
   describe("create", () => {
     beforeEach(async () => {
-      table = await config.createTable()
+      table = await config.createTable(priceTable())
     })
 
     it("returns a success message when the view is successfully created", async () => {
@@ -83,7 +103,7 @@ describe("/views", () => {
 
   describe("fetch", () => {
     beforeEach(async () => {
-      table = await config.createTable()
+      table = await config.createTable(priceTable())
     })
 
     it("returns only custom views", async () => {
@@ -105,7 +125,7 @@ describe("/views", () => {
 
   describe("query", () => {
     beforeEach(async () => {
-      table = await config.createTable()
+      table = await config.createTable(priceTable())
     })
 
     it("returns data for the created view", async () => {
@@ -172,7 +192,7 @@ describe("/views", () => {
 
   describe("destroy", () => {
     it("should be able to delete a view", async () => {
-      const table = await config.createTable()
+      const table = await config.createTable(priceTable())
       const view = await config.createView()
       const res = await request
         .delete(`/api/views/${view.name}`)
@@ -186,7 +206,7 @@ describe("/views", () => {
 
   describe("exportView", () => {
     it("should be able to delete a view", async () => {
-      await config.createTable()
+      await config.createTable(priceTable())
       await config.createRow()
       const view = await config.createView()
       let res = await request

packages/server/src/db/linkedRows/index.js

@@ -11,7 +11,9 @@ const {
 const { flatten } = require("lodash")
 const CouchDB = require("../../db")
 const { FieldTypes } = require("../../constants")
-const { getMultiIDParams } = require("../../db/utils")
+const { getMultiIDParams, USER_METDATA_PREFIX } = require("../../db/utils")
+const { partition } = require("lodash")
+const { getGlobalUsers } = require("../../utilities/global")
 
 /**
  * This functionality makes sure that when rows with links are created, updated or deleted they are processed
@@ -57,6 +59,31 @@ async function getLinksForRows(appId, rows) {
   )
 }
 
+async function getFullLinkedDocs(appId, links) {
+  // create DBs
+  const db = new CouchDB(appId)
+  const linkedRowIds = links.map(link => link.id)
+  let linked = (await db.allDocs(getMultiIDParams(linkedRowIds))).rows.map(
+    row => row.doc
+  )
+  // need to handle users as specific cases
+  let [users, other] = partition(linked, linkRow =>
+    linkRow._id.startsWith(USER_METDATA_PREFIX)
+  )
+  const globalUsers = await getGlobalUsers(appId, users)
+  users = users.map(user => {
+    const globalUser = globalUsers.find(
+      globalUser => globalUser && user._id.includes(globalUser._id)
+    )
+    return {
+      ...globalUser,
+      // doing user second overwrites the id and rev (always metadata)
+      ...user,
+    }
+  })
+  return [...other, ...users]
+}
+
 /**
  * Update link documents for a row or table - this is to be called by the API controller when a change is occurring.
  * @param {string} eventType states what type of change which is occurring, means this can be expanded upon in the
@@ -154,14 +181,13 @@ exports.attachFullLinkedDocs = async (appId, table, rows) => {
   if (linkedTableIds.length === 0) {
     return rows
   }
+  // create DBs
   const db = new CouchDB(appId)
+  // get all the links
   const links = (await getLinksForRows(appId, rows)).filter(link =>
     rows.some(row => row._id === link.thisId)
   )
-  const linkedRowIds = links.map(link => link.id)
+  let linked = await getFullLinkedDocs(appId, links)
-  const linked = (await db.allDocs(getMultiIDParams(linkedRowIds))).rows.map(
-    row => row.doc
-  )
   const linkedTables = []
   for (let row of rows) {
     for (let link of links.filter(link => link.thisId === row._id)) {

packages/server/src/db/utils.js

@@ -6,17 +6,11 @@ const {
   APP_DEV_PREFIX,
   APP_PREFIX,
   SEPARATOR,
+  StaticDatabases,
 } = require("@budibase/auth/db")
 
 const UNICODE_MAX = "\ufff0"
 
-const StaticDatabases = {
-  BUILDER: {
-    name: "builder-db",
-    baseDoc: "builder-doc",
-  },
-}
-
 const AppStatus = {
   DEV: "development",
   ALL: "all",
@@ -54,9 +48,18 @@ const SearchIndexes = {
   ROWS: "rows",
 }
 
+exports.StaticDatabases = {
+  BUILDER: {
+    name: "builder-db",
+    baseDoc: "builder-doc",
+  },
+  ...StaticDatabases,
+}
+
 exports.APP_PREFIX = APP_PREFIX
 exports.APP_DEV_PREFIX = APP_DEV_PREFIX
-exports.StaticDatabases = StaticDatabases
+exports.USER_METDATA_PREFIX = `${DocumentTypes.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
+exports.LINK_USER_METADATA_PREFIX = `${DocumentTypes.LINK}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
 exports.ViewNames = ViewNames
 exports.InternalTables = InternalTables
 exports.DocumentTypes = DocumentTypes

packages/server/src/middleware/currentapp.js

@@ -1,9 +1,16 @@
-const { getAppId, setCookie, getCookie } = require("@budibase/auth").utils
+const {
+  getAppId,
+  setCookie,
+  getCookie,
+  clearCookie,
+} = require("@budibase/auth").utils
 const { Cookies } = require("@budibase/auth").constants
 const { getRole } = require("@budibase/auth/roles")
 const { getGlobalSelf } = require("../utilities/workerRequests")
 const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { generateUserMetadataID } = require("../db/utils")
+const { dbExists } = require("@budibase/auth/db")
+const CouchDB = require("../db")
 
 module.exports = async (ctx, next) => {
   // try to get the appID from the request
@@ -13,6 +20,15 @@ module.exports = async (ctx, next) => {
   if (!appCookie && !requestAppId) {
     return next()
   }
+  // check the app exists referenced in cookie
+  if (appCookie) {
+    const appId = appCookie.appId
+    const exists = await dbExists(CouchDB, appId)
+    if (!exists) {
+      clearCookie(ctx, Cookies.CurrentApp)
+      return next()
+    }
+  }
 
   let updateCookie = false,
     appId,
@@ -33,7 +49,7 @@ module.exports = async (ctx, next) => {
     updateCookie = true
     appId = requestAppId
     // retrieving global user gets the right role
-    roleId = globalUser.roleId
+    roleId = globalUser.roleId || BUILTIN_ROLE_IDS.PUBLIC
   } else if (appCookie != null) {
     appId = appCookie.appId
     roleId = appCookie.roleId || BUILTIN_ROLE_IDS.PUBLIC

packages/server/src/tests/utilities/TestConfiguration.js

@@ -101,7 +101,7 @@ class TestConfiguration {
       userId: GLOBAL_USER_ID,
     }
     const app = {
-      roleId: BUILTIN_ROLE_IDS.BUILDER,
+      roleId: BUILTIN_ROLE_IDS.ADMIN,
       appId: this.appId,
     }
     const authToken = jwt.sign(auth, env.JWT_SECRET)
@@ -306,12 +306,9 @@ class TestConfiguration {
     return await this._req(config, null, controllers.layout.save)
   }
 
-  async createUser(roleId = BUILTIN_ROLE_IDS.POWER) {
+  async createUser(id = null) {
-    const globalId = `us_${Math.random()}`
+    const globalId = !id ? `us_${Math.random()}` : `us_${id}`
-    const resp = await this.globalUser(
+    const resp = await this.globalUser(globalId)
-      globalId,
-      roleId === BUILTIN_ROLE_IDS.BUILDER
-    )
     return {
       ...resp,
       globalId,
@@ -319,7 +316,6 @@ class TestConfiguration {
   }
 
   async login(email, password, { roleId, userId, builder } = {}) {
-    roleId = !roleId ? BUILTIN_ROLE_IDS.BUILDER : roleId
     userId = !userId ? `us_uuid1` : userId
     if (!this.request) {
       throw "Server has not been opened, cannot login."

packages/server/src/tests/utilities/structures.js

@@ -46,7 +46,6 @@ exports.basicRow = tableId => {
   return {
     name: "Test Contact",
     description: "original description",
-    status: "new",
     tableId: tableId,
   }
 }

packages/server/src/utilities/fileSystem/index.js

@@ -18,6 +18,10 @@ const download = require("download")
 const env = require("../../environment")
 const { homedir } = require("os")
 const fetch = require("node-fetch")
+const {
+  USER_METDATA_PREFIX,
+  LINK_USER_METADATA_PREFIX,
+} = require("../../db/utils")
 
 const DEFAULT_AUTOMATION_BUCKET =
   "https://prod-budi-automations.s3-eu-west-1.amazonaws.com"
@@ -117,7 +121,14 @@ exports.performBackup = async (appId, backupName) => {
   const writeStream = fs.createWriteStream(path)
   // perform couch dump
   const instanceDb = new CouchDB(appId)
-  await instanceDb.dump(writeStream, {})
+  await instanceDb.dump(writeStream, {
+    // filter out anything that has a user metadata structure in its ID
+    filter: doc =>
+      !(
+        doc._id.includes(USER_METDATA_PREFIX) ||
+        doc.includes(LINK_USER_METADATA_PREFIX)
+      ),
+  })
   // write the file to the object store
   await streamUpload(
     ObjectStoreBuckets.BACKUPS,

packages/server/src/utilities/fileSystem/newApp.js

@@ -30,5 +30,7 @@ exports.uploadClientLibrary = async appId => {
   const sourcepath = require.resolve("@budibase/client")
   const destPath = join(appId, "budibase-client.js")
 
-  await streamUpload(BUCKET_NAME, destPath, fs.createReadStream(sourcepath))
+  await streamUpload(BUCKET_NAME, destPath, fs.createReadStream(sourcepath), {
+    ContentType: "application/javascript",
+  })
 }

packages/server/src/utilities/global.js

@@ -0,0 +1,64 @@
+const CouchDB = require("../db")
+const {
+  getMultiIDParams,
+  getGlobalIDFromUserMetadataID,
+  StaticDatabases,
+} = require("../db/utils")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { getDeployedAppID } = require("@budibase/auth/db")
+const { getGlobalUserParams } = require("@budibase/auth/db")
+
+exports.updateAppRole = (appId, user) => {
+  if (!user.roles) {
+    return user
+  }
+  if (user.builder && user.builder.global) {
+    user.roleId = BUILTIN_ROLE_IDS.ADMIN
+  } else {
+    // always use the deployed app
+    user.roleId = user.roles[getDeployedAppID(appId)]
+    if (!user.roleId) {
+      user.roleId = BUILTIN_ROLE_IDS.PUBLIC
+    }
+  }
+  delete user.roles
+  return user
+}
+
+exports.getGlobalUser = async (appId, userId) => {
+  const db = CouchDB(StaticDatabases.GLOBAL.name)
+  let user = await db.get(getGlobalIDFromUserMetadataID(userId))
+  if (user) {
+    delete user.password
+  }
+  return exports.updateAppRole(appId, user)
+}
+
+exports.getGlobalUsers = async (appId = null, users = null) => {
+  const db = CouchDB(StaticDatabases.GLOBAL.name)
+  let globalUsers
+  if (users) {
+    const globalIds = users.map(user => getGlobalIDFromUserMetadataID(user._id))
+    globalUsers = (await db.allDocs(getMultiIDParams(globalIds))).rows.map(
+      row => row.doc
+    )
+  } else {
+    globalUsers = (
+      await db.allDocs(
+        getGlobalUserParams(null, {
+          include_docs: true,
+        })
+      )
+    ).rows.map(row => row.doc)
+  }
+  globalUsers = globalUsers
+    .filter(user => user != null)
+    .map(user => {
+      delete user.password
+      return user
+    })
+  if (!appId) {
+    return globalUsers
+  }
+  return globalUsers.map(user => exports.updateAppRole(appId, user))
+}

packages/server/src/utilities/rowProcessor.js

@@ -169,9 +169,14 @@ exports.inputProcessing = (user = {}, table, row) => {
   let clonedRow = cloneDeep(row)
   // need to copy the table so it can be differenced on way out
   const copiedTable = cloneDeep(table)
+  const dontCleanseKeys = ["type", "_id", "_rev", "tableId"]
   for (let [key, value] of Object.entries(clonedRow)) {
     const field = table.schema[key]
+    // cleanse fields that aren't in the schema
     if (!field) {
+      if (dontCleanseKeys.indexOf(key) === -1) {
+        delete clonedRow[key]
+      }
       continue
     }
     clonedRow[key] = exports.coerce(value, field.type)

packages/server/src/utilities/users.js

@@ -1,13 +1,9 @@
 const CouchDB = require("../db")
-const { getGlobalIDFromUserMetadataID, InternalTables } = require("../db/utils")
+const { InternalTables } = require("../db/utils")
-const { getGlobalUsers } = require("../utilities/workerRequests")
+const { getGlobalUser } = require("../utilities/global")
 
 exports.getFullUser = async (ctx, userId) => {
-  const global = await getGlobalUsers(
+  const global = await getGlobalUser(ctx.appId, userId)
-    ctx,
-    ctx.appId,
-    getGlobalIDFromUserMetadataID(userId)
-  )
   let metadata
   try {
     // this will throw an error if the db doesn't exist, or there is no appId

packages/server/src/utilities/workerRequests.js

@@ -1,27 +1,16 @@
 const fetch = require("node-fetch")
 const env = require("../environment")
 const { checkSlashesInUrl } = require("./index")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { getDeployedAppID } = require("@budibase/auth/db")
-const { getGlobalIDFromUserMetadataID } = require("../db/utils")
+const { updateAppRole, getGlobalUser } = require("./global")
 
-function getAppRole(appId, user) {
+function request(ctx, request, noApiKey) {
-  if (!user.roles) {
-    return user
-  }
-  // always use the deployed app
-  user.roleId = user.roles[getDeployedAppID(appId)]
-  if (!user.roleId) {
-    user.roleId = BUILTIN_ROLE_IDS.PUBLIC
-  }
-  delete user.roles
-  return user
-}
-
-function request(ctx, request) {
   if (!request.headers) {
     request.headers = {}
   }
+  if (!noApiKey) {
+    request.headers["x-budibase-api-key"] = env.INTERNAL_API_KEY
+  }
   if (request.body && Object.keys(request.body).length > 0) {
     request.headers["Content-Type"] = "application/json"
     request.body =
@@ -44,9 +33,6 @@ exports.sendSmtpEmail = async (to, from, subject, contents) => {
     checkSlashesInUrl(env.WORKER_URL + `/api/admin/email/send`),
     request(null, {
       method: "POST",
-      headers: {
-        "x-budibase-api-key": env.INTERNAL_API_KEY,
-      },
       body: {
         email: to,
         from,
@@ -86,49 +72,19 @@ exports.getDeployedApps = async ctx => {
   }
 }
 
-exports.deleteGlobalUser = async (ctx, globalId) => {
-  const endpoint = `/api/admin/users/${globalId}`
-  const reqCfg = { method: "DELETE" }
-  const response = await fetch(
-    checkSlashesInUrl(env.WORKER_URL + endpoint),
-    request(ctx, reqCfg)
-  )
-  return response.json()
-}
-
-exports.getGlobalUsers = async (ctx, appId = null, globalId = null) => {
-  const endpoint = globalId
-    ? `/api/admin/users/${globalId}`
-    : `/api/admin/users`
-  const reqCfg = { method: "GET" }
-  const response = await fetch(
-    checkSlashesInUrl(env.WORKER_URL + endpoint),
-    request(ctx, reqCfg)
-  )
-  let users = await response.json()
-  if (!appId) {
-    return users
-  }
-  if (Array.isArray(users)) {
-    users = users.map(user => getAppRole(appId, user))
-  } else {
-    users = getAppRole(appId, users)
-  }
-  return users
-}
-
 exports.getGlobalSelf = async (ctx, appId = null) => {
   const endpoint = `/api/admin/users/self`
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + endpoint),
-    request(ctx, { method: "GET" })
+    // we don't want to use API key when getting self
+    request(ctx, { method: "GET" }, true)
   )
   if (response.status !== 200) {
     ctx.throw(400, "Unable to get self globally.")
   }
   let json = await response.json()
   if (appId) {
-    json = getAppRole(appId, json)
+    json = updateAppRole(appId, json)
   }
   return json
 }
@@ -141,8 +97,7 @@ exports.addAppRoleToUser = async (ctx, appId, roleId, userId = null) => {
     user = await exports.getGlobalSelf(ctx)
     endpoint = `/api/admin/users/self`
   } else {
-    userId = getGlobalIDFromUserMetadataID(userId)
+    user = await getGlobalUser(appId, userId)
-    user = await exports.getGlobalUsers(ctx, appId, userId)
     body._id = userId
     endpoint = `/api/admin/users`
   }
@@ -172,9 +127,6 @@ exports.removeAppFromUserRoles = async appId => {
     checkSlashesInUrl(env.WORKER_URL + `/api/admin/roles/${deployedAppId}`),
     request(null, {
       method: "DELETE",
-      headers: {
-        "x-budibase-api-key": env.INTERNAL_API_KEY,
-      },
     })
   )
   if (response.status !== 200) {

packages/standard-components/manifest.json

@@ -1557,5 +1557,38 @@
     "context": {
       "type": "schema"
     }
+  },
+  "daterangepicker": {
+    "name": "Date Range",
+    "icon": "Date",
+    "styleable": true,
+    "hasChildren": false,
+    "info": "Your data provider will be automatically filtered to the given date range.",
+    "settings": [
+      {
+        "type": "dataProvider",
+        "label": "Provider",
+        "key": "dataProvider"
+      },
+      {
+        "type": "field",
+        "label": "Date field",
+        "key": "field"
+      },
+      {
+        "type": "select",
+        "label": "Default range",
+        "key": "defaultValue",
+        "options": [
+          "Last 1 day",
+          "Last 7 days",
+          "Last 30 days",
+          "Last 3 months",
+          "Last 6 months",
+          "Last 1 year"
+        ],
+        "defaultValue": "Last 30 days"
+      }
+    ]
   }
 }

packages/standard-components/package.json

@@ -29,14 +29,15 @@
   "keywords": [
     "svelte"
   ],
-  "version": "0.9.24",
+  "version": "0.9.34",
   "license": "MIT",
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc",
   "dependencies": {
-    "@budibase/bbui": "^0.9.24",
+    "@budibase/bbui": "^0.9.34",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
     "apexcharts": "^3.22.1",
+    "dayjs": "^1.10.5",
     "svelte-apexcharts": "^1.0.2",
     "svelte-flatpickr": "^3.1.0"
   }

packages/standard-components/src/DataProvider.svelte

@@ -1,6 +1,12 @@
 <script>
   import { getContext } from "svelte"
   import { ProgressCircle, Pagination } from "@budibase/bbui"
+  import {
+    buildLuceneQuery,
+    luceneQuery,
+    luceneSort,
+    luceneLimit,
+  } from "./lucene"
 
   export let dataSource
   export let filter
@@ -25,9 +31,11 @@
   let schema = {}
   let bookmarks = [null]
   let pageNumber = 0
+  let query = null
 
+  $: query = buildLuceneQuery(filter)
   $: internalTable = dataSource?.type === "table"
-  $: query = internalTable ? buildLuceneQuery(filter) : null
+  $: nestedProvider = dataSource?.type === "provider"
   $: hasNextPage = bookmarks[pageNumber + 1] != null
   $: hasPrevPage = pageNumber > 0
   $: getSchema(dataSource)
@@ -48,22 +56,38 @@
     }
   }
   $: {
-    // Sort and limit rows in memory when we aren't searching internal tables
     if (internalTable) {
+      // Internal tables are already processed server-side
       rows = allRows
     } else {
-      const sortedRows = sortRows(allRows, sortColumn, sortOrder)
+      // For anything else we use client-side implementations to filter, sort
-      rows = limitRows(sortedRows, limit)
+      // and limit
+      const filtered = luceneQuery(allRows, query)
+      const sorted = luceneSort(filtered, sortColumn, sortOrder, sortType)
+      rows = luceneLimit(sorted, limit)
     }
   }
   $: actions = [
     {
       type: ActionTypes.RefreshDatasource,
-      callback: () => fetchData(dataSource),
+      callback: () => refresh(),
       metadata: { dataSource },
     },
+    {
+      type: ActionTypes.SetDataProviderQuery,
+      callback: newQuery => (query = newQuery),
+    },
   ]
-  $: dataContext = { rows, schema, rowsLength: rows.length }
+  $: dataContext = {
+    rows,
+    schema,
+    rowsLength: rows.length,
+
+    // Undocumented properties. These aren't supposed to be used in builder
+    // bindings, but are used internally by other components
+    id: $component?.id,
+    state: { query },
+  }
 
   const getSortType = (schema, sortColumn) => {
     if (!schema || !sortColumn || !schema[sortColumn]) {
@@ -73,36 +97,18 @@
     return type === "number" ? "number" : "string"
   }
 
-  const buildLuceneQuery = filter => {
+  const refresh = async () => {
-    let query = {
+    if (schemaLoaded && !nestedProvider) {
-      string: {},
+      fetchData(
-      fuzzy: {},
+        dataSource,
-      range: {},
+        query,
-      equal: {},
+        limit,
-      notEqual: {},
+        sortColumn,
-      empty: {},
+        sortOrder,
-      notEmpty: {},
+        sortType,
+        paginate
+      )
     }
-    if (Array.isArray(filter)) {
-      filter.forEach(({ operator, field, type, value }) => {
-        if (operator.startsWith("range")) {
-          if (!query.range[field]) {
-            query.range[field] = {
-              low: type === "number" ? Number.MIN_SAFE_INTEGER : "0000",
-              high: type === "number" ? Number.MAX_SAFE_INTEGER : "9999",
-            }
-          }
-          if (operator === "rangeLow") {
-            query.range[field].low = value
-          } else if (operator === "rangeHigh") {
-            query.range[field].high = value
-          }
-        } else if (query[operator]) {
-          query[operator][field] = value
-        }
-      })
-    }
-    return query
   }
 
   const fetchData = async (
@@ -116,6 +122,7 @@
   ) => {
     loading = true
     if (dataSource?.type === "table") {
+      // For internal tables we use server-side processing
       const res = await API.searchTable({
         tableId: dataSource.tableId,
         query,
@@ -132,55 +139,27 @@
       } else {
         bookmarks = [null]
       }
+    } else if (dataSource?.type === "provider") {
+      // For providers referencing another provider, just use the rows it
+      // provides
+      allRows = dataSource?.value?.rows ?? []
     } else {
-      const rows = await API.fetchDatasource(dataSource)
+      // For other data sources like queries or views, fetch all rows from the
-      allRows = inMemoryFilterRows(rows, filter)
+      // server
+      allRows = await API.fetchDatasource(dataSource)
     }
     loading = false
     loaded = true
   }
 
-  const inMemoryFilterRows = (rows, filter) => {
-    let filteredData = [...rows]
-    Object.entries(filter || {}).forEach(([field, value]) => {
-      if (value != null && value !== "") {
-        filteredData = filteredData.filter(row => {
-          return row[field] === value
-        })
-      }
-    })
-    return filteredData
-  }
-
-  const sortRows = (rows, sortColumn, sortOrder) => {
-    if (!sortColumn || !sortOrder) {
-      return rows
-    }
-    return rows.slice().sort((a, b) => {
-      const colA = a[sortColumn]
-      const colB = b[sortColumn]
-      if (sortOrder === "Descending") {
-        return colA > colB ? -1 : 1
-      } else {
-        return colA > colB ? 1 : -1
-      }
-    })
-  }
-
-  const limitRows = (rows, limit) => {
-    const numLimit = parseFloat(limit)
-    if (isNaN(numLimit)) {
-      return rows
-    }
-    return rows.slice(0, numLimit)
-  }
-
   const getSchema = async dataSource => {
     if (dataSource?.schema) {
       schema = dataSource.schema
     } else if (dataSource?.tableId) {
       const definition = await API.fetchTableDefinition(dataSource.tableId)
       schema = definition?.schema ?? {}
+    } else if (dataSource?.type === "provider") {
+      schema = dataSource.value?.schema ?? {}
     } else {
       schema = {}
     }

packages/standard-components/src/DateRangePicker.svelte

@@ -0,0 +1,77 @@
+<script>
+  import { Select } from "@budibase/bbui"
+  import { getContext } from "svelte"
+  import dayjs from "dayjs"
+  import utc from "dayjs/plugin/utc"
+  import { onMount } from "svelte"
+
+  dayjs.extend(utc)
+
+  export let dataProvider
+  export let field
+  export let defaultValue
+
+  const dataContext = getContext("context")
+  const component = getContext("component")
+  const { styleable, builderStore, ActionTypes, getAction } = getContext("sdk")
+
+  const setQuery = getAction(dataProvider?.id, ActionTypes.SetDataProviderQuery)
+  const options = [
+    "Last 1 day",
+    "Last 7 days",
+    "Last 30 days",
+    "Last 3 months",
+    "Last 6 months",
+    "Last 1 year",
+  ]
+  let value = options.includes(defaultValue) ? defaultValue : "Last 30 days"
+
+  const updateDateRange = option => {
+    const query = dataProvider?.state?.query
+    if (!query || !setQuery) {
+      return
+    }
+
+    value = option
+    let low = dayjs.utc().subtract(1, "year")
+    let high = dayjs.utc().add(1, "day")
+
+    if (option === "Last 1 day") {
+      low = dayjs.utc().subtract(1, "day")
+    } else if (option === "Last 7 days") {
+      low = dayjs.utc().subtract(7, "days")
+    } else if (option === "Last 30 days") {
+      low = dayjs.utc().subtract(30, "days")
+    } else if (option === "Last 3 months") {
+      low = dayjs.utc().subtract(3, "months")
+    } else if (option === "Last 6 months") {
+      low = dayjs.utc().subtract(6, "months")
+    }
+
+    // Update data provider query with the new filter
+    setQuery({
+      ...query,
+      range: {
+        ...query.range,
+        [field]: {
+          high: high.format(),
+          low: low.format(),
+        },
+      },
+    })
+  }
+
+  // Update the range on mount to the initial value
+  onMount(() => {
+    updateDateRange(value)
+  })
+</script>
+
+<div use:styleable={$component.styles}>
+  <Select
+    placeholder={null}
+    {options}
+    {value}
+    on:change={e => updateDateRange(e.detail)}
+  />
+</div>

packages/standard-components/src/forms/AttachmentField.svelte

@@ -9,8 +9,10 @@
 
   let fieldState
   let fieldApi
+  let fieldSchema
 
   const { API, notifications } = getContext("sdk")
+  const formContext = getContext("form")
   const BYTES_IN_MB = 1000000
 
   export let files = []
@@ -28,7 +30,7 @@
     for (let i = 0; i < fileList.length; i++) {
       data.append("file", fileList[i])
     }
-    return await API.uploadAttachment(data)
+    return await API.uploadAttachment(data, formContext?.dataSource?.tableId)
   }
 </script>
 
@@ -45,7 +47,9 @@
     <CoreDropzone
       value={$fieldState.value}
       disabled={$fieldState.disabled}
-      on:change={e => fieldApi.setValue(e.detail)}
+      on:change={e => {
+        fieldApi.setValue(e.detail)
+      }}
       {processFiles}
       {handleFileTooLarge}
     />

packages/standard-components/src/forms/Form.svelte

@@ -80,7 +80,7 @@
   }
 
   // Provide both form API and state to children
-  setContext("form", { formApi, formState })
+  setContext("form", { formApi, formState, dataSource })
 
   // Action context to pass to children
   $: actions = [{ type: ActionTypes.ValidateForm, callback: formApi.validate }]

packages/standard-components/src/forms/StringField.svelte

@@ -22,6 +22,7 @@
 >
   {#if fieldState}
     <CoreTextField
+      updateOnChange={false}
       value={$fieldState.value}
       on:change={e => fieldApi.setValue(e.detail)}
       disabled={$fieldState.disabled}

packages/standard-components/src/index.js

@@ -28,6 +28,7 @@ export { default as cardhorizontal } from "./CardHorizontal.svelte"
 export { default as cardstat } from "./CardStat.svelte"
 export { default as icon } from "./Icon.svelte"
 export { default as backgroundimage } from "./BackgroundImage.svelte"
+export { default as daterangepicker } from "./DateRangePicker.svelte"
 export * from "./charts"
 export * from "./forms"
 export * from "./table"

packages/standard-components/src/lucene.js

@@ -0,0 +1,146 @@
+/**
+ * Builds a lucene JSON query from the filter structure generated in the builder
+ * @param filter the builder filter structure
+ */
+export const buildLuceneQuery = filter => {
+  let query = {
+    string: {},
+    fuzzy: {},
+    range: {},
+    equal: {},
+    notEqual: {},
+    empty: {},
+    notEmpty: {},
+  }
+  if (Array.isArray(filter)) {
+    filter.forEach(({ operator, field, type, value }) => {
+      if (operator.startsWith("range")) {
+        if (!query.range[field]) {
+          query.range[field] = {
+            low:
+              type === "number"
+                ? Number.MIN_SAFE_INTEGER
+                : "0000-00-00T00:00:00.000Z",
+            high:
+              type === "number"
+                ? Number.MAX_SAFE_INTEGER
+                : "9999-00-00T00:00:00.000Z",
+          }
+        }
+        if (operator === "rangeLow" && value != null && value !== "") {
+          query.range[field].low = value
+        } else if (operator === "rangeHigh" && value != null && value !== "") {
+          query.range[field].high = value
+        }
+      } else if (query[operator]) {
+        query[operator][field] = value
+      }
+    })
+  }
+  return query
+}
+
+/**
+ * Performs a client-side lucene search on an array of data
+ * @param docs the data
+ * @param query the JSON lucene query
+ */
+export const luceneQuery = (docs, query) => {
+  if (!query) {
+    return docs
+  }
+
+  // Iterates over a set of filters and evaluates a fail function against a doc
+  const match = (type, failFn) => doc => {
+    const filters = Object.entries(query[type] || {})
+    for (let i = 0; i < filters.length; i++) {
+      if (failFn(filters[i][0], filters[i][1], doc)) {
+        return false
+      }
+    }
+    return true
+  }
+
+  // Process a string match (fails if the value does not start with the string)
+  const stringMatch = match("string", (key, value, doc) => {
+    return !doc[key] || !doc[key].startsWith(value)
+  })
+
+  // Process a range match
+  const rangeMatch = match("range", (key, value, doc) => {
+    return !doc[key] || doc[key] < value.low || doc[key] > value.high
+  })
+
+  // Process an equal match (fails if the value is different)
+  const equalMatch = match("equal", (key, value, doc) => {
+    return doc[key] !== value
+  })
+
+  // Process a not-equal match (fails if the value is the same)
+  const notEqualMatch = match("notEqual", (key, value, doc) => {
+    return doc[key] === value
+  })
+
+  // Process an empty match (fails if the value is not empty)
+  const emptyMatch = match("empty", (key, value, doc) => {
+    return doc[key] != null && doc[key] !== ""
+  })
+
+  // Process a not-empty match (fails is the value is empty)
+  const notEmptyMatch = match("notEmpty", (key, value, doc) => {
+    return doc[key] == null || doc[key] === ""
+  })
+
+  // Match a document against all criteria
+  const docMatch = doc => {
+    return (
+      stringMatch(doc) &&
+      rangeMatch(doc) &&
+      equalMatch(doc) &&
+      notEqualMatch(doc) &&
+      emptyMatch(doc) &&
+      notEmptyMatch(doc)
+    )
+  }
+
+  // Process all docs
+  return docs.filter(docMatch)
+}
+
+/**
+ * Performs a client-side sort from the equivalent server-side lucene sort
+ * parameters.
+ * @param docs the data
+ * @param sort the sort column
+ * @param sortOrder the sort order ("ascending" or "descending")
+ * @param sortType the type of sort ("string" or "number")
+ */
+export const luceneSort = (docs, sort, sortOrder, sortType = "string") => {
+  if (!sort || !sortOrder || !sortType) {
+    return docs
+  }
+  const parse = sortType === "string" ? x => `${x}` : x => parseFloat(x)
+  return docs.slice().sort((a, b) => {
+    const colA = parse(a[sort])
+    const colB = parse(b[sort])
+    if (sortOrder === "Descending") {
+      return colA > colB ? -1 : 1
+    } else {
+      return colA > colB ? 1 : -1
+    }
+  })
+}
+
+/**
+ * Limits the specified docs to the specified number of rows from the equivalent
+ * server-side lucene limit parameters.
+ * @param docs the data
+ * @param limit the number of docs to limit to
+ */
+export const luceneLimit = (docs, limit) => {
+  const numLimit = parseFloat(limit)
+  if (isNaN(numLimit)) {
+    return docs
+  }
+  return docs.slice(0, numLimit)
+}

packages/standard-components/yarn.lock

@@ -80,6 +80,11 @@ colorette@^1.2.2:
   resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.2.tgz#cbcc79d5e99caea2dbf10eb3a26fd8b3e6acfa94"
   integrity sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w==
 
+dayjs@^1.10.5:
+  version "1.10.5"
+  resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.10.5.tgz#5600df4548fc2453b3f163ebb2abbe965ccfb986"
+  integrity sha512-BUFis41ikLz+65iH6LHQCDm4YPMj5r1YFLdupPIyM4SGcXMmtiLQ7U37i+hGS8urIuqe7I/ou3IS1jVc4nbN4g==
+
 debug@^4.3.2:
   version "4.3.2"
   resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b"

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "0.9.24",
+  "version": "0.9.34",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/string-templates/src/index.cjs

@@ -16,6 +16,9 @@ registerAll(hbsInstance)
  * utility function to check if the object is valid
  */
 function testObject(object) {
+  if (object == null) {
+    throw "Unable to process null object"
+  }
   // JSON stringify will fail if there are any cycles, stops infinite recursion
   try {
     JSON.stringify(object)
@@ -33,7 +36,7 @@ function testObject(object) {
  */
 module.exports.processObject = async (object, context) => {
   testObject(object)
-  for (let key of Object.keys(object)) {
+  for (let key of Object.keys(object || {})) {
     if (object[key] != null) {
       let val = object[key]
       if (typeof val === "string") {
@@ -68,7 +71,7 @@ module.exports.processString = async (string, context) => {
  */
 module.exports.processObjectSync = (object, context) => {
   testObject(object)
-  for (let key of Object.keys(object)) {
+  for (let key of Object.keys(object || {})) {
     let val = object[key]
     if (typeof val === "string") {
       object[key] = module.exports.processStringSync(object[key], context)

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "0.9.24",
+  "version": "0.9.34",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -21,8 +21,8 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.24",
+    "@budibase/auth": "^0.9.34",
-    "@budibase/string-templates": "^0.9.24",
+    "@budibase/string-templates": "^0.9.34",
     "@koa/router": "^8.0.0",
     "aws-sdk": "^2.811.0",
     "bcryptjs": "^2.4.3",

packages/worker/src/api/controllers/admin/roles.js

@@ -8,7 +8,7 @@ const CouchDB = require("../../../db")
 
 exports.fetch = async ctx => {
   // always use the dev apps as they'll be most up to date (true)
-  const apps = await getAllApps({ dev: true })
+  const apps = await getAllApps({ CouchDB, dev: true })
   const promises = []
   for (let app of apps) {
     // use dev app IDs

packages/worker/src/api/controllers/app.js

@@ -1,19 +1,12 @@
-const fetch = require("node-fetch")
 const { DocumentTypes } = require("@budibase/auth").db
 const CouchDB = require("../../db")
-const env = require("../../environment")
 
 const APP_PREFIX = "app_"
 const URL_REGEX_SLASH = /\/|\\/g
 
 exports.getApps = async ctx => {
-  let allDbs
   // allDbs call of CouchDB is very inaccurate in production
-  if (env.COUCH_DB_URL) {
+  const allDbs = await CouchDB.allDbs()
-    allDbs = await (await fetch(`${env.COUCH_DB_URL}/_all_dbs`)).json()
-  } else {
-    allDbs = await CouchDB.allDbs()
-  }
   const appDbNames = allDbs.filter(dbName => dbName.startsWith(APP_PREFIX))
   const appPromises = appDbNames.map(db =>
     new CouchDB(db).get(DocumentTypes.APP_METADATA)