merge

.github/workflows/release.yml

@@ -5,6 +5,11 @@ on:
     branches:
       - master
 
+env:
+  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
+  POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
+  SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+      
 jobs:
   release:
     runs-on: ubuntu-latest 
@@ -18,10 +23,6 @@ jobs:
       - run: yarn lint 
       - run: yarn bootstrap 
       - run: yarn build 
-        env:
-          POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
-          POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
-          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
       - run: yarn test
 
       - name: Configure AWS Credentials

hosting/docker-compose.yaml

@@ -6,6 +6,7 @@ services:
   app-service:
     restart: always
     image: budibase/apps
+    container_name: bbapps
     ports:
       - "${APP_PORT}:4002"
     environment:
@@ -32,6 +33,7 @@ services:
   worker-service:
     restart: always
     image: budibase/worker
+    container_name: bbworker
     ports:
       - "${WORKER_PORT}:4003"
     environment:
@@ -118,7 +120,7 @@ services:
     image: containrrr/watchtower
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
-    command: --debug --http-api-update budibase/apps budibase/worker
+    command: --debug --http-api-update bbapps bbworker
     environment:
       - WATCHTOWER_HTTP_API=true
       - WATCHTOWER_HTTP_API_TOKEN=budibase

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.29",
+  "version": "0.9.38",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.29",
+  "version": "0.9.38",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/auth/src/db/utils.js

@@ -169,7 +169,8 @@ exports.getAllApps = async ({ CouchDB, dev, all } = {}) => {
     dbName.startsWith(exports.APP_PREFIX)
   )
   const appPromises = appDbNames.map(db =>
-    new CouchDB(db).get(DocumentTypes.APP_METADATA)
+    // skip setup otherwise databases could be re-created
+    new CouchDB(db, { skip_setup: true }).get(DocumentTypes.APP_METADATA)
   )
   if (appPromises.length === 0) {
     return []
@@ -194,6 +195,21 @@ exports.getAllApps = async ({ CouchDB, dev, all } = {}) => {
   }
 }
 
+exports.dbExists = async (CouchDB, dbName) => {
+  let exists = false
+  try {
+    const db = CouchDB(dbName, { skip_setup: true })
+    // check if database exists
+    const info = await db.info()
+    if (info && !info.error) {
+      exists = true
+    }
+  } catch (err) {
+    exists = false
+  }
+  return exists
+}
+
 /**
  * Generates a new configuration ID.
  * @returns {string} The new configuration ID which the config doc can be stored under.

packages/auth/src/middleware/passport/google.js

@@ -20,16 +20,10 @@ async function authenticate(token, tokenSecret, profile, done) {
     // use the google profile id
     dbUser = await db.get(userId)
   } catch (err) {
-    console.log("Google user not found. Creating..")
-
-    // create the user
     const user = {
       _id: userId,
       provider: profile.provider,
       roles: {},
-      builder: {
-        global: true,
-      },
       ...profile._json,
     }
 
@@ -50,12 +44,18 @@ async function authenticate(token, tokenSecret, profile, done) {
       user.roles = existing.roles
       user.builder = existing.builder
       user.admin = existing.admin
-    }
 
       const response = await db.post(user)
-
       dbUser = user
       dbUser._rev = response.rev
+    } else {
+      return done(
+        new Error(
+          "email does not yet exist. You must set up your local budibase account first."
+        ),
+        false
+      )
+    }
   }
 
   // authenticate

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.29",
+  "version": "0.9.38",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/bbui/yarn.lock

@@ -402,13 +402,13 @@ braces@~3.0.2:
     fill-range "^7.0.1"
 
 browserslist@^4.0.0:
-  version "4.16.4"
+  version "4.16.6"
-  resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.16.4.tgz#7ebf913487f40caf4637b892b268069951c35d58"
+  resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.16.6.tgz#d7901277a5a88e554ed305b183ec9b0c08f66fa2"
-  integrity sha512-d7rCxYV8I9kj41RH8UKYnvDYCRENUlHRgyXy/Rhr/1BaeLGfiCptEdFE8MIrvGfWbBFNjVYx76SQWvNX1j+/cQ==
+  integrity sha512-Wspk/PqO+4W9qp5iUTJsa1B/QrYn1keNCcEP5OvP7WBwT4KaDly0uONYmC6Xa3Z5IqnUgS0KcgLYu1l74x0ZXQ==
   dependencies:
-    caniuse-lite "^1.0.30001208"
+    caniuse-lite "^1.0.30001219"
     colorette "^1.2.2"
-    electron-to-chromium "^1.3.712"
+    electron-to-chromium "^1.3.723"
     escalade "^3.1.1"
     node-releases "^1.1.71"
 
@@ -469,10 +469,10 @@ caniuse-api@^3.0.0:
     lodash.memoize "^4.1.2"
     lodash.uniq "^4.5.0"
 
-caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001208:
+caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001219:
-  version "1.0.30001209"
+  version "1.0.30001235"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001209.tgz#1bb4be0bd118e98e21cfb7ef617b1ef2164622f4"
+  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001235.tgz#ad5ca75bc5a1f7b12df79ad806d715a43a5ac4ed"
-  integrity sha512-2Ktt4OeRM7EM/JaOZjuLzPYAIqmbwQMNnYbgooT+icoRGrKOyAxA1xhlnotBD1KArRSPsuJp3TdYcZYrL7qNxA==
+  integrity sha512-zWEwIVqnzPkSAXOUlQnPW2oKoYb2aLQ4Q5ejdjBcnH63rfypaW34CxaeBn1VMya2XaEU3P/R2qHpWyj+l0BT1A==
 
 chalk@^2.0.0, chalk@^2.4.1, chalk@^2.4.2:
   version "2.4.2"
@@ -856,10 +856,10 @@ ee-first@1.1.1:
   resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d"
   integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=
 
-electron-to-chromium@^1.3.712:
+electron-to-chromium@^1.3.723:
-  version "1.3.717"
+  version "1.3.749"
-  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.717.tgz#78d4c857070755fb58ab64bcc173db1d51cbc25f"
+  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.749.tgz#0ecebc529ceb49dd2a7c838ae425236644c3439a"
-  integrity sha512-OfzVPIqD1MkJ7fX+yTl2nKyOE4FReeVfMCzzxQS+Kp43hZYwHwThlGP+EGIZRXJsxCM7dqo8Y65NOX/HP12iXQ==
+  integrity sha512-F+v2zxZgw/fMwPz/VUGIggG4ZndDsYy0vlpthi3tjmDZlcfbhN5mYW0evXUsBr2sUtuDANFtle410A9u/sd/4A==
 
 emojis-list@^3.0.0:
   version "3.0.0"
@@ -1554,9 +1554,9 @@ negotiator@0.6.2:
   integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==
 
 node-releases@^1.1.71:
-  version "1.1.71"
+  version "1.1.73"
-  resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.71.tgz#cb1334b179896b1c89ecfdd4b725fb7bbdfc7dbb"
+  resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.73.tgz#dd4e81ddd5277ff846b80b52bb40c49edf7a7b20"
-  integrity sha512-zR6HoT6LrLCRBwukmrVbHv0EpEQjksO6GmFcZQQuCAy139BEsoVKPYnf3jongYW83fAa1torLGYwxxky/p28sg==
+  integrity sha512-uW7fodD6pyW2FZNZnp/Z3hvWKeEW1Y8R1+1CnErE8cXFXzl5blBOoVB41CvMer6P6Q0S5FXDwcHgFd1Wj0U9zg==
 
 nollup@^0.14.1:
   version "0.14.4"
@@ -2564,9 +2564,9 @@ wrappy@1:
   integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=
 
 ws@^5.2.0:
-  version "5.2.2"
+  version "5.2.3"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-5.2.2.tgz#dffef14866b8e8dc9133582514d1befaf96e980f"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-5.2.3.tgz#05541053414921bc29c63bee14b8b0dd50b07b3d"
-  integrity sha512-jaHFD6PFv6UgoIVda6qZllptQsMlDEJkTQcybzzXDYM1XO9Y8em691FGMPmM46WGyLU4z9KMgQN+qrux/nhlHA==
+  integrity sha512-jZArVERrMsKUatIdnLzqvcfydI85dvd/Fp1u/VOpfdDWQ4c9qWXe+VIeAbQ5FrDwciAkr+lzofXLz3Kuf26AOA==
   dependencies:
     async-limiter "~1.0.0"
 

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.29",
+  "version": "0.9.38",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.29",
+    "@budibase/bbui": "^0.9.38",
-    "@budibase/client": "^0.9.29",
+    "@budibase/client": "^0.9.38",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.29",
+    "@budibase/string-templates": "^0.9.38",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/components/design/NavigationPanel/NewScreenModal.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { store, allScreens } from "builderStore"
+  import { store, allScreens, selectedAccessRole } from "builderStore"
   import { tables } from "stores/backend"
   import { roles } from "stores/backend"
   import { Input, Select, ModalContent, Toggle } from "@budibase/bbui"
@@ -14,7 +14,7 @@
   let templateIndex
   let draftScreen
   let createLink = true
-  let roleId = "BASIC"
+  let roleId = $selectedAccessRole || "BASIC"
 
   $: templates = getTemplates($store, $tables.list)
   $: route = !route && $allScreens.length === 0 ? "*" : route

packages/builder/src/components/integration/QueryViewer.svelte

@@ -48,6 +48,8 @@
   $: integrationInfo = $integrations[datasourceType]
   $: queryConfig = integrationInfo?.query
   $: shouldShowQueryConfig = queryConfig && query.queryVerb
+  $: readQuery = query.queryVerb === "read" || query.readable
+  $: queryInvalid = !query.name || (readQuery && data.length === 0)
 
   function newField() {
     fields = [...fields, {}]
@@ -150,11 +152,7 @@
     <div class="viewer-controls">
       <Heading size="S">Results</Heading>
       <ButtonGroup>
-        <Button
+        <Button cta disabled={queryInvalid} on:click={saveQuery}>
-          cta
-          disabled={data.length === 0 || !query.name}
-          on:click={saveQuery}
-        >
           Save Query
         </Button>
         <Button secondary on:click={previewQuery}>Run Query</Button>

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.29",
+  "version": "0.9.38",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -25,7 +25,7 @@
     "inquirer": "^8.0.0",
     "lookpath": "^1.1.0",
     "pkg": "^4.4.9",
-    "posthog-node": "^1.0.7",
+    "posthog-node": "1.0.7",
     "randomstring": "^1.1.5"
   },
   "devDependencies": {

packages/cli/yarn.lock

@@ -1223,7 +1223,7 @@ pkg@^4.4.9:
     resolve "^1.15.1"
     stream-meter "^1.0.4"
 
-posthog-node@^1.0.7:
+posthog-node@1.0.7:
   version "1.0.7"
   resolved "https://registry.yarnpkg.com/posthog-node/-/posthog-node-1.0.7.tgz#a7a9525eebff23312117e57cff3ddac82afb2262"
   integrity sha512-KTCwyU+PU1eAQtjy5ZSJ47mrxv2d/mMkxo+vvV5c+YqfE4mBAY1UPEPMv1nElb5Vq0UnxvyQXaUnOn8d8Xr6Eg==

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.29",
+  "version": "0.9.38",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -18,18 +18,18 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/string-templates": "^0.9.29",
+    "@budibase/string-templates": "^0.9.38",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"
   },
   "devDependencies": {
-    "@budibase/standard-components": "^0.9.29",
+    "@budibase/standard-components": "^0.9.38",
     "@rollup/plugin-commonjs": "^18.0.0",
     "@rollup/plugin-node-resolve": "^11.2.1",
     "fs-extra": "^8.1.0",
     "jsdom": "^16.0.1",
-    "postcss": "^8.2.9",
+    "postcss": "^8.2.10",
     "rollup": "^2.44.0",
     "rollup-plugin-json": "^4.0.0",
     "rollup-plugin-node-builtins": "^2.1.2",

packages/client/src/api/attachments.js

@@ -3,9 +3,9 @@ import API from "./api"
 /**
  * Uploads an attachment to the server.
  */
-export const uploadAttachment = async data => {
+export const uploadAttachment = async (data, tableId = "") => {
   return await API.post({
-    url: "/api/attachments/upload",
+    url: `/api/attachments/${tableId}/upload`,
     body: data,
     json: false,
   })

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.29",
+  "version": "0.9.38",
   "description": "Budibase Web Server",
   "main": "src/electron.js",
   "repository": {
@@ -55,9 +55,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.29",
+    "@budibase/auth": "^0.9.38",
-    "@budibase/client": "^0.9.29",
+    "@budibase/client": "^0.9.38",
-    "@budibase/string-templates": "^0.9.29",
+    "@budibase/string-templates": "^0.9.38",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -109,7 +109,7 @@
   "devDependencies": {
     "@babel/core": "^7.14.3",
     "@babel/preset-env": "^7.14.4",
-    "@budibase/standard-components": "^0.9.29",
+    "@budibase/standard-components": "^0.9.38",
     "@jest/test-sequencer": "^24.8.0",
     "babel-jest": "^27.0.2",
     "docker-compose": "^0.23.6",

packages/server/scripts/exportAppTemplate.js

@@ -5,6 +5,10 @@ const { join } = require("path")
 const CouchDB = require("../src/db")
 // load environment
 const env = require("../src/environment")
+const {
+  USER_METDATA_PREFIX,
+  LINK_USER_METADATA_PREFIX,
+} = require("../src/db/utils")
 
 // Script to export a chosen budibase app into a package
 // Usage: ./scripts/exportAppTemplate.js export --name=Funky --appId=appId
@@ -44,7 +48,13 @@ yargs
       // perform couch dump
 
       const instanceDb = new CouchDB(appId)
-      await instanceDb.dump(writeStream, {})
+      await instanceDb.dump(writeStream, {
+        filter: doc =>
+          !(
+            doc._id.includes(USER_METDATA_PREFIX) ||
+            doc.includes(LINK_USER_METADATA_PREFIX)
+          ),
+      })
       console.log(`Template ${name} exported to ${exportPath}`)
     }
   )

packages/server/src/api/controllers/application.js

@@ -99,18 +99,11 @@ async function createInstance(template) {
   // replicate the template data to the instance DB
   // this is currently very hard to test, downloading and importing template files
   /* istanbul ignore next */
-  let _rev
   if (template && template.useTemplate === "true") {
     const { ok } = await db.load(await getTemplateStream(template))
     if (!ok) {
       throw "Error loading database dump from template."
     }
-    try {
-      const response = await db.get(DocumentTypes.APP_METADATA)
-      _rev = response._rev
-    } catch (err) {
-      _rev = null
-    }
   } else {
     // create the users table
     await db.put(USERS_TABLE_SCHEMA)
@@ -121,7 +114,7 @@ async function createInstance(template) {
   await createRoutingView(appId)
   await createAllSearchIndex(appId)
 
-  return { _id: appId, _rev }
+  return { _id: appId }
 }
 
 exports.fetch = async function (ctx) {
@@ -188,11 +181,21 @@ exports.create = async function (ctx) {
     instanceConfig.file = ctx.request.files.templateFile
   }
   const instance = await createInstance(instanceConfig)
+  const appId = instance._id
 
   const url = await getAppUrlIfNotInUse(ctx)
-  const appId = instance._id
+  const db = new CouchDB(appId)
+  let _rev
+  try {
+    // if template there will be an existing doc
+    const existing = await db.get(DocumentTypes.APP_METADATA)
+    _rev = existing._rev
+  } catch (err) {
+    // nothing to do
+  }
   const newApplication = {
     _id: DocumentTypes.APP_METADATA,
+    _rev,
     appId: instance._id,
     type: "app",
     version: packageJson.version,
@@ -204,11 +207,7 @@ exports.create = async function (ctx) {
     updatedAt: new Date().toISOString(),
     createdAt: new Date().toISOString(),
   }
-  if (instance._rev) {
+  await db.put(newApplication, { force: true })
-    newApplication._rev = instance._rev
-  }
-  const instanceDb = new CouchDB(appId)
-  await instanceDb.put(newApplication)
 
   await createEmptyAppPackage(ctx, newApplication)
   /* istanbul ignore next */

packages/server/src/api/controllers/query.js

@@ -60,7 +60,7 @@ exports.save = async function (ctx) {
   ctx.message = `Query ${query.name} saved successfully.`
 }
 
-async function enrichQueryFields(fields, parameters) {
+async function enrichQueryFields(fields, parameters = {}) {
   const enrichedQuery = {}
 
   // enrich the fields with dynamic parameters

packages/server/src/api/controllers/user.js

@@ -4,10 +4,8 @@ const {
   getUserMetadataParams,
 } = require("../../db/utils")
 const { InternalTables } = require("../../db/utils")
-const {
+const { addAppRoleToUser } = require("../../utilities/workerRequests")
-  getGlobalUsers,
+const { getGlobalUsers } = require("../../utilities/global")
-  addAppRoleToUser,
-} = require("../../utilities/workerRequests")
 const { getFullUser } = require("../../utilities/users")
 
 function removeGlobalProps(user) {
@@ -20,7 +18,7 @@ function removeGlobalProps(user) {
 
 exports.fetchMetadata = async function (ctx) {
   const database = new CouchDB(ctx.appId)
-  const global = await getGlobalUsers(ctx, ctx.appId)
+  const global = await getGlobalUsers(ctx.appId)
   const metadata = (
     await database.allDocs(
       getUserMetadataParams(null, {

packages/server/src/api/routes/static.js

@@ -7,8 +7,8 @@ const {
   PermissionTypes,
   PermissionLevels,
 } = require("@budibase/auth/permissions")
-const usage = require("../../middleware/usageQuota")
 const env = require("../../environment")
+const { paramResource } = require("../../middleware/resourceId")
 
 const router = Router()
 
@@ -39,9 +39,9 @@ router
   .get("/builder/:file*", controller.serveBuilder)
   .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
   .post(
-    "/api/attachments/upload",
+    "/api/attachments/:tableId/upload",
+    paramResource("tableId"),
     authorized(PermissionTypes.TABLE, PermissionLevels.WRITE),
-    usage,
     controller.uploadFile
   )
   .get("/componentlibrary", controller.serveComponentLibrary)

packages/server/src/api/routes/tests/routing.spec.js

@@ -26,11 +26,6 @@ describe("/routing", () => {
 
   describe("fetch", () => {
     it("returns the correct routing for basic user", async () => {
-      workerRequests.getGlobalUsers.mockImplementationOnce((ctx, appId) => {
-        return {
-          roleId: BUILTIN_ROLE_IDS.BASIC,
-        }
-      })
       const res = await request
         .get(`/api/routing/client`)
         .set(await config.roleHeaders({
@@ -52,13 +47,6 @@ describe("/routing", () => {
     })
 
     it("returns the correct routing for power user", async () => {
-      workerRequests.getGlobalUsers.mockImplementationOnce((ctx, appId) => {
-        return {
-          roles: {
-            [appId]: BUILTIN_ROLE_IDS.POWER,
-          }
-        }
-      })
       const res = await request
         .get(`/api/routing/client`)
         .set(await config.roleHeaders({

packages/server/src/api/routes/tests/user.spec.js

@@ -1,7 +1,6 @@
 const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { checkPermissionsEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
-const workerRequests = require("../../../utilities/workerRequests")
 
 jest.mock("../../../utilities/workerRequests", () => ({
   getGlobalUsers: jest.fn(() => {
@@ -25,30 +24,18 @@ describe("/users", () => {
   })
 
   describe("fetch", () => {
-    beforeEach(() => {
-      workerRequests.getGlobalUsers.mockImplementationOnce(() => ([
-          {
-            _id: "us_uuid1",
-          },
-          {
-            _id: "us_uuid2",
-          }
-        ]
-      ))
-    })
-
     it("returns a list of users from an instance db", async () => {
-      await config.createUser("brenda@brenda.com", "brendas_password")
+      await config.createUser("uuidx")
-      await config.createUser("pam@pam.com", "pam_password")
+      await config.createUser("uuidy")
       const res = await request
         .get(`/api/users/metadata`)
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
 
-      expect(res.body.length).toBe(2)
+      expect(res.body.length).toBe(3)
-      expect(res.body.find(u => u._id === `ro_ta_users_us_uuid1`)).toBeDefined()
+      expect(res.body.find(u => u._id === `ro_ta_users_us_uuidx`)).toBeDefined()
-      expect(res.body.find(u => u._id === `ro_ta_users_us_uuid2`)).toBeDefined()
+      expect(res.body.find(u => u._id === `ro_ta_users_us_uuidy`)).toBeDefined()
     })
 
     it("should apply authorization to endpoint", async () => {
@@ -65,9 +52,6 @@ describe("/users", () => {
   })
 
   describe("update", () => {
-    beforeEach(() => {
-    })
-
     it("should be able to update the user", async () => {
       const user = await config.createUser()
       user.roleId = BUILTIN_ROLE_IDS.BASIC
@@ -94,14 +78,6 @@ describe("/users", () => {
   })
 
   describe("find", () => {
-    beforeEach(() => {
-      jest.resetAllMocks()
-      workerRequests.getGlobalUsers.mockImplementationOnce(() => ({
-        _id: "us_uuid1",
-        roleId: BUILTIN_ROLE_IDS.POWER,
-      }))
-    })
-
     it("should be able to find the user", async () => {
       const user = await config.createUser()
       const res = await request
@@ -110,7 +86,7 @@ describe("/users", () => {
         .expect(200)
         .expect("Content-Type", /json/)
       expect(res.body._id).toEqual(user._id)
-      expect(res.body.roleId).toEqual(BUILTIN_ROLE_IDS.POWER)
+      expect(res.body.roleId).toEqual(BUILTIN_ROLE_IDS.ADMIN)
       expect(res.body.tableId).toBeDefined()
     })
   })

packages/server/src/db/linkedRows/index.js

@@ -11,7 +11,9 @@ const {
 const { flatten } = require("lodash")
 const CouchDB = require("../../db")
 const { FieldTypes } = require("../../constants")
-const { getMultiIDParams } = require("../../db/utils")
+const { getMultiIDParams, USER_METDATA_PREFIX } = require("../../db/utils")
+const { partition } = require("lodash")
+const { getGlobalUsers } = require("../../utilities/global")
 
 /**
  * This functionality makes sure that when rows with links are created, updated or deleted they are processed
@@ -57,6 +59,31 @@ async function getLinksForRows(appId, rows) {
   )
 }
 
+async function getFullLinkedDocs(appId, links) {
+  // create DBs
+  const db = new CouchDB(appId)
+  const linkedRowIds = links.map(link => link.id)
+  let linked = (await db.allDocs(getMultiIDParams(linkedRowIds))).rows.map(
+    row => row.doc
+  )
+  // need to handle users as specific cases
+  let [users, other] = partition(linked, linkRow =>
+    linkRow._id.startsWith(USER_METDATA_PREFIX)
+  )
+  const globalUsers = await getGlobalUsers(appId, users)
+  users = users.map(user => {
+    const globalUser = globalUsers.find(
+      globalUser => globalUser && user._id.includes(globalUser._id)
+    )
+    return {
+      ...globalUser,
+      // doing user second overwrites the id and rev (always metadata)
+      ...user,
+    }
+  })
+  return [...other, ...users]
+}
+
 /**
  * Update link documents for a row or table - this is to be called by the API controller when a change is occurring.
  * @param {string} eventType states what type of change which is occurring, means this can be expanded upon in the
@@ -154,14 +181,13 @@ exports.attachFullLinkedDocs = async (appId, table, rows) => {
   if (linkedTableIds.length === 0) {
     return rows
   }
+  // create DBs
   const db = new CouchDB(appId)
+  // get all the links
   const links = (await getLinksForRows(appId, rows)).filter(link =>
     rows.some(row => row._id === link.thisId)
   )
-  const linkedRowIds = links.map(link => link.id)
+  let linked = await getFullLinkedDocs(appId, links)
-  const linked = (await db.allDocs(getMultiIDParams(linkedRowIds))).rows.map(
-    row => row.doc
-  )
   const linkedTables = []
   for (let row of rows) {
     for (let link of links.filter(link => link.thisId === row._id)) {

packages/server/src/db/utils.js

@@ -6,17 +6,11 @@ const {
   APP_DEV_PREFIX,
   APP_PREFIX,
   SEPARATOR,
+  StaticDatabases,
 } = require("@budibase/auth/db")
 
 const UNICODE_MAX = "\ufff0"
 
-const StaticDatabases = {
-  BUILDER: {
-    name: "builder-db",
-    baseDoc: "builder-doc",
-  },
-}
-
 const AppStatus = {
   DEV: "development",
   ALL: "all",
@@ -55,9 +49,18 @@ const SearchIndexes = {
   ROWS: "rows",
 }
 
+exports.StaticDatabases = {
+  BUILDER: {
+    name: "builder-db",
+    baseDoc: "builder-doc",
+  },
+  ...StaticDatabases,
+}
+
 exports.APP_PREFIX = APP_PREFIX
 exports.APP_DEV_PREFIX = APP_DEV_PREFIX
-exports.StaticDatabases = StaticDatabases
+exports.USER_METDATA_PREFIX = `${DocumentTypes.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
+exports.LINK_USER_METADATA_PREFIX = `${DocumentTypes.LINK}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
 exports.ViewNames = ViewNames
 exports.InternalTables = InternalTables
 exports.DocumentTypes = DocumentTypes

packages/server/src/middleware/currentapp.js

@@ -1,9 +1,16 @@
-const { getAppId, setCookie, getCookie } = require("@budibase/auth").utils
+const {
+  getAppId,
+  setCookie,
+  getCookie,
+  clearCookie,
+} = require("@budibase/auth").utils
 const { Cookies } = require("@budibase/auth").constants
 const { getRole } = require("@budibase/auth/roles")
 const { getGlobalSelf } = require("../utilities/workerRequests")
 const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { generateUserMetadataID } = require("../db/utils")
+const { dbExists } = require("@budibase/auth/db")
+const CouchDB = require("../db")
 
 module.exports = async (ctx, next) => {
   // try to get the appID from the request
@@ -13,6 +20,15 @@ module.exports = async (ctx, next) => {
   if (!appCookie && !requestAppId) {
     return next()
   }
+  // check the app exists referenced in cookie
+  if (appCookie) {
+    const appId = appCookie.appId
+    const exists = await dbExists(CouchDB, appId)
+    if (!exists) {
+      clearCookie(ctx, Cookies.CurrentApp)
+      return next()
+    }
+  }
 
   let updateCookie = false,
     appId,

packages/server/src/tests/utilities/TestConfiguration.js

@@ -306,8 +306,8 @@ class TestConfiguration {
     return await this._req(config, null, controllers.layout.save)
   }
 
-  async createUser() {
+  async createUser(id = null) {
-    const globalId = `us_${Math.random()}`
+    const globalId = !id ? `us_${Math.random()}` : `us_${id}`
     const resp = await this.globalUser(globalId)
     return {
       ...resp,

packages/server/src/utilities/fileSystem/index.js

@@ -18,6 +18,10 @@ const download = require("download")
 const env = require("../../environment")
 const { homedir } = require("os")
 const fetch = require("node-fetch")
+const {
+  USER_METDATA_PREFIX,
+  LINK_USER_METADATA_PREFIX,
+} = require("../../db/utils")
 
 const DEFAULT_AUTOMATION_BUCKET =
   "https://prod-budi-automations.s3-eu-west-1.amazonaws.com"
@@ -117,7 +121,14 @@ exports.performBackup = async (appId, backupName) => {
   const writeStream = fs.createWriteStream(path)
   // perform couch dump
   const instanceDb = new CouchDB(appId)
-  await instanceDb.dump(writeStream, {})
+  await instanceDb.dump(writeStream, {
+    // filter out anything that has a user metadata structure in its ID
+    filter: doc =>
+      !(
+        doc._id.includes(USER_METDATA_PREFIX) ||
+        doc.includes(LINK_USER_METADATA_PREFIX)
+      ),
+  })
   // write the file to the object store
   await streamUpload(
     ObjectStoreBuckets.BACKUPS,

packages/server/src/utilities/global.js

@@ -0,0 +1,64 @@
+const CouchDB = require("../db")
+const {
+  getMultiIDParams,
+  getGlobalIDFromUserMetadataID,
+  StaticDatabases,
+} = require("../db/utils")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { getDeployedAppID } = require("@budibase/auth/db")
+const { getGlobalUserParams } = require("@budibase/auth/db")
+
+exports.updateAppRole = (appId, user) => {
+  if (!user.roles) {
+    return user
+  }
+  if (user.builder && user.builder.global) {
+    user.roleId = BUILTIN_ROLE_IDS.ADMIN
+  } else {
+    // always use the deployed app
+    user.roleId = user.roles[getDeployedAppID(appId)]
+    if (!user.roleId) {
+      user.roleId = BUILTIN_ROLE_IDS.PUBLIC
+    }
+  }
+  delete user.roles
+  return user
+}
+
+exports.getGlobalUser = async (appId, userId) => {
+  const db = CouchDB(StaticDatabases.GLOBAL.name)
+  let user = await db.get(getGlobalIDFromUserMetadataID(userId))
+  if (user) {
+    delete user.password
+  }
+  return exports.updateAppRole(appId, user)
+}
+
+exports.getGlobalUsers = async (appId = null, users = null) => {
+  const db = CouchDB(StaticDatabases.GLOBAL.name)
+  let globalUsers
+  if (users) {
+    const globalIds = users.map(user => getGlobalIDFromUserMetadataID(user._id))
+    globalUsers = (await db.allDocs(getMultiIDParams(globalIds))).rows.map(
+      row => row.doc
+    )
+  } else {
+    globalUsers = (
+      await db.allDocs(
+        getGlobalUserParams(null, {
+          include_docs: true,
+        })
+      )
+    ).rows.map(row => row.doc)
+  }
+  globalUsers = globalUsers
+    .filter(user => user != null)
+    .map(user => {
+      delete user.password
+      return user
+    })
+  if (!appId) {
+    return globalUsers
+  }
+  return globalUsers.map(user => exports.updateAppRole(appId, user))
+}

packages/server/src/utilities/users.js

@@ -1,13 +1,9 @@
 const CouchDB = require("../db")
-const { getGlobalIDFromUserMetadataID, InternalTables } = require("../db/utils")
+const { InternalTables } = require("../db/utils")
-const { getGlobalUsers } = require("../utilities/workerRequests")
+const { getGlobalUser } = require("../utilities/global")
 
 exports.getFullUser = async (ctx, userId) => {
-  const global = await getGlobalUsers(
+  const global = await getGlobalUser(ctx.appId, userId)
-    ctx,
-    ctx.appId,
-    getGlobalIDFromUserMetadataID(userId)
-  )
   let metadata
   try {
     // this will throw an error if the db doesn't exist, or there is no appId

packages/server/src/utilities/workerRequests.js

@@ -1,26 +1,8 @@
 const fetch = require("node-fetch")
 const env = require("../environment")
 const { checkSlashesInUrl } = require("./index")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { getDeployedAppID } = require("@budibase/auth/db")
-const { getGlobalIDFromUserMetadataID } = require("../db/utils")
+const { updateAppRole, getGlobalUser } = require("./global")
-
-function getAppRole(appId, user) {
-  if (!user.roles) {
-    return user
-  }
-  if (user.builder && user.builder.global) {
-    user.roleId = BUILTIN_ROLE_IDS.ADMIN
-  } else {
-    // always use the deployed app
-    user.roleId = user.roles[getDeployedAppID(appId)]
-    if (!user.roleId) {
-      user.roleId = BUILTIN_ROLE_IDS.PUBLIC
-    }
-  }
-  delete user.roles
-  return user
-}
 
 function request(ctx, request, noApiKey) {
   if (!request.headers) {
@@ -90,27 +72,6 @@ exports.getDeployedApps = async ctx => {
   }
 }
 
-exports.getGlobalUsers = async (ctx, appId = null, globalId = null) => {
-  const endpoint = globalId
-    ? `/api/admin/users/${globalId}`
-    : `/api/admin/users`
-  const reqCfg = { method: "GET" }
-  const response = await fetch(
-    checkSlashesInUrl(env.WORKER_URL + endpoint),
-    request(ctx, reqCfg)
-  )
-  let users = await response.json()
-  if (!appId) {
-    return users
-  }
-  if (Array.isArray(users)) {
-    users = users.map(user => getAppRole(appId, user))
-  } else {
-    users = getAppRole(appId, users)
-  }
-  return users
-}
-
 exports.getGlobalSelf = async (ctx, appId = null) => {
   const endpoint = `/api/admin/users/self`
   const response = await fetch(
@@ -123,7 +84,7 @@ exports.getGlobalSelf = async (ctx, appId = null) => {
   }
   let json = await response.json()
   if (appId) {
-    json = getAppRole(appId, json)
+    json = updateAppRole(appId, json)
   }
   return json
 }
@@ -136,8 +97,7 @@ exports.addAppRoleToUser = async (ctx, appId, roleId, userId = null) => {
     user = await exports.getGlobalSelf(ctx)
     endpoint = `/api/admin/users/self`
   } else {
-    userId = getGlobalIDFromUserMetadataID(userId)
+    user = await getGlobalUser(appId, userId)
-    user = await exports.getGlobalUsers(ctx, appId, userId)
     body._id = userId
     endpoint = `/api/admin/users`
   }

packages/standard-components/package.json

@@ -29,11 +29,11 @@
   "keywords": [
     "svelte"
   ],
-  "version": "0.9.29",
+  "version": "0.9.38",
   "license": "MIT",
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc",
   "dependencies": {
-    "@budibase/bbui": "^0.9.29",
+    "@budibase/bbui": "^0.9.38",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
     "apexcharts": "^3.22.1",

packages/standard-components/src/forms/AttachmentField.svelte

@@ -9,8 +9,10 @@
 
   let fieldState
   let fieldApi
+  let fieldSchema
 
   const { API, notifications } = getContext("sdk")
+  const formContext = getContext("form")
   const BYTES_IN_MB = 1000000
 
   export let files = []
@@ -28,7 +30,7 @@
     for (let i = 0; i < fileList.length; i++) {
       data.append("file", fileList[i])
     }
-    return await API.uploadAttachment(data)
+    return await API.uploadAttachment(data, formContext?.dataSource?.tableId)
   }
 </script>
 
@@ -45,7 +47,9 @@
     <CoreDropzone
       value={$fieldState.value}
       disabled={$fieldState.disabled}
-      on:change={e => fieldApi.setValue(e.detail)}
+      on:change={e => {
+        fieldApi.setValue(e.detail)
+      }}
       {processFiles}
       {handleFileTooLarge}
     />

packages/standard-components/src/forms/Form.svelte

@@ -80,7 +80,7 @@
   }
 
   // Provide both form API and state to children
-  setContext("form", { formApi, formState })
+  setContext("form", { formApi, formState, dataSource })
 
   // Action context to pass to children
   $: actions = [{ type: ActionTypes.ValidateForm, callback: formApi.validate }]

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "0.9.29",
+  "version": "0.9.38",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/string-templates/scripts/gen-collection-info.js

@@ -19,8 +19,9 @@ const ADDED_HELPERS = {
     date: {
       args: ["datetime", "format"],
       numArgs: 2,
-      example: '{{date now "DD-MM-YYYY"}} -> 21-01-2021',
+      example: '{{date now "DD-MM-YYYY" "America/New_York" }} -> 21-01-2021',
-      description: "Format a date using moment.js date formatting.",
+      description:
+        "Format a date using moment.js date formatting - the timezone is optional and uses the tz database.",
     },
     duration: {
       args: ["time", "durationType"],

packages/string-templates/src/helpers/date.js

@@ -3,6 +3,7 @@ dayjs.extend(require("dayjs/plugin/duration"))
 dayjs.extend(require("dayjs/plugin/advancedFormat"))
 dayjs.extend(require("dayjs/plugin/relativeTime"))
 dayjs.extend(require("dayjs/plugin/utc"))
+dayjs.extend(require("dayjs/plugin/timezone"))
 
 /**
  * This file was largely taken from the helper-date package - we did this for two reasons:
@@ -72,7 +73,8 @@ function setLocale(str, pattern, options) {
   // if options is null then it'll get updated here
   const config = initialConfig(str, pattern, options)
   const defaults = { lang: "en", date: new Date(config.str) }
-  const opts = getContext(this, defaults, config.options)
+  // for now don't allow this to be configurable, don't pass in options
+  const opts = getContext(this, defaults, {})
 
   // set the language to use
   dayjs.locale(opts.lang || opts.language)
@@ -89,7 +91,15 @@ module.exports.date = (str, pattern, options) => {
 
   setLocale(config.str, config.pattern, config.options)
 
-  const date = dayjs(new Date(config.str)).utc()
+  let date = dayjs(new Date(config.str))
+  if (typeof config.options === "string") {
+    date =
+      config.options.toLowerCase() === "utc"
+        ? date.utc()
+        : date.tz(config.options)
+  } else {
+    date = date.tz(dayjs.tz.guess())
+  }
   if (config.pattern === "") {
     return date.toISOString()
   }

packages/string-templates/test/helpers.spec.js

@@ -1,5 +1,6 @@
 const { processString, processObject, isValid } = require("../src/index.cjs")
 const tableJson = require("./examples/table.json")
+const dayjs = require("dayjs")
 
 describe("test the custom helpers we have applied", () => {
   it("should be able to use the object helper", async () => {
@@ -162,7 +163,7 @@ describe("test the date helpers", () => {
   it("should allow use of the date helper", async () => {
     const date = new Date(1611577535000)
     const output = await processString("{{ date time 'YYYY-MM-DD' }}", {
-      time: date.toISOString(),
+      time: date.toUTCString(),
     })
     expect(output).toBe("2021-01-25")
   })
@@ -172,6 +173,25 @@ describe("test the date helpers", () => {
     const output = await processString("{{ date now 'DD' }}", {})
     expect(parseInt(output)).toBe(date.getDate())
   })
+
+  it("should test the timezone capabilities", async () => {
+    const date = new Date(1611577535000)
+    const output = await processString("{{ date time 'HH-mm-ss Z' 'America/New_York' }}", {
+      time: date.toUTCString(),
+    })
+    const formatted = new dayjs(date).tz("America/New_York").format("HH-mm-ss Z")
+    expect(output).toBe(formatted)
+  })
+
+  it("should guess the users timezone when not specified", async () => {
+    const date = new Date()
+    const output = await processString("{{ date time 'Z' }}", {
+      time: date.toUTCString()
+    })
+    const timezone = dayjs.tz.guess()
+    const offset = new dayjs(date).tz(timezone).format("Z")
+    expect(output).toBe(offset)
+  })
 })
 
 describe("test the string helpers", () => {

packages/string-templates/yarn.lock

@@ -2313,9 +2313,9 @@ hmac-drbg@^1.0.1:
     minimalistic-crypto-utils "^1.0.1"
 
 hosted-git-info@^2.1.4:
-  version "2.8.8"
+  version "2.8.9"
-  resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.8.tgz#7539bd4bc1e0e0a895815a2e0262420b12858488"
+  resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9"
-  integrity sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==
+  integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==
 
 html-encoding-sniffer@^2.0.1:
   version "2.0.1"
@@ -4968,9 +4968,9 @@ write-file-atomic@^3.0.0:
     typedarray-to-buffer "^3.1.5"
 
 ws@^7.2.3:
-  version "7.4.2"
+  version "7.4.6"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.2.tgz#782100048e54eb36fe9843363ab1c68672b261dd"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.6.tgz#5654ca8ecdeee47c33a9a4bf6d28e2be2980377c"
-  integrity sha512-T4tewALS3+qsrpGI/8dqNMLIVdq/g/85U98HPMa6F0m6xTbvhXU6RCQLqPH3+SlomNV/LdY6RXEbBpMH6EOJnA==
+  integrity sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==
 
 xml-name-validator@^3.0.0:
   version "3.0.0"

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "0.9.29",
+  "version": "0.9.38",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -21,8 +21,8 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.29",
+    "@budibase/auth": "^0.9.38",
-    "@budibase/string-templates": "^0.9.29",
+    "@budibase/string-templates": "^0.9.38",
     "@koa/router": "^8.0.0",
     "aws-sdk": "^2.811.0",
     "bcryptjs": "^2.4.3",

packages/worker/src/api/controllers/app.js

@@ -1,19 +1,12 @@
-const fetch = require("node-fetch")
 const { DocumentTypes } = require("@budibase/auth").db
 const CouchDB = require("../../db")
-const env = require("../../environment")
 
 const APP_PREFIX = "app_"
 const URL_REGEX_SLASH = /\/|\\/g
 
 exports.getApps = async ctx => {
-  let allDbs
   // allDbs call of CouchDB is very inaccurate in production
-  if (env.COUCH_DB_URL) {
+  const allDbs = await CouchDB.allDbs()
-    allDbs = await (await fetch(`${env.COUCH_DB_URL}/_all_dbs`)).json()
-  } else {
-    allDbs = await CouchDB.allDbs()
-  }
   const appDbNames = allDbs.filter(dbName => dbName.startsWith(APP_PREFIX))
   const appPromises = appDbNames.map(db =>
     new CouchDB(db).get(DocumentTypes.APP_METADATA)

yarn.lock

@@ -2355,8 +2355,9 @@ has@^1.0.3:
     function-bind "^1.1.1"
 
 hosted-git-info@^2.1.4, hosted-git-info@^2.7.1:
-  version "2.8.5"
+  version "2.8.9"
-  resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.5.tgz#759cfcf2c4d156ade59b0b2dfabddc42a6b9c70c"
+  resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9"
+  integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==
 
 http-cache-semantics@^3.8.1:
   version "3.8.1"