Merge branch 'develop' of github.com:Budibase/budibase into feature/custom-css-conditional-ui

.github/workflows/deploy-release.yml

@@ -68,16 +68,28 @@ jobs:
             ]
         env:
           KUBECONFIG_FILE: '${{ secrets.RELEASE_KUBECONFIG }}'
-      
-      - name: Set the base64 kubeconfig
-        run: echo 'RELEASE_KUBECONFIG=${{ secrets.RELEASE_KUBECONFIG }}' | base64
 
-      - name: Re roll the services
+      - name: Re roll app-service
         uses: actions-hub/kubectl@master
         env:
-          KUBE_CONFIG: ${{ env.RELEASE_KUBECONFIG }}
+          KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
         with:
-          args: rollout restart deployment proxy-service -n budibase && kubectl rollout restart deployment app-service -n budibase && kubectl rollout restart deployment worker-service -n budibase
+          args: rollout restart deployment app-service -n budibase
+
+      - name: Re roll proxy-service
+        uses: actions-hub/kubectl@master
+        env:
+          KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
+        with:
+          args: rollout restart deployment proxy-service -n budibase
+
+      - name: Re roll worker-service
+        uses: actions-hub/kubectl@master
+        env:
+          KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
+        with:
+          args: rollout restart deployment worker-service -n budibase
+
 
       - name: Discord Webhook Action
         uses: tsickert/discord-webhook@v4.0.0

.github/workflows/release-develop.yml

@@ -121,15 +121,26 @@ jobs:
         env:
           KUBECONFIG_FILE: '${{ secrets.RELEASE_KUBECONFIG }}'
       
-      - name: Set the base64 kubeconfig
+      - name: Re roll app-service
-        run: echo 'RELEASE_KUBECONFIG=${{ secrets.RELEASE_KUBECONFIG }}' | base64
-
-      - name: Re roll the services
         uses: actions-hub/kubectl@master
         env:
-          KUBE_CONFIG: ${{ env.RELEASE_KUBECONFIG }}
+          KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
         with:
-          args: rollout restart deployment proxy-service -n budibase && kubectl rollout restart deployment app-service -n budibase && kubectl rollout restart deployment worker-service -n budibase
+          args: rollout restart deployment app-service -n budibase
+
+      - name: Re roll proxy-service
+        uses: actions-hub/kubectl@master
+        env:
+          KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
+        with:
+          args: rollout restart deployment proxy-service -n budibase
+
+      - name: Re roll worker-service
+        uses: actions-hub/kubectl@master
+        env:
+          KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
+        with:
+          args: rollout restart deployment worker-service -n budibase
 
       - name: Discord Webhook Action
         uses: tsickert/discord-webhook@v4.0.0

.github/workflows/smoke_test.yaml

@@ -1,4 +1,4 @@
-name: Budibase Smoke Test
+name: Budibase Nightly Tests
 
 on:
  workflow_dispatch:
@@ -6,7 +6,7 @@ on:
   - cron: "0 5 * * *" # every day at 5AM
 
 jobs:
-  release:
+  nightly:
     runs-on: ubuntu-latest
 
     steps:
@@ -43,6 +43,18 @@ jobs:
           name: Test Reports
           path: packages/builder/cypress/reports/testReport.html
 
+      # TODO: enable once running in QA test env
+      # - name: Configure AWS Credentials
+      #   uses: aws-actions/configure-aws-credentials@v1
+      #   with:
+      #     aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      #     aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      #     aws-region: eu-west-1
+      
+      # - name: Upload test results HTML
+      #   uses: aws-actions/configure-aws-credentials@v1
+      #   run: aws s3 cp packages/builder/cypress/reports/testReport.html s3://{{ secrets.BUDI_QA_REPORTS_BUCKET_NAME }}/$GITHUB_RUN_ID/index.html
+
       - name: Cypress Discord Notify
         run: yarn test:e2e:ci:notify
         env:

.prettierrc.json

@@ -4,7 +4,7 @@
   "singleQuote": false,
   "trailingComma": "es5",
   "arrowParens": "avoid",
-  "jsxBracketSameLine": false,
+  "bracketSameLine": false,
   "plugins": ["prettier-plugin-svelte"],
   "svelteSortOrder": "options-scripts-markup-styles"
 }

charts/budibase/templates/app-service-deployment.yaml

@@ -130,6 +130,22 @@ spec:
         - name: BB_ADMIN_USER_PASSWORD
           value: { { .Values.globals.bbAdminUserPassword | quote } }
         {{ end }}
+        {{ if .Values.services.apps.nodeDebug }}
+        - name: NODE_DEBUG
+          value: {{ .Values.services.apps.nodeDebug | quote }}
+        {{ end }}
+        {{ if .Values.globals.elasticApmEnabled }}
+        - name: ELASTIC_APM_ENABLED
+          value: {{ .Values.globals.elasticApmEnabled | quote }}
+        {{ end }}
+        {{ if .Values.globals.elasticApmSecretToken }}
+        - name: ELASTIC_APM_SECRET_TOKEN
+          value: {{ .Values.globals.elasticApmSecretToken | quote }}
+        {{ end }}
+        {{ if .Values.globals.elasticApmServerUrl }}
+        - name: ELASTIC_APM_SERVER_URL
+          value: {{ .Values.globals.elasticApmServerUrl | quote }}
+        {{ end }}
 
         image: budibase/apps:{{ .Values.globals.appVersion }}
         imagePullPolicy: Always

charts/budibase/templates/worker-service-deployment.yaml

@@ -27,6 +27,8 @@ spec:
     spec:
       containers:
       - env:
+        - name: BUDIBASE_ENVIRONMENT
+          value: {{ .Values.globals.budibaseEnv }}
         - name: DEPLOYMENT_ENVIRONMENT
           value: "kubernetes"
         - name: CLUSTER_PORT
@@ -125,6 +127,19 @@ spec:
           value: {{ .Values.globals.google.secret | quote }}
         - name: TENANT_FEATURE_FLAGS
           value: {{ .Values.globals.tenantFeatureFlags | quote }}
+        {{ if .Values.globals.elasticApmEnabled }}
+        - name: ELASTIC_APM_ENABLED
+          value: {{ .Values.globals.elasticApmEnabled | quote }}
+        {{ end }}
+        {{ if .Values.globals.elasticApmSecretToken }}
+        - name: ELASTIC_APM_SECRET_TOKEN
+          value: {{ .Values.globals.elasticApmSecretToken | quote }}
+        {{ end }}
+        {{ if .Values.globals.elasticApmServerUrl }}
+        - name: ELASTIC_APM_SERVER_URL
+          value: {{ .Values.globals.elasticApmServerUrl | quote }}
+        {{ end }}
+
         image: budibase/worker:{{ .Values.globals.appVersion }}
         imagePullPolicy: Always
         livenessProbe:

charts/budibase/values.yaml

@@ -114,6 +114,10 @@ globals:
   smtp:
     enabled: false
 
+#  elasticApmEnabled:
+#  elasticApmSecretToken:
+#  elasticApmServerUrl:
+
 services:
   budibaseVersion: latest
   dns: cluster.local
@@ -126,6 +130,7 @@ services:
     port: 4002
     replicaCount: 1
     logLevel: info
+#    nodeDebug: "" # set the value of NODE_DEBUG
 
   worker:
     port: 4003

hosting/docker-compose.yaml

@@ -78,6 +78,7 @@ services:
     image: budibase/proxy
     environment:
       - PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
+      - PROXY_RATE_LIMIT_API_PER_SECOND=20
     depends_on:
       - minio-service
       - worker-service

hosting/nginx.dev.conf.hbs

@@ -15,7 +15,10 @@ http {
 
   log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                     '$status $body_bytes_sent "$http_referer" '
-                    '"$http_user_agent" "$http_x_forwarded_for"';
+                    '"$http_user_agent" "$http_x_forwarded_for" '
+                    'response_time=$upstream_response_time proxy_host=$proxy_host upstream_addr=$upstream_addr';
+
+  access_log /var/log/nginx/access.log main;
   
   map $http_upgrade $connection_upgrade {
     default     "upgrade";

hosting/nginx.prod.conf.hbs

@@ -11,7 +11,7 @@ events {
 http {
   # rate limiting
   limit_req_status 429;
-  limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=20r/s;
+  limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=${PROXY_RATE_LIMIT_API_PER_SECOND}r/s;
   limit_req_zone $binary_remote_addr zone=webhooks:10m rate=${PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND}r/s;
 
   include       /etc/nginx/mime.types;
@@ -33,7 +33,10 @@ http {
 
   log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                     '$status $body_bytes_sent "$http_referer" '
-                    '"$http_user_agent" "$http_x_forwarded_for"';
+                    '"$http_user_agent" "$http_x_forwarded_for" '
+                    'response_time=$upstream_response_time proxy_host=$proxy_host upstream_addr=$upstream_addr';
+
+  access_log /var/log/nginx/access.log main;
 
   map $http_upgrade $connection_upgrade {
     default     "upgrade";
@@ -85,6 +88,10 @@ http {
       proxy_pass      http://$apps:4002;
     }
 
+    location /preview {
+      proxy_pass      http://$apps:4002;
+    }
+
     location = / {
       proxy_pass      http://$apps:4002;
     }
@@ -94,6 +101,7 @@ http {
       proxy_pass      http://$watchtower:8080;
     }
     {{/if}}
+
     location ~ ^/(builder|app_) {
       proxy_http_version  1.1;
       proxy_set_header    Connection          $connection_upgrade;

hosting/proxy/Dockerfile

@@ -10,4 +10,5 @@ COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template
 COPY error.html /usr/share/nginx/html/error.html
 
 # Default environment
-ENV PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
+ENV PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
+ENV PROXY_RATE_LIMIT_API_PER_SECOND=20

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.2.44-alpha.1",
+  "version": "1.3.4-alpha.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.2.44-alpha.1",
+  "version": "1.3.4-alpha.1",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -20,7 +20,7 @@
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@budibase/types": "1.2.44-alpha.1",
+    "@budibase/types": "1.3.4-alpha.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.0.1",

packages/backend-core/src/auth.ts

@@ -1,11 +1,11 @@
 const passport = require("koa-passport")
 const LocalStrategy = require("passport-local").Strategy
 const JwtStrategy = require("passport-jwt").Strategy
-const { getGlobalDB } = require("./tenancy")
+import { getGlobalDB } from "./tenancy"
 const refresh = require("passport-oauth2-refresh")
-const { Configs } = require("./constants")
+import { Configs } from "./constants"
-const { getScopedConfig } = require("./db/utils")
+import { getScopedConfig } from "./db/utils"
-const {
+import {
   jwt,
   local,
   authenticated,
@@ -13,7 +13,6 @@ const {
   oidc,
   auditLog,
   tenancy,
-  appTenancy,
   authError,
   ssoCallbackUrl,
   csrf,
@@ -22,32 +21,36 @@ const {
   builderOnly,
   builderOrAdmin,
   joiValidator,
-} = require("./middleware")
+} from "./middleware"
-
+import { invalidateUser } from "./cache/user"
-const { invalidateUser } = require("./cache/user")
+import { User } from "@budibase/types"
 
 // Strategies
 passport.use(new LocalStrategy(local.options, local.authenticate))
 passport.use(new JwtStrategy(jwt.options, jwt.authenticate))
 
-passport.serializeUser((user, done) => done(null, user))
+passport.serializeUser((user: User, done: any) => done(null, user))
 
-passport.deserializeUser(async (user, done) => {
+passport.deserializeUser(async (user: User, done: any) => {
   const db = getGlobalDB()
 
   try {
-    const user = await db.get(user._id)
+    const dbUser = await db.get(user._id)
-    return done(null, user)
+    return done(null, dbUser)
   } catch (err) {
     console.error(`User not found`, err)
     return done(null, false, { message: "User not found" })
   }
 })
 
-async function refreshOIDCAccessToken(db, chosenConfig, refreshToken) {
+async function refreshOIDCAccessToken(
+  db: any,
+  chosenConfig: any,
+  refreshToken: string
+) {
   const callbackUrl = await oidc.getCallbackUrl(db, chosenConfig)
-  let enrichedConfig
+  let enrichedConfig: any
-  let strategy
+  let strategy: any
 
   try {
     enrichedConfig = await oidc.fetchStrategyConfig(chosenConfig, callbackUrl)
@@ -70,22 +73,28 @@ async function refreshOIDCAccessToken(db, chosenConfig, refreshToken) {
     refresh.requestNewAccessToken(
       Configs.OIDC,
       refreshToken,
-      (err, accessToken, refreshToken, params) => {
+      (err: any, accessToken: string, refreshToken: any, params: any) => {
         resolve({ err, accessToken, refreshToken, params })
       }
     )
   })
 }
 
-async function refreshGoogleAccessToken(db, config, refreshToken) {
+async function refreshGoogleAccessToken(
+  db: any,
+  config: any,
+  refreshToken: any
+) {
   let callbackUrl = await google.getCallbackUrl(db, config)
 
   let strategy
   try {
     strategy = await google.strategyFactory(config, callbackUrl)
-  } catch (err) {
+  } catch (err: any) {
     console.error(err)
-    throw new Error("Error constructing OIDC refresh strategy", err)
+    throw new Error(
+      `Error constructing OIDC refresh strategy: message=${err.message}`
+    )
   }
 
   refresh.use(strategy)
@@ -94,14 +103,18 @@ async function refreshGoogleAccessToken(db, config, refreshToken) {
     refresh.requestNewAccessToken(
       Configs.GOOGLE,
       refreshToken,
-      (err, accessToken, refreshToken, params) => {
+      (err: any, accessToken: string, refreshToken: string, params: any) => {
         resolve({ err, accessToken, refreshToken, params })
       }
     )
   })
 }
 
-async function refreshOAuthToken(refreshToken, configType, configId) {
+async function refreshOAuthToken(
+  refreshToken: string,
+  configType: string,
+  configId: string
+) {
   const db = getGlobalDB()
 
   const config = await getScopedConfig(db, {
@@ -113,7 +126,7 @@ async function refreshOAuthToken(refreshToken, configType, configId) {
   let refreshResponse
   if (configType === Configs.OIDC) {
     // configId - retrieved from cookie.
-    chosenConfig = config.configs.filter(c => c.uuid === configId)[0]
+    chosenConfig = config.configs.filter((c: any) => c.uuid === configId)[0]
     if (!chosenConfig) {
       throw new Error("Invalid OIDC configuration")
     }
@@ -134,7 +147,7 @@ async function refreshOAuthToken(refreshToken, configType, configId) {
   return refreshResponse
 }
 
-async function updateUserOAuth(userId, oAuthConfig) {
+async function updateUserOAuth(userId: string, oAuthConfig: any) {
   const details = {
     accessToken: oAuthConfig.accessToken,
     refreshToken: oAuthConfig.refreshToken,
@@ -162,14 +175,13 @@ async function updateUserOAuth(userId, oAuthConfig) {
   }
 }
 
-module.exports = {
+export = {
   buildAuthMiddleware: authenticated,
   passport,
   google,
   oidc,
   jwt: require("jsonwebtoken"),
   buildTenancyMiddleware: tenancy,
-  buildAppTenancyMiddleware: appTenancy,
   auditLog,
   authError,
   buildCsrfMiddleware: csrf,

packages/backend-core/src/db/constants.ts

@@ -18,6 +18,7 @@ export enum ViewName {
   LINK = "by_link",
   ROUTING = "screen_routes",
   AUTOMATION_LOGS = "automation_logs",
+  ACCOUNT_BY_EMAIL = "account_by_email",
 }
 
 export const DeprecatedViews = {
@@ -41,6 +42,7 @@ export enum DocumentType {
   MIGRATIONS = "migrations",
   DEV_INFO = "devinfo",
   AUTOMATION_LOG = "log_au",
+  ACCOUNT_METADATA = "acc_metadata",
 }
 
 export const StaticDatabases = {

packages/backend-core/src/db/views.js

@@ -5,6 +5,8 @@ const {
   SEPARATOR,
 } = require("./utils")
 const { getGlobalDB } = require("../tenancy")
+const { StaticDatabases } = require("./constants")
+const { doWithDB } = require("./")
 
 const DESIGN_DB = "_design/database"
 
@@ -56,6 +58,31 @@ exports.createNewUserEmailView = async () => {
   await db.put(designDoc)
 }
 
+exports.createAccountEmailView = async () => {
+  await doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => {
+    let designDoc
+    try {
+      designDoc = await db.get(DESIGN_DB)
+    } catch (err) {
+      // no design doc, make one
+      designDoc = DesignDoc()
+    }
+    const view = {
+      // if using variables in a map function need to inject them before use
+      map: `function(doc) {
+      if (doc._id.startsWith("${DocumentType.ACCOUNT_METADATA}${SEPARATOR}")) {
+        emit(doc.email.toLowerCase(), doc._id)
+      }
+    }`,
+    }
+    designDoc.views = {
+      ...designDoc.views,
+      [ViewName.ACCOUNT_BY_EMAIL]: view,
+    }
+    await db.put(designDoc)
+  })
+}
+
 exports.createUserAppView = async () => {
   const db = getGlobalDB()
   let designDoc
@@ -128,6 +155,39 @@ exports.createUserBuildersView = async () => {
   await db.put(designDoc)
 }
 
+exports.queryView = async (viewName, params, db, CreateFuncByName) => {
+  try {
+    let response = (await db.query(`database/${viewName}`, params)).rows
+    response = response.map(resp =>
+      params.include_docs ? resp.doc : resp.value
+    )
+    if (params.arrayResponse) {
+      return response
+    } else {
+      return response.length <= 1 ? response[0] : response
+    }
+  } catch (err) {
+    if (err != null && err.name === "not_found") {
+      const createFunc = CreateFuncByName[viewName]
+      await removeDeprecated(db, viewName)
+      await createFunc()
+      return exports.queryView(viewName, params, db, CreateFuncByName)
+    } else {
+      throw err
+    }
+  }
+}
+
+exports.queryPlatformView = async (viewName, params) => {
+  const CreateFuncByName = {
+    [ViewName.ACCOUNT_BY_EMAIL]: exports.createAccountEmailView,
+  }
+
+  return doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => {
+    return exports.queryView(viewName, params, db, CreateFuncByName)
+  })
+}
+
 exports.queryGlobalView = async (viewName, params, db = null) => {
   const CreateFuncByName = {
     [ViewName.USER_BY_EMAIL]: exports.createNewUserEmailView,
@@ -139,20 +199,5 @@ exports.queryGlobalView = async (viewName, params, db = null) => {
   if (!db) {
     db = getGlobalDB()
   }
-  try {
+  return exports.queryView(viewName, params, db, CreateFuncByName)
-    let response = (await db.query(`database/${viewName}`, params)).rows
-    response = response.map(resp =>
-      params.include_docs ? resp.doc : resp.value
-    )
-    return response.length <= 1 ? response[0] : response
-  } catch (err) {
-    if (err != null && err.name === "not_found") {
-      const createFunc = CreateFuncByName[viewName]
-      await removeDeprecated(db, viewName)
-      await createFunc()
-      return exports.queryGlobalView(viewName, params)
-    } else {
-      throw err
-    }
-  }
 }

packages/backend-core/src/events/index.ts

@@ -8,4 +8,5 @@ import { processors } from "./processors"
 
 export const shutdown = () => {
   processors.shutdown()
+  console.log("Events shutdown")
 }

packages/backend-core/src/index.ts

@@ -17,6 +17,7 @@ import constants from "./constants"
 import * as dbConstants from "./db/constants"
 import logging from "./logging"
 import pino from "./pino"
+import * as middleware from "./middleware"
 
 // mimic the outer package exports
 import * as db from "./pkg/db"
@@ -57,6 +58,7 @@ const core = {
   roles,
   ...pino,
   ...errorClasses,
+  middleware,
 }
 
 export = core

packages/backend-core/src/middleware/authenticated.ts

@@ -65,7 +65,7 @@ async function checkApiKey(apiKey: string, populateUser?: Function) {
  * The tenancy modules should not be used here and it should be assumed that the tenancy context
  * has not yet been populated.
  */
-module.exports = (
+export = (
   noAuthPatterns = [],
   opts: { publicAllowed: boolean; populateUser?: Function } = {
     publicAllowed: false,

packages/backend-core/src/middleware/index.ts

@@ -13,7 +13,8 @@ const adminOnly = require("./adminOnly")
 const builderOrAdmin = require("./builderOrAdmin")
 const builderOnly = require("./builderOnly")
 const joiValidator = require("./joi-validator")
-module.exports = {
+
+const pkg = {
   google,
   oidc,
   jwt,
@@ -33,3 +34,5 @@ module.exports = {
   builderOrAdmin,
   joiValidator,
 }
+
+export = pkg

packages/backend-core/src/middleware/joi-validator.js

@@ -13,10 +13,13 @@ function validate(schema, property) {
       params = ctx.request[property]
     }
 
-    schema = schema.append({
+    // not all schemas have the append property e.g. array schemas
-      createdAt: Joi.any().optional(),
+    if (schema.append) {
-      updatedAt: Joi.any().optional(),
+      schema = schema.append({
-    })
+        createdAt: Joi.any().optional(),
+        updatedAt: Joi.any().optional(),
+      })
+    }
 
     const { error } = schema.validate(params)
     if (error) {

packages/backend-core/src/objectStore/index.ts

@@ -66,15 +66,13 @@ const PUBLIC_BUCKETS = [ObjectStoreBuckets.APPS, ObjectStoreBuckets.GLOBAL]
  * @constructor
  */
 export const ObjectStore = (bucket: any) => {
-  AWS.config.update({
-    accessKeyId: env.MINIO_ACCESS_KEY,
-    secretAccessKey: env.MINIO_SECRET_KEY,
-    region: env.AWS_REGION,
-  })
   const config: any = {
     s3ForcePathStyle: true,
     signatureVersion: "v4",
     apiVersion: "2006-03-01",
+    accessKeyId: env.MINIO_ACCESS_KEY,
+    secretAccessKey: env.MINIO_SECRET_KEY,
+    region: env.AWS_REGION,
   }
   if (bucket) {
     config.params = {

packages/backend-core/src/security/sessions.ts

@@ -3,17 +3,27 @@ const { v4: uuidv4 } = require("uuid")
 const { logWarn } = require("../logging")
 const env = require("../environment")
 
-interface Session {
+interface CreateSession {
-  key: string
-  userId: string
   sessionId: string
-  lastAccessedAt: string
+  tenantId: string
-  createdAt: string
   csrfToken?: string
-  value: string
 }
 
-type SessionKey = { key: string }[]
+interface Session extends CreateSession {
+  userId: string
+  lastAccessedAt: string
+  createdAt: string
+  // make optional attributes required
+  csrfToken: string
+}
+
+interface SessionKey {
+  key: string
+}
+
+interface ScannedSession {
+  value: Session
+}
 
 // a week in seconds
 const EXPIRY_SECONDS = 86400 * 7
@@ -22,14 +32,14 @@ function makeSessionID(userId: string, sessionId: string) {
   return `${userId}/${sessionId}`
 }
 
-export async function getSessionsForUser(userId: string) {
+export async function getSessionsForUser(userId: string): Promise<Session[]> {
   if (!userId) {
     console.trace("Cannot get sessions for undefined userId")
     return []
   }
   const client = await redis.getSessionClient()
-  const sessions = await client.scan(userId)
+  const sessions: ScannedSession[] = await client.scan(userId)
-  return sessions.map((session: Session) => session.value)
+  return sessions.map(session => session.value)
 }
 
 export async function invalidateSessions(
@@ -39,33 +49,32 @@ export async function invalidateSessions(
   try {
     const reason = opts?.reason || "unknown"
     let sessionIds: string[] = opts.sessionIds || []
-    let sessions: SessionKey
+    let sessionKeys: SessionKey[]
 
     // If no sessionIds, get all the sessions for the user
     if (sessionIds.length === 0) {
-      sessions = await getSessionsForUser(userId)
+      const sessions = await getSessionsForUser(userId)
-      sessions.forEach(
+      sessionKeys = sessions.map(session => ({
-        (session: any) =>
+        key: makeSessionID(session.userId, session.sessionId),
-          (session.key = makeSessionID(session.userId, session.sessionId))
+      }))
-      )
     } else {
       // use the passed array of sessionIds
       sessionIds = Array.isArray(sessionIds) ? sessionIds : [sessionIds]
-      sessions = sessionIds.map((sessionId: string) => ({
+      sessionKeys = sessionIds.map(sessionId => ({
         key: makeSessionID(userId, sessionId),
       }))
     }
 
-    if (sessions && sessions.length > 0) {
+    if (sessionKeys && sessionKeys.length > 0) {
       const client = await redis.getSessionClient()
       const promises = []
-      for (let session of sessions) {
+      for (let sessionKey of sessionKeys) {
-        promises.push(client.delete(session.key))
+        promises.push(client.delete(sessionKey.key))
       }
       if (!env.isTest()) {
         logWarn(
-          `Invalidating sessions for ${userId} (reason: ${reason}) - ${sessions
+          `Invalidating sessions for ${userId} (reason: ${reason}) - ${sessionKeys
-            .map(session => session.key)
+            .map(sessionKey => sessionKey.key)
             .join(", ")}`
         )
       }
@@ -76,22 +85,26 @@ export async function invalidateSessions(
   }
 }
 
-export async function createASession(userId: string, session: Session) {
+export async function createASession(
+  userId: string,
+  createSession: CreateSession
+) {
   // invalidate all other sessions
   await invalidateSessions(userId, { reason: "creation" })
 
   const client = await redis.getSessionClient()
-  const sessionId = session.sessionId
+  const sessionId = createSession.sessionId
-  if (!session.csrfToken) {
+  const csrfToken = createSession.csrfToken ? createSession.csrfToken : uuidv4()
-    session.csrfToken = uuidv4()
+  const key = makeSessionID(userId, sessionId)
-  }
+
-  session = {
+  const session: Session = {
-    ...session,
+    ...createSession,
+    csrfToken,
     createdAt: new Date().toISOString(),
     lastAccessedAt: new Date().toISOString(),
     userId,
   }
-  await client.store(makeSessionID(userId, sessionId), session, EXPIRY_SECONDS)
+  await client.store(key, session, EXPIRY_SECONDS)
 }
 
 export async function updateSessionTTL(session: Session) {
@@ -106,7 +119,10 @@ export async function endSession(userId: string, sessionId: string) {
   await client.delete(makeSessionID(userId, sessionId))
 }
 
-export async function getSession(userId: string, sessionId: string) {
+export async function getSession(
+  userId: string,
+  sessionId: string
+): Promise<Session> {
   if (!userId || !sessionId) {
     throw new Error(`Invalid session details - ${userId} - ${sessionId}`)
   }

packages/backend-core/src/users.js

@@ -11,7 +11,6 @@ const { UNICODE_MAX } = require("./db/constants")
  * Given an email address this will use a view to search through
  * all the users to find one with this email address.
  * @param {string} email the email to lookup the user by.
- * @return {Promise<object|null>}
  */
 exports.getGlobalUserByEmail = async email => {
   if (email == null) {

packages/backend-core/tests/utilities/mocks/accounts.ts

@@ -0,0 +1,7 @@
+export const getAccount = jest.fn()
+export const getAccountByTenantId = jest.fn()
+
+jest.mock("../../../src/cloud/accounts", () => ({
+  getAccount,
+  getAccountByTenantId,
+}))

packages/backend-core/tests/utilities/mocks/date.js

@@ -1,2 +0,0 @@
-exports.MOCK_DATE = new Date("2020-01-01T00:00:00.000Z")
-exports.MOCK_DATE_TIMESTAMP = 1577836800000

packages/backend-core/tests/utilities/mocks/date.ts

@@ -0,0 +1,2 @@
+export const MOCK_DATE = new Date("2020-01-01T00:00:00.000Z")
+export const MOCK_DATE_TIMESTAMP = 1577836800000

packages/backend-core/tests/utilities/mocks/index.js

@@ -1,9 +0,0 @@
-const posthog = require("./posthog")
-const events = require("./events")
-const date = require("./date")
-
-module.exports = {
-  posthog,
-  date,
-  events,
-}

packages/backend-core/tests/utilities/mocks/index.ts

@@ -0,0 +1,4 @@
+import "./posthog"
+import "./events"
+export * as accounts from "./accounts"
+export * as date from "./date"

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.2.44-alpha.1",
+  "version": "1.3.4-alpha.1",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "^1.2.1",
-    "@budibase/string-templates": "1.2.44-alpha.1",
+    "@budibase/string-templates": "1.3.4-alpha.1",
     "@spectrum-css/actionbutton": "^1.0.1",
     "@spectrum-css/actiongroup": "^1.0.1",
     "@spectrum-css/avatar": "^3.0.2",

packages/bbui/src/Actions/position_dropdown.js

@@ -1,4 +1,4 @@
-export default function positionDropdown(element, { anchor, align }) {
+export default function positionDropdown(element, { anchor, align, maxWidth }) {
   let positionSide = "top"
   let maxHeight = 0
   let dimensions = getDimensions(anchor)
@@ -34,13 +34,24 @@ export default function positionDropdown(element, { anchor, align }) {
   }
 
   function calcLeftPosition() {
-    return align === "right"
+    let left
-      ? dimensions.left + dimensions.width - dimensions.containerWidth
+
-      : dimensions.left
+    if (align == "right") {
+      left = dimensions.left + dimensions.width - dimensions.containerWidth
+    } else if (align == "right-side") {
+      left = dimensions.left + dimensions.width
+    } else {
+      left = dimensions.left
+    }
+
+    return left
   }
 
   element.style.position = "absolute"
   element.style.zIndex = "9999"
+  if (maxWidth) {
+    element.style.maxWidth = `${maxWidth}px`
+  }
   element.style.minWidth = `${dimensions.width}px`
   element.style.maxHeight = `${maxHeight.toFixed(0)}px`
   element.style.transformOrigin = `center ${positionSide}`
@@ -54,10 +65,8 @@ export default function positionDropdown(element, { anchor, align }) {
       element.style.left = `${calcLeftPosition(dimensions).toFixed(0)}px`
     })
   })
-
   resizeObserver.observe(anchor)
   resizeObserver.observe(element)
-
   return {
     destroy() {
       resizeObserver.disconnect()

packages/bbui/src/Form/Core/DatePicker.svelte

@@ -67,6 +67,13 @@
 
     // If time only set date component to 2000-01-01
     if (timeOnly) {
+      // Classic flackpickr causing issues.
+      // When selecting a value for the first time for a "time only" field,
+      // the time is always offset by 1 hour for some reason (regardless of time
+      // zone) so we need to correct it.
+      if (!value && newValue) {
+        newValue = new Date(dates[0].getTime() + 60 * 60 * 1000).toISOString()
+      }
       newValue = `2000-01-01T${newValue.split("T")[1]}`
     }
 

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -139,7 +139,13 @@
         <div class="title">
           <div class="filename">
             {#if selectedUrl}
-              <Link href={selectedUrl}>{selectedImage.name}</Link>
+              <Link
+                target="_blank"
+                download={selectedImage.name}
+                href={selectedUrl}
+              >
+                {selectedImage.name}
+              </Link>
             {:else}
               {selectedImage.name}
             {/if}

packages/bbui/src/Form/Core/RadioGroup.svelte

@@ -10,6 +10,7 @@
   export let disabled = false
   export let getOptionLabel = option => option
   export let getOptionValue = option => option
+  export let getOptionTitle = option => option
 
   const dispatch = createEventDispatcher()
   const onChange = e => dispatch("change", e.target.value)
@@ -19,7 +20,7 @@
   {#if options && Array.isArray(options)}
     {#each options as option}
       <div
-        title={getOptionLabel(option)}
+        title={getOptionTitle(option)}
         class="spectrum-Radio spectrum-FieldGroup-item spectrum-Radio--emphasized"
         class:is-invalid={!!error}
       >

packages/bbui/src/Form/RadioGroup.svelte

@@ -12,6 +12,7 @@
   export let direction = "vertical"
   export let getOptionLabel = option => extractProperty(option, "label")
   export let getOptionValue = option => extractProperty(option, "value")
+  export let getOptionTitle = option => extractProperty(option, "label")
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -35,6 +36,7 @@
     {direction}
     {getOptionLabel}
     {getOptionValue}
+    {getOptionTitle}
     on:change={onChange}
   />
 </Field>

packages/bbui/src/Link/Link.svelte

@@ -8,12 +8,14 @@
   export let secondary = false
   export let overBackground = false
   export let target
+  export let download
 </script>
 
 <a
   on:click
   {href}
   {target}
+  {download}
   class:spectrum-Link--primary={primary}
   class:spectrum-Link--secondary={secondary}
   class:spectrum-Link--overBackground={overBackground}

packages/bbui/src/Popover/Popover.svelte

@@ -11,6 +11,7 @@
   export let align = "right"
   export let portalTarget
   export let dataCy
+  export let maxWidth
 
   export let direction = "bottom"
   export let showTip = false
@@ -45,7 +46,7 @@
   <Portal target={portalTarget}>
     <div
       tabindex="0"
-      use:positionDropdown={{ anchor, align }}
+      use:positionDropdown={{ anchor, align, maxWidth }}
       use:clickOutside={hide}
       on:keydown={handleEscape}
       class={"spectrum-Popover is-open " + (tooltipClasses || "")}

packages/bbui/src/Table/AttachmentRenderer.svelte

@@ -15,14 +15,24 @@
 
 {#each attachments as attachment}
   {#if isImage(attachment.extension)}
-    <Link quiet target="_blank" href={attachment.url}>
+    <Link
+      quiet
+      target="_blank"
+      download={attachment.name}
+      href={attachment.url}
+    >
       <div class="center" title={attachment.name}>
         <img src={attachment.url} alt={attachment.extension} />
       </div>
     </Link>
   {:else}
     <div class="file" title={attachment.name}>
-      <Link quiet target="_blank" href={attachment.url}>
+      <Link
+        quiet
+        target="_blank"
+        download={attachment.name}
+        href={attachment.url}
+      >
         {attachment.extension}
       </Link>
     </div>

packages/builder/cypress/integration/adminAndManagement/authentication.spec.js

@@ -0,0 +1,178 @@
+import filterTests from "../../support/filterTests"
+// const interact = require("../support/interact")
+
+filterTests(["smoke", "all"], () => {
+  context("Auth Configuration", () => {
+    before(() => {
+      cy.login()
+    })
+
+    after(() => {
+      cy.get(".spectrum-SideNav li").contains("Auth").click()
+      cy.location().should(loc => {
+        expect(loc.pathname).to.eq("/builder/portal/manage/auth")
+      })
+
+      cy.get("[data-cy=new-scope-input]").clear()
+
+      cy.get("div.content").scrollTo("bottom")
+      cy.get("[data-cy=oidc-active]").click()
+
+      cy.get("[data-cy=oidc-active]").should('not.be.checked')
+
+      cy.intercept("POST", "/api/global/configs").as("updateAuth")
+      cy.get("button[data-cy=oidc-save]").contains("Save").click({force: true})
+      cy.wait("@updateAuth")
+      cy.get("@updateAuth").its("response.statusCode").should("eq", 200)
+
+      cy.get(".spectrum-Toast-content")
+        .contains("Settings saved")
+        .should("be.visible")
+    })
+
+    it("Should allow updating of the OIDC config", () => {
+      cy.get(".spectrum-SideNav li").contains("Auth").click()
+      cy.location().should(loc => {
+        expect(loc.pathname).to.eq("/builder/portal/manage/auth")
+      })
+      cy.get("div.content").scrollTo("bottom")
+      cy.get(".spectrum-Toast .spectrum-ClearButton").click()
+
+      cy.get("input[data-cy=configUrl]").type("http://budi-auth.com/v2")
+      cy.get("input[data-cy=clientID]").type("34ac6a13-f24a-4b52-c70d-fa544ffd11b2")
+      cy.get("input[data-cy=clientSecret]").type("12A8Q~4nS_DWhOOJ2vWIRsNyDVsdtXPD.Zxa9df_")
+
+      cy.get("button[data-cy=oidc-save]").should("not.be.disabled");
+
+      cy.intercept("POST", "/api/global/configs").as("updateAuth")
+      cy.get("button[data-cy=oidc-save]").contains("Save").click({force: true})
+      cy.wait("@updateAuth")
+      cy.get("@updateAuth").its("response.statusCode").should("eq", 200)
+
+      cy.get(".spectrum-Toast-content")
+        .contains("Settings saved")
+        .should("be.visible")
+    })
+
+    it("Should display default scopes in advanced config.", () => {
+      cy.get(".spectrum-SideNav li").contains("Auth").click()
+      cy.location().should(loc => {
+        expect(loc.pathname).to.eq("/builder/portal/manage/auth")
+      })
+      cy.get("div.content").scrollTo("bottom")
+
+      cy.get(".spectrum-Tags").find(".spectrum-Tags-item").its("length").should("eq", 4)
+
+      cy.get(".spectrum-Tags-item").contains("openid")
+      cy.get(".spectrum-Tags-item").contains("openid").find(".spectrum-ClearButton").should("not.exist")
+
+      cy.get(".spectrum-Tags-item").contains("offline_access")
+      cy.get(".spectrum-Tags-item").contains("email")
+      cy.get(".spectrum-Tags-item").contains("profile")
+    })
+
+    it("Add a new scopes", () => {
+      cy.get(".spectrum-SideNav li").contains("Auth").click()
+      cy.location().should(loc => {
+        expect(loc.pathname).to.eq("/builder/portal/manage/auth")
+      })
+      cy.get("div.content").scrollTo("bottom")
+
+      cy.get("[data-cy=new-scope-input]").type("Sample{enter}")
+      cy.get(".spectrum-Tags").find(".spectrum-Tags-item").its("length").should("eq", 5)
+      cy.get(".spectrum-Tags-item").contains("Sample")
+
+      cy.get(".auth-form input.spectrum-Textfield-input").type("Another ")
+      cy.get(".spectrum-Tags").find(".spectrum-Tags-item").its("length").should("eq", 6)
+      cy.get(".spectrum-Tags-item").contains("Another")
+      
+      cy.get("button[data-cy=oidc-save]").should("not.be.disabled");
+
+      cy.intercept("POST", "/api/global/configs").as("updateAuth")
+      cy.get("button[data-cy=oidc-save]").contains("Save").click({force: true})
+      cy.wait("@updateAuth")
+      cy.get("@updateAuth").its("response.statusCode").should("eq", 200)
+
+      cy.reload()
+
+      cy.get("div.content").scrollTo("bottom")
+
+      cy.get(".spectrum-Tags-item").contains("openid")
+      cy.get(".spectrum-Tags-item").contains("offline_access")
+      cy.get(".spectrum-Tags-item").contains("email")
+      cy.get(".spectrum-Tags-item").contains("profile")
+      cy.get(".spectrum-Tags-item").contains("Sample")
+      cy.get(".spectrum-Tags-item").contains("Another")
+    })
+
+    it("Should allow the removal of auth scopes", () => {
+      cy.get(".spectrum-SideNav li").contains("Auth").click()
+      cy.location().should(loc => {
+        expect(loc.pathname).to.eq("/builder/portal/manage/auth")
+      })
+      cy.get("div.content").scrollTo("bottom")
+
+      cy.get(".spectrum-Tags-item").contains("offline_access").parent().find(".spectrum-ClearButton").click()
+      cy.get(".spectrum-Tags-item").contains("profile").parent().find(".spectrum-ClearButton").click()
+
+      cy.get(".spectrum-Tags").find(".spectrum-Tags-item").its("length").should("eq", 4)
+
+      cy.get(".spectrum-Tags-item").contains("offline_access").should("not.exist")
+      cy.get(".spectrum-Tags-item").contains("profile").should("not.exist")
+
+      cy.get("button[data-cy=oidc-save]").should("not.be.disabled");
+
+      cy.intercept("POST", "/api/global/configs").as("updateAuth")
+      cy.get("button[data-cy=oidc-save]").contains("Save").click({force: true})
+      cy.wait("@updateAuth")
+      cy.get("@updateAuth").its("response.statusCode").should("eq", 200)
+
+      cy.get(".spectrum-Toast-content")
+        .contains("Settings saved")
+        .should("be.visible")
+
+      cy.reload()
+
+      cy.get(".spectrum-Tags").find(".spectrum-Tags-item").its("length").should("eq", 4)
+
+      cy.get(".spectrum-Tags-item").contains("offline_access").should("not.exist")
+      cy.get(".spectrum-Tags-item").contains("profile").should("not.exist")
+    })
+
+    it("Should allow auth scopes to be reset to the core defaults.", () => {
+      cy.get(".spectrum-SideNav li").contains("Auth").click()
+
+      cy.get("div.content").scrollTo("bottom")
+
+      cy.get("[data-cy=restore-oidc-default-scopes]").click({force: true})
+
+      cy.get(".spectrum-Tags").find(".spectrum-Tags-item").its("length").should("eq", 4)
+
+      cy.get(".spectrum-Tags-item").contains("openid")
+      cy.get(".spectrum-Tags-item").contains("offline_access")
+      cy.get(".spectrum-Tags-item").contains("email")
+      cy.get(".spectrum-Tags-item").contains("profile")
+    })
+
+    it("Should not allow invalid characters in the auth scopes", () => {
+      cy.get("[data-cy=new-scope-input]").type("thisIsInvalid\\{enter}")
+      cy.get(".spectrum-Form-itemField .error").contains("Auth scopes cannot contain spaces, double quotes or backslashes")
+      cy.get(".spectrum-Tags").find(".spectrum-Tags-item").its("length").should("eq", 4)
+
+      cy.get("[data-cy=new-scope-input]").clear()
+
+      cy.get("[data-cy=new-scope-input]").type("alsoInvalid\"{enter}")
+      cy.get(".spectrum-Form-itemField .error").contains("Auth scopes cannot contain spaces, double quotes or backslashes")
+      cy.get(".spectrum-Tags").find(".spectrum-Tags-item").its("length").should("eq", 4)
+
+      cy.get("[data-cy=new-scope-input]").clear()
+    })
+
+    it("Should not allow duplicate auth scopes", () => {
+      cy.get("[data-cy=new-scope-input]").type("offline_access{enter}")
+      cy.get(".spectrum-Form-itemField .error").contains("Auth scope already exists")
+      cy.get(".spectrum-Tags").find(".spectrum-Tags-item").its("length").should("eq", 4)
+    })
+
+  })
+})

packages/builder/cypress/integration/appPublishWorkflow.spec.js

@@ -102,7 +102,7 @@ filterTests(['all'], () => {
 
       cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 6000 })
       cy.wait(500)
-      cy.get(interact.APP_TABLE_STATUS, { timeout: 1000 }).eq(0).contains("Unpublished")
+      cy.get(interact.APP_TABLE_STATUS, { timeout: 10000 }).eq(0).contains("Unpublished")
 
     })
   })

packages/builder/cypress/integration/createBinding.spec.js

@@ -10,7 +10,7 @@ filterTests(['smoke', 'all'], () => {
 
     it("should add a current user binding", () => {
       cy.searchAndAddComponent("Paragraph").then(() => {
-        addSettingBinding("text", "Current User._id")
+        addSettingBinding("text", ["Current User", "_id"], "Current User._id")
       })
     })
 
@@ -28,7 +28,7 @@ filterTests(['smoke', 'all'], () => {
       const paramName = "foo"
       cy.createScreen(`/test/:${paramName}`)
       cy.searchAndAddComponent("Paragraph").then(componentId => {
-        addSettingBinding("text", `URL.${paramName}`)
+        addSettingBinding("text", ["URL", paramName], `URL.${paramName}`)
         // The builder preview pages don't have a real URL, so all we can do
         // is check that we were able to bind to the property, and that the
         // component exists on the page
@@ -47,11 +47,13 @@ filterTests(['smoke', 'all'], () => {
     })
   })
 
-  const addSettingBinding = (setting, bindingText, clickOption = true) => {
+  const addSettingBinding = (setting, bindingCategories, bindingText, clickOption = true) => {
     cy.get(`[data-cy="setting-${setting}"] [data-cy=text-binding-button]`).click()
+    cy.get(".category-list li").contains(bindingCategories[0])
     cy.get(".drawer").within(() => {
       if (clickOption) {
-        cy.contains(bindingText).click()
+        cy.get(".category-list li").contains(bindingCategories[0]).click()
+        cy.get("li.binding").contains(bindingCategories[1]).click()
         cy.get("textarea").should("have.value", `{{ ${bindingText} }}`)
       } else {
         cy.get("textarea").type(bindingText)

packages/builder/cypress/integration/datasources/mySql.spec.js

@@ -175,7 +175,10 @@ filterTests(["all"], () => {
         cy.get("@query").its("response.statusCode").should("eq", 200)
         cy.get("@query").its("response.body").should("not.be.empty")
         // Save query
+        cy.intercept("POST", "**/queries").as("saveQuery")
         cy.get(".spectrum-Button").contains("Save Query").click({ force: true })
+        cy.wait("@saveQuery")
+        cy.get("@saveQuery").its("response.statusCode").should("eq", 200)
         cy.get(".nav-item").should("contain", queryName)
       })
 

packages/builder/cypress/integration/datasources/postgreSql.spec.js

@@ -252,7 +252,8 @@ filterTests(["all"], () => {
           .contains("Delete Query")
           .click({ force: true })
         // Confirm deletion
-        cy.reload({ timeout: 5000 })
+        cy.reload()
+        cy.get(".nav-item", { timeout: 30000 }).contains(datasource).click({ force: true })
         cy.get(".nav-item", { timeout: 1000 }).should("not.contain", queryRename)
       })
 

packages/builder/cypress/integration/revertApp.spec.js

@@ -48,6 +48,7 @@ filterTests(['smoke', 'all'], () => {
                 cy.get(interact.AREA_LABEL_REVERT).click({ force: true })
             })
             cy.get(interact.SPECTRUM_DIALOG_GRID).within(() => {
+                cy.get("input").type("Cypress Tests")
                 // Click Revert
                 cy.get(interact.SPECTRUM_BUTTON).contains("Revert").click({ force: true })
                 cy.wait(2000) // Wait for app to finish reverting

packages/builder/cypress/support/commands.js

@@ -448,10 +448,7 @@ Cypress.Commands.add("createTable", (tableName, initialTable) => {
         .contains("Continue")
         .click({ force: true })
     })
-  cy.get(".spectrum-Modal", { timeout: 10000 }).should(
+  cy.get(".spectrum-Modal").contains("Create Table", { timeout: 10000 })
-    "not.contain",
-    "Add data source"
-  )
   cy.get(".spectrum-Modal", { timeout: 2000 }).within(() => {
     cy.get("input", { timeout: 2000 }).first().type(tableName).blur()
     cy.get(".spectrum-ButtonGroup").contains("Create").click()
@@ -742,8 +739,15 @@ Cypress.Commands.add("deleteAllScreens", () => {
 Cypress.Commands.add("navigateToFrontend", () => {
   // Clicks on Design tab and then the Home nav item
   cy.wait(500)
+  cy.intercept("**/preview").as("preview")
   cy.contains("Design").click()
-  cy.get(".spectrum-Search", { timeout: 2000 }).type("/")
+  cy.wait("@preview")
+  cy.get("@preview").then(res => {
+    if (res.statusCode != 200) {
+      cy.reload()
+    }
+  })
+  cy.get(".spectrum-Search", { timeout: 20000 }).type("/")
   cy.get(".nav-item", { timeout: 2000 }).contains("home").click({ force: true })
 })
 

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.2.44-alpha.1",
+  "version": "1.3.4-alpha.1",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -13,11 +13,11 @@
     "cy:setup:ci": "node ./cypress/setup.js",
     "cy:open": "cypress open",
     "cy:run": "cypress run",
-    "cy:run:ci": "cypress run --headed --browser chrome --spec  cypress/integration/createApp.spec.js",
+    "cy:run:ci": "cypress run --headed --browser chrome --spec cypress/integration/createApp.spec.js",
     "cy:run:ci:record": "xvfb-run cypress run --headed --browser chrome --record",
     "cy:test": "start-server-and-test cy:setup http://localhost:4100/builder cy:run",
     "cy:ci": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:run:ci",
-    "cy:ci:record": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:run:ci:record && npm run cy:ci:report",
+    "cy:ci:record": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:run:ci:record; npm run cy:ci:report",
     "cy:ci:report": "mochawesome-merge cypress/reports/*.json > cypress/reports/testReport.json && marge cypress/reports/testReport.json --reportDir cypress/reports --inline",
     "cy:ci:notify": "node scripts/cypressResultsWebhook",
     "cy:debug": "start-server-and-test cy:setup http://localhost:4100/builder cy:open",
@@ -69,10 +69,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "1.2.44-alpha.1",
+    "@budibase/bbui": "1.3.4-alpha.1",
-    "@budibase/client": "1.2.44-alpha.1",
+    "@budibase/client": "1.3.4-alpha.1",
-    "@budibase/frontend-core": "1.2.44-alpha.1",
+    "@budibase/frontend-core": "1.3.4-alpha.1",
-    "@budibase/string-templates": "1.2.44-alpha.1",
+    "@budibase/string-templates": "1.3.4-alpha.1",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/scripts/cypressResultsWebhook.js

@@ -5,7 +5,6 @@ const path = require("path")
 const fs = require("fs")
 
 const WEBHOOK_URL = process.env.CYPRESS_WEBHOOK_URL
-const OUTCOME = process.env.CYPRESS_OUTCOME
 const DASHBOARD_URL = process.env.CYPRESS_DASHBOARD_URL
 const GIT_SHA = process.env.GITHUB_SHA
 const GITHUB_ACTIONS_RUN_URL = process.env.GITHUB_ACTIONS_RUN_URL
@@ -35,6 +34,8 @@ async function discordCypressResultsNotification(report) {
     skipped,
   } = report.stats
 
+  const OUTCOME = failures > 0 ? "failure" : "success"
+
   const options = {
     method: "POST",
     headers: {
@@ -114,7 +115,7 @@ async function discordCypressResultsNotification(report) {
   }
   const response = await fetch(WEBHOOK_URL, options)
 
-  if (response.status >= 400) {
+  if (response.status >= 201) {
     const text = await response.text()
     console.error(
       `Error sending discord webhook. \nStatus: ${response.status}. \nResponse Body: ${text}. \nRequest Body: ${options.body}`

packages/builder/src/builderStore/dataBinding.js

@@ -299,7 +299,10 @@ const getProviderContextBindings = (asset, dataProviders) => {
         schema = {}
         const values = context.values || []
         values.forEach(value => {
-          schema[value.key] = { name: value.label, type: "string" }
+          schema[value.key] = {
+            name: value.label,
+            type: value.type || "string",
+          }
         })
       } else if (context.type === "schema") {
         // Schema contexts are generated dynamically depending on their data
@@ -359,6 +362,12 @@ const getProviderContextBindings = (asset, dataProviders) => {
           providerId,
           // Table ID is used by JSON fields to know what table the field is in
           tableId: table?._id,
+          category: component._instanceName,
+          icon: def.icon,
+          display: {
+            name: fieldSchema.name || key,
+            type: fieldSchema.type,
+          },
         })
       })
     })
@@ -385,6 +394,9 @@ const getUserBindings = () => {
       // datasource options, based on bindable properties
       fieldSchema,
       providerId: "user",
+      category: "Current User",
+      icon: "User",
+      display: fieldSchema,
     })
   })
   return bindings
@@ -401,11 +413,17 @@ const getDeviceBindings = () => {
       type: "context",
       runtimeBinding: `${safeDevice}.${makePropSafe("mobile")}`,
       readableBinding: `Device.Mobile`,
+      category: "Device",
+      icon: "DevicePhone",
+      display: { type: "boolean", name: "mobile" },
     })
     bindings.push({
       type: "context",
       runtimeBinding: `${safeDevice}.${makePropSafe("tablet")}`,
       readableBinding: `Device.Tablet`,
+      category: "Device",
+      icon: "DevicePhone",
+      display: { type: "boolean", name: "tablet" },
     })
   }
   return bindings
@@ -429,6 +447,8 @@ const getSelectedRowsBindings = asset => {
           "selectedRows"
         )}`,
         readableBinding: `${table._instanceName}.Selected rows`,
+        category: "Selected rows",
+        icon: "ViewRow",
       }))
     )
 
@@ -460,6 +480,9 @@ const getStateBindings = () => {
       type: "context",
       runtimeBinding: `${safeState}.${makePropSafe(key)}`,
       readableBinding: `State.${key}`,
+      category: "State",
+      icon: "AutomatedSegment",
+      display: { name: key },
     }))
   }
   return bindings
@@ -482,11 +505,17 @@ const getUrlBindings = asset => {
     type: "context",
     runtimeBinding: `${safeURL}.${makePropSafe(param)}`,
     readableBinding: `URL.${param}`,
+    category: "URL",
+    icon: "RailTop",
+    display: { type: "string" },
   }))
   const queryParamsBinding = {
     type: "context",
     runtimeBinding: makePropSafe("query"),
     readableBinding: "Query params",
+    category: "URL",
+    icon: "RailTop",
+    display: { type: "object" },
   }
   return urlParamBindings.concat([queryParamsBinding])
 }
@@ -497,6 +526,9 @@ const getRoleBindings = () => {
       type: "context",
       runtimeBinding: `trim "${role._id}"`,
       readableBinding: `Role.${role.name}`,
+      category: "Role",
+      icon: "UserGroup",
+      display: { type: "string", name: role.name },
     }
   })
 }
@@ -518,6 +550,7 @@ export const getEventContextBindings = (
   // Check if any context bindings are provided by the component for this
   // setting
   const component = findComponent(asset.props, componentId)
+  const def = store.actions.components.getDefinition(component?._component)
   const settings = getComponentSettings(component?._component)
   const eventSetting = settings.find(setting => setting.key === settingKey)
   if (eventSetting?.context?.length) {
@@ -527,6 +560,8 @@ export const getEventContextBindings = (
         runtimeBinding: `${makePropSafe("eventContext")}.${makePropSafe(
           contextEntry.key
         )}`,
+        category: component._instanceName,
+        icon: def.icon,
       })
     })
   }
@@ -548,6 +583,8 @@ export const getEventContextBindings = (
         bindings.push({
           readableBinding: `Action ${idx + 1}.${contextValue.label}`,
           runtimeBinding: `actions.${idx}.${contextValue.value}`,
+          category: "Actions",
+          icon: "JourneyAction",
         })
       })
     }

packages/builder/src/builderStore/store/frontend.js

@@ -19,7 +19,6 @@ import {
   makeComponentUnique,
 } from "../componentUtils"
 import { Helpers } from "@budibase/bbui"
-import { DefaultAppTheme, LAYOUT_NAMES } from "../../constants"
 import { Utils } from "@budibase/frontend-core"
 
 const INITIAL_FRONTEND_STATE = {
@@ -40,6 +39,7 @@ const INITIAL_FRONTEND_STATE = {
     devicePreview: false,
     messagePassing: false,
     continueIfAction: false,
+    showNotificationAction: false,
   },
   errors: [],
   hasAppPackage: false,
@@ -124,35 +124,6 @@ export const getFrontendStore = () => {
       await integrations.init()
       await queries.init()
       await tables.init()
-
-      // Add navigation settings to old apps
-      if (!application.navigation) {
-        const layout = layouts.find(x => x._id === LAYOUT_NAMES.MASTER.PRIVATE)
-        const customTheme = application.customTheme
-        let navigationSettings = {
-          navigation: "Top",
-          title: application.name,
-          navWidth: "Large",
-          navBackground:
-            customTheme?.navBackground || DefaultAppTheme.navBackground,
-          navTextColor:
-            customTheme?.navTextColor || DefaultAppTheme.navTextColor,
-        }
-        if (layout) {
-          navigationSettings.hideLogo = layout.props.hideLogo
-          navigationSettings.hideTitle = layout.props.hideTitle
-          navigationSettings.title = layout.props.title || application.name
-          navigationSettings.logoUrl = layout.props.logoUrl
-          navigationSettings.links = layout.props.links
-          navigationSettings.navigation = layout.props.navigation || "Top"
-          navigationSettings.sticky = layout.props.sticky
-          navigationSettings.navWidth = layout.props.width || "Large"
-          if (navigationSettings.navigation === "None") {
-            navigationSettings.navigation = "Top"
-          }
-        }
-        await store.actions.navigation.save(navigationSettings)
-      }
     },
     theme: {
       save: async theme => {

packages/builder/src/components/automation/AutomationPanel/AutomationList.svelte

@@ -23,7 +23,7 @@
 </script>
 
 <div class="automations-list">
-  {#each $automationStore.automations as automation, idx}
+  {#each $automationStore.automations.sort(aut => aut.name) as automation, idx}
     <NavItem
       border={idx > 0}
       icon="ShareAndroid"

packages/builder/src/components/automation/SetupPanel/FieldSelector.svelte

@@ -14,7 +14,7 @@
   $: {
     let fields = {}
 
-    for (const [key, type] of Object.entries(block?.inputs?.fields)) {
+    for (const [key, type] of Object.entries(block?.inputs?.fields ?? {})) {
       fields = {
         ...fields,
         [key]: {

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -467,6 +467,7 @@
         options={relationshipOptions}
         getOptionLabel={option => option.name}
         getOptionValue={option => option.value}
+        getOptionTitle={option => option.alt}
       />
     {/if}
     <Input

packages/builder/src/components/common/NavItem.svelte

@@ -16,6 +16,7 @@
   export let scrollable = false
   export let highlighted = false
   export let rightAlignIcon = false
+  export let id
 
   const scrollApi = getContext("scroll")
   const dispatch = createEventDispatcher()
@@ -58,6 +59,7 @@
   on:click={onClick}
   ondragover="return false"
   ondragenter="return false"
+  {id}
 >
   <div class="nav-item-content" bind:this={contentRef}>
     {#if withArrow}

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -9,6 +9,9 @@
     Body,
     Layout,
     Button,
+    ActionButton,
+    Icon,
+    Popover,
   } from "@budibase/bbui"
   import { createEventDispatcher, onMount } from "svelte"
   import {
@@ -45,9 +48,25 @@
   let jsValue = initialValueJS ? value : null
   let hbsValue = initialValueJS ? null : value
 
+  let selectedCategory = null
+
+  let popover
+  let popoverAnchor
+  let hoverTarget
+
   $: usingJS = mode === "JavaScript"
   $: searchRgx = new RegExp(search, "ig")
   $: categories = Object.entries(groupBy("category", bindings))
+
+  $: bindingIcons = bindings?.reduce((acc, ele) => {
+    if (ele.icon) {
+      acc[ele.category] = acc[ele.category] || ele.icon
+    }
+    return acc
+  }, {})
+
+  $: categoryIcons = { ...bindingIcons, Helpers: "MagicWand" }
+
   $: filteredCategories = categories
     .map(([name, categoryBindings]) => ({
       name,
@@ -55,10 +74,19 @@
         return binding.readableBinding.match(searchRgx)
       }),
     }))
-    .filter(category => category.bindings?.length > 0)
+    .filter(category => {
+      return (
+        category.bindings?.length > 0 &&
+        (!selectedCategory ? true : selectedCategory === category.name)
+      )
+    })
+
   $: filteredHelpers = helpers?.filter(helper => {
     return helper.label.match(searchRgx) || helper.description.match(searchRgx)
   })
+
+  $: categoryNames = [...categories.map(cat => cat[0]), "Helpers"]
+
   $: codeMirrorHints = bindings?.map(x => `$("${x.readableBinding}")`)
 
   const updateValue = val => {
@@ -140,58 +168,163 @@
   })
 </script>
 
+<span class="detailPopover">
+  <Popover
+    align="right-side"
+    bind:this={popover}
+    anchor={popoverAnchor}
+    maxWidth={300}
+  >
+    <Layout gap="S">
+      <div class="helper">
+        {#if hoverTarget.title}
+          <div class="helper__name">{hoverTarget.title}</div>
+        {/if}
+        {#if hoverTarget.description}
+          <div class="helper__description">
+            {@html hoverTarget.description}
+          </div>
+        {/if}
+        {#if hoverTarget.example}
+          <pre class="helper__example">{hoverTarget.example}</pre>
+        {/if}
+      </div>
+    </Layout>
+  </Popover>
+</span>
+
 <DrawerContent>
   <svelte:fragment slot="sidebar">
-    <div class="container">
+    <Layout noPadding gap="S">
-      <section>
+      {#if selectedCategory}
+        <div>
+          <ActionButton
+            secondary
+            icon={"ArrowLeft"}
+            on:click={() => {
+              selectedCategory = null
+            }}
+          >
+            Back
+          </ActionButton>
+        </div>
+      {/if}
+
+      {#if !selectedCategory}
         <div class="heading">Search</div>
         <Search placeholder="Search" bind:value={search} />
-      </section>
+      {/if}
-      {#each filteredCategories as category}
+
-        {#if category.bindings?.length}
+      {#if !selectedCategory && !search}
-          <section>
+        <ul class="category-list">
-            <div class="heading">{category.name}</div>
+          {#each categoryNames as categoryName}
+            <li
+              on:click={() => {
+                selectedCategory = categoryName
+              }}
+            >
+              <Icon name={categoryIcons[categoryName]} />
+              <span class="category-name">{categoryName} </span>
+              <span class="category-chevron"><Icon name="ChevronRight" /></span>
+            </li>
+          {/each}
+        </ul>
+      {/if}
+
+      {#if selectedCategory || search}
+        {#each filteredCategories as category}
+          {#if category.bindings?.length}
+            <div class="cat-heading">
+              <Icon name={categoryIcons[category.name]} />{category.name}
+            </div>
             <ul>
               {#each category.bindings as binding}
-                <li on:click={() => addBinding(binding)}>
+                <li
-                  <span class="binding__label">{binding.readableBinding}</span>
+                  class="binding"
-                  {#if binding.type}
+                  on:mouseenter={e => {
-                    <span class="binding__type">{binding.type}</span>
+                    popoverAnchor = e.target
-                  {/if}
+                    if (!binding.description) {
-                  {#if binding.description}
+                      return
-                    <br />
+                    }
-                    <div class="binding__description">
+                    hoverTarget = {
-                      {binding.description || ""}
+                      title: binding.display.name || binding.fieldSchema.name,
-                    </div>
+                      description: binding.description,
+                    }
+                    popover.show()
+                    e.stopPropagation()
+                  }}
+                  on:mouseleave={() => {
+                    popover.hide()
+                    popoverAnchor = null
+                    hoverTarget = null
+                  }}
+                  on:focus={() => {}}
+                  on:blur={() => {}}
+                  on:click={() => addBinding(binding)}
+                >
+                  <span class="binding__label">
+                    {#if binding.display?.name}
+                      {binding.display.name}
+                    {:else if binding.fieldSchema?.name}
+                      {binding.fieldSchema?.name}
+                    {:else}
+                      {binding.readableBinding}
+                    {/if}
+                  </span>
+
+                  {#if binding.display?.type || binding.fieldSchema?.type}
+                    <span class="binding__typeWrap">
+                      <span class="binding__type">
+                        {binding.display?.type || binding.fieldSchema?.type}
+                      </span>
+                    </span>
                   {/if}
                 </li>
               {/each}
             </ul>
-          </section>
+          {/if}
+        {/each}
+
+        {#if selectedCategory === "Helpers" || search}
+          {#if filteredHelpers?.length}
+            <div class="heading">Helpers</div>
+            <ul class="helpers">
+              {#each filteredHelpers as helper}
+                <li
+                  class="binding"
+                  on:click={() => addHelper(helper, usingJS)}
+                  on:mouseenter={e => {
+                    popoverAnchor = e.target
+                    if (!helper.displayText && helper.description) {
+                      return
+                    }
+                    hoverTarget = {
+                      title: helper.displayText,
+                      description: helper.description,
+                      example: getHelperExample(helper, usingJS),
+                    }
+                    popover.show()
+                    e.stopPropagation()
+                  }}
+                  on:mouseleave={() => {
+                    popover.hide()
+                    popoverAnchor = null
+                    hoverTarget = null
+                  }}
+                  on:focus={() => {}}
+                  on:blur={() => {}}
+                >
+                  <span class="binding__label">{helper.displayText}</span>
+                  <span class="binding__typeWrap">
+                    <span class="binding__type">function</span>
+                  </span>
+                </li>
+              {/each}
+            </ul>
+          {/if}
         {/if}
-      {/each}
-      {#if filteredHelpers?.length}
-        <section>
-          <div class="heading">Helpers</div>
-          <ul>
-            {#each filteredHelpers as helper}
-              <li on:click={() => addHelper(helper, usingJS)}>
-                <div class="helper">
-                  <div class="helper__name">{helper.displayText}</div>
-                  <div class="helper__description">
-                    {@html helper.description}
-                  </div>
-                  <pre class="helper__example">{getHelperExample(
-                      helper,
-                      usingJS
-                    )}</pre>
-                </div>
-              </li>
-            {/each}
-          </ul>
-        </section>
       {/if}
-    </div>
+    </Layout>
   </svelte:fragment>
   <div class="main">
     <Tabs selected={mode} on:select={onChangeMode}>
@@ -241,6 +374,35 @@
 </DrawerContent>
 
 <style>
+  ul.helpers li * {
+    pointer-events: none;
+  }
+  ul.category-list li {
+    display: flex;
+    gap: var(--spacing-m);
+    align-items: center;
+  }
+  ul.category-list .category-name {
+    font-weight: 600;
+    text-transform: capitalize;
+  }
+  ul.category-list .category-chevron {
+    flex: 1;
+    text-align: right;
+  }
+  ul.category-list .category-chevron :global(div.icon),
+  .cat-heading :global(div.icon) {
+    display: inline-block;
+  }
+  li.binding {
+    display: flex;
+    align-items: center;
+  }
+  li.binding .binding__typeWrap {
+    flex: 1;
+    text-align: right;
+    text-transform: capitalize;
+  }
   .main :global(textarea) {
     min-height: 202px !important;
   }
@@ -251,23 +413,20 @@
     padding: var(--spacing-s) var(--spacing-xl);
   }
 
-  .container {
+  .heading,
-    margin: calc(-1 * var(--spacing-xl));
+  .cat-heading {
-  }
-  .heading {
     font-size: var(--font-size-s);
     font-weight: 600;
     text-transform: uppercase;
     color: var(--spectrum-global-color-gray-600);
-    padding: var(--spacing-xl) 0 var(--spacing-m) 0;
   }
 
-  section {
+  .cat-heading {
-    padding: 0 var(--spacing-xl) var(--spacing-xl) var(--spacing-xl);
+    display: flex;
-  }
+    gap: var(--spacing-m);
-  section:not(:first-child) {
+    align-items: center;
-    border-top: var(--border-light);
   }
+
   ul {
     list-style: none;
     padding: 0;
@@ -278,7 +437,7 @@
     font-size: var(--font-size-s);
     padding: var(--spacing-m);
     border-radius: 4px;
-    border: var(--border-light);
+    background-color: var(--spectrum-global-color-gray-200);
     transition: background-color 130ms ease-in-out, color 130ms ease-in-out,
       border-color 130ms ease-in-out;
     word-wrap: break-word;
@@ -292,22 +451,14 @@
   li:hover {
     color: var(--spectrum-global-color-gray-900);
     background-color: var(--spectrum-global-color-gray-50);
-    border-color: var(--spectrum-global-color-gray-500);
     cursor: pointer;
   }
-  li:hover :global(*) {
-    color: var(--spectrum-global-color-gray-900) !important;
-  }
 
   .binding__label {
     font-weight: 600;
     text-transform: capitalize;
   }
-  .binding__description {
+
-    color: var(--spectrum-global-color-gray-700);
-    margin: 0.5rem 0 0 0;
-    white-space: normal;
-  }
   .binding__type {
     font-family: monospace;
     background-color: var(--spectrum-global-color-gray-200);

packages/builder/src/components/common/bindings/ClientBindingPanel.svelte

@@ -15,7 +15,6 @@
     }
     return bindings?.map(binding => ({
       ...binding,
-      category: "Bindable Values",
       type: null,
     }))
   }

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ShowNotification.svelte

@@ -0,0 +1,61 @@
+<script>
+  import { Select, Label, Checkbox } from "@budibase/bbui"
+  import { onMount } from "svelte"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
+
+  export let parameters
+  export let bindings = []
+
+  const types = [
+    {
+      label: "Success",
+      value: "success",
+    },
+    {
+      label: "Warning",
+      value: "warning",
+    },
+    {
+      label: "Error",
+      value: "error",
+    },
+    {
+      label: "Info",
+      value: "info",
+    },
+  ]
+
+  onMount(() => {
+    if (!parameters.type) {
+      parameters.type = "success"
+    }
+    if (parameters.autoDismiss == null) {
+      parameters.autoDismiss = true
+    }
+  })
+</script>
+
+<div class="root">
+  <Label>Type</Label>
+  <Select bind:value={parameters.type} options={types} placeholder={null} />
+  <Label>Message</Label>
+  <DrawerBindableInput
+    {bindings}
+    value={parameters.message}
+    on:change={e => (parameters.message = e.detail)}
+  />
+  <Label />
+  <Checkbox text="Auto dismiss" bind:value={parameters.autoDismiss} />
+</div>
+
+<style>
+  .root {
+    display: grid;
+    column-gap: var(--spacing-l);
+    row-gap: var(--spacing-s);
+    grid-template-columns: 60px 1fr;
+    align-items: center;
+    max-width: 400px;
+    margin: 0 auto;
+  }
+</style>

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/index.js

@@ -15,3 +15,4 @@ export { default as S3Upload } from "./S3Upload.svelte"
 export { default as ExportData } from "./ExportData.svelte"
 export { default as ContinueIf } from "./ContinueIf.svelte"
 export { default as UpdateFieldValue } from "./UpdateFieldValue.svelte"
+export { default as ShowNotification } from "./ShowNotification.svelte"

packages/builder/src/components/design/settings/controls/ButtonActionEditor/manifest.json

@@ -110,6 +110,12 @@
       "type": "logic",
       "component": "ContinueIf",
       "dependsOnFeature": "continueIfAction"
+    },
+    {
+      "name": "Show Notification",
+      "type": "application",
+      "component": "ShowNotification",
+      "dependsOnFeature": "showNotificationAction"
     }
   ]
 }

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte

@@ -183,7 +183,7 @@
           $goto("./navigation")
         }
       } else if (type === "request-add-component") {
-        $goto(`./components/${$selectedComponent?._id}/new`)
+        toggleAddComponent()
       } else if (type === "highlight-setting") {
         store.actions.settings.highlight(data.setting)
 
@@ -227,9 +227,8 @@
     if (isAddingComponent) {
       $goto(`../${$selectedScreen._id}/components/${$selectedComponent?._id}`)
     } else {
-      $goto(
+      const id = $selectedComponent?._id || $selectedScreen?.props?._id
-        `../${$selectedScreen._id}/components/${$selectedComponent?._id}/new`
+      $goto(`../${$selectedScreen._id}/components/${id}/new`)
-      )
     }
   }
 

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/navigation/ComponentDropdownMenu.svelte

@@ -2,10 +2,20 @@
   import { store } from "builderStore"
   import { ActionMenu, MenuItem, Icon } from "@budibase/bbui"
 
+  export let component
+
   $: noPaste = !$store.componentToPaste
 
   const keyboardEvent = (key, ctrlKey = false) => {
-    document.dispatchEvent(new KeyboardEvent("keydown", { key, ctrlKey }))
+    document.dispatchEvent(
+      new CustomEvent("component-menu", {
+        detail: {
+          key,
+          ctrlKey,
+          id: component?._id,
+        },
+      })
+    )
   }
 </script>
 

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/navigation/ComponentKeyHandler.svelte

@@ -0,0 +1,118 @@
+<script>
+  import { onMount } from "svelte"
+  import { selectedComponent, selectedScreen, store } from "builderStore"
+  import { findComponent } from "builderStore/componentUtils"
+  import { goto, isActive } from "@roxi/routify"
+  import { notifications } from "@budibase/bbui"
+  import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+
+  let confirmDeleteDialog
+  let componentToDelete
+
+  const keyHandlers = {
+    ["^ArrowUp"]: async component => {
+      await store.actions.components.moveUp(component)
+    },
+    ["^ArrowDown"]: async component => {
+      await store.actions.components.moveDown(component)
+    },
+    ["^c"]: component => {
+      store.actions.components.copy(component, false)
+    },
+    ["^x"]: component => {
+      store.actions.components.copy(component, true)
+    },
+    ["^v"]: async component => {
+      await store.actions.components.paste(component, "inside")
+    },
+    ["^d"]: async component => {
+      store.actions.components.copy(component)
+      await store.actions.components.paste(component, "below")
+    },
+    ["^Enter"]: () => {
+      $goto("./new")
+    },
+    ["Delete"]: component => {
+      // Don't show confirmation for the screen itself
+      if (component?._id === $selectedScreen.props._id) {
+        return false
+      }
+      componentToDelete = component
+      confirmDeleteDialog.show()
+    },
+    ["ArrowUp"]: () => {
+      store.actions.components.selectPrevious()
+    },
+    ["ArrowDown"]: () => {
+      store.actions.components.selectNext()
+    },
+    ["Escape"]: () => {
+      if (!$isActive("/new")) {
+        return false
+      }
+      $goto("./")
+    },
+  }
+
+  const handleKeyAction = async (component, key, ctrlKey = false) => {
+    if (!component || !key) {
+      return false
+    }
+    try {
+      // Delete and backspace are the same
+      if (key === "Backspace") {
+        key = "Delete"
+      }
+      // Prefix key with a caret for ctrl modifier
+      if (ctrlKey) {
+        key = "^" + key
+      }
+      const handler = keyHandlers[key]
+      if (!handler) {
+        return false
+      }
+      return handler(component)
+    } catch (error) {
+      console.error(error)
+      notifications.error("Error handling key press")
+    }
+  }
+
+  const handleKeyPress = async e => {
+    // Ignore repeating events
+    if (e.repeat) {
+      return
+    }
+    // Ignore events when typing
+    const activeTag = document.activeElement?.tagName.toLowerCase()
+    if (["input", "textarea"].indexOf(activeTag) !== -1 && e.key !== "Escape") {
+      return
+    }
+    // Key events are always for the selected component
+    return handleKeyAction($selectedComponent, e.key, e.ctrlKey || e.metaKey)
+  }
+
+  const handleComponentMenu = async e => {
+    // Menu events can be for any component
+    const { id, key, ctrlKey } = e.detail
+    const component = findComponent($selectedScreen.props, id)
+    return await handleKeyAction(component, key, ctrlKey)
+  }
+
+  onMount(() => {
+    document.addEventListener("keydown", handleKeyPress)
+    document.addEventListener("component-menu", handleComponentMenu)
+    return () => {
+      document.removeEventListener("keydown", handleKeyPress)
+      document.removeEventListener("component-menu", handleComponentMenu)
+    }
+  })
+</script>
+
+<ConfirmDialog
+  bind:this={confirmDeleteDialog}
+  title="Confirm Deletion"
+  body={`Are you sure you want to delete "${componentToDelete?._instanceName}"?`}
+  okText="Delete Component"
+  onOk={() => store.actions.components.delete(componentToDelete)}
+/>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/navigation/ComponentListPanel.svelte

@@ -2,62 +2,15 @@
   import Panel from "components/design/Panel.svelte"
   import ComponentTree from "./ComponentTree.svelte"
   import { dndStore } from "./dndStore.js"
-  import { goto, isActive } from "@roxi/routify"
+  import { goto } from "@roxi/routify"
-  import { store, selectedScreen, selectedComponent } from "builderStore"
+  import { store, selectedScreen } from "builderStore"
   import NavItem from "components/common/NavItem.svelte"
   import ScreenslotDropdownMenu from "./ScreenslotDropdownMenu.svelte"
-  import { setContext, onMount } from "svelte"
-  import { get } from "svelte/store"
   import DNDPositionIndicator from "./DNDPositionIndicator.svelte"
   import { DropPosition } from "./dndStore"
-  import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import { notifications, Button } from "@budibase/bbui"
-
+  import ComponentKeyHandler from "./ComponentKeyHandler.svelte"
-  let scrollRef
+  import ComponentScrollWrapper from "./ComponentScrollWrapper.svelte"
-  let confirmDeleteDialog
-
-  const scrollTo = bounds => {
-    if (!bounds) {
-      return
-    }
-    const sidebarWidth = 259
-    const navItemHeight = 32
-    const { scrollLeft, scrollTop, offsetHeight } = scrollRef
-    let scrollBounds = scrollRef.getBoundingClientRect()
-    let newOffsets = {}
-
-    // Calculate left offset
-    const offsetX = bounds.left + bounds.width + scrollLeft - 36
-    if (offsetX > sidebarWidth) {
-      newOffsets.left = offsetX - sidebarWidth
-    } else {
-      newOffsets.left = 0
-    }
-    if (newOffsets.left === scrollLeft) {
-      delete newOffsets.left
-    }
-
-    // Calculate top offset
-    const offsetY = bounds.top - scrollBounds?.top + scrollTop
-    if (offsetY > scrollTop + offsetHeight - 2 * navItemHeight) {
-      newOffsets.top = offsetY - offsetHeight + 2 * navItemHeight
-    } else if (offsetY < scrollTop + navItemHeight) {
-      newOffsets.top = offsetY - navItemHeight
-    } else {
-      delete newOffsets.top
-    }
-
-    // Skip if offset is unchanged
-    if (newOffsets.left == null && newOffsets.top == null) {
-      return
-    }
-
-    // Smoothly scroll to the offset
-    scrollRef.scroll({
-      ...newOffsets,
-      behavior: "smooth",
-    })
-  }
 
   const onDrop = async () => {
     try {
@@ -67,95 +20,15 @@
       notifications.error("Error saving component")
     }
   }
-
-  // Set scroll context so components can invoke scrolling when selected
-  setContext("scroll", {
-    scrollTo,
-  })
-
-  const deleteComponent = async () => {
-    await store.actions.components.delete(get(selectedComponent))
-  }
-
-  const handleKeyPress = async e => {
-    // Ignore repeating events
-    if (e.repeat) {
-      return
-    }
-    // Ignore events when typing
-    const activeTag = document.activeElement?.tagName.toLowerCase()
-    if (["input", "textarea"].indexOf(activeTag) !== -1 && e.key !== "Escape") {
-      return
-    }
-    const component = get(selectedComponent)
-    try {
-      if (e.ctrlKey || e.metaKey) {
-        if (e.key === "ArrowUp") {
-          e.preventDefault()
-          await store.actions.components.moveUp(component)
-        } else if (e.key === "ArrowDown") {
-          e.preventDefault()
-          await store.actions.components.moveDown(component)
-        } else if (e.key === "c") {
-          e.preventDefault()
-          await store.actions.components.copy(component, false)
-        } else if (e.key === "x") {
-          e.preventDefault()
-          store.actions.components.copy(component, true)
-        } else if (e.key === "v") {
-          e.preventDefault()
-          await store.actions.components.paste(component, "inside")
-        } else if (e.key === "d") {
-          e.preventDefault()
-          await store.actions.components.copy(component)
-          await store.actions.components.paste(component, "below")
-        } else if (e.key === "Enter") {
-          e.preventDefault()
-          $goto("./new")
-        }
-      } else if (e.key === "Backspace" || e.key === "Delete") {
-        // Don't show confirmation for the screen itself
-        if (component._id === get(selectedScreen).props._id) {
-          return
-        }
-        e.preventDefault()
-        confirmDeleteDialog.show()
-      } else if (e.key === "ArrowUp") {
-        e.preventDefault()
-        await store.actions.components.selectPrevious()
-      } else if (e.key === "ArrowDown") {
-        e.preventDefault()
-        await store.actions.components.selectNext()
-      } else if (e.key === "Escape" && $isActive("./new")) {
-        e.preventDefault()
-        $goto("./")
-      }
-    } catch (error) {
-      console.log(error)
-      notifications.error("Error handling key press")
-    }
-  }
-
-  onMount(() => {
-    document.addEventListener("keydown", handleKeyPress)
-    return () => {
-      document.removeEventListener("keydown", handleKeyPress)
-    }
-  })
 </script>
 
 <Panel title="Components" showExpandIcon borderRight>
   <div class="add-component">
     <Button on:click={() => $goto("./new")} cta>Add component</Button>
   </div>
-  <div class="nav-items-container" bind:this={scrollRef}>
+  <ComponentScrollWrapper>
     <ul>
-      <li
+      <li>
-        on:click={() => {
-          $store.selectedComponentId = $selectedScreen?.props._id
-        }}
-        id={`component-${$selectedScreen?.props._id}`}
-      >
         <NavItem
           text="Screen"
           indentLevel={0}
@@ -164,6 +37,10 @@
           scrollable
           icon="WebPage"
           on:drop={onDrop}
+          on:click={() => {
+            $store.selectedComponentId = $selectedScreen?.props._id
+          }}
+          id={`component-${$selectedScreen?.props._id}`}
         >
           <ScreenslotDropdownMenu component={$selectedScreen?.props} />
         </NavItem>
@@ -187,15 +64,9 @@
         {/if}
       </li>
     </ul>
-  </div>
+  </ComponentScrollWrapper>
 </Panel>
-<ConfirmDialog
+<ComponentKeyHandler />
-  bind:this={confirmDeleteDialog}
-  title="Confirm Deletion"
-  body={`Are you sure you want to delete "${$selectedComponent?._instanceName}"?`}
-  okText="Delete Component"
-  onOk={deleteComponent}
-/>
 
 <style>
   .add-component {
@@ -205,12 +76,6 @@
     flex-direction: column;
     align-items: stretch;
   }
-  .nav-items-container {
-    padding: var(--spacing-xl) 0;
-    flex: 1 1 auto;
-    overflow: auto;
-    height: 0;
-  }
   ul {
     list-style: none;
     padding-left: 0;

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/navigation/ComponentScrollWrapper.svelte

@@ -0,0 +1,82 @@
+<script>
+  import { setContext } from "svelte"
+  import { dndStore } from "./dndStore"
+  import { notifications } from "@budibase/bbui"
+
+  let scrollRef
+
+  const scrollTo = bounds => {
+    if (!bounds) {
+      return
+    }
+    const sidebarWidth = 259
+    const navItemHeight = 32
+    const { scrollLeft, scrollTop, offsetHeight } = scrollRef
+    let scrollBounds = scrollRef.getBoundingClientRect()
+    let newOffsets = {}
+
+    // Calculate left offset
+    const offsetX = bounds.left + bounds.width + scrollLeft - 36
+    if (offsetX > sidebarWidth) {
+      newOffsets.left = offsetX - sidebarWidth
+    } else {
+      newOffsets.left = 0
+    }
+    if (newOffsets.left === scrollLeft) {
+      delete newOffsets.left
+    }
+
+    // Calculate top offset
+    const offsetY = bounds.top - scrollBounds?.top + scrollTop
+    if (offsetY > scrollTop + offsetHeight - 2 * navItemHeight) {
+      newOffsets.top = offsetY - offsetHeight + 2 * navItemHeight
+    } else if (offsetY < scrollTop + navItemHeight) {
+      newOffsets.top = offsetY - navItemHeight
+    } else {
+      delete newOffsets.top
+    }
+
+    // Skip if offset is unchanged
+    if (newOffsets.left == null && newOffsets.top == null) {
+      return
+    }
+
+    // Smoothly scroll to the offset
+    scrollRef.scroll({
+      ...newOffsets,
+      behavior: "smooth",
+    })
+  }
+
+  // Set scroll context so components can invoke scrolling when selected
+  setContext("scroll", {
+    scrollTo,
+  })
+
+  const onDrop = async () => {
+    try {
+      await dndStore.actions.drop()
+    } catch (error) {
+      console.error(error)
+      notifications.error("Error saving component")
+    }
+  }
+</script>
+
+<div
+  bind:this={scrollRef}
+  on:drop={onDrop}
+  ondragover="return false"
+  ondragenter="return false"
+>
+  <slot />
+</div>
+
+<style>
+  div {
+    padding: var(--spacing-xl) 0;
+    flex: 1 1 auto;
+    overflow: auto;
+    height: 0;
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/navigation/ComponentTree.svelte

@@ -68,7 +68,8 @@
     closedNodes = closedNodes
   }
 
-  const onDrop = async () => {
+  const onDrop = async e => {
+    e.stopPropagation()
     try {
       await dndStore.actions.drop()
     } catch (error) {

packages/builder/src/pages/builder/portal/manage/auth/index.svelte

@@ -18,6 +18,8 @@
     Body,
     Select,
     Toggle,
+    Tag,
+    Tags,
   } from "@budibase/bbui"
   import { onMount } from "svelte"
   import { API } from "api"
@@ -29,6 +31,8 @@
     OIDC: "oidc",
   }
 
+  const HasSpacesRegex = /[\\"\s]/
+
   // Some older google configs contain a manually specified value - retain the functionality to edit the field
   // When there is no value or we are in the cloud - prohibit editing the field, must use platform url to change
   $: googleCallbackUrl = undefined
@@ -145,7 +149,6 @@
 
   async function save(docs) {
     let calls = []
-
     // Only if the user has provided an image, upload it
     if (image) {
       let data = new FormData()
@@ -157,7 +160,6 @@
         })
       )
     }
-
     docs.forEach(element => {
       // Delete unsupported fields
       delete element.createdAt
@@ -199,7 +201,6 @@
         }
       }
     })
-
     if (calls.length) {
       Promise.all(calls)
         .then(data => {
@@ -215,6 +216,21 @@
     }
   }
 
+  let defaultScopes = ["profile", "email", "offline_access"]
+
+  const refreshScopes = idx => {
+    providers.oidc.config.configs[idx]["scopes"] =
+      providers.oidc.config.configs[idx]["scopes"]
+  }
+
+  let scopesFields = [
+    {
+      editing: true,
+      inputText: null,
+      error: null,
+    },
+  ]
+
   onMount(async () => {
     try {
       await organisation.init()
@@ -276,7 +292,7 @@
     if (!oidcDoc?._id) {
       providers.oidc = {
         type: ConfigTypes.OIDC,
-        config: { configs: [{ activated: true }] },
+        config: { configs: [{ activated: true, scopes: defaultScopes }] },
       }
     } else {
       originalOidcDoc = cloneDeep(oidcDoc)
@@ -345,6 +361,7 @@
             size="s"
             cta
             on:click={() => save([providers.oidc])}
+            dataCy={"oidc-save"}
           >
             Save
           </Button>
@@ -362,6 +379,7 @@
             bind:value={providers.oidc.config.configs[0][field.name]}
             readonly={field.readonly}
             placeholder={field.placeholder}
+            dataCy={field.name}
           />
         </div>
       {/each}
@@ -392,15 +410,132 @@
       <div class="form-row">
         <Label size="L">Activated</Label>
         <Toggle
+          dataCy={"oidc-active"}
           text=""
           bind:value={providers.oidc.config.configs[0].activated}
         />
       </div>
     </Layout>
+    <span class="advanced-config">
+      <Layout gap="XS" noPadding>
+        <Heading size="XS">
+          <div class="auth-scopes">
+            <div>Advanced</div>
+            <Button
+              secondary
+              newStyles
+              size="S"
+              on:click={() => {
+                providers.oidc.config.configs[0]["scopes"] = [...defaultScopes]
+              }}
+              dataCy={"restore-oidc-default-scopes"}
+            >
+              Restore Defaults
+            </Button>
+          </div>
+        </Heading>
+        <Body size="S">
+          Changes to your authentication scopes will only take effect when you
+          next log in. Please refer to your vendor documentation before
+          modification.
+        </Body>
+
+        <div class="auth-form">
+          <span class="add-new">
+            <Label size="L">{"Auth Scopes"}</Label>
+            <Input
+              dataCy={"new-scope-input"}
+              error={scopesFields[0].error}
+              placeholder={"New Scope"}
+              bind:value={scopesFields[0].inputText}
+              on:keyup={e => {
+                if (!scopesFields[0].inputText) {
+                  scopesFields[0].error = null
+                }
+                if (
+                  e.key === "Enter" ||
+                  e.keyCode === 13 ||
+                  e.code == "Space" ||
+                  e.keyCode == 32
+                ) {
+                  let scopes = providers.oidc.config.configs[0]["scopes"]
+                    ? providers.oidc.config.configs[0]["scopes"]
+                    : [...defaultScopes]
+
+                  let update = scopesFields[0].inputText.trim()
+
+                  if (HasSpacesRegex.test(update)) {
+                    scopesFields[0].error =
+                      "Auth scopes cannot contain spaces, double quotes or backslashes"
+                    return
+                  } else if (scopes.indexOf(update) > -1) {
+                    scopesFields[0].error = "Auth scope already exists"
+                    return
+                  } else if (!update.length) {
+                    scopesFields[0].inputText = null
+                    scopesFields[0].error = null
+                    return
+                  } else {
+                    scopesFields[0].error = null
+                    scopes.push(update)
+                    providers.oidc.config.configs[0]["scopes"] = scopes
+                    scopesFields[0].inputText = null
+                  }
+                }
+              }}
+            />
+          </span>
+          <div class="tag-wrap">
+            <span />
+            <Tags>
+              <Tag closable={false}>openid</Tag>
+              {#each providers.oidc.config.configs[0]["scopes"] || [...defaultScopes] as tag, idx}
+                <Tag
+                  closable={scopesFields[0].editing}
+                  on:click={() => {
+                    let idxScopes = providers.oidc.config.configs[0]["scopes"]
+                    if (idxScopes.length == 1) {
+                      idxScopes.pop()
+                    } else {
+                      idxScopes.splice(idx, 1)
+                      refreshScopes(0)
+                    }
+                  }}
+                >
+                  {tag}
+                </Tag>
+              {/each}
+            </Tags>
+          </div>
+        </div>
+      </Layout>
+    </span>
   {/if}
 </Layout>
 
 <style>
+  .auth-scopes {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+  }
+
+  .advanced-config :global(.spectrum-Tags-item) {
+    margin-left: 0px;
+    margin-top: var(--spacing-m);
+    margin-right: var(--spacing-m);
+  }
+
+  .auth-form > * {
+    display: grid;
+    grid-gap: var(--spacing-l);
+    grid-template-columns: 100px 1fr;
+  }
+
+  .advanced-config .auth-form .tag-wrap {
+    padding: 0px 5px 5px 0px;
+  }
+
   .form-row {
     display: grid;
     grid-template-columns: 100px 1fr;

packages/builder/src/pages/builder/portal/manage/users/_components/AddUserModal.svelte

@@ -44,7 +44,11 @@
     ]
   }
 
-  function validateInput(email, index) {
+  function validateInput(input, index) {
+    if (input.email) {
+      input.email = input.email.trim()
+    }
+    const email = input.email
     if (email) {
       const res = emailValidator(email)
       if (res === true) {
@@ -61,7 +65,7 @@
   const onConfirm = () => {
     let valid = true
     userData.forEach((input, index) => {
-      valid = validateInput(input.email, index) && valid
+      valid = validateInput(input, index) && valid
     })
     if (!valid) {
       return false
@@ -95,7 +99,7 @@
             bind:dropdownValue={input.role}
             options={Constants.BudibaseRoleOptions}
             error={input.error}
-            on:blur={() => validateInput(input.email, index)}
+            on:blur={() => validateInput(input, index)}
           />
         </div>
         <div class="icon">

packages/builder/src/pages/builder/portal/manage/users/_components/DeletionFailureModal.svelte

@@ -0,0 +1,73 @@
+<script>
+  import { Body, ModalContent, Table } from "@budibase/bbui"
+  import { onMount } from "svelte"
+
+  export let userData
+  export let deleteUsersResponse
+
+  let successCount
+  let failureCount
+  let title
+  let unsuccessfulUsers
+  let message
+
+  const setTitle = () => {
+    if (successCount) {
+      title = `${successCount} users deleted`
+    } else {
+      title = "Oops!"
+    }
+  }
+
+  const setMessage = () => {
+    if (successCount) {
+      message = "However there was a problem deleting some users."
+    } else {
+      message = "There was a problem deleting some users."
+    }
+  }
+
+  const setUsers = () => {
+    unsuccessfulUsers = deleteUsersResponse.unsuccessful.map(user => {
+      return {
+        email: user.email,
+        reason: user.reason,
+      }
+    })
+  }
+
+  onMount(() => {
+    successCount = deleteUsersResponse.successful.length
+    failureCount = deleteUsersResponse.unsuccessful.length
+    setTitle()
+    setMessage()
+    setUsers()
+  })
+
+  const schema = {
+    email: {},
+    reason: {},
+  }
+</script>
+
+<ModalContent
+  size="M"
+  {title}
+  confirmText="Close"
+  showCloseIcon={false}
+  showCancelButton={false}
+>
+  <Body size="XS">
+    {message}
+  </Body>
+  <Table
+    {schema}
+    data={unsuccessfulUsers}
+    allowEditColumns={false}
+    allowEditRows={false}
+    allowSelectRows={false}
+  />
+</ModalContent>
+
+<style>
+</style>

packages/builder/src/pages/builder/portal/manage/users/_components/ImportUsersModal.svelte

@@ -62,7 +62,7 @@
       csvString = e.target.result
       files = fileArray
 
-      userEmails = csvString.split("\n")
+      userEmails = csvString.split(/\r?\n/)
     })
     reader.readAsText(fileArray[0])
   }

packages/builder/src/pages/builder/portal/manage/users/_components/InvitedModal.svelte

@@ -0,0 +1,75 @@
+<script>
+  import { Body, ModalContent, Table } from "@budibase/bbui"
+  import { onMount } from "svelte"
+
+  export let inviteUsersResponse
+
+  let hasSuccess
+  let hasFailure
+  let title
+  let failureMessage
+
+  let unsuccessfulUsers
+
+  const setTitle = () => {
+    if (hasSuccess) {
+      title = "Users invited!"
+    } else if (hasFailure) {
+      title = "Oops!"
+    }
+  }
+
+  const setFailureMessage = () => {
+    if (hasSuccess) {
+      failureMessage = "However there was a problem inviting some users."
+    } else {
+      failureMessage = "There was a problem inviting users."
+    }
+  }
+
+  const setUsers = () => {
+    unsuccessfulUsers = inviteUsersResponse.unsuccessful.map(user => {
+      return {
+        email: user.email,
+        reason: user.reason,
+      }
+    })
+  }
+
+  onMount(() => {
+    hasSuccess = inviteUsersResponse.successful.length
+    hasFailure = inviteUsersResponse.unsuccessful.length
+    setTitle()
+    setFailureMessage()
+    setUsers()
+  })
+
+  const failedSchema = {
+    email: {},
+    reason: {},
+  }
+</script>
+
+<ModalContent showCancelButton={false} {title} confirmText="Done">
+  {#if hasSuccess}
+    <Body size="XS">
+      Your users should now receive an email invite to get access to their
+      Budibase account
+    </Body>
+  {/if}
+  {#if hasFailure}
+    <Body size="XS">
+      {failureMessage}
+    </Body>
+    <Table
+      schema={failedSchema}
+      data={unsuccessfulUsers}
+      allowEditColumns={false}
+      allowEditRows={false}
+      allowSelectRows={false}
+    />
+  {/if}
+</ModalContent>
+
+<style>
+</style>

packages/builder/src/pages/builder/portal/manage/users/_components/PasswordModal.svelte

@@ -2,24 +2,78 @@
   import { Body, ModalContent, Table, Icon } from "@budibase/bbui"
   import PasswordCopyRenderer from "./PasswordCopyRenderer.svelte"
   import { parseToCsv } from "helpers/data/utils"
+  import { onMount } from "svelte"
 
   export let userData
+  export let createUsersResponse
 
-  $: mappedData = userData.map(user => {
+  let hasSuccess
-    return {
+  let hasFailure
-      email: user.email,
+  let title
-      password: user.password,
+  let failureMessage
+
+  let userDataIndex
+  let successfulUsers
+  let unsuccessfulUsers
+
+  const setTitle = () => {
+    if (hasSuccess) {
+      title = "Users created!"
+    } else if (hasFailure) {
+      title = "Oops!"
     }
+  }
+
+  const setFailureMessage = () => {
+    if (hasSuccess) {
+      failureMessage = "However there was a problem creating some users."
+    } else {
+      failureMessage = "There was a problem creating some users."
+    }
+  }
+
+  const setUsers = () => {
+    userDataIndex = userData.reduce((prev, current) => {
+      prev[current.email] = current
+      return prev
+    }, {})
+
+    successfulUsers = createUsersResponse.successful.map(user => {
+      return {
+        email: user.email,
+        password: userDataIndex[user.email].password,
+      }
+    })
+
+    unsuccessfulUsers = createUsersResponse.unsuccessful.map(user => {
+      return {
+        email: user.email,
+        reason: user.reason,
+      }
+    })
+  }
+
+  onMount(() => {
+    hasSuccess = createUsersResponse.successful.length
+    hasFailure = createUsersResponse.unsuccessful.length
+    setTitle()
+    setFailureMessage()
+    setUsers()
   })
 
-  const schema = {
+  const successSchema = {
     email: {},
     password: {},
   }
 
+  const failedSchema = {
+    email: {},
+    reason: {},
+  }
+
   const downloadCsvFile = () => {
     const fileName = "passwords.csv"
-    const content = parseToCsv(["email", "password"], mappedData)
+    const content = parseToCsv(["email", "password"], successfulUsers)
 
     download(fileName, content)
   }
@@ -42,36 +96,52 @@
 </script>
 
 <ModalContent
-  size="S"
+  size="M"
-  title="Accounts created!"
+  {title}
   confirmText="Done"
   showCancelButton={false}
   cancelText="Cancel"
   showCloseIcon={false}
 >
-  <Body size="XS">
+  {#if hasFailure}
-    All your new users can be accessed through the autogenerated passwords. Take
+    <Body size="XS">
-    note of these passwords or download the CSV file.
+      {failureMessage}
-  </Body>
+    </Body>
+    <Table
+      schema={failedSchema}
+      data={unsuccessfulUsers}
+      allowEditColumns={false}
+      allowEditRows={false}
+      allowSelectRows={false}
+    />
+  {/if}
+  {#if hasSuccess}
+    <Body size="XS">
+      All your new users can be accessed through the autogenerated passwords.
+      Take note of these passwords or download the CSV file.
+    </Body>
 
-  <div class="container" on:click={downloadCsvFile}>
+    <div class="container" on:click={downloadCsvFile}>
-    <div class="inner">
+      <div class="inner">
-      <Icon name="Download" />
+        <Icon name="Download" />
 
-      <div style="margin-left: var(--spacing-m)">
+        <div style="margin-left: var(--spacing-m)">
-        <Body size="XS">Passwords CSV</Body>
+          <Body size="XS">Passwords CSV</Body>
+        </div>
       </div>
     </div>
-  </div>
 
-  <Table
+    <Table
-    {schema}
+      schema={successSchema}
-    data={mappedData}
+      data={successfulUsers}
-    allowEditColumns={false}
+      allowEditColumns={false}
-    allowEditRows={false}
+      allowEditRows={false}
-    allowSelectRows={false}
+      allowSelectRows={false}
-    customRenderers={[{ column: "password", component: PasswordCopyRenderer }]}
+      customRenderers={[
-  />
+        { column: "password", component: PasswordCopyRenderer },
+      ]}
+    />
+  {/if}
 </ModalContent>
 
 <style>

packages/builder/src/pages/builder/portal/manage/users/index.svelte

@@ -7,7 +7,6 @@
     Table,
     Layout,
     Modal,
-    ModalContent,
     Search,
     notifications,
     Pagination,
@@ -23,6 +22,8 @@
   import { goto } from "@roxi/routify"
   import OnboardingTypeModal from "./_components/OnboardingTypeModal.svelte"
   import PasswordModal from "./_components/PasswordModal.svelte"
+  import InvitedModal from "./_components/InvitedModal.svelte"
+  import DeletionFailureModal from "./_components/DeletionFailureModal.svelte"
   import ImportUsersModal from "./_components/ImportUsersModal.svelte"
   import { createPaginationStore } from "helpers/pagination"
   import { get } from "svelte/store"
@@ -33,7 +34,8 @@
     inviteConfirmationModal,
     onboardingTypeModal,
     passwordModal,
-    importUsersModal
+    importUsersModal,
+    deletionFailureModal
   let pageInfo = createPaginationStore()
   let prevEmail = undefined,
     searchEmail = undefined
@@ -55,6 +57,9 @@
     apps: {},
   }
   $: userData = []
+  $: createUsersResponse = { successful: [], unsuccessful: [] }
+  $: deleteUsersResponse = { successful: [], unsuccessful: [] }
+  $: inviteUsersResponse = { successful: [], unsuccessful: [] }
   $: page = $pageInfo.page
   $: fetchUsers(page, searchEmail)
   $: {
@@ -92,8 +97,7 @@
       admin: user.role === Constants.BudibaseRoles.Admin,
     }))
     try {
-      const res = await users.invite(payload)
+      inviteUsersResponse = await users.invite(payload)
-      notifications.success(res.message)
       inviteConfirmationModal.show()
     } catch (error) {
       notifications.error("Error inviting user")
@@ -116,8 +120,9 @@
       newUsers.push(user)
     }
 
-    if (!newUsers.length)
+    if (!newUsers.length) {
       notifications.info("Duplicated! There is no new users to add.")
+    }
     return { ...userData, users: newUsers }
   }
 
@@ -139,12 +144,14 @@
     userData = await removingDuplicities({ groups, users })
     if (!userData.users.length) return
 
-    return createUser()
+    return createUsers()
   }
 
-  async function createUser() {
+  async function createUsers() {
     try {
-      await users.create(await removingDuplicities(userData))
+      createUsersResponse = await users.create(
+        await removingDuplicities(userData)
+      )
       notifications.success("Successfully created user")
       await groups.actions.init()
       passwordModal.show()
@@ -157,7 +164,7 @@
     if (onboardingType === "emailOnboarding") {
       createUserFlow()
     } else {
-      await createUser()
+      await createUsers()
     }
   }
 
@@ -176,8 +183,15 @@
         notifications.error("You cannot delete yourself")
         return
       }
-      await users.bulkDelete(ids)
+      deleteUsersResponse = await users.bulkDelete(ids)
-      notifications.success(`Successfully deleted ${selectedRows.length} rows`)
+      if (deleteUsersResponse.unsuccessful?.length) {
+        deletionFailureModal.show()
+      } else {
+        notifications.success(
+          `Successfully deleted ${selectedRows.length} users`
+        )
+      }
+
       selectedRows = []
       await fetchUsers(page, searchEmail)
     } catch (error) {
@@ -267,16 +281,7 @@
 </Modal>
 
 <Modal bind:this={inviteConfirmationModal}>
-  <ModalContent
+  <InvitedModal {inviteUsersResponse} />
-    showCancelButton={false}
-    title="Invites sent!"
-    confirmText="Done"
-  >
-    <Body size="S"
-      >Your users should now recieve an email invite to get access to their
-      Budibase account</Body
-    ></ModalContent
-  >
 </Modal>
 
 <Modal bind:this={onboardingTypeModal}>
@@ -284,7 +289,11 @@
 </Modal>
 
 <Modal bind:this={passwordModal}>
-  <PasswordModal userData={userData.users} />
+  <PasswordModal {createUsersResponse} userData={userData.users} />
+</Modal>
+
+<Modal bind:this={deletionFailureModal}>
+  <DeletionFailureModal {deleteUsersResponse} />
 </Modal>
 
 <Modal bind:this={importUsersModal}>

packages/builder/src/stores/portal/users.js

@@ -63,10 +63,14 @@ export function createUsersStore() {
 
       return body
     })
-    await API.createUsers({ users: mappedUsers, groups: data.groups })
+    const response = await API.createUsers({
+      users: mappedUsers,
+      groups: data.groups,
+    })
 
     // re-search from first page
     await search()
+    return response
   }
 
   async function del(id) {
@@ -79,7 +83,7 @@ export function createUsersStore() {
   }
 
   async function bulkDelete(userIds) {
-    await API.deleteUsers(userIds)
+    return API.deleteUsers(userIds)
   }
 
   async function save(user) {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.2.44-alpha.1",
+  "version": "1.3.4-alpha.1",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -26,7 +26,7 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "1.2.41-alpha.5",
+    "@budibase/backend-core": "1.3.4-alpha.1",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",

packages/client/manifest.json

@@ -8,7 +8,8 @@
     "devicePreview": true,
     "messagePassing": true,
     "rowSelection": true,
-    "continueIfAction": true
+    "continueIfAction": true,
+    "showNotificationAction": true
   },
   "layout": {
     "name": "Layout",
@@ -543,7 +544,8 @@
         "values": [
           {
             "label": "Row Index",
-            "key": "index"
+            "key": "index",
+            "type": "number"
           }
         ]
       }
@@ -2313,19 +2315,23 @@
         "values": [
           {
             "label": "Value",
-            "key": "__value"
+            "key": "__value",
+            "type": "object"
           },
           {
             "label": "Valid",
-            "key": "__valid"
+            "key": "__valid",
+            "type": "boolean"
           },
           {
             "label": "Current Step",
-            "key": "__currentStep"
+            "key": "__currentStep",
+            "type": "number"
           },
           {
             "label": "Current Step Valid",
-            "key": "__currentStepValid"
+            "key": "__currentStepValid",
+            "type": "boolean"
           }
         ]
       },
@@ -3549,23 +3555,28 @@
       "values": [
         {
           "label": "Rows",
-          "key": "rows"
+          "key": "rows",
+          "type": "array"
         },
         {
           "label": "Extra Info",
-          "key": "info"
+          "key": "info",
+          "type": "string"
         },
         {
           "label": "Rows Length",
-          "key": "rowsLength"
+          "key": "rowsLength",
+          "type": "number"
         },
         {
           "label": "Schema",
-          "key": "schema"
+          "key": "schema",
+          "type": "object"
         },
         {
           "label": "Page Number",
-          "key": "pageNumber"
+          "key": "pageNumber",
+          "type": "number"
         }
       ]
     }
@@ -4327,23 +4338,28 @@
         "values": [
           {
             "label": "Rows",
-            "key": "rows"
+            "key": "rows",
+            "type": "array"
           },
           {
             "label": "Extra Info",
-            "key": "info"
+            "key": "info",
+            "type": "string"
           },
           {
             "label": "Rows Length",
-            "key": "rowsLength"
+            "key": "rowsLength",
+            "type": "number"
           },
           {
             "label": "Schema",
-            "key": "schema"
+            "key": "schema",
+            "type": "object"
           },
           {
             "label": "Page Number",
-            "key": "pageNumber"
+            "key": "pageNumber",
+            "type": "number"
           }
         ]
       },
@@ -4353,7 +4369,8 @@
         "values": [
           {
             "label": "Row Index",
-            "key": "index"
+            "key": "index",
+            "type": "number"
           }
         ]
       },

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.2.44-alpha.1",
+  "version": "1.3.4-alpha.1",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "1.2.44-alpha.1",
+    "@budibase/bbui": "1.3.4-alpha.1",
-    "@budibase/frontend-core": "1.2.44-alpha.1",
+    "@budibase/frontend-core": "1.3.4-alpha.1",
-    "@budibase/string-templates": "1.2.44-alpha.1",
+    "@budibase/string-templates": "1.3.4-alpha.1",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/client/src/components/Screen.svelte

@@ -3,7 +3,7 @@
   import Component from "./Component.svelte"
   import Provider from "./context/Provider.svelte"
   import { onMount, getContext } from "svelte"
-  import { enrichButtonActions } from "utils/buttonActions.js"
+  import { enrichButtonActions } from "../utils/buttonActions.js"
 
   export let params = {}
 
@@ -29,7 +29,9 @@
         ...$context,
         url: params,
       })
-      actions()
+      if (actions != null) {
+        actions()
+      }
     }
   })
 </script>

packages/client/src/components/app/forms/AttachmentField.svelte

@@ -59,8 +59,8 @@
   }
 
   const handleChange = e => {
-    fieldApi.setValue(e.detail)
+    const changed = fieldApi.setValue(e.detail)
-    if (onChange) {
+    if (onChange && changed) {
       onChange({ value: e.detail })
     }
   }

packages/client/src/components/app/forms/BooleanField.svelte

@@ -28,8 +28,8 @@
   }
 
   const handleChange = e => {
-    fieldApi.setValue(e.detail)
+    const changed = fieldApi.setValue(e.detail)
-    if (onChange) {
+    if (onChange && changed) {
       onChange({ value: e.detail })
     }
   }

packages/client/src/components/app/forms/DateTimeField.svelte

@@ -18,8 +18,8 @@
   let fieldApi
 
   const handleChange = e => {
-    fieldApi.setValue(e.detail)
+    const changed = fieldApi.setValue(e.detail)
-    if (onChange) {
+    if (onChange && changed) {
       onChange({ value: e.detail })
     }
   }

packages/client/src/components/app/forms/InnerForm.svelte

@@ -268,7 +268,7 @@
 
       // Skip if the value is the same
       if (!skipCheck && fieldState.value === value) {
-        return
+        return false
       }
 
       // Update field state

packages/client/src/components/app/forms/JSONField.svelte

@@ -37,8 +37,8 @@
 
   const handleChange = e => {
     const value = parseValue(e.detail)
-    fieldApi.setValue(value)
+    const changed = fieldApi.setValue(value)
-    if (onChange) {
+    if (onChange && changed) {
       onChange({ value })
     }
   }

packages/client/src/components/app/forms/LongFormField.svelte

@@ -47,8 +47,8 @@
   }
 
   const handleChange = e => {
-    fieldApi.setValue(e.detail)
+    const changed = fieldApi.setValue(e.detail)
-    if (onChange) {
+    if (onChange && changed) {
       onChange({ value: e.detail })
     }
   }

packages/client/src/components/app/forms/MultiFieldSelect.svelte

@@ -44,8 +44,8 @@
   }
 
   const handleChange = e => {
-    fieldApi.setValue(e.detail)
+    const changed = fieldApi.setValue(e.detail)
-    if (onChange) {
+    if (onChange && changed) {
       onChange({ value: e.detail })
     }
   }

packages/client/src/components/app/forms/OptionsField.svelte

@@ -34,8 +34,8 @@
   )
 
   const handleChange = e => {
-    fieldApi.setValue(e.detail)
+    const changed = fieldApi.setValue(e.detail)
-    if (onChange) {
+    if (onChange && changed) {
       onChange({ value: e.detail })
     }
   }
@@ -77,6 +77,7 @@
         {direction}
         on:change={handleChange}
         getOptionLabel={flatOptions ? x => x : x => x.label}
+        getOptionTitle={flatOptions ? x => x : x => x.label}
         getOptionValue={flatOptions ? x => x : x => x.value}
       />
     {/if}

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -84,8 +84,8 @@
   }
 
   const handleChange = value => {
-    fieldApi.setValue(value)
+    const changed = fieldApi.setValue(value)
-    if (onChange) {
+    if (onChange && changed) {
       onChange({ value })
     }
   }

packages/client/src/components/app/forms/S3Upload.svelte

@@ -90,8 +90,8 @@
   }
 
   const handleChange = e => {
-    fieldApi.setValue(e.detail)
+    const changed = fieldApi.setValue(e.detail)
-    if (onChange) {
+    if (onChange && changed) {
       onChange({ value: e.detail })
     }
   }

packages/client/src/components/app/forms/StringField.svelte

@@ -16,8 +16,8 @@
   let fieldApi
 
   const handleChange = e => {
-    fieldApi.setValue(e.detail)
+    const changed = fieldApi.setValue(e.detail)
-    if (onChange) {
+    if (onChange && changed) {
       onChange({ value: e.detail })
     }
   }
@@ -29,7 +29,6 @@
   {disabled}
   {validation}
   {defaultValue}
-  {onChange}
   type={type === "number" ? "number" : "string"}
   bind:fieldState
   bind:fieldApi

packages/client/src/stores/notification.js

@@ -62,10 +62,14 @@ const createNotificationStore = () => {
     subscribe: store.subscribe,
     actions: {
       send,
-      info: msg => send(msg, "info", "Info"),
+      info: (msg, autoDismiss) =>
-      success: msg => send(msg, "success", "CheckmarkCircle"),
+        send(msg, "info", "Info", autoDismiss ?? true),
-      warning: msg => send(msg, "warning", "Alert"),
+      success: (msg, autoDismiss) =>
-      error: msg => send(msg, "error", "Alert", false),
+        send(msg, "success", "CheckmarkCircle", autoDismiss ?? true),
+      warning: (msg, autoDismiss) =>
+        send(msg, "warning", "Alert", autoDismiss ?? true),
+      error: (msg, autoDismiss) =>
+        send(msg, "error", "Alert", autoDismiss ?? false),
       blockNotifications,
       dismiss,
     },

packages/client/src/utils/buttonActions.js

@@ -300,6 +300,14 @@ const continueIfHandler = action => {
   }
 }
 
+const showNotificationHandler = action => {
+  const { message, type, autoDismiss } = action.parameters
+  if (!message || !type) {
+    return
+  }
+  notificationStore.actions[type]?.(message, autoDismiss)
+}
+
 const handlerMap = {
   ["Save Row"]: saveRowHandler,
   ["Duplicate Row"]: duplicateRowHandler,
@@ -318,6 +326,7 @@ const handlerMap = {
   ["Upload File to S3"]: s3UploadHandler,
   ["Export Data"]: exportDataHandler,
   ["Continue if / Stop if"]: continueIfHandler,
+  ["Show Notification"]: showNotificationHandler,
 }
 
 const confirmTextMap = {

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "1.2.44-alpha.1",
+  "version": "1.3.4-alpha.1",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "1.2.44-alpha.1",
+    "@budibase/bbui": "1.3.4-alpha.1",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/frontend-core/src/utils/lucene.js

@@ -72,7 +72,7 @@ const cleanupQuery = query => {
       continue
     }
     for (let [key, value] of Object.entries(query[filterField])) {
-      if (!value || value === "") {
+      if (value == null || value === "") {
         delete query[filterField][key]
       }
     }
@@ -186,7 +186,7 @@ export const runLuceneQuery = (docs, query) => {
     return docs
   }
 
-  // make query consistent first
+  // Make query consistent first
   query = cleanupQuery(query)
 
   // Iterates over a set of filters and evaluates a fail function against a doc
@@ -218,7 +218,12 @@ export const runLuceneQuery = (docs, query) => {
 
   // Process a range match
   const rangeMatch = match("range", (docValue, testValue) => {
-    return !docValue || docValue < testValue.low || docValue > testValue.high
+    return (
+      docValue == null ||
+      docValue === "" ||
+      docValue < testValue.low ||
+      docValue > testValue.high
+    )
   })
 
   // Process an equal match (fails if the value is different)

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.2.44-alpha.1",
+  "version": "1.3.4-alpha.1",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "1.2.44-alpha.1",
+    "@budibase/backend-core": "1.3.4-alpha.1",
-    "@budibase/client": "1.2.44-alpha.1",
+    "@budibase/client": "1.3.4-alpha.1",
-    "@budibase/pro": "1.2.44-alpha.1",
+    "@budibase/pro": "1.3.4-alpha.1",
-    "@budibase/string-templates": "1.2.44-alpha.1",
+    "@budibase/string-templates": "1.3.4-alpha.1",
-    "@budibase/types": "1.2.44-alpha.1",
+    "@budibase/types": "1.3.4-alpha.1",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -99,6 +99,7 @@
     "curlconverter": "3.21.0",
     "dotenv": "8.2.0",
     "download": "8.0.0",
+    "elastic-apm-node": "3.38.0",
     "fix-path": "3.0.0",
     "form-data": "4.0.0",
     "fs-extra": "8.1.0",

packages/server/src/api/controllers/application.ts

@@ -47,7 +47,14 @@ import { checkAppMetadata } from "../../automations/logging"
 import { getUniqueRows } from "../../utilities/usageQuota/rows"
 import { quotas } from "@budibase/pro"
 import { errors, events, migrations } from "@budibase/backend-core"
-import { App, MigrationType } from "@budibase/types"
+import {
+  App,
+  Layout,
+  Screen,
+  MigrationType,
+  AppNavigation,
+} from "@budibase/types"
+import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -243,27 +250,19 @@ const performAppCreate = async (ctx: any) => {
   }
   const instance = await createInstance(instanceConfig)
   const appId = instance._id
-
   const db = context.getAppDB()
-  let _rev
+
-  try {
+  let newApplication: App = {
-    // if template there will be an existing doc
-    const existing = await db.get(DocumentType.APP_METADATA)
-    _rev = existing._rev
-  } catch (err) {
-    // nothing to do
-  }
-  const newApplication: App = {
     _id: DocumentType.APP_METADATA,
-    _rev,
+    _rev: undefined,
-    appId: instance._id,
+    appId,
     type: "app",
     version: packageJson.version,
     componentLibraries: ["@budibase/standard-components"],
     name: name,
     url: url,
-    template: ctx.request.body.template,
+    template: templateKey,
-    instance: instance,
+    instance,
     tenantId: getTenantId(),
     updatedAt: new Date().toISOString(),
     createdAt: new Date().toISOString(),
@@ -285,6 +284,36 @@ const performAppCreate = async (ctx: any) => {
       buttonBorderRadius: "16px",
     },
   }
+
+  // If we used a template or imported an app there will be an existing doc.
+  // Fetch and migrate some metadata from the existing app.
+  try {
+    const existing: App = await db.get(DocumentType.APP_METADATA)
+    const keys: (keyof App)[] = [
+      "_rev",
+      "navigation",
+      "theme",
+      "customTheme",
+      "icon",
+    ]
+    keys.forEach(key => {
+      if (existing[key]) {
+        // @ts-ignore
+        newApplication[key] = existing[key]
+      }
+    })
+
+    // Migrate navigation settings and screens if required
+    if (existing && !existing.navigation) {
+      const navigation = await migrateAppNavigation()
+      if (navigation) {
+        newApplication.navigation = navigation
+      }
+    }
+  } catch (err) {
+    // Nothing to do
+  }
+
   const response = await db.put(newApplication, { force: true })
   newApplication._rev = response.rev
 
@@ -567,3 +596,55 @@ const updateAppPackage = async (appPackage: any, appId: any) => {
     return newAppPackage
   })
 }
+
+const migrateAppNavigation = async () => {
+  const db = context.getAppDB()
+  const existing: App = await db.get(DocumentType.APP_METADATA)
+  const layouts: Layout[] = await getLayouts()
+  const screens: Screen[] = await getScreens()
+
+  // Migrate all screens, removing custom layouts
+  for (let screen of screens) {
+    if (!screen.layoutId) {
+      return
+    }
+    const layout = layouts.find(layout => layout._id === screen.layoutId)
+    screen.layoutId = undefined
+    screen.showNavigation = layout?.props.navigation !== "None"
+    screen.width = layout?.props.width || "Large"
+    await db.put(screen)
+  }
+
+  // Migrate layout navigation settings
+  const { name, customTheme } = existing
+  const layout = layouts?.find(
+    (layout: Layout) => layout._id === BASE_LAYOUT_PROP_IDS.PRIVATE
+  )
+  if (layout) {
+    let navigationSettings: any = {
+      navigation: "Top",
+      title: name,
+      navWidth: "Large",
+      navBackground:
+        customTheme?.navBackground || "var(--spectrum-global-color-gray-50)",
+      navTextColor:
+        customTheme?.navTextColor || "var(--spectrum-global-color-gray-800)",
+    }
+    if (layout) {
+      navigationSettings.hideLogo = layout.props.hideLogo
+      navigationSettings.hideTitle = layout.props.hideTitle
+      navigationSettings.title = layout.props.title || name
+      navigationSettings.logoUrl = layout.props.logoUrl
+      navigationSettings.links = layout.props.links
+      navigationSettings.navigation = layout.props.navigation || "Top"
+      navigationSettings.sticky = layout.props.sticky
+      navigationSettings.navWidth = layout.props.width || "Large"
+      if (navigationSettings.navigation === "None") {
+        navigationSettings.navigation = "Top"
+      }
+    }
+    return navigationSettings
+  } else {
+    return null
+  }
+}

packages/server/src/api/controllers/component.js

@@ -3,32 +3,36 @@ const { getComponentLibraryManifest } = require("../../utilities/fileSystem")
 const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.fetchAppComponentDefinitions = async function (ctx) {
-  const db = getAppDB()
+  try {
-  const app = await db.get(DocumentType.APP_METADATA)
+    const db = getAppDB()
+    const app = await db.get(DocumentType.APP_METADATA)
 
-  let componentManifests = await Promise.all(
+    let componentManifests = await Promise.all(
-    app.componentLibraries.map(async library => {
+      app.componentLibraries.map(async library => {
-      let manifest = await getComponentLibraryManifest(library)
+        let manifest = await getComponentLibraryManifest(library)
 
-      return {
+        return {
-        manifest,
+          manifest,
-        library,
+          library,
-      }
+        }
-    })
+      })
-  )
+    )
-  const definitions = {}
+    const definitions = {}
-  for (let { manifest, library } of componentManifests) {
+    for (let { manifest, library } of componentManifests) {
-    for (let key of Object.keys(manifest)) {
+      for (let key of Object.keys(manifest)) {
-      if (key === "features") {
+        if (key === "features") {
-        definitions[key] = manifest[key]
+          definitions[key] = manifest[key]
-      } else {
+        } else {
-        const fullComponentName = `${library}/${key}`.toLowerCase()
+          const fullComponentName = `${library}/${key}`.toLowerCase()
-        definitions[fullComponentName] = {
+          definitions[fullComponentName] = {
-          component: fullComponentName,
+            component: fullComponentName,
-          ...manifest[key],
+            ...manifest[key],
+          }
         }
       }
     }
+    ctx.body = definitions
+  } catch (err) {
+    console.error(`component-definitions=failed`, err)
   }
-  ctx.body = definitions
 }

packages/server/src/api/controllers/row/internal.js

@@ -375,6 +375,7 @@ exports.exportRows = async ctx => {
   const table = await db.get(ctx.params.tableId)
   const rowIds = ctx.request.body.rows
   let format = ctx.query.format
+  const { columns } = ctx.request.body
   let response = (
     await db.allDocs({
       include_docs: true,
@@ -382,7 +383,20 @@ exports.exportRows = async ctx => {
     })
   ).rows.map(row => row.doc)
 
-  let rows = await outputProcessing(table, response)
+  let result = await outputProcessing(table, response)
+  let rows = []
+
+  // Filter data to only specified columns if required
+  if (columns && columns.length) {
+    for (let i = 0; i < result.length; i++) {
+      rows[i] = {}
+      for (let column of columns) {
+        rows[i][column] = result[i][column]
+      }
+    }
+  } else {
+    rows = result
+  }
 
   let headers = Object.keys(rows[0])
   const exporter = exporters[format]

packages/server/src/api/routes/tests/application.spec.js

@@ -17,6 +17,7 @@ const {
   checkBuilderEndpoint,
 } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
+const { basicScreen, basicLayout } = setup.structures
 const { AppStatus } = require("../../../db/utils")
 const { events } = require("@budibase/backend-core")
 
@@ -81,6 +82,31 @@ describe("/applications", () => {
         body: { name: "My App" },
       })
     })
+
+    it("migrates navigation settings from old apps", async () => {
+      const res = await request
+        .post("/api/applications")
+        .field("name", "Old App")
+        .field("useTemplate", "true")
+        .set(config.defaultHeaders())
+        .attach("templateFile", "src/api/routes/tests/data/old-app.txt")
+        .expect("Content-Type", /json/)
+        .expect(200)
+      expect(res.body._id).toBeDefined()
+      expect(res.body.navigation).toBeDefined()
+      expect(res.body.navigation.hideLogo).toBe(true)
+      expect(res.body.navigation.title).toBe("Custom Title")
+      expect(res.body.navigation.hideLogo).toBe(true)
+      expect(res.body.navigation.navigation).toBe("Left")
+      expect(res.body.navigation.navBackground).toBe(
+        "var(--spectrum-global-color-blue-600)"
+      )
+      expect(res.body.navigation.navTextColor).toBe(
+        "var(--spectrum-global-color-gray-50)"
+      )
+      expect(events.app.created).toBeCalledTimes(1)
+      expect(events.app.fileImported).toBeCalledTimes(1)
+    })
   })
 
   describe("fetch", () => {

packages/server/src/api/routes/tests/row.spec.js

@@ -3,7 +3,12 @@ const setup = require("./utilities")
 const { basicRow } = setup.structures
 const { doInAppContext } = require("@budibase/backend-core/context")
 const { doInTenant } = require("@budibase/backend-core/tenancy")
-const { quotas, QuotaUsageType, StaticQuotaName, MonthlyQuotaName } = require("@budibase/pro")
+const {
+  quotas,
+  QuotaUsageType,
+  StaticQuotaName,
+  MonthlyQuotaName,
+} = require("@budibase/pro")
 
 describe("/rows", () => {
   let request = setup.getRequest()
@@ -23,23 +28,30 @@ describe("/rows", () => {
     await request
       .get(`/api/${table._id}/rows/${id}`)
       .set(config.defaultHeaders())
-      .expect('Content-Type', /json/)
+      .expect("Content-Type", /json/)
       .expect(status)
 
   const getRowUsage = async () => {
-    return config.doInContext(null, () => quotas.getCurrentUsageValue(QuotaUsageType.STATIC, StaticQuotaName.ROWS))
+    return config.doInContext(null, () =>
+      quotas.getCurrentUsageValue(QuotaUsageType.STATIC, StaticQuotaName.ROWS)
+    )
   }
 
   const getQueryUsage = async () => {
-    return config.doInContext(null, () => quotas.getCurrentUsageValue(QuotaUsageType.MONTHLY, MonthlyQuotaName.QUERIES))
+    return config.doInContext(null, () =>
+      quotas.getCurrentUsageValue(
+        QuotaUsageType.MONTHLY,
+        MonthlyQuotaName.QUERIES
+      )
+    )
   }
 
-  const assertRowUsage = async (expected) => {
+  const assertRowUsage = async expected => {
     const usage = await getRowUsage()
     expect(usage).toBe(expected)
   }
 
-  const assertQueryUsage = async (expected) => {
+  const assertQueryUsage = async expected => {
     const usage = await getQueryUsage()
     expect(usage).toBe(expected)
   }
@@ -76,10 +88,12 @@ describe("/rows", () => {
           name: "Updated Name",
         })
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
-      expect(res.res.statusMessage).toEqual(`${table.name} updated successfully.`)
+      expect(res.res.statusMessage).toEqual(
+        `${table.name} updated successfully.`
+      )
       expect(res.body.name).toEqual("Updated Name")
       // await assertRowUsage(rowUsage)
       // await assertQueryUsage(queryUsage + 1)
@@ -92,7 +106,7 @@ describe("/rows", () => {
       const res = await request
         .get(`/api/${table._id}/rows/${existing._id}`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       expect(res.body).toEqual({
@@ -110,7 +124,7 @@ describe("/rows", () => {
       const newRow = {
         tableId: table._id,
         name: "Second Contact",
-        status: "new"
+        status: "new",
       }
       await config.createRow()
       await config.createRow(newRow)
@@ -119,7 +133,7 @@ describe("/rows", () => {
       const res = await request
         .get(`/api/${table._id}/rows`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       expect(res.body.length).toBe(2)
@@ -135,17 +149,36 @@ describe("/rows", () => {
       await request
         .get(`/api/${table._id}/rows/not-a-valid-id`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(404)
       await assertQueryUsage(queryUsage) // no change
     })
 
     it("row values are coerced", async () => {
-      const str = {type:"string", constraints: { type: "string", presence: false }}
+      const str = {
-      const attachment = {type:"attachment", constraints: { type: "array", presence: false }}
+        type: "string",
-      const bool = {type:"boolean", constraints: { type: "boolean", presence: false }}
+        constraints: { type: "string", presence: false },
-      const number = {type:"number", constraints: { type: "number", presence: false }}
+      }
-      const datetime = {type:"datetime", constraints: { type: "string", presence: false, datetime: {earliest:"", latest: ""}  }}
+      const attachment = {
+        type: "attachment",
+        constraints: { type: "array", presence: false },
+      }
+      const bool = {
+        type: "boolean",
+        constraints: { type: "boolean", presence: false },
+      }
+      const number = {
+        type: "number",
+        constraints: { type: "number", presence: false },
+      }
+      const datetime = {
+        type: "datetime",
+        constraints: {
+          type: "string",
+          presence: false,
+          datetime: { earliest: "", latest: "" },
+        },
+      }
 
       table = await config.createTable({
         name: "TestTable2",
@@ -171,9 +204,9 @@ describe("/rows", () => {
           boolUndefined: bool,
           boolString: bool,
           boolBool: bool,
-          attachmentNull : attachment,
+          attachmentNull: attachment,
-          attachmentUndefined : attachment,
+          attachmentUndefined: attachment,
-          attachmentEmpty : attachment,
+          attachmentEmpty: attachment,
         },
       })
 
@@ -198,9 +231,9 @@ describe("/rows", () => {
         boolString: "true",
         boolBool: true,
         tableId: table._id,
-        attachmentNull : null,
+        attachmentNull: null,
-        attachmentUndefined : undefined,
+        attachmentUndefined: undefined,
-        attachmentEmpty : "",
+        attachmentEmpty: "",
       }
 
       const id = (await config.createRow(row))._id
@@ -218,7 +251,9 @@ describe("/rows", () => {
       expect(saved.datetimeEmptyString).toBe(null)
       expect(saved.datetimeNull).toBe(null)
       expect(saved.datetimeUndefined).toBe(undefined)
-      expect(saved.datetimeString).toBe(new Date(row.datetimeString).toISOString())
+      expect(saved.datetimeString).toBe(
+        new Date(row.datetimeString).toISOString()
+      )
       expect(saved.datetimeDate).toBe(row.datetimeDate.toISOString())
       expect(saved.boolNull).toBe(null)
       expect(saved.boolEmpty).toBe(null)
@@ -247,10 +282,12 @@ describe("/rows", () => {
           name: "Updated Name",
         })
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
-      
+
-      expect(res.res.statusMessage).toEqual(`${table.name} updated successfully.`)
+      expect(res.res.statusMessage).toEqual(
+        `${table.name} updated successfully.`
+      )
       expect(res.body.name).toEqual("Updated Name")
       expect(res.body.description).toEqual(existing.description)
 
@@ -292,16 +329,14 @@ describe("/rows", () => {
       const res = await request
         .delete(`/api/${table._id}/rows`)
         .send({
-          rows: [
+          rows: [createdRow],
-            createdRow
-          ]
         })
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body[0]._id).toEqual(createdRow._id)
-      await assertRowUsage(rowUsage -1)
+      await assertRowUsage(rowUsage - 1)
-      await assertQueryUsage(queryUsage +1)
+      await assertQueryUsage(queryUsage + 1)
     })
   })
 
@@ -314,9 +349,9 @@ describe("/rows", () => {
         .post(`/api/${table._id}/rows/validate`)
         .send({ name: "ivan" })
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
-      
+
       expect(res.body.valid).toBe(true)
       expect(Object.keys(res.body.errors)).toEqual([])
       await assertRowUsage(rowUsage)
@@ -331,9 +366,9 @@ describe("/rows", () => {
         .post(`/api/${table._id}/rows/validate`)
         .send({ name: 1 })
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
-      
+
       expect(res.body.valid).toBe(false)
       expect(Object.keys(res.body.errors)).toEqual(["name"])
       await assertRowUsage(rowUsage)
@@ -351,19 +386,16 @@ describe("/rows", () => {
       const res = await request
         .delete(`/api/${table._id}/rows`)
         .send({
-          rows: [
+          rows: [row1, row2],
-            row1,
-            row2,
-          ]
         })
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       expect(res.body.length).toEqual(2)
       await loadRow(row1._id, 404)
       await assertRowUsage(rowUsage - 2)
-      await assertQueryUsage(queryUsage +1)
+      await assertQueryUsage(queryUsage + 1)
     })
   })
 
@@ -376,12 +408,12 @@ describe("/rows", () => {
       const res = await request
         .get(`/api/views/${table._id}`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body.length).toEqual(1)
       expect(res.body[0]._id).toEqual(row._id)
       await assertRowUsage(rowUsage)
-      await assertQueryUsage(queryUsage +1)
+      await assertQueryUsage(queryUsage + 1)
     })
 
     it("should throw an error if view doesn't exist", async () => {
@@ -406,7 +438,7 @@ describe("/rows", () => {
       const res = await request
         .get(`/api/views/${view.name}`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body.length).toEqual(1)
       expect(res.body[0]._id).toEqual(row._id)
@@ -418,21 +450,24 @@ describe("/rows", () => {
 
   describe("fetchEnrichedRows", () => {
     it("should allow enriching some linked rows", async () => {
-      const { table, firstRow, secondRow } = await doInTenant(setup.structures.TENANT_ID, async () => {
+      const { table, firstRow, secondRow } = await doInTenant(
-        const table = await config.createLinkedTable()
+        setup.structures.TENANT_ID,
-        const firstRow = await config.createRow({
+        async () => {
-          name: "Test Contact",
+          const table = await config.createLinkedTable()
-          description: "original description",
+          const firstRow = await config.createRow({
-          tableId: table._id
+            name: "Test Contact",
-        })
+            description: "original description",
-        const secondRow = await config.createRow({
+            tableId: table._id,
-          name: "Test 2",
+          })
-          description: "og desc",
+          const secondRow = await config.createRow({
-          link: [{_id: firstRow._id}],
+            name: "Test 2",
-          tableId: table._id,
+            description: "og desc",
-        })
+            link: [{ _id: firstRow._id }],
-        return { table, firstRow, secondRow }
+            tableId: table._id,
-      })
+          })
+          return { table, firstRow, secondRow }
+        }
+      )
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -440,7 +475,7 @@ describe("/rows", () => {
       const resBasic = await request
         .get(`/api/${table._id}/rows/${secondRow._id}`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
       expect(resBasic.body.link[0]._id).toBe(firstRow._id)
       expect(resBasic.body.link[0].primaryDisplay).toBe("Test Contact")
@@ -449,14 +484,14 @@ describe("/rows", () => {
       const resEnriched = await request
         .get(`/api/${table._id}/${secondRow._id}/enrich`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
       expect(resEnriched.body.link.length).toBe(1)
       expect(resEnriched.body.link[0]._id).toBe(firstRow._id)
       expect(resEnriched.body.link[0].name).toBe("Test Contact")
       expect(resEnriched.body.link[0].description).toBe("original description")
       await assertRowUsage(rowUsage)
-      await assertQueryUsage(queryUsage +2)
+      await assertQueryUsage(queryUsage + 2)
     })
   })
 
@@ -466,9 +501,11 @@ describe("/rows", () => {
       const row = await config.createRow({
         name: "test",
         description: "test",
-        attachment: [{
+        attachment: [
-          key: `${config.getAppId()}/attachments/test/thing.csv`,
+          {
-        }],
+            key: `${config.getAppId()}/attachments/test/thing.csv`,
+          },
+        ],
         tableId: table._id,
       })
       // the environment needs configured for this
@@ -482,4 +519,49 @@ describe("/rows", () => {
       })
     })
   })
+
+  describe("exportData", () => {
+    it("should allow exporting all columns", async () => {
+      const existing = await config.createRow()
+      const res = await request
+        .post(`/api/${table._id}/rows/exportRows?format=json`)
+        .set(config.defaultHeaders())
+        .send({
+          rows: [existing._id],
+        })
+        .expect("Content-Type", /json/)
+        .expect(200)
+      const results = JSON.parse(res.text)
+      expect(results.length).toEqual(1)
+      const row = results[0]
+
+      // Ensure all original columns were exported
+      expect(Object.keys(row).length).toBeGreaterThanOrEqual(
+        Object.keys(existing).length
+      )
+      Object.keys(existing).forEach(key => {
+        expect(row[key]).toEqual(existing[key])
+      })
+    })
+
+    it("should allow exporting only certain columns", async () => {
+      const existing = await config.createRow()
+      const res = await request
+        .post(`/api/${table._id}/rows/exportRows?format=json`)
+        .set(config.defaultHeaders())
+        .send({
+          rows: [existing._id],
+          columns: ["_id"],
+        })
+        .expect("Content-Type", /json/)
+        .expect(200)
+      const results = JSON.parse(res.text)
+      expect(results.length).toEqual(1)
+      const row = results[0]
+
+      // Ensure only the _id column was exported
+      expect(Object.keys(row).length).toEqual(1)
+      expect(row._id).toEqual(existing._id)
+    })
+  })
 })