Merge remote-tracking branch 'origin/develop' into feature/form-block-ux-updates

.github/workflows/budibase_ci.yml

@@ -154,7 +154,7 @@ jobs:
           node-version: 14.x
           cache: "yarn"
       - run: yarn
-      - run: yarn build
+      - run: yarn build --projects=@budibase/server,@budibase/worker,@budibase/client
       - name: Run tests
         run: |
           cd qa-core

.gitignore

@@ -101,8 +101,6 @@ packages/builder/cypress.env.json
 packages/builder/cypress/reports
 stats.html
 
-# TypeScript cache
-*.tsbuildinfo
 
 # plugins
 budibase-component

charts/budibase/templates/app-service-deployment.yaml

@@ -201,25 +201,24 @@ spec:
 
         image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
         imagePullPolicy: Always
+        {{- if .Values.services.apps.startupProbe }}
+        {{- with .Values.services.apps.startupProbe }}
+        startupProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.services.apps.livenessProbe }}
+        {{- with .Values.services.apps.livenessProbe }}
         livenessProbe:
-          httpGet:
+            {{- toYaml . | nindent 10 }}
-            path: /health
+        {{- end }}
-            port: {{ .Values.services.apps.port }}
+        {{- end }}
-          initialDelaySeconds: 10
+        {{- if .Values.services.apps.readinessProbe }}
-          periodSeconds: 5
+        {{- with .Values.services.apps.readinessProbe }}
-          successThreshold: 1
-          failureThreshold: 3
-          timeoutSeconds: 3
         readinessProbe:
-          httpGet:
+            {{- toYaml . | nindent 10 }}
-            path: /health
+        {{- end }}
-            port: {{ .Values.services.apps.port }}
+        {{- end }}
-          initialDelaySeconds: 5
-          periodSeconds: 5
-          successThreshold: 1
-          failureThreshold: 3
-          timeoutSeconds: 3
-
         name: bbapps
         ports:
         - containerPort: {{ .Values.services.apps.port }}

charts/budibase/templates/proxy-service-deployment.yaml

@@ -40,24 +40,24 @@ spec:
       - image: budibase/proxy:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
         imagePullPolicy: Always
         name: proxy-service
+        {{- if .Values.services.proxy.startupProbe }}
+        {{- with .Values.services.proxy.startupProbe }}
+        startupProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.services.proxy.livenessProbe }}
+        {{- with .Values.services.proxy.livenessProbe }}
         livenessProbe:
-          httpGet:
+            {{- toYaml . | nindent 10 }}
-            path: /health
+        {{- end }}
-            port: {{ .Values.services.proxy.port }}
+        {{- end }}
-          initialDelaySeconds: 0
+        {{- if .Values.services.proxy.readinessProbe }}
-          periodSeconds: 5
+        {{- with .Values.services.proxy.readinessProbe }}
-          successThreshold: 1
-          failureThreshold: 2
-          timeoutSeconds: 3
         readinessProbe:
-          httpGet:
+            {{- toYaml . | nindent 10 }}
-            path: /health
+        {{- end }}
-            port: {{ .Values.services.proxy.port }}
+        {{- end }}
-          initialDelaySeconds: 0
-          periodSeconds: 5
-          successThreshold: 1
-          failureThreshold: 2
-          timeoutSeconds: 3
         ports:
         - containerPort: {{ .Values.services.proxy.port }}
         env:

charts/budibase/templates/worker-service-deployment.yaml

@@ -190,24 +190,24 @@ spec:
         {{ end }}
         image: budibase/worker:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
         imagePullPolicy: Always
+        {{- if .Values.services.worker.startupProbe }}
+        {{- with .Values.services.worker.startupProbe }}
+        startupProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.services.worker.livenessProbe }}
+        {{- with .Values.services.worker.livenessProbe }}
         livenessProbe:
-          httpGet:
+            {{- toYaml . | nindent 10 }}
-            path: /health
+        {{- end }}
-            port: {{ .Values.services.worker.port }}
+        {{- end }}
-          initialDelaySeconds: 10
+        {{- if .Values.services.worker.readinessProbe }}
-          periodSeconds: 5
+        {{- with .Values.services.worker.readinessProbe }}
-          successThreshold: 1
-          failureThreshold: 3
-          timeoutSeconds: 3
         readinessProbe:
-          httpGet:
+            {{- toYaml . | nindent 10 }}
-            path: /health
+        {{- end }}
-            port: {{ .Values.services.worker.port }}
+        {{- end }}
-          initialDelaySeconds: 5
-          periodSeconds: 5
-          successThreshold: 1
-          failureThreshold: 3
-          timeoutSeconds: 3
         name: bbworker
         ports:
         - containerPort: {{ .Values.services.worker.port }}

charts/budibase/values.yaml

@@ -119,15 +119,37 @@ services:
     port: 10000
     replicaCount: 1
     upstreams:
-      apps: 'http://app-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.apps.port }}'
+      apps: "http://app-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.apps.port }}"
-      worker: 'http://worker-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.worker.port }}'
+      worker: "http://worker-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.worker.port }}"
-      minio: 'http://minio-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.objectStore.port }}'
+      minio: "http://minio-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.objectStore.port }}"
-      couchdb: 'http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}'
+      couchdb: "http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}"
     resources: {}
-#    annotations:
+    startupProbe:
-#      co.elastic.logs/module: nginx
+      httpGet:
-#      co.elastic.logs/fileset.stdout: access
+        path: /health
-#      co.elastic.logs/fileset.stderr: error
+        port: 10000
+        scheme: HTTP
+      failureThreshold: 30
+      periodSeconds: 3
+    readinessProbe:
+      httpGet:
+        path: /health
+        port: 10000
+        scheme: HTTP
+      enabled: true
+      periodSeconds: 3
+      failureThreshold: 1
+    livenessProbe:
+      httpGet:
+        path: /health
+        port: 10000
+        scheme: HTTP
+      failureThreshold: 3
+      periodSeconds: 5
+  #    annotations:
+  #      co.elastic.logs/module: nginx
+  #      co.elastic.logs/fileset.stdout: access
+  #      co.elastic.logs/fileset.stderr: error
 
   apps:
     port: 4002
@@ -135,23 +157,67 @@ services:
     logLevel: info
     httpLogging: 1
     resources: {}
-#    nodeDebug: "" # set the value of NODE_DEBUG
+    startupProbe:
-#    annotations:
+      httpGet:
-#      co.elastic.logs/multiline.type: pattern
+        path: /health
-#      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+        port: 4002
-#      co.elastic.logs/multiline.negate: false
+        scheme: HTTP
-#      co.elastic.logs/multiline.match: after
+      failureThreshold: 30
+      periodSeconds: 3
+    readinessProbe:
+      httpGet:
+        path: /health
+        port: 4002
+        scheme: HTTP
+      enabled: true
+      periodSeconds: 3
+      failureThreshold: 1
+    livenessProbe:
+      httpGet:
+        path: /health
+        port: 4002
+        scheme: HTTP
+      failureThreshold: 3
+      periodSeconds: 5
+  #    nodeDebug: "" # set the value of NODE_DEBUG
+  #    annotations:
+  #      co.elastic.logs/multiline.type: pattern
+  #      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+  #      co.elastic.logs/multiline.negate: false
+  #      co.elastic.logs/multiline.match: after
   worker:
     port: 4003
     replicaCount: 1
     logLevel: info
     httpLogging: 1
     resources: {}
-#    annotations:
+    startupProbe:
-#      co.elastic.logs/multiline.type: pattern
+      httpGet:
-#      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+        path: /health
-#      co.elastic.logs/multiline.negate: false
+        port: 4003
-#      co.elastic.logs/multiline.match: after
+        scheme: HTTP
+      failureThreshold: 30
+      periodSeconds: 3
+    readinessProbe:
+      httpGet:
+        path: /health
+        port: 4003
+        scheme: HTTP
+      enabled: true
+      periodSeconds: 3
+      failureThreshold: 1
+    livenessProbe:
+      httpGet:
+        path: /health
+        port: 4003
+        scheme: HTTP
+      failureThreshold: 3
+      periodSeconds: 5
+  #    annotations:
+  #      co.elastic.logs/multiline.type: pattern
+  #      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+  #      co.elastic.logs/multiline.negate: false
+  #      co.elastic.logs/multiline.match: after
 
   couchdb:
     enabled: true

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.8.29-alpha.3",
+  "version": "2.8.29-alpha.17",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

nx.json

@@ -3,19 +3,10 @@
     "default": {
       "runner": "nx-cloud",
       "options": {
-        "cacheableOperations": ["build", "test"],
+        "cacheableOperations": ["build", "test", "check:types"],
         "accessToken": "MmM4OGYxNzItMDBlYy00ZmE3LTk4MTYtNmJhYWMyZjBjZTUyfHJlYWQ="
       }
     }
   },
-  "targetDefaults": {
+  "targetDefaults": {}
-    "dev:builder": {
-      "dependsOn": [
-        {
-          "projects": ["@budibase/string-templates"],
-          "target": "build"
-        }
-      ]
-    }
-  }
 }

package.json

@@ -36,7 +36,7 @@
     "bootstrap": "./scripts/link-dependencies.sh && echo '***BOOTSTRAP ONLY REQUIRED FOR USE WITH ACCOUNT PORTAL***'",
     "build": "yarn nx run-many -t=build",
     "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
-    "check:types": "lerna run check:types --skip-nx-cache",
+    "check:types": "lerna run check:types",
     "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap",
     "backend:build": "./scripts/scopeBackend.sh 'lerna run --stream build'",
     "build:sdk": "lerna run --stream build:sdk",
@@ -51,9 +51,9 @@
     "kill-builder": "kill-port 3000",
     "kill-server": "kill-port 4001 4002",
     "kill-all": "yarn run kill-builder && yarn run kill-server",
-    "dev": "yarn run kill-all && lerna run --stream --parallel dev:builder",
+    "dev": "yarn run kill-all && lerna run --stream dev:builder",
-    "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
+    "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
-    "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server",
+    "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
     "dev:docker": "yarn build:docker:pre && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
     "test": "lerna run --stream test --stream",

packages/backend-core/jest.config.ts

@@ -9,6 +9,7 @@ const baseConfig: Config.InitialProjectOptions = {
   },
   moduleNameMapper: {
     "@budibase/types": "<rootDir>/../types/src",
+    "@budibase/shared-core": ["<rootDir>/../shared-core/src"],
   },
 }
 

packages/backend-core/package.json

@@ -16,12 +16,14 @@
     "prepack": "cp package.json dist",
     "build": "tsc -p tsconfig.build.json",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
+    "check:types": "tsc -p tsconfig.json --noEmit --paths null",
     "test": "bash scripts/test.sh",
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
     "@budibase/nano": "10.1.2",
     "@budibase/pouchdb-replication-stream": "1.2.10",
+    "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",
@@ -86,20 +88,5 @@
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3"
-  },
+  }
-  "nx": {
-    "targets": {
-      "build": {
-        "dependsOn": [
-          {
-            "projects": [
-              "@budibase/types"
-            ],
-            "target": "build"
-          }
-        ]
-      }
-    }
-  },
-  "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }

packages/backend-core/src/cache/tests/writethrough.spec.ts

@@ -36,7 +36,7 @@ describe("writethrough", () => {
           _id: docId,
           value: 1,
         })
-        const output = await db.get(response.id)
+        const output = await db.get<any>(response.id)
         current = output
         expect(output.value).toBe(1)
       })
@@ -45,7 +45,7 @@ describe("writethrough", () => {
     it("second put shouldn't update DB", async () => {
       await config.doInTenant(async () => {
         const response = await writethrough.put({ ...current, value: 2 })
-        const output = await db.get(response.id)
+        const output = await db.get<any>(response.id)
         expect(current._rev).toBe(output._rev)
         expect(output.value).toBe(1)
       })
@@ -55,7 +55,7 @@ describe("writethrough", () => {
       await config.doInTenant(async () => {
         tk.freeze(Date.now() + DELAY + 1)
         const response = await writethrough.put({ ...current, value: 3 })
-        const output = await db.get(response.id)
+        const output = await db.get<any>(response.id)
         expect(response.rev).not.toBe(current._rev)
         expect(output.value).toBe(3)
 
@@ -79,7 +79,7 @@ describe("writethrough", () => {
           expect.arrayContaining([current._rev, current._rev, newRev])
         )
 
-        const output = await db.get(current._id)
+        const output = await db.get<any>(current._id)
         expect(output.value).toBe(4)
         expect(output._rev).toBe(newRev)
 
@@ -107,7 +107,7 @@ describe("writethrough", () => {
         })
         expect(res.ok).toBe(true)
 
-        const output = await db.get(id)
+        const output = await db.get<any>(id)
         expect(output.value).toBe(3)
         expect(output._rev).toBe(res.rev)
       })
@@ -130,8 +130,8 @@ describe("writethrough", () => {
         const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
         expect(resp1.rev).toBeDefined()
         expect(resp2.rev).toBeDefined()
-        expect((await db.get("db1")).value).toBe("first")
+        expect((await db.get<any>("db1")).value).toBe("first")
-        expect((await db2.get("db1")).value).toBe("second")
+        expect((await db2.get<any>("db1")).value).toBe("second")
       })
     })
   })

packages/backend-core/src/constants/db.ts

@@ -1,5 +1,5 @@
-export const SEPARATOR = "_"
+import { prefixed, DocumentType } from "@budibase/types"
-export const UNICODE_MAX = "\ufff0"
+export { SEPARATOR, UNICODE_MAX, DocumentType } from "@budibase/types"
 
 /**
  * Can be used to create a few different forms of querying a view.
@@ -14,8 +14,6 @@ export enum ViewName {
   USER_BY_APP = "by_app",
   USER_BY_EMAIL = "by_email2",
   BY_API_KEY = "by_api_key",
-  /** @deprecated - could be deleted */
-  USER_BY_BUILDERS = "by_builders",
   LINK = "by_link",
   ROUTING = "screen_routes",
   AUTOMATION_LOGS = "automation_logs",
@@ -36,42 +34,6 @@ export enum InternalTable {
   USER_METADATA = "ta_users",
 }
 
-export enum DocumentType {
-  USER = "us",
-  GROUP = "gr",
-  WORKSPACE = "workspace",
-  CONFIG = "config",
-  TEMPLATE = "template",
-  APP = "app",
-  DEV = "dev",
-  APP_DEV = "app_dev",
-  APP_METADATA = "app_metadata",
-  ROLE = "role",
-  MIGRATIONS = "migrations",
-  DEV_INFO = "devinfo",
-  AUTOMATION_LOG = "log_au",
-  ACCOUNT_METADATA = "acc_metadata",
-  PLUGIN = "plg",
-  DATASOURCE = "datasource",
-  DATASOURCE_PLUS = "datasource_plus",
-  APP_BACKUP = "backup",
-  TABLE = "ta",
-  ROW = "ro",
-  AUTOMATION = "au",
-  LINK = "li",
-  WEBHOOK = "wh",
-  INSTANCE = "inst",
-  LAYOUT = "layout",
-  SCREEN = "screen",
-  QUERY = "query",
-  DEPLOYMENTS = "deployments",
-  METADATA = "metadata",
-  MEM_VIEW = "view",
-  USER_FLAG = "flag",
-  AUTOMATION_METADATA = "meta_au",
-  AUDIT_LOG = "al",
-}
-
 export const StaticDatabases = {
   GLOBAL: {
     name: "global-db",
@@ -95,7 +57,7 @@ export const StaticDatabases = {
   },
 }
 
-export const APP_PREFIX = DocumentType.APP + SEPARATOR
+export const APP_PREFIX = prefixed(DocumentType.APP)
-export const APP_DEV = DocumentType.APP_DEV + SEPARATOR
+export const APP_DEV = prefixed(DocumentType.APP_DEV)
 export const APP_DEV_PREFIX = APP_DEV
 export const BUDIBASE_DATASOURCE_TYPE = "budibase"

packages/backend-core/src/db/views.ts

@@ -105,16 +105,6 @@ export const createApiKeyView = async () => {
   await createView(db, viewJs, ViewName.BY_API_KEY)
 }
 
-export const createUserBuildersView = async () => {
-  const db = getGlobalDB()
-  const viewJs = `function(doc) {
-    if (doc.builder && doc.builder.global === true) {
-      emit(doc._id, doc._id)
-    }
-  }`
-  await createView(db, viewJs, ViewName.USER_BY_BUILDERS)
-}
-
 export interface QueryViewOptions {
   arrayResponse?: boolean
 }
@@ -223,7 +213,6 @@ export const queryPlatformView = async <T>(
 const CreateFuncByName: any = {
   [ViewName.USER_BY_EMAIL]: createNewUserEmailView,
   [ViewName.BY_API_KEY]: createApiKeyView,
-  [ViewName.USER_BY_BUILDERS]: createUserBuildersView,
   [ViewName.USER_BY_APP]: createUserAppView,
 }
 

packages/backend-core/src/environment.ts

@@ -1,4 +1,5 @@
 import { existsSync, readFileSync } from "fs"
+import { ServiceType } from "@budibase/types"
 
 function isTest() {
   return isCypress() || isJest()
@@ -83,10 +84,20 @@ function getPackageJsonFields(): {
   }
 }
 
+function isWorker() {
+  return environment.SERVICE_TYPE === ServiceType.WORKER
+}
+
+function isApps() {
+  return environment.SERVICE_TYPE === ServiceType.APPS
+}
+
 const environment = {
   isTest,
   isJest,
   isDev,
+  isWorker,
+  isApps,
   isProd: () => {
     return !isDev()
   },
@@ -153,6 +164,7 @@ const environment = {
   SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS,
   DISABLE_JWT_WARNING: process.env.DISABLE_JWT_WARNING,
   BLACKLIST_IPS: process.env.BLACKLIST_IPS,
+  SERVICE_TYPE: "unknown",
   /**
    * Enable to allow an admin user to login using a password.
    * This can be useful to prevent lockout when configuring SSO.

packages/backend-core/src/events/identification.ts

@@ -21,6 +21,7 @@ import { processors } from "./processors"
 import { newid } from "../utils"
 import * as installation from "../installation"
 import * as configs from "../configs"
+import * as users from "../users"
 import { withCache, TTL, CacheKey } from "../cache/generic"
 
 /**
@@ -164,8 +165,8 @@ const identifyUser = async (
   const id = user._id as string
   const tenantId = await getEventTenantId(user.tenantId)
   const type = IdentityType.USER
-  let builder = user.builder?.global || false
+  let builder = users.hasBuilderPermissions(user)
-  let admin = user.admin?.global || false
+  let admin = users.hasAdminPermissions(user)
   let providerType
   if (isSSOUser(user)) {
     providerType = user.providerType

packages/backend-core/src/middleware/adminOnly.ts

@@ -1,10 +1,8 @@
-import { BBContext } from "@budibase/types"
+import { UserCtx } from "@budibase/types"
+import { isAdmin } from "../users"
 
-export default async (ctx: BBContext, next: any) => {
+export default async (ctx: UserCtx, next: any) => {
-  if (
+  if (!ctx.internal && !isAdmin(ctx.user)) {
-    !ctx.internal &&
-    (!ctx.user || !ctx.user.admin || !ctx.user.admin.global)
-  ) {
     ctx.throw(403, "Admin user only endpoint.")
   }
   return next()

packages/backend-core/src/middleware/builderOnly.ts

@@ -1,10 +1,19 @@
-import { BBContext } from "@budibase/types"
+import { UserCtx } from "@budibase/types"
+import { isBuilder, hasBuilderPermissions } from "../users"
+import { getAppId } from "../context"
+import env from "../environment"
 
-export default async (ctx: BBContext, next: any) => {
+export default async (ctx: UserCtx, next: any) => {
-  if (
+  const appId = getAppId()
-    !ctx.internal &&
+  const builderFn = env.isWorker()
-    (!ctx.user || !ctx.user.builder || !ctx.user.builder.global)
+    ? hasBuilderPermissions
-  ) {
+    : env.isApps()
+    ? isBuilder
+    : undefined
+  if (!builderFn) {
+    throw new Error("Service name unknown - middleware inactive.")
+  }
+  if (!ctx.internal && !builderFn(ctx.user, appId)) {
     ctx.throw(403, "Builder user only endpoint.")
   }
   return next()

packages/backend-core/src/middleware/builderOrAdmin.ts

@@ -1,12 +1,20 @@
-import { BBContext } from "@budibase/types"
+import { UserCtx } from "@budibase/types"
+import { isBuilder, isAdmin, hasBuilderPermissions } from "../users"
+import { getAppId } from "../context"
+import env from "../environment"
 
-export default async (ctx: BBContext, next: any) => {
+export default async (ctx: UserCtx, next: any) => {
-  if (
+  const appId = getAppId()
-    !ctx.internal &&
+  const builderFn = env.isWorker()
-    (!ctx.user || !ctx.user.builder || !ctx.user.builder.global) &&
+    ? hasBuilderPermissions
-    (!ctx.user || !ctx.user.admin || !ctx.user.admin.global)
+    : env.isApps()
-  ) {
+    ? isBuilder
-    ctx.throw(403, "Builder user only endpoint.")
+    : undefined
+  if (!builderFn) {
+    throw new Error("Service name unknown - middleware inactive.")
+  }
+  if (!ctx.internal && !builderFn(ctx.user, appId) && !isAdmin(ctx.user)) {
+    ctx.throw(403, "Admin/Builder user only endpoint.")
   }
   return next()
 }

packages/backend-core/src/middleware/tests/builder.spec.ts

@@ -0,0 +1,180 @@
+import adminOnly from "../adminOnly"
+import builderOnly from "../builderOnly"
+import builderOrAdmin from "../builderOrAdmin"
+import { structures } from "../../../tests"
+import { ContextUser, ServiceType } from "@budibase/types"
+import { doInAppContext } from "../../context"
+import env from "../../environment"
+env._set("SERVICE_TYPE", ServiceType.APPS)
+
+const appId = "app_aaa"
+const basicUser = structures.users.user()
+const adminUser = structures.users.adminUser()
+const adminOnlyUser = structures.users.adminOnlyUser()
+const builderUser = structures.users.builderUser()
+const appBuilderUser = structures.users.appBuilderUser(appId)
+
+function buildUserCtx(user: ContextUser) {
+  return {
+    internal: false,
+    user,
+    throw: jest.fn(),
+  } as any
+}
+
+function passed(throwFn: jest.Func, nextFn: jest.Func) {
+  expect(throwFn).not.toBeCalled()
+  expect(nextFn).toBeCalled()
+}
+
+function threw(throwFn: jest.Func) {
+  // cant check next, the throw function doesn't actually throw - so it still continues
+  expect(throwFn).toBeCalled()
+}
+
+describe("adminOnly middleware", () => {
+  it("should allow admin user", () => {
+    const ctx = buildUserCtx(adminUser),
+      next = jest.fn()
+    adminOnly(ctx, next)
+    passed(ctx.throw, next)
+  })
+
+  it("should not allow basic user", () => {
+    const ctx = buildUserCtx(basicUser),
+      next = jest.fn()
+    adminOnly(ctx, next)
+    threw(ctx.throw)
+  })
+
+  it("should not allow builder user", () => {
+    const ctx = buildUserCtx(builderUser),
+      next = jest.fn()
+    adminOnly(ctx, next)
+    threw(ctx.throw)
+  })
+})
+
+describe("builderOnly middleware", () => {
+  it("should allow builder user", () => {
+    const ctx = buildUserCtx(builderUser),
+      next = jest.fn()
+    builderOnly(ctx, next)
+    passed(ctx.throw, next)
+  })
+
+  it("should allow app builder user", () => {
+    const ctx = buildUserCtx(appBuilderUser),
+      next = jest.fn()
+    doInAppContext(appId, () => {
+      builderOnly(ctx, next)
+    })
+    passed(ctx.throw, next)
+  })
+
+  it("should allow admin and builder user", () => {
+    const ctx = buildUserCtx(adminUser),
+      next = jest.fn()
+    builderOnly(ctx, next)
+    passed(ctx.throw, next)
+  })
+
+  it("should not allow admin user", () => {
+    const ctx = buildUserCtx(adminOnlyUser),
+      next = jest.fn()
+    builderOnly(ctx, next)
+    threw(ctx.throw)
+  })
+
+  it("should not allow app builder user to different app", () => {
+    const ctx = buildUserCtx(appBuilderUser),
+      next = jest.fn()
+    doInAppContext("app_bbb", () => {
+      builderOnly(ctx, next)
+    })
+    threw(ctx.throw)
+  })
+
+  it("should not allow basic user", () => {
+    const ctx = buildUserCtx(basicUser),
+      next = jest.fn()
+    builderOnly(ctx, next)
+    threw(ctx.throw)
+  })
+})
+
+describe("builderOrAdmin middleware", () => {
+  it("should allow builder user", () => {
+    const ctx = buildUserCtx(builderUser),
+      next = jest.fn()
+    builderOrAdmin(ctx, next)
+    passed(ctx.throw, next)
+  })
+
+  it("should allow builder and admin user", () => {
+    const ctx = buildUserCtx(adminUser),
+      next = jest.fn()
+    builderOrAdmin(ctx, next)
+    passed(ctx.throw, next)
+  })
+
+  it("should allow admin user", () => {
+    const ctx = buildUserCtx(adminOnlyUser),
+      next = jest.fn()
+    builderOrAdmin(ctx, next)
+    passed(ctx.throw, next)
+  })
+
+  it("should allow app builder user", () => {
+    const ctx = buildUserCtx(appBuilderUser),
+      next = jest.fn()
+    doInAppContext(appId, () => {
+      builderOrAdmin(ctx, next)
+    })
+    passed(ctx.throw, next)
+  })
+
+  it("should not allow basic user", () => {
+    const ctx = buildUserCtx(basicUser),
+      next = jest.fn()
+    builderOrAdmin(ctx, next)
+    threw(ctx.throw)
+  })
+})
+
+describe("check service difference", () => {
+  it("should not allow without app ID in apps", () => {
+    env._set("SERVICE_TYPE", ServiceType.APPS)
+    const appId = "app_a"
+    const ctx = buildUserCtx({
+      ...basicUser,
+      builder: {
+        apps: [appId],
+      },
+    })
+    const next = jest.fn()
+    doInAppContext(appId, () => {
+      builderOnly(ctx, next)
+    })
+    passed(ctx.throw, next)
+    doInAppContext("app_b", () => {
+      builderOnly(ctx, next)
+    })
+    threw(ctx.throw)
+  })
+
+  it("should allow without app ID in worker", () => {
+    env._set("SERVICE_TYPE", ServiceType.WORKER)
+    const ctx = buildUserCtx({
+      ...basicUser,
+      builder: {
+        apps: ["app_a"],
+      },
+    })
+    const next = jest.fn()
+    doInAppContext("app_b", () => {
+      builderOnly(ctx, next)
+    })
+    passed(ctx.throw, next)
+  })
+})

packages/backend-core/src/security/permissions.ts

@@ -1,3 +1,5 @@
+import { PermissionType, PermissionLevel } from "@budibase/types"
+export { PermissionType, PermissionLevel } from "@budibase/types"
 import flatten from "lodash/flatten"
 import cloneDeep from "lodash/fp/cloneDeep"
 
@@ -5,25 +7,6 @@ export type RoleHierarchy = {
   permissionId: string
 }[]
 
-export enum PermissionLevel {
-  READ = "read",
-  WRITE = "write",
-  EXECUTE = "execute",
-  ADMIN = "admin",
-}
-
-// these are the global types, that govern the underlying default behaviour
-export enum PermissionType {
-  APP = "app",
-  TABLE = "table",
-  USER = "user",
-  AUTOMATION = "automation",
-  WEBHOOK = "webhook",
-  BUILDER = "builder",
-  VIEW = "view",
-  QUERY = "query",
-}
-
 export class Permission {
   type: PermissionType
   level: PermissionLevel
@@ -173,3 +156,4 @@ export function isPermissionLevelHigherThanRead(level: PermissionLevel) {
 
 // utility as a lot of things need simply the builder permission
 export const BUILDER = PermissionType.BUILDER
+export const GLOBAL_BUILDER = PermissionType.GLOBAL_BUILDER

packages/backend-core/src/users/db.ts

@@ -0,0 +1,460 @@
+import env from "../environment"
+import * as eventHelpers from "./events"
+import * as accounts from "../accounts"
+import * as cache from "../cache"
+import { getIdentity, getTenantId, getGlobalDB } from "../context"
+import * as dbUtils from "../db"
+import { EmailUnavailableError, HTTPError } from "../errors"
+import * as platform from "../platform"
+import * as sessions from "../security/sessions"
+import * as usersCore from "./users"
+import {
+  AllDocsResponse,
+  BulkUserCreated,
+  BulkUserDeleted,
+  RowResponse,
+  SaveUserOpts,
+  User,
+  Account,
+  isSSOUser,
+  isSSOAccount,
+  UserStatus,
+} from "@budibase/types"
+import * as accountSdk from "../accounts"
+import {
+  validateUniqueUser,
+  getAccountHolderFromUserIds,
+  isAdmin,
+} from "./utils"
+import { searchExistingEmails } from "./lookup"
+import { hash } from "../utils"
+
+type QuotaUpdateFn = (change: number, cb?: () => Promise<any>) => Promise<any>
+type GroupUpdateFn = (groupId: string, userIds: string[]) => Promise<any>
+type FeatureFn = () => Promise<Boolean>
+type QuotaFns = { addUsers: QuotaUpdateFn; removeUsers: QuotaUpdateFn }
+type GroupFns = { addUsers: GroupUpdateFn }
+type FeatureFns = { isSSOEnforced: FeatureFn; isAppBuildersEnabled: FeatureFn }
+
+const bulkDeleteProcessing = async (dbUser: User) => {
+  const userId = dbUser._id as string
+  await platform.users.removeUser(dbUser)
+  await eventHelpers.handleDeleteEvents(dbUser)
+  await cache.user.invalidateUser(userId)
+  await sessions.invalidateSessions(userId, { reason: "bulk-deletion" })
+}
+
+export class UserDB {
+  static quotas: QuotaFns
+  static groups: GroupFns
+  static features: FeatureFns
+
+  static init(quotaFns: QuotaFns, groupFns: GroupFns, featureFns: FeatureFns) {
+    UserDB.quotas = quotaFns
+    UserDB.groups = groupFns
+    UserDB.features = featureFns
+  }
+
+  static async isPreventPasswordActions(user: User, account?: Account) {
+    // when in maintenance mode we allow sso users with the admin role
+    // to perform any password action - this prevents lockout
+    if (env.ENABLE_SSO_MAINTENANCE_MODE && isAdmin(user)) {
+      return false
+    }
+
+    // SSO is enforced for all users
+    if (await UserDB.features.isSSOEnforced()) {
+      return true
+    }
+
+    // Check local sso
+    if (isSSOUser(user)) {
+      return true
+    }
+
+    // Check account sso
+    if (!account) {
+      account = await accountSdk.getAccountByTenantId(getTenantId())
+    }
+    return !!(account && account.email === user.email && isSSOAccount(account))
+  }
+
+  static async buildUser(
+    user: User,
+    opts: SaveUserOpts = {
+      hashPassword: true,
+      requirePassword: true,
+    },
+    tenantId: string,
+    dbUser?: any,
+    account?: Account
+  ): Promise<User> {
+    let { password, _id } = user
+
+    // don't require a password if the db user doesn't already have one
+    if (dbUser && !dbUser.password) {
+      opts.requirePassword = false
+    }
+
+    let hashedPassword
+    if (password) {
+      if (await UserDB.isPreventPasswordActions(user, account)) {
+        throw new HTTPError("Password change is disabled for this user", 400)
+      }
+      hashedPassword = opts.hashPassword ? await hash(password) : password
+    } else if (dbUser) {
+      hashedPassword = dbUser.password
+    }
+
+    // passwords are never required if sso is enforced
+    const requirePasswords =
+      opts.requirePassword && !(await UserDB.features.isSSOEnforced())
+    if (!hashedPassword && requirePasswords) {
+      throw "Password must be specified."
+    }
+
+    _id = _id || dbUtils.generateGlobalUserID()
+
+    const fullUser = {
+      createdAt: Date.now(),
+      ...dbUser,
+      ...user,
+      _id,
+      password: hashedPassword,
+      tenantId,
+    }
+    // make sure the roles object is always present
+    if (!fullUser.roles) {
+      fullUser.roles = {}
+    }
+    // add the active status to a user if its not provided
+    if (fullUser.status == null) {
+      fullUser.status = UserStatus.ACTIVE
+    }
+
+    return fullUser
+  }
+
+  static async allUsers() {
+    const db = getGlobalDB()
+    const response = await db.allDocs(
+      dbUtils.getGlobalUserParams(null, {
+        include_docs: true,
+      })
+    )
+    return response.rows.map((row: any) => row.doc)
+  }
+
+  static async countUsersByApp(appId: string) {
+    let response: any = await usersCore.searchGlobalUsersByApp(appId, {})
+    return {
+      userCount: response.length,
+    }
+  }
+
+  static async getUsersByAppAccess(appId?: string) {
+    const opts: any = {
+      include_docs: true,
+      limit: 50,
+    }
+    let response: User[] = await usersCore.searchGlobalUsersByAppAccess(
+      appId,
+      opts
+    )
+    return response
+  }
+
+  static async getUserByEmail(email: string) {
+    return usersCore.getGlobalUserByEmail(email)
+  }
+
+  /**
+   * Gets a user by ID from the global database, based on the current tenancy.
+   */
+  static async getUser(userId: string) {
+    const user = await usersCore.getById(userId)
+    if (user) {
+      delete user.password
+    }
+    return user
+  }
+
+  static async save(user: User, opts: SaveUserOpts = {}): Promise<User> {
+    // default booleans to true
+    if (opts.hashPassword == null) {
+      opts.hashPassword = true
+    }
+    if (opts.requirePassword == null) {
+      opts.requirePassword = true
+    }
+    const tenantId = getTenantId()
+    const db = getGlobalDB()
+
+    let { email, _id, userGroups = [], roles } = user
+
+    if (!email && !_id) {
+      throw new Error("_id or email is required")
+    }
+
+    if (
+      user.builder?.apps?.length &&
+      !(await UserDB.features.isAppBuildersEnabled())
+    ) {
+      throw new Error("Unable to update app builders, please check license")
+    }
+
+    let dbUser: User | undefined
+    if (_id) {
+      // try to get existing user from db
+      try {
+        dbUser = (await db.get(_id)) as User
+        if (email && dbUser.email !== email) {
+          throw "Email address cannot be changed"
+        }
+        email = dbUser.email
+      } catch (e: any) {
+        if (e.status === 404) {
+          // do nothing, save this new user with the id specified - required for SSO auth
+        } else {
+          throw e
+        }
+      }
+    }
+
+    if (!dbUser && email) {
+      // no id was specified - load from email instead
+      dbUser = await usersCore.getGlobalUserByEmail(email)
+      if (dbUser && dbUser._id !== _id) {
+        throw new EmailUnavailableError(email)
+      }
+    }
+
+    const change = dbUser ? 0 : 1 // no change if there is existing user
+    return UserDB.quotas.addUsers(change, async () => {
+      await validateUniqueUser(email, tenantId)
+
+      let builtUser = await UserDB.buildUser(user, opts, tenantId, dbUser)
+      // don't allow a user to update its own roles/perms
+      if (opts.currentUserId && opts.currentUserId === dbUser?._id) {
+        builtUser = usersCore.cleanseUserObject(builtUser, dbUser) as User
+      }
+
+      if (!dbUser && roles?.length) {
+        builtUser.roles = { ...roles }
+      }
+
+      // make sure we set the _id field for a new user
+      // Also if this is a new user, associate groups with them
+      let groupPromises = []
+      if (!_id) {
+        _id = builtUser._id!
+
+        if (userGroups.length > 0) {
+          for (let groupId of userGroups) {
+            groupPromises.push(UserDB.groups.addUsers(groupId, [_id!]))
+          }
+        }
+      }
+
+      try {
+        // save the user to db
+        let response = await db.put(builtUser)
+        builtUser._rev = response.rev
+
+        await eventHelpers.handleSaveEvents(builtUser, dbUser)
+        await platform.users.addUser(tenantId, builtUser._id!, builtUser.email)
+        await cache.user.invalidateUser(response.id)
+
+        await Promise.all(groupPromises)
+
+        // finally returned the saved user from the db
+        return db.get(builtUser._id!)
+      } catch (err: any) {
+        if (err.status === 409) {
+          throw "User exists already"
+        } else {
+          throw err
+        }
+      }
+    })
+  }
+
+  static async bulkCreate(
+    newUsersRequested: User[],
+    groups: string[]
+  ): Promise<BulkUserCreated> {
+    const tenantId = getTenantId()
+
+    let usersToSave: any[] = []
+    let newUsers: any[] = []
+
+    const emails = newUsersRequested.map((user: User) => user.email)
+    const existingEmails = await searchExistingEmails(emails)
+    const unsuccessful: { email: string; reason: string }[] = []
+
+    for (const newUser of newUsersRequested) {
+      if (
+        newUsers.find(
+          (x: User) => x.email.toLowerCase() === newUser.email.toLowerCase()
+        ) ||
+        existingEmails.includes(newUser.email.toLowerCase())
+      ) {
+        unsuccessful.push({
+          email: newUser.email,
+          reason: `Unavailable`,
+        })
+        continue
+      }
+      newUser.userGroups = groups
+      newUsers.push(newUser)
+    }
+
+    const account = await accountSdk.getAccountByTenantId(tenantId)
+    return UserDB.quotas.addUsers(newUsers.length, async () => {
+      // create the promises array that will be called by bulkDocs
+      newUsers.forEach((user: any) => {
+        usersToSave.push(
+          UserDB.buildUser(
+            user,
+            {
+              hashPassword: true,
+              requirePassword: user.requirePassword,
+            },
+            tenantId,
+            undefined, // no dbUser
+            account
+          )
+        )
+      })
+
+      const usersToBulkSave = await Promise.all(usersToSave)
+      await usersCore.bulkUpdateGlobalUsers(usersToBulkSave)
+
+      // Post-processing of bulk added users, e.g. events and cache operations
+      for (const user of usersToBulkSave) {
+        // TODO: Refactor to bulk insert users into the info db
+        // instead of relying on looping tenant creation
+        await platform.users.addUser(tenantId, user._id, user.email)
+        await eventHelpers.handleSaveEvents(user, undefined)
+      }
+
+      const saved = usersToBulkSave.map(user => {
+        return {
+          _id: user._id,
+          email: user.email,
+        }
+      })
+
+      // now update the groups
+      if (Array.isArray(saved) && groups) {
+        const groupPromises = []
+        const createdUserIds = saved.map(user => user._id)
+        for (let groupId of groups) {
+          groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds))
+        }
+        await Promise.all(groupPromises)
+      }
+
+      return {
+        successful: saved,
+        unsuccessful,
+      }
+    })
+  }
+
+  static async bulkDelete(userIds: string[]): Promise<BulkUserDeleted> {
+    const db = getGlobalDB()
+
+    const response: BulkUserDeleted = {
+      successful: [],
+      unsuccessful: [],
+    }
+
+    // remove the account holder from the delete request if present
+    const account = await getAccountHolderFromUserIds(userIds)
+    if (account) {
+      userIds = userIds.filter(u => u !== account.budibaseUserId)
+      // mark user as unsuccessful
+      response.unsuccessful.push({
+        _id: account.budibaseUserId,
+        email: account.email,
+        reason: "Account holder cannot be deleted",
+      })
+    }
+
+    // Get users and delete
+    const allDocsResponse: AllDocsResponse<User> = await db.allDocs({
+      include_docs: true,
+      keys: userIds,
+    })
+    const usersToDelete: User[] = allDocsResponse.rows.map(
+      (user: RowResponse<User>) => {
+        return user.doc
+      }
+    )
+
+    // Delete from DB
+    const toDelete = usersToDelete.map(user => ({
+      ...user,
+      _deleted: true,
+    }))
+    const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete)
+
+    await UserDB.quotas.removeUsers(toDelete.length)
+    for (let user of usersToDelete) {
+      await bulkDeleteProcessing(user)
+    }
+
+    // Build Response
+    // index users by id
+    const userIndex: { [key: string]: User } = {}
+    usersToDelete.reduce((prev, current) => {
+      prev[current._id!] = current
+      return prev
+    }, userIndex)
+
+    // add the successful and unsuccessful users to response
+    dbResponse.forEach(item => {
+      const email = userIndex[item.id].email
+      if (item.ok) {
+        response.successful.push({ _id: item.id, email })
+      } else {
+        response.unsuccessful.push({
+          _id: item.id,
+          email,
+          reason: "Database error",
+        })
+      }
+    })
+
+    return response
+  }
+
+  static async destroy(id: string) {
+    const db = getGlobalDB()
+    const dbUser = (await db.get(id)) as User
+    const userId = dbUser._id as string
+
+    if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+      // root account holder can't be deleted from inside budibase
+      const email = dbUser.email
+      const account = await accounts.getAccount(email)
+      if (account) {
+        if (dbUser.userId === getIdentity()!._id) {
+          throw new HTTPError('Please visit "Account" to delete this user', 400)
+        } else {
+          throw new HTTPError("Account holder cannot be deleted", 400)
+        }
+      }
+    }
+
+    await platform.users.removeUser(dbUser)
+
+    await db.remove(userId, dbUser._rev)
+
+    await UserDB.quotas.removeUsers(1)
+    await eventHelpers.handleDeleteEvents(dbUser)
+    await cache.user.invalidateUser(userId)
+    await sessions.invalidateSessions(userId, { reason: "deletion" })
+  }
+}

packages/backend-core/src/users/events.ts

@@ -1,15 +1,18 @@
-import env from "../../environment"
+import env from "../environment"
-import { events, accounts, tenancy } from "@budibase/backend-core"
+import * as events from "../events"
+import * as accounts from "../accounts"
+import { getTenantId } from "../context"
 import { User, UserRoles, CloudAccount } from "@budibase/types"
+import { hasBuilderPermissions, hasAdminPermissions } from "./utils"
 
 export const handleDeleteEvents = async (user: any) => {
   await events.user.deleted(user)
 
-  if (isBuilder(user)) {
+  if (hasBuilderPermissions(user)) {
     await events.user.permissionBuilderRemoved(user)
   }
 
-  if (isAdmin(user)) {
+  if (hasAdminPermissions(user)) {
     await events.user.permissionAdminRemoved(user)
   }
 }
@@ -55,7 +58,7 @@ export const handleSaveEvents = async (
   user: User,
   existingUser: User | undefined
 ) => {
-  const tenantId = tenancy.getTenantId()
+  const tenantId = getTenantId()
   let tenantAccount: CloudAccount | undefined
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     tenantAccount = await accounts.getAccountByTenantId(tenantId)
@@ -103,23 +106,20 @@ export const handleSaveEvents = async (
   await handleAppRoleEvents(user, existingUser)
 }
 
-const isBuilder = (user: any) => user.builder && user.builder.global
-const isAdmin = (user: any) => user.admin && user.admin.global
-
 export const isAddingBuilder = (user: any, existingUser: any) => {
-  return isAddingPermission(user, existingUser, isBuilder)
+  return isAddingPermission(user, existingUser, hasBuilderPermissions)
 }
 
 export const isRemovingBuilder = (user: any, existingUser: any) => {
-  return isRemovingPermission(user, existingUser, isBuilder)
+  return isRemovingPermission(user, existingUser, hasBuilderPermissions)
 }
 
 const isAddingAdmin = (user: any, existingUser: any) => {
-  return isAddingPermission(user, existingUser, isAdmin)
+  return isAddingPermission(user, existingUser, hasAdminPermissions)
 }
 
 const isRemovingAdmin = (user: any, existingUser: any) => {
-  return isRemovingPermission(user, existingUser, isAdmin)
+  return isRemovingPermission(user, existingUser, hasAdminPermissions)
 }
 
 const isOnboardingComplete = (user: any, existingUser: any) => {

packages/backend-core/src/users/index.ts

@@ -0,0 +1,4 @@
+export * from "./users"
+export * from "./utils"
+export * from "./lookup"
+export { UserDB } from "./db"

packages/backend-core/src/users/lookup.ts

@@ -0,0 +1,102 @@
+import {
+  AccountMetadata,
+  PlatformUser,
+  PlatformUserByEmail,
+  User,
+} from "@budibase/types"
+import * as dbUtils from "../db"
+import { ViewName } from "../constants"
+
+/**
+ * Apply a system-wide search on emails:
+ * - in tenant
+ * - cross tenant
+ * - accounts
+ * return an array of emails that match the supplied emails.
+ */
+export async function searchExistingEmails(emails: string[]) {
+  let matchedEmails: string[] = []
+
+  const existingTenantUsers = await getExistingTenantUsers(emails)
+  matchedEmails.push(...existingTenantUsers.map(user => user.email))
+
+  const existingPlatformUsers = await getExistingPlatformUsers(emails)
+  matchedEmails.push(...existingPlatformUsers.map(user => user._id!))
+
+  const existingAccounts = await getExistingAccounts(emails)
+  matchedEmails.push(...existingAccounts.map(account => account.email))
+
+  return [...new Set(matchedEmails.map(email => email.toLowerCase()))]
+}
+
+// lookup, could be email or userId, either will return a doc
+export async function getPlatformUser(
+  identifier: string
+): Promise<PlatformUser | null> {
+  // use the view here and allow to find anyone regardless of casing
+  // Use lowercase to ensure email login is case insensitive
+  return (await dbUtils.queryPlatformView(ViewName.PLATFORM_USERS_LOWERCASE, {
+    keys: [identifier.toLowerCase()],
+    include_docs: true,
+  })) as PlatformUser
+}
+
+export async function getExistingTenantUsers(
+  emails: string[]
+): Promise<User[]> {
+  const lcEmails = emails.map(email => email.toLowerCase())
+  const params = {
+    keys: lcEmails,
+    include_docs: true,
+  }
+
+  const opts = {
+    arrayResponse: true,
+  }
+
+  return (await dbUtils.queryGlobalView(
+    ViewName.USER_BY_EMAIL,
+    params,
+    undefined,
+    opts
+  )) as User[]
+}
+
+export async function getExistingPlatformUsers(
+  emails: string[]
+): Promise<PlatformUserByEmail[]> {
+  const lcEmails = emails.map(email => email.toLowerCase())
+  const params = {
+    keys: lcEmails,
+    include_docs: true,
+  }
+
+  const opts = {
+    arrayResponse: true,
+  }
+  return (await dbUtils.queryPlatformView(
+    ViewName.PLATFORM_USERS_LOWERCASE,
+    params,
+    opts
+  )) as PlatformUserByEmail[]
+}
+
+export async function getExistingAccounts(
+  emails: string[]
+): Promise<AccountMetadata[]> {
+  const lcEmails = emails.map(email => email.toLowerCase())
+  const params = {
+    keys: lcEmails,
+    include_docs: true,
+  }
+
+  const opts = {
+    arrayResponse: true,
+  }
+
+  return (await dbUtils.queryPlatformView(
+    ViewName.ACCOUNT_BY_EMAIL,
+    params,
+    opts
+  )) as AccountMetadata[]
+}

packages/backend-core/src/users/users.ts

@@ -11,10 +11,16 @@ import {
   SEPARATOR,
   UNICODE_MAX,
   ViewName,
-} from "./db"
+} from "../db"
-import { BulkDocsResponse, SearchUsersRequest, User } from "@budibase/types"
+import {
-import { getGlobalDB } from "./context"
+  BulkDocsResponse,
-import * as context from "./context"
+  SearchUsersRequest,
+  User,
+  ContextUser,
+} from "@budibase/types"
+import { getGlobalDB } from "../context"
+import * as context from "../context"
+import { user as userCache } from "../cache"
 
 type GetOpts = { cleanup?: boolean }
 
@@ -178,7 +184,7 @@ export const getGlobalUserByAppPage = (appId: string, user: User) => {
  * Performs a starts with search on the global email view.
  */
 export const searchGlobalUsersByEmail = async (
-  email: string,
+  email: string | unknown,
   opts: any,
   getOpts?: GetOpts
 ) => {
@@ -248,3 +254,23 @@ export async function getUserCount() {
   })
   return response.total_rows
 }
+
+// used to remove the builder/admin permissions, for processing the
+// user as an app user (they may have some specific role/group
+export function removePortalUserPermissions(user: User | ContextUser) {
+  delete user.admin
+  delete user.builder
+  return user
+}
+
+export function cleanseUserObject(user: User | ContextUser, base?: User) {
+  delete user.admin
+  delete user.builder
+  delete user.roles
+  if (base) {
+    user.admin = base.admin
+    user.builder = base.builder
+    user.roles = base.roles
+  }
+  return user
+}

packages/backend-core/src/users/utils.ts

@@ -0,0 +1,55 @@
+import { CloudAccount } from "@budibase/types"
+import * as accountSdk from "../accounts"
+import env from "../environment"
+import { getPlatformUser } from "./lookup"
+import { EmailUnavailableError } from "../errors"
+import { getTenantId } from "../context"
+import { sdk } from "@budibase/shared-core"
+import { getAccountByTenantId } from "../accounts"
+
+// extract from shared-core to make easily accessible from backend-core
+export const isBuilder = sdk.users.isBuilder
+export const isAdmin = sdk.users.isAdmin
+export const isGlobalBuilder = sdk.users.isGlobalBuilder
+export const isAdminOrBuilder = sdk.users.isAdminOrBuilder
+export const hasAdminPermissions = sdk.users.hasAdminPermissions
+export const hasBuilderPermissions = sdk.users.hasBuilderPermissions
+export const hasAppBuilderPermissions = sdk.users.hasAppBuilderPermissions
+
+export async function validateUniqueUser(email: string, tenantId: string) {
+  // check budibase users in other tenants
+  if (env.MULTI_TENANCY) {
+    const tenantUser = await getPlatformUser(email)
+    if (tenantUser != null && tenantUser.tenantId !== tenantId) {
+      throw new EmailUnavailableError(email)
+    }
+  }
+
+  // check root account users in account portal
+  if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+    const account = await accountSdk.getAccount(email)
+    if (account && account.verified && account.tenantId !== tenantId) {
+      throw new EmailUnavailableError(email)
+    }
+  }
+}
+
+/**
+ * For the given user id's, return the account holder if it is in the ids.
+ */
+export async function getAccountHolderFromUserIds(
+  userIds: string[]
+): Promise<CloudAccount | undefined> {
+  if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+    const tenantId = getTenantId()
+    const account = await getAccountByTenantId(tenantId)
+    if (!account) {
+      throw new Error(`Account not found for tenantId=${tenantId}`)
+    }
+
+    const budibaseUserId = account.budibaseUserId
+    if (userIds.includes(budibaseUserId)) {
+      return account
+    }
+  }
+}

packages/backend-core/tests/core/utilities/mocks/events.ts

@@ -1,5 +1,3 @@
-import * as events from "../../../../src/events"
-
 beforeAll(async () => {
   const processors = await import("../../../../src/events/processors")
   const events = await import("../../../../src/events")

packages/backend-core/tests/core/utilities/mocks/licenses.ts

@@ -94,6 +94,10 @@ export const useSyncAutomations = () => {
   return useFeature(Feature.SYNC_AUTOMATIONS)
 }
 
+export const useAppBuilders = () => {
+  return useFeature(Feature.APP_BUILDERS)
+}
+
 // QUOTAS
 
 export const setAutomationLogsQuota = (value: number) => {

packages/backend-core/tests/core/utilities/structures/users.ts

@@ -1,5 +1,6 @@
 import {
   AdminUser,
+  AdminOnlyUser,
   BuilderUser,
   SSOAuthDetails,
   SSOUser,
@@ -21,6 +22,15 @@ export const adminUser = (userProps?: any): AdminUser => {
   }
 }
 
+export const adminOnlyUser = (userProps?: any): AdminOnlyUser => {
+  return {
+    ...user(userProps),
+    admin: {
+      global: true,
+    },
+  }
+}
+
 export const builderUser = (userProps?: any): BuilderUser => {
   return {
     ...user(userProps),
@@ -30,6 +40,15 @@ export const builderUser = (userProps?: any): BuilderUser => {
   }
 }
 
+export const appBuilderUser = (appId: string, userProps?: any): BuilderUser => {
+  return {
+    ...user(userProps),
+    builder: {
+      apps: [appId],
+    },
+  }
+}
+
 export function ssoUser(
   opts: { user?: any; details?: SSOAuthDetails } = {}
 ): SSOUser {

packages/backend-core/tsconfig.json

@@ -4,9 +4,9 @@
     "composite": true,
     "baseUrl": ".",
     "paths": {
-      "@budibase/types": ["../types/src"]
+      "@budibase/types": ["../types/src"],
+      "@budibase/shared-core": ["../shared-core/src"]
     }
   },
-
   "exclude": ["node_modules", "dist"]
 }

packages/bbui/package.json

@@ -85,7 +85,8 @@
     "dayjs": "^1.10.4",
     "easymde": "^2.16.1",
     "svelte-flatpickr": "3.2.3",
-    "svelte-portal": "^1.0.0"
+    "svelte-portal": "^1.0.0",
+    "svelte-dnd-action": "^0.9.8"
   },
   "resolutions": {
     "loader-utils": "1.4.1"
@@ -97,13 +98,13 @@
           {
             "projects": [
               "@budibase/string-templates",
-              "@budibase/shared-core"
+              "@budibase/shared-core",
+              "@budibase/types"
             ],
             "target": "build"
           }
         ]
       }
     }
-  },
+  }
-  "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }

packages/bbui/src/IconPicker/IconPicker.svelte

@@ -1,5 +1,4 @@
 <script>
-  //import { createEventDispatcher } from "svelte"
   import "@spectrum-css/popover/dist/index-vars.css"
   import clickOutside from "../Actions/click_outside"
   import { fly } from "svelte/transition"

packages/bbui/src/OptionSelectDnD/OptionSelectDnD.svelte

@@ -0,0 +1,252 @@
+<script>
+  import { flip } from "svelte/animate"
+  import { dndzone } from "svelte-dnd-action"
+  import Icon from "../Icon/Icon.svelte"
+  import Popover from "../Popover/Popover.svelte"
+  import { onMount } from "svelte"
+  const flipDurationMs = 150
+
+  export let constraints
+  export let optionColors = {}
+  let options = []
+
+  let colorPopovers = []
+  let anchors = []
+
+  let colorsArray = [
+    "hsla(0, 90%, 75%, 0.3)",
+    "hsla(50, 80%, 75%, 0.3)",
+    "hsla(120, 90%, 75%, 0.3)",
+    "hsla(200, 90%, 75%, 0.3)",
+    "hsla(240, 90%, 75%, 0.3)",
+    "hsla(320, 90%, 75%, 0.3)",
+  ]
+  $: {
+    if (constraints.inclusion.length) {
+      options = constraints.inclusion.map(value => ({
+        name: value,
+        id: Math.random(),
+      }))
+    }
+  }
+  const removeInput = idx => {
+    delete optionColors[options[idx].name]
+    constraints.inclusion = constraints.inclusion.filter((e, i) => i !== idx)
+    options = options.filter((e, i) => i !== idx)
+    colorPopovers.pop(undefined)
+    anchors.pop(undefined)
+  }
+
+  const addNewInput = () => {
+    options = [
+      ...options,
+      { name: `Option ${constraints.inclusion.length + 1}`, id: Math.random() },
+    ]
+    constraints.inclusion = [
+      ...constraints.inclusion,
+      `Option ${constraints.inclusion.length + 1}`,
+    ]
+
+    colorPopovers.push(undefined)
+    anchors.push(undefined)
+  }
+
+  const handleDndConsider = e => {
+    options = e.detail.items
+  }
+  const handleDndFinalize = e => {
+    options = e.detail.items
+    constraints.inclusion = options.map(option => option.name)
+  }
+
+  const handleColorChange = (optionName, color, idx) => {
+    optionColors[optionName] = color
+    colorPopovers[idx].hide()
+  }
+
+  const handleNameChange = (optionName, idx, value) => {
+    constraints.inclusion[idx] = value
+    options[idx].name = value
+    optionColors[value] = optionColors[optionName]
+    delete optionColors[optionName]
+  }
+
+  const openColorPickerPopover = (optionIdx, target) => {
+    colorPopovers[optionIdx].show()
+    anchors[optionIdx] = target
+  }
+
+  onMount(() => {
+    // Initialize anchor arrays on mount, assuming 'options' is already populated
+    colorPopovers = constraints.inclusion.map(() => undefined)
+    anchors = constraints.inclusion.map(() => undefined)
+  })
+</script>
+
+<div>
+  <div
+    class="actions"
+    use:dndzone={{
+      items: options,
+      flipDurationMs,
+      dropTargetStyle: { outline: "none" },
+    }}
+    on:consider={handleDndConsider}
+    on:finalize={handleDndFinalize}
+  >
+    {#each options as option, idx (option.id)}
+      <div
+        class="no-border action-container"
+        animate:flip={{ duration: flipDurationMs }}
+      >
+        <div class="child drag-handle-spacing">
+          <Icon name="DragHandle" size="L" />
+        </div>
+        <div class="child color-picker">
+          <div
+            id="color-picker"
+            bind:this={anchors[idx]}
+            style="--color:{optionColors?.[option.name] ||
+              'hsla(0, 1%, 50%, 0.3)'}"
+            class="circle"
+            on:click={e => openColorPickerPopover(idx, e.target)}
+          >
+            <Popover
+              bind:this={colorPopovers[idx]}
+              anchor={anchors[idx]}
+              align="left"
+              offset={0}
+              style=""
+              popoverTarget={document.getElementById(`color-picker`)}
+              animate={false}
+            >
+              <div class="colors">
+                {#each colorsArray as color}
+                  <div
+                    on:click={() => handleColorChange(option.name, color, idx)}
+                    style="--color:{color};"
+                    class="circle circle-hover"
+                  />
+                {/each}
+              </div>
+            </Popover>
+          </div>
+        </div>
+        <div class="child">
+          <input
+            class="input-field"
+            type="text"
+            on:change={e => handleNameChange(option.name, idx, e.target.value)}
+            value={option.name}
+            placeholder="Option name"
+          />
+        </div>
+        <div class="child">
+          <Icon name="Close" hoverable size="S" on:click={removeInput(idx)} />
+        </div>
+      </div>
+    {/each}
+  </div>
+  <div on:click={addNewInput} class="add-option">
+    <Icon hoverable name="Add" />
+    <div>Add option</div>
+  </div>
+</div>
+
+<style>
+  .action-container {
+    background-color: var(--spectrum-alias-background-color-primary);
+    border-radius: 0px;
+    border: 1px solid var(--spectrum-global-color-gray-300);
+    transition: background-color 130ms ease-in-out, color 130ms ease-in-out,
+      border-color 130ms ease-in-out;
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+  }
+  .no-border {
+    border-bottom: none;
+  }
+
+  .action-container:last-child {
+    border-bottom: 1px solid var(--spectrum-global-color-gray-300) !important;
+  }
+
+  .child {
+    height: 30px;
+  }
+  .child:hover,
+  .child:focus {
+    background: var(--spectrum-global-color-gray-200);
+  }
+  .add-option {
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+    padding: var(--spacing-m);
+    gap: var(--spacing-m);
+    cursor: pointer;
+  }
+
+  .input-field {
+    border: none;
+    outline: none;
+    background-color: transparent;
+    width: 100%;
+    color: var(--text);
+  }
+
+  .child input[type="text"] {
+    padding-left: 10px;
+  }
+
+  .input-field:hover,
+  .input-field:focus {
+    background: var(--spectrum-global-color-gray-200);
+  }
+
+  .action-container > :nth-child(1) {
+    flex-grow: 1;
+    justify-content: center;
+    display: flex;
+  }
+
+  .action-container > :nth-child(2) {
+    flex-grow: 1;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+  }
+
+  .action-container > :nth-child(3) {
+    flex-grow: 4;
+    display: flex;
+  }
+  .action-container > :nth-child(4) {
+    flex-grow: 1;
+    justify-content: center;
+    display: flex;
+  }
+
+  .circle {
+    height: 20px;
+    width: 20px;
+    background-color: var(--color);
+    border-radius: 50%;
+    display: inline-block;
+    box-sizing: border-box;
+  }
+
+  .circle-hover:hover {
+    border: 1px solid var(--spectrum-global-color-blue-400);
+    cursor: pointer;
+  }
+
+  .colors {
+    display: grid;
+    grid-template-columns: 1fr 1fr 1fr;
+    gap: var(--spacing-xl);
+    justify-items: center;
+    margin: var(--spacing-m);
+  }
+</style>

packages/bbui/src/Popover/Popover.svelte

@@ -21,6 +21,7 @@
   export let offset = 5
   export let customHeight
   export let animate = true
+  export let customZindex
 
   $: target = portalTarget || getContext(Context.PopoverRoot) || ".spectrum"
 
@@ -77,8 +78,9 @@
       }}
       on:keydown={handleEscape}
       class="spectrum-Popover is-open"
+      class:customZindex
       role="presentation"
-      style="height: {customHeight}"
+      style="height: {customHeight}; --customZindex: {customZindex};"
       transition:fly|local={{ y: -20, duration: animate ? 200 : 0 }}
     >
       <slot />
@@ -92,4 +94,8 @@
     border-color: var(--spectrum-global-color-gray-300);
     overflow: auto;
   }
+
+  .customZindex {
+    z-index: var(--customZindex) !important;
+  }
 </style>

packages/bbui/src/index.js

@@ -84,7 +84,7 @@ export { default as IconSideNavItem } from "./IconSideNav/IconSideNavItem.svelte
 export { default as Slider } from "./Form/Slider.svelte"
 export { default as Accordion } from "./Accordion/Accordion.svelte"
 export { default as File } from "./Form/File.svelte"
-
+export { default as OptionSelectDnD } from "./OptionSelectDnD/OptionSelectDnD.svelte"
 // Renderers
 export { default as BoldRenderer } from "./Table/BoldRenderer.svelte"
 export { default as CodeRenderer } from "./Table/CodeRenderer.svelte"

packages/builder/package.json

@@ -133,8 +133,21 @@
         "dependsOn": [
           {
             "projects": [
+              "@budibase/shared-core",
               "@budibase/string-templates",
-              "@budibase/shared-core"
+              "@budibase/types"
+            ],
+            "target": "build"
+          }
+        ]
+      },
+      "dev:builder": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/shared-core",
+              "@budibase/string-templates",
+              "@budibase/types"
             ],
             "target": "build"
           }
@@ -145,13 +158,13 @@
           {
             "projects": [
               "@budibase/shared-core",
-              "@budibase/string-templates"
+              "@budibase/string-templates",
+              "@budibase/types"
             ],
             "target": "build"
           }
         ]
       }
     }
-  },
+  }
-  "gitHead": "115189f72a850bfb52b65ec61d932531bf327072"
 }

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -108,10 +108,7 @@
   /****************************************************/
 
   const getInputData = (testData, blockInputs) => {
-    let newInputData = testData || blockInputs
+    let newInputData = cloneDeep(testData || blockInputs)
-    if (block.event === "app:trigger" && !newInputData?.fields) {
-      newInputData = cloneDeep(blockInputs)
-    }
 
     /**
      * TODO - Remove after November 2023

packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte

@@ -50,6 +50,7 @@
         type="string"
         {bindings}
         fillWidth={true}
+        updateOnChange={false}
       />
     {/each}
   </div>

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -64,6 +64,13 @@
     <svelte:fragment slot="filter">
       <GridFilterButton />
     </svelte:fragment>
+    <svelte:fragment slot="edit-column">
+      <GridEditColumnModal />
+    </svelte:fragment>
+    <svelte:fragment slot="add-column">
+      <GridAddColumnModal />
+    </svelte:fragment>
+
     <svelte:fragment slot="controls">
       {#if isInternal}
         <GridCreateViewButton />
@@ -77,9 +84,8 @@
       {:else}
         <GridImportButton />
       {/if}
+
       <GridExportButton />
-      <GridAddColumnModal />
-      <GridEditColumnModal />
       {#if isUsersTable}
         <GridEditUserModal />
       {:else}

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -7,12 +7,12 @@
     Toggle,
     RadioGroup,
     DatePicker,
-    ModalContent,
-    Context,
     Modal,
     notifications,
+    OptionSelectDnD,
+    Layout,
   } from "@budibase/bbui"
-  import { createEventDispatcher } from "svelte"
+  import { createEventDispatcher, getContext } from "svelte"
   import { cloneDeep } from "lodash/fp"
   import { tables, datasources } from "stores/backend"
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
@@ -26,12 +26,10 @@
     SWITCHABLE_TYPES,
   } from "constants/backend"
   import { getAutoColumnInformation, buildAutoColumn } from "builderStore/utils"
-  import ValuesList from "components/common/ValuesList.svelte"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import { truncate } from "lodash"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import { getBindings } from "components/backend/DataTable/formula"
-  import { getContext } from "svelte"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
   import { ValidColumnNameRegex } from "@budibase/shared-core"
 
@@ -45,11 +43,11 @@
 
   const dispatch = createEventDispatcher()
   const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
-  const { hide } = getContext(Context.Modal)
+  const { dispatch: gridDispatch } = getContext("grid")
-  let fieldDefinitions = cloneDeep(FIELDS)
 
   export let field
 
+  let fieldDefinitions = cloneDeep(FIELDS)
   let originalName
   let linkEditDisabled
   let primaryDisplay
@@ -61,11 +59,10 @@
   let savingColumn
   let deleteColName
   let jsonSchemaModal
-
+  let allowedTypes = []
   let editableColumn = {
     type: "string",
     constraints: fieldDefinitions.STRING.constraints,
-
     // Initial value for column name in other table for linked records
     fieldName: $tables.selected.name,
   }
@@ -83,7 +80,23 @@
       primaryDisplay =
         $tables.selected.primaryDisplay == null ||
         $tables.selected.primaryDisplay === editableColumn.name
+    } else if (!savingColumn) {
+      let highestNumber = 0
+      Object.keys(table.schema).forEach(columnName => {
+        const columnNumber = extractColumnNumber(columnName)
+        if (columnNumber > highestNumber) {
+          highestNumber = columnNumber
+        }
+        return highestNumber
+      })
+
+      if (highestNumber >= 1) {
+        editableColumn.name = `Column 0${highestNumber + 1}`
+      } else {
+        editableColumn.name = "Column 01"
+      }
     }
+    allowedTypes = getAllowedTypes()
   }
 
   $: initialiseField(field, savingColumn)
@@ -182,6 +195,8 @@
         indexes,
       })
       dispatch("updatecolumns")
+      gridDispatch("close-edit-column")
+
       if (
         saveColumn.type === LINK_TYPE &&
         saveColumn.relationshipType === RelationshipType.MANY_TO_MANY
@@ -203,6 +218,7 @@
 
   function cancelEdit() {
     editableColumn.name = originalName
+    gridDispatch("close-edit-column")
   }
 
   async function deleteColumn() {
@@ -214,8 +230,8 @@
         await tables.deleteField(editableColumn)
         notifications.success(`Column ${editableColumn.name} deleted`)
         confirmDeleteDialog.hide()
-        hide()
         dispatch("updatecolumns")
+        gridDispatch("close-edit-column")
       }
     } catch (error) {
       notifications.error(`Error deleting column: ${error.message}`)
@@ -251,14 +267,6 @@
     required = req
   }
 
-  function onChangePrimaryDisplay(e) {
-    const isPrimary = e.detail
-    // primary display is always required
-    if (isPrimary) {
-      editableColumn.constraints.presence = { allowEmpty: false }
-    }
-  }
-
   function openJsonSchemaEditor() {
     jsonSchemaModal.show()
   }
@@ -272,6 +280,11 @@
     deleteColName = ""
   }
 
+  function extractColumnNumber(columnName) {
+    const match = columnName.match(/Column (\d+)/)
+    return match ? parseInt(match[1]) : 0
+  }
+
   function getRelationshipOptions(field) {
     if (!field || !field.tableId) {
       return null
@@ -402,15 +415,8 @@
   }
 </script>
 
-<ModalContent
+<Layout noPadding gap="S">
-  title={originalName ? "Edit Column" : "Create Column"}
-  confirmText="Save Column"
-  onConfirm={saveColumn}
-  onCancel={cancelEdit}
-  disabled={invalid}
->
   <Input
-    label="Name"
     bind:value={editableColumn.name}
     disabled={uneditable ||
       (linkEditDisabled && editableColumn.type === LINK_TYPE)}
@@ -419,12 +425,12 @@
 
   <Select
     disabled={!typeEnabled}
-    label="Type"
     bind:value={editableColumn.type}
     on:change={handleTypeChange}
-    options={getAllowedTypes()}
+    options={allowedTypes}
     getOptionLabel={field => field.name}
     getOptionValue={field => field.type}
+    getOptionIcon={field => field.icon}
     isOptionEnabled={option => {
       if (option.type == AUTO_TYPE) {
         return availableAutoColumnKeys?.length > 0
@@ -433,28 +439,6 @@
     }}
   />
 
-  {#if canBeRequired || canBeDisplay}
-    <div>
-      {#if canBeRequired}
-        <Toggle
-          value={required}
-          on:change={onChangeRequired}
-          disabled={primaryDisplay}
-          thin
-          text="Required"
-        />
-      {/if}
-      {#if canBeDisplay}
-        <Toggle
-          bind:value={primaryDisplay}
-          on:change={onChangePrimaryDisplay}
-          thin
-          text="Use as table display column"
-        />
-      {/if}
-    </div>
-  {/if}
-
   {#if editableColumn.type === "string"}
     <Input
       type="number"
@@ -462,9 +446,9 @@
       bind:value={editableColumn.constraints.length.maximum}
     />
   {:else if editableColumn.type === "options"}
-    <ValuesList
+    <OptionSelectDnD
-      label="Options (one per line)"
+      bind:constraints={editableColumn.constraints}
-      bind:values={editableColumn.constraints.inclusion}
+      bind:optionColors={editableColumn.optionColors}
     />
   {:else if editableColumn.type === "longform"}
     <div>
@@ -480,19 +464,28 @@
       />
     </div>
   {:else if editableColumn.type === "array"}
-    <ValuesList
+    <OptionSelectDnD
-      label="Options (one per line)"
+      bind:constraints={editableColumn.constraints}
-      bind:values={editableColumn.constraints.inclusion}
+      bind:optionColors={editableColumn.optionColors}
     />
   {:else if editableColumn.type === "datetime" && !editableColumn.autocolumn}
-    <DatePicker
+    <div class="split-label">
-      label="Earliest"
+      <div class="label-length">
-      bind:value={editableColumn.constraints.datetime.earliest}
+        <Label size="M">Earliest</Label>
-    />
+      </div>
-    <DatePicker
+      <div class="input-length">
-      label="Latest"
+        <DatePicker bind:value={editableColumn.constraints.datetime.earliest} />
-      bind:value={editableColumn.constraints.datetime.latest}
+      </div>
-    />
+    </div>
+
+    <div class="split-label">
+      <div class="label-length">
+        <Label size="M">Latest</Label>
+      </div>
+      <div class="input-length">
+        <DatePicker bind:value={editableColumn.constraints.datetime.latest} />
+      </div>
+    </div>
     {#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER"}
       <div>
         <Label
@@ -509,16 +502,30 @@
       </div>
     {/if}
   {:else if editableColumn.type === "number" && !editableColumn.autocolumn}
-    <Input
+    <div class="split-label">
-      type="number"
+      <div class="label-length">
-      label="Min Value"
+        <Label size="M">Max Value</Label>
-      bind:value={editableColumn.constraints.numericality.greaterThanOrEqualTo}
+      </div>
-    />
+      <div class="input-length">
-    <Input
+        <Input
-      type="number"
+          type="number"
-      label="Max Value"
+          bind:value={editableColumn.constraints.numericality
-      bind:value={editableColumn.constraints.numericality.lessThanOrEqualTo}
+            .greaterThanOrEqualTo}
-    />
+        />
+      </div>
+    </div>
+
+    <div class="split-label">
+      <div class="label-length">
+        <Label size="M">Max Value</Label>
+      </div>
+      <div class="input-length">
+        <Input
+          type="number"
+          bind:value={editableColumn.constraints.numericality.lessThanOrEqualTo}
+        />
+      </div>
+    </div>
   {:else if editableColumn.type === "link"}
     <Select
       label="Table"
@@ -547,32 +554,44 @@
     />
   {:else if editableColumn.type === FORMULA_TYPE}
     {#if !table.sql}
-      <Select
+      <div class="split-label">
-        label="Formula type"
+        <div class="label-length">
-        bind:value={editableColumn.formulaType}
+          <Label size="M">Formula Type</Label>
-        options={[
+        </div>
-          { label: "Dynamic", value: "dynamic" },
+        <div class="input-length">
-          { label: "Static", value: "static" },
+          <Select
-        ]}
+            bind:value={editableColumn.formulaType}
-        getOptionLabel={option => option.label}
+            options={[
-        getOptionValue={option => option.value}
+              { label: "Dynamic", value: "dynamic" },
-        tooltip="Dynamic formula are calculated when retrieved, but cannot be filtered or sorted by,
+              { label: "Static", value: "static" },
+            ]}
+            getOptionLabel={option => option.label}
+            getOptionValue={option => option.value}
+            tooltip="Dynamic formula are calculated when retrieved, but cannot be filtered or sorted by,
          while static formula are calculated when the row is saved."
-      />
+          />
+        </div>
+      </div>
     {/if}
-    <ModalBindableInput
+    <div class="split-label">
-      title="Formula"
+      <div class="label-length">
-      label="Formula"
+        <Label size="M">Formula</Label>
-      value={editableColumn.formula}
+      </div>
-      on:change={e => {
+      <div class="input-length">
-        editableColumn = {
+        <ModalBindableInput
-          ...editableColumn,
+          title="Formula"
-          formula: e.detail,
+          value={editableColumn.formula}
-        }
+          on:change={e => {
-      }}
+            editableColumn = {
-      bindings={getBindings({ table })}
+              ...editableColumn,
-      allowJS
+              formula: e.detail,
-    />
+            }
+          }}
+          bindings={getBindings({ table })}
+          allowJS
+        />
+      </div>
+    </div>
   {:else if editableColumn.type === JSON_TYPE}
     <Button primary text on:click={openJsonSchemaEditor}
       >Open schema editor</Button
@@ -591,12 +610,28 @@
     />
   {/if}
 
-  <div slot="footer">
+  {#if canBeRequired || canBeDisplay}
-    {#if !uneditable && originalName != null}
+    <div>
-      <Button warning text on:click={confirmDelete}>Delete</Button>
+      {#if canBeRequired}
-    {/if}
+        <Toggle
-  </div>
+          value={required}
-</ModalContent>
+          on:change={onChangeRequired}
+          disabled={primaryDisplay}
+          thin
+          text="Required"
+        />
+      {/if}
+    </div>
+  {/if}
+</Layout>
+
+<div class="action-buttons">
+  {#if !uneditable && originalName != null}
+    <Button quiet warning text on:click={confirmDelete}>Delete</Button>
+  {/if}
+  <Button secondary newStyles on:click={cancelEdit}>Cancel</Button>
+  <Button disabled={invalid} newStyles cta on:click={saveColumn}>Save</Button>
+</div>
 <Modal bind:this={jsonSchemaModal}>
   <JSONSchemaModal
     schema={editableColumn.schema}
@@ -607,6 +642,7 @@
     }}
   />
 </Modal>
+
 <ConfirmDialog
   bind:this={confirmDeleteDialog}
   okText="Delete Column"
@@ -622,3 +658,24 @@
   </p>
   <Input bind:value={deleteColName} placeholder={originalName} />
 </ConfirmDialog>
+
+<style>
+  .action-buttons {
+    display: flex;
+    justify-content: flex-end;
+    margin-top: var(--spacing-s);
+    gap: var(--spacing-l);
+  }
+  .split-label {
+    display: flex;
+    align-items: center;
+  }
+
+  .label-length {
+    flex-basis: 40%;
+  }
+
+  .input-length {
+    flex-grow: 1;
+  }
+</style>

packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte

@@ -1,15 +1,8 @@
 <script>
-  import { getContext, onMount } from "svelte"
+  import { getContext } from "svelte"
-  import { Modal } from "@budibase/bbui"
   import CreateEditColumn from "components/backend/DataTable/modals/CreateEditColumn.svelte"
 
-  const { rows, subscribe } = getContext("grid")
+  const { rows } = getContext("grid")
-
-  let modal
-
-  onMount(() => subscribe("add-column", modal.show))
 </script>
 
-<Modal bind:this={modal}>
+<CreateEditColumn on:updatecolumns={rows.actions.refreshTableDefinition} />
-  <CreateEditColumn on:updatecolumns={rows.actions.refreshTableDefinition} />
-</Modal>

packages/builder/src/components/backend/DataTable/modals/grid/GridEditColumnModal.svelte

@@ -1,24 +1,19 @@
 <script>
   import { getContext, onMount } from "svelte"
-  import { Modal } from "@budibase/bbui"
   import CreateEditColumn from "../CreateEditColumn.svelte"
 
   const { rows, subscribe } = getContext("grid")
 
   let editableColumn
-  let editColumnModal
 
   const editColumn = column => {
     editableColumn = column
-    editColumnModal.show()
   }
 
   onMount(() => subscribe("edit-column", editColumn))
 </script>
 
-<Modal bind:this={editColumnModal}>
+<CreateEditColumn
-  <CreateEditColumn
+  field={editableColumn}
-    field={editableColumn}
+  on:updatecolumns={rows.actions.refreshData}
-    on:updatecolumns={rows.actions.refreshData}
+/>
-  />
-</Modal>

packages/builder/src/components/start/AppRow.svelte

@@ -1,16 +1,21 @@
 <script>
   import { Heading, Body, Button, Icon } from "@budibase/bbui"
   import { processStringSync } from "@budibase/string-templates"
+  import { auth } from "stores/portal"
   import { goto } from "@roxi/routify"
   import { UserAvatars } from "@budibase/frontend-core"
+  import { sdk } from "@budibase/shared-core"
 
   export let app
   export let lockedAction
 
   $: editing = app.sessions?.length
+  $: isBuilder = sdk.users.isBuilder($auth.user, app?.devId)
 
   const handleDefaultClick = () => {
-    if (window.innerWidth < 640) {
+    if (!isBuilder) {
+      goToApp()
+    } else if (window.innerWidth < 640) {
       goToOverview()
     } else {
       goToBuilder()
@@ -24,6 +29,10 @@
   const goToOverview = () => {
     $goto(`../../app/${app.devId}/settings`)
   }
+
+  const goToApp = () => {
+    window.open(`/app/${app.name}`, "_blank")
+  }
 </script>
 
 <div class="app-row" on:click={lockedAction || handleDefaultClick}>
@@ -39,7 +48,7 @@
   </div>
 
   <div class="updated">
-    {#if editing}
+    {#if editing && isBuilder}
       Currently editing
       <UserAvatars users={app.sessions} />
     {:else if app.updatedAt}
@@ -56,14 +65,21 @@
     <Body size="S">{app.deployed ? "Published" : "Unpublished"}</Body>
   </div>
 
-  <div class="app-row-actions">
+  {#if isBuilder}
-    <Button size="S" secondary on:click={lockedAction || goToOverview}>
+    <div class="app-row-actions">
-      Manage
+      <Button size="S" secondary on:click={lockedAction || goToOverview}>
-    </Button>
+        Manage
-    <Button size="S" primary on:click={lockedAction || goToBuilder}>
+      </Button>
-      Edit
+      <Button size="S" primary on:click={lockedAction || goToBuilder}>
-    </Button>
+        Edit
-  </div>
+      </Button>
+    </div>
+  {:else}
+    <!-- this can happen if an app builder has app user access to an app -->
+    <div class="app-row-actions">
+      <Button size="S" secondary>View</Button>
+    </div>
+  {/if}
 </div>
 
 <style>

packages/builder/src/constants/backend/index.js

@@ -2,6 +2,7 @@ export const FIELDS = {
   STRING: {
     name: "Text",
     type: "string",
+    icon: "Text",
     constraints: {
       type: "string",
       length: {},
@@ -11,6 +12,7 @@ export const FIELDS = {
   BARCODEQR: {
     name: "Barcode/QR",
     type: "barcodeqr",
+    icon: "Camera",
     constraints: {
       type: "string",
       length: {},
@@ -20,6 +22,7 @@ export const FIELDS = {
   LONGFORM: {
     name: "Long Form Text",
     type: "longform",
+    icon: "TextAlignLeft",
     constraints: {
       type: "string",
       length: {},
@@ -29,6 +32,7 @@ export const FIELDS = {
   OPTIONS: {
     name: "Options",
     type: "options",
+    icon: "Dropdown",
     constraints: {
       type: "string",
       presence: false,
@@ -38,6 +42,7 @@ export const FIELDS = {
   ARRAY: {
     name: "Multi-select",
     type: "array",
+    icon: "Duplicate",
     constraints: {
       type: "array",
       presence: false,
@@ -47,6 +52,7 @@ export const FIELDS = {
   NUMBER: {
     name: "Number",
     type: "number",
+    icon: "123",
     constraints: {
       type: "number",
       presence: false,
@@ -56,10 +62,12 @@ export const FIELDS = {
   BIGINT: {
     name: "BigInt",
     type: "bigint",
+    icon: "TagBold",
   },
   BOOLEAN: {
     name: "Boolean",
     type: "boolean",
+    icon: "Boolean",
     constraints: {
       type: "boolean",
       presence: false,
@@ -68,6 +76,7 @@ export const FIELDS = {
   DATETIME: {
     name: "Date/Time",
     type: "datetime",
+    icon: "Calendar",
     constraints: {
       type: "string",
       length: {},
@@ -81,6 +90,7 @@ export const FIELDS = {
   ATTACHMENT: {
     name: "Attachment",
     type: "attachment",
+    icon: "Folder",
     constraints: {
       type: "array",
       presence: false,
@@ -89,6 +99,7 @@ export const FIELDS = {
   LINK: {
     name: "Relationship",
     type: "link",
+    icon: "Link",
     constraints: {
       type: "array",
       presence: false,
@@ -97,11 +108,13 @@ export const FIELDS = {
   FORMULA: {
     name: "Formula",
     type: "formula",
+    icon: "Calculator",
     constraints: {},
   },
   JSON: {
     name: "JSON",
     type: "json",
+    icon: "Brackets",
     constraints: {
       type: "object",
       presence: false,

packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte

@@ -12,12 +12,12 @@
   } from "@budibase/bbui"
   import { store } from "builderStore"
   import { groups, licensing, apps, users, auth, admin } from "stores/portal"
-  import { fetchData } from "@budibase/frontend-core"
+  import { fetchData, Constants, Utils } from "@budibase/frontend-core"
+  import { sdk } from "@budibase/shared-core"
   import { API } from "api"
   import GroupIcon from "../../../portal/users/groups/_components/GroupIcon.svelte"
   import RoleSelect from "components/common/RoleSelect.svelte"
   import UpgradeModal from "components/common/users/UpgradeModal.svelte"
-  import { Constants, Utils } from "@budibase/frontend-core"
   import { emailValidator } from "helpers/validation"
   import { roles } from "stores/backend"
   import { fly } from "svelte/transition"
@@ -108,9 +108,9 @@
     await usersFetch.refresh()
 
     filteredUsers = $usersFetch.rows.map(user => {
-      const isBuilderOrAdmin = user.admin?.global || user.builder?.global
+      const isAdminOrBuilder = sdk.users.isAdminOrBuilder(user, prodAppId)
       let role = undefined
-      if (isBuilderOrAdmin) {
+      if (isAdminOrBuilder) {
         role = Constants.Roles.ADMIN
       } else {
         const appRole = Object.keys(user.roles).find(x => x === prodAppId)
@@ -122,7 +122,7 @@
       return {
         ...user,
         role,
-        isBuilderOrAdmin,
+        isAdminOrBuilder,
       }
     })
   }
@@ -258,7 +258,7 @@
     }
     // Must exclude users who have explicit privileges
     const userByEmail = filteredUsers.reduce((acc, user) => {
-      if (user.role || user.admin?.global || user.builder?.global) {
+      if (user.role || sdk.users.isAdminOrBuilder(user, prodAppId)) {
         acc.push(user.email)
       }
       return acc
@@ -389,9 +389,9 @@
   }
 
   const userTitle = user => {
-    if (user.admin?.global) {
+    if (sdk.users.isAdmin(user)) {
       return "Admin"
-    } else if (user.builder?.global) {
+    } else if (sdk.users.isBuilder(user, prodAppId)) {
       return "Developer"
     } else {
       return "App user"
@@ -403,7 +403,7 @@
       const role = $roles.find(role => role._id === user.role)
       return `This user has been given ${role?.name} access from the ${user.group} group`
     }
-    if (user.isBuilderOrAdmin) {
+    if (user.isAdminOrBuilder) {
       return "This user's role grants admin access to all apps"
     }
     return null
@@ -614,7 +614,7 @@
                     }}
                     autoWidth
                     align="right"
-                    allowedRoles={user.isBuilderOrAdmin
+                    allowedRoles={user.isAdminOrBuilder
                       ? [Constants.Roles.ADMIN]
                       : null}
                   />

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json

@@ -75,6 +75,14 @@
   {
     "name": "Chart",
     "icon": "GraphBarVertical",
-    "children": ["bar", "line", "area", "candlestick", "pie", "donut"]
+    "children": [
+      "bar",
+      "line",
+      "area",
+      "candlestick",
+      "pie",
+      "donut",
+      "histogram"
+    ]
   }
 ]

packages/builder/src/pages/builder/app/[application]/settings/_layout.svelte

@@ -4,6 +4,8 @@
   import { url, isActive } from "@roxi/routify"
   import DeleteModal from "components/deploy/DeleteModal.svelte"
   import { isOnlyUser } from "builderStore"
+  import { auth } from "stores/portal"
+  import { sdk } from "@budibase/shared-core"
 
   let deleteModal
 </script>
@@ -44,22 +46,24 @@
             url={$url("./version")}
             active={$isActive("./version")}
           />
-          <div class="delete-action">
+          {#if sdk.users.isGlobalBuilder($auth.user)}
-            <AbsTooltip
+            <div class="delete-action">
-              position={TooltipPosition.Bottom}
+              <AbsTooltip
-              text={$isOnlyUser
+                position={TooltipPosition.Bottom}
-                ? null
+                text={$isOnlyUser
-                : "Unavailable - another user is editing this app"}
+                  ? null
-            >
+                  : "Unavailable - another user is editing this app"}
-              <SideNavItem
+              >
-                text="Delete app"
+                <SideNavItem
-                disabled={!$isOnlyUser}
+                  text="Delete app"
-                on:click={() => {
+                  disabled={!$isOnlyUser}
-                  deleteModal.show()
+                  on:click={() => {
-                }}
+                    deleteModal.show()
-              />
+                  }}
-            </AbsTooltip>
+                />
-          </div>
+              </AbsTooltip>
+            </div>
+          {/if}
         </SideNav>
         <slot />
       </Content>

packages/builder/src/pages/builder/apps/index.svelte

@@ -22,53 +22,39 @@
   import Spaceman from "assets/bb-space-man.svg"
   import Logo from "assets/bb-emblem.svg"
   import { UserAvatar } from "@budibase/frontend-core"
-  import { helpers } from "@budibase/shared-core"
+  import { helpers, sdk } from "@budibase/shared-core"
 
   let loaded = false
   let userInfoModal
   let changePasswordModal
 
-  onMount(async () => {
-    try {
-      await organisation.init()
-      await apps.load()
-      await groups.actions.init()
-    } catch (error) {
-      notifications.error("Error loading apps")
-    }
-    loaded = true
-  })
-  const publishedAppsOnly = app => app.status === AppStatus.DEPLOYED
-
   $: userGroups = $groups.filter(group =>
     group.users.find(user => user._id === $auth.user?._id)
   )
-  let userApps = []
+  $: publishedApps = $apps.filter(app => app.status === AppStatus.DEPLOYED)
-  $: publishedApps = $apps.filter(publishedAppsOnly)
+  $: userApps = getUserApps(publishedApps, userGroups, $auth.user)
 
-  $: {
+  function getUserApps(publishedApps, userGroups, user) {
-    if (!Object.keys($auth.user?.roles).length && $auth.user?.userGroups) {
+    if (sdk.users.isAdmin(user)) {
-      userApps =
+      return publishedApps
-        $auth.user?.builder?.global || $auth.user?.admin?.global
-          ? publishedApps
-          : publishedApps.filter(app => {
-              return userGroups.find(group => {
-                return groups.actions
-                  .getGroupAppIds(group)
-                  .map(role => apps.extractAppId(role))
-                  .includes(app.appId)
-              })
-            })
-    } else {
-      userApps =
-        $auth.user?.builder?.global || $auth.user?.admin?.global
-          ? publishedApps
-          : publishedApps.filter(app =>
-              Object.keys($auth.user?.roles)
-                .map(x => apps.extractAppId(x))
-                .includes(app.appId)
-            )
     }
+    return publishedApps.filter(app => {
+      if (sdk.users.isBuilder(user, app.appId)) {
+        return true
+      }
+      if (!Object.keys(user?.roles).length && user?.userGroups) {
+        return userGroups.find(group => {
+          return groups.actions
+            .getGroupAppIds(group)
+            .map(role => apps.extractAppId(role))
+            .includes(app.appId)
+        })
+      } else {
+        return Object.keys($auth.user?.roles)
+          .map(x => apps.extractAppId(x))
+          .includes(app.appId)
+      }
+    })
   }
 
   function getUrl(app) {
@@ -86,6 +72,17 @@
       // Swallow error and do nothing
     }
   }
+
+  onMount(async () => {
+    try {
+      await organisation.init()
+      await apps.load()
+      await groups.actions.init()
+    } catch (error) {
+      notifications.error("Error loading apps")
+    }
+    loaded = true
+  })
 </script>
 
 {#if $auth.user && loaded}
@@ -109,7 +106,7 @@
               >
                 Update password
               </MenuItem>
-              {#if $auth.isBuilder}
+              {#if sdk.users.hasBuilderPermissions($auth.user)}
                 <MenuItem
                   icon="UserDeveloper"
                   on:click={() => $goto("../portal")}

packages/builder/src/pages/builder/index.svelte

@@ -1,11 +1,12 @@
 <script>
   import { redirect } from "@roxi/routify"
   import { auth } from "stores/portal"
+  import { sdk } from "@budibase/shared-core"
 
   auth.checkQueryString()
 
   $: {
-    if ($auth.user?.builder?.global) {
+    if (sdk.users.hasBuilderPermissions($auth.user)) {
       $redirect(`./portal`)
     } else if ($auth.user) {
       $redirect(`./apps`)

packages/builder/src/pages/builder/portal/_components/UpgradeButton.svelte

@@ -3,6 +3,7 @@
   import { goto } from "@roxi/routify"
   import { auth, admin, licensing } from "stores/portal"
   import { isEnabled, TENANT_FEATURE_FLAGS } from "helpers/featureFlags"
+  import { sdk } from "@budibase/shared-core"
 </script>
 
 {#if isEnabled(TENANT_FEATURE_FLAGS.LICENSING) && !$licensing.isEnterprisePlan}
@@ -17,7 +18,7 @@
     >
       Upgrade
     </Button>
-  {:else if !$admin.cloud && $auth.isAdmin}
+  {:else if !$admin.cloud && sdk.users.isAdmin($auth.user)}
     <Button
       cta
       size="S"

packages/builder/src/pages/builder/portal/_layout.svelte

@@ -8,13 +8,14 @@
   import Logo from "./_components/Logo.svelte"
   import UserDropdown from "./_components/UserDropdown.svelte"
   import HelpMenu from "components/common/HelpMenu.svelte"
+  import { sdk } from "@budibase/shared-core"
 
   let loaded = false
   let mobileMenuVisible = false
   let activeTab = "Apps"
 
   $: $url(), updateActiveTab($menu)
-  $: fullscreen = !$apps.length
+  $: isOnboarding = !$apps.length && sdk.users.isGlobalBuilder($auth.user)
 
   const updateActiveTab = menu => {
     for (let entry of menu) {
@@ -33,7 +34,7 @@
   onMount(async () => {
     // Prevent non-builders from accessing the portal
     if ($auth.user) {
-      if (!$auth.user?.builder?.global) {
+      if (!sdk.users.hasBuilderPermissions($auth.user)) {
         $redirect("../")
       } else {
         try {
@@ -49,7 +50,7 @@
 </script>
 
 {#if $auth.user && loaded}
-  {#if fullscreen}
+  {#if isOnboarding}
     <slot />
   {:else}
     <HelpMenu />

packages/builder/src/pages/builder/portal/account/upgrade.svelte

@@ -19,6 +19,7 @@
   import DeleteLicenseKeyModal from "../../../../components/portal/licensing/DeleteLicenseKeyModal.svelte"
   import { API } from "api"
   import { onMount } from "svelte"
+  import { sdk } from "@budibase/shared-core"
 
   $: license = $auth.user.license
   $: upgradeUrl = `${$admin.accountPortalUrl}/portal/upgrade`
@@ -176,7 +177,7 @@
   })
 </script>
 
-{#if $auth.isAdmin}
+{#if sdk.users.isAdmin($auth.user)}
   <DeleteLicenseKeyModal
     bind:this={deleteLicenseKeyModal}
     onConfirm={deleteLicenseKey}

packages/builder/src/pages/builder/portal/account/usage.svelte

@@ -14,6 +14,7 @@
   import { Constants } from "@budibase/frontend-core"
   import { DashCard, Usage } from "components/usage"
   import { PlanModel } from "constants"
+  import { sdk } from "@budibase/shared-core"
 
   let staticUsage = []
   let monthlyUsage = []
@@ -51,7 +52,8 @@
   $: accountPortalAccess = $auth?.user?.accountPortalAccess
   $: quotaReset = quotaUsage?.quotaReset
   $: canManagePlan =
-    ($admin.cloud && accountPortalAccess) || (!$admin.cloud && $auth.isAdmin)
+    ($admin.cloud && accountPortalAccess) ||
+    (!$admin.cloud && sdk.users.isAdmin($auth.user))
 
   $: showButton = !usesInvoicing && accountPortalAccess
 

packages/builder/src/pages/builder/portal/apps/_layout.svelte

@@ -1,11 +1,19 @@
 <script>
   import { notifications } from "@budibase/bbui"
-  import { admin, apps, templates, licensing, groups } from "stores/portal"
+  import {
+    admin,
+    apps,
+    templates,
+    licensing,
+    groups,
+    auth,
+  } from "stores/portal"
   import { onMount } from "svelte"
   import { redirect } from "@roxi/routify"
+  import { sdk } from "@budibase/shared-core"
 
   // Don't block loading if we've already hydrated state
-  let loaded = $apps.length > 0
+  let loaded = $apps.length != null
 
   onMount(async () => {
     try {
@@ -25,7 +33,7 @@
       }
 
       // Go to new app page if no apps exists
-      if (!$apps.length) {
+      if (!$apps.length && sdk.users.isGlobalBuilder($auth.user)) {
         $redirect("./onboarding")
       }
     } catch (error) {

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -15,6 +15,7 @@
   import CreateAppModal from "components/start/CreateAppModal.svelte"
   import AppLimitModal from "components/portal/licensing/AppLimitModal.svelte"
   import AccountLockedModal from "components/portal/licensing/AccountLockedModal.svelte"
+  import { sdk } from "@budibase/shared-core"
 
   import { store, automationStore } from "builderStore"
   import { API } from "api"
@@ -203,40 +204,40 @@
   })
 </script>
 
-{#if $apps.length}
+<Page>
-  <Page>
+  <Layout noPadding gap="L">
-    <Layout noPadding gap="L">
+    {#each Object.keys(automationErrors || {}) as appId}
-      {#each Object.keys(automationErrors || {}) as appId}
+      <Notification
-        <Notification
+        wide
-          wide
+        dismissable
-          dismissable
+        action={() => goToAutomationError(appId)}
-          action={() => goToAutomationError(appId)}
+        type="error"
-          type="error"
+        icon="Alert"
-          icon="Alert"
+        actionMessage={errorCount(automationErrors[appId]) > 1
-          actionMessage={errorCount(automationErrors[appId]) > 1
+          ? "View errors"
-            ? "View errors"
+          : "View error"}
-            : "View error"}
+        on:dismiss={async () => {
-          on:dismiss={async () => {
+          await automationStore.actions.clearLogErrors({ appId })
-            await automationStore.actions.clearLogErrors({ appId })
+          await apps.load()
-            await apps.load()
+        }}
-          }}
+        message={automationErrorMessage(appId)}
-          message={automationErrorMessage(appId)}
+      />
-        />
+    {/each}
-      {/each}
+    <div class="title">
-      <div class="title">
+      <div class="welcome">
-        <div class="welcome">
+        <Layout noPadding gap="XS">
-          <Layout noPadding gap="XS">
+          <Heading size="L">{welcomeHeader}</Heading>
-            <Heading size="L">{welcomeHeader}</Heading>
+          <Body size="M">
-            <Body size="M">
+            Below you'll find the list of apps that you have access to
-              Manage your apps and get a head start with templates
+          </Body>
-            </Body>
+        </Layout>
-          </Layout>
-        </div>
       </div>
+    </div>
 
-      {#if enrichedApps.length}
+    {#if enrichedApps.length}
-        <Layout noPadding gap="L">
+      <Layout noPadding gap="L">
-          <div class="title">
+        <div class="title">
+          {#if $auth.user && sdk.users.isGlobalBuilder($auth.user)}
             <div class="buttons">
               <Button
                 size="M"
@@ -266,41 +267,46 @@
                 </Button>
               {/if}
             </div>
-            {#if enrichedApps.length > 1}
+          {/if}
-              <div class="app-actions">
+          {#if enrichedApps.length > 1}
-                <Select
+            <div class="app-actions">
-                  autoWidth
+              <Select
-                  bind:value={sortBy}
+                autoWidth
-                  placeholder={null}
+                bind:value={sortBy}
-                  options={[
+                placeholder={null}
-                    { label: "Sort by name", value: "name" },
+                options={[
-                    { label: "Sort by recently updated", value: "updated" },
+                  { label: "Sort by name", value: "name" },
-                    { label: "Sort by status", value: "status" },
+                  { label: "Sort by recently updated", value: "updated" },
-                  ]}
+                  { label: "Sort by status", value: "status" },
-                />
+                ]}
-                <Search placeholder="Search" bind:value={searchTerm} />
+              />
-              </div>
+              <Search placeholder="Search" bind:value={searchTerm} />
-            {/if}
+            </div>
-          </div>
+          {/if}
-
-          <div class="app-table">
-            {#each filteredApps as app (app.appId)}
-              <AppRow {app} lockedAction={usersLimitLockAction} />
-            {/each}
-          </div>
-        </Layout>
-      {/if}
-
-      {#if creatingFromTemplate}
-        <div class="empty-wrapper">
-          <img class="img-logo img-size" alt="logo" src={Logo} />
-          <p>Creating your Budibase app from your selected template...</p>
-          <Spinner size="10" />
         </div>
-      {/if}
+
-    </Layout>
+        <div class="app-table">
-  </Page>
+          {#each filteredApps as app (app.appId)}
-{/if}
+            <AppRow {app} lockedAction={usersLimitLockAction} />
+          {/each}
+        </div>
+      </Layout>
+    {:else}
+      <div class="no-apps">
+        <img class="spaceman" alt="spaceman" src={Logo} width="100px" />
+        <Body weight="700">You haven't been given access to any apps yet</Body>
+      </div>
+    {/if}
+
+    {#if creatingFromTemplate}
+      <div class="empty-wrapper">
+        <img class="img-logo img-size" alt="logo" src={Logo} />
+        <p>Creating your Budibase app from your selected template...</p>
+        <Spinner size="10" />
+      </div>
+    {/if}
+  </Layout>
+</Page>
 
 <Modal
   bind:this={creationModal}
@@ -368,6 +374,16 @@
     height: 160px;
   }
 
+  .no-apps {
+    background-color: var(--spectrum-global-color-gray-100);
+    padding: calc(var(--spacing-xl) * 2);
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    flex-direction: column;
+    gap: var(--spacing-xl);
+  }
+
   @media (max-width: 1000px) {
     .img-logo {
       display: none;

packages/builder/src/pages/builder/portal/settings/branding.svelte

@@ -18,6 +18,7 @@
   import { API } from "api"
   import { onMount } from "svelte"
   import { goto } from "@roxi/routify"
+  import { sdk } from "@budibase/shared-core"
 
   const imageExtensions = [
     ".png",
@@ -206,7 +207,7 @@
   })
 </script>
 
-{#if $auth.isAdmin && mounted}
+{#if sdk.users.isAdmin($auth.user) && mounted}
   <Layout noPadding>
     <Layout gap="XS" noPadding>
       <div class="title">
@@ -400,7 +401,7 @@
           on:click={() => {
             if (isCloud && $auth?.user?.accountPortalAccess) {
               window.open($admin.accountPortalUrl + "/portal/upgrade", "_blank")
-            } else if ($auth.isAdmin) {
+            } else if (sdk.users.isAdmin($auth.user)) {
               $goto("/builder/portal/account/upgrade")
             }
           }}

packages/builder/src/pages/builder/portal/settings/diagnostics.svelte

@@ -13,6 +13,7 @@
   import { redirect } from "@roxi/routify"
   import { API } from "api"
   import { onMount } from "svelte"
+  import { sdk } from "@budibase/shared-core"
 
   let diagnosticInfo = ""
 
@@ -46,7 +47,7 @@
   })
 </script>
 
-{#if $auth.isAdmin && diagnosticInfo}
+{#if sdk.users.isAdmin($auth.user) && diagnosticInfo}
   <Layout noPadding>
     <Layout gap="XS">
       <Heading size="M">Diagnostics</Heading>

packages/builder/src/pages/builder/portal/settings/organisation.svelte

@@ -13,10 +13,11 @@
   import { auth, organisation, admin } from "stores/portal"
   import { writable } from "svelte/store"
   import { redirect } from "@roxi/routify"
+  import { sdk } from "@budibase/shared-core"
 
   // Only admins allowed here
   $: {
-    if (!$auth.isAdmin) {
+    if (!sdk.users.isAdmin($auth.user)) {
       $redirect("../../portal")
     }
   }
@@ -50,7 +51,7 @@
   }
 </script>
 
-{#if $auth.isAdmin}
+{#if sdk.users.isAdmin($auth.user)}
   <Layout noPadding>
     <Layout gap="XS" noPadding>
       <Heading size="M">Organisation</Heading>

packages/builder/src/pages/builder/portal/settings/version.svelte

@@ -14,6 +14,7 @@
   import { API } from "api"
   import { auth, admin } from "stores/portal"
   import { redirect } from "@roxi/routify"
+  import { sdk } from "@budibase/shared-core"
 
   let version
   let loaded = false
@@ -25,7 +26,7 @@
 
   // Only admins allowed here
   $: {
-    if (!$auth.isAdmin || $admin.cloud) {
+    if (!sdk.users.isAdmin($auth.user) || $admin.cloud) {
       $redirect("../../portal")
     }
   }
@@ -89,7 +90,7 @@
   })
 </script>
 
-{#if $auth.isAdmin}
+{#if sdk.users.isAdmin($auth.user)}
   <Layout noPadding>
     <Layout gap="XS" noPadding>
       <Heading size="M">Version</Heading>

packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte

@@ -20,6 +20,7 @@
   import CreateEditGroupModal from "./_components/CreateEditGroupModal.svelte"
   import GroupIcon from "./_components/GroupIcon.svelte"
   import GroupUsers from "./_components/GroupUsers.svelte"
+  import { sdk } from "@budibase/shared-core"
 
   export let groupId
 
@@ -46,7 +47,7 @@
   let editModal, deleteModal
 
   $: scimEnabled = $features.isScimEnabled
-  $: readonly = !$auth.isAdmin || scimEnabled
+  $: readonly = !sdk.users.isAdmin($auth.user) || scimEnabled
   $: group = $groups.find(x => x._id === groupId)
   $: groupApps = $apps
     .filter(app =>

packages/builder/src/pages/builder/portal/users/groups/_components/EditUserPicker.svelte

@@ -3,6 +3,7 @@
   import UserGroupPicker from "components/settings/UserGroupPicker.svelte"
   import { createPaginationStore } from "helpers/pagination"
   import { auth, groups, users } from "stores/portal"
+  import { sdk } from "@budibase/shared-core"
 
   export let groupId
   export let onUsersUpdated
@@ -13,7 +14,7 @@
   let prevSearch = undefined
   let pageInfo = createPaginationStore()
 
-  $: readonly = !$auth.isAdmin
+  $: readonly = !sdk.users.isAdmin($auth.user)
   $: page = $pageInfo.page
   $: searchUsers(page, searchTerm)
   $: group = $groups.find(x => x._id === groupId)

packages/builder/src/pages/builder/portal/users/groups/_components/GroupUsers.svelte

@@ -9,6 +9,7 @@
   import { setContext } from "svelte"
   import ScimBanner from "../../_components/SCIMBanner.svelte"
   import RemoveUserTableRenderer from "../_components/RemoveUserTableRenderer.svelte"
+  import { sdk } from "@budibase/shared-core"
 
   export let groupId
 
@@ -49,7 +50,7 @@
   ]
 
   $: scimEnabled = $features.isScimEnabled
-  $: readonly = !$auth.isAdmin || scimEnabled
+  $: readonly = !sdk.users.isAdmin($auth.user) || scimEnabled
 
   const removeUser = async id => {
     await groups.actions.removeUser(groupId, id)

packages/builder/src/pages/builder/portal/users/groups/_components/RemoveUserTableRenderer.svelte

@@ -2,6 +2,7 @@
   import { ActionButton } from "@budibase/bbui"
   import { getContext } from "svelte"
   import { auth } from "stores/portal"
+  import { sdk } from "@budibase/shared-core"
 
   export let value
 
@@ -13,6 +14,10 @@
   }
 </script>
 
-<ActionButton disabled={!$auth.isAdmin} size="S" on:click={onClick}>
+<ActionButton
+  disabled={!sdk.users.isAdmin($auth.user)}
+  size="S"
+  on:click={onClick}
+>
   Remove
 </ActionButton>

packages/builder/src/pages/builder/portal/users/groups/index.svelte

@@ -22,6 +22,7 @@
   import GroupNameTableRenderer from "./_components/GroupNameTableRenderer.svelte"
   import { goto } from "@roxi/routify"
   import ScimBanner from "../_components/SCIMBanner.svelte"
+  import { sdk } from "@budibase/shared-core"
 
   const DefaultGroup = {
     name: "",
@@ -40,7 +41,7 @@
     { column: "roles", component: GroupAppsTableRenderer },
   ]
 
-  $: readonly = !$auth.isAdmin
+  $: readonly = !sdk.users.isAdmin($auth.user)
   $: schema = {
     name: { displayName: "Group", width: "2fr", minWidth: "200px" },
     users: { sortable: false, width: "1fr" },

packages/builder/src/pages/builder/portal/users/users/[userId].svelte

@@ -31,6 +31,7 @@
   import AppNameTableRenderer from "./_components/AppNameTableRenderer.svelte"
   import AppRoleTableRenderer from "./_components/AppRoleTableRenderer.svelte"
   import ScimBanner from "../_components/SCIMBanner.svelte"
+  import { sdk } from "@budibase/shared-core"
 
   export let userId
 
@@ -87,8 +88,8 @@
 
   $: scimEnabled = $features.isScimEnabled
   $: isSSO = !!user?.provider
-  $: readonly = !$auth.isAdmin || scimEnabled
+  $: readonly = !sdk.users.isAdmin($auth.user) || scimEnabled
-  $: privileged = user?.admin?.global || user?.builder?.global
+  $: privileged = sdk.users.isAdminOrBuilder(user)
   $: nameLabel = getNameLabel(user)
   $: filteredGroups = getFilteredGroups($groups, searchTerm)
   $: availableApps = getAvailableApps($apps, privileged, user?.roles)
@@ -97,9 +98,9 @@
       return y._id === userId
     })
   })
-  $: globalRole = user?.admin?.global
+  $: globalRole = sdk.users.isAdmin(user)
     ? "admin"
-    : user?.builder?.global
+    : sdk.users.isBuilder(user)
     ? "developer"
     : "appUser"
 
@@ -285,7 +286,7 @@
           <div class="field">
             <Label size="L">Role</Label>
             <Select
-              disabled={!$auth.isAdmin}
+              disabled={!sdk.users.isAdmin($auth.user)}
               value={globalRole}
               options={Constants.BudibaseRoleOptions}
               on:change={updateUserRole}

packages/builder/src/pages/builder/portal/users/users/_components/AppsTableRenderer.svelte

@@ -1,11 +1,12 @@
 <script>
   import { Icon } from "@budibase/bbui"
   import { apps } from "stores/portal"
+  import { sdk } from "@budibase/shared-core"
 
   export let value
   export let row
 
-  $: priviliged = row?.admin?.global || row?.builder?.global
+  $: priviliged = sdk.users.isAdminOrBuilder(row)
   $: count = priviliged ? $apps.length : value?.length || 0
 </script>
 

packages/builder/src/pages/builder/portal/users/users/_components/RemoveGroupTableRenderer.svelte

@@ -2,6 +2,7 @@
   import { ActionButton } from "@budibase/bbui"
   import { getContext } from "svelte"
   import { auth } from "stores/portal"
+  import { sdk } from "@budibase/shared-core"
 
   export let value
 
@@ -13,6 +14,10 @@
   }
 </script>
 
-<ActionButton disabled={!$auth.isAdmin} size="S" on:click={onClick}>
+<ActionButton
+  disabled={!sdk.users.isAdmin($auth.user)}
+  size="S"
+  on:click={onClick}
+>
   Remove
 </ActionButton>

packages/builder/src/pages/builder/portal/users/users/_components/UpdateRolesModal.svelte

@@ -2,6 +2,7 @@
   import { createEventDispatcher } from "svelte"
   import { Body, Select, ModalContent, notifications } from "@budibase/bbui"
   import { users } from "stores/portal"
+  import { sdk } from "@budibase/shared-core"
 
   export let app
   export let user
@@ -15,7 +16,7 @@
     .filter(role => role._id !== "PUBLIC")
     .map(role => ({ value: role._id, label: role.name }))
 
-  if (!user?.builder?.global) {
+  if (!sdk.users.isBuilder(user, app?.appId)) {
     options.push({ value: NO_ACCESS, label: "No Access" })
   }
   let selectedRole = user?.roles?.[app?._id]

packages/builder/src/pages/builder/portal/users/users/index.svelte

@@ -39,6 +39,7 @@
   import { API } from "api"
   import { OnboardingType } from "../../../../../constants"
   import ScimBanner from "../_components/SCIMBanner.svelte"
+  import { sdk } from "@budibase/shared-core"
 
   const fetch = fetchData({
     API,
@@ -66,7 +67,7 @@
   let userData = []
 
   $: isOwner = $auth.accountPortalAccess && $admin.cloud
-  $: readonly = !$auth.isAdmin || $features.isScimEnabled
+  $: readonly = !sdk.users.isAdmin($auth.user) || $features.isScimEnabled
 
   $: debouncedUpdateFetch(searchEmail)
   $: schema = {

packages/builder/src/stores/portal/auth.js

@@ -2,6 +2,7 @@ import { derived, writable, get } from "svelte/store"
 import { API } from "api"
 import { admin } from "stores/portal"
 import analytics from "analytics"
+import { sdk } from "@budibase/shared-core"
 
 export function createAuthStore() {
   const auth = writable({
@@ -13,13 +14,6 @@ export function createAuthStore() {
     postLogout: false,
   })
   const store = derived(auth, $store => {
-    let isAdmin = false
-    let isBuilder = false
-    if ($store.user) {
-      const user = $store.user
-      isAdmin = !!user.admin?.global
-      isBuilder = !!user.builder?.global
-    }
     return {
       user: $store.user,
       accountPortalAccess: $store.accountPortalAccess,
@@ -27,8 +21,6 @@ export function createAuthStore() {
       tenantSet: $store.tenantSet,
       loaded: $store.loaded,
       postLogout: $store.postLogout,
-      isAdmin,
-      isBuilder,
       isSSO: !!$store.user?.provider,
     }
   })
@@ -57,8 +49,8 @@ export function createAuthStore() {
               name: user.account?.name,
               user_id: user._id,
               tenant: user.tenantId,
-              admin: user?.admin?.global,
+              admin: sdk.users.isAdmin(user),
-              builder: user?.builder?.global,
+              builder: sdk.users.isBuilder(user),
               "Company size": user.account?.size,
               "Job role": user.account?.profession,
             },

packages/builder/src/stores/portal/menu.js

@@ -2,8 +2,12 @@ import { derived } from "svelte/store"
 import { isEnabled, TENANT_FEATURE_FLAGS } from "helpers/featureFlags"
 import { admin } from "./admin"
 import { auth } from "./auth"
+import { sdk } from "@budibase/shared-core"
 
 export const menu = derived([admin, auth], ([$admin, $auth]) => {
+  const user = $auth?.user
+  const isAdmin = sdk.users.isAdmin(user)
+  const cloud = $admin?.cloud
   // Determine user sub pages
   let userSubPages = [
     {
@@ -24,19 +28,21 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
       title: "Apps",
       href: "/builder/portal/apps",
     },
-    {
+  ]
+  if (sdk.users.isGlobalBuilder(user)) {
+    menu.push({
       title: "Users",
       href: "/builder/portal/users",
       subPages: userSubPages,
-    },
+    })
-    {
+    menu.push({
       title: "Plugins",
       href: "/builder/portal/plugins",
-    },
+    })
-  ]
+  }
 
   // Add settings page for admins
-  if ($auth.isAdmin) {
+  if (isAdmin) {
     let settingsSubPages = [
       {
         title: "Auth",
@@ -59,7 +65,7 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
         href: "/builder/portal/settings/environment",
       },
     ]
-    if (!$admin.cloud) {
+    if (!cloud) {
       settingsSubPages.push({
         title: "Version",
         href: "/builder/portal/settings/version",
@@ -84,38 +90,35 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
         href: "/builder/portal/account/usage",
       },
     ]
-    if ($auth.isAdmin) {
+    if (isAdmin) {
       accountSubPages.push({
         title: "Audit Logs",
         href: "/builder/portal/account/auditLogs",
       })
 
-      if (!$admin.cloud) {
+      if (!cloud) {
         accountSubPages.push({
           title: "System Logs",
           href: "/builder/portal/account/systemLogs",
         })
       }
     }
-    if ($admin.cloud && $auth?.user?.accountPortalAccess) {
+    if (cloud && user?.accountPortalAccess) {
       accountSubPages.push({
         title: "Upgrade",
-        href: $admin.accountPortalUrl + "/portal/upgrade",
+        href: $admin?.accountPortalUrl + "/portal/upgrade",
       })
-    } else if (!$admin.cloud && $auth.isAdmin) {
+    } else if (!cloud && isAdmin) {
       accountSubPages.push({
         title: "Upgrade",
         href: "/builder/portal/account/upgrade",
       })
     }
     // add license check here
-    if (
+    if (user?.accountPortalAccess && user.account.stripeCustomerId) {
-      $auth?.user?.accountPortalAccess &&
-      $auth.user.account.stripeCustomerId
-    ) {
       accountSubPages.push({
         title: "Billing",
-        href: $admin.accountPortalUrl + "/portal/billing",
+        href: $admin?.accountPortalUrl + "/portal/billing",
       })
     }
     menu.push({

packages/builder/src/stores/portal/users.js

@@ -2,6 +2,7 @@ import { writable } from "svelte/store"
 import { API } from "api"
 import { update } from "lodash"
 import { licensing } from "."
+import { sdk } from "@budibase/shared-core"
 
 export function createUsersStore() {
   const { subscribe, set } = writable({})
@@ -111,8 +112,12 @@ export function createUsersStore() {
     return await API.saveUser(user)
   }
 
-  const getUserRole = ({ admin, builder }) =>
+  const getUserRole = user =>
-    admin?.global ? "admin" : builder?.global ? "developer" : "appUser"
+    sdk.users.isAdmin(user)
+      ? "admin"
+      : sdk.users.isBuilder(user)
+      ? "developer"
+      : "appUser"
 
   const refreshUsage =
     fn =>

packages/cli/package.json

@@ -14,6 +14,7 @@
     "tsc": "tsc -p tsconfig.build.json",
     "pkg": "pkg . --out-path build --no-bytecode --public --public-packages \"*\" -C GZip",
     "build": "yarn prebuild && yarn rename && yarn tsc && yarn pkg && yarn postbuild",
+    "check:types": "tsc -p tsconfig.json --noEmit --paths null",
     "postbuild": "rm -rf prebuilds 2> /dev/null"
   },
   "pkg": {
@@ -45,7 +46,7 @@
     "lookpath": "1.1.0",
     "node-fetch": "2.6.7",
     "pkg": "5.8.0",
-    "posthog-node": "1.0.7",
+    "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",
     "pouchdb-replication-stream": "1.2.9",
     "randomstring": "1.1.5",
@@ -70,7 +71,8 @@
         "dependsOn": [
           {
             "projects": [
-              "@budibase/backend-core"
+              "@budibase/backend-core",
+              "@budibase/string-templates"
             ],
             "target": "build"
           }

packages/cli/tsconfig.json

@@ -16,7 +16,6 @@
     "require": ["tsconfig-paths/register"],
     "swc": true
   },
-  "references": [{ "path": "../types" }, { "path": "../backend-core" }],
   "include": ["src/**/*", "package.json"],
   "exclude": ["node_modules", "dist"]
 }

packages/client/manifest.json

@@ -2212,6 +2212,147 @@
       }
     ]
   },
+  "histogram": {
+    "name": "Histogram Chart",
+    "description": "Histogram chart",
+    "icon": "Histogram",
+    "size": {
+      "width": 600,
+      "height": 400
+    },
+    "requiredAncestors": ["dataprovider"],
+    "settings": [
+      {
+        "type": "text",
+        "label": "Title",
+        "key": "title"
+      },
+      {
+        "type": "dataProvider",
+        "label": "Provider",
+        "key": "dataProvider",
+        "required": true
+      },
+      {
+        "type": "field",
+        "label": "Data column",
+        "key": "valueColumn",
+        "dependsOn": "dataProvider",
+        "required": true
+      },
+      {
+        "type": "text",
+        "label": "Y axis label",
+        "key": "yAxisLabel",
+        "defaultValue": "Frequency"
+      },
+      {
+        "type": "text",
+        "label": "X axis label",
+        "key": "xAxisLabel"
+      },
+      {
+        "type": "number",
+        "label": "Bucket count",
+        "key": "bucketCount",
+        "defaultValue": 10,
+        "min": 2
+      },
+      {
+        "type": "boolean",
+        "label": "Data labels",
+        "key": "dataLabels",
+        "defaultValue": false
+      },
+      {
+        "type": "text",
+        "label": "Width",
+        "key": "width"
+      },
+      {
+        "type": "text",
+        "label": "Height",
+        "key": "height",
+        "defaultValue": "400"
+      },
+      {
+        "type": "select",
+        "label": "Colors",
+        "key": "palette",
+        "defaultValue": "Palette 1",
+        "options": [
+          "Custom",
+          "Palette 1",
+          "Palette 2",
+          "Palette 3",
+          "Palette 4",
+          "Palette 5",
+          "Palette 6",
+          "Palette 7",
+          "Palette 8",
+          "Palette 9",
+          "Palette 10"
+        ]
+      },
+      {
+        "type": "color",
+        "label": "C1",
+        "key": "c1",
+        "dependsOn": {
+          "setting": "palette",
+          "value": "Custom"
+        }
+      },
+      {
+        "type": "color",
+        "label": "C2",
+        "key": "c2",
+        "dependsOn": {
+          "setting": "palette",
+          "value": "Custom"
+        }
+      },
+      {
+        "type": "color",
+        "label": "C3",
+        "key": "c3",
+        "dependsOn": {
+          "setting": "palette",
+          "value": "Custom"
+        }
+      },
+      {
+        "type": "color",
+        "label": "C4",
+        "key": "c4",
+        "dependsOn": {
+          "setting": "palette",
+          "value": "Custom"
+        }
+      },
+      {
+        "type": "color",
+        "label": "C5",
+        "key": "c5",
+        "dependsOn": {
+          "setting": "palette",
+          "value": "Custom"
+        }
+      },
+      {
+        "type": "boolean",
+        "label": "Animate",
+        "key": "animate",
+        "defaultValue": true
+      },
+      {
+        "type": "boolean",
+        "label": "Horizontal",
+        "key": "horizontal",
+        "defaultValue": false
+      }
+    ]
+  },
   "form": {
     "name": "Form",
     "icon": "Form",
@@ -3965,6 +4106,10 @@
             "label": "Bar",
             "value": "bar"
           },
+          {
+            "label": "Histogram",
+            "value": "histogram"
+          },
           {
             "label": "Line",
             "value": "line"
@@ -4215,6 +4360,47 @@
           }
         ]
       },
+      {
+        "section": true,
+        "name": "Histogram Chart",
+        "icon": "Histogram",
+        "dependsOn": {
+          "setting": "chartType",
+          "value": "histogram"
+        },
+        "settings": [
+          {
+            "type": "field",
+            "label": "Value column",
+            "key": "valueColumn",
+            "dependsOn": "dataSource",
+            "required": true
+          },
+          {
+            "type": "text",
+            "label": "Y axis label",
+            "key": "yAxisLabel"
+          },
+          {
+            "type": "text",
+            "label": "X axis label",
+            "key": "xAxisLabel"
+          },
+          {
+            "type": "boolean",
+            "label": "Horizontal",
+            "key": "horizontal",
+            "defaultValue": false
+          },
+          {
+            "type": "number",
+            "label": "Bucket count",
+            "key": "bucketCount",
+            "defaultValue": 10,
+            "min": 2
+          }
+        ]
+      },
       {
         "section": true,
         "name": "Line Chart",
@@ -5179,11 +5365,7 @@
             "type": "boolean",
             "label": "Hide notifications",
             "key": "notificationOverride",
-            "defaultValue": false,
+            "defaultValue": false
-            "dependsOn": {
-              "setting": "showSaveButton",
-              "value": true
-            }
           }
         ]
       },

packages/client/package.json

@@ -70,14 +70,26 @@
         "dependsOn": [
           {
             "projects": [
+              "@budibase/shared-core",
               "@budibase/string-templates",
-              "@budibase/shared-core"
+              "@budibase/types"
+            ],
+            "target": "build"
+          }
+        ]
+      },
+      "dev:builder": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/shared-core",
+              "@budibase/string-templates",
+              "@budibase/types"
             ],
             "target": "build"
           }
         ]
       }
     }
-  },
+  }
-  "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }

packages/client/rollup.config.js

@@ -25,11 +25,11 @@ const devPaths = production
   : [
       {
         find: "@budibase/shared-core",
-        replacement: path.resolve("../shared-core/dist/mjs/src/index"),
+        replacement: path.resolve("../shared-core/dist/index"),
       },
       {
         find: "@budibase/types",
-        replacement: path.resolve("../types/dist/mjs/index"),
+        replacement: path.resolve("../types/dist/index"),
       },
     ]
 

packages/client/src/components/BlockComponent.svelte

@@ -26,9 +26,9 @@
   $: parentId = $component?.id
   $: inBuilder = $builderStore.inBuilder
   $: instance = {
-    _component: `@budibase/standard-components/${type}`,
+    _component: getComponent(type),
     _id: id,
-    _instanceName: name || type[0].toUpperCase() + type.slice(1),
+    _instanceName: getInstanceName(name, type),
     _styles: {
       ...styles,
       normal: styles?.normal || {},
@@ -45,6 +45,30 @@
     }
   }
 
+  const getComponent = type => {
+    if (!type) {
+      return null
+    }
+    if (type.startsWith("plugin/")) {
+      return type
+    } else {
+      return `@budibase/standard-components/${type}`
+    }
+  }
+
+  const getInstanceName = (name, type) => {
+    if (name) {
+      return name
+    }
+    if (!type) {
+      return "New component"
+    }
+    if (type.startsWith("plugin/")) {
+      type = type.split("plugin/")[1]
+    }
+    return type[0].toUpperCase() + type.slice(1)
+  }
+
   onDestroy(() => {
     if (inBuilder) {
       block.unregisterComponent(id, parentId)

packages/client/src/components/app/blocks/ChartBlock.svelte

@@ -46,6 +46,9 @@
   export let lowColumn
   export let dateColumn
 
+  // Histogram
+  export let bucketCount
+
   let dataProviderId
 
   $: colors = c1 && c2 && c3 && c4 && c5 ? [c1, c2, c3, c4, c5] : null
@@ -92,6 +95,7 @@
           highColumn,
           lowColumn,
           dateColumn,
+          bucketCount,
         }}
       />
     {/if}

packages/client/src/components/app/blocks/form/InnerFormBlock.svelte

@@ -83,6 +83,7 @@
         tableId: dataSource?.tableId,
         rowId: `{{ ${safe(repeaterId)}.${safe("_id")} }}`,
         revId: `{{ ${safe(repeaterId)}.${safe("_rev")} }}`,
+        notificationOverride,
       },
     },
     {

packages/client/src/components/app/charts/HistogramChart.svelte

@@ -0,0 +1,136 @@
+<script>
+  import { ApexOptionsBuilder } from "./ApexOptionsBuilder"
+  import ApexChart from "./ApexChart.svelte"
+
+  export let title
+  export let dataProvider
+  export let valueColumn
+  export let xAxisLabel
+  export let yAxisLabel
+  export let height
+  export let width
+  export let dataLabels
+  export let animate
+  export let palette
+  export let c1, c2, c3, c4, c5
+  export let horizontal
+  export let bucketCount = 10
+
+  $: options = setUpChart(
+    title,
+    dataProvider,
+    valueColumn,
+    xAxisLabel || valueColumn,
+    yAxisLabel,
+    height,
+    width,
+    dataLabels,
+    animate,
+    palette,
+    horizontal,
+    c1 && c2 && c3 && c4 && c5 ? [c1, c2, c3, c4, c5] : null,
+    customColor,
+    bucketCount
+  )
+
+  $: customColor = palette === "Custom"
+
+  const setUpChart = (
+    title,
+    dataProvider,
+    valueColumn,
+    xAxisLabel, //freqAxisLabel
+    yAxisLabel, //valueAxisLabel
+    height,
+    width,
+    dataLabels,
+    animate,
+    palette,
+    horizontal,
+    colors,
+    customColor,
+    bucketCount
+  ) => {
+    const allCols = [valueColumn]
+    if (
+      !dataProvider ||
+      !dataProvider.rows?.length ||
+      allCols.find(x => x == null)
+    ) {
+      return null
+    }
+
+    // Fetch data
+    const { schema, rows } = dataProvider
+    const reducer = row => (valid, column) => valid && row[column] != null
+    const hasAllColumns = row => allCols.reduce(reducer(row), true)
+    const data = rows.filter(row => hasAllColumns(row)).slice(0, 100)
+    if (!schema || !data.length) {
+      return null
+    }
+
+    // Initialise default chart
+    let builder = new ApexOptionsBuilder()
+      .type("bar")
+      .title(title)
+      .width(width)
+      .height(height)
+      .xLabel(horizontal ? yAxisLabel : xAxisLabel)
+      .yLabel(horizontal ? xAxisLabel : yAxisLabel)
+      .dataLabels(dataLabels)
+      .animate(animate)
+      .palette(palette)
+      .horizontal(horizontal)
+      .colors(customColor ? colors : null)
+
+    if (horizontal) {
+      builder = builder.setOption(["plotOptions", "bar", "barHeight"], "90%")
+    } else {
+      builder = builder.setOption(["plotOptions", "bar", "columnWidth"], "99%")
+    }
+
+    // Pull occurences of the value.
+    let flatlist = data.map(row => {
+      return row[valueColumn]
+    })
+
+    // Build range buckets
+    let interval = Math.max(...flatlist) / bucketCount
+    let counts = Array(bucketCount).fill(0)
+
+    // Assign row data to a bucket
+    let buckets = flatlist.reduce((acc, val) => {
+      let dest = Math.min(Math.floor(val / interval), bucketCount - 1)
+      acc[dest] = acc[dest] + 1
+      return acc
+    }, counts)
+
+    const rangeLabel = bucketIdx => {
+      return `${Math.floor(interval * bucketIdx)} - ${Math.floor(
+        interval * (bucketIdx + 1)
+      )}`
+    }
+
+    const series = [
+      {
+        name: yAxisLabel,
+        data: Array.from({ length: buckets.length }, (_, i) => ({
+          x: rangeLabel(i),
+          y: buckets[i],
+        })),
+      },
+    ]
+
+    builder = builder.setOption(["xaxis", "labels"], {
+      formatter: x => {
+        return x + ""
+      },
+    })
+
+    builder = builder.series(series)
+
+    return builder.getOptions()
+  }
+</script>
+
+<ApexChart {options} />

packages/client/src/components/app/charts/index.js

@@ -4,3 +4,4 @@ export { default as pie } from "./PieChart.svelte"
 export { default as donut } from "./DonutChart.svelte"
 export { default as area } from "./AreaChart.svelte"
 export { default as candlestick } from "./CandleStickChart.svelte"
+export { default as histogram } from "./HistogramChart.svelte"

packages/client/src/sdk.js

@@ -18,6 +18,8 @@ import { styleable } from "utils/styleable"
 import { linkable } from "utils/linkable"
 import { getAction } from "utils/getAction"
 import Provider from "components/context/Provider.svelte"
+import Block from "components/Block.svelte"
+import BlockComponent from "components/BlockComponent.svelte"
 import { ActionTypes } from "./constants"
 import { fetchDatasourceSchema } from "./utils/schema.js"
 import { getAPIKey } from "./utils/api.js"
@@ -44,4 +46,6 @@ export default {
   Provider,
   ActionTypes,
   getAPIKey,
+  Block,
+  BlockComponent,
 }

packages/frontend-core/src/components/grid/cells/HeaderCell.svelte

@@ -1,7 +1,7 @@
 <script>
-  import { getContext } from "svelte"
+  import { getContext, onMount, tick } from "svelte"
   import GridCell from "./GridCell.svelte"
-  import { Icon, Popover, Menu, MenuItem } from "@budibase/bbui"
+  import { Icon, Popover, Menu, MenuItem, clickOutside } from "@budibase/bbui"
   import { getColumnIcon } from "../lib/utils"
 
   export let column
@@ -16,6 +16,7 @@
     sort,
     renderedColumns,
     dispatch,
+    subscribe,
     config,
     ui,
     columns,
@@ -32,7 +33,9 @@
 
   let anchor
   let open = false
+  let editIsOpen = false
   let timeout
+  let popover
 
   $: sortedBy = column.name === $sort.column
   $: canMoveLeft = orderable && idx > 0
@@ -44,11 +47,16 @@
     ? "high-low"
     : "Z-A"
 
-  const editColumn = () => {
+  const editColumn = async () => {
+    editIsOpen = true
+    await tick()
     dispatch("edit-column", column.schema)
-    open = false
   }
 
+  const cancelEdit = () => {
+    popover.hide()
+    editIsOpen = false
+  }
   const onMouseDown = e => {
     if (e.button === 0 && orderable) {
       timeout = setTimeout(() => {
@@ -109,6 +117,7 @@
     columns.actions.saveChanges()
     open = false
   }
+  onMount(() => subscribe("close-edit-column", cancelEdit))
 </script>
 
 <div
@@ -157,57 +166,74 @@
 
 <Popover
   bind:open
+  bind:this={popover}
   {anchor}
   align="right"
   offset={0}
   popoverTarget={document.getElementById(`grid-${rand}`)}
   animate={false}
+  customZindex={100}
 >
-  <Menu>
+  {#if editIsOpen}
-    <MenuItem
+    <div
-      icon="Edit"
+      use:clickOutside={() => {
-      on:click={editColumn}
+        editIsOpen = false
-      disabled={!$config.allowSchemaChanges || column.schema.disabled}
+      }}
+      class="content"
     >
-      Edit column
+      <slot />
-    </MenuItem>
+    </div>
-    <MenuItem
+  {:else}
-      icon="Label"
+    <Menu>
-      on:click={makeDisplayColumn}
+      <MenuItem
-      disabled={idx === "sticky" ||
+        icon="Edit"
-        !$config.allowSchemaChanges ||
+        on:click={editColumn}
-        bannedDisplayColumnTypes.includes(column.schema.type)}
+        disabled={!$config.allowSchemaChanges || column.schema.disabled}
-    >
+      >
-      Use as display column
+        Edit column
-    </MenuItem>
+      </MenuItem>
-    <MenuItem
+      <MenuItem
-      icon="SortOrderUp"
+        icon="Label"
-      on:click={sortAscending}
+        on:click={makeDisplayColumn}
-      disabled={column.name === $sort.column && $sort.order === "ascending"}
+        disabled={idx === "sticky" ||
-    >
+          !$config.allowSchemaChanges ||
-      Sort {ascendingLabel}
+          bannedDisplayColumnTypes.includes(column.schema.type)}
-    </MenuItem>
+      >
-    <MenuItem
+        Use as display column
-      icon="SortOrderDown"
+      </MenuItem>
-      on:click={sortDescending}
+      <MenuItem
-      disabled={column.name === $sort.column && $sort.order === "descending"}
+        icon="SortOrderUp"
-    >
+        on:click={sortAscending}
-      Sort {descendingLabel}
+        disabled={column.name === $sort.column && $sort.order === "ascending"}
-    </MenuItem>
+      >
-    <MenuItem disabled={!canMoveLeft} icon="ChevronLeft" on:click={moveLeft}>
+        Sort {ascendingLabel}
-      Move left
+      </MenuItem>
-    </MenuItem>
+      <MenuItem
-    <MenuItem disabled={!canMoveRight} icon="ChevronRight" on:click={moveRight}>
+        icon="SortOrderDown"
-      Move right
+        on:click={sortDescending}
-    </MenuItem>
+        disabled={column.name === $sort.column && $sort.order === "descending"}
-    <MenuItem
+      >
-      disabled={idx === "sticky" || !$config.showControls}
+        Sort {descendingLabel}
-      icon="VisibilityOff"
+      </MenuItem>
-      on:click={hideColumn}
+      <MenuItem disabled={!canMoveLeft} icon="ChevronLeft" on:click={moveLeft}>
-    >
+        Move left
-      Hide column
+      </MenuItem>
-    </MenuItem>
+      <MenuItem
-  </Menu>
+        disabled={!canMoveRight}
+        icon="ChevronRight"
+        on:click={moveRight}
+      >
+        Move right
+      </MenuItem>
+      <MenuItem
+        disabled={idx === "sticky" || !$config.showControls}
+        icon="VisibilityOff"
+        on:click={hideColumn}
+      >
+        Hide column
+      </MenuItem>
+    </Menu>
+  {/if}
 </Popover>
 
 <style>
@@ -255,4 +281,13 @@
   .header-cell:hover .sort-indicator {
     display: none;
   }
+
+  .content {
+    width: 300px;
+    padding: 20px;
+    display: flex;
+    flex-direction: column;
+    gap: 20px;
+    background: var(--spectrum-alias-background-color-secondary);
+  }
 </style>

packages/frontend-core/src/components/grid/cells/OptionsCell.svelte

@@ -18,6 +18,7 @@
   let focusedOptionIdx = null
 
   $: options = schema?.constraints?.inclusion || []
+  $: optionColors = schema?.optionColors || {}
   $: editable = focused && !readonly
   $: values = Array.isArray(value) ? value : [value].filter(x => x != null)
   $: {
@@ -93,7 +94,7 @@
     on:click={editable ? open : null}
   >
     {#each values as val}
-      {@const color = getOptionColor(val)}
+      {@const color = optionColors[val] || getOptionColor(val)}
       {#if color}
         <div class="badge text" style="--color: {color}">
           <span>
@@ -121,7 +122,7 @@
       use:clickOutside={close}
     >
       {#each options as option, idx}
-        {@const color = getOptionColor(option)}
+        {@const color = optionColors[option] || getOptionColor(option)}
         <div
           class="option"
           on:click={() => toggleOption(option)}

packages/frontend-core/src/components/grid/layout/Grid.svelte

@@ -139,9 +139,20 @@
   {#if $loaded}
     <div class="grid-data-outer" use:clickOutside={ui.actions.blur}>
       <div class="grid-data-inner">
-        <StickyColumn />
+        <StickyColumn>
+          <svelte:fragment slot="edit-column">
+            <slot name="edit-column" />
+          </svelte:fragment>
+        </StickyColumn>
         <div class="grid-data-content">
-          <HeaderRow />
+          <HeaderRow>
+            <svelte:fragment slot="add-column">
+              <slot name="add-column" />
+            </svelte:fragment>
+            <svelte:fragment slot="edit-column">
+              <slot name="edit-column" />
+            </svelte:fragment>
+          </HeaderRow>
           <GridBody />
         </div>
         {#if $canAddRows}

packages/frontend-core/src/components/grid/layout/HeaderRow.svelte

@@ -1,34 +1,22 @@
 <script>
+  import NewColumnButton from "./NewColumnButton.svelte"
+
   import { getContext } from "svelte"
   import GridScrollWrapper from "./GridScrollWrapper.svelte"
   import HeaderCell from "../cells/HeaderCell.svelte"
-  import { Icon, TempTooltip, TooltipType } from "@budibase/bbui"
+  import { TempTooltip, TooltipType } from "@budibase/bbui"
 
-  const {
+  const { renderedColumns, config, hasNonAutoColumn, tableId, loading } =
-    renderedColumns,
+    getContext("grid")
-    dispatch,
-    scroll,
-    hiddenColumnsWidth,
-    width,
-    config,
-    hasNonAutoColumn,
-    tableId,
-    loading,
-  } = getContext("grid")
-
-  $: columnsWidth = $renderedColumns.reduce(
-    (total, col) => total + col.width,
-    0
-  )
-  $: end = $hiddenColumnsWidth + columnsWidth - 1 - $scroll.left
-  $: left = Math.min($width - 40, end)
 </script>
 
 <div class="header">
   <GridScrollWrapper scrollHorizontally>
     <div class="row">
       {#each $renderedColumns as column, idx}
-        <HeaderCell {column} {idx} />
+        <HeaderCell {column} {idx}>
+          <slot name="edit-column" />
+        </HeaderCell>
       {/each}
     </div>
   </GridScrollWrapper>
@@ -39,13 +27,9 @@
         type={TooltipType.Info}
         condition={!$hasNonAutoColumn && !$loading}
       >
-        <div
+        <NewColumnButton>
-          class="add"
+          <slot name="add-column" />
-          style="left:{left}px;"
+        </NewColumnButton>
-          on:click={() => dispatch("add-column")}
-        >
-          <Icon name="Add" />
-        </div>
       </TempTooltip>
     {/key}
   {/if}
@@ -61,21 +45,4 @@
   .row {
     display: flex;
   }
-  .add {
-    height: var(--default-row-height);
-    display: grid;
-    place-items: center;
-    width: 40px;
-    position: absolute;
-    top: 0;
-    border-left: var(--cell-border);
-    border-right: var(--cell-border);
-    border-bottom: var(--cell-border);
-    background: var(--grid-background-alt);
-    z-index: 1;
-  }
-  .add:hover {
-    background: var(--spectrum-global-color-gray-200);
-    cursor: pointer;
-  }
 </style>

packages/frontend-core/src/components/grid/layout/NewColumnButton.svelte

@@ -0,0 +1,79 @@
+<script>
+  import { getContext, onMount } from "svelte"
+  import { Icon, Popover, clickOutside } from "@budibase/bbui"
+
+  const { renderedColumns, scroll, hiddenColumnsWidth, width, subscribe } =
+    getContext("grid")
+
+  let anchor
+  let open = false
+  $: columnsWidth = $renderedColumns.reduce(
+    (total, col) => (total += col.width),
+    0
+  )
+  $: end = $hiddenColumnsWidth + columnsWidth - 1 - $scroll.left
+  $: left = Math.min($width - 40, end)
+
+  const close = () => {
+    open = false
+  }
+  onMount(() => subscribe("close-edit-column", close))
+</script>
+
+<div
+  id="add-column-button"
+  bind:this={anchor}
+  class="add"
+  style="left:{left}px"
+  on:click={() => (open = true)}
+>
+  <Icon name="Add" />
+</div>
+<Popover
+  bind:open
+  {anchor}
+  align="right"
+  offset={0}
+  popoverTarget={document.getElementById(`add-column-button`)}
+  animate={false}
+  customZindex={100}
+>
+  <div
+    use:clickOutside={() => {
+      open = false
+    }}
+    class="content"
+  >
+    <slot />
+  </div>
+</Popover>
+
+<style>
+  .add {
+    height: var(--default-row-height);
+    display: grid;
+    place-items: center;
+    width: 40px;
+    position: absolute;
+    top: 0;
+    border-left: var(--cell-border);
+    border-right: var(--cell-border);
+    border-bottom: var(--cell-border);
+    background: var(--grid-background-alt);
+    z-index: 1;
+  }
+  .add:hover {
+    background: var(--spectrum-global-color-gray-200);
+    cursor: pointer;
+  }
+
+  .content {
+    width: 300px;
+    padding: 20px;
+    display: flex;
+    flex-direction: column;
+    gap: 20px;
+    z-index: 2;
+    background: var(--spectrum-alias-background-color-secondary);
+  }
+</style>

packages/frontend-core/src/components/grid/layout/StickyColumn.svelte

@@ -57,7 +57,9 @@
       disabled={!$renderedRows.length}
     />
     {#if $stickyColumn}
-      <HeaderCell column={$stickyColumn} orderable={false} idx="sticky" />
+      <HeaderCell column={$stickyColumn} orderable={false} idx="sticky">
+        <slot name="edit-column" />
+      </HeaderCell>
     {/if}
   </div>
 

packages/pro

@@ -1 +1 @@
-Subproject commit 63fa1b15f6e2afa8a264d597157fd798c9ce031c
+Subproject commit cf3bef2aad9c739111b306fd0712397adc363f81

packages/server/nodemon.json

@@ -1,10 +1,13 @@
 {
-  "watch": ["src", "../backend-core", "../pro"],
+  "watch": [
-  "ext": "js,ts,json",
+    "src",
-  "ignore": [
+    "../backend-core",
-    "src/**/*.spec.ts",
+    "../pro",
-    "src/**/*.spec.js",
+    "../types",
-    "../backend-core/dist/**/*"
+    "../shared-core",
+    "../string-templates"
   ],
+  "ext": "js,ts,json",
+  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../*/dist/**/*"],
   "exec": "yarn build && node ./dist/index.js"
 }

packages/server/package.json

@@ -11,7 +11,7 @@
   "scripts": {
     "prebuild": "rimraf dist/",
     "build": "node ./scripts/build.js",
-    "check:types": "tsc -p tsconfig.build.json --noEmit",
+    "check:types": "tsc -p tsconfig.json --noEmit --paths null",
     "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
@@ -179,31 +179,5 @@
   },
   "optionalDependencies": {
     "oracledb": "5.3.0"
-  },
+  }
-  "nx": {
-    "targets": {
-      "dev:builder": {
-        "dependsOn": [
-          {
-            "projects": [
-              "@budibase/backend-core"
-            ],
-            "target": "build"
-          }
-        ]
-      },
-      "test": {
-        "dependsOn": [
-          {
-            "projects": [
-              "@budibase/string-templates",
-              "@budibase/shared-core"
-            ],
-            "target": "build"
-          }
-        ]
-      }
-    }
-  },
-  "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }

packages/server/scripts/integrations/postgres/init.sql

@@ -1,6 +1,6 @@
 SELECT 'CREATE DATABASE main'
 WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
-CREATE SCHEMA test;
+CREATE SCHEMA "test-1";
 CREATE TYPE person_job AS ENUM ('qa', 'programmer', 'designer');
 CREATE TABLE Persons (
     PersonID SERIAL PRIMARY KEY,
@@ -39,7 +39,7 @@ CREATE TABLE Products_Tasks (
             REFERENCES Tasks(TaskID),
     PRIMARY KEY (ProductID, TaskID)
 );
-CREATE TABLE test.table1 (
+CREATE TABLE "test-1".table1 (
   id SERIAL PRIMARY KEY,
   Name varchar(255)
 );
@@ -60,7 +60,7 @@ INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (1, 1);
 INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (2, 1);
 INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (3, 1);
 INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (1, 2);
-INSERT INTO test.table1 (Name) VALUES ('Test');
+INSERT INTO "test-1".table1 (Name) VALUES ('Test');
 INSERT INTO CompositeTable (KeyPartOne, KeyPartTwo, Name) VALUES ('aaa', 'bbb', 'Michael');
 INSERT INTO CompositeTable (KeyPartOne, KeyPartTwo, Name) VALUES ('bbb', 'ccc', 'Andrew');
 INSERT INTO CompositeTable (KeyPartOne, KeyPartTwo, Name) VALUES ('ddd', '', 'OneKey');

packages/server/scripts/test.sh

@@ -5,8 +5,8 @@ if [[ -n $CI ]]
 then
   # --runInBand performs better in ci where resources are limited
   export NODE_OPTIONS="--max-old-space-size=4096"
-  echo "jest --coverage --runInBand --forceExit"
+  echo "jest --coverage --runInBand --forceExit --bail"
-  jest --coverage --runInBand --forceExit
+  jest --coverage --runInBand --forceExit --bail
 else
   # --maxWorkers performs better in development
   echo "jest --coverage --maxWorkers=2 --forceExit"

packages/server/src/api/controllers/application.ts

@@ -30,6 +30,7 @@ import {
   objectStore,
   roles,
   tenancy,
+  users,
 } from "@budibase/backend-core"
 import { USERS_TABLE_SCHEMA } from "../../constants"
 import {
@@ -49,8 +50,8 @@ import {
   MigrationType,
   PlanType,
   Screen,
-  SocketSession,
   UserCtx,
+  ContextUser,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import sdk from "../../sdk"
@@ -177,32 +178,10 @@ export const addSampleData = async (ctx: UserCtx) => {
 }
 
 export async function fetch(ctx: UserCtx) {
-  const dev = ctx.query && ctx.query.status === AppStatus.DEV
+  ctx.body = await sdk.applications.fetch(
-  const all = ctx.query && ctx.query.status === AppStatus.ALL
+    ctx.query.status as AppStatus,
-  const apps = (await dbCore.getAllApps({ dev, all })) as App[]
+    ctx.user
-
+  )
-  const appIds = apps
-    .filter((app: any) => app.status === "development")
-    .map((app: any) => app.appId)
-
-  // get the locks for all the dev apps
-  if (dev || all) {
-    const locks = await getLocksById(appIds)
-    for (let app of apps) {
-      const lock = locks[app.appId]
-      if (lock) {
-        app.lockedBy = lock
-      } else {
-        // make sure its definitely not present
-        delete app.lockedBy
-      }
-    }
-  }
-
-  // Enrich apps with all builder user sessions
-  const enrichedApps = await sdk.users.sessions.enrichApps(apps)
-
-  ctx.body = await checkAppMetadata(enrichedApps)
 }
 
 export async function fetchAppDefinition(ctx: UserCtx) {
@@ -222,6 +201,7 @@ export async function fetchAppDefinition(ctx: UserCtx) {
 
 export async function fetchAppPackage(ctx: UserCtx) {
   const db = context.getAppDB()
+  const appId = context.getAppId()
   let application = await db.get<any>(DocumentType.APP_METADATA)
   const layouts = await getLayouts()
   let screens = await getScreens()
@@ -233,7 +213,7 @@ export async function fetchAppPackage(ctx: UserCtx) {
   )
 
   // Only filter screens if the user is not a builder
-  if (!(ctx.user.builder && ctx.user.builder.global)) {
+  if (!users.isBuilder(ctx.user, appId)) {
     const userRoleId = getUserRoleId(ctx)
     const accessController = new roles.AccessController()
     screens = await accessController.checkScreensAccess(screens, userRoleId)

packages/server/src/api/controllers/row/index.ts

@@ -11,10 +11,6 @@ import {
   Row,
   PatchRowRequest,
   PatchRowResponse,
-  SearchResponse,
-  SortOrder,
-  SortType,
-  ViewV2,
 } from "@budibase/types"
 import * as utils from "./utils"
 import { gridSocket } from "../../../websockets"
@@ -23,6 +19,7 @@ import { fixRow } from "../public/rows"
 import sdk from "../../../sdk"
 import * as exporters from "../view/exporters"
 import { apiFileReturn } from "../../../utilities/fileSystem"
+export * as views from "./views"
 
 function pickApi(tableId: any) {
   if (isExternalTable(tableId)) {
@@ -37,6 +34,7 @@ export async function patch(
   const appId = ctx.appId
   const tableId = utils.getTableId(ctx)
   const body = ctx.request.body
+
   // if it doesn't have an _id then its save
   if (body && !body._id) {
     return save(ctx)
@@ -62,13 +60,14 @@ export async function patch(
   }
 }
 
-export const save = async (ctx: any) => {
+export const save = async (ctx: UserCtx<Row, Row>) => {
   const appId = ctx.appId
   const tableId = utils.getTableId(ctx)
   const body = ctx.request.body
+
   // if it has an ID already then its a patch
   if (body && body._id) {
-    return patch(ctx)
+    return patch(ctx as UserCtx<PatchRowRequest, PatchRowResponse>)
   }
   const { row, table, squashed } = await quotas.addRow(() =>
     quotas.addQuery(() => pickApi(tableId).save(ctx), {
@@ -147,7 +146,7 @@ async function deleteRows(ctx: UserCtx<DeleteRowRequest>) {
   const rowDeletes: Row[] = await processDeleteRowsRequest(ctx)
   deleteRequest.rows = rowDeletes
 
-  let { rows } = await quotas.addQuery<any>(
+  const { rows } = await quotas.addQuery(
     () => pickApi(tableId).bulkDestroy(ctx),
     {
       datasourceId: tableId,
@@ -167,13 +166,13 @@ async function deleteRow(ctx: UserCtx<DeleteRowRequest>) {
   const appId = ctx.appId
   const tableId = utils.getTableId(ctx)
 
-  let resp = await quotas.addQuery<any>(() => pickApi(tableId).destroy(ctx), {
+  const resp = await quotas.addQuery(() => pickApi(tableId).destroy(ctx), {
     datasourceId: tableId,
   })
   await quotas.removeRow()
 
   ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, resp.row)
-  gridSocket?.emitRowDeletion(ctx, resp.row._id)
+  gridSocket?.emitRowDeletion(ctx, resp.row._id!)
 
   return resp
 }
@@ -212,83 +211,6 @@ export async function search(ctx: any) {
   })
 }
 
-function getSortOptions(
-  ctx: Ctx,
-  view: ViewV2
-):
-  | {
-      sort: string
-      sortOrder?: SortOrder
-      sortType?: SortType
-    }
-  | undefined {
-  const { sort_column, sort_order, sort_type } = ctx.query
-  if (Array.isArray(sort_column)) {
-    ctx.throw(400, "sort_column cannot be an array")
-  }
-  if (Array.isArray(sort_order)) {
-    ctx.throw(400, "sort_order cannot be an array")
-  }
-  if (Array.isArray(sort_type)) {
-    ctx.throw(400, "sort_type cannot be an array")
-  }
-
-  if (sort_column) {
-    return {
-      sort: sort_column,
-      sortOrder: sort_order as SortOrder,
-      sortType: sort_type as SortType,
-    }
-  }
-  if (view.sort) {
-    return {
-      sort: view.sort.field,
-      sortOrder: view.sort.order,
-      sortType: view.sort.type,
-    }
-  }
-
-  return
-}
-
-export async function searchView(ctx: Ctx<void, SearchResponse>) {
-  const { viewId } = ctx.params
-
-  const view = await sdk.views.get(viewId)
-  if (!view) {
-    ctx.throw(404, `View ${viewId} not found`)
-  }
-
-  if (view.version !== 2) {
-    ctx.throw(400, `This method only supports viewsV2`)
-  }
-
-  const table = await sdk.tables.getTable(view?.tableId)
-
-  const viewFields =
-    (view.columns &&
-      Object.entries(view.columns).length &&
-      Object.keys(sdk.views.enrichSchema(view, table.schema).schema)) ||
-    undefined
-
-  ctx.status = 200
-  const result = await quotas.addQuery(
-    () =>
-      sdk.rows.search({
-        tableId: view.tableId,
-        query: view.query || {},
-        fields: viewFields,
-        ...getSortOptions(ctx, view),
-      }),
-    {
-      datasourceId: view.tableId,
-    }
-  )
-
-  result.rows.forEach(r => (r._viewId = view.id))
-  ctx.body = result
-}
-
 export async function validate(ctx: Ctx) {
   const tableId = utils.getTableId(ctx)
   // external tables are hard to validate currently

packages/server/src/api/controllers/row/internal.ts

@@ -93,7 +93,6 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
 }
 
 export async function save(ctx: UserCtx) {
-  const db = context.getAppDB()
   let inputs = ctx.request.body
   inputs.tableId = ctx.params.tableId
 
@@ -177,7 +176,6 @@ export async function destroy(ctx: UserCtx) {
 }
 
 export async function bulkDestroy(ctx: UserCtx) {
-  const db = context.getAppDB()
   const tableId = ctx.params.tableId
   const table = await sdk.tables.getTable(tableId)
   let { rows } = ctx.request.body
@@ -206,6 +204,7 @@ export async function bulkDestroy(ctx: UserCtx) {
       })
     )
   } else {
+    const db = context.getAppDB()
     await db.bulkDocs(processedRows.map(row => ({ ...row, _deleted: true })))
   }
   // remove any attachments that were on the rows from object storage

packages/server/src/api/controllers/row/views.ts

@@ -0,0 +1,86 @@
+import { quotas } from "@budibase/pro"
+import {
+  UserCtx,
+  SearchResponse,
+  SortOrder,
+  SortType,
+  ViewV2,
+} from "@budibase/types"
+import sdk from "../../../sdk"
+
+export async function searchView(ctx: UserCtx<void, SearchResponse>) {
+  const { viewId } = ctx.params
+
+  const view = await sdk.views.get(viewId)
+  if (!view) {
+    ctx.throw(404, `View ${viewId} not found`)
+  }
+
+  if (view.version !== 2) {
+    ctx.throw(400, `This method only supports viewsV2`)
+  }
+
+  const table = await sdk.tables.getTable(view?.tableId)
+
+  const viewFields =
+    (view.columns &&
+      Object.entries(view.columns).length &&
+      Object.keys(sdk.views.enrichSchema(view, table.schema).schema)) ||
+    undefined
+
+  ctx.status = 200
+  const result = await quotas.addQuery(
+    () =>
+      sdk.rows.search({
+        tableId: view.tableId,
+        query: view.query || {},
+        fields: viewFields,
+        ...getSortOptions(ctx, view),
+      }),
+    {
+      datasourceId: view.tableId,
+    }
+  )
+
+  result.rows.forEach(r => (r._viewId = view.id))
+  ctx.body = result
+}
+
+function getSortOptions(
+  ctx: UserCtx,
+  view: ViewV2
+):
+  | {
+      sort: string
+      sortOrder?: SortOrder
+      sortType?: SortType
+    }
+  | undefined {
+  const { sort_column, sort_order, sort_type } = ctx.query
+  if (Array.isArray(sort_column)) {
+    ctx.throw(400, "sort_column cannot be an array")
+  }
+  if (Array.isArray(sort_order)) {
+    ctx.throw(400, "sort_order cannot be an array")
+  }
+  if (Array.isArray(sort_type)) {
+    ctx.throw(400, "sort_type cannot be an array")
+  }
+
+  if (sort_column) {
+    return {
+      sort: sort_column,
+      sortOrder: sort_order as SortOrder,
+      sortType: sort_type as SortType,
+    }
+  }
+  if (view.sort) {
+    return {
+      sort: view.sort.field,
+      sortOrder: view.sort.order,
+      sortType: view.sort.type,
+    }
+  }
+
+  return
+}

packages/server/src/api/controllers/view/viewsV2.ts

@@ -2,15 +2,86 @@ import sdk from "../../../sdk"
 import {
   CreateViewRequest,
   Ctx,
+  UIFieldMetadata,
   UpdateViewRequest,
   ViewResponse,
+  ViewV2,
+  RequiredKeys,
 } from "@budibase/types"
 
+async function parseSchemaUI(ctx: Ctx, view: CreateViewRequest) {
+  if (!view.schema) {
+    return
+  }
+
+  function hasOverrides(
+    newObj: Record<string, any>,
+    existingObj: Record<string, any>
+  ) {
+    const result = Object.entries(newObj).some(([key, value]) => {
+      const isObject = typeof value === "object"
+      const existing = existingObj[key]
+      if (isObject && hasOverrides(value, existing || {})) {
+        return true
+      }
+      if (!isObject && value !== existing) {
+        return true
+      }
+    })
+
+    return result
+  }
+
+  const table = await sdk.tables.getTable(view.tableId)
+  for (const [
+    fieldName,
+    { order, width, visible, icon, ...schemaNonUI },
+  ] of Object.entries(view.schema)) {
+    const overrides = hasOverrides(schemaNonUI, table.schema[fieldName])
+    if (overrides) {
+      ctx.throw(
+        400,
+        "This endpoint does not support overriding non UI fields in the schema"
+      )
+    }
+  }
+
+  const schemaUI =
+    view.schema &&
+    Object.entries(view.schema).reduce((p, [fieldName, schemaValue]) => {
+      const fieldSchema: RequiredKeys<UIFieldMetadata> = {
+        order: schemaValue.order,
+        width: schemaValue.width,
+        visible: schemaValue.visible,
+        icon: schemaValue.icon,
+      }
+      Object.entries(fieldSchema)
+        .filter(([_, val]) => val === undefined)
+        .forEach(([key]) => {
+          delete fieldSchema[key as keyof UIFieldMetadata]
+        })
+      p[fieldName] = fieldSchema
+      return p
+    }, {} as Record<string, RequiredKeys<UIFieldMetadata>>)
+  return schemaUI
+}
+
 export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
   const view = ctx.request.body
   const { tableId } = view
 
-  const result = await sdk.views.create(tableId, view)
+  const schemaUI = await parseSchemaUI(ctx, view)
+
+  const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version"> = {
+    name: view.name,
+    tableId: view.tableId,
+    query: view.query,
+    sort: view.sort,
+    columns: view.schema && Object.keys(view.schema),
+    schemaUI,
+    primaryDisplay: view.primaryDisplay,
+  }
+  const result = await sdk.views.create(tableId, parsedView)
   ctx.status = 201
   ctx.body = {
     data: result,
@@ -30,7 +101,20 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
 
   const { tableId } = view
 
-  const result = await sdk.views.update(tableId, view)
+  const schemaUI = await parseSchemaUI(ctx, view)
+  const parsedView: RequiredKeys<ViewV2> = {
+    id: view.id,
+    name: view.name,
+    version: view.version,
+    tableId: view.tableId,
+    query: view.query,
+    sort: view.sort,
+    columns: view.schema && Object.keys(view.schema),
+    schemaUI,
+    primaryDisplay: view.primaryDisplay,
+  }
+
+  const result = await sdk.views.update(tableId, parsedView)
   ctx.body = {
     data: result,
   }