Merge branch 'cheeks-lab-day-binding-eval' of github.com:Budibase/budibase into cheeks-snippets-poc

.eslintrc.json

@@ -44,7 +44,8 @@
         "no-undef": "off",
         "no-prototype-builtins": "off",
         "local-rules/no-budibase-imports": "error",
-        "local-rules/no-test-com": "error"
+        "local-rules/no-test-com": "error",
+        "local-rules/email-domain-example-com": "error"
       }
     },
     {

eslint-local-rules/index.js

@@ -51,4 +51,41 @@ module.exports = {
       }
     },
   },
+  "email-domain-example-com": {
+    meta: {
+      type: "problem",
+      docs: {
+        description:
+          "enforce using the example.com domain for generator.email calls",
+        category: "Possible Errors",
+        recommended: false,
+      },
+      fixable: "code",
+      schema: [],
+    },
+    create: function (context) {
+      return {
+        CallExpression(node) {
+          if (
+            node.callee.type === "MemberExpression" &&
+            node.callee.object.name === "generator" &&
+            node.callee.property.name === "email" &&
+            node.arguments.length === 0
+          ) {
+            context.report({
+              node,
+              message:
+                "Prefer using generator.email with the domain \"{ domain: 'example.com' }\".",
+              fix: function (fixer) {
+                return fixer.replaceText(
+                  node,
+                  'generator.email({ domain: "example.com" })'
+                )
+              },
+            })
+          }
+        },
+      }
+    },
+  },
 }

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.21.2",
+  "version": "2.21.6",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/package.json

@@ -67,7 +67,7 @@
     "@types/lodash": "4.14.200",
     "@types/node-fetch": "2.6.4",
     "@types/pouchdb": "6.4.0",
-    "@types/redlock": "4.0.3",
+    "@types/redlock": "4.0.7",
     "@types/semver": "7.3.7",
     "@types/tar-fs": "2.0.1",
     "@types/uuid": "8.3.4",
@@ -78,6 +78,7 @@
     "jest-serial-runner": "1.2.1",
     "pino-pretty": "10.0.0",
     "pouchdb-adapter-memory": "7.2.2",
+    "testcontainers": "^10.7.2",
     "timekeeper": "2.2.0",
     "typescript": "5.2.2"
   },

packages/backend-core/src/cache/base/index.ts

@@ -23,6 +23,18 @@ export default class BaseCache {
     return client.keys(pattern)
   }
 
+  async exists(key: string, opts = { useTenancy: true }) {
+    key = opts.useTenancy ? generateTenantKey(key) : key
+    const client = await this.getClient()
+    return client.exists(key)
+  }
+
+  async scan(key: string, opts = { useTenancy: true }) {
+    key = opts.useTenancy ? generateTenantKey(key) : key
+    const client = await this.getClient()
+    return client.scan(key)
+  }
+
   /**
    * Read only from the cache.
    */
@@ -32,6 +44,15 @@ export default class BaseCache {
     return client.get(key)
   }
 
+  /**
+   * Read only from the cache.
+   */
+  async bulkGet<T>(keys: string[], opts = { useTenancy: true }) {
+    keys = opts.useTenancy ? keys.map(key => generateTenantKey(key)) : keys
+    const client = await this.getClient()
+    return client.bulkGet<T>(keys)
+  }
+
   /**
    * Write to the cache.
    */
@@ -46,6 +67,25 @@ export default class BaseCache {
     await client.store(key, value, ttl)
   }
 
+  /**
+   * Bulk write to the cache.
+   */
+  async bulkStore(
+    data: Record<string, any>,
+    ttl: number | null = null,
+    opts = { useTenancy: true }
+  ) {
+    if (opts.useTenancy) {
+      data = Object.entries(data).reduce((acc, [key, value]) => {
+        acc[generateTenantKey(key)] = value
+        return acc
+      }, {} as Record<string, any>)
+    }
+
+    const client = await this.getClient()
+    await client.bulkStore(data, ttl)
+  }
+
   /**
    * Remove from cache.
    */
@@ -55,15 +95,24 @@ export default class BaseCache {
     return client.delete(key)
   }
 
+  /**
+   * Remove from cache.
+   */
+  async bulkDelete(keys: string[], opts = { useTenancy: true }) {
+    keys = opts.useTenancy ? keys.map(key => generateTenantKey(key)) : keys
+    const client = await this.getClient()
+    return client.bulkDelete(keys)
+  }
+
   /**
    * Read from the cache. Write to the cache if not exists.
    */
-  async withCache(
+  async withCache<T>(
     key: string,
-    ttl: number,
+    ttl: number | null = null,
-    fetchFn: any,
+    fetchFn: () => Promise<T> | T,
     opts = { useTenancy: true }
-  ) {
+  ): Promise<T> {
     const cachedValue = await this.get(key, opts)
     if (cachedValue) {
       return cachedValue
@@ -89,4 +138,13 @@ export default class BaseCache {
       throw err
     }
   }
+
+  /**
+   * Delete the entry if the provided value matches the stored one.
+   */
+  async deleteIfValue(key: string, value: any, opts = { useTenancy: true }) {
+    key = opts.useTenancy ? generateTenantKey(key) : key
+    const client = await this.getClient()
+    await client.deleteIfValue(key, value)
+  }
 }

packages/backend-core/src/cache/docWritethrough.ts

@@ -0,0 +1,97 @@
+import { AnyDocument, Database } from "@budibase/types"
+
+import { JobQueue, createQueue } from "../queue"
+import * as dbUtils from "../db"
+
+interface ProcessDocMessage {
+  dbName: string
+  docId: string
+  data: Record<string, any>
+}
+
+const PERSIST_MAX_ATTEMPTS = 100
+let processor: DocWritethroughProcessor | undefined
+
+export const docWritethroughProcessorQueue = createQueue<ProcessDocMessage>(
+  JobQueue.DOC_WRITETHROUGH_QUEUE,
+  {
+    jobOptions: {
+      attempts: PERSIST_MAX_ATTEMPTS,
+    },
+  }
+)
+
+class DocWritethroughProcessor {
+  init() {
+    docWritethroughProcessorQueue.process(async message => {
+      try {
+        await this.persistToDb(message.data)
+      } catch (err: any) {
+        if (err.status === 409) {
+          // If we get a 409, it means that another job updated it meanwhile. We want to retry it to persist it again.
+          throw new Error(
+            `Conflict persisting message ${message.id}. Attempt ${message.attemptsMade}`
+          )
+        }
+
+        throw err
+      }
+    })
+    return this
+  }
+
+  private async persistToDb({
+    dbName,
+    docId,
+    data,
+  }: {
+    dbName: string
+    docId: string
+    data: Record<string, any>
+  }) {
+    const db = dbUtils.getDB(dbName)
+    let doc: AnyDocument | undefined
+    try {
+      doc = await db.get(docId)
+    } catch {
+      doc = { _id: docId }
+    }
+
+    doc = { ...doc, ...data }
+    await db.put(doc)
+  }
+}
+
+export class DocWritethrough {
+  private db: Database
+  private _docId: string
+
+  constructor(db: Database, docId: string) {
+    this.db = db
+    this._docId = docId
+  }
+
+  get docId() {
+    return this._docId
+  }
+
+  async patch(data: Record<string, any>) {
+    await docWritethroughProcessorQueue.add({
+      dbName: this.db.name,
+      docId: this.docId,
+      data,
+    })
+  }
+}
+
+export function init(): DocWritethroughProcessor {
+  processor = new DocWritethroughProcessor().init()
+  return processor
+}
+
+export function getProcessor(): DocWritethroughProcessor {
+  if (!processor) {
+    return init()
+  }
+  return processor
+}

packages/backend-core/src/cache/generic.ts

@@ -26,7 +26,8 @@ export const store = (...args: Parameters<typeof GENERIC.store>) =>
   GENERIC.store(...args)
 export const destroy = (...args: Parameters<typeof GENERIC.delete>) =>
   GENERIC.delete(...args)
-export const withCache = (...args: Parameters<typeof GENERIC.withCache>) =>
+export const withCache = <T>(
-  GENERIC.withCache(...args)
+  ...args: Parameters<typeof GENERIC.withCache<T>>
+) => GENERIC.withCache(...args)
 export const bustCache = (...args: Parameters<typeof GENERIC.bustCache>) =>
   GENERIC.bustCache(...args)

packages/backend-core/src/cache/index.ts

@@ -5,3 +5,4 @@ export * as writethrough from "./writethrough"
 export * as invite from "./invite"
 export * as passwordReset from "./passwordReset"
 export * from "./generic"
+export * as docWritethrough from "./docWritethrough"

packages/backend-core/src/cache/tests/docWritethrough.spec.ts

@@ -0,0 +1,293 @@
+import tk from "timekeeper"
+
+import _ from "lodash"
+import { DBTestConfiguration, generator, structures } from "../../../tests"
+import { getDB } from "../../db"
+
+import {
+  DocWritethrough,
+  docWritethroughProcessorQueue,
+  init,
+} from "../docWritethrough"
+
+import InMemoryQueue from "../../queue/inMemoryQueue"
+
+const initialTime = Date.now()
+
+async function waitForQueueCompletion() {
+  const queue: InMemoryQueue = docWritethroughProcessorQueue as never
+  await queue.waitForCompletion()
+}
+
+describe("docWritethrough", () => {
+  beforeAll(() => {
+    init()
+  })
+
+  const config = new DBTestConfiguration()
+
+  const db = getDB(structures.db.id())
+  let documentId: string
+  let docWritethrough: DocWritethrough
+
+  describe("patch", () => {
+    function generatePatchObject(fieldCount: number) {
+      const keys = generator.unique(() => generator.word(), fieldCount)
+      return keys.reduce((acc, c) => {
+        acc[c] = generator.word()
+        return acc
+      }, {} as Record<string, any>)
+    }
+
+    beforeEach(async () => {
+      jest.clearAllMocks()
+      documentId = structures.uuid()
+      docWritethrough = new DocWritethrough(db, documentId)
+    })
+
+    it("patching will not persist until the messages are persisted", async () => {
+      await config.doInTenant(async () => {
+        await docWritethrough.patch(generatePatchObject(2))
+        await docWritethrough.patch(generatePatchObject(2))
+
+        expect(await db.exists(documentId)).toBe(false)
+      })
+    })
+
+    it("patching will persist when the messages are persisted", async () => {
+      await config.doInTenant(async () => {
+        const patch1 = generatePatchObject(2)
+        const patch2 = generatePatchObject(2)
+        await docWritethrough.patch(patch1)
+        await docWritethrough.patch(patch2)
+
+        await waitForQueueCompletion()
+
+        // This will not be persisted
+        const patch3 = generatePatchObject(3)
+        await docWritethrough.patch(patch3)
+
+        expect(await db.get(documentId)).toEqual({
+          _id: documentId,
+          ...patch1,
+          ...patch2,
+          _rev: expect.stringMatching(/2-.+/),
+          createdAt: new Date(initialTime).toISOString(),
+          updatedAt: new Date(initialTime).toISOString(),
+        })
+      })
+    })
+
+    it("patching will persist keeping the previous data", async () => {
+      await config.doInTenant(async () => {
+        const patch1 = generatePatchObject(2)
+        const patch2 = generatePatchObject(2)
+        await docWritethrough.patch(patch1)
+        await docWritethrough.patch(patch2)
+
+        await waitForQueueCompletion()
+
+        const patch3 = generatePatchObject(3)
+        await docWritethrough.patch(patch3)
+
+        await waitForQueueCompletion()
+
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining({
+            _id: documentId,
+            ...patch1,
+            ...patch2,
+            ...patch3,
+          })
+        )
+      })
+    })
+
+    it("date audit fields are set correctly when persisting", async () => {
+      await config.doInTenant(async () => {
+        const patch1 = generatePatchObject(2)
+        const patch2 = generatePatchObject(2)
+        await docWritethrough.patch(patch1)
+        const date1 = new Date()
+        await waitForQueueCompletion()
+        await docWritethrough.patch(patch2)
+
+        tk.travel(Date.now() + 100)
+        const date2 = new Date()
+        await waitForQueueCompletion()
+
+        expect(date1).not.toEqual(date2)
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining({
+            createdAt: date1.toISOString(),
+            updatedAt: date2.toISOString(),
+          })
+        )
+      })
+    })
+
+    it("concurrent patches will override keys", async () => {
+      await config.doInTenant(async () => {
+        const patch1 = generatePatchObject(2)
+        await docWritethrough.patch(patch1)
+        await waitForQueueCompletion()
+        const patch2 = generatePatchObject(1)
+        await docWritethrough.patch(patch2)
+
+        const keyToOverride = _.sample(Object.keys(patch1))!
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining({
+            [keyToOverride]: patch1[keyToOverride],
+          })
+        )
+
+        await waitForQueueCompletion()
+
+        const patch3 = {
+          ...generatePatchObject(3),
+          [keyToOverride]: generator.word(),
+        }
+        await docWritethrough.patch(patch3)
+        await waitForQueueCompletion()
+
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining({
+            ...patch1,
+            ...patch2,
+            ...patch3,
+          })
+        )
+      })
+    })
+
+    it("concurrent patches to different docWritethrough will not pollute each other", async () => {
+      await config.doInTenant(async () => {
+        const secondDocWritethrough = new DocWritethrough(
+          db,
+          structures.db.id()
+        )
+
+        const doc1Patch = generatePatchObject(2)
+        await docWritethrough.patch(doc1Patch)
+        const doc2Patch = generatePatchObject(1)
+        await secondDocWritethrough.patch(doc2Patch)
+
+        await waitForQueueCompletion()
+
+        const doc1Patch2 = generatePatchObject(3)
+        await docWritethrough.patch(doc1Patch2)
+        const doc2Patch2 = generatePatchObject(3)
+        await secondDocWritethrough.patch(doc2Patch2)
+        await waitForQueueCompletion()
+
+        expect(await db.get(docWritethrough.docId)).toEqual(
+          expect.objectContaining({
+            ...doc1Patch,
+            ...doc1Patch2,
+          })
+        )
+
+        expect(await db.get(secondDocWritethrough.docId)).toEqual(
+          expect.objectContaining({
+            ...doc2Patch,
+            ...doc2Patch2,
+          })
+        )
+      })
+    })
+
+    it("cached values are persisted only once", async () => {
+      await config.doInTenant(async () => {
+        const initialPatch = generatePatchObject(5)
+
+        await docWritethrough.patch(initialPatch)
+        await waitForQueueCompletion()
+
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining(initialPatch)
+        )
+
+        await db.remove(await db.get(documentId))
+
+        await waitForQueueCompletion()
+        const extraPatch = generatePatchObject(5)
+        await docWritethrough.patch(extraPatch)
+        await waitForQueueCompletion()
+
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining(extraPatch)
+        )
+        expect(await db.get(documentId)).not.toEqual(
+          expect.objectContaining(initialPatch)
+        )
+      })
+    })
+
+    it("concurrent calls will not cause conflicts", async () => {
+      async function parallelPatch(count: number) {
+        const patches = Array.from({ length: count }).map(() =>
+          generatePatchObject(1)
+        )
+        await Promise.all(patches.map(p => docWritethrough.patch(p)))
+
+        return patches.reduce((acc, c) => {
+          acc = { ...acc, ...c }
+          return acc
+        }, {})
+      }
+      const queueMessageSpy = jest.spyOn(docWritethroughProcessorQueue, "add")
+
+      await config.doInTenant(async () => {
+        let patches = await parallelPatch(5)
+        expect(queueMessageSpy).toBeCalledTimes(5)
+
+        await waitForQueueCompletion()
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining(patches)
+        )
+
+        patches = { ...patches, ...(await parallelPatch(40)) }
+        expect(queueMessageSpy).toBeCalledTimes(45)
+
+        await waitForQueueCompletion()
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining(patches)
+        )
+
+        patches = { ...patches, ...(await parallelPatch(10)) }
+        expect(queueMessageSpy).toBeCalledTimes(55)
+
+        await waitForQueueCompletion()
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining(patches)
+        )
+      })
+    })
+
+    // This is not yet supported
+    it.skip("patches will execute in order", async () => {
+      let incrementalValue = 0
+      const keyToOverride = generator.word()
+      async function incrementalPatches(count: number) {
+        for (let i = 0; i < count; i++) {
+          await docWritethrough.patch({ [keyToOverride]: incrementalValue++ })
+        }
+      }
+
+      await config.doInTenant(async () => {
+        await incrementalPatches(5)
+
+        await waitForQueueCompletion()
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining({ [keyToOverride]: 5 })
+        )
+
+        await incrementalPatches(40)
+        await waitForQueueCompletion()
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining({ [keyToOverride]: 45 })
+        )
+      })
+    })
+  })
+})

packages/backend-core/src/cache/user.ts

@@ -6,7 +6,7 @@ import env from "../environment"
 import * as accounts from "../accounts"
 import { UserDB } from "../users"
 import { sdk } from "@budibase/shared-core"
-import { User } from "@budibase/types"
+import { User, UserMetadata } from "@budibase/types"
 
 const EXPIRY_SECONDS = 3600
 
@@ -15,7 +15,7 @@ const EXPIRY_SECONDS = 3600
  */
 async function populateFromDB(userId: string, tenantId: string) {
   const db = tenancy.getTenantDB(tenantId)
-  const user = await db.get<any>(userId)
+  const user = await db.get<UserMetadata>(userId)
   user.budibaseAccess = true
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(user.email)

packages/backend-core/src/constants/db.ts

@@ -57,6 +57,9 @@ export const StaticDatabases = {
   AUDIT_LOGS: {
     name: "audit-logs",
   },
+  SCIM_LOGS: {
+    name: "scim-logs",
+  },
 }
 
 export const APP_PREFIX = prefixed(DocumentType.APP)

packages/backend-core/src/context/mainContext.ts

@@ -35,6 +35,17 @@ export function getAuditLogDBName(tenantId?: string) {
   }
 }
 
+export function getScimDBName(tenantId?: string) {
+  if (!tenantId) {
+    tenantId = getTenantId()
+  }
+  if (tenantId === DEFAULT_TENANT_ID) {
+    return StaticDatabases.SCIM_LOGS.name
+  } else {
+    return `${tenantId}${SEPARATOR}${StaticDatabases.SCIM_LOGS.name}`
+  }
+}
+
 export function baseGlobalDBName(tenantId: string | undefined | null) {
   if (!tenantId || tenantId === DEFAULT_TENANT_ID) {
     return StaticDatabases.GLOBAL.name

packages/backend-core/src/db/Replication.ts

@@ -1,66 +1,57 @@
+import PouchDB from "pouchdb"
 import { getPouchDB, closePouchDB } from "./couch"
 import { DocumentType } from "../constants"
 
 class Replication {
-  source: any
+  source: PouchDB.Database
-  target: any
+  target: PouchDB.Database
-  replication: any
 
-  /**
+  constructor({ source, target }: { source: string; target: string }) {
-   *
-   * @param source - the DB you want to replicate or rollback to
-   * @param target - the DB you want to replicate to, or rollback from
-   */
-  constructor({ source, target }: any) {
     this.source = getPouchDB(source)
     this.target = getPouchDB(target)
   }
 
-  close() {
+  async close() {
-    return Promise.all([closePouchDB(this.source), closePouchDB(this.target)])
+    await Promise.all([closePouchDB(this.source), closePouchDB(this.target)])
   }
 
-  promisify(operation: any, opts = {}) {
+  replicate(opts: PouchDB.Replication.ReplicateOptions = {}) {
-    return new Promise(resolve => {
+    return new Promise<PouchDB.Replication.ReplicationResult<{}>>(resolve => {
-      operation(this.target, opts)
+      this.source.replicate
-        .on("denied", function (err: any) {
+        .to(this.target, opts)
+        .on("denied", function (err) {
           // a document failed to replicate (e.g. due to permissions)
           throw new Error(`Denied: Document failed to replicate ${err}`)
         })
-        .on("complete", function (info: any) {
+        .on("complete", function (info) {
           return resolve(info)
         })
-        .on("error", function (err: any) {
+        .on("error", function (err) {
           throw new Error(`Replication Error: ${err}`)
         })
     })
   }
 
-  /**
+  appReplicateOpts(
-   * Two way replication operation, intended to be promise based.
+    opts: PouchDB.Replication.ReplicateOptions = {}
-   * @param opts - PouchDB replication options
+  ): PouchDB.Replication.ReplicateOptions {
-   */
+    if (typeof opts.filter === "string") {
-  sync(opts = {}) {
+      return opts
-    this.replication = this.promisify(this.source.sync, opts)
+    }
-    return this.replication
-  }
 
-  /**
+    const filter = opts.filter
-   * One way replication operation, intended to be promise based.
+    delete opts.filter
-   * @param opts - PouchDB replication options
-   */
-  replicate(opts = {}) {
-    this.replication = this.promisify(this.source.replicate.to, opts)
-    return this.replication
-  }
 
-  appReplicateOpts() {
     return {
-      filter: (doc: any) => {
+      ...opts,
+      filter: (doc: any, params: any) => {
         if (doc._id && doc._id.startsWith(DocumentType.AUTOMATION_LOG)) {
           return false
         }
-        return doc._id !== DocumentType.APP_METADATA
+        if (doc._id === DocumentType.APP_METADATA) {
+          return false
+        }
+        return filter ? filter(doc, params) : true
       },
     }
   }
@@ -75,10 +66,6 @@ class Replication {
     // take the opportunity to remove deleted tombstones
     await this.replicate()
   }
-
-  cancel() {
-    this.replication.cancel()
-  }
 }
 
 export default Replication

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -70,7 +70,15 @@ export class DatabaseImpl implements Database {
     DatabaseImpl.nano = buildNano(couchInfo)
   }
 
-  async exists() {
+  exists(docId?: string) {
+    if (docId === undefined) {
+      return this.dbExists()
+    }
+
+    return this.docExists(docId)
+  }
+
+  private async dbExists() {
     const response = await directCouchUrlCall({
       url: `${this.couchInfo.url}/${this.name}`,
       method: "HEAD",
@@ -79,6 +87,15 @@ export class DatabaseImpl implements Database {
     return response.status === 200
   }
 
+  private async docExists(id: string): Promise<boolean> {
+    try {
+      await this.performCall(db => () => db.head(id))
+      return true
+    } catch {
+      return false
+    }
+  }
+
   private nano() {
     return this.instanceNano || DatabaseImpl.nano
   }

packages/backend-core/src/db/instrumentation.ts

@@ -24,9 +24,12 @@ export class DDInstrumentedDatabase implements Database {
     return this.db.name
   }
 
-  exists(): Promise<boolean> {
+  exists(docId?: string): Promise<boolean> {
     return tracer.trace("db.exists", span => {
-      span?.addTags({ db_name: this.name })
+      span?.addTags({ db_name: this.name, doc_id: docId })
+      if (docId) {
+        return this.db.exists(docId)
+      }
       return this.db.exists()
     })
   }

packages/backend-core/src/db/tests/DatabaseImpl.spec.ts

@@ -0,0 +1,55 @@
+import _ from "lodash"
+import { AnyDocument } from "@budibase/types"
+import { generator } from "../../../tests"
+import { DatabaseImpl } from "../couch"
+import { newid } from "../../utils"
+
+describe("DatabaseImpl", () => {
+  const database = new DatabaseImpl(generator.word())
+  const documents: AnyDocument[] = []
+
+  beforeAll(async () => {
+    const docsToCreate = Array.from({ length: 10 }).map(() => ({
+      _id: newid(),
+    }))
+    const createdDocs = await database.bulkDocs(docsToCreate)
+
+    documents.push(...createdDocs.map((x: any) => ({ _id: x.id, _rev: x.rev })))
+  })
+
+  describe("document exists", () => {
+    it("can check existing docs by id", async () => {
+      const existingDoc = _.sample(documents)
+      const result = await database.exists(existingDoc!._id!)
+
+      expect(result).toBe(true)
+    })
+
+    it("can check non existing docs by id", async () => {
+      const result = await database.exists(newid())
+
+      expect(result).toBe(false)
+    })
+
+    it("can check an existing doc by id multiple times", async () => {
+      const existingDoc = _.sample(documents)
+      const id = existingDoc!._id!
+
+      const results = []
+      results.push(await database.exists(id))
+      results.push(await database.exists(id))
+      results.push(await database.exists(id))
+
+      expect(results).toEqual([true, true, true])
+    })
+
+    it("returns false after the doc is deleted", async () => {
+      const existingDoc = _.sample(documents)
+      const id = existingDoc!._id!
+      expect(await database.exists(id)).toBe(true)
+
+      await database.remove(existingDoc!)
+      expect(await database.exists(id)).toBe(false)
+    })
+  })
+})

packages/backend-core/src/environment.ts

@@ -186,6 +186,7 @@ const environment = {
     environment[key] = value
   },
   ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M",
+  DISABLE_SCIM_CALLS: process.env.DISABLE_SCIM_CALLS,
 }
 
 // clean up any environment variable edge cases

packages/backend-core/src/queue/constants.ts

@@ -4,4 +4,5 @@ export enum JobQueue {
   AUDIT_LOG = "auditLogQueue",
   SYSTEM_EVENT_QUEUE = "systemEventQueue",
   APP_MIGRATION = "appMigration",
+  DOC_WRITETHROUGH_QUEUE = "docWritethroughQueue",
 }

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -1,5 +1,14 @@
 import events from "events"
-import { timeout } from "../utils"
+import { newid, timeout } from "../utils"
+import { Queue, QueueOptions, JobOptions } from "./queue"
+
+interface JobMessage {
+  id: string
+  timestamp: number
+  queue: string
+  data: any
+  opts?: JobOptions
+}
 
 /**
  * Bull works with a Job wrapper around all messages that contains a lot more information about
@@ -10,12 +19,13 @@ import { timeout } from "../utils"
  * @returns A new job which can now be put onto the queue, this is mostly an
  * internal structure so that an in memory queue can be easily swapped for a Bull queue.
  */
-function newJob(queue: string, message: any) {
+function newJob(queue: string, message: any, opts?: JobOptions): JobMessage {
   return {
+    id: newid(),
     timestamp: Date.now(),
     queue: queue,
     data: message,
-    opts: {},
+    opts,
   }
 }
 
@@ -24,26 +34,29 @@ function newJob(queue: string, message: any) {
  * It is relatively simple, using an event emitter internally to register when messages are available
  * to the consumers - in can support many inputs and many consumers.
  */
-class InMemoryQueue {
+class InMemoryQueue implements Partial<Queue> {
   _name: string
-  _opts?: any
+  _opts?: QueueOptions
-  _messages: any[]
+  _messages: JobMessage[]
+  _queuedJobIds: Set<string>
   _emitter: EventEmitter
   _runCount: number
   _addCount: number
+
   /**
    * The constructor the queue, exactly the same as that of Bulls.
    * @param name The name of the queue which is being configured.
    * @param opts This is not used by the in memory queue as there is no real use
    * case when in memory, but is the same API as Bull
    */
-  constructor(name: string, opts?: any) {
+  constructor(name: string, opts?: QueueOptions) {
     this._name = name
     this._opts = opts
     this._messages = []
     this._emitter = new events.EventEmitter()
     this._runCount = 0
     this._addCount = 0
+    this._queuedJobIds = new Set<string>()
   }
 
   /**
@@ -55,22 +68,42 @@ class InMemoryQueue {
    * note this is incredibly limited compared to Bull as in reality the Job would contain
    * a lot more information about the queue and current status of Bull cluster.
    */
-  process(func: any) {
+  async process(func: any) {
     this._emitter.on("message", async () => {
       if (this._messages.length <= 0) {
         return
       }
       let msg = this._messages.shift()
+
       let resp = func(msg)
+
+      async function retryFunc(fnc: any) {
+        try {
+          await fnc
+        } catch (e: any) {
+          await new Promise<void>(r => setTimeout(() => r(), 50))
+
+          await retryFunc(func(msg))
+        }
+      }
+
       if (resp.then != null) {
-        await resp
+        try {
+          await retryFunc(resp)
+        } catch (e: any) {
+          console.error(e)
+        }
       }
       this._runCount++
+      const jobId = msg?.opts?.jobId?.toString()
+      if (jobId && msg?.opts?.removeOnComplete) {
+        this._queuedJobIds.delete(jobId)
+      }
     })
   }
 
   async isReady() {
-    return true
+    return this as any
   }
 
   // simply puts a message to the queue and emits to the queue for processing
@@ -83,27 +116,45 @@ class InMemoryQueue {
    * @param repeat serves no purpose for the import queue.
    */
   // eslint-disable-next-line no-unused-vars
-  add(msg: any, repeat: boolean) {
+  async add(data: any, opts?: JobOptions) {
-    if (typeof msg !== "object") {
+    const jobId = opts?.jobId?.toString()
+    if (jobId && this._queuedJobIds.has(jobId)) {
+      console.log(`Ignoring already queued job ${jobId}`)
+      return
+    }
+
+    if (typeof data !== "object") {
       throw "Queue only supports carrying JSON."
     }
-    this._messages.push(newJob(this._name, msg))
+    if (jobId) {
-    this._addCount++
+      this._queuedJobIds.add(jobId)
-    this._emitter.emit("message")
+    }
+
+    const pushMessage = () => {
+      this._messages.push(newJob(this._name, data, opts))
+      this._addCount++
+      this._emitter.emit("message")
+    }
+
+    const delay = opts?.delay
+    if (delay) {
+      setTimeout(pushMessage, delay)
+    } else {
+      pushMessage()
+    }
+    return {} as any
   }
 
   /**
    * replicating the close function from bull, which waits for jobs to finish.
    */
-  async close() {
+  async close() {}
-    return []
-  }
 
   /**
    * This removes a cron which has been implemented, this is part of Bull API.
    * @param cronJobId The cron which is to be removed.
    */
-  removeRepeatableByKey(cronJobId: string) {
+  async removeRepeatableByKey(cronJobId: string) {
     // TODO: implement for testing
     console.log(cronJobId)
   }
@@ -111,12 +162,12 @@ class InMemoryQueue {
   /**
    * Implemented for tests
    */
-  getRepeatableJobs() {
+  async getRepeatableJobs() {
     return []
   }
 
   // eslint-disable-next-line no-unused-vars
-  removeJobs(pattern: string) {
+  async removeJobs(pattern: string) {
     // no-op
   }
 
@@ -128,18 +179,22 @@ class InMemoryQueue {
   }
 
   async getJob() {
-    return {}
+    return null
   }
 
   on() {
     // do nothing
-    return this
+    return this as any
   }
 
   async waitForCompletion() {
     do {
       await timeout(50)
-    } while (this._addCount < this._runCount)
+    } while (this.hasRunningJobs())
+  }
+
+  hasRunningJobs() {
+    return this._addCount > this._runCount
   }
 }
 

packages/backend-core/src/queue/listeners.ts

@@ -88,6 +88,7 @@ enum QueueEventType {
   AUDIT_LOG_EVENT = "audit-log-event",
   SYSTEM_EVENT = "system-event",
   APP_MIGRATION = "app-migration",
+  DOC_WRITETHROUGH = "doc-writethrough",
 }
 
 const EventTypeMap: { [key in JobQueue]: QueueEventType } = {
@@ -96,6 +97,7 @@ const EventTypeMap: { [key in JobQueue]: QueueEventType } = {
   [JobQueue.AUDIT_LOG]: QueueEventType.AUDIT_LOG_EVENT,
   [JobQueue.SYSTEM_EVENT_QUEUE]: QueueEventType.SYSTEM_EVENT,
   [JobQueue.APP_MIGRATION]: QueueEventType.APP_MIGRATION,
+  [JobQueue.DOC_WRITETHROUGH_QUEUE]: QueueEventType.DOC_WRITETHROUGH,
 }
 
 function logging(queue: Queue, jobQueue: JobQueue) {

packages/backend-core/src/queue/queue.ts

@@ -7,6 +7,8 @@ import { addListeners, StalledFn } from "./listeners"
 import { Duration } from "../utils"
 import * as timers from "../timers"
 
+export { QueueOptions, Queue, JobOptions } from "bull"
+
 // the queue lock is held for 5 minutes
 const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs()
 // queue lock is refreshed every 30 seconds

packages/backend-core/src/redis/init.ts

@@ -9,7 +9,8 @@ let userClient: Client,
   lockClient: Client,
   socketClient: Client,
   inviteClient: Client,
-  passwordResetClient: Client
+  passwordResetClient: Client,
+  docWritethroughClient: Client
 
 export async function init() {
   userClient = await new Client(utils.Databases.USER_CACHE).init()
@@ -24,6 +25,9 @@ export async function init() {
     utils.Databases.SOCKET_IO,
     utils.SelectableDatabase.SOCKET_IO
   ).init()
+  docWritethroughClient = await new Client(
+    utils.Databases.DOC_WRITE_THROUGH
+  ).init()
 }
 
 export async function shutdown() {
@@ -104,3 +108,10 @@ export async function getPasswordResetClient() {
   }
   return passwordResetClient
 }
+
+export async function getDocWritethroughClient() {
+  if (!writethroughClient) {
+    await init()
+  }
+  return writethroughClient
+}

packages/backend-core/src/redis/redis.ts

@@ -1,5 +1,5 @@
 import env from "../environment"
-import Redis from "ioredis"
+import Redis, { Cluster } from "ioredis"
 // mock-redis doesn't have any typing
 let MockRedis: any | undefined
 if (env.MOCK_REDIS) {
@@ -28,7 +28,7 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT
 
 // for testing just generate the client once
 let CLOSED = false
-let CLIENTS: { [key: number]: any } = {}
+const CLIENTS: Record<number, Redis> = {}
 let CONNECTED = false
 
 // mock redis always connected
@@ -36,7 +36,7 @@ if (env.MOCK_REDIS) {
   CONNECTED = true
 }
 
-function pickClient(selectDb: number): any {
+function pickClient(selectDb: number) {
   return CLIENTS[selectDb]
 }
 
@@ -201,12 +201,15 @@ class RedisWrapper {
     key = `${db}${SEPARATOR}${key}`
     let stream
     if (CLUSTERED) {
-      let node = this.getClient().nodes("master")
+      let node = (this.getClient() as never as Cluster).nodes("master")
       stream = node[0].scanStream({ match: key + "*", count: 100 })
     } else {
-      stream = this.getClient().scanStream({ match: key + "*", count: 100 })
+      stream = (this.getClient() as Redis).scanStream({
+        match: key + "*",
+        count: 100,
+      })
     }
-    return promisifyStream(stream, this.getClient())
+    return promisifyStream(stream, this.getClient() as any)
   }
 
   async keys(pattern: string) {
@@ -221,14 +224,16 @@ class RedisWrapper {
 
   async get(key: string) {
     const db = this._db
-    let response = await this.getClient().get(addDbPrefix(db, key))
+    const response = await this.getClient().get(addDbPrefix(db, key))
     // overwrite the prefixed key
+    // @ts-ignore
     if (response != null && response.key) {
+      // @ts-ignore
       response.key = key
     }
     // if its not an object just return the response
     try {
-      return JSON.parse(response)
+      return JSON.parse(response!)
     } catch (err) {
       return response
     }
@@ -274,13 +279,37 @@ class RedisWrapper {
     }
   }
 
+  async bulkStore(
+    data: Record<string, any>,
+    expirySeconds: number | null = null
+  ) {
+    const client = this.getClient()
+
+    const dataToStore = Object.entries(data).reduce((acc, [key, value]) => {
+      acc[addDbPrefix(this._db, key)] =
+        typeof value === "object" ? JSON.stringify(value) : value
+      return acc
+    }, {} as Record<string, any>)
+
+    const pipeline = client.pipeline()
+    pipeline.mset(dataToStore)
+
+    if (expirySeconds !== null) {
+      for (const key of Object.keys(dataToStore)) {
+        pipeline.expire(key, expirySeconds)
+      }
+    }
+
+    await pipeline.exec()
+  }
+
   async getTTL(key: string) {
     const db = this._db
     const prefixedKey = addDbPrefix(db, key)
     return this.getClient().ttl(prefixedKey)
   }
 
-  async setExpiry(key: string, expirySeconds: number | null) {
+  async setExpiry(key: string, expirySeconds: number) {
     const db = this._db
     const prefixedKey = addDbPrefix(db, key)
     await this.getClient().expire(prefixedKey, expirySeconds)
@@ -291,10 +320,35 @@ class RedisWrapper {
     await this.getClient().del(addDbPrefix(db, key))
   }
 
+  async bulkDelete(keys: string[]) {
+    const db = this._db
+    await this.getClient().del(keys.map(key => addDbPrefix(db, key)))
+  }
+
   async clear() {
     let items = await this.scan()
     await Promise.all(items.map((obj: any) => this.delete(obj.key)))
   }
+
+  async increment(key: string) {
+    const result = await this.getClient().incr(addDbPrefix(this._db, key))
+    if (isNaN(result)) {
+      throw new Error(`Redis ${key} does not contain a number`)
+    }
+    return result
+  }
+
+  async deleteIfValue(key: string, value: any) {
+    const client = this.getClient()
+
+    const luaScript = `
+      if redis.call('GET', KEYS[1]) == ARGV[1] then
+        redis.call('DEL', KEYS[1])
+      end
+      `
+
+    await client.eval(luaScript, 1, addDbPrefix(this._db, key), value)
+  }
 }
 
 export default RedisWrapper

packages/backend-core/src/redis/redlockImpl.ts

@@ -72,7 +72,7 @@ const OPTIONS: Record<keyof typeof LockType, Redlock.Options> = {
 export async function newRedlock(opts: Redlock.Options = {}) {
   const options = { ...OPTIONS.DEFAULT, ...opts }
   const redisWrapper = await getLockClient()
-  const client = redisWrapper.getClient()
+  const client = redisWrapper.getClient() as any
   return new Redlock([client], options)
 }
 

packages/backend-core/src/redis/tests/redis.spec.ts

@@ -0,0 +1,214 @@
+import { GenericContainer, StartedTestContainer } from "testcontainers"
+import { generator, structures } from "../../../tests"
+import RedisWrapper from "../redis"
+import { env } from "../.."
+
+jest.setTimeout(30000)
+
+describe("redis", () => {
+  let redis: RedisWrapper
+  let container: StartedTestContainer
+
+  beforeAll(async () => {
+    const container = await new GenericContainer("redis")
+      .withExposedPorts(6379)
+      .start()
+
+    env._set(
+      "REDIS_URL",
+      `${container.getHost()}:${container.getMappedPort(6379)}`
+    )
+    env._set("MOCK_REDIS", 0)
+    env._set("REDIS_PASSWORD", 0)
+  })
+
+  afterAll(() => container?.stop())
+
+  beforeEach(async () => {
+    redis = new RedisWrapper(structures.db.id())
+    await redis.init()
+  })
+
+  describe("store", () => {
+    it("a basic value can be persisted", async () => {
+      const key = structures.uuid()
+      const value = generator.word()
+
+      await redis.store(key, value)
+
+      expect(await redis.get(key)).toEqual(value)
+    })
+
+    it("objects can be persisted", async () => {
+      const key = structures.uuid()
+      const value = { [generator.word()]: generator.word() }
+
+      await redis.store(key, value)
+
+      expect(await redis.get(key)).toEqual(value)
+    })
+  })
+
+  describe("bulkStore", () => {
+    function createRandomObject(
+      keyLength: number,
+      valueGenerator: () => any = () => generator.word()
+    ) {
+      return generator
+        .unique(() => generator.word(), keyLength)
+        .reduce((acc, key) => {
+          acc[key] = valueGenerator()
+          return acc
+        }, {} as Record<string, string>)
+    }
+
+    it("a basic object can be persisted", async () => {
+      const data = createRandomObject(10)
+
+      await redis.bulkStore(data)
+
+      for (const [key, value] of Object.entries(data)) {
+        expect(await redis.get(key)).toEqual(value)
+      }
+
+      expect(await redis.keys("*")).toHaveLength(10)
+    })
+
+    it("a complex object can be persisted", async () => {
+      const data = {
+        ...createRandomObject(10, () => createRandomObject(5)),
+        ...createRandomObject(5),
+      }
+
+      await redis.bulkStore(data)
+
+      for (const [key, value] of Object.entries(data)) {
+        expect(await redis.get(key)).toEqual(value)
+      }
+
+      expect(await redis.keys("*")).toHaveLength(15)
+    })
+
+    it("no TTL is set by default", async () => {
+      const data = createRandomObject(10)
+
+      await redis.bulkStore(data)
+
+      for (const [key, value] of Object.entries(data)) {
+        expect(await redis.get(key)).toEqual(value)
+        expect(await redis.getTTL(key)).toEqual(-1)
+      }
+    })
+
+    it("a bulk store can be persisted with TTL", async () => {
+      const ttl = 500
+      const data = createRandomObject(8)
+
+      await redis.bulkStore(data, ttl)
+
+      for (const [key, value] of Object.entries(data)) {
+        expect(await redis.get(key)).toEqual(value)
+        expect(await redis.getTTL(key)).toEqual(ttl)
+      }
+
+      expect(await redis.keys("*")).toHaveLength(8)
+    })
+
+    it("setting a TTL of -1 will not persist the key", async () => {
+      const ttl = -1
+      const data = createRandomObject(5)
+
+      await redis.bulkStore(data, ttl)
+
+      for (const [key, value] of Object.entries(data)) {
+        expect(await redis.get(key)).toBe(null)
+      }
+
+      expect(await redis.keys("*")).toHaveLength(0)
+    })
+  })
+
+  describe("increment", () => {
+    it("can increment on a new key", async () => {
+      const key = structures.uuid()
+      const result = await redis.increment(key)
+      expect(result).toBe(1)
+    })
+
+    it("can increment multiple times", async () => {
+      const key = structures.uuid()
+      const results = [
+        await redis.increment(key),
+        await redis.increment(key),
+        await redis.increment(key),
+        await redis.increment(key),
+        await redis.increment(key),
+      ]
+      expect(results).toEqual([1, 2, 3, 4, 5])
+    })
+
+    it("can increment on a new key", async () => {
+      const key1 = structures.uuid()
+      const key2 = structures.uuid()
+
+      const result1 = await redis.increment(key1)
+      expect(result1).toBe(1)
+
+      const result2 = await redis.increment(key2)
+      expect(result2).toBe(1)
+    })
+
+    it("can increment multiple times in parallel", async () => {
+      const key = structures.uuid()
+      const results = await Promise.all(
+        Array.from({ length: 100 }).map(() => redis.increment(key))
+      )
+      expect(results).toHaveLength(100)
+      expect(results).toEqual(Array.from({ length: 100 }).map((_, i) => i + 1))
+    })
+
+    it("can increment existing set keys", async () => {
+      const key = structures.uuid()
+      await redis.store(key, 70)
+      await redis.increment(key)
+
+      const result = await redis.increment(key)
+      expect(result).toBe(72)
+    })
+
+    it.each([
+      generator.word(),
+      generator.bool(),
+      { [generator.word()]: generator.word() },
+    ])("cannot increment if the store value is not a number", async value => {
+      const key = structures.uuid()
+      await redis.store(key, value)
+
+      await expect(redis.increment(key)).rejects.toThrowError(
+        "ERR value is not an integer or out of range"
+      )
+    })
+  })
+
+  describe("deleteIfValue", () => {
+    it("can delete if the value matches", async () => {
+      const key = structures.uuid()
+      const value = generator.word()
+      await redis.store(key, value)
+
+      await redis.deleteIfValue(key, value)
+
+      expect(await redis.get(key)).toBeNull()
+    })
+
+    it("will not delete if the value does not matches", async () => {
+      const key = structures.uuid()
+      const value = generator.word()
+      await redis.store(key, value)
+
+      await redis.deleteIfValue(key, generator.word())
+
+      expect(await redis.get(key)).toEqual(value)
+    })
+  })
+})

packages/backend-core/src/redis/utils.ts

@@ -30,6 +30,7 @@ export enum Databases {
   LOCKS = "locks",
   SOCKET_IO = "socket_io",
   BPM_EVENTS = "bpmEvents",
+  DOC_WRITE_THROUGH = "docWriteThrough",
 }
 
 /**

packages/backend-core/src/security/roles.ts

@@ -84,25 +84,24 @@ export function getBuiltinRoles(): { [key: string]: RoleDoc } {
   return cloneDeep(BUILTIN_ROLES)
 }
 
-export const BUILTIN_ROLE_ID_ARRAY = Object.values(BUILTIN_ROLES).map(
+export function isBuiltin(role: string) {
-  role => role._id
+  return getBuiltinRole(role) !== undefined
-)
+}
 
-export const BUILTIN_ROLE_NAME_ARRAY = Object.values(BUILTIN_ROLES).map(
+export function getBuiltinRole(roleId: string): Role | undefined {
-  role => role.name
+  const role = Object.values(BUILTIN_ROLES).find(role =>
-)
+    roleId.includes(role._id)
-
+  )
-export function isBuiltin(role?: string) {
+  if (!role) {
-  return BUILTIN_ROLE_ID_ARRAY.some(builtin => role?.includes(builtin))
+    return undefined
+  }
+  return cloneDeep(role)
 }
 
 /**
  * Works through the inheritance ranks to see how far up the builtin stack this ID is.
  */
-export function builtinRoleToNumber(id?: string) {
+export function builtinRoleToNumber(id: string) {
-  if (!id) {
-    return 0
-  }
   const builtins = getBuiltinRoles()
   const MAX = Object.values(builtins).length + 1
   if (id === BUILTIN_IDS.ADMIN || id === BUILTIN_IDS.BUILDER) {
@@ -123,7 +122,7 @@ export function builtinRoleToNumber(id?: string) {
 /**
  * Converts any role to a number, but has to be async to get the roles from db.
  */
-export async function roleToNumber(id?: string) {
+export async function roleToNumber(id: string) {
   if (isBuiltin(id)) {
     return builtinRoleToNumber(id)
   }
@@ -131,7 +130,7 @@ export async function roleToNumber(id?: string) {
     defaultPublic: true,
   })) as RoleDoc[]
   for (let role of hierarchy) {
-    if (isBuiltin(role?.inherits)) {
+    if (role?.inherits && isBuiltin(role.inherits)) {
       return builtinRoleToNumber(role.inherits) + 1
     }
   }
@@ -161,35 +160,28 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string): string {
  * @returns The role object, which may contain an "inherits" property.
  */
 export async function getRole(
-  roleId?: string,
+  roleId: string,
   opts?: { defaultPublic?: boolean }
-): Promise<RoleDoc | undefined> {
+): Promise<RoleDoc> {
-  if (!roleId) {
-    return undefined
-  }
-  let role: any = {}
   // built in roles mostly come from the in-code implementation,
   // but can be extended by a doc stored about them (e.g. permissions)
-  if (isBuiltin(roleId)) {
+  let role: RoleDoc | undefined = getBuiltinRole(roleId)
-    role = cloneDeep(
+  if (!role) {
-      Object.values(BUILTIN_ROLES).find(role => role._id === roleId)
-    )
-  } else {
     // make sure has the prefix (if it has it then it won't be added)
     roleId = prefixRoleID(roleId)
   }
   try {
     const db = getAppDB()
-    const dbRole = await db.get(getDBRoleID(roleId))
+    const dbRole = await db.get<RoleDoc>(getDBRoleID(roleId))
-    role = Object.assign(role, dbRole)
+    role = Object.assign(role || {}, dbRole)
     // finalise the ID
-    role._id = getExternalRoleID(role._id, role.version)
+    role._id = getExternalRoleID(role._id!, role.version)
   } catch (err) {
     if (!isBuiltin(roleId) && opts?.defaultPublic) {
       return cloneDeep(BUILTIN_ROLES.PUBLIC)
     }
     // only throw an error if there is no role at all
-    if (Object.keys(role).length === 0) {
+    if (!role || Object.keys(role).length === 0) {
       throw err
     }
   }
@@ -200,7 +192,7 @@ export async function getRole(
  * Simple function to get all the roles based on the top level user role ID.
  */
 async function getAllUserRoles(
-  userRoleId?: string,
+  userRoleId: string,
   opts?: { defaultPublic?: boolean }
 ): Promise<RoleDoc[]> {
   // admins have access to all roles
@@ -226,7 +218,7 @@ async function getAllUserRoles(
 }
 
 export async function getUserRoleIdHierarchy(
-  userRoleId?: string
+  userRoleId: string
 ): Promise<string[]> {
   const roles = await getUserRoleHierarchy(userRoleId)
   return roles.map(role => role._id!)
@@ -241,7 +233,7 @@ export async function getUserRoleIdHierarchy(
  * highest level of access and the last being the lowest level.
  */
 export async function getUserRoleHierarchy(
-  userRoleId?: string,
+  userRoleId: string,
   opts?: { defaultPublic?: boolean }
 ) {
   // special case, if they don't have a role then they are a public user
@@ -265,9 +257,9 @@ export function checkForRoleResourceArray(
   return rolePerms
 }
 
-export async function getAllRoleIds(appId?: string) {
+export async function getAllRoleIds(appId: string): Promise<string[]> {
   const roles = await getAllRoles(appId)
-  return roles.map(role => role._id)
+  return roles.map(role => role._id!)
 }
 
 /**

packages/backend-core/tests/core/utilities/structures/accounts.ts

@@ -18,7 +18,7 @@ export const account = (partial: Partial<Account> = {}): Account => {
   return {
     accountId: uuid(),
     tenantId: generator.word(),
-    email: generator.email(),
+    email: generator.email({ domain: "example.com" }),
     tenantName: generator.word(),
     hosting: Hosting.SELF,
     createdAt: Date.now(),

packages/backend-core/tests/core/utilities/structures/scim.ts

@@ -13,7 +13,7 @@ interface CreateUserRequestFields {
 export function createUserRequest(userData?: Partial<CreateUserRequestFields>) {
   const defaultValues = {
     externalId: uuid(),
-    email: generator.email(),
+    email: `${uuid()}@example.com`,
     firstName: generator.first(),
     lastName: generator.last(),
     username: generator.name(),

packages/bbui/src/Actions/position_dropdown.js

@@ -36,7 +36,10 @@ export default function positionDropdown(element, opts) {
     }
 
     if (typeof customUpdate === "function") {
-      styles = customUpdate(anchorBounds, elementBounds, styles)
+      styles = customUpdate(anchorBounds, elementBounds, {
+        ...styles,
+        offset: opts.offset,
+      })
     } else {
       // Determine vertical styles
       if (align === "right-outside" || align === "left-outside") {

packages/bbui/src/Icon/IconAvatar.svelte

@@ -1,22 +1,41 @@
 <script>
   import Icon from "./Icon.svelte"
 
+  import Tooltip from "../Tooltip/Tooltip.svelte"
+  import { fade } from "svelte/transition"
+
   export let icon
   export let background
   export let color
   export let size = "M"
+  export let tooltip
+
+  let showTooltip = false
 </script>
 
+<!-- svelte-ignore a11y-no-static-element-interactions -->
+<!-- svelte-ignore a11y-click-events-have-key-events -->
 <div
   class="icon size--{size}"
   style="background: {background || `transparent`};"
   class:filled={!!background}
+  on:mouseover={() => (showTooltip = true)}
+  on:mouseleave={() => (showTooltip = false)}
+  on:focus={() => (showTooltip = true)}
+  on:blur={() => (showTooltip = false)}
+  on:click={() => (showTooltip = false)}
 >
   <Icon name={icon} color={background ? "white" : color} />
+  {#if tooltip && showTooltip}
+    <div class="tooltip" in:fade={{ duration: 130, delay: 250 }}>
+      <Tooltip textWrapping direction="right" text={tooltip} />
+    </div>
+  {/if}
 </div>
 
 <style>
   .icon {
+    position: relative;
     width: 28px;
     height: 28px;
     flex: 0 0 28px;
@@ -32,6 +51,15 @@
     width: 16px;
     height: 16px;
   }
+  .icon.size--XS {
+    width: 18px;
+    height: 18px;
+    flex: 0 0 18px;
+  }
+  .icon.size--XS :global(.spectrum-Icon) {
+    width: 10px;
+    height: 10px;
+  }
   .icon.size--S {
     width: 22px;
     height: 22px;
@@ -58,4 +86,14 @@
     width: 22px;
     height: 22px;
   }
+
+  .tooltip {
+    position: absolute;
+    pointer-events: none;
+    left: calc(50% + 8px);
+    bottom: calc(-50% + 6px);
+    /* transform: translateY(-50%); */
+    text-align: center;
+    z-index: 1;
+  }
 </style>

packages/bbui/src/Modal/Modal.svelte

@@ -10,6 +10,7 @@
   export let inline = false
   export let disableCancel = false
   export let autoFocus = true
+  export let zIndex = 999
 
   const dispatch = createEventDispatcher()
   let visible = fixed || inline
@@ -101,7 +102,11 @@
   <Portal target=".modal-container">
     {#if visible}
       <!-- svelte-ignore a11y-no-static-element-interactions -->
-      <div class="spectrum-Underlay is-open" on:mousedown|self={cancel}>
+      <div
+        class="spectrum-Underlay is-open"
+        on:mousedown|self={cancel}
+        style="z-index:{zIndex || 999}"
+      >
         <div
           class="background"
           in:fade={{ duration: 200 }}
@@ -132,7 +137,6 @@
     flex-direction: row;
     justify-content: center;
     align-items: center;
-    z-index: 999;
     overflow: auto;
     overflow-x: hidden;
     background: transparent;

packages/bbui/src/Table/Table.svelte

@@ -470,6 +470,7 @@
     --table-border: 1px solid var(--spectrum-alias-border-color-mid);
     --cell-padding: var(--spectrum-global-dimension-size-250);
     overflow: auto;
+    display: contents;
   }
   .wrapper--quiet {
     --table-bg: var(--spectrum-alias-background-color-transparent);

packages/builder/src/analytics/index.js

@@ -9,13 +9,17 @@ const intercom = new IntercomClient(process.env.INTERCOM_TOKEN)
 class AnalyticsHub {
   constructor() {
     this.clients = [posthog, intercom]
+    this.initialised = false
   }
 
   async activate() {
     // Check analytics are enabled
     const analyticsStatus = await API.getAnalyticsStatus()
-    if (analyticsStatus.enabled) {
+    if (analyticsStatus.enabled && !this.initialised) {
-      this.clients.forEach(client => client.init())
+      this.clients.forEach(client => {
+        client.init()
+      })
+      this.initialised = true
     }
   }
 

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -40,8 +40,15 @@
       part2: PrettyRelationshipDefinitions.MANY,
     },
   }
-  let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions)
+  $: relationshipOpts1 =
-  let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions)
+    relationshipPart2 === PrettyRelationshipDefinitions.ONE
+      ? [PrettyRelationshipDefinitions.MANY]
+      : Object.values(PrettyRelationshipDefinitions)
+
+  $: relationshipOpts2 =
+    relationshipPart1 === PrettyRelationshipDefinitions.ONE
+      ? [PrettyRelationshipDefinitions.MANY]
+      : Object.values(PrettyRelationshipDefinitions)
 
   let relationshipPart1 = PrettyRelationshipDefinitions.ONE
   let relationshipPart2 = PrettyRelationshipDefinitions.MANY

packages/builder/src/components/common/CodeEditor/CodeEditor.svelte

@@ -55,6 +55,8 @@
   export let jsBindingWrapping = true
   export let readonly = false
 
+  const dispatch = createEventDispatcher()
+
   // Export a function to expose caret position
   export const getCaretPosition = () => {
     const selection_range = editor.state.selection.ranges[0]
@@ -106,8 +108,6 @@
     }
   )
 
-  const dispatch = createEventDispatcher()
-
   // Theming!
   let currentTheme = $themeStore?.theme
   let isDark = !currentTheme.includes("light")

packages/builder/src/components/common/CodeEditor/index.js

@@ -163,26 +163,12 @@ export const jsAutocomplete = baseCompletions => {
 }
 
 export const buildBindingInfoNode = (completion, binding) => {
+  if (!binding.valueHTML || binding.value == null) {
+    return null
+  }
   const ele = document.createElement("div")
   ele.classList.add("info-bubble")
-
+  ele.innerHTML = `<div class="binding__example">${binding.valueHTML}</div>`
-  if (binding.value != null && binding.valueHTML) {
-    ele.innerHTML = `<div class="binding__example">${binding.valueHTML}</div>`
-    return ele
-  }
-
-  const exampleNodeHtml = binding.readableBinding
-    ? `<div class="binding__example">{{ ${binding.readableBinding} }}</div>`
-    : ""
-
-  const descriptionNodeHtml = binding.description
-    ? `<div class="binding__description">${binding.description}</div>`
-    : ""
-
-  ele.innerHTML = `
-    ${exampleNodeHtml}
-    ${descriptionNodeHtml}
-  `
   return ele
 }
 

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -89,6 +89,18 @@
       sidePanel = sidePanelOptions[0]
     }
   }
+  $: hbsCompletions = [
+    hbAutocomplete([
+      ...bindingCompletions,
+      ...getHelperCompletions(EditorModes.Handlebars),
+    ]),
+  ]
+  $: jsCompletions = [
+    jsAutocomplete([
+      ...bindingCompletions,
+      ...getHelperCompletions(EditorModes.JS),
+    ]),
+  ]
 
   const getModeOptions = (allowHBS, allowJS) => {
     let options = []
@@ -148,6 +160,9 @@
 
   const enrichBindings = (bindings, context, snippets) => {
     return bindings.map(binding => {
+      if (!context) {
+        return binding
+      }
       const value = getBindingValue(binding, context, snippets)
       return {
         ...binding,
@@ -265,38 +280,34 @@
       {/if}
       <div class="editor">
         {#if mode === Modes.Text}
-          <CodeEditor
+          {#key hbsCompletions}
-            value={hbsValue}
+            <CodeEditor
-            on:change={onChangeHBSValue}
+              value={hbsValue}
-            bind:getCaretPosition
+              on:change={onChangeHBSValue}
-            bind:insertAtPos
+              bind:getCaretPosition
-            completions={[
+              bind:insertAtPos
-              hbAutocomplete([
+              completions={hbsCompletions}
-                ...bindingCompletions,
+              autofocus={autofocusEditor}
-                ...getHelperCompletions(editorMode),
+              placeholder={placeholder ||
-              ]),
+                "Add bindings by typing {{ or use the menu on the right"}
-            ]}
+              jsBindingWrapping={false}
-            autofocus={autofocusEditor}
+            />
-            placeholder={placeholder ||
+          {/key}
-              "Add bindings by typing {{ or use the menu on the right"}
-          />
         {:else if mode === Modes.JavaScript}
-          <CodeEditor
+          {#key jsCompletions}
-            value={decodeJSBinding(jsValue)}
+            <CodeEditor
-            on:change={onChangeJSValue}
+              value={decodeJSBinding(jsValue)}
-            completions={[
+              on:change={onChangeJSValue}
-              jsAutocomplete([
+              completions={jsCompletions}
-                ...bindingCompletions,
+              mode={EditorModes.JS}
-                ...getHelperCompletions(editorMode),
+              bind:getCaretPosition
-              ]),
+              bind:insertAtPos
-            ]}
+              autofocus={autofocusEditor}
-            mode={EditorModes.JS}
+              placeholder={placeholder ||
-            bind:getCaretPosition
+                "Add bindings by typing $ or use the menu on the right"}
-            bind:insertAtPos
+              jsBindingWrapping
-            autofocus={autofocusEditor}
+            />
-            placeholder={placeholder ||
+          {/key}
-              "Add bindings by typing $ or use the menu on the right"}
-          />
         {/if}
         {#if targetMode}
           <div class="mode-overlay">
@@ -332,7 +343,7 @@
           {context}
           addHelper={onSelectHelper}
           addBinding={onSelectBinding}
-          mode={editorMode}
+          {mode}
         />
       {:else if sidePanel === SidePanels.Evaluation}
         <EvaluationSidePanel

packages/builder/src/components/common/bindings/ClientBindingPanel.svelte

@@ -4,7 +4,6 @@
   import { onMount } from "svelte"
 
   export let bindings = []
-  export let valid
   export let value = ""
   export let allowJS = false
   export let allowHelpers = true
@@ -27,7 +26,6 @@
 </script>
 
 <BindingPanel
-  bind:valid
   bindings={enrichedBindings}
   context={$previewStore.selectedComponentContext}
   snippets={$snippets}

packages/builder/src/components/common/bindings/DrawerBindableCombobox.svelte

@@ -22,7 +22,6 @@
 
   const dispatch = createEventDispatcher()
   let bindingDrawer
-  let valid = true
 
   $: readableValue = runtimeToReadableBinding(bindings, value)
   $: tempValue = readableValue
@@ -80,15 +79,12 @@
 </div>
 
 <Drawer bind:this={bindingDrawer} title={title ?? placeholder ?? "Bindings"}>
-  <Button cta slot="buttons" on:click={handleClose} disabled={!valid}>
+  <Button cta slot="buttons" on:click={handleClose}>Save</Button>
-    Save
-  </Button>
   <svelte:component
     this={panel}
     slot="body"
     value={readableValue}
     close={handleClose}
-    bind:valid
     on:change={event => (tempValue = event.detail)}
     {bindings}
     {allowJS}

packages/builder/src/components/common/bindings/DrawerBindableInput.svelte

@@ -28,7 +28,6 @@
   const dispatch = createEventDispatcher()
 
   let bindingDrawer
-  let valid = true
   let currentVal = value
 
   $: readableValue = runtimeToReadableBinding(bindings, value)
@@ -93,13 +92,10 @@
   title={title ?? placeholder ?? "Bindings"}
   {forceModal}
 >
-  <Button cta slot="buttons" disabled={!valid} on:click={saveBinding}>
+  <Button cta slot="buttons" on:click={saveBinding}>Save</Button>
-    Save
-  </Button>
   <svelte:component
     this={panel}
     slot="body"
-    bind:valid
     value={readableValue}
     on:change={event => (tempValue = event.detail)}
     {bindings}

packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte

@@ -25,7 +25,6 @@
 
   const dispatch = createEventDispatcher()
   let bindingDrawer
-  let valid = true
   let currentVal = value
 
   $: readableValue = runtimeToReadableBinding(bindings, value)
@@ -176,13 +175,10 @@
   title={title ?? placeholder ?? "Bindings"}
   left={drawerLeft}
 >
-  <Button cta slot="buttons" disabled={!valid} on:click={saveBinding}>
+  <Button cta slot="buttons" on:click={saveBinding}>Save</Button>
-    Save
-  </Button>
   <svelte:component
     this={panel}
     slot="body"
-    bind:valid
     value={readableValue}
     on:change={event => (tempValue = event.detail)}
     {bindings}

packages/builder/src/components/common/bindings/ServerBindingPanel.svelte

@@ -3,7 +3,6 @@
   import { snippets } from "stores/builder"
 
   export let bindings = []
-  export let valid
   export let value = ""
   export let allowJS = false
   export let context = null
@@ -21,7 +20,6 @@
 </script>
 
 <BindingPanel
-  bind:valid
   bindings={enrichedBindings}
   snippets={$snippets}
   {value}

packages/builder/src/components/deploy/AppActions.svelte

@@ -164,9 +164,10 @@
       </div>
     {/if}
     <TourWrap
-      tourStepKey={$builderStore.onboarding
+      stepKeys={[
-        ? TOUR_STEP_KEYS.BUILDER_USER_MANAGEMENT
+        TOUR_STEP_KEYS.BUILDER_USER_MANAGEMENT,
-        : TOUR_STEP_KEYS.FEATURE_USER_MANAGEMENT}
+        TOUR_STEP_KEYS.FEATURE_USER_MANAGEMENT,
+      ]}
     >
       <div class="app-action-button users">
         <div class="app-action" id="builder-app-users-button">
@@ -209,7 +210,7 @@
       <div bind:this={appActionPopoverAnchor}>
         <div class="app-action">
           <Icon name={isPublished ? "GlobeCheck" : "GlobeStrike"} />
-          <TourWrap tourStepKey={TOUR_STEP_KEYS.BUILDER_APP_PUBLISH}>
+          <TourWrap stepKeys={[TOUR_STEP_KEYS.BUILDER_APP_PUBLISH]}>
             <span class="publish-open" id="builder-app-publish-button">
               Publish
               <Icon

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte

@@ -45,7 +45,10 @@
     <Checkbox text="Require confirmation" bind:value={parameters.confirm} />
 
     {#if parameters.confirm}
-      <Label small>Confirm text</Label>
+      <Label small>Title</Label>
+      <Input placeholder="Delete Row" bind:value={parameters.customTitleText} />
+
+      <Label small>Text</Label>
       <Input
         placeholder="Are you sure you want to delete?"
         bind:value={parameters.confirmText}

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DuplicateRow.svelte

@@ -72,7 +72,13 @@
     <Checkbox text="Require confirmation" bind:value={parameters.confirm} />
 
     {#if parameters.confirm}
-      <Label small>Confirm text</Label>
+      <Label small>Title</Label>
+      <Input
+        placeholder="Duplicate Row"
+        bind:value={parameters.customTitleText}
+      />
+
+      <Label small>Text</Label>
       <Input
         placeholder="Are you sure you want to duplicate this row?"
         bind:value={parameters.confirmText}

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte

@@ -64,7 +64,13 @@
 
       {#if parameters.confirm}
         <Input
-          label="Confirm text"
+          label="Title"
+          placeholder="Execute Query"
+          bind:value={parameters.customTitleText}
+        />
+
+        <Input
+          label="Text"
           placeholder="Are you sure you want to execute this query?"
           bind:value={parameters.confirmText}
         />

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte

@@ -72,7 +72,10 @@
     <Checkbox text="Require confirmation" bind:value={parameters.confirm} />
 
     {#if parameters.confirm}
-      <Label small>Confirm text</Label>
+      <Label small>Title</Label>
+      <Input placeholder="Save Row" bind:value={parameters.customTitleText} />
+
+      <Label small>Text</Label>
       <Input
         placeholder="Are you sure you want to save this row?"
         bind:value={parameters.confirmText}

packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte

@@ -1,5 +1,5 @@
 <script>
-  import EditComponentPopover from "../EditComponentPopover.svelte"
+  import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte"
   import { Icon } from "@budibase/bbui"
   import { runtimeToReadableBinding } from "dataBinding"
   import { isJSBinding } from "@budibase/string-templates"

packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte

@@ -48,7 +48,6 @@
   let drawer
   let tmpQueryParams
   let tmpCustomData
-  let customDataValid = true
   let modal
 
   $: text = value?.label ?? "Choose an option"
@@ -267,14 +266,11 @@
     <Drawer title="Custom data" bind:this={drawer}>
       <div slot="buttons" style="display:contents">
         <Button primary on:click={promptForCSV}>Load CSV</Button>
-        <Button cta on:click={saveCustomData} disabled={!customDataValid}>
+        <Button cta on:click={saveCustomData}>Save</Button>
-          Save
-        </Button>
       </div>
       <div slot="description">Provide a JSON array to use as data</div>
       <ClientBindingPanel
         slot="body"
-        bind:valid={customDataValid}
         value={tmpCustomData}
         on:change={event => (tmpCustomData = event.detail)}
         {bindings}

packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte

@@ -3,6 +3,7 @@
   import { componentStore } from "stores/builder"
   import { cloneDeep } from "lodash/fp"
   import { createEventDispatcher, getContext } from "svelte"
+  import { customPositionHandler } from "."
   import ComponentSettingsSection from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte"
 
   export let anchor
@@ -54,25 +55,6 @@
 
     dispatch("change", nestedComponentInstance)
   }
-
-  const customPositionHandler = (anchorBounds, eleBounds, cfg) => {
-    let { left, top } = cfg
-    let percentageOffset = 30
-    // left-outside
-    left = anchorBounds.left - eleBounds.width - 18
-
-    // shift up from the anchor, if space allows
-    let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset
-    let defaultTop = anchorBounds.top - offsetPos
-
-    if (window.innerHeight - defaultTop < eleBounds.height) {
-      top = window.innerHeight - eleBounds.height - 5
-    } else {
-      top = anchorBounds.top - offsetPos
-    }
-
-    return { ...cfg, left, top }
-  }
 </script>
 
 <Icon
@@ -104,6 +86,7 @@
   showPopover={drawers.length === 0}
   clickOutsideOverride={drawers.length > 0}
   maxHeight={600}
+  offset={18}
   handlePostionUpdate={customPositionHandler}
 >
   <span class="popover-wrap">

packages/builder/src/components/design/settings/controls/EditComponentPopover/index.js

@@ -0,0 +1,18 @@
+export const customPositionHandler = (anchorBounds, eleBounds, cfg) => {
+  let { left, top, offset } = cfg
+  let percentageOffset = 30
+  // left-outside
+  left = anchorBounds.left - eleBounds.width - (offset || 5)
+
+  // shift up from the anchor, if space allows
+  let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset
+  let defaultTop = anchorBounds.top - offsetPos
+
+  if (window.innerHeight - defaultTop < eleBounds.height) {
+    top = window.innerHeight - eleBounds.height - 5
+  } else {
+    top = anchorBounds.top - offsetPos
+  }
+
+  return { ...cfg, left, top }
+}

packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte

@@ -1,5 +1,5 @@
 <script>
-  import EditComponentPopover from "../EditComponentPopover.svelte"
+  import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte"
   import { Toggle, Icon } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { cloneDeep } from "lodash/fp"

packages/builder/src/components/design/settings/controls/FormStepConfiguration.svelte

@@ -52,8 +52,8 @@
     _id: Helpers.uuid(),
     _component: componentType,
     _instanceName: `Step ${currentStep + 1}`,
-    title: stepSettings.title ?? defaults.title,
+    title: stepSettings.title ?? defaults?.title,
-    buttons: stepSettings.buttons || defaults.buttons,
+    buttons: stepSettings.buttons || defaults?.buttons,
     fields: stepSettings.fields,
     desc: stepSettings.desc,
 

packages/builder/src/components/design/settings/controls/GridColumnConfiguration/FieldSetting.svelte

@@ -1,5 +1,5 @@
 <script>
-  import EditComponentPopover from "../EditComponentPopover.svelte"
+  import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte"
   import { Toggle, Icon } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { cloneDeep } from "lodash/fp"

packages/builder/src/components/design/settings/controls/GridColumnConfiguration/PrimaryColumnFieldSetting.svelte

@@ -1,5 +1,5 @@
 <script>
-  import EditComponentPopover from "../EditComponentPopover.svelte"
+  import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte"
   import { Icon } from "@budibase/bbui"
   import { setContext } from "svelte"
   import { writable } from "svelte/store"

packages/builder/src/components/design/settings/controls/IconSelect/IconSelect.svelte

@@ -139,10 +139,22 @@
         {/each}
       </div>
       <div class="search-input">
-        <div class="input-wrapper">
+        <div class="input-wrapper" style={`width: ${value ? "425" : "510"}px`}>
-          <Input bind:value={searchTerm} thin placeholder="Search Icon" />
+          <Input
+            bind:value={searchTerm}
+            on:keyup={event => {
+              if (event.key === "Enter") {
+                searchForIcon()
+              }
+            }}
+            thin
+            placeholder="Search Icon"
+          />
         </div>
         <Button secondary on:click={searchForIcon}>Search</Button>
+        {#if value}
+          <Button primary on:click={() => (value = null)}>Clear</Button>
+        {/if}
       </div>
       <div class="page-area">
         <div class="pager">
@@ -239,6 +251,7 @@
     flex-flow: row nowrap;
     width: 100%;
     padding-right: 15px;
+    gap: 10px;
   }
   .input-wrapper {
     width: 510px;

packages/builder/src/components/design/settings/controls/PropertyControl.svelte

@@ -20,17 +20,23 @@
   export let bindings = []
   export let componentBindings = []
   export let nested = false
-  export let highlighted = false
   export let propertyFocus = false
   export let info = null
   export let disableBindings = false
   export let wide
 
-  $: nullishValue = value == null || value === ""
+  let highlightType
+
+  $: highlightedProp = $builderStore.highlightedSetting
   $: allBindings = getAllBindings(bindings, componentBindings, nested)
   $: safeValue = getSafeValue(value, defaultValue, allBindings)
   $: replaceBindings = val => readableToRuntimeBinding(allBindings, val)
 
+  $: if (!Array.isArray(value)) {
+    highlightType =
+      highlightedProp?.key === key ? `highlighted-${highlightedProp?.type}` : ""
+  }
+
   const getAllBindings = (bindings, componentBindings, nested) => {
     if (!nested) {
       return bindings
@@ -71,16 +77,17 @@
   }
 
   onDestroy(() => {
-    if (highlighted) {
+    if (highlightedProp) {
       builderStore.highlightSetting(null)
     }
   })
 </script>
 
 <div
-  class="property-control"
+  id={`${key}-prop-control-wrap`}
+  class={`property-control ${highlightType}`}
   class:wide={!label || labelHidden || wide === true}
-  class:highlighted={highlighted && nullishValue}
+  class:highlighted={highlightType}
   class:property-focus={propertyFocus}
 >
   {#if label && !labelHidden}
@@ -116,6 +123,16 @@
 </div>
 
 <style>
+  .property-control.highlighted.highlighted-info {
+    border-color: var(--spectrum-semantic-informative-color-background);
+  }
+  .property-control.highlighted.highlighted-error {
+    border-color: var(--spectrum-global-color-static-red-600);
+  }
+  .property-control.highlighted.highlighted-warning {
+    border-color: var(--spectrum-global-color-static-orange-700);
+  }
+
   .property-control {
     position: relative;
     display: grid;
@@ -133,6 +150,10 @@
   .property-control.highlighted {
     background: var(--spectrum-global-color-gray-300);
     border-color: var(--spectrum-global-color-static-red-600);
+    margin-top: -3.5px;
+    margin-bottom: -3.5px;
+    padding-bottom: 3.5px;
+    padding-top: 3.5px;
   }
 
   .property-control.property-focus :global(input) {

packages/builder/src/components/portal/onboarding/TourPopover.svelte

@@ -1,6 +1,6 @@
 <script>
   import { Popover, Layout, Heading, Body, Button, Link } from "@budibase/bbui"
-  import { TOURS } from "./tours.js"
+  import { TOURS, getCurrentStepIdx } from "./tours.js"
   import { goto, layout, isActive } from "@roxi/routify"
   import { builderStore } from "stores/builder"
 
@@ -20,6 +20,13 @@
 
   const updateTourStep = (targetStepKey, tourKey) => {
     if (!tourKey) {
+      tourSteps = null
+      tourStepIdx = null
+      lastStep = null
+      tourStep = null
+      popoverAnchor = null
+      popover = null
+      skipping = false
       return
     }
     if (!tourSteps?.length) {
@@ -78,16 +85,6 @@
       }
     }
   }
-
-  const getCurrentStepIdx = (steps, tourStepKey) => {
-    if (!steps?.length) {
-      return
-    }
-    if (steps?.length && !tourStepKey) {
-      return 0
-    }
-    return steps.findIndex(step => step.id === tourStepKey)
-  }
 </script>
 
 {#if tourKey}
@@ -98,7 +95,9 @@
       anchor={popoverAnchor}
       maxWidth={300}
       dismissible={false}
-      offset={15}
+      offset={12}
+      handlePostionUpdate={tourStep?.positionHandler}
+      customZindex={3}
     >
       <div class="tour-content">
         <Layout noPadding gap="M">
@@ -119,7 +118,7 @@
           </Body>
           <div class="tour-footer">
             <div class="tour-navigation">
-              {#if typeof tourOnSkip === "function"}
+              {#if typeof tourOnSkip === "function" && !lastStep}
                 <Link
                   secondary
                   quiet

packages/builder/src/components/portal/onboarding/TourWrap.svelte

@@ -1,39 +1,63 @@
 <script>
   import { tourHandler } from "./tourHandler"
-  import { TOURS } from "./tours"
+  import { TOURSBYSTEP, TOURS, getCurrentStepIdx } from "./tours"
   import { onMount, onDestroy } from "svelte"
   import { builderStore } from "stores/builder"
 
-  export let tourStepKey
+  export let stepKeys = []
 
-  let currentTourStep
   let ready = false
-  let registered = false
+  let registered = {}
-  let handler
 
   const registerTourNode = (tourKey, stepKey) => {
-    if (ready && !registered && tourKey) {
+    const step = TOURSBYSTEP[stepKey]
-      currentTourStep = TOURS[tourKey].steps.find(step => step.id === stepKey)
+    if (ready && step && !registered[stepKey] && step?.tour === tourKey) {
-      if (!currentTourStep) {
+      const elem = document.querySelector(step.query)
-        return
+      registered[stepKey] = tourHandler(elem, stepKey)
+    }
+  }
+
+  const scrollToStep = () => {
+    let tourStepIdx = getCurrentStepIdx(
+      TOURS[tourKeyWatch]?.steps,
+      tourStepKeyWatch
+    )
+    let currentStep = TOURS[tourKeyWatch]?.steps?.[tourStepIdx]
+    if (currentStep?.scrollIntoView) {
+      let currentNode = $builderStore.tourNodes?.[currentStep.id]
+      if (currentNode) {
+        currentNode.scrollIntoView({ behavior: "smooth", block: "center" })
       }
-      const elem = document.querySelector(currentTourStep.query)
-      handler = tourHandler(elem, stepKey)
-      registered = true
     }
   }
 
   $: tourKeyWatch = $builderStore.tourKey
-  $: registerTourNode(tourKeyWatch, tourStepKey, ready)
+  $: tourStepKeyWatch = $builderStore.tourStepKey
+  $: if (tourKeyWatch || stepKeys || ready) {
+    stepKeys.forEach(tourStepKey => {
+      registerTourNode(tourKeyWatch, tourStepKey)
+    })
+  }
+  $: scrollToStep(tourKeyWatch, tourStepKeyWatch)
 
   onMount(() => {
     ready = true
   })
 
   onDestroy(() => {
-    if (handler) {
+    Object.entries(registered).forEach(entry => {
+      const handler = entry[1]
+      const stepKey = entry[0]
+      // Run step destroy, de-register nodes in the builderStore and local cache
       handler.destroy()
-    }
+      delete registered[stepKey]
+
+      // Check if the step is part of an active tour. End the tour if that is the case
+      const step = TOURSBYSTEP[stepKey]
+      if (step.tour === tourKeyWatch) {
+        builderStore.setTour()
+      }
+    })
   })
 </script>
 

packages/builder/src/components/portal/onboarding/steps/NewFormSteps.svelte

@@ -0,0 +1,9 @@
+<div>
+  When faced with a sizable form, consider implementing a multi-step approach to
+  enhance user experience.
+
+  <p>
+    Breaking the form into multiple steps can significantly improve usability by
+    making the process more digestible for your users.
+  </p>
+</div>

packages/builder/src/components/portal/onboarding/steps/NewViewUpdateFormRowId.svelte

@@ -0,0 +1,17 @@
+<div>
+  You can use bindings to set the Row ID on your form.
+  <p>
+    This will allow you to pull the correct information into your form and allow
+    you to update!
+  </p>
+  <a href="https://docs.budibase.com/docs/form-block" target="_blank">
+    How to pass a row ID using bindings
+  </a>
+</div>
+
+<style>
+  a {
+    color: inherit;
+    text-decoration: underline;
+  }
+</style>

packages/builder/src/components/portal/onboarding/steps/index.js

@@ -1,3 +1,5 @@
 export { default as OnboardingData } from "./OnboardingData.svelte"
 export { default as OnboardingDesign } from "./OnboardingDesign.svelte"
 export { default as OnboardingPublish } from "./OnboardingPublish.svelte"
+export { default as NewViewUpdateFormRowId } from "./NewViewUpdateFormRowId.svelte"
+export { default as NewFormSteps } from "./NewFormSteps.svelte"

packages/builder/src/components/portal/onboarding/tours.js

@@ -2,8 +2,15 @@ import { get } from "svelte/store"
 import { builderStore } from "stores/builder"
 import { auth } from "stores/portal"
 import analytics from "analytics"
-import { OnboardingData, OnboardingDesign, OnboardingPublish } from "./steps"
+import {
+  OnboardingData,
+  OnboardingDesign,
+  OnboardingPublish,
+  NewViewUpdateFormRowId,
+  NewFormSteps,
+} from "./steps"
 import { API } from "api"
+import { customPositionHandler } from "components/design/settings/controls/EditComponentPopover"
 
 const ONBOARDING_EVENT_PREFIX = "onboarding"
 
@@ -14,11 +21,26 @@ export const TOUR_STEP_KEYS = {
   BUILDER_USER_MANAGEMENT: "builder-user-management",
   BUILDER_AUTOMATION_SECTION: "builder-automation-section",
   FEATURE_USER_MANAGEMENT: "feature-user-management",
+  BUILDER_FORM_CREATE_STEPS: "builder-form-create-steps",
+  BUILDER_FORM_VIEW_UPDATE_STEPS: "builder-form-view-update-steps",
+  BUILDER_FORM_ROW_ID: "builder-form-row-id",
 }
 
 export const TOUR_KEYS = {
   TOUR_BUILDER_ONBOARDING: "builder-onboarding",
   FEATURE_ONBOARDING: "feature-onboarding",
+  BUILDER_FORM_CREATE: "builder-form-create",
+  BUILDER_FORM_VIEW_UPDATE: "builder-form-view-update",
+}
+
+export const getCurrentStepIdx = (steps, tourStepKey) => {
+  if (!steps?.length) {
+    return
+  }
+  if (steps?.length && !tourStepKey) {
+    return 0
+  }
+  return steps.findIndex(step => step.id === tourStepKey)
 }
 
 const endUserOnboarding = async ({ skipped = false } = {}) => {
@@ -37,13 +59,8 @@ const endUserOnboarding = async ({ skipped = false } = {}) => {
       // Update the cached user
       await auth.getSelf()
 
-      builderStore.update(state => ({
+      builderStore.endBuilderOnboarding()
-        ...state,
+      builderStore.setTour()
-        tourNodes: null,
-        tourKey: null,
-        tourStepKey: null,
-        onboarding: false,
-      }))
     } catch (e) {
       console.error("Onboarding failed", e)
       return false
@@ -52,9 +69,29 @@ const endUserOnboarding = async ({ skipped = false } = {}) => {
   }
 }
 
-const tourEvent = eventKey => {
+const endTour = async ({ key, skipped = false } = {}) => {
+  const { tours = {} } = get(auth).user
+  tours[key] = new Date().toISOString()
+
+  await API.updateSelf({
+    tours,
+  })
+
+  if (skipped) {
+    tourEvent(key, skipped)
+  }
+
+  // Update the cached user
+  await auth.getSelf()
+
+  // Reset tour state
+  builderStore.setTour()
+}
+
+const tourEvent = (eventKey, skipped) => {
   analytics.captureEvent(`${ONBOARDING_EVENT_PREFIX}:${eventKey}`, {
     eventSource: EventSource.PORTAL,
+    skipped,
   })
 }
 
@@ -135,7 +172,71 @@ const getTours = () => {
         },
       ],
     },
+    [TOUR_KEYS.BUILDER_FORM_CREATE]: {
+      steps: [
+        {
+          id: TOUR_STEP_KEYS.BUILDER_FORM_CREATE_STEPS,
+          title: "Add multiple steps",
+          layout: NewFormSteps,
+          query: "#steps-prop-control-wrap",
+          onComplete: () => {
+            builderStore.highlightSetting()
+            endTour({ key: TOUR_KEYS.BUILDER_FORM_CREATE })
+          },
+          onLoad: () => {
+            tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_CREATE_STEPS)
+            builderStore.highlightSetting("steps", "info")
+          },
+          positionHandler: customPositionHandler,
+          align: "left-outside",
+        },
+      ],
+    },
+    [TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE]: {
+      steps: [
+        {
+          id: TOUR_STEP_KEYS.BUILDER_FORM_ROW_ID,
+          title: "Add row ID to update a row",
+          layout: NewViewUpdateFormRowId,
+          query: "#rowId-prop-control-wrap",
+          onLoad: () => {
+            tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_ROW_ID)
+            builderStore.highlightSetting("rowId", "info")
+          },
+          positionHandler: customPositionHandler,
+          align: "left-outside",
+        },
+        {
+          id: TOUR_STEP_KEYS.BUILDER_FORM_VIEW_UPDATE_STEPS,
+          title: "Add multiple steps",
+          layout: NewFormSteps,
+          query: "#steps-prop-control-wrap",
+          onComplete: () => {
+            builderStore.highlightSetting()
+            endTour({ key: TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE })
+          },
+          onLoad: () => {
+            tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_VIEW_UPDATE_STEPS)
+            builderStore.highlightSetting("steps", "info")
+          },
+          positionHandler: customPositionHandler,
+          align: "left-outside",
+          scrollIntoView: true,
+        },
+      ],
+      onSkip: async () => {
+        builderStore.highlightSetting()
+        endTour({ key: TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE, skipped: true })
+      },
+    },
   }
 }
 
 export const TOURS = getTours()
+export const TOURSBYSTEP = Object.keys(TOURS).reduce((acc, tour) => {
+  TOURS[tour].steps.forEach(element => {
+    acc[element.id] = element
+    acc[element.id]["tour"] = tour
+  })
+  return acc
+}, {})

packages/builder/src/dataBinding.js

@@ -1146,7 +1146,7 @@ export const getAllStateVariables = () => {
         "@budibase/standard-components/multistepformblockstep"
       )
 
-      steps.forEach(step => {
+      steps?.forEach(step => {
         parseComponentSettings(stepDefinition, step)
       })
     })

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -96,7 +96,7 @@
         const release_date = new Date("2023-03-01T00:00:00.000Z")
         const onboarded = new Date($auth.user?.onboardedAt)
         if (onboarded < release_date) {
-          builderStore.startTour(TOUR_KEYS.FEATURE_ONBOARDING)
+          builderStore.setTour(TOUR_KEYS.FEATURE_ONBOARDING)
         }
       }
     }
@@ -144,7 +144,7 @@
         </span>
         <Tabs {selected} size="M">
           {#each $layout.children as { path, title }}
-            <TourWrap tourStepKey={`builder-${title}-section`}>
+            <TourWrap stepKeys={[`builder-${title}-section`]}>
               <Tab
                 quiet
                 selected={$isActive(path)}
@@ -188,7 +188,7 @@
 {/if}
 
 <svelte:window on:keydown={handleKeyDown} />
-<Modal bind:this={commandPaletteModal}>
+<Modal bind:this={commandPaletteModal} zIndex={999999}>
   <CommandPalette />
 </Modal>
 

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte

@@ -16,6 +16,14 @@
   } from "dataBinding"
   import { ActionButton, notifications } from "@budibase/bbui"
   import { capitalise } from "helpers"
+  import TourWrap from "components/portal/onboarding/TourWrap.svelte"
+  import { TOUR_STEP_KEYS } from "components/portal/onboarding/tours.js"
+
+  const {
+    BUILDER_FORM_CREATE_STEPS,
+    BUILDER_FORM_VIEW_UPDATE_STEPS,
+    BUILDER_FORM_ROW_ID,
+  } = TOUR_STEP_KEYS
 
   const onUpdateName = async value => {
     try {
@@ -46,7 +54,6 @@
 
   $: id = $selectedComponent?._id
   $: id, (section = tabs[0])
-
   $: componentName = getComponentName(componentInstance)
 </script>
 
@@ -92,13 +99,21 @@
         </div>
       </span>
       {#if section == "settings"}
-        <ComponentSettingsSection
+        <TourWrap
-          {componentInstance}
+          stepKeys={[
-          {componentDefinition}
+            BUILDER_FORM_CREATE_STEPS,
-          {bindings}
+            BUILDER_FORM_VIEW_UPDATE_STEPS,
-          {componentBindings}
+            BUILDER_FORM_ROW_ID,
-          {isScreen}
+          ]}
-        />
+        >
+          <ComponentSettingsSection
+            {componentInstance}
+            {componentDefinition}
+            {bindings}
+            {componentBindings}
+            {isScreen}
+          />
+        </TourWrap>
       {/if}
       {#if section == "styles"}
         <DesignSection

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte

@@ -1,7 +1,7 @@
 <script>
   import { helpers } from "@budibase/shared-core"
   import { DetailSummary, notifications } from "@budibase/bbui"
-  import { componentStore } from "stores/builder"
+  import { componentStore, builderStore } from "stores/builder"
   import PropertyControl from "components/design/settings/controls/PropertyControl.svelte"
   import ResetFieldsButton from "components/design/settings/controls/ResetFieldsButton.svelte"
   import EjectBlockButton from "components/design/settings/controls/EjectBlockButton.svelte"
@@ -177,9 +177,7 @@
               defaultValue={setting.defaultValue}
               nested={setting.nested}
               onChange={val => updateSetting(setting, val)}
-              highlighted={$componentStore.highlightedSettingKey ===
+              propertyFocus={$builderStore.propertyFocus === setting.key}
-                setting.key}
-              propertyFocus={$componentStore.propertyFocus === setting.key}
               info={setting.info}
               disableBindings={setting.disableBindings}
               props={{

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte

@@ -176,7 +176,7 @@
     } else if (type === "request-add-component") {
       toggleAddComponent()
     } else if (type === "highlight-setting") {
-      builderStore.highlightSetting(data.setting)
+      builderStore.highlightSetting(data.setting, "error")
 
       // Also scroll setting into view
       const selector = `#${data.setting}-prop-control`

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte

@@ -3,13 +3,23 @@
   import DatasourceModal from "./DatasourceModal.svelte"
   import ScreenRoleModal from "./ScreenRoleModal.svelte"
   import sanitizeUrl from "helpers/sanitizeUrl"
+  import FormTypeModal from "./FormTypeModal.svelte"
   import { Modal, notifications } from "@budibase/bbui"
-  import { screenStore, navigationStore, tables } from "stores/builder"
+  import {
+    screenStore,
+    navigationStore,
+    tables,
+    builderStore,
+  } from "stores/builder"
+  import { auth } from "stores/portal"
   import { get } from "svelte/store"
   import getTemplates from "templates"
   import { Roles } from "constants/backend"
   import { capitalise } from "helpers"
   import { goto } from "@roxi/routify"
+  import { TOUR_KEYS } from "components/portal/onboarding/tours.js"
+  import formScreen from "templates/formScreen"
+  import rowListScreen from "templates/rowListScreen"
 
   let mode
   let pendingScreen
@@ -18,12 +28,18 @@
   let screenDetailsModal
   let datasourceModal
   let screenAccessRoleModal
+  let formTypeModal
 
   // Cache variables for workflow
   let screenAccessRole = Roles.BASIC
-  let selectedTemplates = null
+
+  let templates = null
+  let screens = null
+
+  let selectedDatasources = null
   let blankScreenUrl = null
   let screenMode = null
+  let formType = null
 
   // Creates an array of screens, checking and sanitising their URLs
   const createScreens = async ({ screens, screenAccessRole }) => {
@@ -32,7 +48,7 @@
     }
 
     try {
-      let screenId
+      let createdScreens = []
 
       for (let screen of screens) {
         // Check we aren't clashing with an existing URL
@@ -56,7 +72,7 @@
 
         // Create the screen
         const response = await screenStore.save(screen)
-        screenId = response._id
+        createdScreens.push(response)
 
         // Add link in layout. We only ever actually create 1 screen now, even
         // for autoscreens, so it's always safe to do this.
@@ -66,9 +82,7 @@
         )
       }
 
-      // Go to new screen
+      return createdScreens
-      $goto(`./${screenId}`)
-      screenStore.select(screenId)
     } catch (error) {
       console.error(error)
       notifications.error("Error creating screens")
@@ -104,13 +118,16 @@
   // Handler for NewScreenModal
   export const show = newMode => {
     mode = newMode
-    selectedTemplates = null
+    templates = null
+    screens = null
+    selectedDatasources = null
     blankScreenUrl = null
     screenMode = mode
     pendingScreen = null
     screenAccessRole = Roles.BASIC
+    formType = null
 
-    if (mode === "table" || mode === "grid") {
+    if (mode === "table" || mode === "grid" || mode === "form") {
       datasourceModal.show()
     } else if (mode === "blank") {
       let templates = getTemplates($tables.list)
@@ -125,19 +142,26 @@
   }
 
   // Handler for DatasourceModal confirmation, move to screen access select
-  const confirmScreenDatasources = async ({ templates }) => {
+  const confirmScreenDatasources = async ({ datasources }) => {
-    selectedTemplates = templates
+    selectedDatasources = datasources
-    screenAccessRoleModal.show()
+    if (screenMode === "form") {
+      formTypeModal.show()
+    } else {
+      screenAccessRoleModal.show()
+    }
   }
 
   // Handler for Datasource Screen Creation
   const completeDatasourceScreenCreation = async () => {
-    const screens = selectedTemplates.map(template => {
+    templates = rowListScreen(selectedDatasources, mode)
+
+    const screens = templates.map(template => {
       let screenTemplate = template.create()
       screenTemplate.autoTableId = template.resourceId
       return screenTemplate
     })
-    await createScreens({ screens, screenAccessRole })
+    const createdScreens = await createScreens({ screens, screenAccessRole })
+    loadNewScreen(createdScreens)
   }
 
   const confirmScreenBlank = async ({ screenUrl }) => {
@@ -154,7 +178,54 @@
       return
     }
     pendingScreen.routing.route = screenUrl
-    await createScreens({ screens: [pendingScreen], screenAccessRole })
+    const createdScreens = await createScreens({
+      screens: [pendingScreen],
+      screenAccessRole,
+    })
+    loadNewScreen(createdScreens)
+  }
+
+  const onConfirmFormType = () => {
+    screenAccessRoleModal.show()
+  }
+
+  const loadNewScreen = createdScreens => {
+    const lastScreen = createdScreens.slice(-1)[0]
+
+    // Go to new screen
+    if (lastScreen?.props?._children.length) {
+      // Focus on the main component for the streen type
+      const mainComponent = lastScreen?.props?._children?.[0]._id
+      $goto(`./${lastScreen._id}/${mainComponent}`)
+    } else {
+      $goto(`./${lastScreen._id}`)
+    }
+
+    screenStore.select(lastScreen._id)
+  }
+
+  const confirmFormScreenCreation = async () => {
+    templates = formScreen(selectedDatasources, { actionType: formType })
+    screens = templates.map(template => {
+      let screenTemplate = template.create()
+      return screenTemplate
+    })
+    const createdScreens = await createScreens({ screens, screenAccessRole })
+
+    if (formType === "Update" || formType === "Create") {
+      const associatedTour =
+        formType === "Update"
+          ? TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE
+          : TOUR_KEYS.BUILDER_FORM_CREATE
+
+      const tourRequired = !$auth?.user?.tours?.[associatedTour]
+      if (tourRequired) {
+        builderStore.setTour(associatedTour)
+      }
+    }
+
+    // Go to new screen
+    loadNewScreen(createdScreens)
   }
 
   // Submit screen config for creation.
@@ -164,6 +235,8 @@
         screenUrl: blankScreenUrl,
         screenAccessRole,
       })
+    } else if (screenMode === "form") {
+      confirmFormScreenCreation()
     } else {
       completeDatasourceScreenCreation()
     }
@@ -179,19 +252,18 @@
 </script>
 
 <Modal bind:this={datasourceModal} autoFocus={false}>
-  <DatasourceModal
+  <DatasourceModal {mode} onConfirm={confirmScreenDatasources} />
-    {mode}
-    onConfirm={confirmScreenDatasources}
-    initialScreens={!selectedTemplates ? [] : [...selectedTemplates]}
-  />
 </Modal>
 
 <Modal bind:this={screenAccessRoleModal}>
   <ScreenRoleModal
-    onConfirm={confirmScreenCreation}
+    onConfirm={() => {
-    onCancel={roleSelectBack}
+      confirmScreenCreation()
+    }}
     bind:screenAccessRole
+    onCancel={roleSelectBack}
     screenUrl={blankScreenUrl}
+    confirmText={screenMode === "form" ? "Confirm" : "Done"}
   />
 </Modal>
 
@@ -201,3 +273,17 @@
     initialUrl={blankScreenUrl}
   />
 </Modal>
+
+<Modal bind:this={formTypeModal}>
+  <FormTypeModal
+    onConfirm={onConfirmFormType}
+    onCancel={() => {
+      formTypeModal.hide()
+      datasourceModal.show()
+    }}
+    on:select={e => {
+      formType = e.detail
+    }}
+    type={formType}
+  />
+</Modal>

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte

@@ -4,37 +4,33 @@
   import ICONS from "components/backend/DatasourceNavigator/icons"
   import { IntegrationNames } from "constants"
   import { onMount } from "svelte"
-  import rowListScreen from "templates/rowListScreen"
   import DatasourceTemplateRow from "./DatasourceTemplateRow.svelte"
 
-  export let mode
   export let onCancel
   export let onConfirm
-  export let initialScreens = []
 
-  let selectedScreens = [...initialScreens]
+  let selectedSources = []
 
   $: filteredSources = $datasources.list?.filter(datasource => {
     return datasource.source !== IntegrationNames.REST && datasource["entities"]
   })
 
   const toggleSelection = datasource => {
-    const { resourceId } = datasource
+    const exists = selectedSources.find(
-    if (selectedScreens.find(s => s.resourceId === resourceId)) {
+      d => d.resourceId === datasource.resourceId
-      selectedScreens = selectedScreens.filter(
+    )
-        screen => screen.resourceId !== resourceId
+    if (exists) {
+      selectedSources = selectedSources.filter(
+        d => d.resourceId === datasource.resourceId
       )
     } else {
-      selectedScreens = [
+      selectedSources = [...selectedSources, datasource]
-        ...selectedScreens,
-        rowListScreen([datasource], mode)[0],
-      ]
     }
   }
 
   const confirmDatasourceSelection = async () => {
     await onConfirm({
-      templates: selectedScreens,
+      datasources: selectedSources,
     })
   }
 
@@ -54,7 +50,7 @@
     cancelText="Back"
     onConfirm={confirmDatasourceSelection}
     {onCancel}
-    disabled={!selectedScreens.length}
+    disabled={!selectedSources.length}
     size="L"
   >
     <Body size="S">
@@ -85,8 +81,8 @@
               resourceId: table._id,
               type: "table",
             }}
-            {@const selected = selectedScreens.find(
+            {@const selected = selectedSources.find(
-              screen => screen.resourceId === tableDS.resourceId
+              datasource => datasource.resourceId === tableDS.resourceId
             )}
             <DatasourceTemplateRow
               on:click={() => toggleSelection(tableDS)}
@@ -103,7 +99,7 @@
                 tableId: view.tableId,
                 type: "viewV2",
               }}
-              {@const selected = selectedScreens.find(
+              {@const selected = selectedSources.find(
                 x => x.resourceId === viewDS.resourceId
               )}
               <DatasourceTemplateRow

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/FormTypeModal.svelte

@@ -0,0 +1,123 @@
+<script>
+  import { ModalContent, Layout, Body, Icon } from "@budibase/bbui"
+  import { createEventDispatcher } from "svelte"
+
+  export let onCancel = () => {}
+  export let onConfirm = () => {}
+  export let type
+
+  const dispatch = createEventDispatcher()
+</script>
+
+<span>
+  <ModalContent
+    title="Select form type"
+    confirmText="Done"
+    cancelText="Back"
+    {onConfirm}
+    {onCancel}
+    disabled={!type}
+    size="L"
+  >
+    <!-- svelte-ignore a11y-click-events-have-key-events -->
+    <!-- svelte-ignore a11y-no-static-element-interactions -->
+    <Layout noPadding gap="S">
+      <div
+        class="form-type"
+        class:selected={type === "Create"}
+        on:click={() => {
+          dispatch("select", "Create")
+        }}
+      >
+        <div class="form-type-wrap">
+          <div class="form-type-content">
+            <Body noPadding>Create a new row</Body>
+            <Body size="S">
+              For capturing and storing new data from your users
+            </Body>
+          </div>
+          {#if type === "Create"}
+            <span class="form-type-check">
+              <Icon size="S" name="CheckmarkCircle" />
+            </span>
+          {/if}
+        </div>
+      </div>
+      <div
+        class="form-type"
+        class:selected={type === "Update"}
+        on:click={() => {
+          dispatch("select", "Update")
+        }}
+      >
+        <div class="form-type-wrap">
+          <div class="form-type-content">
+            <Body noPadding>Update an existing row</Body>
+            <Body size="S">For viewing and updating existing data</Body>
+          </div>
+          {#if type === "Update"}
+            <span class="form-type-check">
+              <Icon size="S" name="CheckmarkCircle" />
+            </span>
+          {/if}
+        </div>
+      </div>
+      <div
+        class="form-type"
+        class:selected={type === "View"}
+        on:click={() => {
+          dispatch("select", "View")
+        }}
+      >
+        <div class="form-type-wrap">
+          <div class="form-type-content">
+            <Body noPadding>View an existing row</Body>
+            <Body size="S">For a read only view of your data</Body>
+          </div>
+          {#if type === "View"}
+            <span class="form-type-check">
+              <Icon size="S" name="CheckmarkCircle" />
+            </span>
+          {/if}
+        </div>
+      </div>
+    </Layout>
+  </ModalContent>
+</span>
+
+<style>
+  .form-type {
+    cursor: pointer;
+    gap: var(--spacing-s);
+    padding: var(--spacing-m) var(--spacing-xl);
+    background: var(--spectrum-alias-background-color-secondary);
+    transition: 0.3s all;
+    border: 1px solid var(--spectrum-global-color-gray-300);
+    border-radius: 4px;
+    display: flex;
+    flex-direction: column;
+  }
+  .selected,
+  .form-type:hover {
+    background: var(--spectrum-alias-background-color-tertiary);
+  }
+  .form-type-wrap {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+  }
+  .form-type :global(p:nth-child(2)) {
+    color: var(--grey-6);
+  }
+  .form-type-check {
+    margin-left: auto;
+  }
+  .form-type-check :global(.spectrum-Icon) {
+    color: var(--spectrum-global-color-green-600);
+  }
+  .form-type-content {
+    gap: var(--spacing-s);
+    display: flex;
+    flex-direction: column;
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte

@@ -9,6 +9,7 @@
   export let onCancel
   export let screenUrl
   export let screenAccessRole
+  export let confirmText = "Done"
 
   let error
 
@@ -40,7 +41,7 @@
 
 <ModalContent
   title="Access"
-  confirmText="Done"
+  {confirmText}
   cancelText="Back"
   {onConfirm}
   {onCancel}

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte

@@ -1,9 +1,10 @@
 <script>
   import { Body } from "@budibase/bbui"
   import CreationPage from "components/common/CreationPage.svelte"
-  import blankImage from "./blank.png"
+  import blankImage from "./images/blank.png"
-  import tableImage from "./table.png"
+  import tableImage from "./images/table.png"
-  import gridImage from "./grid.png"
+  import gridImage from "./images/grid.png"
+  import formImage from "./images/form.png"
   import CreateScreenModal from "./CreateScreenModal.svelte"
   import { screenStore } from "stores/builder"
 
@@ -56,6 +57,16 @@
           <Body size="XS">View and manipulate rows on a grid</Body>
         </div>
       </div>
+
+      <div class="card" on:click={() => createScreenModal.show("form")}>
+        <div class="image">
+          <img alt="" src={formImage} />
+        </div>
+        <div class="text">
+          <Body size="S">Form</Body>
+          <Body size="XS">Capture data from your users</Body>
+        </div>
+      </div>
     </div>
   </CreationPage>
 </div>

packages/builder/src/pages/builder/portal/users/_components/ActiveDirectoryInfo.svelte

@@ -0,0 +1,23 @@
+<script>
+  import { IconAvatar } from "@budibase/bbui"
+
+  export let text = ""
+  export let iconSize = "S"
+</script>
+
+<div class="scim-banner">
+  <IconAvatar
+    icon="Sync"
+    size={iconSize}
+    background={"var(--spectrum-global-color-gray-500)"}
+    tooltip="Synced from your AD"
+  />
+  {text}
+</div>
+
+<style>
+  .scim-banner {
+    display: flex;
+    gap: var(--spacing-s);
+  }
+</style>

packages/builder/src/pages/builder/portal/users/_components/SCIMBanner.svelte

@@ -1,15 +0,0 @@
-<script>
-  import { Icon } from "@budibase/bbui"
-</script>
-
-<div class="scim-banner">
-  <Icon name="Info" size="S" />
-  Users are synced from your AD
-</div>
-
-<style>
-  .scim-banner {
-    display: flex;
-    gap: var(--spacing-s);
-  }
-</style>

packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte

@@ -13,7 +13,7 @@
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import { Breadcrumb, Breadcrumbs } from "components/portal/page"
   import { roles } from "stores/builder"
-  import { apps, auth, features, groups } from "stores/portal"
+  import { apps, auth, groups } from "stores/portal"
   import { onMount, setContext } from "svelte"
   import AppNameTableRenderer from "../users/_components/AppNameTableRenderer.svelte"
   import AppRoleTableRenderer from "../users/_components/AppRoleTableRenderer.svelte"
@@ -47,9 +47,9 @@
   let loaded = false
   let editModal, deleteModal
 
-  $: scimEnabled = $features.isScimEnabled
-  $: readonly = !sdk.users.isAdmin($auth.user) || scimEnabled
   $: group = $groups.find(x => x._id === groupId)
+  $: isScimGroup = group?.scimInfo?.isSync
+  $: readonly = !sdk.users.isAdmin($auth.user) || isScimGroup
   $: groupApps = $apps
     .filter(app =>
       groups.actions
@@ -119,23 +119,27 @@
     <div class="header">
       <GroupIcon {group} size="L" />
       <Heading>{group?.name}</Heading>
-      {#if !readonly}
+      <ActionMenu align="right">
-        <ActionMenu align="right">
+        <span slot="control">
-          <span slot="control">
+          <Icon hoverable name="More" />
-            <Icon hoverable name="More" />
+        </span>
-          </span>
+        <MenuItem icon="Refresh" on:click={() => editModal.show()}>
-          <MenuItem icon="Refresh" on:click={() => editModal.show()}>
+          Edit
-            Edit
+        </MenuItem>
-          </MenuItem>
+        <div title={isScimGroup && "Group synced from your AD"}>
-          <MenuItem icon="Delete" on:click={() => deleteModal.show()}>
+          <MenuItem
+            icon="Delete"
+            on:click={() => deleteModal.show()}
+            disabled={isScimGroup}
+          >
             Delete
           </MenuItem>
-        </ActionMenu>
+        </div>
-      {/if}
+      </ActionMenu>
     </div>
 
     <Layout noPadding gap="S">
-      <GroupUsers {groupId} />
+      <GroupUsers {groupId} {readonly} />
     </Layout>
 
     <Layout noPadding gap="S">

packages/builder/src/pages/builder/portal/users/groups/_components/CreateEditGroupModal.svelte

@@ -11,6 +11,8 @@
   export let group
   export let saveGroup
 
+  let readonlyTitle = group?.scimInfo?.isSync
+
   let nameError
 </script>
 
@@ -26,7 +28,12 @@
   title={group?._rev ? "Edit group" : "Create group"}
   confirmText="Save"
 >
-  <Input bind:value={group.name} label="Name" error={nameError} />
+  <Input
+    bind:value={group.name}
+    label="Name"
+    error={nameError}
+    disabled={readonlyTitle}
+  />
   <div class="modal-format">
     <div class="modal-inner">
       <Body size="XS">Icon</Body>

packages/builder/src/pages/builder/portal/users/groups/_components/EditUserPicker.svelte

@@ -2,8 +2,7 @@
   import { Button, Popover, notifications } from "@budibase/bbui"
   import UserGroupPicker from "components/settings/UserGroupPicker.svelte"
   import { createPaginationStore } from "helpers/pagination"
-  import { auth, groups, users } from "stores/portal"
+  import { groups, users } from "stores/portal"
-  import { sdk } from "@budibase/shared-core"
 
   export let groupId
   export let onUsersUpdated
@@ -14,7 +13,6 @@
   let prevSearch = undefined
   let pageInfo = createPaginationStore()
 
-  $: readonly = !sdk.users.isAdmin($auth.user)
   $: page = $pageInfo.page
   $: searchUsers(page, searchTerm)
   $: group = $groups.find(x => x._id === groupId)
@@ -43,7 +41,7 @@
 </script>
 
 <div bind:this={popoverAnchor}>
-  <Button disabled={readonly} on:click={popover.show()} cta>Add user</Button>
+  <Button on:click={popover.show()} cta>Add user</Button>
 </div>
 <Popover align="right" bind:this={popover} anchor={popoverAnchor}>
   <UserGroupPicker

packages/builder/src/pages/builder/portal/users/groups/_components/GroupIcon.svelte

@@ -5,4 +5,12 @@
   export let size = "M"
 </script>
 
-<IconAvatar icon={group?.icon} background={group?.color} {size} />
+<div class="icon-group">
+  <IconAvatar icon={group?.icon} background={group?.color} {size} />
+</div>
+
+<style>
+  .icon-group {
+    position: relative;
+  }
+</style>

packages/builder/src/pages/builder/portal/users/groups/_components/GroupNameTableRenderer.svelte

@@ -1,5 +1,6 @@
 <script>
   import GroupIcon from "./GroupIcon.svelte"
+  import ActiveDirectoryInfo from "../../_components/ActiveDirectoryInfo.svelte"
 
   export let value
   export let row
@@ -14,13 +15,16 @@
   {:else}
     <div class="text">-</div>
   {/if}
+  {#if row.scimInfo?.isSync}
+    <ActiveDirectoryInfo iconSize="XS" />
+  {/if}
 </div>
 
 <style>
   .align {
     display: flex;
     align-items: center;
-    overflow: hidden;
+    overflow: visible;
     gap: var(--spacing-m);
     max-width: var(--max-cell-width);
     flex: 1 1 auto;

packages/builder/src/pages/builder/portal/users/groups/_components/GroupUsers.svelte

@@ -5,13 +5,14 @@
   import { fetchData } from "@budibase/frontend-core"
   import { goto } from "@roxi/routify"
   import { API } from "api"
-  import { auth, features, groups } from "stores/portal"
+  import { groups } from "stores/portal"
   import { setContext } from "svelte"
-  import ScimBanner from "../../_components/SCIMBanner.svelte"
+
   import RemoveUserTableRenderer from "../_components/RemoveUserTableRenderer.svelte"
-  import { sdk } from "@budibase/shared-core"
+  import ActiveDirectoryInfo from "../../_components/ActiveDirectoryInfo.svelte"
 
   export let groupId
+  export let readonly
 
   let emailSearch
   let fetchGroupUsers
@@ -49,9 +50,6 @@
     },
   ]
 
-  $: scimEnabled = $features.isScimEnabled
-  $: readonly = !sdk.users.isAdmin($auth.user) || scimEnabled
-
   const removeUser = async id => {
     await groups.actions.removeUser(groupId, id)
     fetchGroupUsers.refresh()
@@ -63,10 +61,10 @@
 </script>
 
 <div class="header">
-  {#if !scimEnabled}
+  {#if !readonly}
     <EditUserPicker {groupId} onUsersUpdated={fetchGroupUsers.getInitialData} />
   {:else}
-    <ScimBanner />
+    <ActiveDirectoryInfo text="Users synced from your AD" />
   {/if}
 
   <div class="controls-right">

packages/builder/src/pages/builder/portal/users/groups/index.svelte

@@ -13,7 +13,7 @@
     Search,
     notifications,
   } from "@budibase/bbui"
-  import { groups, auth, licensing, admin, features } from "stores/portal"
+  import { groups, auth, licensing, admin } from "stores/portal"
   import { onMount } from "svelte"
   import CreateEditGroupModal from "./_components/CreateEditGroupModal.svelte"
   import { cloneDeep } from "lodash/fp"
@@ -21,7 +21,6 @@
   import UsersTableRenderer from "./_components/UsersTableRenderer.svelte"
   import GroupNameTableRenderer from "./_components/GroupNameTableRenderer.svelte"
   import { goto } from "@roxi/routify"
-  import ScimBanner from "../_components/SCIMBanner.svelte"
   import { sdk } from "@budibase/shared-core"
 
   const DefaultGroup = {
@@ -110,14 +109,10 @@
   <div class="controls">
     <ButtonGroup>
       {#if $licensing.groupsEnabled}
-        {#if !$features.isScimEnabled}
+        <!--Show the group create button-->
-          <!--Show the group create button-->
+        <Button disabled={readonly} cta on:click={showCreateGroupModal}>
-          <Button disabled={readonly} cta on:click={showCreateGroupModal}>
+          Add group
-            Add group
+        </Button>
-          </Button>
-        {:else}
-          <ScimBanner />
-        {/if}
       {:else}
         <Button
           primary

packages/builder/src/pages/builder/portal/users/users/[userId].svelte

@@ -18,7 +18,7 @@
     Table,
   } from "@budibase/bbui"
   import { onMount, setContext } from "svelte"
-  import { users, auth, groups, apps, licensing, features } from "stores/portal"
+  import { users, auth, groups, apps, licensing } from "stores/portal"
   import { roles } from "stores/builder"
   import ForceResetPasswordModal from "./_components/ForceResetPasswordModal.svelte"
   import UserGroupPicker from "components/settings/UserGroupPicker.svelte"
@@ -30,8 +30,8 @@
   import GroupNameTableRenderer from "../groups/_components/GroupNameTableRenderer.svelte"
   import AppNameTableRenderer from "./_components/AppNameTableRenderer.svelte"
   import AppRoleTableRenderer from "./_components/AppRoleTableRenderer.svelte"
-  import ScimBanner from "../_components/SCIMBanner.svelte"
   import { sdk } from "@budibase/shared-core"
+  import ActiveDirectoryInfo from "../_components/ActiveDirectoryInfo.svelte"
 
   export let userId
 
@@ -87,12 +87,13 @@
   let user
   let loaded = false
 
-  $: scimEnabled = $features.isScimEnabled
+  $: internalGroups = $groups?.filter(g => !g?.scimInfo?.isSync)
+
   $: isSSO = !!user?.provider
-  $: readonly = !sdk.users.isAdmin($auth.user) || scimEnabled
+  $: readonly = !sdk.users.isAdmin($auth.user) || user?.scimInfo?.isSync
   $: privileged = sdk.users.isAdminOrGlobalBuilder(user)
   $: nameLabel = getNameLabel(user)
-  $: filteredGroups = getFilteredGroups($groups, searchTerm)
+  $: filteredGroups = getFilteredGroups(internalGroups, searchTerm)
   $: availableApps = getAvailableApps($apps, privileged, user?.roles)
   $: userGroups = $groups.filter(x => {
     return x.users?.find(y => {
@@ -274,8 +275,8 @@
     <Layout noPadding gap="S">
       <div class="details-title">
         <Heading size="S">Details</Heading>
-        {#if scimEnabled}
+        {#if user?.scimInfo?.isSync}
-          <ScimBanner />
+          <ActiveDirectoryInfo text="User synced from your AD" />
         {/if}
       </div>
       <div class="fields">
@@ -321,11 +322,11 @@
       <Layout gap="S" noPadding>
         <div class="tableTitle">
           <Heading size="S">Groups</Heading>
-          <div bind:this={popoverAnchor}>
+          {#if internalGroups?.length}
-            <Button disabled={readonly} on:click={popover.show()} secondary>
+            <div bind:this={popoverAnchor}>
-              Add to group
+              <Button on:click={popover.show()} secondary>Add to group</Button>
-            </Button>
+            </div>
-          </div>
+          {/if}
           <Popover align="right" bind:this={popover} anchor={popoverAnchor}>
             <UserGroupPicker
               labelKey="name"

packages/builder/src/pages/builder/portal/users/users/_components/AddUserModal.svelte

@@ -33,6 +33,8 @@
   $: reached = licensing.usersLimitReached(userCount)
   $: exceeded = licensing.usersLimitExceeded(userCount)
 
+  $: internalGroups = $groups?.filter(g => !g?.scimInfo?.isSync)
+
   function removeInput(idx) {
     userData = userData.filter((e, i) => i !== idx)
   }
@@ -133,12 +135,12 @@
     {/if}
   </Layout>
 
-  {#if $licensing.groupsEnabled}
+  {#if $licensing.groupsEnabled && internalGroups?.length}
     <Multiselect
       bind:value={userGroups}
       placeholder="No groups"
       label="Groups"
-      options={$groups}
+      options={internalGroups}
       getOptionLabel={option => option.name}
       getOptionValue={option => option._id}
     />

packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte

@@ -0,0 +1,11 @@
+<script>
+  import ActiveDirectoryInfo from "../../_components/ActiveDirectoryInfo.svelte"
+
+  export let value
+  export let row
+</script>
+
+{value}
+{#if row.scimInfo?.isSync}
+  <ActiveDirectoryInfo iconSize="XS" />
+{/if}

packages/builder/src/pages/builder/portal/users/users/_components/ImportUsersModal.svelte

@@ -34,6 +34,8 @@
     label: `${option.label} - ${option.subtitle}`,
   }))
 
+  $: internalGroups = $groups?.filter(g => !g?.scimInfo?.isSync)
+
   const validEmails = userEmails => {
     if ($admin.cloud && userEmails.length > MAX_USERS_UPLOAD_LIMIT) {
       notifications.error(
@@ -106,12 +108,12 @@
   {/if}
   <RadioGroup bind:value={usersRole} options={roleOptions} />
 
-  {#if $licensing.groupsEnabled}
+  {#if $licensing.groupsEnabled && internalGroups?.length}
     <Multiselect
       bind:value={userGroups}
       placeholder="No groups"
       label="Groups"
-      options={$groups}
+      options={internalGroups}
       getOptionLabel={option => option.name}
       getOptionValue={option => option._id}
     />

packages/builder/src/pages/builder/portal/users/users/_components/RemoveGroupTableRenderer.svelte

@@ -5,6 +5,7 @@
   import { sdk } from "@budibase/shared-core"
 
   export let value
+  export let row
 
   const groupContext = getContext("groups")
 
@@ -12,12 +13,11 @@
     e.stopPropagation()
     groupContext.removeGroup(value)
   }
+
+  $: disabled = !sdk.users.isAdmin($auth.user) || row?.scimInfo?.isSync
+  $: tooltip = row?.scimInfo?.isSync && "This group is managed via your AD"
 </script>
 
-<ActionButton
+<ActionButton {disabled} size="S" on:click={onClick} {tooltip}
-  disabled={!sdk.users.isAdmin($auth.user)}
+  >Remove</ActionButton
-  size="S"
-  on:click={onClick}
 >
-  Remove
-</ActionButton>

packages/builder/src/pages/builder/portal/users/users/index.svelte

@@ -19,7 +19,6 @@
     auth,
     licensing,
     organisation,
-    features,
     admin,
   } from "stores/portal"
   import { onMount } from "svelte"
@@ -28,6 +27,7 @@
   import GroupsTableRenderer from "./_components/GroupsTableRenderer.svelte"
   import AppsTableRenderer from "./_components/AppsTableRenderer.svelte"
   import RoleTableRenderer from "./_components/RoleTableRenderer.svelte"
+  import EmailTableRenderer from "./_components/EmailTableRenderer.svelte"
   import { goto } from "@roxi/routify"
   import OnboardingTypeModal from "./_components/OnboardingTypeModal.svelte"
   import PasswordModal from "./_components/PasswordModal.svelte"
@@ -37,7 +37,6 @@
   import { Constants, Utils, fetchData } from "@budibase/frontend-core"
   import { API } from "api"
   import { OnboardingType } from "constants"
-  import ScimBanner from "../_components/SCIMBanner.svelte"
   import { sdk } from "@budibase/shared-core"
 
   const fetch = fetchData({
@@ -63,6 +62,7 @@
   let selectedRows = []
   let bulkSaveResponse
   let customRenderers = [
+    { column: "email", component: EmailTableRenderer },
     { column: "userGroups", component: GroupsTableRenderer },
     { column: "apps", component: AppsTableRenderer },
     { column: "role", component: RoleTableRenderer },
@@ -73,7 +73,7 @@
   let parsedInvites = []
 
   $: isOwner = $auth.accountPortalAccess && $admin.cloud
-  $: readonly = !sdk.users.isAdmin($auth.user) || $features.isScimEnabled
+  $: readonly = !sdk.users.isAdmin($auth.user)
   $: debouncedUpdateFetch(searchEmail)
   $: schema = {
     email: {
@@ -247,19 +247,25 @@
     }
   }
 
-  const deleteRows = async () => {
+  const deleteUsers = async () => {
     try {
       let ids = selectedRows.map(user => user._id)
       if (ids.includes(get(auth).user._id)) {
         notifications.error("You cannot delete yourself")
         return
       }
+
+      if (selectedRows.some(u => u.scimInfo?.isSync)) {
+        notifications.error("You cannot remove users created via your AD")
+        return
+      }
+
       await users.bulkDelete(ids)
       notifications.success(`Successfully deleted ${selectedRows.length} rows`)
       selectedRows = []
       await fetch.refresh()
     } catch (error) {
-      notifications.error("Error deleting rows")
+      notifications.error("Error deleting users")
     }
   }
 
@@ -320,8 +326,6 @@
           Import
         </Button>
       </div>
-    {:else}
-      <ScimBanner />
     {/if}
     <div class="controls-right">
       <Search bind:value={searchEmail} placeholder="Search" />
@@ -330,7 +334,7 @@
           item="user"
           on:updaterows
           {selectedRows}
-          {deleteRows}
+          deleteRows={deleteUsers}
         />
       {/if}
     </div>
@@ -346,6 +350,7 @@
     {customRenderers}
     loading={!$fetch.loaded || !groupsLoaded}
   />
+
   <div class="pagination">
     <Pagination
       page={$fetch.pageNumber + 1}
@@ -355,6 +360,7 @@
       goToNextPage={fetch.nextPage}
     />
   </div>
+
   <Table
     schema={pendingSchema}
     data={parsedInvites}
@@ -402,6 +408,7 @@
     display: flex;
     flex-direction: row;
     justify-content: flex-end;
+    margin-left: auto;
   }
   .controls {
     display: flex;

packages/builder/src/stores/builder/app.js

@@ -29,6 +29,7 @@ export const INITIAL_APP_META_STATE = {
   initialised: false,
   hasAppPackage: false,
   usedPlugins: null,
+  automations: {},
   routes: {},
 }
 
@@ -63,6 +64,7 @@ export class AppMetaStore extends BudiStore {
         ...app.features,
       },
       initialised: true,
+      automations: app.automations || {},
       hasAppPackage: true,
     }))
   }

packages/builder/src/stores/builder/builder.js

@@ -2,12 +2,11 @@ import { get } from "svelte/store"
 import { createBuilderWebsocket } from "./websocket.js"
 import { BuilderSocketEvent } from "@budibase/shared-core"
 import BudiStore from "./BudiStore"
-import { previewStore } from "./preview.js"
 import { TOUR_KEYS } from "components/portal/onboarding/tours.js"
 
 export const INITIAL_BUILDER_STATE = {
   previousTopNavPath: {},
-  highlightedSettingKey: null,
+  highlightedSetting: null,
   propertyFocus: null,
   builderSidePanel: false,
   onboarding: false,
@@ -26,7 +25,6 @@ export class BuilderStore extends BudiStore {
     this.reset = this.reset.bind(this)
     this.highlightSetting = this.highlightSetting.bind(this)
     this.propertyFocus = this.propertyFocus.bind(this)
-    this.hover = this.hover.bind(this)
     this.hideBuilderSidePanel = this.hideBuilderSidePanel.bind(this)
     this.showBuilderSidePanel = this.showBuilderSidePanel.bind(this)
     this.setPreviousTopNavPath = this.setPreviousTopNavPath.bind(this)
@@ -58,10 +56,10 @@ export class BuilderStore extends BudiStore {
     this.websocket = null
   }
 
-  highlightSetting(key) {
+  highlightSetting(key, type) {
     this.update(state => ({
       ...state,
-      highlightedSettingKey: key,
+      highlightedSetting: key ? { key, type: type || "info" } : null,
     }))
   }
 
@@ -135,25 +133,20 @@ export class BuilderStore extends BudiStore {
     }))
   }
 
-  startTour(tourKey) {
+  endBuilderOnboarding() {
     this.update(state => ({
       ...state,
-      tourKey: tourKey,
+      onboarding: false,
     }))
   }
 
-  hover(componentId, notifyClient = true) {
+  setTour(tourKey) {
-    const store = get(this.store)
+    this.update(state => ({
-    if (componentId === store.hoveredComponentId) {
+      ...state,
-      return
+      tourStepKey: null,
-    }
+      tourNodes: null,
-    this.update(state => {
+      tourKey: tourKey,
-      state.hoveredComponentId = componentId
+    }))
-      return state
-    })
-    if (notifyClient) {
-      previewStore.sendEvent("hover-component", componentId)
-    }
   }
 }
 

packages/builder/src/stores/builder/tests/builder.test.js

@@ -88,14 +88,42 @@ describe("Builder store", () => {
     )
   })
 
-  it("Sync a highlighted setting key to state", ctx => {
+  it("Sync a highlighted setting key to state. Default to info type", ctx => {
-    expect(ctx.test.store.highlightedSettingKey).toBeNull()
+    expect(ctx.test.store.highlightedSetting).toBeNull()
 
     ctx.test.builderStore.highlightSetting("testing")
 
     expect(ctx.test.store).toStrictEqual({
       ...INITIAL_BUILDER_STATE,
-      highlightedSettingKey: "testing",
+      highlightedSetting: {
+        key: "testing",
+        type: "info",
+      },
+    })
+  })
+
+  it("Sync a highlighted setting key to state. Use provided type", ctx => {
+    expect(ctx.test.store.highlightedSetting).toBeNull()
+
+    ctx.test.builderStore.highlightSetting("testing", "error")
+
+    expect(ctx.test.store).toStrictEqual({
+      ...INITIAL_BUILDER_STATE,
+      highlightedSetting: {
+        key: "testing",
+        type: "error",
+      },
+    })
+  })
+
+  it("Sync a highlighted setting key to state. Unset when no value is passed", ctx => {
+    expect(ctx.test.store.highlightedSetting).toBeNull()
+
+    ctx.test.builderStore.highlightSetting("testing", "error")
+    ctx.test.builderStore.highlightSetting()
+
+    expect(ctx.test.store).toStrictEqual({
+      ...INITIAL_BUILDER_STATE,
     })
   })
 

packages/builder/src/stores/portal/groups.js

@@ -35,7 +35,8 @@ export function createGroupsStore() {
     get: getGroup,
 
     save: async group => {
-      const response = await API.saveGroup(group)
+      const { _scimInfo, ...dataToSave } = group
+      const response = await API.saveGroup(dataToSave)
       group._id = response._id
       group._rev = response._rev
       updateStore(group)

packages/builder/src/templates/formScreen.js

@@ -0,0 +1,43 @@
+import { Screen } from "./Screen"
+import { Component } from "./Component"
+import sanitizeUrl from "helpers/sanitizeUrl"
+
+export const FORM_TEMPLATE = "FORM_TEMPLATE"
+export const formUrl = datasource => sanitizeUrl(`/${datasource.label}-form`)
+
+// Mode not really necessary
+export default function (datasources, config) {
+  if (!Array.isArray(datasources)) {
+    return []
+  }
+  return datasources.map(datasource => {
+    return {
+      name: `${datasource.label} - Form`,
+      create: () => createScreen(datasource, config),
+      id: FORM_TEMPLATE,
+      resourceId: datasource.resourceId,
+    }
+  })
+}
+
+const generateMultistepFormBlock = (dataSource, { actionType } = {}) => {
+  const multistepFormBlock = new Component(
+    "@budibase/standard-components/multistepformblock"
+  )
+  multistepFormBlock
+    .customProps({
+      actionType,
+      dataSource,
+      steps: [{}],
+    })
+    .instanceName(`${dataSource.label} - Multistep Form block`)
+  return multistepFormBlock
+}
+
+const createScreen = (datasource, config) => {
+  return new Screen()
+    .route(formUrl(datasource))
+    .instanceName(`${datasource.label} - Form`)
+    .addChild(generateMultistepFormBlock(datasource, config))
+    .json()
+}

packages/builder/src/templates/index.js

@@ -1,7 +1,11 @@
 import rowListScreen from "./rowListScreen"
 import createFromScratchScreen from "./createFromScratchScreen"
+import formScreen from "./formScreen"
 
-const allTemplates = datasources => [...rowListScreen(datasources)]
+const allTemplates = datasources => [
+  ...rowListScreen(datasources),
+  ...formScreen(datasources),
+]
 
 // Allows us to apply common behaviour to all create() functions
 const createTemplateOverride = template => () => {
@@ -19,6 +23,7 @@ export default datasources => {
   })
   const fromScratch = enrichTemplate(createFromScratchScreen)
   const tableTemplates = allTemplates(datasources).map(enrichTemplate)
+
   return [
     fromScratch,
     ...tableTemplates.sort((templateA, templateB) => {

packages/client/manifest.json

@@ -525,6 +525,38 @@
         "barTitle": "Disable button",
         "key": "disabled"
       },
+      {
+        "type": "icon",
+        "label": "Icon",
+        "key": "icon"
+      },
+      {
+        "type": "select",
+        "label": "Gap",
+        "key": "gap",
+        "showInBar": true,
+        "barStyle": "picker",
+        "dependsOn": "icon",
+        "options": [
+          {
+            "label": "None",
+            "value": "N"
+          },
+          {
+            "label": "Small",
+            "value": "S"
+          },
+          {
+            "label": "Medium",
+            "value": "M"
+          },
+          {
+            "label": "Large",
+            "value": "L"
+          }
+        ],
+        "defaultValue": "M"
+      },
       {
         "type": "event",
         "label": "On click",
@@ -1105,6 +1137,12 @@
         "key": "color",
         "showInBar": true
       },
+      {
+        "type": "color",
+        "label": "Text Color",
+        "key": "textColor",
+        "showInBar": true
+      },
       {
         "type": "boolean",
         "label": "Allow delete",