Merge branch 'master' into feature/app-list-actions

.eslintrc.json

@@ -44,7 +44,8 @@
         "no-undef": "off",
         "no-prototype-builtins": "off",
         "local-rules/no-budibase-imports": "error",
-        "local-rules/no-test-com": "error"
+        "local-rules/no-test-com": "error",
+        "local-rules/email-domain-example-com": "error"
       }
     },
     {

eslint-local-rules/index.js

@@ -51,4 +51,41 @@ module.exports = {
       }
     },
   },
+  "email-domain-example-com": {
+    meta: {
+      type: "problem",
+      docs: {
+        description:
+          "enforce using the example.com domain for generator.email calls",
+        category: "Possible Errors",
+        recommended: false,
+      },
+      fixable: "code",
+      schema: [],
+    },
+    create: function (context) {
+      return {
+        CallExpression(node) {
+          if (
+            node.callee.type === "MemberExpression" &&
+            node.callee.object.name === "generator" &&
+            node.callee.property.name === "email" &&
+            node.arguments.length === 0
+          ) {
+            context.report({
+              node,
+              message:
+                "Prefer using generator.email with the domain \"{ domain: 'example.com' }\".",
+              fix: function (fixer) {
+                return fixer.replaceText(
+                  node,
+                  'generator.email({ domain: "example.com" })'
+                )
+              },
+            })
+          }
+        },
+      }
+    },
+  },
 }

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.21.3",
+  "version": "2.21.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/package.json

@@ -67,7 +67,7 @@
     "@types/lodash": "4.14.200",
     "@types/node-fetch": "2.6.4",
     "@types/pouchdb": "6.4.0",
-    "@types/redlock": "4.0.3",
+    "@types/redlock": "4.0.7",
     "@types/semver": "7.3.7",
     "@types/tar-fs": "2.0.1",
     "@types/uuid": "8.3.4",
@@ -78,6 +78,7 @@
     "jest-serial-runner": "1.2.1",
     "pino-pretty": "10.0.0",
     "pouchdb-adapter-memory": "7.2.2",
+    "testcontainers": "^10.7.2",
     "timekeeper": "2.2.0",
     "typescript": "5.2.2"
   },

packages/backend-core/src/cache/base/index.ts

@@ -23,6 +23,18 @@ export default class BaseCache {
     return client.keys(pattern)
   }
 
+  async exists(key: string, opts = { useTenancy: true }) {
+    key = opts.useTenancy ? generateTenantKey(key) : key
+    const client = await this.getClient()
+    return client.exists(key)
+  }
+
+  async scan(key: string, opts = { useTenancy: true }) {
+    key = opts.useTenancy ? generateTenantKey(key) : key
+    const client = await this.getClient()
+    return client.scan(key)
+  }
+
   /**
    * Read only from the cache.
    */
@@ -32,6 +44,15 @@ export default class BaseCache {
     return client.get(key)
   }
 
+  /**
+   * Read only from the cache.
+   */
+  async bulkGet<T>(keys: string[], opts = { useTenancy: true }) {
+    keys = opts.useTenancy ? keys.map(key => generateTenantKey(key)) : keys
+    const client = await this.getClient()
+    return client.bulkGet<T>(keys)
+  }
+
   /**
    * Write to the cache.
    */
@@ -46,6 +67,25 @@ export default class BaseCache {
     await client.store(key, value, ttl)
   }
 
+  /**
+   * Bulk write to the cache.
+   */
+  async bulkStore(
+    data: Record<string, any>,
+    ttl: number | null = null,
+    opts = { useTenancy: true }
+  ) {
+    if (opts.useTenancy) {
+      data = Object.entries(data).reduce((acc, [key, value]) => {
+        acc[generateTenantKey(key)] = value
+        return acc
+      }, {} as Record<string, any>)
+    }
+
+    const client = await this.getClient()
+    await client.bulkStore(data, ttl)
+  }
+
   /**
    * Remove from cache.
    */
@@ -55,15 +95,24 @@ export default class BaseCache {
     return client.delete(key)
   }
 
+  /**
+   * Remove from cache.
+   */
+  async bulkDelete(keys: string[], opts = { useTenancy: true }) {
+    keys = opts.useTenancy ? keys.map(key => generateTenantKey(key)) : keys
+    const client = await this.getClient()
+    return client.bulkDelete(keys)
+  }
+
   /**
    * Read from the cache. Write to the cache if not exists.
    */
-  async withCache(
+  async withCache<T>(
     key: string,
-    ttl: number,
+    ttl: number | null = null,
-    fetchFn: any,
+    fetchFn: () => Promise<T> | T,
     opts = { useTenancy: true }
-  ) {
+  ): Promise<T> {
     const cachedValue = await this.get(key, opts)
     if (cachedValue) {
       return cachedValue
@@ -89,4 +138,13 @@ export default class BaseCache {
       throw err
     }
   }
+
+  /**
+   * Delete the entry if the provided value matches the stored one.
+   */
+  async deleteIfValue(key: string, value: any, opts = { useTenancy: true }) {
+    key = opts.useTenancy ? generateTenantKey(key) : key
+    const client = await this.getClient()
+    await client.deleteIfValue(key, value)
+  }
 }

packages/backend-core/src/cache/docWritethrough.ts

@@ -0,0 +1,86 @@
+import { AnyDocument, Database } from "@budibase/types"
+
+import { JobQueue, createQueue } from "../queue"
+import * as dbUtils from "../db"
+
+interface ProcessDocMessage {
+  dbName: string
+  docId: string
+  data: Record<string, any>
+}
+
+const PERSIST_MAX_ATTEMPTS = 100
+
+export const docWritethroughProcessorQueue = createQueue<ProcessDocMessage>(
+  JobQueue.DOC_WRITETHROUGH_QUEUE,
+  {
+    jobOptions: {
+      attempts: PERSIST_MAX_ATTEMPTS,
+    },
+  }
+)
+
+class DocWritethroughProcessor {
+  init() {
+    docWritethroughProcessorQueue.process(async message => {
+      try {
+        await this.persistToDb(message.data)
+      } catch (err: any) {
+        if (err.status === 409) {
+          // If we get a 409, it means that another job updated it meanwhile. We want to retry it to persist it again.
+          throw new Error(
+            `Conflict persisting message ${message.id}. Attempt ${message.attemptsMade}`
+          )
+        }
+
+        throw err
+      }
+    })
+    return this
+  }
+
+  private async persistToDb({
+    dbName,
+    docId,
+    data,
+  }: {
+    dbName: string
+    docId: string
+    data: Record<string, any>
+  }) {
+    const db = dbUtils.getDB(dbName)
+    let doc: AnyDocument | undefined
+    try {
+      doc = await db.get(docId)
+    } catch {
+      doc = { _id: docId }
+    }
+
+    doc = { ...doc, ...data }
+    await db.put(doc)
+  }
+}
+
+export const processor = new DocWritethroughProcessor().init()
+
+export class DocWritethrough {
+  private db: Database
+  private _docId: string
+
+  constructor(db: Database, docId: string) {
+    this.db = db
+    this._docId = docId
+  }
+
+  get docId() {
+    return this._docId
+  }
+
+  async patch(data: Record<string, any>) {
+    await docWritethroughProcessorQueue.add({
+      dbName: this.db.name,
+      docId: this.docId,
+      data,
+    })
+  }
+}

packages/backend-core/src/cache/generic.ts

@@ -26,7 +26,8 @@ export const store = (...args: Parameters<typeof GENERIC.store>) =>
   GENERIC.store(...args)
 export const destroy = (...args: Parameters<typeof GENERIC.delete>) =>
   GENERIC.delete(...args)
-export const withCache = (...args: Parameters<typeof GENERIC.withCache>) =>
+export const withCache = <T>(
-  GENERIC.withCache(...args)
+  ...args: Parameters<typeof GENERIC.withCache<T>>
+) => GENERIC.withCache(...args)
 export const bustCache = (...args: Parameters<typeof GENERIC.bustCache>) =>
   GENERIC.bustCache(...args)

packages/backend-core/src/cache/index.ts

@@ -5,3 +5,4 @@ export * as writethrough from "./writethrough"
 export * as invite from "./invite"
 export * as passwordReset from "./passwordReset"
 export * from "./generic"
+export * as docWritethrough from "./docWritethrough"

packages/backend-core/src/cache/tests/docWritethrough.spec.ts

@@ -0,0 +1,288 @@
+import tk from "timekeeper"
+
+import _ from "lodash"
+import { DBTestConfiguration, generator, structures } from "../../../tests"
+import { getDB } from "../../db"
+
+import {
+  DocWritethrough,
+  docWritethroughProcessorQueue,
+} from "../docWritethrough"
+
+import InMemoryQueue from "../../queue/inMemoryQueue"
+
+const initialTime = Date.now()
+
+async function waitForQueueCompletion() {
+  const queue: InMemoryQueue = docWritethroughProcessorQueue as never
+  await queue.waitForCompletion()
+}
+
+describe("docWritethrough", () => {
+  const config = new DBTestConfiguration()
+
+  const db = getDB(structures.db.id())
+  let documentId: string
+  let docWritethrough: DocWritethrough
+
+  describe("patch", () => {
+    function generatePatchObject(fieldCount: number) {
+      const keys = generator.unique(() => generator.word(), fieldCount)
+      return keys.reduce((acc, c) => {
+        acc[c] = generator.word()
+        return acc
+      }, {} as Record<string, any>)
+    }
+
+    beforeEach(async () => {
+      jest.clearAllMocks()
+      documentId = structures.uuid()
+      docWritethrough = new DocWritethrough(db, documentId)
+    })
+
+    it("patching will not persist until the messages are persisted", async () => {
+      await config.doInTenant(async () => {
+        await docWritethrough.patch(generatePatchObject(2))
+        await docWritethrough.patch(generatePatchObject(2))
+
+        expect(await db.exists(documentId)).toBe(false)
+      })
+    })
+
+    it("patching will persist when the messages are persisted", async () => {
+      await config.doInTenant(async () => {
+        const patch1 = generatePatchObject(2)
+        const patch2 = generatePatchObject(2)
+        await docWritethrough.patch(patch1)
+        await docWritethrough.patch(patch2)
+
+        await waitForQueueCompletion()
+
+        // This will not be persisted
+        const patch3 = generatePatchObject(3)
+        await docWritethrough.patch(patch3)
+
+        expect(await db.get(documentId)).toEqual({
+          _id: documentId,
+          ...patch1,
+          ...patch2,
+          _rev: expect.stringMatching(/2-.+/),
+          createdAt: new Date(initialTime).toISOString(),
+          updatedAt: new Date(initialTime).toISOString(),
+        })
+      })
+    })
+
+    it("patching will persist keeping the previous data", async () => {
+      await config.doInTenant(async () => {
+        const patch1 = generatePatchObject(2)
+        const patch2 = generatePatchObject(2)
+        await docWritethrough.patch(patch1)
+        await docWritethrough.patch(patch2)
+
+        await waitForQueueCompletion()
+
+        const patch3 = generatePatchObject(3)
+        await docWritethrough.patch(patch3)
+
+        await waitForQueueCompletion()
+
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining({
+            _id: documentId,
+            ...patch1,
+            ...patch2,
+            ...patch3,
+          })
+        )
+      })
+    })
+
+    it("date audit fields are set correctly when persisting", async () => {
+      await config.doInTenant(async () => {
+        const patch1 = generatePatchObject(2)
+        const patch2 = generatePatchObject(2)
+        await docWritethrough.patch(patch1)
+        const date1 = new Date()
+        await waitForQueueCompletion()
+        await docWritethrough.patch(patch2)
+
+        tk.travel(Date.now() + 100)
+        const date2 = new Date()
+        await waitForQueueCompletion()
+
+        expect(date1).not.toEqual(date2)
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining({
+            createdAt: date1.toISOString(),
+            updatedAt: date2.toISOString(),
+          })
+        )
+      })
+    })
+
+    it("concurrent patches will override keys", async () => {
+      await config.doInTenant(async () => {
+        const patch1 = generatePatchObject(2)
+        await docWritethrough.patch(patch1)
+        await waitForQueueCompletion()
+        const patch2 = generatePatchObject(1)
+        await docWritethrough.patch(patch2)
+
+        const keyToOverride = _.sample(Object.keys(patch1))!
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining({
+            [keyToOverride]: patch1[keyToOverride],
+          })
+        )
+
+        await waitForQueueCompletion()
+
+        const patch3 = {
+          ...generatePatchObject(3),
+          [keyToOverride]: generator.word(),
+        }
+        await docWritethrough.patch(patch3)
+        await waitForQueueCompletion()
+
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining({
+            ...patch1,
+            ...patch2,
+            ...patch3,
+          })
+        )
+      })
+    })
+
+    it("concurrent patches to different docWritethrough will not pollute each other", async () => {
+      await config.doInTenant(async () => {
+        const secondDocWritethrough = new DocWritethrough(
+          db,
+          structures.db.id()
+        )
+
+        const doc1Patch = generatePatchObject(2)
+        await docWritethrough.patch(doc1Patch)
+        const doc2Patch = generatePatchObject(1)
+        await secondDocWritethrough.patch(doc2Patch)
+
+        await waitForQueueCompletion()
+
+        const doc1Patch2 = generatePatchObject(3)
+        await docWritethrough.patch(doc1Patch2)
+        const doc2Patch2 = generatePatchObject(3)
+        await secondDocWritethrough.patch(doc2Patch2)
+        await waitForQueueCompletion()
+
+        expect(await db.get(docWritethrough.docId)).toEqual(
+          expect.objectContaining({
+            ...doc1Patch,
+            ...doc1Patch2,
+          })
+        )
+
+        expect(await db.get(secondDocWritethrough.docId)).toEqual(
+          expect.objectContaining({
+            ...doc2Patch,
+            ...doc2Patch2,
+          })
+        )
+      })
+    })
+
+    it("cached values are persisted only once", async () => {
+      await config.doInTenant(async () => {
+        const initialPatch = generatePatchObject(5)
+
+        await docWritethrough.patch(initialPatch)
+        await waitForQueueCompletion()
+
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining(initialPatch)
+        )
+
+        await db.remove(await db.get(documentId))
+
+        await waitForQueueCompletion()
+        const extraPatch = generatePatchObject(5)
+        await docWritethrough.patch(extraPatch)
+        await waitForQueueCompletion()
+
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining(extraPatch)
+        )
+        expect(await db.get(documentId)).not.toEqual(
+          expect.objectContaining(initialPatch)
+        )
+      })
+    })
+
+    it("concurrent calls will not cause conflicts", async () => {
+      async function parallelPatch(count: number) {
+        const patches = Array.from({ length: count }).map(() =>
+          generatePatchObject(1)
+        )
+        await Promise.all(patches.map(p => docWritethrough.patch(p)))
+
+        return patches.reduce((acc, c) => {
+          acc = { ...acc, ...c }
+          return acc
+        }, {})
+      }
+      const queueMessageSpy = jest.spyOn(docWritethroughProcessorQueue, "add")
+
+      await config.doInTenant(async () => {
+        let patches = await parallelPatch(5)
+        expect(queueMessageSpy).toBeCalledTimes(5)
+
+        await waitForQueueCompletion()
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining(patches)
+        )
+
+        patches = { ...patches, ...(await parallelPatch(40)) }
+        expect(queueMessageSpy).toBeCalledTimes(45)
+
+        await waitForQueueCompletion()
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining(patches)
+        )
+
+        patches = { ...patches, ...(await parallelPatch(10)) }
+        expect(queueMessageSpy).toBeCalledTimes(55)
+
+        await waitForQueueCompletion()
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining(patches)
+        )
+      })
+    })
+
+    // This is not yet supported
+    it.skip("patches will execute in order", async () => {
+      let incrementalValue = 0
+      const keyToOverride = generator.word()
+      async function incrementalPatches(count: number) {
+        for (let i = 0; i < count; i++) {
+          await docWritethrough.patch({ [keyToOverride]: incrementalValue++ })
+        }
+      }
+
+      await config.doInTenant(async () => {
+        await incrementalPatches(5)
+
+        await waitForQueueCompletion()
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining({ [keyToOverride]: 5 })
+        )
+
+        await incrementalPatches(40)
+        await waitForQueueCompletion()
+        expect(await db.get(documentId)).toEqual(
+          expect.objectContaining({ [keyToOverride]: 45 })
+        )
+      })
+    })
+  })
+})

packages/backend-core/src/cache/user.ts

@@ -6,7 +6,7 @@ import env from "../environment"
 import * as accounts from "../accounts"
 import { UserDB } from "../users"
 import { sdk } from "@budibase/shared-core"
-import { User } from "@budibase/types"
+import { User, UserMetadata } from "@budibase/types"
 
 const EXPIRY_SECONDS = 3600
 
@@ -15,7 +15,7 @@ const EXPIRY_SECONDS = 3600
  */
 async function populateFromDB(userId: string, tenantId: string) {
   const db = tenancy.getTenantDB(tenantId)
-  const user = await db.get<any>(userId)
+  const user = await db.get<UserMetadata>(userId)
   user.budibaseAccess = true
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(user.email)

packages/backend-core/src/constants/db.ts

@@ -57,6 +57,9 @@ export const StaticDatabases = {
   AUDIT_LOGS: {
     name: "audit-logs",
   },
+  SCIM_LOGS: {
+    name: "scim-logs",
+  },
 }
 
 export const APP_PREFIX = prefixed(DocumentType.APP)

packages/backend-core/src/context/mainContext.ts

@@ -35,6 +35,17 @@ export function getAuditLogDBName(tenantId?: string) {
   }
 }
 
+export function getScimDBName(tenantId?: string) {
+  if (!tenantId) {
+    tenantId = getTenantId()
+  }
+  if (tenantId === DEFAULT_TENANT_ID) {
+    return StaticDatabases.SCIM_LOGS.name
+  } else {
+    return `${tenantId}${SEPARATOR}${StaticDatabases.SCIM_LOGS.name}`
+  }
+}
+
 export function baseGlobalDBName(tenantId: string | undefined | null) {
   if (!tenantId || tenantId === DEFAULT_TENANT_ID) {
     return StaticDatabases.GLOBAL.name

packages/backend-core/src/db/Replication.ts

@@ -1,66 +1,57 @@
+import PouchDB from "pouchdb"
 import { getPouchDB, closePouchDB } from "./couch"
 import { DocumentType } from "../constants"
 
 class Replication {
-  source: any
+  source: PouchDB.Database
-  target: any
+  target: PouchDB.Database
-  replication: any
 
-  /**
+  constructor({ source, target }: { source: string; target: string }) {
-   *
-   * @param source - the DB you want to replicate or rollback to
-   * @param target - the DB you want to replicate to, or rollback from
-   */
-  constructor({ source, target }: any) {
     this.source = getPouchDB(source)
     this.target = getPouchDB(target)
   }
 
-  close() {
+  async close() {
-    return Promise.all([closePouchDB(this.source), closePouchDB(this.target)])
+    await Promise.all([closePouchDB(this.source), closePouchDB(this.target)])
   }
 
-  promisify(operation: any, opts = {}) {
+  replicate(opts: PouchDB.Replication.ReplicateOptions = {}) {
-    return new Promise(resolve => {
+    return new Promise<PouchDB.Replication.ReplicationResult<{}>>(resolve => {
-      operation(this.target, opts)
+      this.source.replicate
-        .on("denied", function (err: any) {
+        .to(this.target, opts)
+        .on("denied", function (err) {
           // a document failed to replicate (e.g. due to permissions)
           throw new Error(`Denied: Document failed to replicate ${err}`)
         })
-        .on("complete", function (info: any) {
+        .on("complete", function (info) {
           return resolve(info)
         })
-        .on("error", function (err: any) {
+        .on("error", function (err) {
           throw new Error(`Replication Error: ${err}`)
         })
     })
   }
 
-  /**
+  appReplicateOpts(
-   * Two way replication operation, intended to be promise based.
+    opts: PouchDB.Replication.ReplicateOptions = {}
-   * @param opts - PouchDB replication options
+  ): PouchDB.Replication.ReplicateOptions {
-   */
+    if (typeof opts.filter === "string") {
-  sync(opts = {}) {
+      return opts
-    this.replication = this.promisify(this.source.sync, opts)
+    }
-    return this.replication
-  }
 
-  /**
+    const filter = opts.filter
-   * One way replication operation, intended to be promise based.
+    delete opts.filter
-   * @param opts - PouchDB replication options
-   */
-  replicate(opts = {}) {
-    this.replication = this.promisify(this.source.replicate.to, opts)
-    return this.replication
-  }
 
-  appReplicateOpts() {
     return {
-      filter: (doc: any) => {
+      ...opts,
+      filter: (doc: any, params: any) => {
         if (doc._id && doc._id.startsWith(DocumentType.AUTOMATION_LOG)) {
           return false
         }
-        return doc._id !== DocumentType.APP_METADATA
+        if (doc._id === DocumentType.APP_METADATA) {
+          return false
+        }
+        return filter ? filter(doc, params) : true
       },
     }
   }
@@ -75,10 +66,6 @@ class Replication {
     // take the opportunity to remove deleted tombstones
     await this.replicate()
   }
-
-  cancel() {
-    this.replication.cancel()
-  }
 }
 
 export default Replication

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -70,7 +70,15 @@ export class DatabaseImpl implements Database {
     DatabaseImpl.nano = buildNano(couchInfo)
   }
 
-  async exists() {
+  exists(docId?: string) {
+    if (docId === undefined) {
+      return this.dbExists()
+    }
+
+    return this.docExists(docId)
+  }
+
+  private async dbExists() {
     const response = await directCouchUrlCall({
       url: `${this.couchInfo.url}/${this.name}`,
       method: "HEAD",
@@ -79,6 +87,15 @@ export class DatabaseImpl implements Database {
     return response.status === 200
   }
 
+  private async docExists(id: string): Promise<boolean> {
+    try {
+      await this.performCall(db => () => db.head(id))
+      return true
+    } catch {
+      return false
+    }
+  }
+
   private nano() {
     return this.instanceNano || DatabaseImpl.nano
   }

packages/backend-core/src/db/instrumentation.ts

@@ -24,9 +24,12 @@ export class DDInstrumentedDatabase implements Database {
     return this.db.name
   }
 
-  exists(): Promise<boolean> {
+  exists(docId?: string): Promise<boolean> {
     return tracer.trace("db.exists", span => {
-      span?.addTags({ db_name: this.name })
+      span?.addTags({ db_name: this.name, doc_id: docId })
+      if (docId) {
+        return this.db.exists(docId)
+      }
       return this.db.exists()
     })
   }

packages/backend-core/src/db/tests/DatabaseImpl.spec.ts

@@ -0,0 +1,55 @@
+import _ from "lodash"
+import { AnyDocument } from "@budibase/types"
+import { generator } from "../../../tests"
+import { DatabaseImpl } from "../couch"
+import { newid } from "../../utils"
+
+describe("DatabaseImpl", () => {
+  const database = new DatabaseImpl(generator.word())
+  const documents: AnyDocument[] = []
+
+  beforeAll(async () => {
+    const docsToCreate = Array.from({ length: 10 }).map(() => ({
+      _id: newid(),
+    }))
+    const createdDocs = await database.bulkDocs(docsToCreate)
+
+    documents.push(...createdDocs.map((x: any) => ({ _id: x.id, _rev: x.rev })))
+  })
+
+  describe("document exists", () => {
+    it("can check existing docs by id", async () => {
+      const existingDoc = _.sample(documents)
+      const result = await database.exists(existingDoc!._id!)
+
+      expect(result).toBe(true)
+    })
+
+    it("can check non existing docs by id", async () => {
+      const result = await database.exists(newid())
+
+      expect(result).toBe(false)
+    })
+
+    it("can check an existing doc by id multiple times", async () => {
+      const existingDoc = _.sample(documents)
+      const id = existingDoc!._id!
+
+      const results = []
+      results.push(await database.exists(id))
+      results.push(await database.exists(id))
+      results.push(await database.exists(id))
+
+      expect(results).toEqual([true, true, true])
+    })
+
+    it("returns false after the doc is deleted", async () => {
+      const existingDoc = _.sample(documents)
+      const id = existingDoc!._id!
+      expect(await database.exists(id)).toBe(true)
+
+      await database.remove(existingDoc!)
+      expect(await database.exists(id)).toBe(false)
+    })
+  })
+})

packages/backend-core/src/environment.ts

@@ -186,6 +186,7 @@ const environment = {
     environment[key] = value
   },
   ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M",
+  DISABLE_SCIM_CALLS: process.env.DISABLE_SCIM_CALLS,
 }
 
 // clean up any environment variable edge cases

packages/backend-core/src/queue/constants.ts

@@ -4,4 +4,5 @@ export enum JobQueue {
   AUDIT_LOG = "auditLogQueue",
   SYSTEM_EVENT_QUEUE = "systemEventQueue",
   APP_MIGRATION = "appMigration",
+  DOC_WRITETHROUGH_QUEUE = "docWritethroughQueue",
 }

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -1,5 +1,14 @@
 import events from "events"
-import { timeout } from "../utils"
+import { newid, timeout } from "../utils"
+import { Queue, QueueOptions, JobOptions } from "./queue"
+
+interface JobMessage {
+  id: string
+  timestamp: number
+  queue: string
+  data: any
+  opts?: JobOptions
+}
 
 /**
  * Bull works with a Job wrapper around all messages that contains a lot more information about
@@ -10,12 +19,13 @@ import { timeout } from "../utils"
  * @returns A new job which can now be put onto the queue, this is mostly an
  * internal structure so that an in memory queue can be easily swapped for a Bull queue.
  */
-function newJob(queue: string, message: any) {
+function newJob(queue: string, message: any, opts?: JobOptions): JobMessage {
   return {
+    id: newid(),
     timestamp: Date.now(),
     queue: queue,
     data: message,
-    opts: {},
+    opts,
   }
 }
 
@@ -24,26 +34,29 @@ function newJob(queue: string, message: any) {
  * It is relatively simple, using an event emitter internally to register when messages are available
  * to the consumers - in can support many inputs and many consumers.
  */
-class InMemoryQueue {
+class InMemoryQueue implements Partial<Queue> {
   _name: string
-  _opts?: any
+  _opts?: QueueOptions
-  _messages: any[]
+  _messages: JobMessage[]
+  _queuedJobIds: Set<string>
   _emitter: EventEmitter
   _runCount: number
   _addCount: number
+
   /**
    * The constructor the queue, exactly the same as that of Bulls.
    * @param name The name of the queue which is being configured.
    * @param opts This is not used by the in memory queue as there is no real use
    * case when in memory, but is the same API as Bull
    */
-  constructor(name: string, opts?: any) {
+  constructor(name: string, opts?: QueueOptions) {
     this._name = name
     this._opts = opts
     this._messages = []
     this._emitter = new events.EventEmitter()
     this._runCount = 0
     this._addCount = 0
+    this._queuedJobIds = new Set<string>()
   }
 
   /**
@@ -55,22 +68,42 @@ class InMemoryQueue {
    * note this is incredibly limited compared to Bull as in reality the Job would contain
    * a lot more information about the queue and current status of Bull cluster.
    */
-  process(func: any) {
+  async process(func: any) {
     this._emitter.on("message", async () => {
       if (this._messages.length <= 0) {
         return
       }
       let msg = this._messages.shift()
+
       let resp = func(msg)
+
+      async function retryFunc(fnc: any) {
+        try {
+          await fnc
+        } catch (e: any) {
+          await new Promise<void>(r => setTimeout(() => r(), 50))
+
+          await retryFunc(func(msg))
+        }
+      }
+
       if (resp.then != null) {
-        await resp
+        try {
+          await retryFunc(resp)
+        } catch (e: any) {
+          console.error(e)
+        }
       }
       this._runCount++
+      const jobId = msg?.opts?.jobId?.toString()
+      if (jobId && msg?.opts?.removeOnComplete) {
+        this._queuedJobIds.delete(jobId)
+      }
     })
   }
 
   async isReady() {
-    return true
+    return this as any
   }
 
   // simply puts a message to the queue and emits to the queue for processing
@@ -83,27 +116,45 @@ class InMemoryQueue {
    * @param repeat serves no purpose for the import queue.
    */
   // eslint-disable-next-line no-unused-vars
-  add(msg: any, repeat: boolean) {
+  async add(data: any, opts?: JobOptions) {
-    if (typeof msg !== "object") {
+    const jobId = opts?.jobId?.toString()
+    if (jobId && this._queuedJobIds.has(jobId)) {
+      console.log(`Ignoring already queued job ${jobId}`)
+      return
+    }
+
+    if (typeof data !== "object") {
       throw "Queue only supports carrying JSON."
     }
-    this._messages.push(newJob(this._name, msg))
+    if (jobId) {
-    this._addCount++
+      this._queuedJobIds.add(jobId)
-    this._emitter.emit("message")
+    }
+
+    const pushMessage = () => {
+      this._messages.push(newJob(this._name, data, opts))
+      this._addCount++
+      this._emitter.emit("message")
+    }
+
+    const delay = opts?.delay
+    if (delay) {
+      setTimeout(pushMessage, delay)
+    } else {
+      pushMessage()
+    }
+    return {} as any
   }
 
   /**
    * replicating the close function from bull, which waits for jobs to finish.
    */
-  async close() {
+  async close() {}
-    return []
-  }
 
   /**
    * This removes a cron which has been implemented, this is part of Bull API.
    * @param cronJobId The cron which is to be removed.
    */
-  removeRepeatableByKey(cronJobId: string) {
+  async removeRepeatableByKey(cronJobId: string) {
     // TODO: implement for testing
     console.log(cronJobId)
   }
@@ -111,12 +162,12 @@ class InMemoryQueue {
   /**
    * Implemented for tests
    */
-  getRepeatableJobs() {
+  async getRepeatableJobs() {
     return []
   }
 
   // eslint-disable-next-line no-unused-vars
-  removeJobs(pattern: string) {
+  async removeJobs(pattern: string) {
     // no-op
   }
 
@@ -128,18 +179,22 @@ class InMemoryQueue {
   }
 
   async getJob() {
-    return {}
+    return null
   }
 
   on() {
     // do nothing
-    return this
+    return this as any
   }
 
   async waitForCompletion() {
     do {
       await timeout(50)
-    } while (this._addCount < this._runCount)
+    } while (this.hasRunningJobs())
+  }
+
+  hasRunningJobs() {
+    return this._addCount > this._runCount
   }
 }
 

packages/backend-core/src/queue/listeners.ts

@@ -88,6 +88,7 @@ enum QueueEventType {
   AUDIT_LOG_EVENT = "audit-log-event",
   SYSTEM_EVENT = "system-event",
   APP_MIGRATION = "app-migration",
+  DOC_WRITETHROUGH = "doc-writethrough",
 }
 
 const EventTypeMap: { [key in JobQueue]: QueueEventType } = {
@@ -96,6 +97,7 @@ const EventTypeMap: { [key in JobQueue]: QueueEventType } = {
   [JobQueue.AUDIT_LOG]: QueueEventType.AUDIT_LOG_EVENT,
   [JobQueue.SYSTEM_EVENT_QUEUE]: QueueEventType.SYSTEM_EVENT,
   [JobQueue.APP_MIGRATION]: QueueEventType.APP_MIGRATION,
+  [JobQueue.DOC_WRITETHROUGH_QUEUE]: QueueEventType.DOC_WRITETHROUGH,
 }
 
 function logging(queue: Queue, jobQueue: JobQueue) {

packages/backend-core/src/queue/queue.ts

@@ -7,6 +7,8 @@ import { addListeners, StalledFn } from "./listeners"
 import { Duration } from "../utils"
 import * as timers from "../timers"
 
+export { QueueOptions, Queue, JobOptions } from "bull"
+
 // the queue lock is held for 5 minutes
 const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs()
 // queue lock is refreshed every 30 seconds

packages/backend-core/src/redis/init.ts

@@ -9,7 +9,8 @@ let userClient: Client,
   lockClient: Client,
   socketClient: Client,
   inviteClient: Client,
-  passwordResetClient: Client
+  passwordResetClient: Client,
+  docWritethroughClient: Client
 
 export async function init() {
   userClient = await new Client(utils.Databases.USER_CACHE).init()
@@ -24,6 +25,9 @@ export async function init() {
     utils.Databases.SOCKET_IO,
     utils.SelectableDatabase.SOCKET_IO
   ).init()
+  docWritethroughClient = await new Client(
+    utils.Databases.DOC_WRITE_THROUGH
+  ).init()
 }
 
 export async function shutdown() {
@@ -104,3 +108,10 @@ export async function getPasswordResetClient() {
   }
   return passwordResetClient
 }
+
+export async function getDocWritethroughClient() {
+  if (!writethroughClient) {
+    await init()
+  }
+  return writethroughClient
+}

packages/backend-core/src/redis/redis.ts

@@ -1,5 +1,5 @@
 import env from "../environment"
-import Redis from "ioredis"
+import Redis, { Cluster } from "ioredis"
 // mock-redis doesn't have any typing
 let MockRedis: any | undefined
 if (env.MOCK_REDIS) {
@@ -28,7 +28,7 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT
 
 // for testing just generate the client once
 let CLOSED = false
-let CLIENTS: { [key: number]: any } = {}
+const CLIENTS: Record<number, Redis> = {}
 let CONNECTED = false
 
 // mock redis always connected
@@ -36,7 +36,7 @@ if (env.MOCK_REDIS) {
   CONNECTED = true
 }
 
-function pickClient(selectDb: number): any {
+function pickClient(selectDb: number) {
   return CLIENTS[selectDb]
 }
 
@@ -201,12 +201,15 @@ class RedisWrapper {
     key = `${db}${SEPARATOR}${key}`
     let stream
     if (CLUSTERED) {
-      let node = this.getClient().nodes("master")
+      let node = (this.getClient() as never as Cluster).nodes("master")
       stream = node[0].scanStream({ match: key + "*", count: 100 })
     } else {
-      stream = this.getClient().scanStream({ match: key + "*", count: 100 })
+      stream = (this.getClient() as Redis).scanStream({
+        match: key + "*",
+        count: 100,
+      })
     }
-    return promisifyStream(stream, this.getClient())
+    return promisifyStream(stream, this.getClient() as any)
   }
 
   async keys(pattern: string) {
@@ -221,14 +224,16 @@ class RedisWrapper {
 
   async get(key: string) {
     const db = this._db
-    let response = await this.getClient().get(addDbPrefix(db, key))
+    const response = await this.getClient().get(addDbPrefix(db, key))
     // overwrite the prefixed key
+    // @ts-ignore
     if (response != null && response.key) {
+      // @ts-ignore
       response.key = key
     }
     // if its not an object just return the response
     try {
-      return JSON.parse(response)
+      return JSON.parse(response!)
     } catch (err) {
       return response
     }
@@ -274,13 +279,37 @@ class RedisWrapper {
     }
   }
 
+  async bulkStore(
+    data: Record<string, any>,
+    expirySeconds: number | null = null
+  ) {
+    const client = this.getClient()
+
+    const dataToStore = Object.entries(data).reduce((acc, [key, value]) => {
+      acc[addDbPrefix(this._db, key)] =
+        typeof value === "object" ? JSON.stringify(value) : value
+      return acc
+    }, {} as Record<string, any>)
+
+    const pipeline = client.pipeline()
+    pipeline.mset(dataToStore)
+
+    if (expirySeconds !== null) {
+      for (const key of Object.keys(dataToStore)) {
+        pipeline.expire(key, expirySeconds)
+      }
+    }
+
+    await pipeline.exec()
+  }
+
   async getTTL(key: string) {
     const db = this._db
     const prefixedKey = addDbPrefix(db, key)
     return this.getClient().ttl(prefixedKey)
   }
 
-  async setExpiry(key: string, expirySeconds: number | null) {
+  async setExpiry(key: string, expirySeconds: number) {
     const db = this._db
     const prefixedKey = addDbPrefix(db, key)
     await this.getClient().expire(prefixedKey, expirySeconds)
@@ -291,10 +320,35 @@ class RedisWrapper {
     await this.getClient().del(addDbPrefix(db, key))
   }
 
+  async bulkDelete(keys: string[]) {
+    const db = this._db
+    await this.getClient().del(keys.map(key => addDbPrefix(db, key)))
+  }
+
   async clear() {
     let items = await this.scan()
     await Promise.all(items.map((obj: any) => this.delete(obj.key)))
   }
+
+  async increment(key: string) {
+    const result = await this.getClient().incr(addDbPrefix(this._db, key))
+    if (isNaN(result)) {
+      throw new Error(`Redis ${key} does not contain a number`)
+    }
+    return result
+  }
+
+  async deleteIfValue(key: string, value: any) {
+    const client = this.getClient()
+
+    const luaScript = `
+      if redis.call('GET', KEYS[1]) == ARGV[1] then
+        redis.call('DEL', KEYS[1])
+      end
+      `
+
+    await client.eval(luaScript, 1, addDbPrefix(this._db, key), value)
+  }
 }
 
 export default RedisWrapper

packages/backend-core/src/redis/redlockImpl.ts

@@ -72,7 +72,7 @@ const OPTIONS: Record<keyof typeof LockType, Redlock.Options> = {
 export async function newRedlock(opts: Redlock.Options = {}) {
   const options = { ...OPTIONS.DEFAULT, ...opts }
   const redisWrapper = await getLockClient()
-  const client = redisWrapper.getClient()
+  const client = redisWrapper.getClient() as any
   return new Redlock([client], options)
 }
 

packages/backend-core/src/redis/tests/redis.spec.ts

@@ -0,0 +1,214 @@
+import { GenericContainer, StartedTestContainer } from "testcontainers"
+import { generator, structures } from "../../../tests"
+import RedisWrapper from "../redis"
+import { env } from "../.."
+
+jest.setTimeout(30000)
+
+describe("redis", () => {
+  let redis: RedisWrapper
+  let container: StartedTestContainer
+
+  beforeAll(async () => {
+    const container = await new GenericContainer("redis")
+      .withExposedPorts(6379)
+      .start()
+
+    env._set(
+      "REDIS_URL",
+      `${container.getHost()}:${container.getMappedPort(6379)}`
+    )
+    env._set("MOCK_REDIS", 0)
+    env._set("REDIS_PASSWORD", 0)
+  })
+
+  afterAll(() => container?.stop())
+
+  beforeEach(async () => {
+    redis = new RedisWrapper(structures.db.id())
+    await redis.init()
+  })
+
+  describe("store", () => {
+    it("a basic value can be persisted", async () => {
+      const key = structures.uuid()
+      const value = generator.word()
+
+      await redis.store(key, value)
+
+      expect(await redis.get(key)).toEqual(value)
+    })
+
+    it("objects can be persisted", async () => {
+      const key = structures.uuid()
+      const value = { [generator.word()]: generator.word() }
+
+      await redis.store(key, value)
+
+      expect(await redis.get(key)).toEqual(value)
+    })
+  })
+
+  describe("bulkStore", () => {
+    function createRandomObject(
+      keyLength: number,
+      valueGenerator: () => any = () => generator.word()
+    ) {
+      return generator
+        .unique(() => generator.word(), keyLength)
+        .reduce((acc, key) => {
+          acc[key] = valueGenerator()
+          return acc
+        }, {} as Record<string, string>)
+    }
+
+    it("a basic object can be persisted", async () => {
+      const data = createRandomObject(10)
+
+      await redis.bulkStore(data)
+
+      for (const [key, value] of Object.entries(data)) {
+        expect(await redis.get(key)).toEqual(value)
+      }
+
+      expect(await redis.keys("*")).toHaveLength(10)
+    })
+
+    it("a complex object can be persisted", async () => {
+      const data = {
+        ...createRandomObject(10, () => createRandomObject(5)),
+        ...createRandomObject(5),
+      }
+
+      await redis.bulkStore(data)
+
+      for (const [key, value] of Object.entries(data)) {
+        expect(await redis.get(key)).toEqual(value)
+      }
+
+      expect(await redis.keys("*")).toHaveLength(15)
+    })
+
+    it("no TTL is set by default", async () => {
+      const data = createRandomObject(10)
+
+      await redis.bulkStore(data)
+
+      for (const [key, value] of Object.entries(data)) {
+        expect(await redis.get(key)).toEqual(value)
+        expect(await redis.getTTL(key)).toEqual(-1)
+      }
+    })
+
+    it("a bulk store can be persisted with TTL", async () => {
+      const ttl = 500
+      const data = createRandomObject(8)
+
+      await redis.bulkStore(data, ttl)
+
+      for (const [key, value] of Object.entries(data)) {
+        expect(await redis.get(key)).toEqual(value)
+        expect(await redis.getTTL(key)).toEqual(ttl)
+      }
+
+      expect(await redis.keys("*")).toHaveLength(8)
+    })
+
+    it("setting a TTL of -1 will not persist the key", async () => {
+      const ttl = -1
+      const data = createRandomObject(5)
+
+      await redis.bulkStore(data, ttl)
+
+      for (const [key, value] of Object.entries(data)) {
+        expect(await redis.get(key)).toBe(null)
+      }
+
+      expect(await redis.keys("*")).toHaveLength(0)
+    })
+  })
+
+  describe("increment", () => {
+    it("can increment on a new key", async () => {
+      const key = structures.uuid()
+      const result = await redis.increment(key)
+      expect(result).toBe(1)
+    })
+
+    it("can increment multiple times", async () => {
+      const key = structures.uuid()
+      const results = [
+        await redis.increment(key),
+        await redis.increment(key),
+        await redis.increment(key),
+        await redis.increment(key),
+        await redis.increment(key),
+      ]
+      expect(results).toEqual([1, 2, 3, 4, 5])
+    })
+
+    it("can increment on a new key", async () => {
+      const key1 = structures.uuid()
+      const key2 = structures.uuid()
+
+      const result1 = await redis.increment(key1)
+      expect(result1).toBe(1)
+
+      const result2 = await redis.increment(key2)
+      expect(result2).toBe(1)
+    })
+
+    it("can increment multiple times in parallel", async () => {
+      const key = structures.uuid()
+      const results = await Promise.all(
+        Array.from({ length: 100 }).map(() => redis.increment(key))
+      )
+      expect(results).toHaveLength(100)
+      expect(results).toEqual(Array.from({ length: 100 }).map((_, i) => i + 1))
+    })
+
+    it("can increment existing set keys", async () => {
+      const key = structures.uuid()
+      await redis.store(key, 70)
+      await redis.increment(key)
+
+      const result = await redis.increment(key)
+      expect(result).toBe(72)
+    })
+
+    it.each([
+      generator.word(),
+      generator.bool(),
+      { [generator.word()]: generator.word() },
+    ])("cannot increment if the store value is not a number", async value => {
+      const key = structures.uuid()
+      await redis.store(key, value)
+
+      await expect(redis.increment(key)).rejects.toThrowError(
+        "ERR value is not an integer or out of range"
+      )
+    })
+  })
+
+  describe("deleteIfValue", () => {
+    it("can delete if the value matches", async () => {
+      const key = structures.uuid()
+      const value = generator.word()
+      await redis.store(key, value)
+
+      await redis.deleteIfValue(key, value)
+
+      expect(await redis.get(key)).toBeNull()
+    })
+
+    it("will not delete if the value does not matches", async () => {
+      const key = structures.uuid()
+      const value = generator.word()
+      await redis.store(key, value)
+
+      await redis.deleteIfValue(key, generator.word())
+
+      expect(await redis.get(key)).toEqual(value)
+    })
+  })
+})

packages/backend-core/src/redis/utils.ts

@@ -30,6 +30,7 @@ export enum Databases {
   LOCKS = "locks",
   SOCKET_IO = "socket_io",
   BPM_EVENTS = "bpmEvents",
+  DOC_WRITE_THROUGH = "docWriteThrough",
 }
 
 /**

packages/backend-core/src/security/roles.ts

@@ -101,10 +101,7 @@ export function getBuiltinRole(roleId: string): Role | undefined {
 /**
  * Works through the inheritance ranks to see how far up the builtin stack this ID is.
  */
-export function builtinRoleToNumber(id?: string) {
+export function builtinRoleToNumber(id: string) {
-  if (!id) {
-    return 0
-  }
   const builtins = getBuiltinRoles()
   const MAX = Object.values(builtins).length + 1
   if (id === BUILTIN_IDS.ADMIN || id === BUILTIN_IDS.BUILDER) {

packages/backend-core/tests/core/utilities/structures/accounts.ts

@@ -18,7 +18,7 @@ export const account = (partial: Partial<Account> = {}): Account => {
   return {
     accountId: uuid(),
     tenantId: generator.word(),
-    email: generator.email(),
+    email: generator.email({ domain: "example.com" }),
     tenantName: generator.word(),
     hosting: Hosting.SELF,
     createdAt: Date.now(),

packages/backend-core/tests/core/utilities/structures/scim.ts

@@ -13,7 +13,7 @@ interface CreateUserRequestFields {
 export function createUserRequest(userData?: Partial<CreateUserRequestFields>) {
   const defaultValues = {
     externalId: uuid(),
-    email: generator.email(),
+    email: `${uuid()}@example.com`,
     firstName: generator.first(),
     lastName: generator.last(),
     username: generator.name(),

packages/bbui/src/Actions/position_dropdown.js

@@ -35,7 +35,10 @@ export default function positionDropdown(element, opts) {
     }
 
     if (typeof customUpdate === "function") {
-      styles = customUpdate(anchorBounds, elementBounds, styles)
+      styles = customUpdate(anchorBounds, elementBounds, {
+        ...styles,
+        offset: opts.offset,
+      })
     } else {
       // Determine vertical styles
       if (align === "right-outside") {

packages/builder/src/analytics/index.js

@@ -9,13 +9,17 @@ const intercom = new IntercomClient(process.env.INTERCOM_TOKEN)
 class AnalyticsHub {
   constructor() {
     this.clients = [posthog, intercom]
+    this.initialised = false
   }
 
   async activate() {
     // Check analytics are enabled
     const analyticsStatus = await API.getAnalyticsStatus()
-    if (analyticsStatus.enabled) {
+    if (analyticsStatus.enabled && !this.initialised) {
-      this.clients.forEach(client => client.init())
+      this.clients.forEach(client => {
+        client.init()
+      })
+      this.initialised = true
     }
   }
 

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -40,8 +40,15 @@
       part2: PrettyRelationshipDefinitions.MANY,
     },
   }
-  let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions)
+  $: relationshipOpts1 =
-  let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions)
+    relationshipPart2 === PrettyRelationshipDefinitions.ONE
+      ? [PrettyRelationshipDefinitions.MANY]
+      : Object.values(PrettyRelationshipDefinitions)
+
+  $: relationshipOpts2 =
+    relationshipPart1 === PrettyRelationshipDefinitions.ONE
+      ? [PrettyRelationshipDefinitions.MANY]
+      : Object.values(PrettyRelationshipDefinitions)
 
   let relationshipPart1 = PrettyRelationshipDefinitions.ONE
   let relationshipPart2 = PrettyRelationshipDefinitions.MANY

packages/builder/src/components/deploy/AppActions.svelte

@@ -164,9 +164,10 @@
       </div>
     {/if}
     <TourWrap
-      tourStepKey={$builderStore.onboarding
+      stepKeys={[
-        ? TOUR_STEP_KEYS.BUILDER_USER_MANAGEMENT
+        TOUR_STEP_KEYS.BUILDER_USER_MANAGEMENT,
-        : TOUR_STEP_KEYS.FEATURE_USER_MANAGEMENT}
+        TOUR_STEP_KEYS.FEATURE_USER_MANAGEMENT,
+      ]}
     >
       <div class="app-action-button users">
         <div class="app-action" id="builder-app-users-button">
@@ -209,7 +210,7 @@
       <div bind:this={appActionPopoverAnchor}>
         <div class="app-action">
           <Icon name={isPublished ? "GlobeCheck" : "GlobeStrike"} />
-          <TourWrap tourStepKey={TOUR_STEP_KEYS.BUILDER_APP_PUBLISH}>
+          <TourWrap stepKeys={[TOUR_STEP_KEYS.BUILDER_APP_PUBLISH]}>
             <span class="publish-open" id="builder-app-publish-button">
               Publish
               <Icon

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte

@@ -45,7 +45,10 @@
     <Checkbox text="Require confirmation" bind:value={parameters.confirm} />
 
     {#if parameters.confirm}
-      <Label small>Confirm text</Label>
+      <Label small>Title</Label>
+      <Input placeholder="Delete Row" bind:value={parameters.customTitleText} />
+
+      <Label small>Text</Label>
       <Input
         placeholder="Are you sure you want to delete?"
         bind:value={parameters.confirmText}

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DuplicateRow.svelte

@@ -72,7 +72,13 @@
     <Checkbox text="Require confirmation" bind:value={parameters.confirm} />
 
     {#if parameters.confirm}
-      <Label small>Confirm text</Label>
+      <Label small>Title</Label>
+      <Input
+        placeholder="Duplicate Row"
+        bind:value={parameters.customTitleText}
+      />
+
+      <Label small>Text</Label>
       <Input
         placeholder="Are you sure you want to duplicate this row?"
         bind:value={parameters.confirmText}

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte

@@ -64,7 +64,13 @@
 
       {#if parameters.confirm}
         <Input
-          label="Confirm text"
+          label="Title"
+          placeholder="Execute Query"
+          bind:value={parameters.customTitleText}
+        />
+
+        <Input
+          label="Text"
           placeholder="Are you sure you want to execute this query?"
           bind:value={parameters.confirmText}
         />

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte

@@ -72,7 +72,10 @@
     <Checkbox text="Require confirmation" bind:value={parameters.confirm} />
 
     {#if parameters.confirm}
-      <Label small>Confirm text</Label>
+      <Label small>Title</Label>
+      <Input placeholder="Save Row" bind:value={parameters.customTitleText} />
+
+      <Label small>Text</Label>
       <Input
         placeholder="Are you sure you want to save this row?"
         bind:value={parameters.confirmText}

packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte

@@ -1,5 +1,5 @@
 <script>
-  import EditComponentPopover from "../EditComponentPopover.svelte"
+  import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte"
   import { Icon } from "@budibase/bbui"
   import { runtimeToReadableBinding } from "dataBinding"
   import { isJSBinding } from "@budibase/string-templates"

packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte

@@ -3,6 +3,7 @@
   import { componentStore } from "stores/builder"
   import { cloneDeep } from "lodash/fp"
   import { createEventDispatcher, getContext } from "svelte"
+  import { customPositionHandler } from "."
   import ComponentSettingsSection from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte"
 
   export let anchor
@@ -54,25 +55,6 @@
 
     dispatch("change", nestedComponentInstance)
   }
-
-  const customPositionHandler = (anchorBounds, eleBounds, cfg) => {
-    let { left, top } = cfg
-    let percentageOffset = 30
-    // left-outside
-    left = anchorBounds.left - eleBounds.width - 18
-
-    // shift up from the anchor, if space allows
-    let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset
-    let defaultTop = anchorBounds.top - offsetPos
-
-    if (window.innerHeight - defaultTop < eleBounds.height) {
-      top = window.innerHeight - eleBounds.height - 5
-    } else {
-      top = anchorBounds.top - offsetPos
-    }
-
-    return { ...cfg, left, top }
-  }
 </script>
 
 <Icon
@@ -104,6 +86,7 @@
   showPopover={drawers.length === 0}
   clickOutsideOverride={drawers.length > 0}
   maxHeight={600}
+  offset={18}
   handlePostionUpdate={customPositionHandler}
 >
   <span class="popover-wrap">

packages/builder/src/components/design/settings/controls/EditComponentPopover/index.js

@@ -0,0 +1,18 @@
+export const customPositionHandler = (anchorBounds, eleBounds, cfg) => {
+  let { left, top, offset } = cfg
+  let percentageOffset = 30
+  // left-outside
+  left = anchorBounds.left - eleBounds.width - (offset || 5)
+
+  // shift up from the anchor, if space allows
+  let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset
+  let defaultTop = anchorBounds.top - offsetPos
+
+  if (window.innerHeight - defaultTop < eleBounds.height) {
+    top = window.innerHeight - eleBounds.height - 5
+  } else {
+    top = anchorBounds.top - offsetPos
+  }
+
+  return { ...cfg, left, top }
+}

packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte

@@ -1,5 +1,5 @@
 <script>
-  import EditComponentPopover from "../EditComponentPopover.svelte"
+  import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte"
   import { Toggle, Icon } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { cloneDeep } from "lodash/fp"

packages/builder/src/components/design/settings/controls/FormStepConfiguration.svelte

@@ -52,8 +52,8 @@
     _id: Helpers.uuid(),
     _component: componentType,
     _instanceName: `Step ${currentStep + 1}`,
-    title: stepSettings.title ?? defaults.title,
+    title: stepSettings.title ?? defaults?.title,
-    buttons: stepSettings.buttons || defaults.buttons,
+    buttons: stepSettings.buttons || defaults?.buttons,
     fields: stepSettings.fields,
     desc: stepSettings.desc,
 

packages/builder/src/components/design/settings/controls/GridColumnConfiguration/FieldSetting.svelte

@@ -1,5 +1,5 @@
 <script>
-  import EditComponentPopover from "../EditComponentPopover.svelte"
+  import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte"
   import { Toggle, Icon } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { cloneDeep } from "lodash/fp"

packages/builder/src/components/design/settings/controls/GridColumnConfiguration/PrimaryColumnFieldSetting.svelte

@@ -1,5 +1,5 @@
 <script>
-  import EditComponentPopover from "../EditComponentPopover.svelte"
+  import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte"
   import { Icon } from "@budibase/bbui"
   import { setContext } from "svelte"
   import { writable } from "svelte/store"

packages/builder/src/components/design/settings/controls/IconSelect/IconSelect.svelte

@@ -139,10 +139,22 @@
         {/each}
       </div>
       <div class="search-input">
-        <div class="input-wrapper">
+        <div class="input-wrapper" style={`width: ${value ? "425" : "510"}px`}>
-          <Input bind:value={searchTerm} thin placeholder="Search Icon" />
+          <Input
+            bind:value={searchTerm}
+            on:keyup={event => {
+              if (event.key === "Enter") {
+                searchForIcon()
+              }
+            }}
+            thin
+            placeholder="Search Icon"
+          />
         </div>
         <Button secondary on:click={searchForIcon}>Search</Button>
+        {#if value}
+          <Button primary on:click={() => (value = null)}>Clear</Button>
+        {/if}
       </div>
       <div class="page-area">
         <div class="pager">
@@ -239,6 +251,7 @@
     flex-flow: row nowrap;
     width: 100%;
     padding-right: 15px;
+    gap: 10px;
   }
   .input-wrapper {
     width: 510px;

packages/builder/src/components/design/settings/controls/PropertyControl.svelte

@@ -20,17 +20,23 @@
   export let bindings = []
   export let componentBindings = []
   export let nested = false
-  export let highlighted = false
   export let propertyFocus = false
   export let info = null
   export let disableBindings = false
   export let wide
 
-  $: nullishValue = value == null || value === ""
+  let highlightType
+
+  $: highlightedProp = $builderStore.highlightedSetting
   $: allBindings = getAllBindings(bindings, componentBindings, nested)
   $: safeValue = getSafeValue(value, defaultValue, allBindings)
   $: replaceBindings = val => readableToRuntimeBinding(allBindings, val)
 
+  $: if (!Array.isArray(value)) {
+    highlightType =
+      highlightedProp?.key === key ? `highlighted-${highlightedProp?.type}` : ""
+  }
+
   const getAllBindings = (bindings, componentBindings, nested) => {
     if (!nested) {
       return bindings
@@ -71,16 +77,17 @@
   }
 
   onDestroy(() => {
-    if (highlighted) {
+    if (highlightedProp) {
       builderStore.highlightSetting(null)
     }
   })
 </script>
 
 <div
-  class="property-control"
+  id={`${key}-prop-control-wrap`}
+  class={`property-control ${highlightType}`}
   class:wide={!label || labelHidden || wide === true}
-  class:highlighted={highlighted && nullishValue}
+  class:highlighted={highlightType}
   class:property-focus={propertyFocus}
 >
   {#if label && !labelHidden}
@@ -115,6 +122,16 @@
 </div>
 
 <style>
+  .property-control.highlighted.highlighted-info {
+    border-color: var(--spectrum-semantic-informative-color-background);
+  }
+  .property-control.highlighted.highlighted-error {
+    border-color: var(--spectrum-global-color-static-red-600);
+  }
+  .property-control.highlighted.highlighted-warning {
+    border-color: var(--spectrum-global-color-static-orange-700);
+  }
+
   .property-control {
     position: relative;
     display: grid;
@@ -132,6 +149,10 @@
   .property-control.highlighted {
     background: var(--spectrum-global-color-gray-300);
     border-color: var(--spectrum-global-color-static-red-600);
+    margin-top: -3.5px;
+    margin-bottom: -3.5px;
+    padding-bottom: 3.5px;
+    padding-top: 3.5px;
   }
 
   .property-control.property-focus :global(input) {

packages/builder/src/components/portal/onboarding/TourPopover.svelte

@@ -1,6 +1,6 @@
 <script>
   import { Popover, Layout, Heading, Body, Button, Link } from "@budibase/bbui"
-  import { TOURS } from "./tours.js"
+  import { TOURS, getCurrentStepIdx } from "./tours.js"
   import { goto, layout, isActive } from "@roxi/routify"
   import { builderStore } from "stores/builder"
 
@@ -20,6 +20,13 @@
 
   const updateTourStep = (targetStepKey, tourKey) => {
     if (!tourKey) {
+      tourSteps = null
+      tourStepIdx = null
+      lastStep = null
+      tourStep = null
+      popoverAnchor = null
+      popover = null
+      skipping = false
       return
     }
     if (!tourSteps?.length) {
@@ -78,16 +85,6 @@
       }
     }
   }
-
-  const getCurrentStepIdx = (steps, tourStepKey) => {
-    if (!steps?.length) {
-      return
-    }
-    if (steps?.length && !tourStepKey) {
-      return 0
-    }
-    return steps.findIndex(step => step.id === tourStepKey)
-  }
 </script>
 
 {#if tourKey}
@@ -98,7 +95,9 @@
       anchor={popoverAnchor}
       maxWidth={300}
       dismissible={false}
-      offset={15}
+      offset={12}
+      handlePostionUpdate={tourStep?.positionHandler}
+      customZindex={3}
     >
       <div class="tour-content">
         <Layout noPadding gap="M">
@@ -119,7 +118,7 @@
           </Body>
           <div class="tour-footer">
             <div class="tour-navigation">
-              {#if typeof tourOnSkip === "function"}
+              {#if typeof tourOnSkip === "function" && !lastStep}
                 <Link
                   secondary
                   quiet

packages/builder/src/components/portal/onboarding/TourWrap.svelte

@@ -1,39 +1,63 @@
 <script>
   import { tourHandler } from "./tourHandler"
-  import { TOURS } from "./tours"
+  import { TOURSBYSTEP, TOURS, getCurrentStepIdx } from "./tours"
   import { onMount, onDestroy } from "svelte"
   import { builderStore } from "stores/builder"
 
-  export let tourStepKey
+  export let stepKeys = []
 
-  let currentTourStep
   let ready = false
-  let registered = false
+  let registered = {}
-  let handler
 
   const registerTourNode = (tourKey, stepKey) => {
-    if (ready && !registered && tourKey) {
+    const step = TOURSBYSTEP[stepKey]
-      currentTourStep = TOURS[tourKey].steps.find(step => step.id === stepKey)
+    if (ready && step && !registered[stepKey] && step?.tour === tourKey) {
-      if (!currentTourStep) {
+      const elem = document.querySelector(step.query)
-        return
+      registered[stepKey] = tourHandler(elem, stepKey)
+    }
+  }
+
+  const scrollToStep = () => {
+    let tourStepIdx = getCurrentStepIdx(
+      TOURS[tourKeyWatch]?.steps,
+      tourStepKeyWatch
+    )
+    let currentStep = TOURS[tourKeyWatch]?.steps?.[tourStepIdx]
+    if (currentStep?.scrollIntoView) {
+      let currentNode = $builderStore.tourNodes?.[currentStep.id]
+      if (currentNode) {
+        currentNode.scrollIntoView({ behavior: "smooth", block: "center" })
       }
-      const elem = document.querySelector(currentTourStep.query)
-      handler = tourHandler(elem, stepKey)
-      registered = true
     }
   }
 
   $: tourKeyWatch = $builderStore.tourKey
-  $: registerTourNode(tourKeyWatch, tourStepKey, ready)
+  $: tourStepKeyWatch = $builderStore.tourStepKey
+  $: if (tourKeyWatch || stepKeys || ready) {
+    stepKeys.forEach(tourStepKey => {
+      registerTourNode(tourKeyWatch, tourStepKey)
+    })
+  }
+  $: scrollToStep(tourKeyWatch, tourStepKeyWatch)
 
   onMount(() => {
     ready = true
   })
 
   onDestroy(() => {
-    if (handler) {
+    Object.entries(registered).forEach(entry => {
+      const handler = entry[1]
+      const stepKey = entry[0]
+      // Run step destroy, de-register nodes in the builderStore and local cache
       handler.destroy()
-    }
+      delete registered[stepKey]
+
+      // Check if the step is part of an active tour. End the tour if that is the case
+      const step = TOURSBYSTEP[stepKey]
+      if (step.tour === tourKeyWatch) {
+        builderStore.setTour()
+      }
+    })
   })
 </script>
 

packages/builder/src/components/portal/onboarding/steps/NewFormSteps.svelte

@@ -0,0 +1,9 @@
+<div>
+  When faced with a sizable form, consider implementing a multi-step approach to
+  enhance user experience.
+
+  <p>
+    Breaking the form into multiple steps can significantly improve usability by
+    making the process more digestible for your users.
+  </p>
+</div>

packages/builder/src/components/portal/onboarding/steps/NewViewUpdateFormRowId.svelte

@@ -0,0 +1,17 @@
+<div>
+  You can use bindings to set the Row ID on your form.
+  <p>
+    This will allow you to pull the correct information into your form and allow
+    you to update!
+  </p>
+  <a href="https://docs.budibase.com/docs/form-block" target="_blank">
+    How to pass a row ID using bindings
+  </a>
+</div>
+
+<style>
+  a {
+    color: inherit;
+    text-decoration: underline;
+  }
+</style>

packages/builder/src/components/portal/onboarding/steps/index.js

@@ -1,3 +1,5 @@
 export { default as OnboardingData } from "./OnboardingData.svelte"
 export { default as OnboardingDesign } from "./OnboardingDesign.svelte"
 export { default as OnboardingPublish } from "./OnboardingPublish.svelte"
+export { default as NewViewUpdateFormRowId } from "./NewViewUpdateFormRowId.svelte"
+export { default as NewFormSteps } from "./NewFormSteps.svelte"

packages/builder/src/components/portal/onboarding/tours.js

@@ -2,8 +2,15 @@ import { get } from "svelte/store"
 import { builderStore } from "stores/builder"
 import { auth } from "stores/portal"
 import analytics from "analytics"
-import { OnboardingData, OnboardingDesign, OnboardingPublish } from "./steps"
+import {
+  OnboardingData,
+  OnboardingDesign,
+  OnboardingPublish,
+  NewViewUpdateFormRowId,
+  NewFormSteps,
+} from "./steps"
 import { API } from "api"
+import { customPositionHandler } from "components/design/settings/controls/EditComponentPopover"
 
 const ONBOARDING_EVENT_PREFIX = "onboarding"
 
@@ -14,11 +21,26 @@ export const TOUR_STEP_KEYS = {
   BUILDER_USER_MANAGEMENT: "builder-user-management",
   BUILDER_AUTOMATION_SECTION: "builder-automation-section",
   FEATURE_USER_MANAGEMENT: "feature-user-management",
+  BUILDER_FORM_CREATE_STEPS: "builder-form-create-steps",
+  BUILDER_FORM_VIEW_UPDATE_STEPS: "builder-form-view-update-steps",
+  BUILDER_FORM_ROW_ID: "builder-form-row-id",
 }
 
 export const TOUR_KEYS = {
   TOUR_BUILDER_ONBOARDING: "builder-onboarding",
   FEATURE_ONBOARDING: "feature-onboarding",
+  BUILDER_FORM_CREATE: "builder-form-create",
+  BUILDER_FORM_VIEW_UPDATE: "builder-form-view-update",
+}
+
+export const getCurrentStepIdx = (steps, tourStepKey) => {
+  if (!steps?.length) {
+    return
+  }
+  if (steps?.length && !tourStepKey) {
+    return 0
+  }
+  return steps.findIndex(step => step.id === tourStepKey)
 }
 
 const endUserOnboarding = async ({ skipped = false } = {}) => {
@@ -37,13 +59,8 @@ const endUserOnboarding = async ({ skipped = false } = {}) => {
       // Update the cached user
       await auth.getSelf()
 
-      builderStore.update(state => ({
+      builderStore.endBuilderOnboarding()
-        ...state,
+      builderStore.setTour()
-        tourNodes: null,
-        tourKey: null,
-        tourStepKey: null,
-        onboarding: false,
-      }))
     } catch (e) {
       console.error("Onboarding failed", e)
       return false
@@ -52,9 +69,29 @@ const endUserOnboarding = async ({ skipped = false } = {}) => {
   }
 }
 
-const tourEvent = eventKey => {
+const endTour = async ({ key, skipped = false } = {}) => {
+  const { tours = {} } = get(auth).user
+  tours[key] = new Date().toISOString()
+
+  await API.updateSelf({
+    tours,
+  })
+
+  if (skipped) {
+    tourEvent(key, skipped)
+  }
+
+  // Update the cached user
+  await auth.getSelf()
+
+  // Reset tour state
+  builderStore.setTour()
+}
+
+const tourEvent = (eventKey, skipped) => {
   analytics.captureEvent(`${ONBOARDING_EVENT_PREFIX}:${eventKey}`, {
     eventSource: EventSource.PORTAL,
+    skipped,
   })
 }
 
@@ -135,7 +172,71 @@ const getTours = () => {
         },
       ],
     },
+    [TOUR_KEYS.BUILDER_FORM_CREATE]: {
+      steps: [
+        {
+          id: TOUR_STEP_KEYS.BUILDER_FORM_CREATE_STEPS,
+          title: "Add multiple steps",
+          layout: NewFormSteps,
+          query: "#steps-prop-control-wrap",
+          onComplete: () => {
+            builderStore.highlightSetting()
+            endTour({ key: TOUR_KEYS.BUILDER_FORM_CREATE })
+          },
+          onLoad: () => {
+            tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_CREATE_STEPS)
+            builderStore.highlightSetting("steps", "info")
+          },
+          positionHandler: customPositionHandler,
+          align: "left-outside",
+        },
+      ],
+    },
+    [TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE]: {
+      steps: [
+        {
+          id: TOUR_STEP_KEYS.BUILDER_FORM_ROW_ID,
+          title: "Add row ID to update a row",
+          layout: NewViewUpdateFormRowId,
+          query: "#rowId-prop-control-wrap",
+          onLoad: () => {
+            tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_ROW_ID)
+            builderStore.highlightSetting("rowId", "info")
+          },
+          positionHandler: customPositionHandler,
+          align: "left-outside",
+        },
+        {
+          id: TOUR_STEP_KEYS.BUILDER_FORM_VIEW_UPDATE_STEPS,
+          title: "Add multiple steps",
+          layout: NewFormSteps,
+          query: "#steps-prop-control-wrap",
+          onComplete: () => {
+            builderStore.highlightSetting()
+            endTour({ key: TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE })
+          },
+          onLoad: () => {
+            tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_VIEW_UPDATE_STEPS)
+            builderStore.highlightSetting("steps", "info")
+          },
+          positionHandler: customPositionHandler,
+          align: "left-outside",
+          scrollIntoView: true,
+        },
+      ],
+      onSkip: async () => {
+        builderStore.highlightSetting()
+        endTour({ key: TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE, skipped: true })
+      },
+    },
   }
 }
 
 export const TOURS = getTours()
+export const TOURSBYSTEP = Object.keys(TOURS).reduce((acc, tour) => {
+  TOURS[tour].steps.forEach(element => {
+    acc[element.id] = element
+    acc[element.id]["tour"] = tour
+  })
+  return acc
+}, {})

packages/builder/src/dataBinding.js

@@ -1146,7 +1146,7 @@ export const getAllStateVariables = () => {
         "@budibase/standard-components/multistepformblockstep"
       )
 
-      steps.forEach(step => {
+      steps?.forEach(step => {
         parseComponentSettings(stepDefinition, step)
       })
     })

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -96,7 +96,7 @@
         const release_date = new Date("2023-03-01T00:00:00.000Z")
         const onboarded = new Date($auth.user?.onboardedAt)
         if (onboarded < release_date) {
-          builderStore.startTour(TOUR_KEYS.FEATURE_ONBOARDING)
+          builderStore.setTour(TOUR_KEYS.FEATURE_ONBOARDING)
         }
       }
     }
@@ -144,7 +144,7 @@
         </span>
         <Tabs {selected} size="M">
           {#each $layout.children as { path, title }}
-            <TourWrap tourStepKey={`builder-${title}-section`}>
+            <TourWrap stepKeys={[`builder-${title}-section`]}>
               <Tab
                 quiet
                 selected={$isActive(path)}

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte

@@ -16,6 +16,14 @@
   } from "dataBinding"
   import { ActionButton, notifications } from "@budibase/bbui"
   import { capitalise } from "helpers"
+  import TourWrap from "components/portal/onboarding/TourWrap.svelte"
+  import { TOUR_STEP_KEYS } from "components/portal/onboarding/tours.js"
+
+  const {
+    BUILDER_FORM_CREATE_STEPS,
+    BUILDER_FORM_VIEW_UPDATE_STEPS,
+    BUILDER_FORM_ROW_ID,
+  } = TOUR_STEP_KEYS
 
   const onUpdateName = async value => {
     try {
@@ -46,7 +54,6 @@
 
   $: id = $selectedComponent?._id
   $: id, (section = tabs[0])
-
   $: componentName = getComponentName(componentInstance)
 </script>
 
@@ -92,13 +99,21 @@
         </div>
       </span>
       {#if section == "settings"}
-        <ComponentSettingsSection
+        <TourWrap
-          {componentInstance}
+          stepKeys={[
-          {componentDefinition}
+            BUILDER_FORM_CREATE_STEPS,
-          {bindings}
+            BUILDER_FORM_VIEW_UPDATE_STEPS,
-          {componentBindings}
+            BUILDER_FORM_ROW_ID,
-          {isScreen}
+          ]}
-        />
+        >
+          <ComponentSettingsSection
+            {componentInstance}
+            {componentDefinition}
+            {bindings}
+            {componentBindings}
+            {isScreen}
+          />
+        </TourWrap>
       {/if}
       {#if section == "styles"}
         <DesignSection

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte

@@ -1,7 +1,7 @@
 <script>
   import { helpers } from "@budibase/shared-core"
   import { DetailSummary, notifications } from "@budibase/bbui"
-  import { componentStore } from "stores/builder"
+  import { componentStore, builderStore } from "stores/builder"
   import PropertyControl from "components/design/settings/controls/PropertyControl.svelte"
   import ResetFieldsButton from "components/design/settings/controls/ResetFieldsButton.svelte"
   import EjectBlockButton from "components/design/settings/controls/EjectBlockButton.svelte"
@@ -177,9 +177,7 @@
               defaultValue={setting.defaultValue}
               nested={setting.nested}
               onChange={val => updateSetting(setting, val)}
-              highlighted={$componentStore.highlightedSettingKey ===
+              propertyFocus={$builderStore.propertyFocus === setting.key}
-                setting.key}
-              propertyFocus={$componentStore.propertyFocus === setting.key}
               info={setting.info}
               disableBindings={setting.disableBindings}
               props={{

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte

@@ -174,7 +174,7 @@
     } else if (type === "request-add-component") {
       toggleAddComponent()
     } else if (type === "highlight-setting") {
-      builderStore.highlightSetting(data.setting)
+      builderStore.highlightSetting(data.setting, "error")
 
       // Also scroll setting into view
       const selector = `#${data.setting}-prop-control`

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte

@@ -3,13 +3,23 @@
   import DatasourceModal from "./DatasourceModal.svelte"
   import ScreenRoleModal from "./ScreenRoleModal.svelte"
   import sanitizeUrl from "helpers/sanitizeUrl"
+  import FormTypeModal from "./FormTypeModal.svelte"
   import { Modal, notifications } from "@budibase/bbui"
-  import { screenStore, navigationStore, tables } from "stores/builder"
+  import {
+    screenStore,
+    navigationStore,
+    tables,
+    builderStore,
+  } from "stores/builder"
+  import { auth } from "stores/portal"
   import { get } from "svelte/store"
   import getTemplates from "templates"
   import { Roles } from "constants/backend"
   import { capitalise } from "helpers"
   import { goto } from "@roxi/routify"
+  import { TOUR_KEYS } from "components/portal/onboarding/tours.js"
+  import formScreen from "templates/formScreen"
+  import rowListScreen from "templates/rowListScreen"
 
   let mode
   let pendingScreen
@@ -18,12 +28,18 @@
   let screenDetailsModal
   let datasourceModal
   let screenAccessRoleModal
+  let formTypeModal
 
   // Cache variables for workflow
   let screenAccessRole = Roles.BASIC
-  let selectedTemplates = null
+
+  let templates = null
+  let screens = null
+
+  let selectedDatasources = null
   let blankScreenUrl = null
   let screenMode = null
+  let formType = null
 
   // Creates an array of screens, checking and sanitising their URLs
   const createScreens = async ({ screens, screenAccessRole }) => {
@@ -32,7 +48,7 @@
     }
 
     try {
-      let screenId
+      let createdScreens = []
 
       for (let screen of screens) {
         // Check we aren't clashing with an existing URL
@@ -56,7 +72,7 @@
 
         // Create the screen
         const response = await screenStore.save(screen)
-        screenId = response._id
+        createdScreens.push(response)
 
         // Add link in layout. We only ever actually create 1 screen now, even
         // for autoscreens, so it's always safe to do this.
@@ -66,9 +82,7 @@
         )
       }
 
-      // Go to new screen
+      return createdScreens
-      $goto(`./${screenId}`)
-      screenStore.select(screenId)
     } catch (error) {
       console.error(error)
       notifications.error("Error creating screens")
@@ -104,13 +118,16 @@
   // Handler for NewScreenModal
   export const show = newMode => {
     mode = newMode
-    selectedTemplates = null
+    templates = null
+    screens = null
+    selectedDatasources = null
     blankScreenUrl = null
     screenMode = mode
     pendingScreen = null
     screenAccessRole = Roles.BASIC
+    formType = null
 
-    if (mode === "table" || mode === "grid") {
+    if (mode === "table" || mode === "grid" || mode === "form") {
       datasourceModal.show()
     } else if (mode === "blank") {
       let templates = getTemplates($tables.list)
@@ -125,19 +142,26 @@
   }
 
   // Handler for DatasourceModal confirmation, move to screen access select
-  const confirmScreenDatasources = async ({ templates }) => {
+  const confirmScreenDatasources = async ({ datasources }) => {
-    selectedTemplates = templates
+    selectedDatasources = datasources
-    screenAccessRoleModal.show()
+    if (screenMode === "form") {
+      formTypeModal.show()
+    } else {
+      screenAccessRoleModal.show()
+    }
   }
 
   // Handler for Datasource Screen Creation
   const completeDatasourceScreenCreation = async () => {
-    const screens = selectedTemplates.map(template => {
+    templates = rowListScreen(selectedDatasources, mode)
+
+    const screens = templates.map(template => {
       let screenTemplate = template.create()
       screenTemplate.autoTableId = template.resourceId
       return screenTemplate
     })
-    await createScreens({ screens, screenAccessRole })
+    const createdScreens = await createScreens({ screens, screenAccessRole })
+    loadNewScreen(createdScreens)
   }
 
   const confirmScreenBlank = async ({ screenUrl }) => {
@@ -154,7 +178,54 @@
       return
     }
     pendingScreen.routing.route = screenUrl
-    await createScreens({ screens: [pendingScreen], screenAccessRole })
+    const createdScreens = await createScreens({
+      screens: [pendingScreen],
+      screenAccessRole,
+    })
+    loadNewScreen(createdScreens)
+  }
+
+  const onConfirmFormType = () => {
+    screenAccessRoleModal.show()
+  }
+
+  const loadNewScreen = createdScreens => {
+    const lastScreen = createdScreens.slice(-1)[0]
+
+    // Go to new screen
+    if (lastScreen?.props?._children.length) {
+      // Focus on the main component for the streen type
+      const mainComponent = lastScreen?.props?._children?.[0]._id
+      $goto(`./${lastScreen._id}/${mainComponent}`)
+    } else {
+      $goto(`./${lastScreen._id}`)
+    }
+
+    screenStore.select(lastScreen._id)
+  }
+
+  const confirmFormScreenCreation = async () => {
+    templates = formScreen(selectedDatasources, { actionType: formType })
+    screens = templates.map(template => {
+      let screenTemplate = template.create()
+      return screenTemplate
+    })
+    const createdScreens = await createScreens({ screens, screenAccessRole })
+
+    if (formType === "Update" || formType === "Create") {
+      const associatedTour =
+        formType === "Update"
+          ? TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE
+          : TOUR_KEYS.BUILDER_FORM_CREATE
+
+      const tourRequired = !$auth?.user?.tours?.[associatedTour]
+      if (tourRequired) {
+        builderStore.setTour(associatedTour)
+      }
+    }
+
+    // Go to new screen
+    loadNewScreen(createdScreens)
   }
 
   // Submit screen config for creation.
@@ -164,6 +235,8 @@
         screenUrl: blankScreenUrl,
         screenAccessRole,
       })
+    } else if (screenMode === "form") {
+      confirmFormScreenCreation()
     } else {
       completeDatasourceScreenCreation()
     }
@@ -179,19 +252,18 @@
 </script>
 
 <Modal bind:this={datasourceModal} autoFocus={false}>
-  <DatasourceModal
+  <DatasourceModal {mode} onConfirm={confirmScreenDatasources} />
-    {mode}
-    onConfirm={confirmScreenDatasources}
-    initialScreens={!selectedTemplates ? [] : [...selectedTemplates]}
-  />
 </Modal>
 
 <Modal bind:this={screenAccessRoleModal}>
   <ScreenRoleModal
-    onConfirm={confirmScreenCreation}
+    onConfirm={() => {
-    onCancel={roleSelectBack}
+      confirmScreenCreation()
+    }}
     bind:screenAccessRole
+    onCancel={roleSelectBack}
     screenUrl={blankScreenUrl}
+    confirmText={screenMode === "form" ? "Confirm" : "Done"}
   />
 </Modal>
 
@@ -201,3 +273,17 @@
     initialUrl={blankScreenUrl}
   />
 </Modal>
+
+<Modal bind:this={formTypeModal}>
+  <FormTypeModal
+    onConfirm={onConfirmFormType}
+    onCancel={() => {
+      formTypeModal.hide()
+      datasourceModal.show()
+    }}
+    on:select={e => {
+      formType = e.detail
+    }}
+    type={formType}
+  />
+</Modal>

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte

@@ -4,37 +4,33 @@
   import ICONS from "components/backend/DatasourceNavigator/icons"
   import { IntegrationNames } from "constants"
   import { onMount } from "svelte"
-  import rowListScreen from "templates/rowListScreen"
   import DatasourceTemplateRow from "./DatasourceTemplateRow.svelte"
 
-  export let mode
   export let onCancel
   export let onConfirm
-  export let initialScreens = []
 
-  let selectedScreens = [...initialScreens]
+  let selectedSources = []
 
   $: filteredSources = $datasources.list?.filter(datasource => {
     return datasource.source !== IntegrationNames.REST && datasource["entities"]
   })
 
   const toggleSelection = datasource => {
-    const { resourceId } = datasource
+    const exists = selectedSources.find(
-    if (selectedScreens.find(s => s.resourceId === resourceId)) {
+      d => d.resourceId === datasource.resourceId
-      selectedScreens = selectedScreens.filter(
+    )
-        screen => screen.resourceId !== resourceId
+    if (exists) {
+      selectedSources = selectedSources.filter(
+        d => d.resourceId === datasource.resourceId
       )
     } else {
-      selectedScreens = [
+      selectedSources = [...selectedSources, datasource]
-        ...selectedScreens,
-        rowListScreen([datasource], mode)[0],
-      ]
     }
   }
 
   const confirmDatasourceSelection = async () => {
     await onConfirm({
-      templates: selectedScreens,
+      datasources: selectedSources,
     })
   }
 
@@ -54,7 +50,7 @@
     cancelText="Back"
     onConfirm={confirmDatasourceSelection}
     {onCancel}
-    disabled={!selectedScreens.length}
+    disabled={!selectedSources.length}
     size="L"
   >
     <Body size="S">
@@ -85,8 +81,8 @@
               resourceId: table._id,
               type: "table",
             }}
-            {@const selected = selectedScreens.find(
+            {@const selected = selectedSources.find(
-              screen => screen.resourceId === tableDS.resourceId
+              datasource => datasource.resourceId === tableDS.resourceId
             )}
             <DatasourceTemplateRow
               on:click={() => toggleSelection(tableDS)}
@@ -103,7 +99,7 @@
                 tableId: view.tableId,
                 type: "viewV2",
               }}
-              {@const selected = selectedScreens.find(
+              {@const selected = selectedSources.find(
                 x => x.resourceId === viewDS.resourceId
               )}
               <DatasourceTemplateRow

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/FormTypeModal.svelte

@@ -0,0 +1,123 @@
+<script>
+  import { ModalContent, Layout, Body, Icon } from "@budibase/bbui"
+  import { createEventDispatcher } from "svelte"
+
+  export let onCancel = () => {}
+  export let onConfirm = () => {}
+  export let type
+
+  const dispatch = createEventDispatcher()
+</script>
+
+<span>
+  <ModalContent
+    title="Select form type"
+    confirmText="Done"
+    cancelText="Back"
+    {onConfirm}
+    {onCancel}
+    disabled={!type}
+    size="L"
+  >
+    <!-- svelte-ignore a11y-click-events-have-key-events -->
+    <!-- svelte-ignore a11y-no-static-element-interactions -->
+    <Layout noPadding gap="S">
+      <div
+        class="form-type"
+        class:selected={type === "Create"}
+        on:click={() => {
+          dispatch("select", "Create")
+        }}
+      >
+        <div class="form-type-wrap">
+          <div class="form-type-content">
+            <Body noPadding>Create a new row</Body>
+            <Body size="S">
+              For capturing and storing new data from your users
+            </Body>
+          </div>
+          {#if type === "Create"}
+            <span class="form-type-check">
+              <Icon size="S" name="CheckmarkCircle" />
+            </span>
+          {/if}
+        </div>
+      </div>
+      <div
+        class="form-type"
+        class:selected={type === "Update"}
+        on:click={() => {
+          dispatch("select", "Update")
+        }}
+      >
+        <div class="form-type-wrap">
+          <div class="form-type-content">
+            <Body noPadding>Update an existing row</Body>
+            <Body size="S">For viewing and updating existing data</Body>
+          </div>
+          {#if type === "Update"}
+            <span class="form-type-check">
+              <Icon size="S" name="CheckmarkCircle" />
+            </span>
+          {/if}
+        </div>
+      </div>
+      <div
+        class="form-type"
+        class:selected={type === "View"}
+        on:click={() => {
+          dispatch("select", "View")
+        }}
+      >
+        <div class="form-type-wrap">
+          <div class="form-type-content">
+            <Body noPadding>View an existing row</Body>
+            <Body size="S">For a read only view of your data</Body>
+          </div>
+          {#if type === "View"}
+            <span class="form-type-check">
+              <Icon size="S" name="CheckmarkCircle" />
+            </span>
+          {/if}
+        </div>
+      </div>
+    </Layout>
+  </ModalContent>
+</span>
+
+<style>
+  .form-type {
+    cursor: pointer;
+    gap: var(--spacing-s);
+    padding: var(--spacing-m) var(--spacing-xl);
+    background: var(--spectrum-alias-background-color-secondary);
+    transition: 0.3s all;
+    border: 1px solid var(--spectrum-global-color-gray-300);
+    border-radius: 4px;
+    display: flex;
+    flex-direction: column;
+  }
+  .selected,
+  .form-type:hover {
+    background: var(--spectrum-alias-background-color-tertiary);
+  }
+  .form-type-wrap {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+  }
+  .form-type :global(p:nth-child(2)) {
+    color: var(--grey-6);
+  }
+  .form-type-check {
+    margin-left: auto;
+  }
+  .form-type-check :global(.spectrum-Icon) {
+    color: var(--spectrum-global-color-green-600);
+  }
+  .form-type-content {
+    gap: var(--spacing-s);
+    display: flex;
+    flex-direction: column;
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte

@@ -9,6 +9,7 @@
   export let onCancel
   export let screenUrl
   export let screenAccessRole
+  export let confirmText = "Done"
 
   let error
 
@@ -40,7 +41,7 @@
 
 <ModalContent
   title="Access"
-  confirmText="Done"
+  {confirmText}
   cancelText="Back"
   {onConfirm}
   {onCancel}

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte

@@ -1,9 +1,10 @@
 <script>
   import { Body } from "@budibase/bbui"
   import CreationPage from "components/common/CreationPage.svelte"
-  import blankImage from "./blank.png"
+  import blankImage from "./images/blank.png"
-  import tableImage from "./table.png"
+  import tableImage from "./images/table.png"
-  import gridImage from "./grid.png"
+  import gridImage from "./images/grid.png"
+  import formImage from "./images/form.png"
   import CreateScreenModal from "./CreateScreenModal.svelte"
   import { screenStore } from "stores/builder"
 
@@ -56,6 +57,16 @@
           <Body size="XS">View and manipulate rows on a grid</Body>
         </div>
       </div>
+
+      <div class="card" on:click={() => createScreenModal.show("form")}>
+        <div class="image">
+          <img alt="" src={formImage} />
+        </div>
+        <div class="text">
+          <Body size="S">Form</Body>
+          <Body size="XS">Capture data from your users</Body>
+        </div>
+      </div>
     </div>
   </CreationPage>
 </div>

packages/builder/src/stores/builder/app.js

@@ -29,6 +29,7 @@ export const INITIAL_APP_META_STATE = {
   initialised: false,
   hasAppPackage: false,
   usedPlugins: null,
+  automations: {},
   routes: {},
 }
 
@@ -63,6 +64,7 @@ export class AppMetaStore extends BudiStore {
         ...app.features,
       },
       initialised: true,
+      automations: app.automations || {},
       hasAppPackage: true,
     }))
   }

packages/builder/src/stores/builder/builder.js

@@ -2,12 +2,11 @@ import { get } from "svelte/store"
 import { createBuilderWebsocket } from "./websocket.js"
 import { BuilderSocketEvent } from "@budibase/shared-core"
 import BudiStore from "./BudiStore"
-import { previewStore } from "./preview.js"
 import { TOUR_KEYS } from "components/portal/onboarding/tours.js"
 
 export const INITIAL_BUILDER_STATE = {
   previousTopNavPath: {},
-  highlightedSettingKey: null,
+  highlightedSetting: null,
   propertyFocus: null,
   builderSidePanel: false,
   onboarding: false,
@@ -26,7 +25,6 @@ export class BuilderStore extends BudiStore {
     this.reset = this.reset.bind(this)
     this.highlightSetting = this.highlightSetting.bind(this)
     this.propertyFocus = this.propertyFocus.bind(this)
-    this.hover = this.hover.bind(this)
     this.hideBuilderSidePanel = this.hideBuilderSidePanel.bind(this)
     this.showBuilderSidePanel = this.showBuilderSidePanel.bind(this)
     this.setPreviousTopNavPath = this.setPreviousTopNavPath.bind(this)
@@ -58,10 +56,10 @@ export class BuilderStore extends BudiStore {
     this.websocket = null
   }
 
-  highlightSetting(key) {
+  highlightSetting(key, type) {
     this.update(state => ({
       ...state,
-      highlightedSettingKey: key,
+      highlightedSetting: key ? { key, type: type || "info" } : null,
     }))
   }
 
@@ -135,25 +133,20 @@ export class BuilderStore extends BudiStore {
     }))
   }
 
-  startTour(tourKey) {
+  endBuilderOnboarding() {
     this.update(state => ({
       ...state,
-      tourKey: tourKey,
+      onboarding: false,
     }))
   }
 
-  hover(componentId, notifyClient = true) {
+  setTour(tourKey) {
-    const store = get(this.store)
+    this.update(state => ({
-    if (componentId === store.hoveredComponentId) {
+      ...state,
-      return
+      tourStepKey: null,
-    }
+      tourNodes: null,
-    this.update(state => {
+      tourKey: tourKey,
-      state.hoveredComponentId = componentId
+    }))
-      return state
-    })
-    if (notifyClient) {
-      previewStore.sendEvent("hover-component", componentId)
-    }
   }
 }
 

packages/builder/src/stores/builder/tests/builder.test.js

@@ -88,14 +88,42 @@ describe("Builder store", () => {
     )
   })
 
-  it("Sync a highlighted setting key to state", ctx => {
+  it("Sync a highlighted setting key to state. Default to info type", ctx => {
-    expect(ctx.test.store.highlightedSettingKey).toBeNull()
+    expect(ctx.test.store.highlightedSetting).toBeNull()
 
     ctx.test.builderStore.highlightSetting("testing")
 
     expect(ctx.test.store).toStrictEqual({
       ...INITIAL_BUILDER_STATE,
-      highlightedSettingKey: "testing",
+      highlightedSetting: {
+        key: "testing",
+        type: "info",
+      },
+    })
+  })
+
+  it("Sync a highlighted setting key to state. Use provided type", ctx => {
+    expect(ctx.test.store.highlightedSetting).toBeNull()
+
+    ctx.test.builderStore.highlightSetting("testing", "error")
+
+    expect(ctx.test.store).toStrictEqual({
+      ...INITIAL_BUILDER_STATE,
+      highlightedSetting: {
+        key: "testing",
+        type: "error",
+      },
+    })
+  })
+
+  it("Sync a highlighted setting key to state. Unset when no value is passed", ctx => {
+    expect(ctx.test.store.highlightedSetting).toBeNull()
+
+    ctx.test.builderStore.highlightSetting("testing", "error")
+    ctx.test.builderStore.highlightSetting()
+
+    expect(ctx.test.store).toStrictEqual({
+      ...INITIAL_BUILDER_STATE,
     })
   })
 

packages/builder/src/templates/formScreen.js

@@ -0,0 +1,43 @@
+import { Screen } from "./Screen"
+import { Component } from "./Component"
+import sanitizeUrl from "helpers/sanitizeUrl"
+
+export const FORM_TEMPLATE = "FORM_TEMPLATE"
+export const formUrl = datasource => sanitizeUrl(`/${datasource.label}-form`)
+
+// Mode not really necessary
+export default function (datasources, config) {
+  if (!Array.isArray(datasources)) {
+    return []
+  }
+  return datasources.map(datasource => {
+    return {
+      name: `${datasource.label} - Form`,
+      create: () => createScreen(datasource, config),
+      id: FORM_TEMPLATE,
+      resourceId: datasource.resourceId,
+    }
+  })
+}
+
+const generateMultistepFormBlock = (dataSource, { actionType } = {}) => {
+  const multistepFormBlock = new Component(
+    "@budibase/standard-components/multistepformblock"
+  )
+  multistepFormBlock
+    .customProps({
+      actionType,
+      dataSource,
+      steps: [{}],
+    })
+    .instanceName(`${dataSource.label} - Multistep Form block`)
+  return multistepFormBlock
+}
+
+const createScreen = (datasource, config) => {
+  return new Screen()
+    .route(formUrl(datasource))
+    .instanceName(`${datasource.label} - Form`)
+    .addChild(generateMultistepFormBlock(datasource, config))
+    .json()
+}

packages/builder/src/templates/index.js

@@ -1,7 +1,11 @@
 import rowListScreen from "./rowListScreen"
 import createFromScratchScreen from "./createFromScratchScreen"
+import formScreen from "./formScreen"
 
-const allTemplates = datasources => [...rowListScreen(datasources)]
+const allTemplates = datasources => [
+  ...rowListScreen(datasources),
+  ...formScreen(datasources),
+]
 
 // Allows us to apply common behaviour to all create() functions
 const createTemplateOverride = template => () => {
@@ -19,6 +23,7 @@ export default datasources => {
   })
   const fromScratch = enrichTemplate(createFromScratchScreen)
   const tableTemplates = allTemplates(datasources).map(enrichTemplate)
+
   return [
     fromScratch,
     ...tableTemplates.sort((templateA, templateB) => {

packages/client/manifest.json

@@ -525,6 +525,38 @@
         "barTitle": "Disable button",
         "key": "disabled"
       },
+      {
+        "type": "icon",
+        "label": "Icon",
+        "key": "icon"
+      },
+      {
+        "type": "select",
+        "label": "Gap",
+        "key": "gap",
+        "showInBar": true,
+        "barStyle": "picker",
+        "dependsOn": "icon",
+        "options": [
+          {
+            "label": "None",
+            "value": "N"
+          },
+          {
+            "label": "Small",
+            "value": "S"
+          },
+          {
+            "label": "Medium",
+            "value": "M"
+          },
+          {
+            "label": "Large",
+            "value": "L"
+          }
+        ],
+        "defaultValue": "M"
+      },
       {
         "type": "event",
         "label": "On click",
@@ -1105,6 +1137,12 @@
         "key": "color",
         "showInBar": true
       },
+      {
+        "type": "color",
+        "label": "Text Color",
+        "key": "textColor",
+        "showInBar": true
+      },
       {
         "type": "boolean",
         "label": "Allow delete",

packages/client/src/components/app/Button.svelte

@@ -13,9 +13,10 @@
   export let size = "M"
   export let type = "cta"
   export let quiet = false
+  export let icon = null
+  export let gap = "M"
 
   // For internal use only for now - not defined in the manifest
-  export let icon = null
   export let active = false
 
   const handleOnClick = async () => {
@@ -47,7 +48,7 @@
 
 {#key $component.editing}
   <button
-    class={`spectrum-Button spectrum-Button--size${size} spectrum-Button--${type}`}
+    class={`spectrum-Button spectrum-Button--size${size} spectrum-Button--${type} gap-${gap}`}
     class:spectrum-Button--quiet={quiet}
     disabled={disabled || handlingOnClick}
     use:styleable={$component.styles}
@@ -58,15 +59,7 @@
     class:active
   >
     {#if icon}
-      <svg
+      <i class="{icon} {size}" />
-        class:hasText={componentText?.length > 0}
-        class="spectrum-Icon spectrum-Icon--size{size.toUpperCase()}"
-        focusable="false"
-        aria-hidden="true"
-        aria-label={icon}
-      >
-        <use xlink:href="#spectrum-icon-18-{icon}" />
-      </svg>
     {/if}
     {componentText}
   </button>
@@ -92,4 +85,13 @@
   .active {
     color: var(--spectrum-global-color-blue-600);
   }
+  .gap-S {
+    gap: 8px;
+  }
+  .gap-M {
+    gap: 16px;
+  }
+  .gap-L {
+    gap: 32px;
+  }
 </style>

packages/client/src/components/app/ButtonGroup.svelte

@@ -20,7 +20,7 @@
       wrap: true,
     }}
   >
-    {#each buttons as { text, type, quiet, disabled, onClick, size }}
+    {#each buttons as { text, type, quiet, disabled, onClick, size, icon, gap }}
       <BlockComponent
         type="button"
         props={{
@@ -29,6 +29,8 @@
           type,
           quiet,
           disabled,
+          icon,
+          gap,
           size: size || "M",
         }}
       />

packages/client/src/components/app/Tag.svelte

@@ -6,6 +6,7 @@
   export let onClick
   export let text = ""
   export let color
+  export let textColor
   export let closable = false
   export let size = "M"
 
@@ -14,7 +15,7 @@
 
   // Add color styles to main styles object, otherwise the styleable helper
   // overrides the color when it's passed as inline style.
-  $: styles = enrichStyles($component.styles, color)
+  $: styles = enrichStyles($component.styles, color, textColor)
   $: componentText = getComponentText(text, $builderStore, $component)
 
   const getComponentText = (text, builderState, componentState) => {
@@ -24,7 +25,7 @@
     return text || componentState.name || "Placeholder text"
   }
 
-  const enrichStyles = (styles, color) => {
+  const enrichStyles = (styles, color, textColor) => {
     if (!color) {
       return styles
     }
@@ -34,7 +35,7 @@
         ...styles?.normal,
         "background-color": color,
         "border-color": color,
-        color: "white",
+        color: textColor || "white",
         "--spectrum-clearbutton-medium-icon-color": "white",
       },
     }

packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte

@@ -92,9 +92,9 @@
 {#if schemaLoaded}
   <Button
     onClick={openEditor}
-    icon="Properties"
+    icon="ri-filter-3-line"
     text="Filter"
-    {size}
+    size="XL"
     type="secondary"
     quiet
     active={filters?.length > 0}

packages/pro

@@ -1 +1 @@
-Subproject commit 22a278da720d92991dabdcd4cb6c96e7abe29781
+Subproject commit 80a4d8ff998895fc298ee510158a82ce7daebc67

packages/server/__mocks__/node-fetch.ts

@@ -8,6 +8,7 @@ module FetchMock {
   let mockSearch = false
 
   const func = async (url: any, opts: any) => {
+    const { host, pathname } = new URL(url)
     function json(body: any, status = 200) {
       return {
         status,
@@ -34,7 +35,7 @@ module FetchMock {
       }
     }
 
-    if (url.includes("/api/global")) {
+    if (pathname.includes("/api/global")) {
       const user = {
         email: "test@example.com",
         _id: "us_test@example.com",
@@ -47,31 +48,31 @@ module FetchMock {
           global: false,
         },
       }
-      return url.endsWith("/users") && opts.method === "GET"
+      return pathname.endsWith("/users") && opts.method === "GET"
         ? json([user])
         : json(user)
     }
     // mocked data based on url
-    else if (url.includes("api/apps")) {
+    else if (pathname.includes("api/apps")) {
       return json({
         app1: {
           url: "/app1",
         },
       })
-    } else if (url.includes("example.com")) {
+    } else if (host.includes("example.com")) {
       return json({
         body: opts.body,
         url,
         method: opts.method,
       })
-    } else if (url.includes("invalid.com")) {
+    } else if (host.includes("invalid.com")) {
       return json(
         {
           invalid: true,
         },
         404
       )
-    } else if (mockSearch && url.includes("_search")) {
+    } else if (mockSearch && pathname.includes("_search")) {
       const body = opts.body
       const parts = body.split("tableId:")
       let tableId
@@ -90,7 +91,7 @@ module FetchMock {
         ],
         bookmark: "test",
       })
-    } else if (url.includes("google.com")) {
+    } else if (host.includes("google.com")) {
       return json({
         url,
         opts,
@@ -177,7 +178,7 @@ module FetchMock {
     } else if (url === "https://www.googleapis.com/oauth2/v4/token") {
       // any valid response
       return json({})
-    } else if (url.includes("failonce.com")) {
+    } else if (host.includes("failonce.com")) {
       failCount++
       if (failCount === 1) {
         return json({ message: "error" }, 500)

packages/server/package.json

@@ -149,7 +149,7 @@
     "rimraf": "3.0.2",
     "supertest": "6.3.3",
     "swagger-jsdoc": "6.1.0",
-    "testcontainers": "10.6.0",
+    "testcontainers": "10.7.2",
     "timekeeper": "2.2.0",
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",

packages/server/src/api/controllers/role.ts

@@ -106,6 +106,21 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
   )
   role._rev = result.rev
   ctx.body = role
+
+  const devDb = context.getDevAppDB()
+  const prodDb = context.getProdAppDB()
+
+  if (await prodDb.exists()) {
+    const replication = new dbCore.Replication({
+      source: devDb.name,
+      target: prodDb.name,
+    })
+    await replication.replicate({
+      filter: (doc: any, params: any) => {
+        return doc._id && doc._id.startsWith("role_")
+      },
+    })
+  }
 }
 
 export async function destroy(ctx: UserCtx<void, DestroyRoleResponse>) {

packages/server/src/api/controllers/user.ts

@@ -1,6 +1,6 @@
 import { generateUserFlagID, InternalTables } from "../../db/utils"
 import { getFullUser } from "../../utilities/users"
-import { context } from "@budibase/backend-core"
+import { cache, context } from "@budibase/backend-core"
 import {
   ContextUserMetadata,
   Ctx,

packages/server/src/api/routes/tests/application.spec.ts

@@ -16,8 +16,9 @@ import * as setup from "./utilities"
 import { AppStatus } from "../../../db/utils"
 import { events, utils, context } from "@budibase/backend-core"
 import env from "../../../environment"
-import type { App } from "@budibase/types"
+import { type App } from "@budibase/types"
 import tk from "timekeeper"
+import * as uuid from "uuid"
 
 describe("/applications", () => {
   let config = setup.getConfig()
@@ -321,7 +322,7 @@ describe("/applications", () => {
 
   describe("permissions", () => {
     it("should only return apps a user has access to", async () => {
-      const user = await config.createUser({
+      let user = await config.createUser({
         builder: { global: false },
         admin: { global: false },
       })
@@ -330,6 +331,81 @@ describe("/applications", () => {
         const apps = await config.api.application.fetch()
         expect(apps).toHaveLength(0)
       })
+
+      user = await config.globalUser({
+        ...user,
+        builder: {
+          apps: [config.getProdAppId()],
+        },
+      })
+
+      await config.withUser(user, async () => {
+        const apps = await config.api.application.fetch()
+        expect(apps).toHaveLength(1)
+      })
+    })
+
+    it("should only return apps a user has access to through a custom role", async () => {
+      let user = await config.createUser({
+        builder: { global: false },
+        admin: { global: false },
+      })
+
+      await config.withUser(user, async () => {
+        const apps = await config.api.application.fetch()
+        expect(apps).toHaveLength(0)
+      })
+
+      const role = await config.api.roles.save({
+        name: "Test",
+        inherits: "PUBLIC",
+        permissionId: "read_only",
+        version: "name",
+      })
+
+      user = await config.globalUser({
+        ...user,
+        roles: {
+          [config.getProdAppId()]: role.name,
+        },
+      })
+
+      await config.withUser(user, async () => {
+        const apps = await config.api.application.fetch()
+        expect(apps).toHaveLength(1)
+      })
+    })
+
+    it.only("should only return apps a user has access to through a custom role on a group", async () => {
+      let user = await config.createUser({
+        builder: { global: false },
+        admin: { global: false },
+      })
+
+      await config.withUser(user, async () => {
+        const apps = await config.api.application.fetch()
+        expect(apps).toHaveLength(0)
+      })
+
+      const roleName = uuid.v4().replace(/-/g, "")
+      const role = await config.api.roles.save({
+        name: roleName,
+        inherits: "PUBLIC",
+        permissionId: "read_only",
+        version: "name",
+      })
+
+      const group = await config.createGroup(role._id!)
+
+      user = await config.globalUser({
+        ...user,
+        userGroups: [group._id!],
+      })
+
+      await config.withUser(user, async () => {
+        const apps = await config.api.application.fetch()
+        expect(apps).toHaveLength(1)
+      })
     })
   })
 })

packages/server/src/tests/jestEnv.ts

@@ -10,3 +10,4 @@ process.env.MOCK_REDIS = "1"
 process.env.PLATFORM_URL = "http://localhost:10000"
 process.env.REDIS_PASSWORD = "budibase"
 process.env.BUDIBASE_VERSION = "0.0.0+jest"
+process.env.WORKER_URL = "http://localhost:10000"

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -299,11 +299,11 @@ export default class TestConfiguration {
     }
   }
 
-  withUser(user: User, f: () => Promise<void>) {
+  async withUser(user: User, f: () => Promise<void>) {
     const oldUser = this.user
     this.user = user
     try {
-      return f()
+      return await f()
     } finally {
       this.user = oldUser
     }
@@ -347,7 +347,7 @@ export default class TestConfiguration {
       lastName = generator.last(),
       builder = { global: true },
       admin = { global: false },
-      email = generator.email(),
+      email = generator.email({ domain: "example.com" }),
       tenantId = this.getTenantId(),
       roles = {},
     } = config
@@ -363,6 +363,7 @@ export default class TestConfiguration {
       _id,
       ...existing,
       ...config,
+      _rev: existing._rev,
       email,
       roles,
       tenantId,
@@ -372,11 +373,12 @@ export default class TestConfiguration {
       admin,
     }
     await sessions.createASession(_id, {
-      sessionId: "sessionid",
+      sessionId: this.sessionIdForUser(_id),
       tenantId: this.getTenantId(),
       csrfToken: this.csrfToken,
     })
     const resp = await db.put(user)
+    await cache.user.invalidateUser(_id)
     return {
       _rev: resp.rev,
       ...user,
@@ -384,9 +386,7 @@ export default class TestConfiguration {
   }
 
   async createUser(user: Partial<User> = {}): Promise<User> {
-    const resp = await this.globalUser(user)
+    return await this.globalUser(user)
-    await cache.user.invalidateUser(resp._id!)
-    return resp
   }
 
   async createGroup(roleId: string = roles.BUILTIN_ROLE_IDS.BASIC) {
@@ -416,6 +416,10 @@ export default class TestConfiguration {
     })
   }
 
+  sessionIdForUser(userId: string): string {
+    return `sessionid-${userId}`
+  }
+
   async login({
     roleId,
     userId,
@@ -442,13 +446,13 @@ export default class TestConfiguration {
         })
       }
       await sessions.createASession(userId, {
-        sessionId: "sessionid",
+        sessionId: this.sessionIdForUser(userId),
         tenantId: this.getTenantId(),
       })
       // have to fake this
       const authObj = {
         userId,
-        sessionId: "sessionid",
+        sessionId: this.sessionIdForUser(userId),
         tenantId: this.getTenantId(),
       }
       const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret)
@@ -470,7 +474,7 @@ export default class TestConfiguration {
     const user = this.getUser()
     const authObj: AuthToken = {
       userId: user._id!,
-      sessionId: "sessionid",
+      sessionId: this.sessionIdForUser(user._id!),
       tenantId,
     }
     const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret)
@@ -508,7 +512,7 @@ export default class TestConfiguration {
 
   async basicRoleHeaders() {
     return await this.roleHeaders({
-      email: generator.email(),
+      email: generator.email({ domain: "example.com" }),
       builder: false,
       prodApp: true,
       roleId: roles.BUILTIN_ROLE_IDS.BASIC,
@@ -516,7 +520,7 @@ export default class TestConfiguration {
   }
 
   async roleHeaders({
-    email = generator.email(),
+    email = generator.email({ domain: "example.com" }),
     roleId = roles.BUILTIN_ROLE_IDS.ADMIN,
     builder = false,
     prodApp = true,

packages/server/src/tests/utilities/api/index.ts

@@ -11,6 +11,7 @@ import { BackupAPI } from "./backup"
 import { AttachmentAPI } from "./attachment"
 import { UserAPI } from "./user"
 import { QueryAPI } from "./query"
+import { RoleAPI } from "./role"
 
 export default class API {
   table: TableAPI
@@ -25,6 +26,7 @@ export default class API {
   attachment: AttachmentAPI
   user: UserAPI
   query: QueryAPI
+  roles: RoleAPI
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
@@ -39,5 +41,6 @@ export default class API {
     this.attachment = new AttachmentAPI(config)
     this.user = new UserAPI(config)
     this.query = new QueryAPI(config)
+    this.roles = new RoleAPI(config)
   }
 }

packages/server/src/tests/utilities/api/role.ts

@@ -0,0 +1,41 @@
+import {
+  AccessibleRolesResponse,
+  FetchRolesResponse,
+  FindRoleResponse,
+  SaveRoleRequest,
+  SaveRoleResponse,
+} from "@budibase/types"
+import { Expectations, TestAPI } from "./base"
+
+export class RoleAPI extends TestAPI {
+  fetch = async (expectations?: Expectations) => {
+    return await this._get<FetchRolesResponse>(`/api/roles`, {
+      expectations,
+    })
+  }
+
+  find = async (roleId: string, expectations?: Expectations) => {
+    return await this._get<FindRoleResponse>(`/api/roles/${roleId}`, {
+      expectations,
+    })
+  }
+
+  save = async (body: SaveRoleRequest, expectations?: Expectations) => {
+    return await this._post<SaveRoleResponse>(`/api/roles`, {
+      body,
+      expectations,
+    })
+  }
+
+  destroy = async (roleId: string, expectations?: Expectations) => {
+    return await this._delete(`/api/roles/${roleId}`, {
+      expectations,
+    })
+  }
+
+  accesssible = async (expectations?: Expectations) => {
+    return await this._get<AccessibleRolesResponse>(`/api/roles/accessible`, {
+      expectations,
+    })
+  }
+}

packages/shared-core/src/filters.ts

@@ -390,24 +390,41 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => {
     }
   )
 
-  // Match a document against all criteria
   const docMatch = (doc: any) => {
-    return (
+    const filterFunctions: Record<SearchQueryOperators, (doc: any) => boolean> =
-      stringMatch(doc) &&
+      {
-      fuzzyMatch(doc) &&
+        string: stringMatch,
-      rangeMatch(doc) &&
+        fuzzy: fuzzyMatch,
-      equalMatch(doc) &&
+        range: rangeMatch,
-      notEqualMatch(doc) &&
+        equal: equalMatch,
-      emptyMatch(doc) &&
+        notEqual: notEqualMatch,
-      notEmptyMatch(doc) &&
+        empty: emptyMatch,
-      oneOf(doc) &&
+        notEmpty: notEmptyMatch,
-      contains(doc) &&
+        oneOf: oneOf,
-      containsAny(doc) &&
+        contains: contains,
-      notContains(doc)
+        containsAny: containsAny,
-    )
+        notContains: notContains,
-  }
+      }
 
-  // Process all docs
+    const activeFilterKeys: SearchQueryOperators[] = Object.entries(query || {})
+      .filter(
+        ([key, value]: [string, any]) =>
+          !["allOr", "onEmptyFilter"].includes(key) &&
+          value &&
+          Object.keys(value as Record<string, any>).length > 0
+      )
+      .map(([key]) => key as any)
+
+    const results: boolean[] = activeFilterKeys.map(filterKey => {
+      return filterFunctions[filterKey]?.(doc) ?? false
+    })
+
+    if (query!.allOr) {
+      return results.some(result => result === true)
+    } else {
+      return results.every(result => result === true)
+    }
+  }
   return docs.filter(docMatch)
 }
 

packages/shared-core/src/tests/filters.test.ts

@@ -47,10 +47,7 @@ describe("runLuceneQuery", () => {
     },
   ]
 
-  function buildQuery(
+  function buildQuery(filters: { [filterKey: string]: any }): SearchQuery {
-    filterKey: string,
-    value: { [key: string]: any }
-  ): SearchQuery {
     const query: SearchQuery = {
       string: {},
       fuzzy: {},
@@ -63,8 +60,13 @@ describe("runLuceneQuery", () => {
       notContains: {},
       oneOf: {},
       containsAny: {},
+      allOr: false,
     }
-    query[filterKey as SearchQueryOperators] = value
+
+    for (const filterKey in filters) {
+      query[filterKey as SearchQueryOperators] = filters[filterKey]
+    }
+
     return query
   }
 
@@ -73,16 +75,17 @@ describe("runLuceneQuery", () => {
   })
 
   it("should return matching rows for equal filter", () => {
-    const query = buildQuery("equal", {
+    const query = buildQuery({
-      order_status: 4,
+      equal: { order_status: 4 },
     })
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1, 2])
   })
 
   it("should return matching row for notEqual filter", () => {
-    const query = buildQuery("notEqual", {
+    const query = buildQuery({
-      order_status: 4,
+      notEqual: { order_status: 4 },
     })
+
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3])
   })
 
@@ -90,48 +93,56 @@ describe("runLuceneQuery", () => {
     expect(
       runLuceneQuery(
         docs,
-        buildQuery("fuzzy", {
+        buildQuery({
-          description: "sm",
+          fuzzy: { description: "sm" },
         })
       ).map(row => row.description)
     ).toEqual(["Small box"])
     expect(
       runLuceneQuery(
         docs,
-        buildQuery("string", {
+        buildQuery({
-          description: "SM",
+          string: { description: "SM" },
         })
       ).map(row => row.description)
     ).toEqual(["Small box"])
   })
 
   it("should return rows within a range filter", () => {
-    const query = buildQuery("range", {
+    const query = buildQuery({
-      customer_id: {
+      range: {
-        low: 500,
+        customer_id: {
-        high: 1000,
+          low: 500,
+          high: 1000,
+        },
       },
     })
+
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3])
   })
 
   it("should return rows with numeric strings within a range filter", () => {
-    const query = buildQuery("range", {
+    const query = buildQuery({
-      customer_id: {
+      range: {
-        low: "500",
+        customer_id: {
-        high: "1000",
+          low: "500",
+          high: "1000",
+        },
       },
     })
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3])
   })
 
   it("should return rows with ISO date strings within a range filter", () => {
-    const query = buildQuery("range", {
+    const query = buildQuery({
-      order_date: {
+      range: {
-        low: "2016-01-04T00:00:00.000Z",
+        order_date: {
-        high: "2016-01-11T00:00:00.000Z",
+          low: "2016-01-04T00:00:00.000Z",
+          high: "2016-01-11T00:00:00.000Z",
+        },
       },
     })
+
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2])
   })
 
@@ -150,40 +161,88 @@ describe("runLuceneQuery", () => {
         label: "",
       },
     ]
-    const query = buildQuery("range", {
+
-      order_date: {
+    const query = buildQuery({
-        low: "2016-01-04T00:00:00.000Z",
+      range: {
-        high: "2016-01-11T00:00:00.000Z",
+        order_date: {
+          low: "2016-01-04T00:00:00.000Z",
+          high: "2016-01-11T00:00:00.000Z",
+        },
       },
     })
+
     expect(runLuceneQuery(docs, query)).toEqual(docs)
   })
 
   it("should return rows with matches on empty filter", () => {
-    const query = buildQuery("empty", {
+    const query = buildQuery({
-      label: null,
+      empty: {
+        label: null,
+      },
     })
+
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1])
   })
 
   it("should return rows with matches on notEmpty filter", () => {
-    const query = buildQuery("notEmpty", {
+    const query = buildQuery({
-      label: null,
+      notEmpty: {
+        label: null,
+      },
     })
+
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2, 3])
   })
 
   test.each([[523, 259], "523,259"])(
     "should return rows with matches on numeric oneOf filter",
     input => {
-      let query = buildQuery("oneOf", {
+      const query = buildQuery({
-        customer_id: input,
+        oneOf: {
+          customer_id: input,
+        },
       })
+
       expect(runLuceneQuery(docs, query).map(row => row.customer_id)).toEqual([
         259, 523,
       ])
     }
   )
+
+  test.each([
+    [false, []],
+    [true, [1, 2, 3]],
+  ])("should return %s  if allOr is %s ", (allOr, expectedResult) => {
+    const query = buildQuery({
+      allOr,
+      oneOf: { staff_id: [10] },
+      contains: { description: ["box"] },
+    })
+
+    expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual(
+      expectedResult
+    )
+  })
+
+  it("should return matching results if allOr is true and only one filter matches with different operands", () => {
+    const query = buildQuery({
+      allOr: true,
+      equal: { order_status: 4 },
+      oneOf: { label: ["FRAGILE"] },
+    })
+
+    expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1, 2])
+  })
+
+  it("should handle when a value is null or undefined", () => {
+    const query = buildQuery({
+      allOr: true,
+      equal: { order_status: null },
+      oneOf: { label: ["FRAGILE"] },
+    })
+
+    expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2])
+  })
 })
 
 describe("buildLuceneQuery", () => {

packages/types/package.json

@@ -18,7 +18,7 @@
     "@budibase/nano": "10.1.5",
     "@types/koa": "2.13.4",
     "@types/pouchdb": "6.4.0",
-    "@types/redlock": "4.0.3",
+    "@types/redlock": "4.0.7",
     "rimraf": "3.0.2",
     "typescript": "5.2.2"
   },

packages/types/src/api/web/auth.ts

@@ -18,6 +18,7 @@ export interface UpdateSelfRequest {
   password?: string
   forceResetPassword?: boolean
   onboardedAt?: string
+  tours?: Record<string, Date>
 }
 
 export interface UpdateSelfResponse {

packages/types/src/documents/app/role.ts

@@ -5,4 +5,5 @@ export interface Role extends Document {
   inherits?: string
   permissions: { [key: string]: string[] }
   version?: string
+  name: string
 }

packages/types/src/documents/document.ts

@@ -38,6 +38,7 @@ export enum DocumentType {
   AUTOMATION_METADATA = "meta_au",
   AUDIT_LOG = "al",
   APP_MIGRATION_METADATA = "_design/migrations",
+  SCIM_LOG = "scimlog",
 }
 
 // these are the core documents that make up the data, design

packages/types/src/documents/global/user.ts

@@ -55,6 +55,7 @@ export interface User extends Document {
   dayPassRecordedAt?: string
   userGroups?: string[]
   onboardedAt?: string
+  tours?: Record<string, Date>
   scimInfo?: { isSync: true } & Record<string, any>
   ssoId?: string
 }

packages/types/src/documents/global/userGroup.ts

@@ -24,7 +24,7 @@ export interface GroupUser {
 }
 
 export interface UserGroupRoles {
-  [key: string]: string
+  [key: string]: string | undefined
 }
 
 export interface SearchGroupRequest {}

packages/types/src/sdk/db.ts

@@ -128,6 +128,7 @@ export interface Database {
 
   exists(): Promise<boolean>
   get<T extends Document>(id?: string): Promise<T>
+  exists(docId: string): Promise<boolean>
   getMultiple<T extends Document>(
     ids: string[],
     opts?: { allowMissing?: boolean }

packages/types/src/sdk/events/userGroup.ts

@@ -48,7 +48,7 @@ export interface GroupAddedOnboardingEvent extends BaseEvent {
 }
 
 export interface GroupPermissionsEditedEvent extends BaseEvent {
-  permissions: Record<string, string>
+  permissions: Record<string, string | undefined>
   groupId: string
   audited: {
     name: string

packages/worker/src/api/routes/global/tests/groups.spec.ts

@@ -147,7 +147,7 @@ describe("/api/global/groups", () => {
 
         await Promise.all(
           Array.from({ length: 30 }).map(async (_, i) => {
-            const email = `user${i}@${generator.domain()}`
+            const email = `user${i}@example.com`
             const user = await config.api.users.saveUser({
               ...structures.users.user(),
               email,

packages/worker/src/api/routes/validation/users.ts

@@ -26,6 +26,7 @@ export const buildSelfSaveValidation = () => {
     firstName: OPTIONAL_STRING,
     lastName: OPTIONAL_STRING,
     onboardedAt: Joi.string().optional(),
+    tours: Joi.object().optional(),
   }
   return auth.joiValidator.body(Joi.object(schema).required().unknown(false))
 }

packages/worker/src/initPro.ts

@@ -1,5 +1,4 @@
 import { sdk as proSdk } from "@budibase/pro"
-import * as userSdk from "./sdk/users"
 
 export const initPro = async () => {
   await proSdk.init({})

qa-core/src/account-api/tests/accounts/accounts.cloud.spec.ts

@@ -84,7 +84,7 @@ describe("Accounts", () => {
     })
 
     it("searches by email", async () => {
-      const email = generator.email()
+      const email = generator.email({ domain: "example.com" })
 
       // Empty result
       const [_, emptyBody] = await config.api.accounts.search(email, "email")