Merge branch 'master' into BUDI-8064/doc-writethrough

.eslintrc.json

@@ -44,7 +44,8 @@
         "no-undef": "off",
         "no-prototype-builtins": "off",
         "local-rules/no-budibase-imports": "error",
-        "local-rules/no-test-com": "error"
+        "local-rules/no-test-com": "error",
+        "local-rules/email-domain-example-com": "error"
       }
     },
     {

eslint-local-rules/index.js

@@ -51,4 +51,41 @@ module.exports = {
       }
     },
   },
+  "email-domain-example-com": {
+    meta: {
+      type: "problem",
+      docs: {
+        description:
+          "enforce using the example.com domain for generator.email calls",
+        category: "Possible Errors",
+        recommended: false,
+      },
+      fixable: "code",
+      schema: [],
+    },
+    create: function (context) {
+      return {
+        CallExpression(node) {
+          if (
+            node.callee.type === "MemberExpression" &&
+            node.callee.object.name === "generator" &&
+            node.callee.property.name === "email" &&
+            node.arguments.length === 0
+          ) {
+            context.report({
+              node,
+              message:
+                "Prefer using generator.email with the domain \"{ domain: 'example.com' }\".",
+              fix: function (fixer) {
+                return fixer.replaceText(
+                  node,
+                  'generator.email({ domain: "example.com" })'
+                )
+              },
+            })
+          }
+        },
+      }
+    },
+  },
 }

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.21.2",
+  "version": "2.21.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/cache/user.ts

@@ -6,7 +6,7 @@ import env from "../environment"
 import * as accounts from "../accounts"
 import { UserDB } from "../users"
 import { sdk } from "@budibase/shared-core"
-import { User } from "@budibase/types"
+import { User, UserMetadata } from "@budibase/types"
 
 const EXPIRY_SECONDS = 3600
 
@@ -15,7 +15,7 @@ const EXPIRY_SECONDS = 3600
  */
 async function populateFromDB(userId: string, tenantId: string) {
   const db = tenancy.getTenantDB(tenantId)
-  const user = await db.get<any>(userId)
+  const user = await db.get<UserMetadata>(userId)
   user.budibaseAccess = true
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(user.email)

packages/backend-core/src/db/Replication.ts

@@ -1,66 +1,57 @@
+import PouchDB from "pouchdb"
 import { getPouchDB, closePouchDB } from "./couch"
 import { DocumentType } from "../constants"
 
 class Replication {
-  source: any
-  target: any
-  replication: any
+  source: PouchDB.Database
+  target: PouchDB.Database
 
-  /**
-   *
-   * @param source - the DB you want to replicate or rollback to
-   * @param target - the DB you want to replicate to, or rollback from
-   */
-  constructor({ source, target }: any) {
+  constructor({ source, target }: { source: string; target: string }) {
     this.source = getPouchDB(source)
     this.target = getPouchDB(target)
   }
 
-  close() {
-    return Promise.all([closePouchDB(this.source), closePouchDB(this.target)])
+  async close() {
+    await Promise.all([closePouchDB(this.source), closePouchDB(this.target)])
   }
 
-  promisify(operation: any, opts = {}) {
-    return new Promise(resolve => {
-      operation(this.target, opts)
-        .on("denied", function (err: any) {
+  replicate(opts: PouchDB.Replication.ReplicateOptions = {}) {
+    return new Promise<PouchDB.Replication.ReplicationResult<{}>>(resolve => {
+      this.source.replicate
+        .to(this.target, opts)
+        .on("denied", function (err) {
           // a document failed to replicate (e.g. due to permissions)
           throw new Error(`Denied: Document failed to replicate ${err}`)
         })
-        .on("complete", function (info: any) {
+        .on("complete", function (info) {
           return resolve(info)
         })
-        .on("error", function (err: any) {
+        .on("error", function (err) {
           throw new Error(`Replication Error: ${err}`)
         })
     })
   }
 
-  /**
-   * Two way replication operation, intended to be promise based.
-   * @param opts - PouchDB replication options
-   */
-  sync(opts = {}) {
-    this.replication = this.promisify(this.source.sync, opts)
-    return this.replication
-  }
+  appReplicateOpts(
+    opts: PouchDB.Replication.ReplicateOptions = {}
+  ): PouchDB.Replication.ReplicateOptions {
+    if (typeof opts.filter === "string") {
+      return opts
+    }
 
-  /**
-   * One way replication operation, intended to be promise based.
-   * @param opts - PouchDB replication options
-   */
-  replicate(opts = {}) {
-    this.replication = this.promisify(this.source.replicate.to, opts)
-    return this.replication
-  }
+    const filter = opts.filter
+    delete opts.filter
 
-  appReplicateOpts() {
     return {
-      filter: (doc: any) => {
+      ...opts,
+      filter: (doc: any, params: any) => {
         if (doc._id && doc._id.startsWith(DocumentType.AUTOMATION_LOG)) {
           return false
         }
-        return doc._id !== DocumentType.APP_METADATA
+        if (doc._id === DocumentType.APP_METADATA) {
+          return false
+        }
+        return filter ? filter(doc, params) : true
       },
     }
   }
@@ -75,10 +66,6 @@ class Replication {
     // take the opportunity to remove deleted tombstones
     await this.replicate()
   }
-
-  cancel() {
-    this.replication.cancel()
-  }
 }
 
 export default Replication

packages/backend-core/src/redis/redis.ts

@@ -291,23 +291,16 @@ class RedisWrapper {
       return acc
     }, {} as Record<string, any>)
 
-    const luaScript = `
-        for i, key in ipairs(KEYS) do
-            redis.call('MSET', key, ARGV[i])
-            ${
-              expirySeconds !== null
-                ? `redis.call('EXPIRE', key, ARGV[#ARGV])`
-                : ""
-            }
-        end
-        `
-    const keys = Object.keys(dataToStore)
-    const values = Object.values(dataToStore)
+    const pipeline = client.pipeline()
+    pipeline.mset(dataToStore)
+
     if (expirySeconds !== null) {
-      values.push(expirySeconds)
+      for (const key of Object.keys(dataToStore)) {
+        pipeline.expire(key, expirySeconds)
+      }
     }
 
-    await client.eval(luaScript, keys.length, ...keys, ...values)
+    await pipeline.exec()
   }
 
   async getTTL(key: string) {
@@ -340,7 +333,7 @@ class RedisWrapper {
   async increment(key: string) {
     const result = await this.getClient().incr(addDbPrefix(this._db, key))
     if (isNaN(result)) {
-      throw new Error(`Redis ${key} does not contains a number`)
+      throw new Error(`Redis ${key} does not contain a number`)
     }
     return result
   }

packages/backend-core/src/redis/tests/redis.spec.ts

@@ -3,6 +3,8 @@ import { generator, structures } from "../../../tests"
 import RedisWrapper from "../redis"
 import { env } from "../.."
 
+jest.setTimeout(30000)
+
 describe("redis", () => {
   let redis: RedisWrapper
   let container: StartedTestContainer

packages/backend-core/src/security/roles.ts

@@ -101,10 +101,7 @@ export function getBuiltinRole(roleId: string): Role | undefined {
 /**
  * Works through the inheritance ranks to see how far up the builtin stack this ID is.
  */
-export function builtinRoleToNumber(id?: string) {
-  if (!id) {
-    return 0
-  }
+export function builtinRoleToNumber(id: string) {
   const builtins = getBuiltinRoles()
   const MAX = Object.values(builtins).length + 1
   if (id === BUILTIN_IDS.ADMIN || id === BUILTIN_IDS.BUILDER) {

packages/backend-core/tests/core/utilities/structures/accounts.ts

@@ -18,7 +18,7 @@ export const account = (partial: Partial<Account> = {}): Account => {
   return {
     accountId: uuid(),
     tenantId: generator.word(),
-    email: generator.email(),
+    email: generator.email({ domain: "example.com" }),
     tenantName: generator.word(),
     hosting: Hosting.SELF,
     createdAt: Date.now(),

packages/backend-core/tests/core/utilities/structures/scim.ts

@@ -13,7 +13,7 @@ interface CreateUserRequestFields {
 export function createUserRequest(userData?: Partial<CreateUserRequestFields>) {
   const defaultValues = {
     externalId: uuid(),
-    email: generator.email(),
+    email: `${uuid()}@example.com`,
     firstName: generator.first(),
     lastName: generator.last(),
     username: generator.name(),

packages/bbui/src/Actions/position_dropdown.js

@@ -35,7 +35,10 @@ export default function positionDropdown(element, opts) {
     }
 
     if (typeof customUpdate === "function") {
-      styles = customUpdate(anchorBounds, elementBounds, styles)
+      styles = customUpdate(anchorBounds, elementBounds, {
+        ...styles,
+        offset: opts.offset,
+      })
     } else {
       // Determine vertical styles
       if (align === "right-outside") {

packages/builder/src/analytics/index.js

@@ -9,13 +9,17 @@ const intercom = new IntercomClient(process.env.INTERCOM_TOKEN)
 class AnalyticsHub {
   constructor() {
     this.clients = [posthog, intercom]
+    this.initialised = false
   }
 
   async activate() {
     // Check analytics are enabled
     const analyticsStatus = await API.getAnalyticsStatus()
-    if (analyticsStatus.enabled) {
-      this.clients.forEach(client => client.init())
+    if (analyticsStatus.enabled && !this.initialised) {
+      this.clients.forEach(client => {
+        client.init()
+      })
+      this.initialised = true
     }
   }
 

packages/builder/src/components/deploy/AppActions.svelte

@@ -164,9 +164,10 @@
       </div>
     {/if}
     <TourWrap
-      tourStepKey={$builderStore.onboarding
-        ? TOUR_STEP_KEYS.BUILDER_USER_MANAGEMENT
-        : TOUR_STEP_KEYS.FEATURE_USER_MANAGEMENT}
+      stepKeys={[
+        TOUR_STEP_KEYS.BUILDER_USER_MANAGEMENT,
+        TOUR_STEP_KEYS.FEATURE_USER_MANAGEMENT,
+      ]}
     >
       <div class="app-action-button users">
         <div class="app-action" id="builder-app-users-button">
@@ -209,7 +210,7 @@
       <div bind:this={appActionPopoverAnchor}>
         <div class="app-action">
           <Icon name={isPublished ? "GlobeCheck" : "GlobeStrike"} />
-          <TourWrap tourStepKey={TOUR_STEP_KEYS.BUILDER_APP_PUBLISH}>
+          <TourWrap stepKeys={[TOUR_STEP_KEYS.BUILDER_APP_PUBLISH]}>
             <span class="publish-open" id="builder-app-publish-button">
               Publish
               <Icon

packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte

@@ -1,5 +1,5 @@
 <script>
-  import EditComponentPopover from "../EditComponentPopover.svelte"
+  import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte"
   import { Icon } from "@budibase/bbui"
   import { runtimeToReadableBinding } from "dataBinding"
   import { isJSBinding } from "@budibase/string-templates"

packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte

@@ -3,6 +3,7 @@
   import { componentStore } from "stores/builder"
   import { cloneDeep } from "lodash/fp"
   import { createEventDispatcher, getContext } from "svelte"
+  import { customPositionHandler } from "."
   import ComponentSettingsSection from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte"
 
   export let anchor
@@ -54,25 +55,6 @@
 
     dispatch("change", nestedComponentInstance)
   }
-
-  const customPositionHandler = (anchorBounds, eleBounds, cfg) => {
-    let { left, top } = cfg
-    let percentageOffset = 30
-    // left-outside
-    left = anchorBounds.left - eleBounds.width - 18
-
-    // shift up from the anchor, if space allows
-    let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset
-    let defaultTop = anchorBounds.top - offsetPos
-
-    if (window.innerHeight - defaultTop < eleBounds.height) {
-      top = window.innerHeight - eleBounds.height - 5
-    } else {
-      top = anchorBounds.top - offsetPos
-    }
-
-    return { ...cfg, left, top }
-  }
 </script>
 
 <Icon
@@ -104,6 +86,7 @@
   showPopover={drawers.length === 0}
   clickOutsideOverride={drawers.length > 0}
   maxHeight={600}
+  offset={18}
   handlePostionUpdate={customPositionHandler}
 >
   <span class="popover-wrap">

packages/builder/src/components/design/settings/controls/EditComponentPopover/index.js

@@ -0,0 +1,18 @@
+export const customPositionHandler = (anchorBounds, eleBounds, cfg) => {
+  let { left, top, offset } = cfg
+  let percentageOffset = 30
+  // left-outside
+  left = anchorBounds.left - eleBounds.width - (offset || 5)
+
+  // shift up from the anchor, if space allows
+  let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset
+  let defaultTop = anchorBounds.top - offsetPos
+
+  if (window.innerHeight - defaultTop < eleBounds.height) {
+    top = window.innerHeight - eleBounds.height - 5
+  } else {
+    top = anchorBounds.top - offsetPos
+  }
+
+  return { ...cfg, left, top }
+}

packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte

@@ -1,5 +1,5 @@
 <script>
-  import EditComponentPopover from "../EditComponentPopover.svelte"
+  import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte"
   import { Toggle, Icon } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { cloneDeep } from "lodash/fp"

packages/builder/src/components/design/settings/controls/FormStepConfiguration.svelte

@@ -52,8 +52,8 @@
     _id: Helpers.uuid(),
     _component: componentType,
     _instanceName: `Step ${currentStep + 1}`,
-    title: stepSettings.title ?? defaults.title,
-    buttons: stepSettings.buttons || defaults.buttons,
+    title: stepSettings.title ?? defaults?.title,
+    buttons: stepSettings.buttons || defaults?.buttons,
     fields: stepSettings.fields,
     desc: stepSettings.desc,
 

packages/builder/src/components/design/settings/controls/GridColumnConfiguration/FieldSetting.svelte

@@ -1,5 +1,5 @@
 <script>
-  import EditComponentPopover from "../EditComponentPopover.svelte"
+  import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte"
   import { Toggle, Icon } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { cloneDeep } from "lodash/fp"

packages/builder/src/components/design/settings/controls/GridColumnConfiguration/PrimaryColumnFieldSetting.svelte

@@ -1,5 +1,5 @@
 <script>
-  import EditComponentPopover from "../EditComponentPopover.svelte"
+  import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte"
   import { Icon } from "@budibase/bbui"
   import { setContext } from "svelte"
   import { writable } from "svelte/store"

packages/builder/src/components/design/settings/controls/IconSelect/IconSelect.svelte

@@ -139,10 +139,22 @@
         {/each}
       </div>
       <div class="search-input">
-        <div class="input-wrapper">
-          <Input bind:value={searchTerm} thin placeholder="Search Icon" />
+        <div class="input-wrapper" style={`width: ${value ? "425" : "510"}px`}>
+          <Input
+            bind:value={searchTerm}
+            on:keyup={event => {
+              if (event.key === "Enter") {
+                searchForIcon()
+              }
+            }}
+            thin
+            placeholder="Search Icon"
+          />
         </div>
         <Button secondary on:click={searchForIcon}>Search</Button>
+        {#if value}
+          <Button primary on:click={() => (value = null)}>Clear</Button>
+        {/if}
       </div>
       <div class="page-area">
         <div class="pager">
@@ -239,6 +251,7 @@
     flex-flow: row nowrap;
     width: 100%;
     padding-right: 15px;
+    gap: 10px;
   }
   .input-wrapper {
     width: 510px;

packages/builder/src/components/design/settings/controls/PropertyControl.svelte

@@ -20,17 +20,23 @@
   export let bindings = []
   export let componentBindings = []
   export let nested = false
-  export let highlighted = false
   export let propertyFocus = false
   export let info = null
   export let disableBindings = false
   export let wide
 
-  $: nullishValue = value == null || value === ""
+  let highlightType
+
+  $: highlightedProp = $builderStore.highlightedSetting
   $: allBindings = getAllBindings(bindings, componentBindings, nested)
   $: safeValue = getSafeValue(value, defaultValue, allBindings)
   $: replaceBindings = val => readableToRuntimeBinding(allBindings, val)
 
+  $: if (!Array.isArray(value)) {
+    highlightType =
+      highlightedProp?.key === key ? `highlighted-${highlightedProp?.type}` : ""
+  }
+
   const getAllBindings = (bindings, componentBindings, nested) => {
     if (!nested) {
       return bindings
@@ -71,16 +77,17 @@
   }
 
   onDestroy(() => {
-    if (highlighted) {
+    if (highlightedProp) {
       builderStore.highlightSetting(null)
     }
   })
 </script>
 
 <div
-  class="property-control"
+  id={`${key}-prop-control-wrap`}
+  class={`property-control ${highlightType}`}
   class:wide={!label || labelHidden || wide === true}
-  class:highlighted={highlighted && nullishValue}
+  class:highlighted={highlightType}
   class:property-focus={propertyFocus}
 >
   {#if label && !labelHidden}
@@ -115,6 +122,16 @@
 </div>
 
 <style>
+  .property-control.highlighted.highlighted-info {
+    border-color: var(--spectrum-semantic-informative-color-background);
+  }
+  .property-control.highlighted.highlighted-error {
+    border-color: var(--spectrum-global-color-static-red-600);
+  }
+  .property-control.highlighted.highlighted-warning {
+    border-color: var(--spectrum-global-color-static-orange-700);
+  }
+
   .property-control {
     position: relative;
     display: grid;
@@ -132,6 +149,10 @@
   .property-control.highlighted {
     background: var(--spectrum-global-color-gray-300);
     border-color: var(--spectrum-global-color-static-red-600);
+    margin-top: -3.5px;
+    margin-bottom: -3.5px;
+    padding-bottom: 3.5px;
+    padding-top: 3.5px;
   }
 
   .property-control.property-focus :global(input) {

packages/builder/src/components/portal/onboarding/TourPopover.svelte

@@ -1,6 +1,6 @@
 <script>
   import { Popover, Layout, Heading, Body, Button, Link } from "@budibase/bbui"
-  import { TOURS } from "./tours.js"
+  import { TOURS, getCurrentStepIdx } from "./tours.js"
   import { goto, layout, isActive } from "@roxi/routify"
   import { builderStore } from "stores/builder"
 
@@ -20,6 +20,13 @@
 
   const updateTourStep = (targetStepKey, tourKey) => {
     if (!tourKey) {
+      tourSteps = null
+      tourStepIdx = null
+      lastStep = null
+      tourStep = null
+      popoverAnchor = null
+      popover = null
+      skipping = false
       return
     }
     if (!tourSteps?.length) {
@@ -78,16 +85,6 @@
       }
     }
   }
-
-  const getCurrentStepIdx = (steps, tourStepKey) => {
-    if (!steps?.length) {
-      return
-    }
-    if (steps?.length && !tourStepKey) {
-      return 0
-    }
-    return steps.findIndex(step => step.id === tourStepKey)
-  }
 </script>
 
 {#if tourKey}
@@ -98,7 +95,9 @@
       anchor={popoverAnchor}
       maxWidth={300}
       dismissible={false}
-      offset={15}
+      offset={12}
+      handlePostionUpdate={tourStep?.positionHandler}
+      customZindex={3}
     >
       <div class="tour-content">
         <Layout noPadding gap="M">
@@ -119,7 +118,7 @@
           </Body>
           <div class="tour-footer">
             <div class="tour-navigation">
-              {#if typeof tourOnSkip === "function"}
+              {#if typeof tourOnSkip === "function" && !lastStep}
                 <Link
                   secondary
                   quiet

packages/builder/src/components/portal/onboarding/TourWrap.svelte

@@ -1,39 +1,63 @@
 <script>
   import { tourHandler } from "./tourHandler"
-  import { TOURS } from "./tours"
+  import { TOURSBYSTEP, TOURS, getCurrentStepIdx } from "./tours"
   import { onMount, onDestroy } from "svelte"
   import { builderStore } from "stores/builder"
 
-  export let tourStepKey
+  export let stepKeys = []
 
-  let currentTourStep
   let ready = false
-  let registered = false
-  let handler
+  let registered = {}
 
   const registerTourNode = (tourKey, stepKey) => {
-    if (ready && !registered && tourKey) {
-      currentTourStep = TOURS[tourKey].steps.find(step => step.id === stepKey)
-      if (!currentTourStep) {
-        return
+    const step = TOURSBYSTEP[stepKey]
+    if (ready && step && !registered[stepKey] && step?.tour === tourKey) {
+      const elem = document.querySelector(step.query)
+      registered[stepKey] = tourHandler(elem, stepKey)
+    }
+  }
+
+  const scrollToStep = () => {
+    let tourStepIdx = getCurrentStepIdx(
+      TOURS[tourKeyWatch]?.steps,
+      tourStepKeyWatch
+    )
+    let currentStep = TOURS[tourKeyWatch]?.steps?.[tourStepIdx]
+    if (currentStep?.scrollIntoView) {
+      let currentNode = $builderStore.tourNodes?.[currentStep.id]
+      if (currentNode) {
+        currentNode.scrollIntoView({ behavior: "smooth", block: "center" })
       }
-      const elem = document.querySelector(currentTourStep.query)
-      handler = tourHandler(elem, stepKey)
-      registered = true
     }
   }
 
   $: tourKeyWatch = $builderStore.tourKey
-  $: registerTourNode(tourKeyWatch, tourStepKey, ready)
+  $: tourStepKeyWatch = $builderStore.tourStepKey
+  $: if (tourKeyWatch || stepKeys || ready) {
+    stepKeys.forEach(tourStepKey => {
+      registerTourNode(tourKeyWatch, tourStepKey)
+    })
+  }
+  $: scrollToStep(tourKeyWatch, tourStepKeyWatch)
 
   onMount(() => {
     ready = true
   })
 
   onDestroy(() => {
-    if (handler) {
+    Object.entries(registered).forEach(entry => {
+      const handler = entry[1]
+      const stepKey = entry[0]
+      // Run step destroy, de-register nodes in the builderStore and local cache
       handler.destroy()
-    }
+      delete registered[stepKey]
+
+      // Check if the step is part of an active tour. End the tour if that is the case
+      const step = TOURSBYSTEP[stepKey]
+      if (step.tour === tourKeyWatch) {
+        builderStore.setTour()
+      }
+    })
   })
 </script>
 

packages/builder/src/components/portal/onboarding/steps/NewFormSteps.svelte

@@ -0,0 +1,9 @@
+<div>
+  When faced with a sizable form, consider implementing a multi-step approach to
+  enhance user experience.
+
+  <p>
+    Breaking the form into multiple steps can significantly improve usability by
+    making the process more digestible for your users.
+  </p>
+</div>

packages/builder/src/components/portal/onboarding/steps/NewViewUpdateFormRowId.svelte

@@ -0,0 +1,17 @@
+<div>
+  You can use bindings to set the Row ID on your form.
+  <p>
+    This will allow you to pull the correct information into your form and allow
+    you to update!
+  </p>
+  <a href="https://docs.budibase.com/docs/form-block" target="_blank">
+    How to pass a row ID using bindings
+  </a>
+</div>
+
+<style>
+  a {
+    color: inherit;
+    text-decoration: underline;
+  }
+</style>

packages/builder/src/components/portal/onboarding/steps/index.js

@@ -1,3 +1,5 @@
 export { default as OnboardingData } from "./OnboardingData.svelte"
 export { default as OnboardingDesign } from "./OnboardingDesign.svelte"
 export { default as OnboardingPublish } from "./OnboardingPublish.svelte"
+export { default as NewViewUpdateFormRowId } from "./NewViewUpdateFormRowId.svelte"
+export { default as NewFormSteps } from "./NewFormSteps.svelte"

packages/builder/src/components/portal/onboarding/tours.js

@@ -2,8 +2,15 @@ import { get } from "svelte/store"
 import { builderStore } from "stores/builder"
 import { auth } from "stores/portal"
 import analytics from "analytics"
-import { OnboardingData, OnboardingDesign, OnboardingPublish } from "./steps"
+import {
+  OnboardingData,
+  OnboardingDesign,
+  OnboardingPublish,
+  NewViewUpdateFormRowId,
+  NewFormSteps,
+} from "./steps"
 import { API } from "api"
+import { customPositionHandler } from "components/design/settings/controls/EditComponentPopover"
 
 const ONBOARDING_EVENT_PREFIX = "onboarding"
 
@@ -14,11 +21,26 @@ export const TOUR_STEP_KEYS = {
   BUILDER_USER_MANAGEMENT: "builder-user-management",
   BUILDER_AUTOMATION_SECTION: "builder-automation-section",
   FEATURE_USER_MANAGEMENT: "feature-user-management",
+  BUILDER_FORM_CREATE_STEPS: "builder-form-create-steps",
+  BUILDER_FORM_VIEW_UPDATE_STEPS: "builder-form-view-update-steps",
+  BUILDER_FORM_ROW_ID: "builder-form-row-id",
 }
 
 export const TOUR_KEYS = {
   TOUR_BUILDER_ONBOARDING: "builder-onboarding",
   FEATURE_ONBOARDING: "feature-onboarding",
+  BUILDER_FORM_CREATE: "builder-form-create",
+  BUILDER_FORM_VIEW_UPDATE: "builder-form-view-update",
+}
+
+export const getCurrentStepIdx = (steps, tourStepKey) => {
+  if (!steps?.length) {
+    return
+  }
+  if (steps?.length && !tourStepKey) {
+    return 0
+  }
+  return steps.findIndex(step => step.id === tourStepKey)
 }
 
 const endUserOnboarding = async ({ skipped = false } = {}) => {
@@ -37,13 +59,8 @@ const endUserOnboarding = async ({ skipped = false } = {}) => {
       // Update the cached user
       await auth.getSelf()
 
-      builderStore.update(state => ({
-        ...state,
-        tourNodes: null,
-        tourKey: null,
-        tourStepKey: null,
-        onboarding: false,
-      }))
+      builderStore.endBuilderOnboarding()
+      builderStore.setTour()
     } catch (e) {
       console.error("Onboarding failed", e)
       return false
@@ -52,9 +69,29 @@ const endUserOnboarding = async ({ skipped = false } = {}) => {
   }
 }
 
-const tourEvent = eventKey => {
+const endTour = async ({ key, skipped = false } = {}) => {
+  const { tours = {} } = get(auth).user
+  tours[key] = new Date().toISOString()
+
+  await API.updateSelf({
+    tours,
+  })
+
+  if (skipped) {
+    tourEvent(key, skipped)
+  }
+
+  // Update the cached user
+  await auth.getSelf()
+
+  // Reset tour state
+  builderStore.setTour()
+}
+
+const tourEvent = (eventKey, skipped) => {
   analytics.captureEvent(`${ONBOARDING_EVENT_PREFIX}:${eventKey}`, {
     eventSource: EventSource.PORTAL,
+    skipped,
   })
 }
 
@@ -135,7 +172,71 @@ const getTours = () => {
         },
       ],
     },
+    [TOUR_KEYS.BUILDER_FORM_CREATE]: {
+      steps: [
+        {
+          id: TOUR_STEP_KEYS.BUILDER_FORM_CREATE_STEPS,
+          title: "Add multiple steps",
+          layout: NewFormSteps,
+          query: "#steps-prop-control-wrap",
+          onComplete: () => {
+            builderStore.highlightSetting()
+            endTour({ key: TOUR_KEYS.BUILDER_FORM_CREATE })
+          },
+          onLoad: () => {
+            tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_CREATE_STEPS)
+            builderStore.highlightSetting("steps", "info")
+          },
+          positionHandler: customPositionHandler,
+          align: "left-outside",
+        },
+      ],
+    },
+    [TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE]: {
+      steps: [
+        {
+          id: TOUR_STEP_KEYS.BUILDER_FORM_ROW_ID,
+          title: "Add row ID to update a row",
+          layout: NewViewUpdateFormRowId,
+          query: "#rowId-prop-control-wrap",
+          onLoad: () => {
+            tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_ROW_ID)
+            builderStore.highlightSetting("rowId", "info")
+          },
+          positionHandler: customPositionHandler,
+          align: "left-outside",
+        },
+        {
+          id: TOUR_STEP_KEYS.BUILDER_FORM_VIEW_UPDATE_STEPS,
+          title: "Add multiple steps",
+          layout: NewFormSteps,
+          query: "#steps-prop-control-wrap",
+          onComplete: () => {
+            builderStore.highlightSetting()
+            endTour({ key: TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE })
+          },
+          onLoad: () => {
+            tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_VIEW_UPDATE_STEPS)
+            builderStore.highlightSetting("steps", "info")
+          },
+          positionHandler: customPositionHandler,
+          align: "left-outside",
+          scrollIntoView: true,
+        },
+      ],
+      onSkip: async () => {
+        builderStore.highlightSetting()
+        endTour({ key: TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE, skipped: true })
+      },
+    },
   }
 }
 
 export const TOURS = getTours()
+export const TOURSBYSTEP = Object.keys(TOURS).reduce((acc, tour) => {
+  TOURS[tour].steps.forEach(element => {
+    acc[element.id] = element
+    acc[element.id]["tour"] = tour
+  })
+  return acc
+}, {})

packages/builder/src/dataBinding.js

@@ -1146,7 +1146,7 @@ export const getAllStateVariables = () => {
         "@budibase/standard-components/multistepformblockstep"
       )
 
-      steps.forEach(step => {
+      steps?.forEach(step => {
         parseComponentSettings(stepDefinition, step)
       })
     })

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -96,7 +96,7 @@
         const release_date = new Date("2023-03-01T00:00:00.000Z")
         const onboarded = new Date($auth.user?.onboardedAt)
         if (onboarded < release_date) {
-          builderStore.startTour(TOUR_KEYS.FEATURE_ONBOARDING)
+          builderStore.setTour(TOUR_KEYS.FEATURE_ONBOARDING)
         }
       }
     }
@@ -144,7 +144,7 @@
         </span>
         <Tabs {selected} size="M">
           {#each $layout.children as { path, title }}
-            <TourWrap tourStepKey={`builder-${title}-section`}>
+            <TourWrap stepKeys={[`builder-${title}-section`]}>
               <Tab
                 quiet
                 selected={$isActive(path)}

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte

@@ -16,6 +16,14 @@
   } from "dataBinding"
   import { ActionButton, notifications } from "@budibase/bbui"
   import { capitalise } from "helpers"
+  import TourWrap from "components/portal/onboarding/TourWrap.svelte"
+  import { TOUR_STEP_KEYS } from "components/portal/onboarding/tours.js"
+
+  const {
+    BUILDER_FORM_CREATE_STEPS,
+    BUILDER_FORM_VIEW_UPDATE_STEPS,
+    BUILDER_FORM_ROW_ID,
+  } = TOUR_STEP_KEYS
 
   const onUpdateName = async value => {
     try {
@@ -46,7 +54,6 @@
 
   $: id = $selectedComponent?._id
   $: id, (section = tabs[0])
-
   $: componentName = getComponentName(componentInstance)
 </script>
 
@@ -92,13 +99,21 @@
         </div>
       </span>
       {#if section == "settings"}
-        <ComponentSettingsSection
-          {componentInstance}
-          {componentDefinition}
-          {bindings}
-          {componentBindings}
-          {isScreen}
-        />
+        <TourWrap
+          stepKeys={[
+            BUILDER_FORM_CREATE_STEPS,
+            BUILDER_FORM_VIEW_UPDATE_STEPS,
+            BUILDER_FORM_ROW_ID,
+          ]}
+        >
+          <ComponentSettingsSection
+            {componentInstance}
+            {componentDefinition}
+            {bindings}
+            {componentBindings}
+            {isScreen}
+          />
+        </TourWrap>
       {/if}
       {#if section == "styles"}
         <DesignSection

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte

@@ -1,7 +1,7 @@
 <script>
   import { helpers } from "@budibase/shared-core"
   import { DetailSummary, notifications } from "@budibase/bbui"
-  import { componentStore } from "stores/builder"
+  import { componentStore, builderStore } from "stores/builder"
   import PropertyControl from "components/design/settings/controls/PropertyControl.svelte"
   import ResetFieldsButton from "components/design/settings/controls/ResetFieldsButton.svelte"
   import EjectBlockButton from "components/design/settings/controls/EjectBlockButton.svelte"
@@ -177,9 +177,7 @@
               defaultValue={setting.defaultValue}
               nested={setting.nested}
               onChange={val => updateSetting(setting, val)}
-              highlighted={$componentStore.highlightedSettingKey ===
-                setting.key}
-              propertyFocus={$componentStore.propertyFocus === setting.key}
+              propertyFocus={$builderStore.propertyFocus === setting.key}
               info={setting.info}
               disableBindings={setting.disableBindings}
               props={{

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte

@@ -174,7 +174,7 @@
     } else if (type === "request-add-component") {
       toggleAddComponent()
     } else if (type === "highlight-setting") {
-      builderStore.highlightSetting(data.setting)
+      builderStore.highlightSetting(data.setting, "error")
 
       // Also scroll setting into view
       const selector = `#${data.setting}-prop-control`

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte

@@ -3,13 +3,23 @@
   import DatasourceModal from "./DatasourceModal.svelte"
   import ScreenRoleModal from "./ScreenRoleModal.svelte"
   import sanitizeUrl from "helpers/sanitizeUrl"
+  import FormTypeModal from "./FormTypeModal.svelte"
   import { Modal, notifications } from "@budibase/bbui"
-  import { screenStore, navigationStore, tables } from "stores/builder"
+  import {
+    screenStore,
+    navigationStore,
+    tables,
+    builderStore,
+  } from "stores/builder"
+  import { auth } from "stores/portal"
   import { get } from "svelte/store"
   import getTemplates from "templates"
   import { Roles } from "constants/backend"
   import { capitalise } from "helpers"
   import { goto } from "@roxi/routify"
+  import { TOUR_KEYS } from "components/portal/onboarding/tours.js"
+  import formScreen from "templates/formScreen"
+  import rowListScreen from "templates/rowListScreen"
 
   let mode
   let pendingScreen
@@ -18,12 +28,18 @@
   let screenDetailsModal
   let datasourceModal
   let screenAccessRoleModal
+  let formTypeModal
 
   // Cache variables for workflow
   let screenAccessRole = Roles.BASIC
-  let selectedTemplates = null
+
+  let templates = null
+  let screens = null
+
+  let selectedDatasources = null
   let blankScreenUrl = null
   let screenMode = null
+  let formType = null
 
   // Creates an array of screens, checking and sanitising their URLs
   const createScreens = async ({ screens, screenAccessRole }) => {
@@ -32,7 +48,7 @@
     }
 
     try {
-      let screenId
+      let createdScreens = []
 
       for (let screen of screens) {
         // Check we aren't clashing with an existing URL
@@ -56,7 +72,7 @@
 
         // Create the screen
         const response = await screenStore.save(screen)
-        screenId = response._id
+        createdScreens.push(response)
 
         // Add link in layout. We only ever actually create 1 screen now, even
         // for autoscreens, so it's always safe to do this.
@@ -66,9 +82,7 @@
         )
       }
 
-      // Go to new screen
-      $goto(`./${screenId}`)
-      screenStore.select(screenId)
+      return createdScreens
     } catch (error) {
       console.error(error)
       notifications.error("Error creating screens")
@@ -104,13 +118,16 @@
   // Handler for NewScreenModal
   export const show = newMode => {
     mode = newMode
-    selectedTemplates = null
+    templates = null
+    screens = null
+    selectedDatasources = null
     blankScreenUrl = null
     screenMode = mode
     pendingScreen = null
     screenAccessRole = Roles.BASIC
+    formType = null
 
-    if (mode === "table" || mode === "grid") {
+    if (mode === "table" || mode === "grid" || mode === "form") {
       datasourceModal.show()
     } else if (mode === "blank") {
       let templates = getTemplates($tables.list)
@@ -125,19 +142,26 @@
   }
 
   // Handler for DatasourceModal confirmation, move to screen access select
-  const confirmScreenDatasources = async ({ templates }) => {
-    selectedTemplates = templates
-    screenAccessRoleModal.show()
+  const confirmScreenDatasources = async ({ datasources }) => {
+    selectedDatasources = datasources
+    if (screenMode === "form") {
+      formTypeModal.show()
+    } else {
+      screenAccessRoleModal.show()
+    }
   }
 
   // Handler for Datasource Screen Creation
   const completeDatasourceScreenCreation = async () => {
-    const screens = selectedTemplates.map(template => {
+    templates = rowListScreen(selectedDatasources, mode)
+
+    const screens = templates.map(template => {
       let screenTemplate = template.create()
       screenTemplate.autoTableId = template.resourceId
       return screenTemplate
     })
-    await createScreens({ screens, screenAccessRole })
+    const createdScreens = await createScreens({ screens, screenAccessRole })
+    loadNewScreen(createdScreens)
   }
 
   const confirmScreenBlank = async ({ screenUrl }) => {
@@ -154,7 +178,54 @@
       return
     }
     pendingScreen.routing.route = screenUrl
-    await createScreens({ screens: [pendingScreen], screenAccessRole })
+    const createdScreens = await createScreens({
+      screens: [pendingScreen],
+      screenAccessRole,
+    })
+    loadNewScreen(createdScreens)
+  }
+
+  const onConfirmFormType = () => {
+    screenAccessRoleModal.show()
+  }
+
+  const loadNewScreen = createdScreens => {
+    const lastScreen = createdScreens.slice(-1)[0]
+
+    // Go to new screen
+    if (lastScreen?.props?._children.length) {
+      // Focus on the main component for the streen type
+      const mainComponent = lastScreen?.props?._children?.[0]._id
+      $goto(`./${lastScreen._id}/${mainComponent}`)
+    } else {
+      $goto(`./${lastScreen._id}`)
+    }
+
+    screenStore.select(lastScreen._id)
+  }
+
+  const confirmFormScreenCreation = async () => {
+    templates = formScreen(selectedDatasources, { actionType: formType })
+    screens = templates.map(template => {
+      let screenTemplate = template.create()
+      return screenTemplate
+    })
+    const createdScreens = await createScreens({ screens, screenAccessRole })
+
+    if (formType === "Update" || formType === "Create") {
+      const associatedTour =
+        formType === "Update"
+          ? TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE
+          : TOUR_KEYS.BUILDER_FORM_CREATE
+
+      const tourRequired = !$auth?.user?.tours?.[associatedTour]
+      if (tourRequired) {
+        builderStore.setTour(associatedTour)
+      }
+    }
+
+    // Go to new screen
+    loadNewScreen(createdScreens)
   }
 
   // Submit screen config for creation.
@@ -164,6 +235,8 @@
         screenUrl: blankScreenUrl,
         screenAccessRole,
       })
+    } else if (screenMode === "form") {
+      confirmFormScreenCreation()
     } else {
       completeDatasourceScreenCreation()
     }
@@ -179,19 +252,18 @@
 </script>
 
 <Modal bind:this={datasourceModal} autoFocus={false}>
-  <DatasourceModal
-    {mode}
-    onConfirm={confirmScreenDatasources}
-    initialScreens={!selectedTemplates ? [] : [...selectedTemplates]}
-  />
+  <DatasourceModal {mode} onConfirm={confirmScreenDatasources} />
 </Modal>
 
 <Modal bind:this={screenAccessRoleModal}>
   <ScreenRoleModal
-    onConfirm={confirmScreenCreation}
-    onCancel={roleSelectBack}
+    onConfirm={() => {
+      confirmScreenCreation()
+    }}
     bind:screenAccessRole
+    onCancel={roleSelectBack}
     screenUrl={blankScreenUrl}
+    confirmText={screenMode === "form" ? "Confirm" : "Done"}
   />
 </Modal>
 
@@ -201,3 +273,17 @@
     initialUrl={blankScreenUrl}
   />
 </Modal>
+
+<Modal bind:this={formTypeModal}>
+  <FormTypeModal
+    onConfirm={onConfirmFormType}
+    onCancel={() => {
+      formTypeModal.hide()
+      datasourceModal.show()
+    }}
+    on:select={e => {
+      formType = e.detail
+    }}
+    type={formType}
+  />
+</Modal>

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte

@@ -4,37 +4,33 @@
   import ICONS from "components/backend/DatasourceNavigator/icons"
   import { IntegrationNames } from "constants"
   import { onMount } from "svelte"
-  import rowListScreen from "templates/rowListScreen"
   import DatasourceTemplateRow from "./DatasourceTemplateRow.svelte"
 
-  export let mode
   export let onCancel
   export let onConfirm
-  export let initialScreens = []
 
-  let selectedScreens = [...initialScreens]
+  let selectedSources = []
 
   $: filteredSources = $datasources.list?.filter(datasource => {
     return datasource.source !== IntegrationNames.REST && datasource["entities"]
   })
 
   const toggleSelection = datasource => {
-    const { resourceId } = datasource
-    if (selectedScreens.find(s => s.resourceId === resourceId)) {
-      selectedScreens = selectedScreens.filter(
-        screen => screen.resourceId !== resourceId
+    const exists = selectedSources.find(
+      d => d.resourceId === datasource.resourceId
+    )
+    if (exists) {
+      selectedSources = selectedSources.filter(
+        d => d.resourceId === datasource.resourceId
       )
     } else {
-      selectedScreens = [
-        ...selectedScreens,
-        rowListScreen([datasource], mode)[0],
-      ]
+      selectedSources = [...selectedSources, datasource]
     }
   }
 
   const confirmDatasourceSelection = async () => {
     await onConfirm({
-      templates: selectedScreens,
+      datasources: selectedSources,
     })
   }
 
@@ -54,7 +50,7 @@
     cancelText="Back"
     onConfirm={confirmDatasourceSelection}
     {onCancel}
-    disabled={!selectedScreens.length}
+    disabled={!selectedSources.length}
     size="L"
   >
     <Body size="S">
@@ -85,8 +81,8 @@
               resourceId: table._id,
               type: "table",
             }}
-            {@const selected = selectedScreens.find(
-              screen => screen.resourceId === tableDS.resourceId
+            {@const selected = selectedSources.find(
+              datasource => datasource.resourceId === tableDS.resourceId
             )}
             <DatasourceTemplateRow
               on:click={() => toggleSelection(tableDS)}
@@ -103,7 +99,7 @@
                 tableId: view.tableId,
                 type: "viewV2",
               }}
-              {@const selected = selectedScreens.find(
+              {@const selected = selectedSources.find(
                 x => x.resourceId === viewDS.resourceId
               )}
               <DatasourceTemplateRow

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/FormTypeModal.svelte

@@ -0,0 +1,123 @@
+<script>
+  import { ModalContent, Layout, Body, Icon } from "@budibase/bbui"
+  import { createEventDispatcher } from "svelte"
+
+  export let onCancel = () => {}
+  export let onConfirm = () => {}
+  export let type
+
+  const dispatch = createEventDispatcher()
+</script>
+
+<span>
+  <ModalContent
+    title="Select form type"
+    confirmText="Done"
+    cancelText="Back"
+    {onConfirm}
+    {onCancel}
+    disabled={!type}
+    size="L"
+  >
+    <!-- svelte-ignore a11y-click-events-have-key-events -->
+    <!-- svelte-ignore a11y-no-static-element-interactions -->
+    <Layout noPadding gap="S">
+      <div
+        class="form-type"
+        class:selected={type === "Create"}
+        on:click={() => {
+          dispatch("select", "Create")
+        }}
+      >
+        <div class="form-type-wrap">
+          <div class="form-type-content">
+            <Body noPadding>Create a new row</Body>
+            <Body size="S">
+              For capturing and storing new data from your users
+            </Body>
+          </div>
+          {#if type === "Create"}
+            <span class="form-type-check">
+              <Icon size="S" name="CheckmarkCircle" />
+            </span>
+          {/if}
+        </div>
+      </div>
+      <div
+        class="form-type"
+        class:selected={type === "Update"}
+        on:click={() => {
+          dispatch("select", "Update")
+        }}
+      >
+        <div class="form-type-wrap">
+          <div class="form-type-content">
+            <Body noPadding>Update an existing row</Body>
+            <Body size="S">For viewing and updating existing data</Body>
+          </div>
+          {#if type === "Update"}
+            <span class="form-type-check">
+              <Icon size="S" name="CheckmarkCircle" />
+            </span>
+          {/if}
+        </div>
+      </div>
+      <div
+        class="form-type"
+        class:selected={type === "View"}
+        on:click={() => {
+          dispatch("select", "View")
+        }}
+      >
+        <div class="form-type-wrap">
+          <div class="form-type-content">
+            <Body noPadding>View an existing row</Body>
+            <Body size="S">For a read only view of your data</Body>
+          </div>
+          {#if type === "View"}
+            <span class="form-type-check">
+              <Icon size="S" name="CheckmarkCircle" />
+            </span>
+          {/if}
+        </div>
+      </div>
+    </Layout>
+  </ModalContent>
+</span>
+
+<style>
+  .form-type {
+    cursor: pointer;
+    gap: var(--spacing-s);
+    padding: var(--spacing-m) var(--spacing-xl);
+    background: var(--spectrum-alias-background-color-secondary);
+    transition: 0.3s all;
+    border: 1px solid var(--spectrum-global-color-gray-300);
+    border-radius: 4px;
+    display: flex;
+    flex-direction: column;
+  }
+  .selected,
+  .form-type:hover {
+    background: var(--spectrum-alias-background-color-tertiary);
+  }
+  .form-type-wrap {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+  }
+  .form-type :global(p:nth-child(2)) {
+    color: var(--grey-6);
+  }
+  .form-type-check {
+    margin-left: auto;
+  }
+  .form-type-check :global(.spectrum-Icon) {
+    color: var(--spectrum-global-color-green-600);
+  }
+  .form-type-content {
+    gap: var(--spacing-s);
+    display: flex;
+    flex-direction: column;
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte

@@ -9,6 +9,7 @@
   export let onCancel
   export let screenUrl
   export let screenAccessRole
+  export let confirmText = "Done"
 
   let error
 
@@ -40,7 +41,7 @@
 
 <ModalContent
   title="Access"
-  confirmText="Done"
+  {confirmText}
   cancelText="Back"
   {onConfirm}
   {onCancel}

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte

@@ -1,9 +1,10 @@
 <script>
   import { Body } from "@budibase/bbui"
   import CreationPage from "components/common/CreationPage.svelte"
-  import blankImage from "./blank.png"
-  import tableImage from "./table.png"
-  import gridImage from "./grid.png"
+  import blankImage from "./images/blank.png"
+  import tableImage from "./images/table.png"
+  import gridImage from "./images/grid.png"
+  import formImage from "./images/form.png"
   import CreateScreenModal from "./CreateScreenModal.svelte"
   import { screenStore } from "stores/builder"
 
@@ -56,6 +57,16 @@
           <Body size="XS">View and manipulate rows on a grid</Body>
         </div>
       </div>
+
+      <div class="card" on:click={() => createScreenModal.show("form")}>
+        <div class="image">
+          <img alt="" src={formImage} />
+        </div>
+        <div class="text">
+          <Body size="S">Form</Body>
+          <Body size="XS">Capture data from your users</Body>
+        </div>
+      </div>
     </div>
   </CreationPage>
 </div>

packages/builder/src/stores/builder/app.js

@@ -29,6 +29,7 @@ export const INITIAL_APP_META_STATE = {
   initialised: false,
   hasAppPackage: false,
   usedPlugins: null,
+  automations: {},
   routes: {},
 }
 
@@ -63,6 +64,7 @@ export class AppMetaStore extends BudiStore {
         ...app.features,
       },
       initialised: true,
+      automations: app.automations || {},
       hasAppPackage: true,
     }))
   }

packages/builder/src/stores/builder/builder.js

@@ -2,12 +2,11 @@ import { get } from "svelte/store"
 import { createBuilderWebsocket } from "./websocket.js"
 import { BuilderSocketEvent } from "@budibase/shared-core"
 import BudiStore from "./BudiStore"
-import { previewStore } from "./preview.js"
 import { TOUR_KEYS } from "components/portal/onboarding/tours.js"
 
 export const INITIAL_BUILDER_STATE = {
   previousTopNavPath: {},
-  highlightedSettingKey: null,
+  highlightedSetting: null,
   propertyFocus: null,
   builderSidePanel: false,
   onboarding: false,
@@ -26,7 +25,6 @@ export class BuilderStore extends BudiStore {
     this.reset = this.reset.bind(this)
     this.highlightSetting = this.highlightSetting.bind(this)
     this.propertyFocus = this.propertyFocus.bind(this)
-    this.hover = this.hover.bind(this)
     this.hideBuilderSidePanel = this.hideBuilderSidePanel.bind(this)
     this.showBuilderSidePanel = this.showBuilderSidePanel.bind(this)
     this.setPreviousTopNavPath = this.setPreviousTopNavPath.bind(this)
@@ -58,10 +56,10 @@ export class BuilderStore extends BudiStore {
     this.websocket = null
   }
 
-  highlightSetting(key) {
+  highlightSetting(key, type) {
     this.update(state => ({
       ...state,
-      highlightedSettingKey: key,
+      highlightedSetting: key ? { key, type: type || "info" } : null,
     }))
   }
 
@@ -135,25 +133,20 @@ export class BuilderStore extends BudiStore {
     }))
   }
 
-  startTour(tourKey) {
+  endBuilderOnboarding() {
     this.update(state => ({
       ...state,
-      tourKey: tourKey,
+      onboarding: false,
     }))
   }
 
-  hover(componentId, notifyClient = true) {
-    const store = get(this.store)
-    if (componentId === store.hoveredComponentId) {
-      return
-    }
-    this.update(state => {
-      state.hoveredComponentId = componentId
-      return state
-    })
-    if (notifyClient) {
-      previewStore.sendEvent("hover-component", componentId)
-    }
+  setTour(tourKey) {
+    this.update(state => ({
+      ...state,
+      tourStepKey: null,
+      tourNodes: null,
+      tourKey: tourKey,
+    }))
   }
 }
 

packages/builder/src/stores/builder/tests/builder.test.js

@@ -88,14 +88,42 @@ describe("Builder store", () => {
     )
   })
 
-  it("Sync a highlighted setting key to state", ctx => {
-    expect(ctx.test.store.highlightedSettingKey).toBeNull()
+  it("Sync a highlighted setting key to state. Default to info type", ctx => {
+    expect(ctx.test.store.highlightedSetting).toBeNull()
 
     ctx.test.builderStore.highlightSetting("testing")
 
     expect(ctx.test.store).toStrictEqual({
       ...INITIAL_BUILDER_STATE,
-      highlightedSettingKey: "testing",
+      highlightedSetting: {
+        key: "testing",
+        type: "info",
+      },
+    })
+  })
+
+  it("Sync a highlighted setting key to state. Use provided type", ctx => {
+    expect(ctx.test.store.highlightedSetting).toBeNull()
+
+    ctx.test.builderStore.highlightSetting("testing", "error")
+
+    expect(ctx.test.store).toStrictEqual({
+      ...INITIAL_BUILDER_STATE,
+      highlightedSetting: {
+        key: "testing",
+        type: "error",
+      },
+    })
+  })
+
+  it("Sync a highlighted setting key to state. Unset when no value is passed", ctx => {
+    expect(ctx.test.store.highlightedSetting).toBeNull()
+
+    ctx.test.builderStore.highlightSetting("testing", "error")
+    ctx.test.builderStore.highlightSetting()
+
+    expect(ctx.test.store).toStrictEqual({
+      ...INITIAL_BUILDER_STATE,
     })
   })
 

packages/builder/src/templates/formScreen.js

@@ -0,0 +1,43 @@
+import { Screen } from "./Screen"
+import { Component } from "./Component"
+import sanitizeUrl from "helpers/sanitizeUrl"
+
+export const FORM_TEMPLATE = "FORM_TEMPLATE"
+export const formUrl = datasource => sanitizeUrl(`/${datasource.label}-form`)
+
+// Mode not really necessary
+export default function (datasources, config) {
+  if (!Array.isArray(datasources)) {
+    return []
+  }
+  return datasources.map(datasource => {
+    return {
+      name: `${datasource.label} - Form`,
+      create: () => createScreen(datasource, config),
+      id: FORM_TEMPLATE,
+      resourceId: datasource.resourceId,
+    }
+  })
+}
+
+const generateMultistepFormBlock = (dataSource, { actionType } = {}) => {
+  const multistepFormBlock = new Component(
+    "@budibase/standard-components/multistepformblock"
+  )
+  multistepFormBlock
+    .customProps({
+      actionType,
+      dataSource,
+      steps: [{}],
+    })
+    .instanceName(`${dataSource.label} - Multistep Form block`)
+  return multistepFormBlock
+}
+
+const createScreen = (datasource, config) => {
+  return new Screen()
+    .route(formUrl(datasource))
+    .instanceName(`${datasource.label} - Form`)
+    .addChild(generateMultistepFormBlock(datasource, config))
+    .json()
+}

packages/builder/src/templates/index.js

@@ -1,7 +1,11 @@
 import rowListScreen from "./rowListScreen"
 import createFromScratchScreen from "./createFromScratchScreen"
+import formScreen from "./formScreen"
 
-const allTemplates = datasources => [...rowListScreen(datasources)]
+const allTemplates = datasources => [
+  ...rowListScreen(datasources),
+  ...formScreen(datasources),
+]
 
 // Allows us to apply common behaviour to all create() functions
 const createTemplateOverride = template => () => {
@@ -19,6 +23,7 @@ export default datasources => {
   })
   const fromScratch = enrichTemplate(createFromScratchScreen)
   const tableTemplates = allTemplates(datasources).map(enrichTemplate)
+
   return [
     fromScratch,
     ...tableTemplates.sort((templateA, templateB) => {

packages/client/manifest.json

@@ -525,6 +525,38 @@
         "barTitle": "Disable button",
         "key": "disabled"
       },
+      {
+        "type": "icon",
+        "label": "Icon",
+        "key": "icon"
+      },
+      {
+        "type": "select",
+        "label": "Gap",
+        "key": "gap",
+        "showInBar": true,
+        "barStyle": "picker",
+        "dependsOn": "icon",
+        "options": [
+          {
+            "label": "None",
+            "value": "N"
+          },
+          {
+            "label": "Small",
+            "value": "S"
+          },
+          {
+            "label": "Medium",
+            "value": "M"
+          },
+          {
+            "label": "Large",
+            "value": "L"
+          }
+        ],
+        "defaultValue": "M"
+      },
       {
         "type": "event",
         "label": "On click",

packages/client/src/components/app/Button.svelte

@@ -13,9 +13,10 @@
   export let size = "M"
   export let type = "cta"
   export let quiet = false
+  export let icon = null
+  export let gap = "M"
 
   // For internal use only for now - not defined in the manifest
-  export let icon = null
   export let active = false
 
   const handleOnClick = async () => {
@@ -47,7 +48,7 @@
 
 {#key $component.editing}
   <button
-    class={`spectrum-Button spectrum-Button--size${size} spectrum-Button--${type}`}
+    class={`spectrum-Button spectrum-Button--size${size} spectrum-Button--${type} gap-${gap}`}
     class:spectrum-Button--quiet={quiet}
     disabled={disabled || handlingOnClick}
     use:styleable={$component.styles}
@@ -58,15 +59,7 @@
     class:active
   >
     {#if icon}
-      <svg
-        class:hasText={componentText?.length > 0}
-        class="spectrum-Icon spectrum-Icon--size{size.toUpperCase()}"
-        focusable="false"
-        aria-hidden="true"
-        aria-label={icon}
-      >
-        <use xlink:href="#spectrum-icon-18-{icon}" />
-      </svg>
+      <i class="{icon} {size}" />
     {/if}
     {componentText}
   </button>
@@ -92,4 +85,13 @@
   .active {
     color: var(--spectrum-global-color-blue-600);
   }
+  .gap-S {
+    gap: 8px;
+  }
+  .gap-M {
+    gap: 16px;
+  }
+  .gap-L {
+    gap: 32px;
+  }
 </style>

packages/client/src/components/app/ButtonGroup.svelte

@@ -20,7 +20,7 @@
       wrap: true,
     }}
   >
-    {#each buttons as { text, type, quiet, disabled, onClick, size }}
+    {#each buttons as { text, type, quiet, disabled, onClick, size, icon, gap }}
       <BlockComponent
         type="button"
         props={{
@@ -29,6 +29,8 @@
           type,
           quiet,
           disabled,
+          icon,
+          gap,
           size: size || "M",
         }}
       />

packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte

@@ -92,9 +92,9 @@
 {#if schemaLoaded}
   <Button
     onClick={openEditor}
-    icon="Properties"
+    icon="ri-filter-3-line"
     text="Filter"
-    {size}
+    size="XL"
     type="secondary"
     quiet
     active={filters?.length > 0}

packages/pro

@@ -1 +1 @@
-Subproject commit 22a278da720d92991dabdcd4cb6c96e7abe29781
+Subproject commit 2b322d0f4b71ba96664d383b94c30445ead7ac5b

packages/server/__mocks__/node-fetch.ts

@@ -8,6 +8,7 @@ module FetchMock {
   let mockSearch = false
 
   const func = async (url: any, opts: any) => {
+    const { host, pathname } = new URL(url)
     function json(body: any, status = 200) {
       return {
         status,
@@ -34,7 +35,7 @@ module FetchMock {
       }
     }
 
-    if (url.includes("/api/global")) {
+    if (pathname.includes("/api/global")) {
       const user = {
         email: "test@example.com",
         _id: "us_test@example.com",
@@ -47,31 +48,31 @@ module FetchMock {
           global: false,
         },
       }
-      return url.endsWith("/users") && opts.method === "GET"
+      return pathname.endsWith("/users") && opts.method === "GET"
         ? json([user])
         : json(user)
     }
     // mocked data based on url
-    else if (url.includes("api/apps")) {
+    else if (pathname.includes("api/apps")) {
       return json({
         app1: {
           url: "/app1",
         },
       })
-    } else if (url.includes("example.com")) {
+    } else if (host.includes("example.com")) {
       return json({
         body: opts.body,
         url,
         method: opts.method,
       })
-    } else if (url.includes("invalid.com")) {
+    } else if (host.includes("invalid.com")) {
       return json(
         {
           invalid: true,
         },
         404
       )
-    } else if (mockSearch && url.includes("_search")) {
+    } else if (mockSearch && pathname.includes("_search")) {
       const body = opts.body
       const parts = body.split("tableId:")
       let tableId
@@ -90,7 +91,7 @@ module FetchMock {
         ],
         bookmark: "test",
       })
-    } else if (url.includes("google.com")) {
+    } else if (host.includes("google.com")) {
       return json({
         url,
         opts,
@@ -177,7 +178,7 @@ module FetchMock {
     } else if (url === "https://www.googleapis.com/oauth2/v4/token") {
       // any valid response
       return json({})
-    } else if (url.includes("failonce.com")) {
+    } else if (host.includes("failonce.com")) {
       failCount++
       if (failCount === 1) {
         return json({ message: "error" }, 500)

packages/server/src/api/controllers/role.ts

@@ -106,6 +106,21 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
   )
   role._rev = result.rev
   ctx.body = role
+
+  const devDb = context.getDevAppDB()
+  const prodDb = context.getProdAppDB()
+
+  if (await prodDb.exists()) {
+    const replication = new dbCore.Replication({
+      source: devDb.name,
+      target: prodDb.name,
+    })
+    await replication.replicate({
+      filter: (doc: any, params: any) => {
+        return doc._id && doc._id.startsWith("role_")
+      },
+    })
+  }
 }
 
 export async function destroy(ctx: UserCtx<void, DestroyRoleResponse>) {

packages/server/src/api/controllers/row/alias.ts

@@ -1,12 +1,27 @@
 import {
-  QueryJson,
-  SearchFilters,
-  Table,
-  Row,
+  Datasource,
   DatasourcePlusQueryResponse,
+  Operation,
+  QueryJson,
+  Row,
+  SearchFilters,
 } from "@budibase/types"
-import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
+import { getSQLClient } from "../../../sdk/app/rows/utils"
 import { cloneDeep } from "lodash"
+import sdk from "../../../sdk"
+import { makeExternalQuery } from "../../../integrations/base/query"
+import { SqlClient } from "../../../integrations/utils"
+
+const WRITE_OPERATIONS: Operation[] = [
+  Operation.CREATE,
+  Operation.UPDATE,
+  Operation.DELETE,
+]
+const DISABLED_WRITE_CLIENTS: SqlClient[] = [
+  SqlClient.MY_SQL,
+  SqlClient.MS_SQL,
+  SqlClient.ORACLE,
+]
 
 class CharSequence {
   static alphabet = "abcdefghijklmnopqrstuvwxyz"
@@ -43,6 +58,25 @@ export default class AliasTables {
     this.charSeq = new CharSequence()
   }
 
+  isAliasingEnabled(json: QueryJson, datasource: Datasource) {
+    const fieldLength = json.resource?.fields?.length
+    if (!fieldLength || fieldLength <= 0) {
+      return false
+    }
+    try {
+      const sqlClient = getSQLClient(datasource)
+      const isWrite = WRITE_OPERATIONS.includes(json.endpoint.operation)
+      const isDisabledClient = DISABLED_WRITE_CLIENTS.includes(sqlClient)
+      if (isWrite && isDisabledClient) {
+        return false
+      }
+    } catch (err) {
+      // if we can't get an SQL client, we can't alias
+      return false
+    }
+    return true
+  }
+
   getAlias(tableName: string) {
     if (this.aliases[tableName]) {
       return this.aliases[tableName]
@@ -111,8 +145,10 @@ export default class AliasTables {
   }
 
   async queryWithAliasing(json: QueryJson): DatasourcePlusQueryResponse {
-    const fieldLength = json.resource?.fields?.length
-    const aliasingEnabled = fieldLength && fieldLength > 0
+    const datasourceId = json.endpoint.datasourceId
+    const datasource = await sdk.datasources.get(datasourceId)
+
+    const aliasingEnabled = this.isAliasingEnabled(json, datasource)
     if (aliasingEnabled) {
       json = cloneDeep(json)
       // run through the query json to update anywhere a table may be used
@@ -158,7 +194,7 @@ export default class AliasTables {
       }
       json.tableAliases = invertedTableAliases
     }
-    const response = await getDatasourceAndQuery(json)
+    const response = await makeExternalQuery(datasource, json)
     if (Array.isArray(response) && aliasingEnabled) {
       return this.reverse(response)
     } else {

packages/server/src/api/controllers/user.ts

@@ -1,6 +1,6 @@
 import { generateUserFlagID, InternalTables } from "../../db/utils"
 import { getFullUser } from "../../utilities/users"
-import { context } from "@budibase/backend-core"
+import { cache, context } from "@budibase/backend-core"
 import {
   ContextUserMetadata,
   Ctx,

packages/server/src/api/routes/tests/application.spec.ts

@@ -16,8 +16,9 @@ import * as setup from "./utilities"
 import { AppStatus } from "../../../db/utils"
 import { events, utils, context } from "@budibase/backend-core"
 import env from "../../../environment"
-import type { App } from "@budibase/types"
+import { type App } from "@budibase/types"
 import tk from "timekeeper"
+import * as uuid from "uuid"
 
 describe("/applications", () => {
   let config = setup.getConfig()
@@ -251,7 +252,7 @@ describe("/applications", () => {
 
   describe("permissions", () => {
     it("should only return apps a user has access to", async () => {
-      const user = await config.createUser({
+      let user = await config.createUser({
         builder: { global: false },
         admin: { global: false },
       })
@@ -260,6 +261,81 @@ describe("/applications", () => {
         const apps = await config.api.application.fetch()
         expect(apps).toHaveLength(0)
       })
+
+      user = await config.globalUser({
+        ...user,
+        builder: {
+          apps: [config.getProdAppId()],
+        },
+      })
+
+      await config.withUser(user, async () => {
+        const apps = await config.api.application.fetch()
+        expect(apps).toHaveLength(1)
+      })
+    })
+
+    it("should only return apps a user has access to through a custom role", async () => {
+      let user = await config.createUser({
+        builder: { global: false },
+        admin: { global: false },
+      })
+
+      await config.withUser(user, async () => {
+        const apps = await config.api.application.fetch()
+        expect(apps).toHaveLength(0)
+      })
+
+      const role = await config.api.roles.save({
+        name: "Test",
+        inherits: "PUBLIC",
+        permissionId: "read_only",
+        version: "name",
+      })
+
+      user = await config.globalUser({
+        ...user,
+        roles: {
+          [config.getProdAppId()]: role.name,
+        },
+      })
+
+      await config.withUser(user, async () => {
+        const apps = await config.api.application.fetch()
+        expect(apps).toHaveLength(1)
+      })
+    })
+
+    it.only("should only return apps a user has access to through a custom role on a group", async () => {
+      let user = await config.createUser({
+        builder: { global: false },
+        admin: { global: false },
+      })
+
+      await config.withUser(user, async () => {
+        const apps = await config.api.application.fetch()
+        expect(apps).toHaveLength(0)
+      })
+
+      const roleName = uuid.v4().replace(/-/g, "")
+      const role = await config.api.roles.save({
+        name: roleName,
+        inherits: "PUBLIC",
+        permissionId: "read_only",
+        version: "name",
+      })
+
+      const group = await config.createGroup(role._id!)
+
+      user = await config.globalUser({
+        ...user,
+        userGroups: [group._id!],
+      })
+
+      await config.withUser(user, async () => {
+        const apps = await config.api.application.fetch()
+        expect(apps).toHaveLength(1)
+      })
     })
   })
 })

packages/server/src/integrations/base/sql.ts

@@ -435,10 +435,12 @@ class InternalBuilder {
     aliases?: QueryJson["tableAliases"]
   ): Knex.QueryBuilder {
     const tableName = endpoint.entityId
-    const tableAliased = aliases?.[tableName]
-      ? `${tableName} as ${aliases?.[tableName]}`
-      : tableName
-    let query = knex(tableAliased)
+    const tableAlias = aliases?.[tableName]
+    let table: string | Record<string, string> = tableName
+    if (tableAlias) {
+      table = { [tableAlias]: tableName }
+    }
+    let query = knex(table)
     if (endpoint.schema) {
       query = query.withSchema(endpoint.schema)
     }

packages/server/src/integrations/index.ts

@@ -14,7 +14,12 @@ import firebase from "./firebase"
 import redis from "./redis"
 import snowflake from "./snowflake"
 import oracle from "./oracle"
-import { SourceName, Integration, PluginType } from "@budibase/types"
+import {
+  SourceName,
+  Integration,
+  PluginType,
+  IntegrationBase,
+} from "@budibase/types"
 import { getDatasourcePlugin } from "../utilities/fileSystem"
 import env from "../environment"
 import cloneDeep from "lodash/cloneDeep"
@@ -40,25 +45,28 @@ const DEFINITIONS: Record<SourceName, Integration | undefined> = {
   [SourceName.BUDIBASE]: undefined,
 }
 
-const INTEGRATIONS: Record<SourceName, any> = {
-  [SourceName.POSTGRES]: postgres.integration,
-  [SourceName.DYNAMODB]: dynamodb.integration,
-  [SourceName.MONGODB]: mongodb.integration,
-  [SourceName.ELASTICSEARCH]: elasticsearch.integration,
-  [SourceName.COUCHDB]: couchdb.integration,
-  [SourceName.SQL_SERVER]: sqlServer.integration,
-  [SourceName.S3]: s3.integration,
-  [SourceName.AIRTABLE]: airtable.integration,
-  [SourceName.MYSQL]: mysql.integration,
-  [SourceName.ARANGODB]: arangodb.integration,
-  [SourceName.REST]: rest.integration,
-  [SourceName.FIRESTORE]: firebase.integration,
-  [SourceName.GOOGLE_SHEETS]: googlesheets.integration,
-  [SourceName.REDIS]: redis.integration,
-  [SourceName.SNOWFLAKE]: snowflake.integration,
-  [SourceName.ORACLE]: undefined,
-  [SourceName.BUDIBASE]: undefined,
-}
+type IntegrationBaseConstructor = new (...args: any[]) => IntegrationBase
+
+const INTEGRATIONS: Record<SourceName, IntegrationBaseConstructor | undefined> =
+  {
+    [SourceName.POSTGRES]: postgres.integration,
+    [SourceName.DYNAMODB]: dynamodb.integration,
+    [SourceName.MONGODB]: mongodb.integration,
+    [SourceName.ELASTICSEARCH]: elasticsearch.integration,
+    [SourceName.COUCHDB]: couchdb.integration,
+    [SourceName.SQL_SERVER]: sqlServer.integration,
+    [SourceName.S3]: s3.integration,
+    [SourceName.AIRTABLE]: airtable.integration,
+    [SourceName.MYSQL]: mysql.integration,
+    [SourceName.ARANGODB]: arangodb.integration,
+    [SourceName.REST]: rest.integration,
+    [SourceName.FIRESTORE]: firebase.integration,
+    [SourceName.GOOGLE_SHEETS]: googlesheets.integration,
+    [SourceName.REDIS]: redis.integration,
+    [SourceName.SNOWFLAKE]: snowflake.integration,
+    [SourceName.ORACLE]: undefined,
+    [SourceName.BUDIBASE]: undefined,
+  }
 
 // optionally add oracle integration if the oracle binary can be installed
 if (

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -1,4 +1,4 @@
-import { QueryJson } from "@budibase/types"
+import { Datasource, Operation, QueryJson, SourceName } from "@budibase/types"
 import { join } from "path"
 import Sql from "../base/sql"
 import { SqlClient } from "../utils"
@@ -198,6 +198,114 @@ describe("Captures of real examples", () => {
     })
   })
 
+  describe("check aliasing is disabled/enabled", () => {
+    const tables = ["tableA", "tableB"]
+
+    function getDatasource(source: SourceName): Datasource {
+      return {
+        source,
+        type: "datasource",
+        isSQL: true,
+      }
+    }
+
+    function getQuery(op: Operation, fields: string[] = ["a"]): QueryJson {
+      return {
+        endpoint: { datasourceId: "", entityId: "", operation: op },
+        resource: {
+          fields,
+        },
+      }
+    }
+
+    it("should check for Postgres aliased status", () => {
+      const aliasing = new AliasTables(tables)
+      const datasource = getDatasource(SourceName.POSTGRES)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
+      ).toEqual(true)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
+      ).toEqual(true)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
+      ).toEqual(true)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
+      ).toEqual(true)
+    })
+
+    it("should check for MS-SQL aliased status", () => {
+      const aliasing = new AliasTables(tables)
+      const datasource = getDatasource(SourceName.SQL_SERVER)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
+      ).toEqual(false)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
+      ).toEqual(true)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
+      ).toEqual(false)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
+      ).toEqual(false)
+    })
+
+    it("should check for MySQL aliased status", () => {
+      const aliasing = new AliasTables(tables)
+      const datasource = getDatasource(SourceName.MYSQL)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
+      ).toEqual(false)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
+      ).toEqual(true)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
+      ).toEqual(false)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
+      ).toEqual(false)
+    })
+
+    it("should check for Oracle aliased status", () => {
+      const aliasing = new AliasTables(tables)
+      const datasource = getDatasource(SourceName.ORACLE)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
+      ).toEqual(false)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
+      ).toEqual(true)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
+      ).toEqual(false)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
+      ).toEqual(false)
+    })
+
+    it("should disable aliasing for non-SQL datasources", () => {
+      const aliasing = new AliasTables(tables)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.READ), {
+          source: SourceName.GOOGLE_SHEETS,
+          type: "datasource",
+          isSQL: false,
+        })
+      )
+    })
+
+    it("should disable when no fields", () => {
+      const aliasing = new AliasTables(tables)
+      const datasource = getDatasource(SourceName.POSTGRES)
+      expect(
+        aliasing.isAliasingEnabled(getQuery(Operation.READ, []), datasource)
+      ).toEqual(false)
+    })
+  })
+
   describe("check some edge cases", () => {
     const tableNames = ["hello", "world"]
 

packages/server/src/sdk/app/rows/utils.ts

@@ -1,17 +1,51 @@
 import cloneDeep from "lodash/cloneDeep"
 import validateJs from "validate.js"
 import {
+  Datasource,
+  DatasourcePlusQueryResponse,
   FieldType,
   QueryJson,
   Row,
+  SourceName,
   Table,
   TableSchema,
-  DatasourcePlusQueryResponse,
 } from "@budibase/types"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Format } from "../../../api/controllers/view/exporters"
 import sdk from "../.."
 import { isRelationshipColumn } from "../../../db/utils"
+import { SqlClient } from "../../../integrations/utils"
+
+const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
+  [SourceName.POSTGRES]: SqlClient.POSTGRES,
+  [SourceName.MYSQL]: SqlClient.MY_SQL,
+  [SourceName.SQL_SERVER]: SqlClient.MS_SQL,
+  [SourceName.ORACLE]: SqlClient.ORACLE,
+  [SourceName.DYNAMODB]: undefined,
+  [SourceName.MONGODB]: undefined,
+  [SourceName.ELASTICSEARCH]: undefined,
+  [SourceName.COUCHDB]: undefined,
+  [SourceName.S3]: undefined,
+  [SourceName.AIRTABLE]: undefined,
+  [SourceName.ARANGODB]: undefined,
+  [SourceName.REST]: undefined,
+  [SourceName.FIRESTORE]: undefined,
+  [SourceName.GOOGLE_SHEETS]: undefined,
+  [SourceName.REDIS]: undefined,
+  [SourceName.SNOWFLAKE]: undefined,
+  [SourceName.BUDIBASE]: undefined,
+}
+
+export function getSQLClient(datasource: Datasource): SqlClient {
+  if (!datasource.isSQL) {
+    throw new Error("Cannot get SQL Client for non-SQL datasource")
+  }
+  const lookup = SQL_CLIENT_SOURCE_MAP[datasource.source]
+  if (lookup) {
+    return lookup
+  }
+  throw new Error("Unable to determine client for SQL datasource")
+}
 
 export async function getDatasourceAndQuery(
   json: QueryJson

packages/server/src/tests/jestEnv.ts

@@ -10,3 +10,4 @@ process.env.MOCK_REDIS = "1"
 process.env.PLATFORM_URL = "http://localhost:10000"
 process.env.REDIS_PASSWORD = "budibase"
 process.env.BUDIBASE_VERSION = "0.0.0+jest"
+process.env.WORKER_URL = "http://localhost:10000"

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -299,11 +299,11 @@ export default class TestConfiguration {
     }
   }
 
-  withUser(user: User, f: () => Promise<void>) {
+  async withUser(user: User, f: () => Promise<void>) {
     const oldUser = this.user
     this.user = user
     try {
-      return f()
+      return await f()
     } finally {
       this.user = oldUser
     }
@@ -347,7 +347,7 @@ export default class TestConfiguration {
       lastName = generator.last(),
       builder = { global: true },
       admin = { global: false },
-      email = generator.email(),
+      email = generator.email({ domain: "example.com" }),
       tenantId = this.getTenantId(),
       roles = {},
     } = config
@@ -363,6 +363,7 @@ export default class TestConfiguration {
       _id,
       ...existing,
       ...config,
+      _rev: existing._rev,
       email,
       roles,
       tenantId,
@@ -372,11 +373,12 @@ export default class TestConfiguration {
       admin,
     }
     await sessions.createASession(_id, {
-      sessionId: "sessionid",
+      sessionId: this.sessionIdForUser(_id),
       tenantId: this.getTenantId(),
       csrfToken: this.csrfToken,
     })
     const resp = await db.put(user)
+    await cache.user.invalidateUser(_id)
     return {
       _rev: resp.rev,
       ...user,
@@ -384,9 +386,7 @@ export default class TestConfiguration {
   }
 
   async createUser(user: Partial<User> = {}): Promise<User> {
-    const resp = await this.globalUser(user)
-    await cache.user.invalidateUser(resp._id!)
-    return resp
+    return await this.globalUser(user)
   }
 
   async createGroup(roleId: string = roles.BUILTIN_ROLE_IDS.BASIC) {
@@ -416,6 +416,10 @@ export default class TestConfiguration {
     })
   }
 
+  sessionIdForUser(userId: string): string {
+    return `sessionid-${userId}`
+  }
+
   async login({
     roleId,
     userId,
@@ -442,13 +446,13 @@ export default class TestConfiguration {
         })
       }
       await sessions.createASession(userId, {
-        sessionId: "sessionid",
+        sessionId: this.sessionIdForUser(userId),
         tenantId: this.getTenantId(),
       })
       // have to fake this
       const authObj = {
         userId,
-        sessionId: "sessionid",
+        sessionId: this.sessionIdForUser(userId),
         tenantId: this.getTenantId(),
       }
       const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret)
@@ -470,7 +474,7 @@ export default class TestConfiguration {
     const user = this.getUser()
     const authObj: AuthToken = {
       userId: user._id!,
-      sessionId: "sessionid",
+      sessionId: this.sessionIdForUser(user._id!),
       tenantId,
     }
     const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret)
@@ -508,7 +512,7 @@ export default class TestConfiguration {
 
   async basicRoleHeaders() {
     return await this.roleHeaders({
-      email: generator.email(),
+      email: generator.email({ domain: "example.com" }),
       builder: false,
       prodApp: true,
       roleId: roles.BUILTIN_ROLE_IDS.BASIC,
@@ -516,7 +520,7 @@ export default class TestConfiguration {
   }
 
   async roleHeaders({
-    email = generator.email(),
+    email = generator.email({ domain: "example.com" }),
     roleId = roles.BUILTIN_ROLE_IDS.ADMIN,
     builder = false,
     prodApp = true,

packages/server/src/tests/utilities/api/index.ts

@@ -11,6 +11,7 @@ import { BackupAPI } from "./backup"
 import { AttachmentAPI } from "./attachment"
 import { UserAPI } from "./user"
 import { QueryAPI } from "./query"
+import { RoleAPI } from "./role"
 
 export default class API {
   table: TableAPI
@@ -25,6 +26,7 @@ export default class API {
   attachment: AttachmentAPI
   user: UserAPI
   query: QueryAPI
+  roles: RoleAPI
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
@@ -39,5 +41,6 @@ export default class API {
     this.attachment = new AttachmentAPI(config)
     this.user = new UserAPI(config)
     this.query = new QueryAPI(config)
+    this.roles = new RoleAPI(config)
   }
 }

packages/server/src/tests/utilities/api/role.ts

@@ -0,0 +1,41 @@
+import {
+  AccessibleRolesResponse,
+  FetchRolesResponse,
+  FindRoleResponse,
+  SaveRoleRequest,
+  SaveRoleResponse,
+} from "@budibase/types"
+import { Expectations, TestAPI } from "./base"
+
+export class RoleAPI extends TestAPI {
+  fetch = async (expectations?: Expectations) => {
+    return await this._get<FetchRolesResponse>(`/api/roles`, {
+      expectations,
+    })
+  }
+
+  find = async (roleId: string, expectations?: Expectations) => {
+    return await this._get<FindRoleResponse>(`/api/roles/${roleId}`, {
+      expectations,
+    })
+  }
+
+  save = async (body: SaveRoleRequest, expectations?: Expectations) => {
+    return await this._post<SaveRoleResponse>(`/api/roles`, {
+      body,
+      expectations,
+    })
+  }
+
+  destroy = async (roleId: string, expectations?: Expectations) => {
+    return await this._delete(`/api/roles/${roleId}`, {
+      expectations,
+    })
+  }
+
+  accesssible = async (expectations?: Expectations) => {
+    return await this._get<AccessibleRolesResponse>(`/api/roles/accessible`, {
+      expectations,
+    })
+  }
+}

packages/shared-core/src/filters.ts

@@ -390,24 +390,41 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => {
     }
   )
 
-  // Match a document against all criteria
   const docMatch = (doc: any) => {
-    return (
-      stringMatch(doc) &&
-      fuzzyMatch(doc) &&
-      rangeMatch(doc) &&
-      equalMatch(doc) &&
-      notEqualMatch(doc) &&
-      emptyMatch(doc) &&
-      notEmptyMatch(doc) &&
-      oneOf(doc) &&
-      contains(doc) &&
-      containsAny(doc) &&
-      notContains(doc)
-    )
-  }
+    const filterFunctions: Record<SearchQueryOperators, (doc: any) => boolean> =
+      {
+        string: stringMatch,
+        fuzzy: fuzzyMatch,
+        range: rangeMatch,
+        equal: equalMatch,
+        notEqual: notEqualMatch,
+        empty: emptyMatch,
+        notEmpty: notEmptyMatch,
+        oneOf: oneOf,
+        contains: contains,
+        containsAny: containsAny,
+        notContains: notContains,
+      }
 
-  // Process all docs
+    const activeFilterKeys: SearchQueryOperators[] = Object.entries(query || {})
+      .filter(
+        ([key, value]: [string, any]) =>
+          !["allOr", "onEmptyFilter"].includes(key) &&
+          value &&
+          Object.keys(value as Record<string, any>).length > 0
+      )
+      .map(([key]) => key as any)
+
+    const results: boolean[] = activeFilterKeys.map(filterKey => {
+      return filterFunctions[filterKey]?.(doc) ?? false
+    })
+
+    if (query!.allOr) {
+      return results.some(result => result === true)
+    } else {
+      return results.every(result => result === true)
+    }
+  }
   return docs.filter(docMatch)
 }
 

packages/shared-core/src/tests/filters.test.ts

@@ -47,10 +47,7 @@ describe("runLuceneQuery", () => {
     },
   ]
 
-  function buildQuery(
-    filterKey: string,
-    value: { [key: string]: any }
-  ): SearchQuery {
+  function buildQuery(filters: { [filterKey: string]: any }): SearchQuery {
     const query: SearchQuery = {
       string: {},
       fuzzy: {},
@@ -63,8 +60,13 @@ describe("runLuceneQuery", () => {
       notContains: {},
       oneOf: {},
       containsAny: {},
+      allOr: false,
     }
-    query[filterKey as SearchQueryOperators] = value
+
+    for (const filterKey in filters) {
+      query[filterKey as SearchQueryOperators] = filters[filterKey]
+    }
+
     return query
   }
 
@@ -73,16 +75,17 @@ describe("runLuceneQuery", () => {
   })
 
   it("should return matching rows for equal filter", () => {
-    const query = buildQuery("equal", {
-      order_status: 4,
+    const query = buildQuery({
+      equal: { order_status: 4 },
     })
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1, 2])
   })
 
   it("should return matching row for notEqual filter", () => {
-    const query = buildQuery("notEqual", {
-      order_status: 4,
+    const query = buildQuery({
+      notEqual: { order_status: 4 },
     })
+
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3])
   })
 
@@ -90,48 +93,56 @@ describe("runLuceneQuery", () => {
     expect(
       runLuceneQuery(
         docs,
-        buildQuery("fuzzy", {
-          description: "sm",
+        buildQuery({
+          fuzzy: { description: "sm" },
         })
       ).map(row => row.description)
     ).toEqual(["Small box"])
     expect(
       runLuceneQuery(
         docs,
-        buildQuery("string", {
-          description: "SM",
+        buildQuery({
+          string: { description: "SM" },
         })
       ).map(row => row.description)
     ).toEqual(["Small box"])
   })
 
   it("should return rows within a range filter", () => {
-    const query = buildQuery("range", {
-      customer_id: {
-        low: 500,
-        high: 1000,
+    const query = buildQuery({
+      range: {
+        customer_id: {
+          low: 500,
+          high: 1000,
+        },
       },
     })
+
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3])
   })
 
   it("should return rows with numeric strings within a range filter", () => {
-    const query = buildQuery("range", {
-      customer_id: {
-        low: "500",
-        high: "1000",
+    const query = buildQuery({
+      range: {
+        customer_id: {
+          low: "500",
+          high: "1000",
+        },
       },
     })
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3])
   })
 
   it("should return rows with ISO date strings within a range filter", () => {
-    const query = buildQuery("range", {
-      order_date: {
-        low: "2016-01-04T00:00:00.000Z",
-        high: "2016-01-11T00:00:00.000Z",
+    const query = buildQuery({
+      range: {
+        order_date: {
+          low: "2016-01-04T00:00:00.000Z",
+          high: "2016-01-11T00:00:00.000Z",
+        },
       },
     })
+
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2])
   })
 
@@ -150,40 +161,88 @@ describe("runLuceneQuery", () => {
         label: "",
       },
     ]
-    const query = buildQuery("range", {
-      order_date: {
-        low: "2016-01-04T00:00:00.000Z",
-        high: "2016-01-11T00:00:00.000Z",
+
+    const query = buildQuery({
+      range: {
+        order_date: {
+          low: "2016-01-04T00:00:00.000Z",
+          high: "2016-01-11T00:00:00.000Z",
+        },
       },
     })
+
     expect(runLuceneQuery(docs, query)).toEqual(docs)
   })
 
   it("should return rows with matches on empty filter", () => {
-    const query = buildQuery("empty", {
-      label: null,
+    const query = buildQuery({
+      empty: {
+        label: null,
+      },
     })
+
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1])
   })
 
   it("should return rows with matches on notEmpty filter", () => {
-    const query = buildQuery("notEmpty", {
-      label: null,
+    const query = buildQuery({
+      notEmpty: {
+        label: null,
+      },
     })
+
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2, 3])
   })
 
   test.each([[523, 259], "523,259"])(
     "should return rows with matches on numeric oneOf filter",
     input => {
-      let query = buildQuery("oneOf", {
-        customer_id: input,
+      const query = buildQuery({
+        oneOf: {
+          customer_id: input,
+        },
       })
+
       expect(runLuceneQuery(docs, query).map(row => row.customer_id)).toEqual([
         259, 523,
       ])
     }
   )
+
+  test.each([
+    [false, []],
+    [true, [1, 2, 3]],
+  ])("should return %s  if allOr is %s ", (allOr, expectedResult) => {
+    const query = buildQuery({
+      allOr,
+      oneOf: { staff_id: [10] },
+      contains: { description: ["box"] },
+    })
+
+    expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual(
+      expectedResult
+    )
+  })
+
+  it("should return matching results if allOr is true and only one filter matches with different operands", () => {
+    const query = buildQuery({
+      allOr: true,
+      equal: { order_status: 4 },
+      oneOf: { label: ["FRAGILE"] },
+    })
+
+    expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1, 2])
+  })
+
+  it("should handle when a value is null or undefined", () => {
+    const query = buildQuery({
+      allOr: true,
+      equal: { order_status: null },
+      oneOf: { label: ["FRAGILE"] },
+    })
+
+    expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2])
+  })
 })
 
 describe("buildLuceneQuery", () => {

packages/types/src/api/web/auth.ts

@@ -18,6 +18,7 @@ export interface UpdateSelfRequest {
   password?: string
   forceResetPassword?: boolean
   onboardedAt?: string
+  tours?: Record<string, Date>
 }
 
 export interface UpdateSelfResponse {

packages/types/src/documents/app/role.ts

@@ -5,4 +5,5 @@ export interface Role extends Document {
   inherits?: string
   permissions: { [key: string]: string[] }
   version?: string
+  name: string
 }

packages/types/src/documents/global/user.ts

@@ -55,6 +55,7 @@ export interface User extends Document {
   dayPassRecordedAt?: string
   userGroups?: string[]
   onboardedAt?: string
+  tours?: Record<string, Date>
   scimInfo?: { isSync: true } & Record<string, any>
   ssoId?: string
 }

packages/worker/src/api/routes/global/tests/groups.spec.ts

@@ -147,7 +147,7 @@ describe("/api/global/groups", () => {
 
         await Promise.all(
           Array.from({ length: 30 }).map(async (_, i) => {
-            const email = `user${i}@${generator.domain()}`
+            const email = `user${i}@example.com`
             const user = await config.api.users.saveUser({
               ...structures.users.user(),
               email,

packages/worker/src/api/routes/validation/users.ts

@@ -26,6 +26,7 @@ export const buildSelfSaveValidation = () => {
     firstName: OPTIONAL_STRING,
     lastName: OPTIONAL_STRING,
     onboardedAt: Joi.string().optional(),
+    tours: Joi.object().optional(),
   }
   return auth.joiValidator.body(Joi.object(schema).required().unknown(false))
 }

qa-core/src/account-api/tests/accounts/accounts.cloud.spec.ts

@@ -84,7 +84,7 @@ describe("Accounts", () => {
     })
 
     it("searches by email", async () => {
-      const email = generator.email()
+      const email = generator.email({ domain: "example.com" })
 
       // Empty result
       const [_, emptyBody] = await config.api.accounts.search(email, "email")

qa-core/src/public-api/fixtures/users.ts

@@ -4,7 +4,7 @@ import { generator } from "../../shared"
 export const generateUser = (
   overrides: Partial<User> = {}
 ): CreateUserParams => ({
-  email: generator.email(),
+  email: generator.email({ domain: "example.com" }),
   roles: {
     [generator.string({ length: 32, alpha: true, numeric: true })]:
       generator.word(),