Merge remote-tracking branch 'origin/master' into feat/automation-features

hosting/couchdb/runner.sh

@@ -76,6 +76,6 @@ done
 
 # CouchDB needs the `_users` and `_replicator` databases to exist before it will
 # function correctly, so we create them here.
-curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_users
+curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users
-curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_replicator
+curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator
 sleep infinity

hosting/docker-compose.yaml

@@ -26,7 +26,7 @@ services:
       BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL}
       BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
       PLUGINS_DIR: ${PLUGINS_DIR}
-      OFFLINE_MODE: ${OFFLINE_MODE}
+      OFFLINE_MODE: ${OFFLINE_MODE:-}
     depends_on:
       - worker-service
       - redis-service
@@ -53,7 +53,7 @@ services:
       INTERNAL_API_KEY: ${INTERNAL_API_KEY}
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}
-      OFFLINE_MODE: ${OFFLINE_MODE}
+      OFFLINE_MODE: ${OFFLINE_MODE:-}
     depends_on:
       - redis-service
       - minio-service
@@ -109,7 +109,7 @@ services:
   redis-service:
     restart: unless-stopped
     image: redis
-    command: redis-server --requirepass ${REDIS_PASSWORD}
+    command: redis-server --requirepass "${REDIS_PASSWORD}"
     volumes:
       - redis_data:/data
 

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.14.2",
+  "version": "2.14.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/account-portal

@@ -1 +1 @@
-Subproject commit b11e6b47370d9b77c63648b45929c86bfed6360c
+Subproject commit b23fb3b17961fb04badd9487913a683fcf26dbe6

packages/backend-core/src/environment.ts

@@ -166,6 +166,8 @@ const environment = {
   DISABLE_JWT_WARNING: process.env.DISABLE_JWT_WARNING,
   BLACKLIST_IPS: process.env.BLACKLIST_IPS,
   SERVICE_TYPE: "unknown",
+  PASSWORD_MIN_LENGTH: process.env.PASSWORD_MIN_LENGTH,
+  PASSWORD_MAX_LENGTH: process.env.PASSWORD_MAX_LENGTH,
   /**
    * Enable to allow an admin user to login using a password.
    * This can be useful to prevent lockout when configuring SSO.

packages/backend-core/src/middleware/authenticated.ts

@@ -15,6 +15,7 @@ import * as identity from "../context/identity"
 import env from "../environment"
 import { Ctx, EndpointMatcher, SessionCookie } from "@budibase/types"
 import { InvalidAPIKeyError, ErrorCode } from "../errors"
+import tracer from "dd-trace"
 
 const ONE_MINUTE = env.SESSION_UPDATE_PERIOD
   ? parseInt(env.SESSION_UPDATE_PERIOD)
@@ -166,6 +167,16 @@ export default function (
       if (!authenticated) {
         authenticated = false
       }
+
+      if (user) {
+        tracer.setUser({
+          id: user?._id,
+          tenantId: user?.tenantId,
+          budibaseAccess: user?.budibaseAccess,
+          status: user?.status,
+        })
+      }
+
       // isAuthenticated is a function, so use a variable to be able to check authed state
       finalise(ctx, { authenticated, user, internal, version, publicEndpoint })
 

packages/backend-core/src/redis/redlockImpl.ts

@@ -2,7 +2,6 @@ import Redlock from "redlock"
 import { getLockClient } from "./init"
 import { LockOptions, LockType } from "@budibase/types"
 import * as context from "../context"
-import { logWarn } from "../logging"
 import { utils } from "@budibase/shared-core"
 import { Duration } from "../utils"
 

packages/backend-core/src/security/auth.ts

@@ -1,7 +1,7 @@
-import { env } from ".."
+import env from "../environment"
 
-export const PASSWORD_MIN_LENGTH = +(process.env.PASSWORD_MIN_LENGTH || 8)
+export const PASSWORD_MIN_LENGTH = +(env.PASSWORD_MIN_LENGTH || 8)
-export const PASSWORD_MAX_LENGTH = +(process.env.PASSWORD_MAX_LENGTH || 512)
+export const PASSWORD_MAX_LENGTH = +(env.PASSWORD_MAX_LENGTH || 512)
 
 export function validatePassword(
   password: string

packages/backend-core/src/users/db.ts

@@ -44,6 +44,12 @@ type GroupFns = {
   getBulk: GroupGetFn
   getGroupBuilderAppIds: GroupBuildersFn
 }
+type CreateAdminUserOpts = {
+  ssoId?: string
+  hashPassword?: boolean
+  requirePassword?: boolean
+  skipPasswordValidation?: boolean
+}
 type FeatureFns = { isSSOEnforced: FeatureFn; isAppBuildersEnabled: FeatureFn }
 
 const bulkDeleteProcessing = async (dbUser: User) => {
@@ -112,9 +118,11 @@ export class UserDB {
         throw new HTTPError("Password change is disabled for this user", 400)
       }
 
-      const passwordValidation = validatePassword(password)
+      if (!opts.skipPasswordValidation) {
-      if (!passwordValidation.valid) {
+        const passwordValidation = validatePassword(password)
-        throw new HTTPError(passwordValidation.error, 400)
+        if (!passwordValidation.valid) {
+          throw new HTTPError(passwordValidation.error, 400)
+        }
       }
 
       hashedPassword = opts.hashPassword ? await hash(password) : password
@@ -489,7 +497,7 @@ export class UserDB {
     email: string,
     password: string,
     tenantId: string,
-    opts?: { ssoId?: string; hashPassword?: boolean; requirePassword?: boolean }
+    opts?: CreateAdminUserOpts
   ) {
     const user: User = {
       email: email,
@@ -513,6 +521,7 @@ export class UserDB {
     return await UserDB.save(user, {
       hashPassword: opts?.hashPassword,
       requirePassword: opts?.requirePassword,
+      skipPasswordValidation: opts?.skipPasswordValidation,
     })
   }
 

packages/bbui/src/ActionButton/ActionButton.svelte

@@ -130,5 +130,6 @@
     max-width: 150px;
     transform: translateX(-50%);
     text-align: center;
+    z-index: 1;
   }
 </style>

packages/bbui/src/DetailSummary/DetailSummary.svelte

@@ -78,7 +78,7 @@
       var(--spacing-xl);
   }
   .property-panel.no-title {
-    padding: var(--spacing-xl);
+    padding-top: var(--spacing-xl);
   }
 
   .show {

packages/bbui/src/Form/Field.svelte

@@ -51,15 +51,13 @@
     margin-top: var(--spectrum-global-dimension-size-75);
     align-items: center;
   }
-
   .helpText :global(svg) {
-    width: 14px;
+    width: 13px;
-    color: var(--grey-5);
+    color: var(--spectrum-global-color-gray-600);
     margin-right: 6px;
   }
-
   .helpText span {
-    color: var(--grey-7);
+    color: var(--spectrum-global-color-gray-800);
     font-size: var(--spectrum-global-dimension-font-size-75);
   }
 </style>

packages/builder/src/api.js

@@ -5,7 +5,7 @@ import {
 } from "@budibase/frontend-core"
 import { store } from "./builderStore"
 import { get } from "svelte/store"
-import { auth } from "./stores/portal"
+import { auth, navigation } from "./stores/portal"
 
 export const API = createAPIClient({
   attachHeaders: headers => {
@@ -45,4 +45,15 @@ export const API = createAPIClient({
       }
     }
   },
+  onMigrationDetected: appId => {
+    const updatingUrl = `/builder/app/updating/${appId}`
+
+    if (window.location.pathname === updatingUrl) {
+      return
+    }
+
+    get(navigation).goto(
+      `${updatingUrl}?returnUrl=${encodeURIComponent(window.location.pathname)}`
+    )
+  },
 })

packages/builder/src/builderStore/dataBinding.js

@@ -465,8 +465,8 @@ const filterCategoryByContext = (component, context) => {
   const { _component } = component
   if (_component.endsWith("formblock")) {
     if (
-      (component.actionType == "Create" && context.type === "schema") ||
+      (component.actionType === "Create" && context.type === "schema") ||
-      (component.actionType == "View" && context.type === "form")
+      (component.actionType === "View" && context.type === "form")
     ) {
       return false
     }
@@ -474,20 +474,21 @@ const filterCategoryByContext = (component, context) => {
   return true
 }
 
+// Enrich binding category information for certain components
 const getComponentBindingCategory = (component, context, def) => {
   let icon = def.icon
   let category = component._instanceName
 
   if (component._component.endsWith("formblock")) {
-    let contextCategorySuffix = {
+    if (context.type === "form") {
-      form: "Fields",
+      category = `${component._instanceName} - Fields`
-      schema: "Row",
+      icon = "Form"
+    } else if (context.type === "schema") {
+      category = `${component._instanceName} - Row`
+      icon = "Data"
     }
-    category = `${component._instanceName} - ${
-      contextCategorySuffix[context.type]
-    }`
-    icon = context.type === "form" ? "Form" : "Data"
   }
+
   return {
     icon,
     category,

packages/builder/src/builderStore/store/frontend.js

@@ -85,7 +85,6 @@ const INITIAL_FRONTEND_STATE = {
   selectedScreenId: null,
   selectedComponentId: null,
   selectedLayoutId: null,
-  hoverComponentId: null,
 
   // Client state
   selectedComponentInstance: null,
@@ -93,6 +92,9 @@ const INITIAL_FRONTEND_STATE = {
   // Onboarding
   onboarding: false,
   tourNodes: null,
+
+  // UI state
+  hoveredComponentId: null,
 }
 
 export const getFrontendStore = () => {
@@ -610,12 +612,12 @@ export const getFrontendStore = () => {
           // Use default config if the 'buttons' prop has never been initialised
           if (!("buttons" in enrichedComponent)) {
             enrichedComponent["buttons"] =
-              Utils.buildDynamicButtonConfig(enrichedComponent)
+              Utils.buildFormBlockButtonConfig(enrichedComponent)
             migrated = true
           } else if (enrichedComponent["buttons"] == null) {
             // Ignore legacy config if 'buttons' has been reset by 'resetOn'
             const { _id, actionType, dataSource } = enrichedComponent
-            enrichedComponent["buttons"] = Utils.buildDynamicButtonConfig({
+            enrichedComponent["buttons"] = Utils.buildFormBlockButtonConfig({
               _id,
               actionType,
               dataSource,
@@ -1289,15 +1291,14 @@ export const getFrontendStore = () => {
           const settings = getComponentSettings(component._component)
           const updatedSetting = settings.find(setting => setting.key === name)
 
-          // Can be a single string or array of strings
+          // Reset dependent fields
-          const resetFields = settings.filter(setting => {
+          settings.forEach(setting => {
-            return (
+            const needsReset =
               name === setting.resetOn ||
               (Array.isArray(setting.resetOn) && setting.resetOn.includes(name))
-            )
+            if (needsReset) {
-          })
+              component[setting.key] = setting.defaultValue || null
-          resetFields?.forEach(setting => {
+            }
-            component[setting.key] = null
           })
 
           if (
@@ -1414,6 +1415,18 @@ export const getFrontendStore = () => {
           return state
         })
       },
+      hover: (componentId, notifyClient = true) => {
+        if (componentId === get(store).hoveredComponentId) {
+          return
+        }
+        store.update(state => {
+          state.hoveredComponentId = componentId
+          return state
+        })
+        if (notifyClient) {
+          store.actions.preview.sendEvent("hover-component", componentId)
+        }
+      },
     },
     links: {
       save: async (url, title) => {

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte

@@ -158,7 +158,7 @@
             {#if isDisabled && !syncAutomationsEnabled && !triggerAutomationsEnabled && lockedFeatures.includes(action.stepId)}
               <div class="tag-color">
                 <Tags>
-                  <Tag icon="LockClosed">Business</Tag>
+                  <Tag icon="LockClosed">Premium</Tag>
                 </Tags>
               </div>
             {:else if isDisabled}

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -41,7 +41,7 @@
       { label: "False", value: "false" },
     ]}
   />
-{:else if schema.type === "array"}
+{:else if schemaHasOptions(schema) && schema.type === "array"}
   <Multiselect
     bind:value={value[field]}
     options={schema.constraints.inclusion}
@@ -77,7 +77,7 @@
     on:change={e => onChange(e, field)}
     useLabel={false}
   />
-{:else if ["string", "number", "bigint", "barcodeqr"].includes(schema.type)}
+{:else if ["string", "number", "bigint", "barcodeqr", "array"].includes(schema.type)}
   <svelte:component
     this={isTestModal ? ModalBindableInput : DrawerBindableInput}
     panel={AutomationBindingPanel}

packages/builder/src/components/design/settings/componentSettings.js

@@ -25,6 +25,8 @@ import BarButtonList from "./controls/BarButtonList.svelte"
 import FieldConfiguration from "./controls/FieldConfiguration/FieldConfiguration.svelte"
 import ButtonConfiguration from "./controls/ButtonConfiguration/ButtonConfiguration.svelte"
 import RelationshipFilterEditor from "./controls/RelationshipFilterEditor.svelte"
+import FormStepConfiguration from "./controls/FormStepConfiguration.svelte"
+import FormStepControls from "components/design/settings/controls/FormStepControls.svelte"
 
 const componentMap = {
   text: DrawerBindableInput,
@@ -51,6 +53,8 @@ const componentMap = {
   url: URLSelect,
   fieldConfiguration: FieldConfiguration,
   buttonConfiguration: ButtonConfiguration,
+  stepConfiguration: FormStepConfiguration,
+  formStepControls: FormStepControls,
   columns: ColumnEditor,
   "columns/basic": BasicColumnEditor,
   "columns/grid": GridColumnEditor,

packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte

@@ -34,6 +34,9 @@
   $: canAddButtons = max == null || buttonList.length < max
 
   const sanitizeValue = val => {
+    if (!Array.isArray(val)) {
+      return null
+    }
     return val?.map(button => {
       return button._component ? button : buildPseudoInstance(button)
     })

packages/builder/src/components/design/settings/controls/DraggableList/DraggableList.svelte

@@ -13,6 +13,8 @@
   export let draggable = true
   export let focus
 
+  let zoneType = generate()
+
   let store = writable({
     selected: null,
     actions: {
@@ -46,6 +48,7 @@
         return {
           id: listItemKey ? item[listItemKey] : generate(),
           item,
+          type: zoneType,
         }
       })
       .filter(item => item.id)
@@ -83,6 +86,8 @@
     items: draggableItems,
     dropTargetStyle: { outline: "none" },
     dragDisabled: !draggable || inactive,
+    type: zoneType,
+    dropFromOthersDisabled: true,
   }}
   on:finalize={handleFinalize}
   on:consider={updateRowOrder}

packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte

@@ -14,6 +14,7 @@
   import { convertOldFieldFormat, getComponentForField } from "./utils"
 
   export let componentInstance
+  export let bindings
   export let value
 
   const dispatch = createEventDispatcher()
@@ -28,7 +29,9 @@
 
   let selectAll = true
 
-  $: bindings = getBindableProperties($selectedScreen, componentInstance._id)
+  $: resolvedBindings =
+    bindings || getBindableProperties($selectedScreen, componentInstance._id)
+
   $: actionType = componentInstance.actionType
   let componentBindings = []
 
@@ -39,7 +42,10 @@
     )
   }
 
-  $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
+  $: datasource =
+    componentInstance.dataSource ||
+    getDatasourceForProvider($currentAsset, componentInstance)
+
   $: resourceId = datasource?.resourceId || datasource?.tableId
 
   $: if (!isEqual(value, cachedValue)) {
@@ -179,7 +185,7 @@
       listType={FieldSetting}
       listTypeProps={{
         componentBindings,
-        bindings,
+        bindings: resolvedBindings,
       }}
     />
   {/if}

packages/builder/src/components/design/settings/controls/FormStepConfiguration.svelte

@@ -0,0 +1,182 @@
+<script>
+  import { createEventDispatcher, setContext } from "svelte"
+  import ComponentSettingsSection from "../../../../pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte"
+  import { getDatasourceForProvider } from "builderStore/dataBinding"
+  import { currentAsset, store } from "builderStore"
+  import { Helpers } from "@budibase/bbui"
+  import { derived, writable } from "svelte/store"
+  import { Utils } from "@budibase/frontend-core"
+  import { cloneDeep, isEqual } from "lodash"
+
+  export let componentInstance
+  export let componentBindings
+  export let value
+  export let bindings
+
+  const dispatch = createEventDispatcher()
+  const multiStepStore = writable({
+    stepCount: value?.length ?? 0,
+    currentStep: 0,
+  })
+  const currentStep = derived(multiStepStore, state => state.currentStep)
+  const componentType = "@budibase/standard-components/multistepformblockstep"
+
+  let cachedValue
+  let cachedInstance = {}
+
+  $: if (!isEqual(cachedValue, value)) {
+    cachedValue = value
+  }
+
+  $: if (!isEqual(componentInstance, cachedInstance)) {
+    cachedInstance = componentInstance
+  }
+
+  setContext("multi-step-form-block", multiStepStore)
+
+  $: stepCount = cachedValue?.length || 0
+  $: updateStore(stepCount)
+  $: dataSource = getDatasourceForProvider($currentAsset, cachedInstance)
+  $: emitCurrentStep($currentStep)
+  $: stepLabel = getStepLabel($multiStepStore)
+  $: stepDef = getDefinition(stepLabel)
+  $: stepSettings = cachedValue?.[$currentStep] || {}
+  $: defaults = Utils.buildMultiStepFormBlockDefaultProps({
+    _id: cachedInstance._id,
+    stepCount: $multiStepStore.stepCount,
+    currentStep: $multiStepStore.currentStep,
+    actionType: cachedInstance.actionType,
+    dataSource: cachedInstance.dataSource,
+  })
+  $: stepInstance = {
+    _id: Helpers.uuid(),
+    _component: componentType,
+    _instanceName: `Step ${currentStep + 1}`,
+    title: stepSettings.title ?? defaults.title,
+    buttons: stepSettings.buttons || defaults.buttons,
+    fields: stepSettings.fields,
+    desc: stepSettings.desc,
+
+    // Needed for field configuration
+    dataSource,
+  }
+
+  const getDefinition = stepLabel => {
+    let def = cloneDeep(store.actions.components.getDefinition(componentType))
+    def.settings.find(x => x.key === "steps").label = stepLabel
+    return def
+  }
+
+  const updateStore = stepCount => {
+    multiStepStore.update(state => {
+      state.stepCount = stepCount
+      if (state.currentStep >= stepCount) {
+        state.currentStep = 0
+      }
+      return { ...state }
+    })
+  }
+
+  const getStepLabel = ({ stepCount, currentStep }) => {
+    if (stepCount <= 1) {
+      return "Steps"
+    }
+    return `Steps (${currentStep + 1}/${stepCount})`
+  }
+
+  const emitCurrentStep = step => {
+    store.actions.preview.sendEvent("builder-meta", {
+      componentId: componentInstance._id,
+      step: step,
+    })
+  }
+
+  const addStep = () => {
+    value = value.toSpliced($currentStep + 1, 0, {})
+    dispatch("change", value)
+    multiStepStore.update(state => ({
+      ...state,
+      currentStep: $currentStep + 1,
+    }))
+  }
+
+  const removeStep = () => {
+    value = value.toSpliced($currentStep, 1)
+    dispatch("change", value)
+    multiStepStore.update(state => ({
+      ...state,
+      currentStep: Math.min($currentStep, stepCount - 2),
+    }))
+  }
+
+  const previousStep = () => {
+    multiStepStore.update(state => ({
+      ...state,
+      currentStep: Math.max($currentStep - 1, 0),
+    }))
+  }
+
+  const nextStep = () => {
+    multiStepStore.update(state => ({
+      ...state,
+      currentStep: Math.min($currentStep + 1, value.length - 1),
+    }))
+  }
+
+  const updateStep = (field, val) => {
+    const newStep = {
+      ...value[$currentStep],
+      [field.key]: val,
+    }
+    value = value.toSpliced($currentStep, 1, newStep)
+    dispatch("change", value)
+  }
+
+  const handleStepAction = action => {
+    switch (action) {
+      case "addStep":
+        addStep()
+        break
+      case "removeStep":
+        removeStep()
+        break
+      case "nextStep":
+        nextStep()
+        break
+      case "previousStep":
+        previousStep()
+    }
+  }
+
+  const processUpdate = (field, val) => {
+    if (field.key === "steps") {
+      handleStepAction(val.action)
+    } else {
+      updateStep(field, val)
+    }
+  }
+</script>
+
+<div class="nested-section">
+  <ComponentSettingsSection
+    includeHidden
+    componentInstance={stepInstance}
+    componentDefinition={stepDef}
+    onUpdateSetting={processUpdate}
+    showSectionTitle={false}
+    isScreen={false}
+    nested={true}
+    {bindings}
+    {componentBindings}
+  />
+</div>
+
+<style>
+  .nested-section {
+    margin: 0 calc(-1 * var(--spacing-xl)) calc(-1 * var(--spacing-xl))
+      calc(-1 * var(--spacing-xl));
+  }
+  .nested-section :global(.property-panel) {
+    padding-top: 0;
+  }
+</style>

packages/builder/src/components/design/settings/controls/FormStepControls.svelte

@@ -0,0 +1,84 @@
+<script>
+  import { createEventDispatcher, getContext } from "svelte"
+  import { ActionButton } from "@budibase/bbui"
+
+  const multiStepStore = getContext("multi-step-form-block")
+  const dispatch = createEventDispatcher()
+
+  $: ({ stepCount, currentStep } = $multiStepStore)
+
+  const stepAction = action => {
+    dispatch("change", {
+      action,
+    })
+  }
+</script>
+
+{#if stepCount === 1}
+  <div class="stretch">
+    <ActionButton
+      icon="MultipleAdd"
+      secondary
+      on:click={() => {
+        stepAction("addStep")
+      }}
+    >
+      Add Step
+    </ActionButton>
+  </div>
+{:else}
+  <div class="step-actions">
+    <ActionButton
+      size="S"
+      secondary
+      icon="ChevronLeft"
+      disabled={currentStep === 0}
+      on:click={() => {
+        stepAction("previousStep")
+      }}
+      tooltip={"Previous step"}
+    />
+    <ActionButton
+      size="S"
+      secondary
+      disabled={currentStep === stepCount - 1}
+      icon="ChevronRight"
+      on:click={() => {
+        stepAction("nextStep")
+      }}
+      tooltip={"Next step"}
+    />
+    <ActionButton
+      size="S"
+      secondary
+      icon="Close"
+      disabled={stepCount === 1}
+      on:click={() => {
+        stepAction("removeStep")
+      }}
+      tooltip={"Remove step"}
+    />
+    <ActionButton
+      size="S"
+      secondary
+      icon="MultipleAdd"
+      on:click={() => {
+        stepAction("addStep")
+      }}
+      tooltip={"Add step"}
+    />
+  </div>
+{/if}
+
+<style>
+  .stretch :global(.spectrum-ActionButton) {
+    width: 100%;
+  }
+  .step-actions {
+    display: flex;
+    gap: var(--spacing-s);
+  }
+  .step-actions :global(.spectrum-ActionButton) {
+    height: 32px;
+  }
+</style>

packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js

@@ -114,7 +114,7 @@ const getColumns = ({
     primary,
     sortable,
     updateSortable: newDraggableList => {
-      onChange(toGridFormat(newDraggableList.concat(primary)))
+      onChange(toGridFormat(newDraggableList.concat(primary || [])))
     },
     update: newEntry => {
       const newDraggableList = draggableList.map(entry => {

packages/builder/src/components/design/settings/controls/PropertyControl.svelte

@@ -24,6 +24,7 @@
   export let propertyFocus = false
   export let info = null
   export let disableBindings = false
+  export let wide
 
   $: nullishValue = value == null || value === ""
   $: allBindings = getAllBindings(bindings, componentBindings, nested)
@@ -78,7 +79,7 @@
 
 <div
   class="property-control"
-  class:wide={!label || labelHidden}
+  class:wide={!label || labelHidden || wide === true}
   class:highlighted={highlighted && nullishValue}
   class:property-focus={propertyFocus}
 >
@@ -104,6 +105,7 @@
       {...props}
       on:drawerHide
       on:drawerShow
+      on:meta
     />
   </div>
   {#if info}
@@ -146,15 +148,28 @@
   .control {
     position: relative;
   }
-  .property-control.wide .control {
-    grid-column: 1 / -1;
-  }
   .text {
     font-size: var(--spectrum-global-dimension-font-size-75);
     color: var(--grey-6);
     grid-column: 2 / 2;
   }
+
+  .property-control.wide .control {
+    flex: 1;
+  }
+  .property-control.wide {
+    grid-template-columns: unset;
+    display: flex;
+    flex-direction: column;
+    width: 100%;
+  }
+  .property-control.wide > * {
+    width: 100%;
+  }
   .property-control.wide .text {
     grid-column: 1 / -1;
   }
+  .property-control.wide .label {
+    margin-bottom: -8px;
+  }
 </style>

packages/builder/src/components/design/settings/controls/RelationshipFilterEditor.svelte

@@ -1,6 +1,9 @@
 <script>
-  import { currentAsset } from "builderStore"
+  import { currentAsset, store } from "builderStore"
-  import { findClosestMatchingComponent } from "builderStore/componentUtils"
+  import {
+    findClosestMatchingComponent,
+    findComponent,
+  } from "builderStore/componentUtils"
   import {
     getDatasourceForProvider,
     getSchemaForDatasource,
@@ -20,8 +23,23 @@
     component => component._component.endsWith("/form")
   )
 
+  const resolveDatasource = (currentAsset, componentInstance, form) => {
+    if (!form && componentInstance._id != $store.selectedComponentId) {
+      const block = findComponent(
+        currentAsset.props,
+        $store.selectedComponentId
+      )
+      const def = store.actions.components.getDefinition(block._component)
+      return def?.block === true
+        ? getDatasourceForProvider(currentAsset, block)
+        : {}
+    } else {
+      return getDatasourceForProvider(currentAsset, form)
+    }
+  }
+
   // Get that form's schema
-  $: datasource = getDatasourceForProvider($currentAsset, form)
+  $: datasource = resolveDatasource($currentAsset, componentInstance, form)
   $: formSchema = getSchemaForDatasource($currentAsset, datasource)?.schema
 
   // Get the schema for the relationship field that this picker is using

packages/builder/src/components/design/settings/controls/ValidationEditor/ValidationDrawer.svelte

@@ -12,7 +12,10 @@
   } from "@budibase/bbui"
   import { currentAsset, selectedComponent } from "builderStore"
   import { findClosestMatchingComponent } from "builderStore/componentUtils"
-  import { getSchemaForDatasource } from "builderStore/dataBinding"
+  import {
+    getSchemaForDatasource,
+    getDatasourceForProvider,
+  } from "builderStore/dataBinding"
   import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
   import { generate } from "shortid"
 
@@ -124,6 +127,12 @@
     ],
   }
 
+  const resolveDatasource = (currentAsset, componentInstance, parent) => {
+    return (
+      getDatasourceForProvider(currentAsset, parent || componentInstance) || {}
+    )
+  }
+
   $: dataSourceSchema = getDataSourceSchema($currentAsset, $selectedComponent)
   $: field = fieldName || $selectedComponent?.field
   $: schemaRules = parseRulesFromSchema(field, dataSourceSchema || {})
@@ -146,8 +155,8 @@
         component._component.endsWith("/formblock") ||
         component._component.endsWith("/tableblock")
     )
-
+    const dataSource = resolveDatasource(asset, component, formParent)
-    return getSchemaForDatasource(asset, formParent?.dataSource)
+    return getSchemaForDatasource(asset, dataSource)
   }
 
   const parseRulesFromSchema = (field, dataSourceSchema) => {

packages/builder/src/pages/builder/_layout.svelte

@@ -1,6 +1,6 @@
 <script>
   import { isActive, redirect, params } from "@roxi/routify"
-  import { admin, auth, licensing } from "stores/portal"
+  import { admin, auth, licensing, navigation } from "stores/portal"
   import { onMount } from "svelte"
   import { CookieUtils, Constants } from "@budibase/frontend-core"
   import { API } from "api"
@@ -17,6 +17,8 @@
 
   $: useAccountPortal = cloud && !$admin.disableAccountPortal
 
+  navigation.actions.init($redirect)
+
   const validateTenantId = async () => {
     const host = window.location.host
     if (host.includes("localhost:") || !baseUrl) {

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte

@@ -108,6 +108,8 @@
           {componentInstance}
           {componentDefinition}
           {bindings}
+          iconTooltip={componentName}
+          componentTitle={title}
         />
       {/if}
       {#if section == "conditions"}

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte

@@ -32,21 +32,19 @@
     const generalSettings = settings.filter(
       setting => !setting.section && setting.tag === tag
     )
-
     const customSections = settings.filter(
       setting => setting.section && setting.tag === tag
     )
-    let sections = [
+    let sections = []
-      ...(generalSettings?.length
+    if (generalSettings.length) {
-        ? [
+      sections.push({
-            {
+        name: "General",
-              name: "General",
+        settings: generalSettings,
-              settings: generalSettings,
+      })
-            },
+    }
-          ]
+    if (customSections.length) {
-        : []),
+      sections = sections.concat(customSections)
-      ...(customSections || []),
+    }
-    ]
 
     // Filter out settings which shouldn't be rendered
     sections.forEach(section => {
@@ -153,6 +151,7 @@
     <DetailSummary
       name={showSectionTitle ? section.name : ""}
       initiallyShow={section.collapsed !== true}
+      collapsible={section.name !== "General"}
     >
       {#if section.info}
         <div class="section-info">
@@ -172,6 +171,7 @@
               control={getComponentForSetting(setting)}
               label={setting.label}
               labelHidden={setting.labelHidden}
+              wide={setting.wide}
               key={setting.key}
               value={componentInstance[setting.key]}
               defaultValue={setting.defaultValue}
@@ -208,7 +208,7 @@
     </DetailSummary>
   {/if}
 {/each}
-{#if componentDefinition?.block && !tag}
+{#if componentDefinition?.block && !tag && componentDefinition.ejectable !== false}
   <DetailSummary name="Eject" collapsible={false}>
     <EjectBlockButton />
   </DetailSummary>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/CustomStylesSection.svelte

@@ -5,6 +5,9 @@
     Drawer,
     Button,
     notifications,
+    AbsTooltip,
+    Icon,
+    Body,
   } from "@budibase/bbui"
   import { selectedScreen, store } from "builderStore"
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
@@ -15,6 +18,9 @@
   } from "builderStore/dataBinding"
 
   export let componentInstance
+  export let componentDefinition
+  export let iconTooltip
+  export let componentTitle
 
   let tempValue
   let drawer
@@ -24,6 +30,8 @@
     $store.selectedComponentId
   )
 
+  $: icon = componentDefinition?.icon
+
   const openDrawer = () => {
     tempValue = runtimeToReadableBinding(
       bindings,
@@ -54,7 +62,19 @@
 {#key componentInstance?._id}
   <Drawer bind:this={drawer} title="Custom CSS">
     <svelte:fragment slot="description">
-      Custom CSS overrides all other component styles.
+      <div class="header">
+        Your CSS will overwrite styles for:
+        {#if icon}
+          <AbsTooltip type="info" text={iconTooltip}>
+            <Icon
+              color={`var(--spectrum-global-color-gray-600)`}
+              size="S"
+              name={icon}
+            />
+          </AbsTooltip>
+          <Body size="S"><b>{componentTitle || ""}</b></Body>
+        {/if}
+      </div>
     </svelte:fragment>
     <Button cta slot="buttons" on:click={save}>Save</Button>
     <svelte:component
@@ -68,3 +88,13 @@
     />
   </Drawer>
 {/key}
+
+<style>
+  .header {
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+    align-items: center;
+    gap: var(--spacing-m);
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json

@@ -8,6 +8,7 @@
       "cardsblock",
       "repeaterblock",
       "formblock",
+      "multistepformblock",
       "chartblock",
       "rowexplorer"
     ]

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte

@@ -36,14 +36,12 @@
 
   // Determine selected component ID
   $: selectedComponentId = $store.selectedComponentId
-  $: hoverComponentId = $store.hoverComponentId
 
   $: previewData = {
     appId: $store.appId,
     layout,
     screen,
     selectedComponentId,
-    hoverComponentId,
     theme: $store.theme,
     customTheme: $store.customTheme,
     previewDevice: $store.previewDevice,
@@ -119,8 +117,8 @@
       error = event.error || "An unknown error occurred"
     } else if (type === "select-component" && data.id) {
       $store.selectedComponentId = data.id
-    } else if (type === "hover-component" && data.id) {
+    } else if (type === "hover-component") {
-      $store.hoverComponentId = data.id
+      store.actions.components.hover(data.id, false)
     } else if (type === "update-prop") {
       await store.actions.components.updateSetting(data.prop, data.value)
     } else if (type === "update-styles") {

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentDropdownMenu.svelte

@@ -7,6 +7,7 @@
   $: definition = store.actions.components.getDefinition(component?._component)
   $: noPaste = !$store.componentToPaste
   $: isBlock = definition?.block === true
+  $: canEject = !(definition?.ejectable === false)
 
   const keyboardEvent = (key, ctrlKey = false) => {
     document.dispatchEvent(
@@ -32,7 +33,7 @@
   >
     Delete
   </MenuItem>
-  {#if isBlock}
+  {#if isBlock && canEject}
     <MenuItem
       icon="Export"
       keyBind="Ctrl+E"

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentKeyHandler.svelte

@@ -32,8 +32,15 @@
       await store.actions.components.paste(component, "below")
     },
     ["Ctrl+e"]: component => {
-      componentToEject = component
+      const definition = store.actions.components.getDefinition(
-      confirmEjectDialog.show()
+        component._component
+      )
+      const isBlock = definition?.block === true
+      const canEject = !(definition?.ejectable === false)
+      if (isBlock && canEject) {
+        componentToEject = component
+        confirmEjectDialog.show()
+      }
     },
     ["Ctrl+Enter"]: () => {
       $goto(`./:componentId/new`)

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentTree.svelte

@@ -90,16 +90,7 @@
     return findComponentPath($selectedComponent, component._id)?.length > 0
   }
 
-  const handleMouseover = componentId => {
+  const hover = store.actions.components.hover
-    if ($store.hoverComponentId !== componentId) {
-      $store.hoverComponentId = componentId
-    }
-  }
-  const handleMouseout = componentId => {
-    if ($store.hoverComponentId === componentId) {
-      $store.hoverComponentId = null
-    }
-  }
 </script>
 
 <ul>
@@ -120,9 +111,9 @@
         on:dragover={dragover(component, index)}
         on:iconClick={() => toggleNodeOpen(component._id)}
         on:drop={onDrop}
-        hovering={$store.hoverComponentId === component._id}
+        hovering={$store.hoveredComponentId === component._id}
-        on:mouseenter={() => handleMouseover(component._id)}
+        on:mouseenter={() => hover(component._id)}
-        on:mouseleave={() => handleMouseout(component._id)}
+        on:mouseleave={() => hover(null)}
         text={getComponentText(component)}
         icon={getComponentIcon(component)}
         iconTooltip={getComponentName(component)}

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/index.svelte

@@ -12,6 +12,9 @@
 
   let scrolling = false
 
+  $: screenComponentId = `${$store.selectedScreenId}-screen`
+  $: navComponentId = `${$store.selectedScreenId}-navigation`
+
   const toNewComponentRoute = () => {
     if ($isActive(`./:componentId/new`)) {
       $goto(`./:componentId`)
@@ -33,16 +36,7 @@
     scrolling = e.target.scrollTop !== 0
   }
 
-  const handleMouseover = componentId => {
+  const hover = store.actions.components.hover
-    if ($store.hoverComponentId !== componentId) {
-      $store.hoverComponentId = componentId
-    }
-  }
-  const handleMouseout = componentId => {
-    if ($store.hoverComponentId === componentId) {
-      $store.hoverComponentId = null
-    }
-  }
 </script>
 
 <div class="components">
@@ -65,46 +59,31 @@
             scrollable
             icon="WebPage"
             on:drop={onDrop}
-            on:click={() => {
+            on:click={() => ($store.selectedComponentId = screenComponentId)}
-              $store.selectedComponentId = `${$store.selectedScreenId}-screen`
+            hovering={$store.hoveredComponentId === screenComponentId}
-            }}
+            on:mouseenter={() => hover(screenComponentId)}
-            hovering={$store.hoverComponentId ===
+            on:mouseleave={() => hover(null)}
-              `${$store.selectedScreenId}-screen`}
+            id="component-screen"
-            on:mouseenter={() =>
+            selectedBy={$userSelectedResourceMap[screenComponentId]}
-              handleMouseover(`${$store.selectedScreenId}-screen`)}
-            on:mouseleave={() =>
-              handleMouseout(`${$store.selectedScreenId}-screen`)}
-            id={`component-screen`}
-            selectedBy={$userSelectedResourceMap[
-              `${$store.selectedScreenId}-screen`
-            ]}
           >
             <ScreenslotDropdownMenu component={$selectedScreen?.props} />
           </NavItem>
           <NavItem
             text="Navigation"
             indentLevel={0}
-            selected={$store.selectedComponentId ===
+            selected={$store.selectedComponentId === navComponentId}
-              `${$store.selectedScreenId}-navigation`}
             opened
             scrollable
             icon={$selectedScreen.showNavigation
               ? "Visibility"
               : "VisibilityOff"}
             on:drop={onDrop}
-            on:click={() => {
+            on:click={() => ($store.selectedComponentId = navComponentId)}
-              $store.selectedComponentId = `${$store.selectedScreenId}-navigation`
+            hovering={$store.hoveredComponentId === navComponentId}
-            }}
+            on:mouseenter={() => hover(navComponentId)}
-            hovering={$store.hoverComponentId ===
+            on:mouseleave={() => hover(null)}
-              `${$store.selectedScreenId}-navigation`}
+            id="component-nav"
-            on:mouseenter={() =>
+            selectedBy={$userSelectedResourceMap[navComponentId]}
-              handleMouseover(`${$store.selectedScreenId}-navigation`)}
-            on:mouseleave={() =>
-              handleMouseout(`${$store.selectedScreenId}-navigation`)}
-            id={`component-nav`}
-            selectedBy={$userSelectedResourceMap[
-              `${$store.selectedScreenId}-navigation`
-            ]}
           />
           <ComponentTree
             level={0}

packages/builder/src/pages/builder/app/updating/[application].svelte

@@ -0,0 +1,19 @@
+<script>
+  import { Updating } from "@budibase/frontend-core"
+  import { redirect, params } from "@roxi/routify"
+
+  import { API } from "api"
+
+  async function isMigrationDone() {
+    const response = await API.getMigrationStatus()
+    return response.migrated
+  }
+
+  async function onMigrationDone() {
+    // For some reason routify params is not stripping the ? properly, so we need to check both with and without ?
+    const returnUrl = $params.returnUrl || $params["?returnUrl"]
+    $redirect(returnUrl)
+  }
+</script>
+
+<Updating {isMigrationDone} {onMigrationDone} />

packages/builder/src/pages/builder/portal/settings/branding.svelte

@@ -214,7 +214,7 @@
         <Heading size="M">Branding</Heading>
         {#if !isCloud && !brandingEnabled}
           <Tags>
-            <Tag icon="LockClosed">Business</Tag>
+            <Tag icon="LockClosed">Premium</Tag>
           </Tags>
         {/if}
         {#if isCloud && !brandingEnabled}

packages/builder/src/pages/builder/portal/users/groups/index.svelte

@@ -97,7 +97,7 @@
       <Heading size="M">Groups</Heading>
       {#if !$licensing.groupsEnabled}
         <Tags>
-          <Tag icon="LockClosed">Business</Tag>
+          <Tag icon="LockClosed">Enterpise</Tag>
         </Tags>
       {/if}
     </div>

packages/builder/src/stores/portal/index.js

@@ -16,5 +16,6 @@ export { environment } from "./environment"
 export { menu } from "./menu"
 export { auditLogs } from "./auditLogs"
 export { features } from "./features"
+export { navigation } from "./navigation"
 
 export const sideBarCollapsed = writable(false)

packages/builder/src/stores/portal/navigation.js

@@ -0,0 +1,31 @@
+import { writable } from "svelte/store"
+
+export function createNavigationStore() {
+  const store = writable({
+    initialisated: false,
+    goto: undefined,
+  })
+  const { set, subscribe } = store
+
+  const init = gotoFunc => {
+    if (typeof gotoFunc !== "function") {
+      throw new Error(
+        `gotoFunc must be a function, found a "${typeof gotoFunc}" instead`
+      )
+    }
+
+    set({
+      initialisated: true,
+      goto: gotoFunc,
+    })
+  }
+
+  return {
+    subscribe,
+    actions: {
+      init,
+    },
+  }
+}
+
+export const navigation = createNavigationStore()

packages/client/manifest.json

@@ -4879,7 +4879,7 @@
   },
   "chartblock": {
     "block": true,
-    "name": "Chart block",
+    "name": "Chart Block",
     "icon": "GraphPie",
     "hasChildren": false,
     "settings": [
@@ -5369,7 +5369,7 @@
   },
   "tableblock": {
     "block": true,
-    "name": "Table block",
+    "name": "Table Block",
     "icon": "Table",
     "styles": ["size"],
     "size": {
@@ -5615,7 +5615,7 @@
   },
   "cardsblock": {
     "block": true,
-    "name": "Cards block",
+    "name": "Cards Block",
     "icon": "PersonalizationField",
     "styles": ["size"],
     "size": {
@@ -5795,7 +5795,7 @@
   },
   "repeaterblock": {
     "block": true,
-    "name": "Repeater block",
+    "name": "Repeater Block",
     "icon": "ViewList",
     "illegalChildren": ["section"],
     "hasChildren": true,
@@ -6035,6 +6035,164 @@
       }
     ]
   },
+  "multistepformblock": {
+    "name": "Multi-step Form Block",
+    "icon": "AssetsAdded",
+    "block": true,
+    "hasChildren": false,
+    "ejectable": false,
+    "size": {
+      "width": 400,
+      "height": 400
+    },
+    "styles": ["size"],
+    "settings": [
+      {
+        "type": "table",
+        "label": "Data",
+        "key": "dataSource"
+      },
+      {
+        "type": "radio",
+        "label": "Type",
+        "key": "actionType",
+        "options": ["Create", "Update", "View"],
+        "defaultValue": "Create"
+      },
+      {
+        "section": true,
+        "dependsOn": {
+          "setting": "actionType",
+          "value": "Create",
+          "invert": true
+        },
+        "name": "Row ID",
+        "info": "<a href='https://docs.budibase.com/docs/form-block' target='_blank'>How to pass a row ID using bindings</a>",
+        "settings": [
+          {
+            "type": "text",
+            "label": "Row ID",
+            "key": "rowId",
+            "nested": true
+          },
+          {
+            "type": "text",
+            "label": "No rows found",
+            "key": "noRowsMessage",
+            "defaultValue": "We couldn't find a row to display",
+            "nested": true
+          }
+        ]
+      },
+      {
+        "section": true,
+        "name": "Details",
+        "settings": [
+          {
+            "type": "stepConfiguration",
+            "key": "steps",
+            "nested": true,
+            "labelHidden": true,
+            "resetOn": [
+              "dataSource",
+              "actionType"
+            ],
+            "defaultValue": [
+              {}
+            ]
+          }
+        ]
+      }
+    ],
+    "actions": [
+      {
+        "type": "ValidateForm",
+        "suffix": "form"
+      },
+      {
+        "type": "ClearForm",
+        "suffix": "form"
+      },
+      {
+        "type": "UpdateFieldValue",
+        "suffix": "form"
+      },
+      {
+        "type": "ScrollTo",
+        "suffix": "form"
+      },
+      {
+        "type": "ChangeFormStep",
+        "suffix": "form"
+      }
+    ],
+    "context": [
+      {
+        "type": "form",
+        "suffix": "form"
+      },
+      {
+        "type": "static",
+        "suffix": "form",
+        "values": [
+          {
+            "label": "Value",
+            "key": "__value",
+            "type": "object"
+          },
+          {
+            "label": "Valid",
+            "key": "__valid",
+            "type": "boolean"
+          },
+          {
+            "label": "Current Step",
+            "key": "__currentStep",
+            "type": "number"
+          },
+          {
+            "label": "Current Step Valid",
+            "key": "__currentStepValid",
+            "type": "boolean"
+          }
+        ]
+      }
+    ]
+  },
+  "multistepformblockstep": {
+    "name": "Multi-step Form Block Step",
+    "settings": [
+      {
+        "type": "formStepControls",
+        "label": "Steps",
+        "key": "steps"
+      },
+      {
+        "type": "text",
+        "label": "Title",
+        "key": "title",
+        "nested": true
+      },
+      {
+        "type": "text",
+        "label": "Description",
+        "key": "desc",
+        "nested": true
+      },
+      {
+        "type": "fieldConfiguration",
+        "key": "fields",
+        "nested": true
+      },
+      {
+        "type": "buttonConfiguration",
+        "label": "Buttons",
+        "key": "buttons",
+        "wide": true,
+        "nested": true
+      }
+    ]
+  },
   "formblock": {
     "name": "Form Block",
     "icon": "Form",
@@ -6290,7 +6448,7 @@
     }
   },
   "gridblock": {
-    "name": "Grid block",
+    "name": "Grid Block",
     "icon": "Table",
     "styles": ["size"],
     "size": {

packages/client/package.json

@@ -37,7 +37,6 @@
     "downloadjs": "1.4.7",
     "html5-qrcode": "^2.2.1",
     "leaflet": "^1.7.1",
-    "regexparam": "^1.3.0",
     "sanitize-html": "^2.7.0",
     "screenfull": "^6.0.1",
     "shortid": "^2.2.15",

packages/client/src/api/api.js

@@ -77,4 +77,10 @@ export const API = createAPIClient({
     // Log all errors to console
     console.warn(`[Client] HTTP ${status} on ${method}:${url}\n\t${message}`)
   },
+  onMigrationDetected: _appId => {
+    if (!window.MIGRATING_APP) {
+      // We will force a reload, that will display the updating screen until the migration is running
+      window.location.reload()
+    }
+  },
 })

packages/client/src/components/UpdatingApp.svelte

@@ -0,0 +1,23 @@
+<script>
+  import { Updating } from "@budibase/frontend-core"
+  import { API } from "../api"
+
+  async function isMigrationDone() {
+    const response = await API.getMigrationStatus()
+    return response.migrated
+  }
+
+  async function onMigrationDone() {
+    window.location.reload()
+  }
+</script>
+
+<div class="updating">
+  <Updating {isMigrationDone} {onMigrationDone} />
+</div>
+
+<style>
+  .updating {
+    font-family: var(--font-sans);
+  }
+</style>

packages/client/src/components/app/blocks/MultiStepFormblock.svelte

@@ -0,0 +1,204 @@
+<script>
+  import BlockComponent from "components/BlockComponent.svelte"
+  import { getContext, setContext } from "svelte"
+  import { builderStore } from "stores"
+  import { Utils } from "@budibase/frontend-core"
+  import FormBlockWrapper from "./form/FormBlockWrapper.svelte"
+  import { writable } from "svelte/store"
+
+  export let actionType
+  export let rowId
+  export let noRowsMessage
+  export let steps
+  export let dataSource
+
+  const { fetchDatasourceSchema } = getContext("sdk")
+  const component = getContext("component")
+  const context = getContext("context")
+
+  // Set current step context to force child form to use it
+  const currentStep = writable(1)
+  setContext("current-step", currentStep)
+
+  const FieldTypeToComponentMap = {
+    string: "stringfield",
+    number: "numberfield",
+    bigint: "bigintfield",
+    options: "optionsfield",
+    array: "multifieldselect",
+    boolean: "booleanfield",
+    longform: "longformfield",
+    datetime: "datetimefield",
+    attachment: "attachmentfield",
+    link: "relationshipfield",
+    json: "jsonfield",
+    barcodeqr: "codescanner",
+    bb_reference: "bbreferencefield",
+  }
+
+  let schema
+
+  $: fetchSchema(dataSource)
+  $: enrichedSteps = enrichSteps(steps, schema, $component.id)
+  $: updateCurrentStep(enrichedSteps, $builderStore, $component)
+
+  const updateCurrentStep = (steps, builderStore, component) => {
+    const { componentId, step } = builderStore.metadata || {}
+
+    // If we aren't in the builder or aren't selected then don't update the step
+    // context at all, allowing the normal form to take control.
+    if (
+      !component.selected ||
+      !builderStore.inBuilder ||
+      componentId !== component.id
+    ) {
+      return
+    }
+
+    // Ensure we have a valid step selected
+    let newStep = Math.min(step || 0, steps.length - 1)
+
+    // Sanity check
+    newStep = Math.max(newStep, 0)
+
+    // Add 1 because the form component expects 1 indexed rather than 0 indexed
+    currentStep.set(newStep + 1)
+  }
+
+  const getPropsForField = field => {
+    if (field._component) {
+      return field
+    }
+    return {
+      field: field.name,
+      label: field.name,
+      placeholder: field.name,
+      _instanceName: field.name,
+    }
+  }
+
+  const getComponentForField = field => {
+    const fieldSchemaName = field.field || field.name
+    if (!fieldSchemaName || !schema?.[fieldSchemaName]) {
+      return null
+    }
+    const type = schema[fieldSchemaName].type
+    return FieldTypeToComponentMap[type]
+  }
+
+  const fetchSchema = async () => {
+    schema = (await fetchDatasourceSchema(dataSource)) || {}
+  }
+
+  const getDefaultFields = (fields, schema) => {
+    if (fields?.length) {
+      return fields.filter(field => field.active)
+    }
+    return Object.values(schema || {})
+      .filter(field => !field.autocolumn)
+      .map(field => ({
+        name: field.name,
+      }))
+  }
+
+  const enrichSteps = (steps, schema, id) => {
+    const safeSteps = steps?.length ? steps : [{}]
+    return safeSteps.map((step, idx) => {
+      const { title, desc, fields, buttons } = step
+      const defaultProps = Utils.buildMultiStepFormBlockDefaultProps({
+        _id: id,
+        stepCount: safeSteps.length,
+        currentStep: idx,
+        actionType,
+        dataSource,
+      })
+      return {
+        fields: getDefaultFields(fields || [], schema),
+        title: title ?? defaultProps.title,
+        desc,
+        buttons: buttons || defaultProps.buttons,
+      }
+    })
+  }
+</script>
+
+<FormBlockWrapper {actionType} {dataSource} {rowId} {noRowsMessage}>
+  <BlockComponent
+    type="form"
+    context="form"
+    props={{
+      dataSource,
+      actionType: actionType === "Create" ? "Create" : "Update",
+      readonly: actionType === "View",
+    }}
+    styles={{
+      normal: {
+        width: "600px",
+        "margin-left": "auto",
+        "margin-right": "auto",
+      },
+    }}
+  >
+    {#each enrichedSteps as step, stepIdx}
+      <BlockComponent
+        type="formstep"
+        props={{ step: stepIdx + 1, _instanceName: `Step ${stepIdx + 1}` }}
+      >
+        <BlockComponent
+          type="container"
+          props={{
+            gap: "M",
+            direction: "column",
+            hAlign: "stretch",
+            vAlign: "top",
+            size: "shrink",
+          }}
+        >
+          <BlockComponent type="container" order={0}>
+            <BlockComponent type="heading" props={{ text: step.title }} />
+          </BlockComponent>
+          <BlockComponent type="text" props={{ text: step.desc }} order={1} />
+          <BlockComponent type="container" order={2}>
+            <div
+              class="form-block fields"
+              class:mobile={$context.device.mobile}
+            >
+              {#each step.fields as field, fieldIdx (`${field.field || field.name}_${stepIdx}_${fieldIdx}`)}
+                {#if getComponentForField(field)}
+                  <BlockComponent
+                    type={getComponentForField(field)}
+                    props={getPropsForField(field)}
+                    order={fieldIdx}
+                    interactive
+                    name={field.field}
+                  />
+                {/if}
+              {/each}
+            </div>
+          </BlockComponent>
+          <BlockComponent
+            type="buttongroup"
+            props={{ buttons: step.buttons }}
+            styles={{
+              normal: {
+                "margin-top": "16px",
+              },
+            }}
+            order={3}
+          />
+        </BlockComponent>
+      </BlockComponent>
+    {/each}
+  </BlockComponent>
+</FormBlockWrapper>
+
+<style>
+  .fields {
+    display: grid;
+    grid-template-columns: repeat(6, 1fr);
+    gap: 8px 16px;
+  }
+  .fields.mobile :global(.spectrum-Form-item) {
+    grid-column: span 6 !important;
+  }
+</style>

packages/client/src/components/app/blocks/TableBlock.svelte

@@ -265,7 +265,7 @@
             props={{
               dataSource,
               buttonPosition: "top",
-              buttons: Utils.buildDynamicButtonConfig({
+              buttons: Utils.buildFormBlockButtonConfig({
                 _id: $component.id + "-form-edit",
                 showDeleteButton: deleteLabel !== "",
                 showSaveButton: true,
@@ -299,7 +299,7 @@
             props={{
               dataSource,
               buttonPosition: "top",
-              buttons: Utils.buildDynamicButtonConfig({
+              buttons: Utils.buildFormBlockButtonConfig({
                 _id: $component.id + "-form-new",
                 showDeleteButton: false,
                 showSaveButton: true,

packages/client/src/components/app/blocks/form/FormBlock.svelte

@@ -1,10 +1,8 @@
 <script>
   import { getContext } from "svelte"
-  import BlockComponent from "components/BlockComponent.svelte"
-  import Block from "components/Block.svelte"
-  import { makePropSafe as safe } from "@budibase/string-templates"
   import InnerFormBlock from "./InnerFormBlock.svelte"
   import { Utils } from "@budibase/frontend-core"
+  import FormBlockWrapper from "./FormBlockWrapper.svelte"
 
   export let actionType
   export let dataSource
@@ -71,22 +69,10 @@
   }
 
   let schema
-  let providerId
-  let repeaterId
 
   $: formattedFields = convertOldFieldFormat(fields)
   $: fieldsOrDefault = getDefaultFields(formattedFields, schema)
   $: fetchSchema(dataSource)
-  $: dataProvider = `{{ literal ${safe(providerId)} }}`
-  $: filter = [
-    {
-      field: "_id",
-      operator: "equal",
-      type: "string",
-      value: !rowId ? `{{ ${safe("url")}.${safe("id")} }}` : rowId,
-      valueType: "Binding",
-    },
-  ]
   // We could simply spread $$props into the inner form and append our
   // additions, but that would create svelte warnings about unused props and
   // make maintenance in future more confusing as we typically always have a
@@ -102,11 +88,10 @@
     title,
     description,
     schema,
-    repeaterId,
     notificationOverride,
     buttons:
       buttons ||
-      Utils.buildDynamicButtonConfig({
+      Utils.buildFormBlockButtonConfig({
         _id: $component.id,
         showDeleteButton,
         showSaveButton,
@@ -124,43 +109,6 @@
   }
 </script>
 
-<Block>
+<FormBlockWrapper {actionType} {dataSource} {rowId} {noRowsMessage}>
-  {#if actionType === "Create"}
+  <InnerFormBlock {...innerProps} />
-    <BlockComponent
+</FormBlockWrapper>
-      type="container"
-      props={{
-        direction: "column",
-        hAlign: "left",
-        vAlign: "stretch",
-      }}
-    >
-      <InnerFormBlock {...innerProps} />
-    </BlockComponent>
-  {:else}
-    <BlockComponent
-      type="dataprovider"
-      context="provider"
-      bind:id={providerId}
-      props={{
-        dataSource,
-        filter,
-        limit: 1,
-        paginate: false,
-      }}
-    >
-      <BlockComponent
-        type="repeater"
-        context="repeater"
-        bind:id={repeaterId}
-        props={{
-          dataProvider,
-          noRowsMessage: noRowsMessage || "We couldn't find a row to display",
-          direction: "column",
-          hAlign: "center",
-        }}
-      >
-        <InnerFormBlock {...innerProps} />
-      </BlockComponent>
-    </BlockComponent>
-  {/if}
-</Block>

packages/client/src/components/app/blocks/form/FormBlockWrapper.svelte

@@ -0,0 +1,64 @@
+<script>
+  import BlockComponent from "components/BlockComponent.svelte"
+  import Block from "components/Block.svelte"
+  import { makePropSafe as safe } from "@budibase/string-templates"
+  import { getContext } from "svelte"
+
+  export let actionType
+  export let dataSource
+  export let rowId
+  export let noRowsMessage
+
+  const component = getContext("component")
+
+  $: providerId = `${$component.id}-provider`
+  $: dataProvider = `{{ literal ${safe(providerId)} }}`
+  $: filter = [
+    {
+      field: "_id",
+      operator: "equal",
+      type: "string",
+      value: !rowId ? `{{ ${safe("url")}.${safe("id")} }}` : rowId,
+      valueType: "Binding",
+    },
+  ]
+</script>
+
+<Block>
+  {#if actionType === "Create"}
+    <BlockComponent
+      type="container"
+      props={{
+        direction: "column",
+        hAlign: "left",
+        vAlign: "stretch",
+      }}
+    >
+      <slot />
+    </BlockComponent>
+  {:else}
+    <BlockComponent
+      type="dataprovider"
+      context="provider"
+      props={{
+        dataSource,
+        filter,
+        limit: 1,
+        paginate: false,
+      }}
+    >
+      <BlockComponent
+        type="repeater"
+        context="repeater"
+        props={{
+          dataProvider,
+          noRowsMessage: noRowsMessage || "We couldn't find a row to display",
+          direction: "column",
+          hAlign: "center",
+        }}
+      >
+        <slot />
+      </BlockComponent>
+    </BlockComponent>
+  {/if}
+</Block>

packages/client/src/components/app/blocks/index.js

@@ -4,3 +4,4 @@ export { default as repeaterblock } from "./RepeaterBlock.svelte"
 export { default as formblock } from "./form/FormBlock.svelte"
 export { default as chartblock } from "./ChartBlock.svelte"
 export { default as rowexplorer } from "./RowExplorer.svelte"
+export { default as multistepformblock } from "./MultiStepFormblock.svelte"

packages/client/src/components/app/forms/Field.svelte

@@ -137,21 +137,23 @@
     width: 100%;
   }
 
+  .error :global(svg),
+  .helpText :global(svg) {
+    width: 13px;
+    margin-right: 6px;
+  }
+
   .error {
     display: flex;
     margin-top: var(--spectrum-global-dimension-size-75);
     align-items: center;
   }
-
   .error :global(svg) {
-    width: 14px;
     color: var(
       --spectrum-semantic-negative-color-default,
       var(--spectrum-global-color-red-500)
     );
-    margin-right: 4px;
   }
-
   .error span {
     color: var(
       --spectrum-semantic-negative-color-default,
@@ -165,17 +167,14 @@
     margin-top: var(--spectrum-global-dimension-size-75);
     align-items: center;
   }
-
   .helpText :global(svg) {
-    width: 14px;
+    color: var(--spectrum-global-color-gray-600);
-    color: var(--grey-7);
-    margin-right: 6px;
   }
-
   .helpText span {
-    color: var(--grey-5);
+    color: var(--spectrum-global-color-gray-800);
     font-size: var(--spectrum-global-dimension-font-size-75);
   }
+
   .spectrum-FieldLabel--right,
   .spectrum-FieldLabel--left {
     padding-right: var(--spectrum-global-dimension-size-200);

packages/client/src/components/app/forms/Form.svelte

@@ -34,7 +34,7 @@
   let loaded = false
   let schema
   let table
-  let currentStep = writable(getInitialFormStep())
+  let currentStep = getContext("current-step") || writable(getInitialFormStep())
 
   $: fetchSchema(dataSource)
   $: schemaKey = generateSchemaKey(schema)

packages/client/src/components/app/forms/InnerForm.svelte

@@ -423,10 +423,14 @@
     }
     const fieldId = field.fieldState.fieldId
     const fieldElement = document.getElementById(fieldId)
-    fieldElement.focus({ preventScroll: true })
+    if (fieldElement) {
+      fieldElement.focus({ preventScroll: true })
+    }
     const label = document.querySelector(`label[for="${fieldId}"]`)
-    label.style.scrollMargin = "100px"
+    if (label) {
-    label.scrollIntoView({ behavior: "smooth", block: "nearest" })
+      label.style.scrollMargin = "100px"
+      label.scrollIntoView({ behavior: "smooth", block: "nearest" })
+    }
   }
 
   // Action context to pass to children

packages/client/src/components/preview/HoverIndicator.svelte

@@ -1,9 +1,9 @@
 <script>
   import { onMount, onDestroy } from "svelte"
   import IndicatorSet from "./IndicatorSet.svelte"
-  import { builderStore, dndIsDragging } from "stores"
+  import { builderStore, dndIsDragging, hoverStore } from "stores"
 
-  $: componentId = $builderStore.hoverComponentId
+  $: componentId = $hoverStore.hoveredComponentId
   $: zIndex = componentId === $builderStore.selectedComponentId ? 900 : 920
 
   const onMouseOver = e => {
@@ -23,12 +23,12 @@
     }
 
     if (newId !== componentId) {
-      builderStore.actions.hoverComponent(newId)
+      hoverStore.actions.hoverComponent(newId)
     }
   }
 
   const onMouseLeave = () => {
-    builderStore.actions.hoverComponent(null)
+    hoverStore.actions.hoverComponent(null)
   }
 
   onMount(() => {

packages/client/src/index.js

@@ -1,4 +1,5 @@
 import ClientApp from "./components/ClientApp.svelte"
+import UpdatingApp from "./components/UpdatingApp.svelte"
 import {
   builderStore,
   appStore,
@@ -7,6 +8,7 @@ import {
   environmentStore,
   dndStore,
   eventStore,
+  hoverStore,
 } from "./stores"
 import loadSpectrumIcons from "@budibase/bbui/spectrum-icons-rollup.js"
 import { get } from "svelte/store"
@@ -32,7 +34,6 @@ const loadBudibase = async () => {
     layout: window["##BUDIBASE_PREVIEW_LAYOUT##"],
     screen: window["##BUDIBASE_PREVIEW_SCREEN##"],
     selectedComponentId: window["##BUDIBASE_SELECTED_COMPONENT_ID##"],
-    hoverComponentId: window["##BUDIBASE_HOVER_COMPONENT_ID##"],
     previewId: window["##BUDIBASE_PREVIEW_ID##"],
     theme: window["##BUDIBASE_PREVIEW_THEME##"],
     customTheme: window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"],
@@ -52,6 +53,13 @@ const loadBudibase = async () => {
     window["##BUDIBASE_APP_EMBEDDED##"] === "true"
   )
 
+  if (window.MIGRATING_APP) {
+    new UpdatingApp({
+      target: window.document.body,
+    })
+    return
+  }
+
   // Fetch environment info
   if (!get(environmentStore)?.loaded) {
     await environmentStore.actions.fetchEnvironment()
@@ -76,6 +84,10 @@ const loadBudibase = async () => {
       } else {
         dndStore.actions.reset()
       }
+    } else if (type === "hover-component") {
+      hoverStore.actions.hoverComponent(data)
+    } else if (type === "builder-meta") {
+      builderStore.actions.setMetadata(data)
     }
   }
 

packages/client/src/stores/builder.js

@@ -8,7 +8,6 @@ const createBuilderStore = () => {
     inBuilder: false,
     screen: null,
     selectedComponentId: null,
-    hoverComponentId: null,
     editMode: false,
     previewId: null,
     theme: null,
@@ -18,22 +17,13 @@ const createBuilderStore = () => {
     hiddenComponentIds: [],
     usedPlugins: null,
     eventResolvers: {},
+    metadata: null,
 
     // Legacy - allow the builder to specify a layout
     layout: null,
   }
   const store = writable(initialState)
   const actions = {
-    hoverComponent: id => {
-      if (id === get(store).hoverComponentId) {
-        return
-      }
-      store.update(state => ({
-        ...state,
-        hoverComponentId: id,
-      }))
-      eventStore.actions.dispatchEvent("hover-component", { id })
-    },
     selectComponent: id => {
       if (id === get(store).selectedComponentId) {
         return
@@ -123,6 +113,12 @@ const createBuilderStore = () => {
         parentType,
       })
     },
+    setMetadata: metadata => {
+      store.update(state => ({
+        ...state,
+        metadata,
+      }))
+    },
   }
   return {
     ...store,

packages/client/src/stores/hover.js

@@ -0,0 +1,25 @@
+import { get, writable } from "svelte/store"
+import { eventStore } from "./events.js"
+
+const createHoverStore = () => {
+  const store = writable({
+    hoveredComponentId: null,
+  })
+
+  const hoverComponent = id => {
+    if (id === get(store).hoveredComponentId) {
+      return
+    }
+    store.set({ hoveredComponentId: id })
+    eventStore.actions.dispatchEvent("hover-component", { id })
+  }
+
+  return {
+    ...store,
+    actions: {
+      hoverComponent,
+    },
+  }
+}
+
+export const hoverStore = createHoverStore()

packages/client/src/stores/index.js

@@ -27,6 +27,7 @@ export {
   dndIsDragging,
 } from "./dnd"
 export { sidePanelStore } from "./sidePanel"
+export { hoverStore } from "./hover"
 
 // Context stores are layered and duplicated, so it is not a singleton
 export { createContextStore } from "./context"

packages/frontend-core/src/api/index.js

@@ -33,6 +33,7 @@ import { buildEnvironmentVariableEndpoints } from "./environmentVariables"
 import { buildEventEndpoints } from "./events"
 import { buildAuditLogsEndpoints } from "./auditLogs"
 import { buildLogsEndpoints } from "./logs"
+import { buildMigrationEndpoints } from "./migrations"
 
 /**
  * Random identifier to uniquely identify a session in a tab. This is
@@ -298,6 +299,7 @@ export const createAPIClient = config => {
     ...buildEventEndpoints(API),
     ...buildAuditLogsEndpoints(API),
     ...buildLogsEndpoints(API),
+    ...buildMigrationEndpoints(API),
     viewV2: buildViewV2Endpoints(API),
   }
 }

packages/frontend-core/src/api/migrations.js

@@ -0,0 +1,10 @@
+export const buildMigrationEndpoints = API => ({
+  /**
+   * Gets the info about the current app migration
+   */
+  getMigrationStatus: async () => {
+    return await API.get({
+      url: "/api/migrations/status",
+    })
+  },
+})

packages/frontend-core/src/components/Updating.svelte

@@ -0,0 +1,79 @@
+<script>
+  export let isMigrationDone
+  export let onMigrationDone
+  export let timeoutSeconds = 10 // 3 minutes
+
+  const loadTime = Date.now()
+  let timedOut = false
+
+  async function checkMigrationsFinished() {
+    setTimeout(async () => {
+      const isMigrated = await isMigrationDone()
+
+      const timeoutMs = timeoutSeconds * 1000
+      if (!isMigrated) {
+        if (loadTime + timeoutMs > Date.now()) {
+          return checkMigrationsFinished()
+        }
+
+        return migrationTimeout()
+      }
+
+      onMigrationDone()
+    }, 1000)
+  }
+
+  checkMigrationsFinished()
+
+  function migrationTimeout() {
+    timedOut = true
+  }
+</script>
+
+<div class="loading" class:timeout={timedOut}>
+  <span class="header">
+    {#if !timedOut}
+      System update
+    {:else}
+      Something went wrong!
+    {/if}
+  </span>
+  <span class="subtext">
+    {#if !timedOut}
+      Please wait and we will be back in a second!
+    {:else}
+      An error occurred, please try again later.
+      <br />
+      Contact
+      <a href="https://budibase.com/support/" target="_blank">support</a> if the
+      issue persists.
+    {/if}</span
+  >
+</div>
+
+<style>
+  .loading {
+    display: flex;
+    justify-content: center;
+    flex-direction: column;
+    gap: var(--spacing-l);
+    height: 100vh;
+    text-align: center;
+    font-size: 18px;
+  }
+  .header {
+    font-weight: 700;
+  }
+  .timeout .header {
+    color: rgb(196, 46, 46);
+  }
+  .subtext {
+    font-size: 16px;
+    color: var(--grey-7);
+  }
+
+  .subtext a {
+    color: var(--grey-7);
+    font-weight: 700;
+  }
+</style>

packages/frontend-core/src/components/index.js

@@ -3,4 +3,5 @@ export { default as TestimonialPage } from "./TestimonialPage.svelte"
 export { default as Testimonial } from "./Testimonial.svelte"
 export { default as UserAvatar } from "./UserAvatar.svelte"
 export { default as UserAvatars } from "./UserAvatars.svelte"
+export { default as Updating } from "./Updating.svelte"
 export { Grid } from "./grid"

packages/frontend-core/src/utils/utils.js

@@ -116,7 +116,7 @@ export const domDebounce = callback => {
  *
  * @param {any} props
  * */
-export const buildDynamicButtonConfig = props => {
+export const buildFormBlockButtonConfig = props => {
   const {
     _id,
     actionType,
@@ -130,7 +130,6 @@ export const buildDynamicButtonConfig = props => {
   } = props || {}
 
   if (!_id) {
-    console.log("MISSING ID")
     return
   }
   const formId = `${_id}-form`
@@ -228,7 +227,7 @@ export const buildDynamicButtonConfig = props => {
     })
   }
 
-  if (actionType == "Update" && showDeleteButton !== false) {
+  if (actionType === "Update" && showDeleteButton !== false) {
     defaultButtons.push({
       text: deleteText || "Delete",
       _id: Helpers.uuid(),
@@ -241,3 +240,108 @@ export const buildDynamicButtonConfig = props => {
 
   return defaultButtons
 }
+
+export const buildMultiStepFormBlockDefaultProps = props => {
+  const { _id, stepCount, currentStep, actionType, dataSource } = props || {}
+
+  // Sanity check
+  if (!_id || !stepCount) {
+    return
+  }
+
+  const title = `Step {{ [${_id}-form].[__currentStep] }}`
+  const resourceId = dataSource?.resourceId
+  const formId = `${_id}-form`
+  let buttons = []
+
+  // Add previous step button if we aren't the first step
+  if (currentStep !== 0) {
+    buttons.push({
+      _id: Helpers.uuid(),
+      _component: "@budibase/standard-components/button",
+      _instanceName: Helpers.uuid(),
+      text: "Back",
+      type: "secondary",
+      size: "M",
+      onClick: [
+        {
+          parameters: {
+            type: "prev",
+            componentId: formId,
+          },
+          "##eventHandlerType": "Change Form Step",
+        },
+      ],
+    })
+  }
+
+  // Add a next button if we aren't the last step
+  if (currentStep !== stepCount - 1) {
+    buttons.push({
+      _id: Helpers.uuid(),
+      _component: "@budibase/standard-components/button",
+      _instanceName: Helpers.uuid(),
+      text: "Next",
+      type: "cta",
+      size: "M",
+      onClick: [
+        {
+          "##eventHandlerType": "Validate Form",
+          parameters: {
+            componentId: formId,
+          },
+        },
+        {
+          parameters: {
+            type: "next",
+            componentId: formId,
+          },
+          "##eventHandlerType": "Change Form Step",
+        },
+      ],
+    })
+  }
+
+  // Add save button if we are the last step
+  if (actionType !== "View" && currentStep === stepCount - 1) {
+    buttons.push({
+      _id: Helpers.uuid(),
+      _component: "@budibase/standard-components/button",
+      _instanceName: Helpers.uuid(),
+      text: "Save",
+      type: "cta",
+      size: "M",
+      onClick: [
+        {
+          "##eventHandlerType": "Validate Form",
+          parameters: {
+            componentId: formId,
+          },
+        },
+        {
+          "##eventHandlerType": "Save Row",
+          parameters: {
+            tableId: resourceId,
+            providerId: formId,
+          },
+        },
+        // Clear a create form once submitted
+        ...(actionType !== "Create"
+          ? []
+          : [
+              {
+                "##eventHandlerType": "Clear Form",
+                parameters: {
+                  componentId: formId,
+                },
+              },
+            ]),
+      ],
+    })
+  }
+
+  return {
+    buttons,
+    title,
+  }
+}

packages/pro

@@ -1 +1 @@
-Subproject commit 82de3443fd03b272555d23c42ead3a611302277d
+Subproject commit a838419616a35923cef029f5264e71e3dbd22a55

packages/server/nodemon.json

@@ -7,7 +7,7 @@
     "../shared-core",
     "../string-templates"
   ],
-  "ext": "js,ts,json",
+  "ext": "js,ts,json,svelte",
   "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../*/dist/**/*"],
   "exec": "yarn build && node ./dist/index.js"
 }

packages/server/package.json

@@ -80,7 +80,7 @@
     "koa": "2.13.4",
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
-    "koa-send": "5.0.0",
+    "koa-send": "5.0.1",
     "koa-useragent": "^4.1.0",
     "koa2-ratelimit": "1.1.1",
     "lodash": "4.17.21",
@@ -120,6 +120,7 @@
     "@types/jest": "29.5.5",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
+    "@types/koa-send": "^4.1.6",
     "@types/lodash": "4.14.200",
     "@types/mssql": "9.1.4",
     "@types/node-fetch": "2.6.4",

packages/server/src/api/controllers/datasource.ts

@@ -1,23 +1,17 @@
-import {
+import { getQueryParams, getTableParams } from "../../db/utils"
-  DocumentType,
-  generateDatasourceID,
-  getQueryParams,
-  getTableParams,
-} from "../../db/utils"
 import { getIntegration } from "../../integrations"
 import { invalidateDynamicVariables } from "../../threads/utils"
 import { context, db as dbCore, events } from "@budibase/backend-core"
 import {
+  BuildSchemaFromSourceRequest,
+  BuildSchemaFromSourceResponse,
   CreateDatasourceRequest,
   CreateDatasourceResponse,
   Datasource,
   DatasourcePlus,
   FetchDatasourceInfoRequest,
   FetchDatasourceInfoResponse,
-  IntegrationBase,
-  Schema,
   SourceName,
-  Table,
   UpdateDatasourceResponse,
   UserCtx,
   VerifyDatasourceRequest,
@@ -25,68 +19,8 @@ import {
 } from "@budibase/types"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
-import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets"
 import { isEqual } from "lodash"
 
-async function getConnector(
-  datasource: Datasource
-): Promise<IntegrationBase | DatasourcePlus> {
-  const Connector = await getIntegration(datasource.source)
-  // can't enrich if it doesn't have an ID yet
-  if (datasource._id) {
-    datasource = await sdk.datasources.enrich(datasource)
-  }
-  // Connect to the DB and build the schema
-  return new Connector(datasource.config)
-}
-
-async function getAndMergeDatasource(datasource: Datasource) {
-  let existingDatasource: undefined | Datasource
-  if (datasource._id) {
-    existingDatasource = await sdk.datasources.get(datasource._id)
-  }
-  let enrichedDatasource = datasource
-  if (existingDatasource) {
-    enrichedDatasource = sdk.datasources.mergeConfigs(
-      datasource,
-      existingDatasource
-    )
-  }
-  return await sdk.datasources.enrich(enrichedDatasource)
-}
-
-async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
-  const connector = (await getConnector(datasource)) as DatasourcePlus
-  return await connector.buildSchema(
-    datasource._id!,
-    datasource.entities! as Record<string, Table>
-  )
-}
-
-async function buildFilteredSchema(
-  datasource: Datasource,
-  filter?: string[]
-): Promise<Schema> {
-  let schema = await buildSchemaHelper(datasource)
-  if (!filter) {
-    return schema
-  }
-
-  let filteredSchema: Schema = { tables: {}, errors: {} }
-  for (let key in schema.tables) {
-    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
-      filteredSchema.tables[key] = schema.tables[key]
-    }
-  }
-
-  for (let key in schema.errors) {
-    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
-      filteredSchema.errors[key] = schema.errors[key]
-    }
-  }
-  return filteredSchema
-}
-
 export async function fetch(ctx: UserCtx) {
   ctx.body = await sdk.datasources.fetch()
 }
@@ -95,8 +29,10 @@ export async function verify(
   ctx: UserCtx<VerifyDatasourceRequest, VerifyDatasourceResponse>
 ) {
   const { datasource } = ctx.request.body
-  const enrichedDatasource = await getAndMergeDatasource(datasource)
+  const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(
-  const connector = await getConnector(enrichedDatasource)
+    datasource
+  )
+  const connector = await sdk.datasources.getConnector(enrichedDatasource)
   if (!connector.testConnection) {
     ctx.throw(400, "Connection information verification not supported")
   }
@@ -112,8 +48,12 @@ export async function information(
   ctx: UserCtx<FetchDatasourceInfoRequest, FetchDatasourceInfoResponse>
 ) {
   const { datasource } = ctx.request.body
-  const enrichedDatasource = await getAndMergeDatasource(datasource)
+  const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(
-  const connector = (await getConnector(enrichedDatasource)) as DatasourcePlus
+    datasource
+  )
+  const connector = (await sdk.datasources.getConnector(
+    enrichedDatasource
+  )) as DatasourcePlus
   if (!connector.getTableNames) {
     ctx.throw(400, "Table name fetching not supported by datasource")
   }
@@ -123,19 +63,16 @@ export async function information(
   }
 }
 
-export async function buildSchemaFromDb(ctx: UserCtx) {
+export async function buildSchemaFromSource(
-  const db = context.getAppDB()
+  ctx: UserCtx<BuildSchemaFromSourceRequest, BuildSchemaFromSourceResponse>
+) {
+  const datasourceId = ctx.params.datasourceId
   const tablesFilter = ctx.request.body.tablesFilter
-  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
 
-  const { tables, errors } = await buildFilteredSchema(datasource, tablesFilter)
+  const { datasource, errors } = await sdk.datasources.buildSchemaFromSource(
-  datasource.entities = tables
+    datasourceId,
-
+    tablesFilter
-  setDefaultDisplayColumns(datasource)
-  const dbResp = await db.put(
-    sdk.tables.populateExternalTableSchemas(datasource)
   )
-  datasource._rev = dbResp.rev
 
   ctx.body = {
     datasource: await sdk.datasources.removeSecretSingle(datasource),
@@ -143,24 +80,6 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
   }
 }
 
-/**
- * Make sure all datasource entities have a display name selected
- */
-function setDefaultDisplayColumns(datasource: Datasource) {
-  //
-  for (let entity of Object.values(datasource.entities || {})) {
-    if (entity.primaryDisplay) {
-      continue
-    }
-    const notAutoColumn = Object.values(entity.schema).find(
-      schema => !schema.autocolumn
-    )
-    if (notAutoColumn) {
-      entity.primaryDisplay = notAutoColumn.name
-    }
-  }
-}
-
 /**
  * Check for variables that have been updated or removed and invalidate them.
  */
@@ -258,51 +177,18 @@ export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
   }
 }
 
-const preSaveAction: Partial<Record<SourceName, any>> = {
-  [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => {
-    await googleSetupCreationAuth(datasource.config as any)
-  },
-}
-
 export async function save(
   ctx: UserCtx<CreateDatasourceRequest, CreateDatasourceResponse>
 ) {
-  const db = context.getAppDB()
+  const {
-  const plus = ctx.request.body.datasource.plus
+    datasource: datasourceData,
-  const fetchSchema = ctx.request.body.fetchSchema
+    fetchSchema,
-  const tablesFilter = ctx.request.body.tablesFilter
+    tablesFilter,
-
+  } = ctx.request.body
-  const datasource = {
+  const { datasource, errors } = await sdk.datasources.save(datasourceData, {
-    _id: generateDatasourceID({ plus }),
+    fetchSchema,
-    ...ctx.request.body.datasource,
+    tablesFilter,
-    type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE,
+  })
-  }
-
-  let errors: Record<string, string> = {}
-  if (fetchSchema) {
-    const schema = await buildFilteredSchema(datasource, tablesFilter)
-    datasource.entities = schema.tables
-    setDefaultDisplayColumns(datasource)
-    errors = schema.errors
-  }
-
-  if (preSaveAction[datasource.source]) {
-    await preSaveAction[datasource.source](datasource)
-  }
-
-  const dbResp = await db.put(
-    sdk.tables.populateExternalTableSchemas(datasource)
-  )
-  await events.datasource.created(datasource)
-  datasource._rev = dbResp.rev
-
-  // Drain connection pools when configuration is changed
-  if (datasource.source) {
-    const source = await getIntegration(datasource.source)
-    if (source && source.pool) {
-      await source.pool.end()
-    }
-  }
 
   ctx.body = {
     datasource: await sdk.datasources.removeSecretSingle(datasource),
@@ -384,8 +270,10 @@ export async function query(ctx: UserCtx) {
 
 export async function getExternalSchema(ctx: UserCtx) {
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
-  const enrichedDatasource = await getAndMergeDatasource(datasource)
+  const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(
-  const connector = await getConnector(enrichedDatasource)
+    datasource
+  )
+  const connector = await sdk.datasources.getConnector(enrichedDatasource)
 
   if (!connector.getExternalSchema) {
     ctx.throw(400, "Datasource does not support exporting external schema")

packages/server/src/api/controllers/query/index.ts

@@ -161,11 +161,8 @@ export async function preview(ctx: UserCtx) {
         auth: { ...authConfigCtx },
       },
     }
-    const runFn = () => Runner.run(inputs)
 
-    const { rows, keys, info, extra } = await quotas.addQuery<any>(runFn, {
+    const { rows, keys, info, extra } = (await Runner.run(inputs)) as any
-      datasourceId: datasource._id,
-    })
     const schemaFields: any = {}
     if (rows?.length > 0) {
       for (let key of [...new Set(keys)] as string[]) {
@@ -259,14 +256,8 @@ async function execute(
       },
       schema: query.schema,
     }
-    const runFn = () => Runner.run(inputs)
 
-    const { rows, pagination, extra, info } = await quotas.addQuery<any>(
+    const { rows, pagination, extra, info } = (await Runner.run(inputs)) as any
-      runFn,
-      {
-        datasourceId: datasource._id,
-      }
-    )
     // remove the raw from execution incase transformer being used to hide data
     if (extra?.raw) {
       delete extra.raw

packages/server/src/api/controllers/row/index.ts

@@ -4,20 +4,20 @@ import * as external from "./external"
 import { isExternalTableID } from "../../../integrations/utils"
 import {
   Ctx,
-  UserCtx,
-  DeleteRowRequest,
   DeleteRow,
+  DeleteRowRequest,
   DeleteRows,
-  Row,
-  PatchRowRequest,
-  PatchRowResponse,
-  SearchRowResponse,
-  SearchRowRequest,
-  SearchParams,
-  GetRowResponse,
-  ValidateResponse,
   ExportRowsRequest,
   ExportRowsResponse,
+  GetRowResponse,
+  PatchRowRequest,
+  PatchRowResponse,
+  Row,
+  SearchParams,
+  SearchRowRequest,
+  SearchRowResponse,
+  UserCtx,
+  ValidateResponse,
 } from "@budibase/types"
 import * as utils from "./utils"
 import { gridSocket } from "../../../websockets"
@@ -25,8 +25,8 @@ import { addRev } from "../public/utils"
 import { fixRow } from "../public/rows"
 import sdk from "../../../sdk"
 import * as exporters from "../view/exporters"
-import { apiFileReturn } from "../../../utilities/fileSystem"
 import { Format } from "../view/exporters"
+import { apiFileReturn } from "../../../utilities/fileSystem"
 
 export * as views from "./views"
 
@@ -49,12 +49,7 @@ export async function patch(
     return save(ctx)
   }
   try {
-    const { row, table } = await quotas.addQuery(
+    const { row, table } = await pickApi(tableId).patch(ctx)
-      () => pickApi(tableId).patch(ctx),
-      {
-        datasourceId: tableId,
-      }
-    )
     if (!row) {
       ctx.throw(404, "Row not found")
     }
@@ -84,12 +79,7 @@ export const save = async (ctx: UserCtx<Row, Row>) => {
     return patch(ctx as UserCtx<PatchRowRequest, PatchRowResponse>)
   }
   const { row, table, squashed } = await quotas.addRow(() =>
-    quotas.addQuery(
+    sdk.rows.save(tableId, ctx.request.body, ctx.user?._id)
-      () => sdk.rows.save(tableId, ctx.request.body, ctx.user?._id),
-      {
-        datasourceId: tableId,
-      }
-    )
   )
   ctx.status = 200
   ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:save`, appId, row, table)
@@ -105,31 +95,21 @@ export async function fetchView(ctx: any) {
 
   const { calculation, group, field } = ctx.query
 
-  ctx.body = await quotas.addQuery(
+  ctx.body = await sdk.rows.fetchView(tableId, viewName, {
-    () =>
+    calculation,
-      sdk.rows.fetchView(tableId, viewName, {
+    group: calculation ? group : null,
-        calculation,
+    field,
-        group: calculation ? group : null,
+  })
-        field,
-      }),
-    {
-      datasourceId: tableId,
-    }
-  )
 }
 
 export async function fetch(ctx: any) {
   const tableId = utils.getTableId(ctx)
-  ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId), {
+  ctx.body = await sdk.rows.fetch(tableId)
-    datasourceId: tableId,
-  })
 }
 
 export async function find(ctx: UserCtx<void, GetRowResponse>) {
   const tableId = utils.getTableId(ctx)
-  ctx.body = await quotas.addQuery(() => pickApi(tableId).find(ctx), {
+  ctx.body = await pickApi(tableId).find(ctx)
-    datasourceId: tableId,
-  })
 }
 
 function isDeleteRows(input: any): input is DeleteRows {
@@ -160,15 +140,9 @@ async function deleteRows(ctx: UserCtx<DeleteRowRequest>) {
 
   let deleteRequest = ctx.request.body as DeleteRows
 
-  const rowDeletes: Row[] = await processDeleteRowsRequest(ctx)
+  deleteRequest.rows = await processDeleteRowsRequest(ctx)
-  deleteRequest.rows = rowDeletes
 
-  const { rows } = await quotas.addQuery(
+  const { rows } = await pickApi(tableId).bulkDestroy(ctx)
-    () => pickApi(tableId).bulkDestroy(ctx),
-    {
-      datasourceId: tableId,
-    }
-  )
   await quotas.removeRows(rows.length)
 
   for (let row of rows) {
@@ -183,9 +157,7 @@ async function deleteRow(ctx: UserCtx<DeleteRowRequest>) {
   const appId = ctx.appId
   const tableId = utils.getTableId(ctx)
 
-  const resp = await quotas.addQuery(() => pickApi(tableId).destroy(ctx), {
+  const resp = await pickApi(tableId).destroy(ctx)
-    datasourceId: tableId,
-  })
   await quotas.removeRow()
 
   ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, resp.row)
@@ -223,9 +195,7 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
   }
 
   ctx.status = 200
-  ctx.body = await quotas.addQuery(() => sdk.rows.search(searchParams), {
+  ctx.body = await sdk.rows.search(searchParams)
-    datasourceId: tableId,
-  })
 }
 
 export async function validate(ctx: Ctx<Row, ValidateResponse>) {
@@ -243,12 +213,7 @@ export async function validate(ctx: Ctx<Row, ValidateResponse>) {
 
 export async function fetchEnrichedRow(ctx: any) {
   const tableId = utils.getTableId(ctx)
-  ctx.body = await quotas.addQuery(
+  ctx.body = await pickApi(tableId).fetchEnrichedRow(ctx)
-    () => pickApi(tableId).fetchEnrichedRow(ctx),
-    {
-      datasourceId: tableId,
-    }
-  )
 }
 
 export const exportRows = async (
@@ -268,22 +233,15 @@ export const exportRows = async (
     )
   }
 
-  ctx.body = await quotas.addQuery(
+  const { fileName, content } = await sdk.rows.exportRows({
-    async () => {
+    tableId,
-      const { fileName, content } = await sdk.rows.exportRows({
+    format: format as Format,
-        tableId,
+    rowIds: rows,
-        format: format as Format,
+    columns,
-        rowIds: rows,
+    query,
-        columns,
+    sort,
-        query,
+    sortOrder,
-        sort,
+  })
-        sortOrder,
+  ctx.attachment(fileName)
-      })
+  ctx.body = apiFileReturn(content)
-      ctx.attachment(fileName)
-      return apiFileReturn(content)
-    },
-    {
-      datasourceId: tableId,
-    }
-  )
 }

packages/server/src/api/controllers/row/views.ts

@@ -68,10 +68,7 @@ export async function searchView(
     paginate: body.paginate,
   }
 
-  const result = await quotas.addQuery(() => sdk.rows.search(searchOptions), {
+  const result = await sdk.rows.search(searchOptions)
-    datasourceId: view.tableId,
-  })
-
   result.rows.forEach(r => (r._viewId = view.id))
   ctx.body = result
 }

packages/server/src/api/controllers/static/index.ts

@@ -25,8 +25,12 @@ import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
 import { App, Ctx, ProcessAttachmentResponse } from "@budibase/types"
+import {
+  getAppMigrationVersion,
+  getLatestMigrationId,
+} from "../../../appMigrations"
 
-const send = require("koa-send")
+import send from "koa-send"
 
 export const toggleBetaUiFeature = async function (ctx: Ctx) {
   const cookieName = `beta:${ctx.params.feature}`
@@ -125,7 +129,26 @@ export const deleteObjects = async function (ctx: Ctx) {
   )
 }
 
+const requiresMigration = async (ctx: Ctx) => {
+  const appId = context.getAppId()
+  if (!appId) {
+    ctx.throw("AppId could not be found")
+  }
+
+  const latestMigration = getLatestMigrationId()
+  if (!latestMigration) {
+    return false
+  }
+
+  const latestMigrationApplied = await getAppMigrationVersion(appId)
+
+  const requiresMigrations = latestMigrationApplied !== latestMigration
+  return requiresMigrations
+}
+
 export const serveApp = async function (ctx: Ctx) {
+  const needMigrations = await requiresMigration(ctx)
+
   const bbHeaderEmbed =
     ctx.request.get("x-budibase-embed")?.toLowerCase() === "true"
 
@@ -145,8 +168,8 @@ export const serveApp = async function (ctx: Ctx) {
     let appId = context.getAppId()
 
     if (!env.isJest()) {
-      const App = require("./templates/BudibaseApp.svelte").default
       const plugins = objectStore.enrichPluginURLs(appInfo.usedPlugins)
+      const App = require("./templates/BudibaseApp.svelte").default
       const { head, html, css } = App.render({
         metaImage:
           branding?.metaImageUrl ||
@@ -167,6 +190,7 @@ export const serveApp = async function (ctx: Ctx) {
           config?.logoUrl !== ""
             ? objectStore.getGlobalFileUrl("settings", "logoUrl")
             : "",
+        appMigrating: needMigrations,
       })
       const appHbs = loadHandlebarsFile(appHbsPath)
       ctx.body = await processString(appHbs, {
@@ -273,7 +297,6 @@ export const getSignedUploadURL = async function (ctx: Ctx) {
     const { bucket, key } = ctx.request.body || {}
     if (!bucket || !key) {
       ctx.throw(400, "bucket and key values are required")
-      return
     }
     try {
       const s3 = new AWS.S3({

packages/server/src/api/controllers/static/templates/BudibaseApp.svelte

@@ -8,6 +8,7 @@
 
   export let clientLibPath
   export let usedPlugins
+  export let appMigrating
 </script>
 
 <svelte:head>
@@ -110,6 +111,11 @@
   <script type="application/javascript">
     window.INIT_TIME = Date.now()
   </script>
+  {#if appMigrating}
+    <script type="application/javascript">
+      window.MIGRATING_APP = true
+    </script>
+  {/if}
   <script type="application/javascript" src={clientLibPath}>
   </script>
   <!-- Custom components need inserted after the core client library -->

packages/server/src/api/controllers/static/templates/preview.hbs

@@ -63,7 +63,6 @@
       // Extract data from message
       const {
         selectedComponentId,
-        hoverComponentId,
         layout,
         screen,
         appId,
@@ -82,7 +81,6 @@
       window["##BUDIBASE_PREVIEW_LAYOUT##"] = layout
       window["##BUDIBASE_PREVIEW_SCREEN##"] = screen
       window["##BUDIBASE_SELECTED_COMPONENT_ID##"] = selectedComponentId
-      window["##BUDIBASE_HOVER_COMPONENT_ID##"] = hoverComponentId
       window["##BUDIBASE_PREVIEW_ID##"] = Math.random()
       window["##BUDIBASE_PREVIEW_THEME##"] = theme
       window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"] = customTheme

packages/server/src/api/routes/datasource.ts

@@ -53,7 +53,7 @@ router
   .post(
     "/api/datasources/:datasourceId/schema",
     authorized(permissions.BUILDER),
-    datasourceController.buildSchemaFromDb
+    datasourceController.buildSchemaFromSource
   )
   .post(
     "/api/datasources",

packages/server/src/api/routes/tests/row.spec.ts

@@ -132,11 +132,6 @@ describe.each([
     expect(usage).toBe(expected)
   }
 
-  const assertQueryUsage = async (expected: number) => {
-    const usage = await getQueryUsage()
-    expect(usage).toBe(expected)
-  }
-
   const defaultRowFields = isInternal
     ? {
         type: "row",
@@ -181,7 +176,6 @@ describe.each([
       expect(res.body.name).toEqual("Test Contact")
       expect(res.body._rev).toBeDefined()
       await assertRowUsage(rowUsage + 1)
-      await assertQueryUsage(queryUsage + 1)
     })
 
     it("Increment row autoId per create row request", async () => {
@@ -232,7 +226,6 @@ describe.each([
       }
 
       await assertRowUsage(rowUsage + ids.length)
-      await assertQueryUsage(queryUsage + ids.length)
     })
 
     it("updates a row successfully", async () => {
@@ -249,7 +242,6 @@ describe.each([
 
       expect(res.name).toEqual("Updated Name")
       await assertRowUsage(rowUsage)
-      await assertQueryUsage(queryUsage + 1)
     })
 
     it("should load a row", async () => {
@@ -262,7 +254,6 @@ describe.each([
         ...existing,
         ...defaultRowFields,
       })
-      await assertQueryUsage(queryUsage + 1)
     })
 
     it("should list all rows for given tableId", async () => {
@@ -284,7 +275,6 @@ describe.each([
       expect(res.length).toBe(2)
       expect(res.find((r: Row) => r.name === newRow.name)).toBeDefined()
       expect(res.find((r: Row) => r.name === firstRow.name)).toBeDefined()
-      await assertQueryUsage(queryUsage + 1)
     })
 
     it("load should return 404 when row does not exist", async () => {
@@ -294,7 +284,6 @@ describe.each([
       await config.api.row.get(tableId, "1234567", {
         expectStatus: 404,
       })
-      await assertQueryUsage(queryUsage) // no change
     })
 
     isInternal &&
@@ -558,7 +547,6 @@ describe.each([
       expect(savedRow.body.description).toEqual(existing.description)
       expect(savedRow.body.name).toEqual("Updated Name")
       await assertRowUsage(rowUsage)
-      await assertQueryUsage(queryUsage + 2) // account for the second load
     })
 
     it("should throw an error when given improper types", async () => {
@@ -578,7 +566,6 @@ describe.each([
       )
 
       await assertRowUsage(rowUsage)
-      await assertQueryUsage(queryUsage)
     })
 
     it("should not overwrite links if those links are not set", async () => {
@@ -668,7 +655,6 @@ describe.each([
       const res = await config.api.row.delete(table._id!, [createdRow])
       expect(res.body[0]._id).toEqual(createdRow._id)
       await assertRowUsage(rowUsage - 1)
-      await assertQueryUsage(queryUsage + 1)
     })
   })
 
@@ -687,7 +673,6 @@ describe.each([
       expect(res.valid).toBe(true)
       expect(Object.keys(res.errors)).toEqual([])
       await assertRowUsage(rowUsage)
-      await assertQueryUsage(queryUsage)
     })
 
     it("should errors on invalid row", async () => {
@@ -705,7 +690,6 @@ describe.each([
         expect(Object.keys(res.errors)).toEqual([])
       }
       await assertRowUsage(rowUsage)
-      await assertQueryUsage(queryUsage)
     })
   })
 
@@ -726,7 +710,6 @@ describe.each([
       expect(res.body.length).toEqual(2)
       await loadRow(row1._id!, table._id!, 404)
       await assertRowUsage(rowUsage - 2)
-      await assertQueryUsage(queryUsage + 1)
     })
 
     it("should be able to delete a variety of row set types", async () => {
@@ -747,7 +730,6 @@ describe.each([
       expect(res.body.length).toEqual(3)
       await loadRow(row1._id!, table._id!, 404)
       await assertRowUsage(rowUsage - 3)
-      await assertQueryUsage(queryUsage + 1)
     })
 
     it("should accept a valid row object and delete the row", async () => {
@@ -760,7 +742,6 @@ describe.each([
       expect(res.body.id).toEqual(row1._id)
       await loadRow(row1._id!, table._id!, 404)
       await assertRowUsage(rowUsage - 1)
-      await assertQueryUsage(queryUsage + 1)
     })
 
     it("Should ignore malformed/invalid delete requests", async () => {
@@ -787,7 +768,6 @@ describe.each([
       expect(res3.body.message).toEqual("Invalid delete rows request")
 
       await assertRowUsage(rowUsage)
-      await assertQueryUsage(queryUsage)
     })
   })
 
@@ -808,7 +788,6 @@ describe.each([
         expect(res.body.length).toEqual(1)
         expect(res.body[0]._id).toEqual(row._id)
         await assertRowUsage(rowUsage)
-        await assertQueryUsage(queryUsage + 1)
       })
 
       it("should throw an error if view doesn't exist", async () => {
@@ -818,7 +797,6 @@ describe.each([
         await config.api.legacyView.get("derp", { expectStatus: 404 })
 
         await assertRowUsage(rowUsage)
-        await assertQueryUsage(queryUsage)
       })
 
       it("should be able to run on a view", async () => {
@@ -837,7 +815,6 @@ describe.each([
         expect(res.body[0]._id).toEqual(row._id)
 
         await assertRowUsage(rowUsage)
-        await assertQueryUsage(queryUsage + 1)
       })
     })
 
@@ -910,7 +887,6 @@ describe.each([
       expect(resEnriched.body.link[0].name).toBe("Test Contact")
       expect(resEnriched.body.link[0].description).toBe("original description")
       await assertRowUsage(rowUsage)
-      await assertQueryUsage(queryUsage + 2)
     })
   })
 
@@ -1129,7 +1105,6 @@ describe.each([
         await config.api.row.delete(view.id, [createdRow])
 
         await assertRowUsage(rowUsage - 1)
-        await assertQueryUsage(queryUsage + 1)
 
         await config.api.row.get(tableId, createdRow._id!, {
           expectStatus: 404,
@@ -1157,7 +1132,6 @@ describe.each([
         await config.api.row.delete(view.id, [rows[0], rows[2]])
 
         await assertRowUsage(rowUsage - 2)
-        await assertQueryUsage(queryUsage + 1)
 
         await config.api.row.get(tableId, rows[0]._id!, {
           expectStatus: 404,

packages/server/src/appMigrations/index.ts

@@ -17,7 +17,7 @@ export const getLatestMigrationId = () =>
     .sort()
     .reverse()[0]
 
-const getTimestamp = (versionId: string) => versionId?.split("_")[0]
+const getTimestamp = (versionId: string) => versionId?.split("_")[0] || ""
 
 export async function checkMissingMigrations(
   ctx: UserCtx,

packages/server/src/automations/steps/queryRows.ts

@@ -103,8 +103,7 @@ function typeCoercion(filters: SearchFilters, table: Table) {
     return filters
   }
   for (let key of Object.keys(filters)) {
-    // @ts-ignore
+    const searchParam = filters[key as keyof SearchFilters]
-    const searchParam = filters[key]
     if (typeof searchParam === "object") {
       for (let [property, value] of Object.entries(searchParam)) {
         // We need to strip numerical prefixes here, so that we can look up
@@ -117,7 +116,13 @@ function typeCoercion(filters: SearchFilters, table: Table) {
           continue
         }
         if (column.type === FieldTypes.NUMBER) {
-          searchParam[property] = parseFloat(value)
+          if (key === "oneOf") {
+            searchParam[property] = value
+              .split(",")
+              .map(item => parseFloat(item))
+          } else {
+            searchParam[property] = parseFloat(value)
+          }
         }
       }
     }

packages/server/src/ddApm.ts

@@ -3,5 +3,9 @@ import apm from "dd-trace"
 // enable APM if configured
 if (process.env.DD_APM_ENABLED) {
   console.log("Starting dd-trace")
-  apm.init()
+  apm.init({
+    // @ts-ignore for some reason dd-trace types don't include this options,
+    // even though it's spoken about in the docs.
+    debug: process.env.DD_ENV === "qa",
+  })
 }

packages/server/src/integration-test/postgres.spec.ts

@@ -1118,4 +1118,76 @@ describe("postgres integrations", () => {
       })
     })
   })
+
+  describe("Integration compatibility with postgres search_path", () => {
+    let client: Client, pathDatasource: Datasource
+    const schema1 = "test1",
+      schema2 = "test-2"
+
+    beforeAll(async () => {
+      const dsConfig = await databaseTestProviders.postgres.getDsConfig()
+      const dbConfig = dsConfig.config!
+
+      client = new Client(dbConfig)
+      await client.connect()
+      await client.query(`CREATE SCHEMA "${schema1}";`)
+      await client.query(`CREATE SCHEMA "${schema2}";`)
+
+      const pathConfig: any = {
+        ...dsConfig,
+        config: {
+          ...dbConfig,
+          schema: `${schema1}, ${schema2}`,
+        },
+      }
+      pathDatasource = await config.api.datasource.create(pathConfig)
+    })
+
+    afterAll(async () => {
+      await client.query(`DROP SCHEMA "${schema1}" CASCADE;`)
+      await client.query(`DROP SCHEMA "${schema2}" CASCADE;`)
+      await client.end()
+    })
+
+    it("discovers tables from any schema in search path", async () => {
+      await client.query(
+        `CREATE TABLE "${schema1}".table1 (id1 SERIAL PRIMARY KEY);`
+      )
+      await client.query(
+        `CREATE TABLE "${schema2}".table2 (id2 SERIAL PRIMARY KEY);`
+      )
+      const response = await makeRequest("post", "/api/datasources/info", {
+        datasource: pathDatasource,
+      })
+      expect(response.status).toBe(200)
+      expect(response.body.tableNames).toBeDefined()
+      expect(response.body.tableNames).toEqual(
+        expect.arrayContaining(["table1", "table2"])
+      )
+    })
+
+    it("does not mix columns from different tables", async () => {
+      const repeated_table_name = "table_same_name"
+      await client.query(
+        `CREATE TABLE "${schema1}".${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);`
+      )
+      await client.query(
+        `CREATE TABLE "${schema2}".${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);`
+      )
+      const response = await makeRequest(
+        "post",
+        `/api/datasources/${pathDatasource._id}/schema`,
+        {
+          tablesFilter: [repeated_table_name],
+        }
+      )
+      expect(response.status).toBe(200)
+      expect(
+        response.body.datasource.entities[repeated_table_name].schema
+      ).toBeDefined()
+      const schema =
+        response.body.datasource.entities[repeated_table_name].schema
+      expect(Object.keys(schema).sort()).toEqual(["id", "val1"])
+    })
+  })
 })

packages/server/src/integrations/postgres.ts

@@ -149,8 +149,6 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
   private index: number = 1
   private open: boolean
 
-  COLUMNS_SQL!: string
-
   PRIMARY_KEYS_SQL = () => `
   SELECT pg_namespace.nspname table_schema
      , pg_class.relname table_name
@@ -159,7 +157,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
   JOIN pg_index ON pg_class.oid = pg_index.indrelid AND pg_index.indisprimary
   JOIN pg_attribute ON pg_attribute.attrelid = pg_class.oid AND pg_attribute.attnum = ANY(pg_index.indkey)
   JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace
-  WHERE pg_namespace.nspname = '${this.config.schema}';
+  WHERE pg_namespace.nspname = ANY(current_schemas(false))
+  AND pg_table_is_visible(pg_class.oid);
   `
 
   ENUM_VALUES = () => `
@@ -170,6 +169,11 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
   JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace;
   `
 
+  COLUMNS_SQL = () => `
+    select * from information_schema.columns where table_schema = ANY(current_schemas(false)) 
+      AND pg_table_is_visible(to_regclass(format('%I.%I', table_schema, table_name)));
+  `
+
   constructor(config: PostgresConfig) {
     super(SqlClient.POSTGRES)
     this.config = config
@@ -219,8 +223,10 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
     if (!this.config.schema) {
       this.config.schema = "public"
     }
-    await this.client.query(`SET search_path TO "${this.config.schema}"`)
+    const search_path = this.config.schema
-    this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'`
+      .split(",")
+      .map(item => `"${item.trim()}"`)
+    await this.client.query(`SET search_path TO ${search_path.join(",")};`)
     this.open = true
   }
 
@@ -307,7 +313,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
 
     try {
       const columnsResponse: { rows: PostgresColumn[] } =
-        await this.client.query(this.COLUMNS_SQL)
+        await this.client.query(this.COLUMNS_SQL())
 
       const tables: { [key: string]: Table } = {}
 
@@ -362,8 +368,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
         })
       }
 
-      let finalizedTables = finaliseExternalTables(tables, entities)
+      const finalizedTables = finaliseExternalTables(tables, entities)
-      let errors = checkExternalTables(finalizedTables)
+      const errors = checkExternalTables(finalizedTables)
       return { tables: finalizedTables, errors }
     } catch (err) {
       // @ts-ignore
@@ -377,7 +383,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
     try {
       await this.openConnection()
       const columnsResponse: { rows: PostgresColumn[] } =
-        await this.client.query(this.COLUMNS_SQL)
+        await this.client.query(this.COLUMNS_SQL())
       const names = columnsResponse.rows.map(row => row.table_name)
       return [...new Set(names)]
     } finally {

packages/server/src/jsRunner.ts

@@ -12,11 +12,19 @@ export function init() {
       const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
       let track: TrackerFn = f => f()
       if (perRequestLimit) {
-        const bbCtx = context.getCurrentContext()
+        const bbCtx = tracer.trace("runJS.getCurrentContext", {}, span =>
+          context.getCurrentContext()
+        )
         if (bbCtx) {
           if (!bbCtx.jsExecutionTracker) {
-            bbCtx.jsExecutionTracker =
+            span?.addTags({
-              timers.ExecutionTimeTracker.withLimit(perRequestLimit)
+              createdExecutionTracker: true,
+            })
+            bbCtx.jsExecutionTracker = tracer.trace(
+              "runJS.createExecutionTimeTracker",
+              {},
+              span => timers.ExecutionTimeTracker.withLimit(perRequestLimit)
+            )
           }
           span?.addTags({
             js: {
@@ -26,8 +34,12 @@ export function init() {
           })
           // We call checkLimit() here to prevent paying the cost of creating
           // a new VM context below when we don't need to.
-          bbCtx.jsExecutionTracker.checkLimit()
+          tracer.trace("runJS.checkLimitAndBind", {}, span => {
-          track = bbCtx.jsExecutionTracker.track.bind(bbCtx.jsExecutionTracker)
+            bbCtx.jsExecutionTracker!.checkLimit()
+            track = bbCtx.jsExecutionTracker!.track.bind(
+              bbCtx.jsExecutionTracker
+            )
+          })
         }
       }
 
@@ -37,6 +49,7 @@ export function init() {
         setInterval: undefined,
         setTimeout: undefined,
       }
+
       vm.createContext(ctx)
       return track(() =>
         vm.runInNewContext(js, ctx, {

packages/server/src/sdk/app/datasources/datasources.ts

@@ -1,4 +1,4 @@
-import { context, db as dbCore } from "@budibase/backend-core"
+import { context, db as dbCore, events } from "@budibase/backend-core"
 import { findHBSBlocks, processObjectSync } from "@budibase/string-templates"
 import {
   Datasource,
@@ -14,16 +14,22 @@ import {
 } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import { getEnvironmentVariables } from "../../utils"
-import { getDefinitions, getDefinition } from "../../../integrations"
+import {
+  getDefinitions,
+  getDefinition,
+  getIntegration,
+} from "../../../integrations"
 import merge from "lodash/merge"
 import {
   BudibaseInternalDB,
+  generateDatasourceID,
   getDatasourceParams,
   getDatasourcePlusParams,
   getTableParams,
+  DocumentType,
 } from "../../../db/utils"
 import sdk from "../../index"
-import datasource from "../../../api/routes/datasource"
+import { setupCreationAuth as googleSetupCreationAuth } from "../../../integrations/googlesheets"
 
 const ENV_VAR_PREFIX = "env."
 
@@ -273,3 +279,75 @@ export async function getExternalDatasources(): Promise<Datasource[]> {
 
   return externalDatasources.rows.map(r => r.doc!)
 }
+
+export async function save(
+  datasource: Datasource,
+  opts?: { fetchSchema?: boolean; tablesFilter?: string[] }
+): Promise<{ datasource: Datasource; errors: Record<string, string> }> {
+  const db = context.getAppDB()
+  const plus = datasource.plus
+
+  const fetchSchema = opts?.fetchSchema || false
+  const tablesFilter = opts?.tablesFilter || []
+
+  datasource = {
+    _id: generateDatasourceID({ plus }),
+    ...datasource,
+    type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE,
+  }
+
+  let errors: Record<string, string> = {}
+  if (fetchSchema) {
+    const schema = await sdk.datasources.buildFilteredSchema(
+      datasource,
+      tablesFilter
+    )
+    datasource.entities = schema.tables
+    setDefaultDisplayColumns(datasource)
+    errors = schema.errors
+  }
+
+  if (preSaveAction[datasource.source]) {
+    await preSaveAction[datasource.source](datasource)
+  }
+
+  const dbResp = await db.put(
+    sdk.tables.populateExternalTableSchemas(datasource)
+  )
+  await events.datasource.created(datasource)
+  datasource._rev = dbResp.rev
+
+  // Drain connection pools when configuration is changed
+  if (datasource.source) {
+    const source = await getIntegration(datasource.source)
+    if (source && source.pool) {
+      await source.pool.end()
+    }
+  }
+
+  return { datasource, errors }
+}
+
+const preSaveAction: Partial<Record<SourceName, any>> = {
+  [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => {
+    await googleSetupCreationAuth(datasource.config as any)
+  },
+}
+
+/**
+ * Make sure all datasource entities have a display name selected
+ */
+export function setDefaultDisplayColumns(datasource: Datasource) {
+  //
+  for (let entity of Object.values(datasource.entities || {})) {
+    if (entity.primaryDisplay) {
+      continue
+    }
+    const notAutoColumn = Object.values(entity.schema).find(
+      schema => !schema.autocolumn
+    )
+    if (notAutoColumn) {
+      entity.primaryDisplay = notAutoColumn.name
+    }
+  }
+}

packages/server/src/sdk/app/datasources/index.ts

@@ -1,5 +1,7 @@
 import * as datasources from "./datasources"
+import * as plus from "./plus"
 
 export default {
   ...datasources,
+  ...plus,
 }

packages/server/src/sdk/app/datasources/plus.ts

@@ -0,0 +1,85 @@
+import {
+  Datasource,
+  DatasourcePlus,
+  IntegrationBase,
+  Schema,
+} from "@budibase/types"
+import * as datasources from "./datasources"
+import tableSdk from "../tables"
+import { getIntegration } from "../../../integrations"
+import { context } from "@budibase/backend-core"
+
+export async function buildFilteredSchema(
+  datasource: Datasource,
+  filter?: string[]
+): Promise<Schema> {
+  const schema = await buildSchemaHelper(datasource)
+  if (!filter) {
+    return schema
+  }
+
+  let filteredSchema: Schema = { tables: {}, errors: {} }
+  for (let key in schema.tables) {
+    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
+      filteredSchema.tables[key] = schema.tables[key]
+    }
+  }
+
+  for (let key in schema.errors) {
+    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
+      filteredSchema.errors[key] = schema.errors[key]
+    }
+  }
+  return filteredSchema
+}
+
+async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
+  const connector = (await getConnector(datasource)) as DatasourcePlus
+  const externalSchema = await connector.buildSchema(
+    datasource._id!,
+    datasource.entities!
+  )
+  return externalSchema
+}
+
+export async function getConnector(
+  datasource: Datasource
+): Promise<IntegrationBase | DatasourcePlus> {
+  const Connector = await getIntegration(datasource.source)
+  // can't enrich if it doesn't have an ID yet
+  if (datasource._id) {
+    datasource = await datasources.enrich(datasource)
+  }
+  // Connect to the DB and build the schema
+  return new Connector(datasource.config)
+}
+
+export async function getAndMergeDatasource(datasource: Datasource) {
+  if (datasource._id) {
+    const existingDatasource = await datasources.get(datasource._id)
+
+    datasource = datasources.mergeConfigs(datasource, existingDatasource)
+  }
+  return await datasources.enrich(datasource)
+}
+
+export async function buildSchemaFromSource(
+  datasourceId: string,
+  tablesFilter?: string[]
+) {
+  const db = context.getAppDB()
+
+  const datasource = await datasources.get(datasourceId)
+
+  const { tables, errors } = await buildFilteredSchema(datasource, tablesFilter)
+  datasource.entities = tables
+
+  datasources.setDefaultDisplayColumns(datasource)
+  const dbResp = await db.put(tableSdk.populateExternalTableSchemas(datasource))
+  datasource._rev = dbResp.rev
+
+  return {
+    datasource,
+    errors,
+  }
+}

packages/server/src/startup.ts

@@ -138,7 +138,11 @@ export async function startup(app?: Koa, server?: Server) {
             bbAdminEmail,
             bbAdminPassword,
             tenantId,
-            { hashPassword: true, requirePassword: true }
+            {
+              hashPassword: true,
+              requirePassword: true,
+              skipPasswordValidation: true,
+            }
           )
           // Need to set up an API key for automated integration tests
           if (env.isTest()) {

packages/shared-core/src/filters.ts

@@ -143,100 +143,104 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => {
     oneOf: {},
     containsAny: {},
   }
-  if (Array.isArray(filter)) {
+
-    filter.forEach(expression => {
+  if (!Array.isArray(filter)) {
-      let { operator, field, type, value, externalType, onEmptyFilter } =
+    return query
-        expression
+  }
-      const isHbs =
+
-        typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
+  filter.forEach(expression => {
-      // Parse all values into correct types
+    let { operator, field, type, value, externalType, onEmptyFilter } =
-      if (operator === "allOr") {
+      expression
-        query.allOr = true
+    const isHbs =
+      typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
+    // Parse all values into correct types
+    if (operator === "allOr") {
+      query.allOr = true
+      return
+    }
+    if (onEmptyFilter) {
+      query.onEmptyFilter = onEmptyFilter
+      return
+    }
+    if (
+      type === "datetime" &&
+      !isHbs &&
+      operator !== "empty" &&
+      operator !== "notEmpty"
+    ) {
+      // Ensure date value is a valid date and parse into correct format
+      if (!value) {
         return
       }
-      if (onEmptyFilter) {
+      try {
-        query.onEmptyFilter = onEmptyFilter
+        value = new Date(value).toISOString()
+      } catch (error) {
         return
       }
-      if (
+    }
-        type === "datetime" &&
+    if (type === "number" && typeof value === "string" && !isHbs) {
-        !isHbs &&
+      if (operator === "oneOf") {
-        operator !== "empty" &&
+        value = value.split(",").map(item => parseFloat(item))
-        operator !== "notEmpty"
+      } else {
+        value = parseFloat(value)
+      }
+    }
+    if (type === "boolean") {
+      value = `${value}`?.toLowerCase() === "true"
+    }
+    if (
+      ["contains", "notContains", "containsAny"].includes(operator) &&
+      type === "array" &&
+      typeof value === "string"
+    ) {
+      value = value.split(",")
+    }
+    if (operator.startsWith("range") && query.range) {
+      const minint =
+        SqlNumberTypeRangeMap[
+          externalType as keyof typeof SqlNumberTypeRangeMap
+        ]?.min || Number.MIN_SAFE_INTEGER
+      const maxint =
+        SqlNumberTypeRangeMap[
+          externalType as keyof typeof SqlNumberTypeRangeMap
+        ]?.max || Number.MAX_SAFE_INTEGER
+      if (!query.range[field]) {
+        query.range[field] = {
+          low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
+          high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
+        }
+      }
+      if ((operator as any) === "rangeLow" && value != null && value !== "") {
+        query.range[field].low = value
+      } else if (
+        (operator as any) === "rangeHigh" &&
+        value != null &&
+        value !== ""
       ) {
-        // Ensure date value is a valid date and parse into correct format
+        query.range[field].high = value
-        if (!value) {
-          return
-        }
-        try {
-          value = new Date(value).toISOString()
-        } catch (error) {
-          return
-        }
-      }
-      if (type === "number" && typeof value === "string") {
-        if (operator === "oneOf") {
-          value = value.split(",").map(item => parseFloat(item))
-        } else if (!isHbs) {
-          value = parseFloat(value)
-        }
       }
+    } else if (query[operator] && operator !== "onEmptyFilter") {
       if (type === "boolean") {
-        value = `${value}`?.toLowerCase() === "true"
+        // Transform boolean filters to cope with null.
-      }
+        // "equals false" needs to be "not equals true"
-      if (
+        // "not equals false" needs to be "equals true"
-        ["contains", "notContains", "containsAny"].includes(operator) &&
+        if (operator === "equal" && value === false) {
-        type === "array" &&
+          query.notEqual = query.notEqual || {}
-        typeof value === "string"
+          query.notEqual[field] = true
-      ) {
+        } else if (operator === "notEqual" && value === false) {
-        value = value.split(",")
+          query.equal = query.equal || {}
-      }
+          query.equal[field] = true
-      if (operator.startsWith("range") && query.range) {
-        const minint =
-          SqlNumberTypeRangeMap[
-            externalType as keyof typeof SqlNumberTypeRangeMap
-          ]?.min || Number.MIN_SAFE_INTEGER
-        const maxint =
-          SqlNumberTypeRangeMap[
-            externalType as keyof typeof SqlNumberTypeRangeMap
-          ]?.max || Number.MAX_SAFE_INTEGER
-        if (!query.range[field]) {
-          query.range[field] = {
-            low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
-            high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
-          }
-        }
-        if ((operator as any) === "rangeLow" && value != null && value !== "") {
-          query.range[field].low = value
-        } else if (
-          (operator as any) === "rangeHigh" &&
-          value != null &&
-          value !== ""
-        ) {
-          query.range[field].high = value
-        }
-      } else if (query[operator] && operator !== "onEmptyFilter") {
-        if (type === "boolean") {
-          // Transform boolean filters to cope with null.
-          // "equals false" needs to be "not equals true"
-          // "not equals false" needs to be "equals true"
-          if (operator === "equal" && value === false) {
-            query.notEqual = query.notEqual || {}
-            query.notEqual[field] = true
-          } else if (operator === "notEqual" && value === false) {
-            query.equal = query.equal || {}
-            query.equal[field] = true
-          } else {
-            query[operator] = query[operator] || {}
-            query[operator]![field] = value
-          }
         } else {
           query[operator] = query[operator] || {}
           query[operator]![field] = value
         }
+      } else {
+        query[operator] = query[operator] || {}
+        query[operator]![field] = value
       }
-    })
+    }
-  }
+  })
+
   return query
 }
 

packages/shared-core/src/tests/filters.test.ts

@@ -1,6 +1,11 @@
-import { SearchQuery, SearchQueryOperators } from "@budibase/types"
+import {
-import { runLuceneQuery } from "../filters"
+  SearchQuery,
-import { expect, describe, it } from "vitest"
+  SearchQueryOperators,
+  FieldType,
+  SearchFilter,
+} from "@budibase/types"
+import { buildLuceneQuery, runLuceneQuery } from "../filters"
+import { expect, describe, it, test } from "vitest"
 
 describe("runLuceneQuery", () => {
   const docs = [
@@ -167,4 +172,186 @@ describe("runLuceneQuery", () => {
     })
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2, 3])
   })
+
+  test.each([[523, 259], "523,259"])(
+    "should return rows with matches on numeric oneOf filter",
+    input => {
+      let query = buildQuery("oneOf", {
+        customer_id: input,
+      })
+      expect(runLuceneQuery(docs, query).map(row => row.customer_id)).toEqual([
+        259, 523,
+      ])
+    }
+  )
+})
+
+describe("buildLuceneQuery", () => {
+  it("should return a basic search query template if the input is not an array", () => {
+    const filter: any = "NOT_AN_ARRAY"
+    expect(buildLuceneQuery(filter)).toEqual({
+      string: {},
+      fuzzy: {},
+      range: {},
+      equal: {},
+      notEqual: {},
+      empty: {},
+      notEmpty: {},
+      contains: {},
+      notContains: {},
+      oneOf: {},
+      containsAny: {},
+    })
+  })
+
+  it("should parseFloat if the type is a number, but the value is a numeric string", () => {
+    const filter: SearchFilter[] = [
+      {
+        operator: SearchQueryOperators.EQUAL,
+        field: "customer_id",
+        type: FieldType.NUMBER,
+        value: "1212",
+      },
+      {
+        operator: SearchQueryOperators.ONE_OF,
+        field: "customer_id",
+        type: FieldType.NUMBER,
+        value: "1000,1212,3400",
+      },
+    ]
+    expect(buildLuceneQuery(filter)).toEqual({
+      string: {},
+      fuzzy: {},
+      range: {},
+      equal: {
+        customer_id: 1212,
+      },
+      notEqual: {},
+      empty: {},
+      notEmpty: {},
+      contains: {},
+      notContains: {},
+      oneOf: {
+        customer_id: [1000, 1212, 3400],
+      },
+      containsAny: {},
+    })
+  })
+
+  it("should not parseFloat if the type is a number, but the value is a handlebars binding string", () => {
+    const filter: SearchFilter[] = [
+      {
+        operator: SearchQueryOperators.EQUAL,
+        field: "customer_id",
+        type: FieldType.NUMBER,
+        value: "{{ customer_id }}",
+      },
+      {
+        operator: SearchQueryOperators.ONE_OF,
+        field: "customer_id",
+        type: FieldType.NUMBER,
+        value: "{{ list_of_customer_ids }}",
+      },
+    ]
+    expect(buildLuceneQuery(filter)).toEqual({
+      string: {},
+      fuzzy: {},
+      range: {},
+      equal: {
+        customer_id: "{{ customer_id }}",
+      },
+      notEqual: {},
+      empty: {},
+      notEmpty: {},
+      contains: {},
+      notContains: {},
+      oneOf: {
+        customer_id: "{{ list_of_customer_ids }}",
+      },
+      containsAny: {},
+    })
+  })
+
+  it("should cast string to boolean if the type is boolean", () => {
+    const filter: SearchFilter[] = [
+      {
+        operator: SearchQueryOperators.EQUAL,
+        field: "a",
+        type: FieldType.BOOLEAN,
+        value: "not_true",
+      },
+      {
+        operator: SearchQueryOperators.NOT_EQUAL,
+        field: "b",
+        type: FieldType.BOOLEAN,
+        value: "not_true",
+      },
+      {
+        operator: SearchQueryOperators.EQUAL,
+        field: "c",
+        type: FieldType.BOOLEAN,
+        value: "true",
+      },
+    ]
+    expect(buildLuceneQuery(filter)).toEqual({
+      string: {},
+      fuzzy: {},
+      range: {},
+      equal: {
+        b: true,
+        c: true,
+      },
+      notEqual: {
+        a: true,
+      },
+      empty: {},
+      notEmpty: {},
+      contains: {},
+      notContains: {},
+      oneOf: {},
+      containsAny: {},
+    })
+  })
+
+  it("should split the string for contains operators", () => {
+    const filter: SearchFilter[] = [
+      {
+        operator: SearchQueryOperators.CONTAINS,
+        field: "description",
+        type: FieldType.ARRAY,
+        value: "Large box,Heavy box,Small box",
+      },
+      {
+        operator: SearchQueryOperators.NOT_CONTAINS,
+        field: "description",
+        type: FieldType.ARRAY,
+        value: "Large box,Heavy box,Small box",
+      },
+      {
+        operator: SearchQueryOperators.CONTAINS_ANY,
+        field: "description",
+        type: FieldType.ARRAY,
+        value: "Large box,Heavy box,Small box",
+      },
+    ]
+    expect(buildLuceneQuery(filter)).toEqual({
+      string: {},
+      fuzzy: {},
+      range: {},
+      equal: {},
+      notEqual: {},
+      empty: {},
+      notEmpty: {},
+      contains: {
+        description: ["Large box", "Heavy box", "Small box"],
+      },
+      notContains: {
+        description: ["Large box", "Heavy box", "Small box"],
+      },
+      oneOf: {},
+      containsAny: {
+        description: ["Large box", "Heavy box", "Small box"],
+      },
+    })
+  })
 })

packages/types/src/api/web/app/datasource.ts

@@ -35,3 +35,12 @@ export interface FetchDatasourceInfoResponse {
 export interface UpdateDatasourceRequest extends Datasource {
   datasource: Datasource
 }
+
+export interface BuildSchemaFromSourceRequest {
+  tablesFilter?: string[]
+}
+
+export interface BuildSchemaFromSourceResponse {
+  datasource: Datasource
+  errors: Record<string, string>
+}

packages/types/src/sdk/user.ts

@@ -2,4 +2,5 @@ export interface SaveUserOpts {
   hashPassword?: boolean
   requirePassword?: boolean
   currentUserId?: string
+  skipPasswordValidation?: boolean
 }

qa-core/src/account-api/tests/licensing/license.manage.spec.ts

@@ -39,10 +39,10 @@ describe("license management", () => {
     let premiumPriceId = null
     let businessPriceId = ""
     for (const plan of planBody) {
-      if (plan.type === PlanType.PREMIUM) {
+      if (plan.type === PlanType.PREMIUM_PLUS) {
         premiumPriceId = plan.prices[0].priceId
       }
-      if (plan.type === PlanType.BUSINESS) {
+      if (plan.type === PlanType.ENTERPRISE_BASIC) {
         businessPriceId = plan.prices[0].priceId
       }
     }
@@ -97,7 +97,7 @@ describe("license management", () => {
       await config.loginAsAccount(createAccountRequest)
       await config.api.stripe.linkStripeCustomer(account.accountId, customer.id)
       const [_, selfBodyPremium] = await config.api.accounts.self()
-      expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM)
+      expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM_PLUS)
 
       // Create portal session - Check URL
       const [portalRes, portalSessionBody] =
@@ -109,7 +109,7 @@ describe("license management", () => {
 
       // License updated to Business
       const [selfRes, selfBodyBusiness] = await config.api.accounts.self()
-      expect(selfBodyBusiness.license.plan.type).toBe(PlanType.BUSINESS)
+      expect(selfBodyBusiness.license.plan.type).toBe(PlanType.ENTERPRISE_BASIC)
     })
   })
 })

yarn.lock

@@ -639,15 +639,7 @@
   dependencies:
     tslib "^2.2.0"
 
-"@azure/core-auth@^1.3.0", "@azure/core-auth@^1.4.0":
+"@azure/core-auth@^1.3.0", "@azure/core-auth@^1.4.0", "@azure/core-auth@^1.5.0":
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/@azure/core-auth/-/core-auth-1.4.0.tgz#6fa9661c1705857820dbc216df5ba5665ac36a9e"
-  integrity sha512-HFrcTgmuSuukRf/EdPmqBrc5l6Q5Uu+2TbuhaKbgaCpP2TfAeiNaQPAadxO+CYBRHGUzIDteMAjFspFLDLnKVQ==
-  dependencies:
-    "@azure/abort-controller" "^1.0.0"
-    tslib "^2.2.0"
-
-"@azure/core-auth@^1.5.0":
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/@azure/core-auth/-/core-auth-1.5.0.tgz#a41848c5c31cb3b7c84c409885267d55a2c92e44"
   integrity sha512-udzoBuYG1VBoHVohDTrvKjyzel34zt77Bhp7dQntVGGD0ehVq48owENbBG8fIgkHRNUBQH5k1r0hpoMu5L8+kw==
@@ -744,15 +736,7 @@
   dependencies:
     tslib "^2.2.0"
 
-"@azure/core-util@^1.0.0", "@azure/core-util@^1.1.1", "@azure/core-util@^1.3.0":
+"@azure/core-util@^1.0.0", "@azure/core-util@^1.1.0", "@azure/core-util@^1.1.1", "@azure/core-util@^1.3.0", "@azure/core-util@^1.6.1":
-  version "1.3.2"
-  resolved "https://registry.yarnpkg.com/@azure/core-util/-/core-util-1.3.2.tgz#3f8cfda1e87fac0ce84f8c1a42fcd6d2a986632d"
-  integrity sha512-2bECOUh88RvL1pMZTcc6OzfobBeWDBf5oBbhjIhT1MV9otMVWCzpOJkkiKtrnO88y5GGBelgY8At73KGAdbkeQ==
-  dependencies:
-    "@azure/abort-controller" "^1.0.0"
-    tslib "^2.2.0"
-
-"@azure/core-util@^1.1.0", "@azure/core-util@^1.6.1":
   version "1.6.1"
   resolved "https://registry.yarnpkg.com/@azure/core-util/-/core-util-1.6.1.tgz#fea221c4fa43c26543bccf799beb30c1c7878f5a"
   integrity sha512-h5taHeySlsV9qxuK64KZxy4iln1BtMYlNt5jbuEFN3UFSAd1EwKg/Gjl5a6tZ/W8t6li3xPnutOx7zbDyXnPmQ==
@@ -1996,17 +1980,10 @@
   resolved "https://registry.yarnpkg.com/@babel/regjsgen/-/regjsgen-0.8.0.tgz#f0ba69b075e1f05fb2825b7fad991e7adbb18310"
   integrity sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA==
 
-"@babel/runtime@^7.12.5", "@babel/runtime@^7.15.4", "@babel/runtime@^7.7.2", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2":
+"@babel/runtime@^7.12.5", "@babel/runtime@^7.13.10", "@babel/runtime@^7.15.4", "@babel/runtime@^7.7.2", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2":
-  version "7.23.6"
+  version "7.23.8"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.23.6.tgz#c05e610dc228855dc92ef1b53d07389ed8ab521d"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.23.8.tgz#8ee6fe1ac47add7122902f257b8ddf55c898f650"
-  integrity sha512-zHd0eUrf5GZoOWVCXp6koAKQTfZV07eit6bGPmJgnZdnSAvvZee6zniW2XMF7Cmc4ISOOnPy3QaSiIJGJkVEDQ==
+  integrity sha512-Y7KbAP984rn1VGMbGqKmBLio9V7y5Je9GvU4rQPCPinCyNfUcToxIXl06d59URp/F3LwinvODxab5N/G6qggkw==
-  dependencies:
-    regenerator-runtime "^0.14.0"
-
-"@babel/runtime@^7.13.10":
-  version "7.23.7"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.23.7.tgz#dd7c88deeb218a0f8bd34d5db1aa242e0f203193"
-  integrity sha512-w06OXVOFso7LcbzMiDGt+3X7Rh7Ho8MmgPoWU3rarH+8upf+wSU/grlGbWzQyr3DkdN6ZeuMFjpdwW0Q+HxobA==
   dependencies:
     regenerator-runtime "^0.14.0"
 
@@ -2019,15 +1996,6 @@
     "@babel/parser" "^7.22.15"
     "@babel/types" "^7.22.15"
 
-"@babel/template@^7.22.5", "@babel/template@^7.3.3":
-  version "7.22.5"
-  resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.5.tgz#0c8c4d944509875849bd0344ff0050756eefc6ec"
-  integrity sha512-X7yV7eiwAxdj9k94NEylvbVHLiVG1nvzCV2EAowhxLTwODV1jl9UzZ48leOC0sH7OnuHrIkllaBgneUykIcZaw==
-  dependencies:
-    "@babel/code-frame" "^7.22.5"
-    "@babel/parser" "^7.22.5"
-    "@babel/types" "^7.22.5"
-
 "@babel/traverse@^7.22.5":
   version "7.23.6"
   resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.6.tgz#b53526a2367a0dd6edc423637f3d2d0f2521abc5"
@@ -3381,7 +3349,7 @@
   resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24"
   integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==
 
-"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.13", "@jridgewell/sourcemap-codec@^1.4.14":
+"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.14", "@jridgewell/sourcemap-codec@^1.4.15":
   version "1.4.15"
   resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32"
   integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==
@@ -4043,12 +4011,12 @@
     magic-string "^0.25.7"
 
 "@rollup/plugin-replace@^5.0.2", "@rollup/plugin-replace@^5.0.3":
-  version "5.0.3"
+  version "5.0.5"
-  resolved "https://registry.yarnpkg.com/@rollup/plugin-replace/-/plugin-replace-5.0.3.tgz#55a4550bd6d5e83a65df3d201e0b3d219be7b4b2"
+  resolved "https://registry.yarnpkg.com/@rollup/plugin-replace/-/plugin-replace-5.0.5.tgz#33d5653dce6d03cb24ef98bef7f6d25b57faefdf"
-  integrity sha512-je7fu05B800IrMlWjb2wzJcdXzHYW46iTipfChnBDbIbDXhASZs27W1B58T2Yf45jZtJUONegpbce+9Ut2Ti/Q==
+  integrity sha512-rYO4fOi8lMaTg/z5Jb+hKnrHHVn8j2lwkqwyS4kTRhKyWOLf2wST2sWXr4WzWiTcoHTp2sTjqUbqIj2E39slKQ==
   dependencies:
     "@rollup/pluginutils" "^5.0.1"
-    magic-string "^0.27.0"
+    magic-string "^0.30.3"
 
 "@rollup/plugin-typescript@8.3.0":
   version "8.3.0"
@@ -5473,7 +5441,7 @@
     "@types/koa" "*"
     "@types/passport" "*"
 
-"@types/koa-send@*":
+"@types/koa-send@*", "@types/koa-send@^4.1.6":
   version "4.1.6"
   resolved "https://registry.yarnpkg.com/@types/koa-send/-/koa-send-4.1.6.tgz#15d90e95e3ccce669a15b6a3c56c3a650a167cea"
   integrity sha512-vgnNGoOJkx7FrF0Jl6rbK1f8bBecqAchKpXtKuXzqIEdXTDO6dsSTjr+eZ5m7ltSjH4K/E7auNJEQCAd0McUPA==
@@ -5605,10 +5573,10 @@
     "@types/node" "*"
     form-data "^3.0.0"
 
-"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=12.12.47", "@types/node@>=13.13.4", "@types/node@>=13.7.0":
+"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=12.12.47", "@types/node@>=13.13.4", "@types/node@>=13.7.0", "@types/node@>=8.1.0":
-  version "20.10.5"
+  version "20.10.7"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.10.5.tgz#47ad460b514096b7ed63a1dae26fad0914ed3ab2"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.10.7.tgz#40fe8faf25418a75de9fe68a8775546732a3a901"
-  integrity sha512-nNPsNE65wjMxEKI93yOP+NPGGBJz/PoN3kZsVLee0XMiJolxSekEVD8wRwBUBqkwc7UWop0edW50yrCQW4CyRw==
+  integrity sha512-fRbIKb8C/Y2lXxB5eVMj4IU7xpdox0Lh8bUPEdtLysaylsml1hOOx1+STloRs/B9nf7C6kPRmmg/V7aQW7usNg==
   dependencies:
     undici-types "~5.26.4"
 
@@ -5617,11 +5585,6 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-16.9.1.tgz#0611b37db4246c937feef529ddcc018cf8e35708"
   integrity sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g==
 
-"@types/node@18.17.0":
-  version "18.17.0"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.17.0.tgz#35d44267a33dd46b49ee0f73d31b05fd7407e290"
-  integrity sha512-GXZxEtOxYGFchyUzxvKI14iff9KZ2DI+A6a37o6EQevtg6uO9t+aUZKcaC1Te5Ng1OnLM7K9NVVj+FbecD9cJg==
-
 "@types/node@20.10.0":
   version "20.10.0"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-20.10.0.tgz#16ddf9c0a72b832ec4fcce35b8249cf149214617"
@@ -5639,13 +5602,6 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.37.tgz#0bfcd173e8e1e328337473a8317e37b3b14fd30d"
   integrity sha512-7GgtHCs/QZrBrDzgIJnQtuSvhFSwhyYSI2uafSwZoNt1iOGhEN5fwNrQMjtONyHm9+/LoA4453jH0CMYcr06Pg==
 
-"@types/node@>=8.1.0":
-  version "20.10.6"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.10.6.tgz#a3ec84c22965802bf763da55b2394424f22bfbb5"
-  integrity sha512-Vac8H+NlRNNlAmDfGUP7b5h/KA+AtWIzuXy0E6OyP8f1tCLYAtPvKRRDJjAPqhpCb0t6U2j7/xqAuLEebW2kiw==
-  dependencies:
-    undici-types "~5.26.4"
-
 "@types/nodemailer@^6.4.4":
   version "6.4.14"
   resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.14.tgz#5c81a5e856db7f8ede80013e6dbad7c5fb2283e2"
@@ -11183,9 +11139,9 @@ formidable@^2.1.2:
     qs "^6.11.0"
 
 fp-ts@^2.5.1:
-  version "2.16.1"
+  version "2.16.2"
-  resolved "https://registry.yarnpkg.com/fp-ts/-/fp-ts-2.16.1.tgz#6abc401ce42b65364ca8f0b0d995c5840c68a930"
+  resolved "https://registry.yarnpkg.com/fp-ts/-/fp-ts-2.16.2.tgz#7faa90f6fc2e8cf84c711d2c4e606afe2be9e342"
-  integrity sha512-by7U5W8dkIzcvDofUcO42yl9JbnHTEDBrzu3pt5fKT+Z4Oy85I21K80EYJYdjQGC2qum4Vo55Ag57iiIK4FYuA==
+  integrity sha512-CkqAjnIKFqvo3sCyoBTqgJvF+bHrSik584S9nhTjtBESLx26cbtVMR/T9a6ApChOcSDAaM3JydDmWDUn4EEXng==
 
 fragment-cache@^0.2.1:
   version "0.2.1"
@@ -11265,9 +11221,9 @@ fs.realpath@^1.0.0:
   integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==
 
 fsevents@^2.3.2, fsevents@~2.3.1, fsevents@~2.3.2:
-  version "2.3.2"
+  version "2.3.3"
-  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
+  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6"
-  integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
+  integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==
 
 function-bind@^1.1.1, function-bind@^1.1.2:
   version "1.1.2"
@@ -14326,16 +14282,6 @@ koa-router@^10.0.0:
     methods "^1.1.2"
     path-to-regexp "^6.1.0"
 
-koa-send@5.0.0:
-  version "5.0.0"
-  resolved "https://registry.yarnpkg.com/koa-send/-/koa-send-5.0.0.tgz#5e8441e07ef55737734d7ced25b842e50646e7eb"
-  integrity sha512-90ZotV7t0p3uN9sRwW2D484rAaKIsD8tAVtypw/aBU+ryfV+fR2xrcAwhI8Wl6WRkojLUs/cB9SBSCuIb+IanQ==
-  dependencies:
-    debug "^3.1.0"
-    http-errors "^1.6.3"
-    mz "^2.7.0"
-    resolve-path "^1.4.0"
-
 koa-send@5.0.1, koa-send@^5.0.0:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/koa-send/-/koa-send-5.0.1.tgz#39dceebfafb395d0d60beaffba3a70b4f543fe79"
@@ -15217,12 +15163,12 @@ magic-string@^0.26.7:
   dependencies:
     sourcemap-codec "^1.4.8"
 
-magic-string@^0.27.0:
+magic-string@^0.30.3:
-  version "0.27.0"
+  version "0.30.5"
-  resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.27.0.tgz#e4a3413b4bab6d98d2becffd48b4a257effdbbf3"
+  resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.30.5.tgz#1994d980bd1c8835dc6e78db7cbd4ae4f24746f9"
-  integrity sha512-8UnnX2PeRAPZuN12svgR9j7M1uWMovg/CEnIwIG0LFkXSJJe4PdfUGiTGl8V9bsBHFUtfVINcSyYxd7q+kx9fA==
+  integrity sha512-7xlpfBaQaP/T6Vh8MO/EqXSW5En6INHEvEXQiuff7Gku0PWjU3uf6w/j9o7O+SpB5fOAkrI5HeoNgwjEO0pFsA==
   dependencies:
-    "@jridgewell/sourcemap-codec" "^1.4.13"
+    "@jridgewell/sourcemap-codec" "^1.4.15"
 
 make-dir@3.1.0, make-dir@^3.0.0, make-dir@^3.1.0:
   version "3.1.0"
@@ -15936,7 +15882,7 @@ mysql2@3.5.2:
     seq-queue "^0.0.5"
     sqlstring "^2.3.2"
 
-mz@^2.4.0, mz@^2.7.0:
+mz@^2.4.0:
   version "2.7.0"
   resolved "https://registry.yarnpkg.com/mz/-/mz-2.7.0.tgz#95008057a56cafadc2bc63dde7f9ff6955948e32"
   integrity sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==
@@ -18774,17 +18720,7 @@ readable-stream@^2.0.0, readable-stream@^2.2.2, readable-stream@^2.3.0, readable
     string_decoder "~1.1.1"
     util-deprecate "~1.0.1"
 
-readable-stream@^4.0.0:
+readable-stream@^4.0.0, readable-stream@^4.2.0:
-  version "4.3.0"
-  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.3.0.tgz#0914d0c72db03b316c9733bb3461d64a3cc50cba"
-  integrity sha512-MuEnA0lbSi7JS8XM+WNJlWZkHAAdm7gETHdFK//Q/mChGyj2akEFtdLZh32jSdkWGbRwCW9pn6g3LWDdDeZnBQ==
-  dependencies:
-    abort-controller "^3.0.0"
-    buffer "^6.0.3"
-    events "^3.3.0"
-    process "^0.11.10"
-
-readable-stream@^4.2.0:
   version "4.5.1"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.5.1.tgz#3f2e4e66eab45606ac8f31597b9edb80c13b12ab"
   integrity sha512-uQjbf34vmf/asGnOHQEw07Q4llgMACQZTWWa4MmICS0IKJoHbLwKCy71H3eR99Dw5iYejc6W+pqZZEeqRtUFAw==
@@ -18965,11 +18901,6 @@ regexparam@2.0.1:
   resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-2.0.1.tgz#c912f5dae371e3798100b3c9ce22b7414d0889fa"
   integrity sha512-zRgSaYemnNYxUv+/5SeoHI0eJIgTL/A2pUtXUPLHQxUldagouJ9p+K6IbIZ/JiQuCEv2E2B1O11SjVQy3aMCkw==
 
-regexparam@^1.3.0:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-1.3.0.tgz#2fe42c93e32a40eff6235d635e0ffa344b92965f"
-  integrity sha512-6IQpFBv6e5vz1QAqI+V4k8P2e/3gRrqfCJ9FI+O1FLQTO+Uz6RXZEZOPmTJ6hlGj7gkERzY5BRCv09whKP96/g==
-
 regexpu-core@^5.3.1:
   version "5.3.1"
   resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-5.3.1.tgz#66900860f88def39a5cb79ebd9490e84f17bcdfb"
@@ -21070,11 +21001,16 @@ timed-out@^4.0.1:
   resolved "https://registry.yarnpkg.com/timed-out/-/timed-out-4.0.1.tgz#f32eacac5a175bea25d7fab565ab3ed8741ef56f"
   integrity sha512-G7r3AhovYtr5YKOWQkta8RKAPb+J9IsO4uVmzjl8AZwfhs8UcUwTiD6gcJYSgOtzyjvQKrKYn41syHbUWMkafA==
 
-timekeeper@2.2.0, timekeeper@^2.2.0:
+timekeeper@2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/timekeeper/-/timekeeper-2.2.0.tgz#9645731fce9e3280a18614a57a9d1b72af3ca368"
   integrity sha512-W3AmPTJWZkRwu+iSNxPIsLZ2ByADsOLbbLxe46UJyWj3mlYLlwucKiq+/dPm0l9wTzqoF3/2PH0AGFCebjq23A==
 
+timekeeper@^2.2.0:
+  version "2.3.1"
+  resolved "https://registry.yarnpkg.com/timekeeper/-/timekeeper-2.3.1.tgz#2deb6e0b95d93625fda84c18d47f84a99e4eba01"
+  integrity sha512-LeQRS7/4JcC0PgdSFnfUiStQEdiuySlCj/5SJ18D+T1n9BoY7PxKFfCwLulpHXoLUFr67HxBddQdEX47lDGx1g==
+
 timm@^1.6.1:
   version "1.7.1"
   resolved "https://registry.yarnpkg.com/timm/-/timm-1.7.1.tgz#96bab60c7d45b5a10a8a4d0f0117c6b7e5aff76f"