Merge branch 'master' into screen-store-ts-conversion

eslint-local-rules/index.js

@@ -41,11 +41,12 @@ module.exports = {
           if (
             /^@budibase\/[^/]+\/.*$/.test(importPath) &&
             importPath !== "@budibase/backend-core/tests" &&
-            importPath !== "@budibase/string-templates/test/utils"
+            importPath !== "@budibase/string-templates/test/utils" &&
+            importPath !== "@budibase/client/manifest.json"
           ) {
             context.report({
               node,
-              message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests and @budibase/string-templates/test/utils.`,
+              message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests, @budibase/string-templates/test/utils and @budibase/client/manifest.json.`,
             })
           }
         },

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.46",
+  "version": "3.3.1",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/src/index.ts

@@ -1,6 +1,5 @@
 export * as configs from "./configs"
 export * as events from "./events"
-export * as migrations from "./migrations"
 export * as users from "./users"
 export * as userUtils from "./users/utils"
 export * as roles from "./security/roles"

packages/backend-core/src/migrations/definitions.ts

@@ -1,40 +0,0 @@
-import {
-  MigrationType,
-  MigrationName,
-  MigrationDefinition,
-} from "@budibase/types"
-
-export const DEFINITIONS: MigrationDefinition[] = [
-  {
-    type: MigrationType.GLOBAL,
-    name: MigrationName.USER_EMAIL_VIEW_CASING,
-  },
-  {
-    type: MigrationType.GLOBAL,
-    name: MigrationName.SYNC_QUOTAS,
-  },
-  {
-    type: MigrationType.APP,
-    name: MigrationName.APP_URLS,
-  },
-  {
-    type: MigrationType.APP,
-    name: MigrationName.EVENT_APP_BACKFILL,
-  },
-  {
-    type: MigrationType.APP,
-    name: MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS,
-  },
-  {
-    type: MigrationType.GLOBAL,
-    name: MigrationName.EVENT_GLOBAL_BACKFILL,
-  },
-  {
-    type: MigrationType.INSTALLATION,
-    name: MigrationName.EVENT_INSTALLATION_BACKFILL,
-  },
-  {
-    type: MigrationType.GLOBAL,
-    name: MigrationName.GLOBAL_INFO_SYNC_USERS,
-  },
-]

packages/backend-core/src/migrations/index.ts

@@ -1,2 +0,0 @@
-export * from "./migrations"
-export * from "./definitions"

packages/backend-core/src/migrations/migrations.ts

@@ -1,186 +0,0 @@
-import { DEFAULT_TENANT_ID } from "../constants"
-import {
-  DocumentType,
-  StaticDatabases,
-  getAllApps,
-  getGlobalDBName,
-  getDB,
-} from "../db"
-import environment from "../environment"
-import * as platform from "../platform"
-import * as context from "../context"
-import { DEFINITIONS } from "."
-import {
-  Migration,
-  MigrationOptions,
-  MigrationType,
-  MigrationNoOpOptions,
-  App,
-} from "@budibase/types"
-
-export const getMigrationsDoc = async (db: any) => {
-  // get the migrations doc
-  try {
-    return await db.get(DocumentType.MIGRATIONS)
-  } catch (err: any) {
-    if (err.status && err.status === 404) {
-      return { _id: DocumentType.MIGRATIONS }
-    } else {
-      throw err
-    }
-  }
-}
-
-export const backPopulateMigrations = async (opts: MigrationNoOpOptions) => {
-  // filter migrations to the type and populate a no-op migration
-  const migrations: Migration[] = DEFINITIONS.filter(
-    def => def.type === opts.type
-  ).map(d => ({ ...d, fn: async () => {} }))
-  await runMigrations(migrations, { noOp: opts })
-}
-
-export const runMigration = async (
-  migration: Migration,
-  options: MigrationOptions = {}
-) => {
-  const migrationType = migration.type
-  const migrationName = migration.name
-  const silent = migration.silent
-
-  const log = (message: string) => {
-    if (!silent) {
-      console.log(message)
-    }
-  }
-
-  // get the db to store the migration in
-  let dbNames: string[]
-  if (migrationType === MigrationType.GLOBAL) {
-    dbNames = [getGlobalDBName()]
-  } else if (migrationType === MigrationType.APP) {
-    if (options.noOp) {
-      if (!options.noOp.appId) {
-        throw new Error("appId is required for noOp app migration")
-      }
-      dbNames = [options.noOp.appId]
-    } else {
-      const apps = (await getAllApps(migration.appOpts)) as App[]
-      dbNames = apps.map(app => app.appId)
-    }
-  } else if (migrationType === MigrationType.INSTALLATION) {
-    dbNames = [StaticDatabases.PLATFORM_INFO.name]
-  } else {
-    throw new Error(`Unrecognised migration type [${migrationType}]`)
-  }
-
-  const length = dbNames.length
-  let count = 0
-
-  // run the migration against each db
-  for (const dbName of dbNames) {
-    count++
-    const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
-
-    const db = getDB(dbName)
-
-    try {
-      const doc = await getMigrationsDoc(db)
-
-      // the migration has already been run
-      if (doc[migrationName]) {
-        // check for force
-        if (
-          options.force &&
-          options.force[migrationType] &&
-          options.force[migrationType].includes(migrationName)
-        ) {
-          log(`[Migration: ${migrationName}] [DB: ${dbName}] Forcing`)
-        } else {
-          // no force, exit
-          return
-        }
-      }
-
-      // check if the migration is not a no-op
-      if (!options.noOp) {
-        log(
-          `[Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
-        )
-
-        if (migration.preventRetry) {
-          // eagerly set the completion date
-          // so that we never run this migration twice even upon failure
-          doc[migrationName] = Date.now()
-          const response = await db.put(doc)
-          doc._rev = response.rev
-        }
-
-        // run the migration
-        if (migrationType === MigrationType.APP) {
-          await context.doInAppContext(db.name, async () => {
-            await migration.fn(db)
-          })
-        } else {
-          await migration.fn(db)
-        }
-
-        log(`[Migration: ${migrationName}] [DB: ${dbName}] Complete`)
-      }
-
-      // mark as complete
-      doc[migrationName] = Date.now()
-      await db.put(doc)
-    } catch (err) {
-      console.error(
-        `[Migration: ${migrationName}] [DB: ${dbName}] Error: `,
-        err
-      )
-      throw err
-    }
-  }
-}
-
-export const runMigrations = async (
-  migrations: Migration[],
-  options: MigrationOptions = {}
-) => {
-  let tenantIds
-
-  if (environment.MULTI_TENANCY) {
-    if (options.noOp) {
-      tenantIds = [options.noOp.tenantId]
-    } else if (!options.tenantIds || !options.tenantIds.length) {
-      // run for all tenants
-      tenantIds = await platform.tenants.getTenantIds()
-    } else {
-      tenantIds = options.tenantIds
-    }
-  } else {
-    // single tenancy
-    tenantIds = [DEFAULT_TENANT_ID]
-  }
-
-  if (tenantIds.length > 1) {
-    console.log(`Checking migrations for ${tenantIds.length} tenants`)
-  } else {
-    console.log("Checking migrations")
-  }
-
-  let count = 0
-  // for all tenants
-  for (const tenantId of tenantIds) {
-    count++
-    if (tenantIds.length > 1) {
-      console.log(`Progress [${count}/${tenantIds.length}]`)
-    }
-    // for all migrations
-    for (const migration of migrations) {
-      // run the migration
-      await context.doInTenant(
-        tenantId,
-        async () => await runMigration(migration, options)
-      )
-    }
-  }
-  console.log("Migrations complete")
-}

packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap

@@ -1,11 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`migrations should match snapshot 1`] = `
-{
-  "_id": "migrations",
-  "_rev": "1-2f64479842a0513aa8b97f356b0b9127",
-  "createdAt": "2020-01-01T00:00:00.000Z",
-  "test": 1577836800000,
-  "updatedAt": "2020-01-01T00:00:00.000Z",
-}
-`;

packages/backend-core/src/migrations/tests/migrations.spec.ts

@@ -1,64 +0,0 @@
-import { testEnv, DBTestConfiguration } from "../../../tests/extra"
-import * as migrations from "../index"
-import * as context from "../../context"
-import { MigrationType } from "@budibase/types"
-
-testEnv.multiTenant()
-
-describe("migrations", () => {
-  const config = new DBTestConfiguration()
-
-  const migrationFunction = jest.fn()
-
-  const MIGRATIONS = [
-    {
-      type: MigrationType.GLOBAL,
-      name: "test" as any,
-      fn: migrationFunction,
-    },
-  ]
-
-  beforeEach(() => {
-    config.newTenant()
-  })
-
-  afterEach(async () => {
-    jest.clearAllMocks()
-  })
-
-  const migrate = () => {
-    return migrations.runMigrations(MIGRATIONS, {
-      tenantIds: [config.tenantId],
-    })
-  }
-
-  it("should run a new migration", async () => {
-    await config.doInTenant(async () => {
-      await migrate()
-      expect(migrationFunction).toHaveBeenCalled()
-      const db = context.getGlobalDB()
-      const doc = await migrations.getMigrationsDoc(db)
-      expect(doc.test).toBeDefined()
-    })
-  })
-
-  it("should match snapshot", async () => {
-    await config.doInTenant(async () => {
-      await migrate()
-      const doc = await migrations.getMigrationsDoc(context.getGlobalDB())
-      expect(doc).toMatchSnapshot()
-    })
-  })
-
-  it("should skip a previously run migration", async () => {
-    await config.doInTenant(async () => {
-      const db = context.getGlobalDB()
-      await migrate()
-      const previousDoc = await migrations.getMigrationsDoc(db)
-      await migrate()
-      const currentDoc = await migrations.getMigrationsDoc(db)
-      expect(migrationFunction).toHaveBeenCalledTimes(1)
-      expect(currentDoc.test).toBe(previousDoc.test)
-    })
-  })
-})

packages/bbui/src/Icon/Icon.svelte

@@ -1,23 +1,23 @@
-<script>
+<script lang="ts">
   import {
     default as AbsTooltip,
     TooltipPosition,
     TooltipType,
   } from "../Tooltip/AbsTooltip.svelte"
 
-  export let name = "Add"
-  export let hidden = false
+  export let name: string = "Add"
+  export let hidden: boolean = false
   export let size = "M"
-  export let hoverable = false
-  export let disabled = false
-  export let color = undefined
-  export let hoverColor = undefined
-  export let tooltip = undefined
+  export let hoverable: boolean = false
+  export let disabled: boolean = false
+  export let color: string | undefined = undefined
+  export let hoverColor: string | undefined = undefined
+  export let tooltip: string | undefined = undefined
   export let tooltipPosition = TooltipPosition.Bottom
   export let tooltipType = TooltipType.Default
-  export let tooltipColor = undefined
-  export let tooltipWrap = true
-  export let newStyles = false
+  export let tooltipColor: string | undefined = undefined
+  export let tooltipWrap: boolean = true
+  export let newStyles: boolean = false
 </script>
 
 <AbsTooltip

packages/bbui/src/Tooltip/AbsTooltip.svelte

@@ -23,7 +23,7 @@
   export let type = TooltipType.Default
   export let text = ""
   export let fixed = false
-  export let color = null
+  export let color = ""
   export let noWrap = false
 
   let wrapper

packages/bbui/src/bbui.css

@@ -45,6 +45,11 @@
   --purple: #806fde;
   --purple-dark: #130080;
 
+  --error-bg: rgba(226, 109, 105, 0.3);
+  --warning-bg: rgba(255, 210, 106, 0.3);
+  --error-content: rgba(226, 109, 105, 0.6);
+  --warning-content: rgba(255, 210, 106, 0.6);
+
   --rounded-small: 4px;
   --rounded-medium: 8px;
   --rounded-large: 16px;

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -293,7 +293,7 @@
               type: RowSelector,
               props: {
                 row: inputData["oldRow"] || {
-                  tableId: inputData["row"].tableId,
+                  tableId: inputData["row"]?.tableId,
                 },
                 meta: {
                   fields: inputData["meta"]?.oldFields || {},

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -12,7 +12,7 @@
     decodeJSBinding,
     encodeJSBinding,
     processObjectSync,
-    processStringSync,
+    processStringWithLogsSync,
   } from "@budibase/string-templates"
   import { readableToRuntimeBinding } from "@/dataBinding"
   import CodeEditor from "../CodeEditor/CodeEditor.svelte"
@@ -41,6 +41,7 @@
     InsertAtPositionFn,
     JSONValue,
   } from "@budibase/types"
+  import type { Log } from "@budibase/string-templates"
   import type { CompletionContext } from "@codemirror/autocomplete"
 
   const dispatch = createEventDispatcher()
@@ -66,6 +67,7 @@
   let insertAtPos: InsertAtPositionFn | undefined
   let targetMode: BindingMode | null = null
   let expressionResult: string | undefined
+  let expressionLogs: Log[] | undefined
   let expressionError: string | undefined
   let evaluating = false
 
@@ -157,7 +159,7 @@
     (expression: string | null, context: any, snippets: Snippet[]) => {
       try {
         expressionError = undefined
-        expressionResult = processStringSync(
+        const output = processStringWithLogsSync(
           expression || "",
           {
             ...context,
@@ -167,6 +169,8 @@
             noThrow: false,
           }
         )
+        expressionResult = output.result
+        expressionLogs = output.logs
       } catch (err: any) {
         expressionResult = undefined
         expressionError = err
@@ -421,6 +425,7 @@
         <EvaluationSidePanel
           {expressionResult}
           {expressionError}
+          {expressionLogs}
           {evaluating}
           expression={editorValue ? editorValue : ""}
         />

packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte

@@ -4,11 +4,13 @@
   import { Helpers } from "@budibase/bbui"
   import { fade } from "svelte/transition"
   import { UserScriptError } from "@budibase/string-templates"
+  import type { Log } from "@budibase/string-templates"
   import type { JSONValue } from "@budibase/types"
 
   // this can be essentially any primitive response from the JS function
   export let expressionResult: JSONValue | undefined = undefined
   export let expressionError: string | undefined = undefined
+  export let expressionLogs: Log[] = []
   export let evaluating = false
   export let expression: string | null = null
 
@@ -16,6 +18,11 @@
   $: empty = expression == null || expression?.trim() === ""
   $: success = !error && !empty
   $: highlightedResult = highlight(expressionResult)
+  $: highlightedLogs = expressionLogs.map(l => ({
+    log: highlight(l.log.join(", ")),
+    line: l.line,
+    type: l.type,
+  }))
 
   const formatError = (err: any) => {
     if (err.code === UserScriptError.code) {
@@ -25,14 +32,14 @@
   }
 
   // json can be any primitive type
-  const highlight = (json?: any | null) => {
+  const highlight = (json?: JSONValue | null) => {
     if (json == null) {
       return ""
     }
 
     // Attempt to parse and then stringify, in case this is valid result
     try {
-      json = JSON.stringify(JSON.parse(json), null, 2)
+      json = JSON.stringify(JSON.parse(json as any), null, 2)
     } catch (err) {
       // couldn't parse/stringify, just treat it as the raw input
     }
@@ -61,7 +68,7 @@
   <div class="header" class:success class:error>
     <div class="header-content">
       {#if error}
-        <Icon name="Alert" color="var(--spectrum-global-color-red-600)" />
+        <Icon name="Alert" color="var(--error-content)" />
         <div>Error</div>
         {#if evaluating}
           <div transition:fade|local={{ duration: 130 }}>
@@ -90,8 +97,36 @@
     {:else if error}
       {formatError(expressionError)}
     {:else}
-      <!-- eslint-disable-next-line svelte/no-at-html-tags-->
-      {@html highlightedResult}
+      <div class="output-lines">
+        {#each highlightedLogs as logLine}
+          <div
+            class="line"
+            class:error-log={logLine.type === "error"}
+            class:warn-log={logLine.type === "warn"}
+          >
+            <div class="icon-log">
+              {#if logLine.type === "error"}
+                <Icon
+                  size="XS"
+                  name="CloseCircle"
+                  color="var(--error-content)"
+                />
+              {:else if logLine.type === "warn"}
+                <Icon size="XS" name="Alert" color="var(--warning-content)" />
+              {/if}
+              <!-- eslint-disable-next-line svelte/no-at-html-tags-->
+              <span>{@html logLine.log}</span>
+            </div>
+            {#if logLine.line}
+              <span style="color: var(--blue)">:{logLine.line}</span>
+            {/if}
+          </div>
+        {/each}
+        <div class="line">
+          <!-- eslint-disable-next-line svelte/no-at-html-tags-->
+          {@html highlightedResult}
+        </div>
+      </div>
     {/if}
   </div>
 </div>
@@ -130,20 +165,37 @@
     height: 100%;
     z-index: 1;
     position: absolute;
-    opacity: 10%;
   }
   .header.error::before {
-    background: var(--spectrum-global-color-red-400);
+    background: var(--error-bg);
   }
   .body {
     flex: 1 1 auto;
     padding: var(--spacing-m) var(--spacing-l);
     font-family: var(--font-mono);
     font-size: 12px;
-    overflow-y: scroll;
+    overflow-y: auto;
     overflow-x: hidden;
-    white-space: pre-wrap;
+    white-space: pre-line;
     word-wrap: break-word;
     height: 0;
   }
+  .output-lines {
+    display: flex;
+    flex-direction: column;
+    gap: var(--spacing-xs);
+  }
+  .line {
+    border-bottom: var(--border-light);
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+    align-items: end;
+    padding: var(--spacing-s);
+  }
+  .icon-log {
+    display: flex;
+    gap: var(--spacing-s);
+    align-items: start;
+  }
 </style>

packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte

@@ -1,4 +1,5 @@
 <script>
+  import { datasources } from "@/stores/builder"
   import { Divider, Heading } from "@budibase/bbui"
 
   export let dividerState
@@ -6,6 +7,21 @@
   export let dataSet
   export let value
   export let onSelect
+  export let identifiers = ["resourceId"]
+
+  $: displayDatasourceName = $datasources.list.length > 1
+
+  function isSelected(entry) {
+    if (!identifiers.length) {
+      return false
+    }
+    for (const identifier of identifiers) {
+      if (entry[identifier] !== value?.[identifier]) {
+        return false
+      }
+    }
+    return true
+  }
 </script>
 
 {#if dividerState}
@@ -21,15 +37,16 @@
   {#each dataSet as data}
     <li
       class="spectrum-Menu-item"
-      class:is-selected={value?.label === data.label &&
-        value?.type === data.type}
+      class:is-selected={isSelected(data) && value?.type === data.type}
       role="option"
       aria-selected="true"
       tabindex="0"
       on:click={() => onSelect(data)}
     >
       <span class="spectrum-Menu-itemLabel">
-        {data.datasourceName ? `${data.datasourceName} - ` : ""}{data.label}
+        {data.datasourceName && displayDatasourceName
+          ? `${data.datasourceName} - `
+          : ""}{data.label}
       </span>
       <svg
         class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"

packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte

@@ -34,7 +34,7 @@
   import ClientBindingPanel from "@/components/common/bindings/ClientBindingPanel.svelte"
   import DataSourceCategory from "@/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte"
   import { API } from "@/api"
-  import { datasourceSelect as format } from "@/helpers/data/format"
+  import { sortAndFormat } from "@/helpers/data/format"
 
   export let value = {}
   export let otherSources
@@ -51,25 +51,13 @@
   let modal
 
   $: text = value?.label ?? "Choose an option"
-  $: tables = $tablesStore.list
-    .map(table => format.table(table, $datasources.list))
-    .sort((a, b) => {
-      // sort tables alphabetically, grouped by datasource
-      const dsA = a.datasourceName ?? ""
-      const dsB = b.datasourceName ?? ""
-
-      const dsComparison = dsA.localeCompare(dsB)
-      if (dsComparison !== 0) {
-        return dsComparison
-      }
-      return a.label.localeCompare(b.label)
-    })
+  $: tables = sortAndFormat.tables($tablesStore.list, $datasources.list)
   $: viewsV1 = $viewsStore.list.map(view => ({
     ...view,
     label: view.name,
     type: "view",
   }))
-  $: viewsV2 = $viewsV2Store.list.map(format.viewV2)
+  $: viewsV2 = sortAndFormat.viewsV2($viewsV2Store.list, $datasources.list)
   $: views = [...(viewsV1 || []), ...(viewsV2 || [])]
   $: queries = $queriesStore.list
     .filter(q => showAllQueries || q.queryVerb === "read" || q.readable)
@@ -303,6 +291,7 @@
         dataSet={views}
         {value}
         onSelect={handleSelected}
+        identifiers={["tableId", "name"]}
       />
     {/if}
     {#if queries?.length}
@@ -312,6 +301,7 @@
         dataSet={queries}
         {value}
         onSelect={handleSelected}
+        identifiers={["_id"]}
       />
     {/if}
     {#if links?.length}
@@ -321,6 +311,7 @@
         dataSet={links}
         {value}
         onSelect={handleSelected}
+        identifiers={["tableId", "fieldName"]}
       />
     {/if}
     {#if fields?.length}
@@ -330,6 +321,7 @@
         dataSet={fields}
         {value}
         onSelect={handleSelected}
+        identifiers={["providerId", "tableId", "fieldName"]}
       />
     {/if}
     {#if jsonArrays?.length}
@@ -339,6 +331,7 @@
         dataSet={jsonArrays}
         {value}
         onSelect={handleSelected}
+        identifiers={["providerId", "tableId", "fieldName"]}
       />
     {/if}
     {#if showDataProviders && dataProviders?.length}
@@ -348,6 +341,7 @@
         dataSet={dataProviders}
         {value}
         onSelect={handleSelected}
+        identifiers={["providerId"]}
       />
     {/if}
     <DataSourceCategory

packages/builder/src/components/design/settings/controls/TableSelect.svelte

@@ -1,22 +1,32 @@
 <script>
-  import { Select } from "@budibase/bbui"
+  import { Popover, Select } from "@budibase/bbui"
   import { createEventDispatcher, onMount } from "svelte"
-  import { tables as tablesStore, viewsV2 } from "@/stores/builder"
-  import { tableSelect as format } from "@/helpers/data/format"
+  import {
+    tables as tableStore,
+    datasources as datasourceStore,
+    viewsV2 as viewsV2Store,
+  } from "@/stores/builder"
+  import DataSourceCategory from "./DataSourceSelect/DataSourceCategory.svelte"
+  import { sortAndFormat } from "@/helpers/data/format"
 
   export let value
 
+  let anchorRight, dropdownRight
+
   const dispatch = createEventDispatcher()
 
-  $: tables = $tablesStore.list.map(format.table)
-  $: views = $viewsV2.list.map(format.viewV2)
+  $: tables = sortAndFormat.tables($tableStore.list, $datasourceStore.list)
+  $: views = sortAndFormat.viewsV2($viewsV2Store.list, $datasourceStore.list)
   $: options = [...(tables || []), ...(views || [])]
 
+  $: text = value?.label ?? "Choose an option"
+
   const onChange = e => {
     dispatch(
       "change",
-      options.find(x => x.resourceId === e.detail)
+      options.find(x => x.resourceId === e.resourceId)
     )
+    dropdownRight.hide()
   }
 
   onMount(() => {
@@ -29,10 +39,47 @@
   })
 </script>
 
-<Select
-  on:change={onChange}
-  value={value?.resourceId}
-  {options}
-  getOptionValue={x => x.resourceId}
-  getOptionLabel={x => x.label}
-/>
+<div class="container" bind:this={anchorRight}>
+  <Select
+    readonly
+    value={text}
+    options={[text]}
+    on:click={dropdownRight.show}
+  />
+</div>
+<Popover bind:this={dropdownRight} anchor={anchorRight}>
+  <div class="dropdown">
+    <DataSourceCategory
+      heading="Tables"
+      dataSet={tables}
+      {value}
+      onSelect={onChange}
+    />
+    {#if views?.length}
+      <DataSourceCategory
+        dividerState={true}
+        heading="Views"
+        dataSet={views}
+        {value}
+        onSelect={onChange}
+      />
+    {/if}
+  </div>
+</Popover>
+
+<style>
+  .container {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+  }
+  .container :global(:first-child) {
+    flex: 1 1 auto;
+  }
+
+  .dropdown {
+    padding: var(--spacing-m) 0;
+    z-index: 99999999;
+  }
+</style>

packages/builder/src/components/portal/licensing/LicensingOverlays.svelte

@@ -20,7 +20,7 @@
 
   const processModals = () => {
     const defaultCacheFn = key => {
-      temporalStore.actions.setExpiring(key, {}, oneDayInSeconds)
+      temporalStore.setExpiring(key, {}, oneDayInSeconds)
     }
 
     const dismissableModals = [
@@ -50,7 +50,7 @@
       },
     ]
     return dismissableModals.filter(modal => {
-      return !temporalStore.actions.getExpiring(modal.key) && modal.criteria()
+      return !temporalStore.getExpiring(modal.key) && modal.criteria()
     })
   }
 

packages/builder/src/components/portal/licensing/licensingBanners.js

@@ -6,7 +6,7 @@ import { BANNER_TYPES } from "@budibase/bbui"
 const oneDayInSeconds = 86400
 
 const defaultCacheFn = key => {
-  temporalStore.actions.setExpiring(key, {}, oneDayInSeconds)
+  temporalStore.setExpiring(key, {}, oneDayInSeconds)
 }
 
 const upgradeAction = key => {
@@ -148,7 +148,7 @@ export const getBanners = () => {
     buildUsersAboveLimitBanner(ExpiringKeys.LICENSING_USERS_ABOVE_LIMIT_BANNER),
   ].filter(licensingBanner => {
     return (
-      !temporalStore.actions.getExpiring(licensingBanner.key) &&
+      !temporalStore.getExpiring(licensingBanner.key) &&
       licensingBanner.criteria()
     )
   })

packages/builder/src/helpers/data/format.js

@@ -9,11 +9,18 @@ export const datasourceSelect = {
       datasourceName: datasource?.name,
     }
   },
-  viewV2: view => ({
-    ...view,
-    label: view.name,
-    type: "viewV2",
-  }),
+  viewV2: (view, datasources) => {
+    const datasource = datasources
+      ?.filter(f => f.entities)
+      .flatMap(d => d.entities)
+      .find(ds => ds._id === view.tableId)
+    return {
+      ...view,
+      label: view.name,
+      type: "viewV2",
+      datasourceName: datasource?.name,
+    }
+  },
 }
 
 export const tableSelect = {
@@ -31,3 +38,36 @@ export const tableSelect = {
     resourceId: view.id,
   }),
 }
+
+export const sortAndFormat = {
+  tables: (tables, datasources) => {
+    return tables
+      .map(table => {
+        const formatted = datasourceSelect.table(table, datasources)
+        return {
+          ...formatted,
+          resourceId: table._id,
+        }
+      })
+      .sort((a, b) => {
+        // sort tables alphabetically, grouped by datasource
+        const dsA = a.datasourceName ?? ""
+        const dsB = b.datasourceName ?? ""
+
+        const dsComparison = dsA.localeCompare(dsB)
+        if (dsComparison !== 0) {
+          return dsComparison
+        }
+        return a.label.localeCompare(b.label)
+      })
+  },
+  viewsV2: (views, datasources) => {
+    return views.map(view => {
+      const formatted = datasourceSelect.viewV2(view, datasources)
+      return {
+        ...formatted,
+        resourceId: view.id,
+      }
+    })
+  },
+}

packages/builder/src/helpers/index.ts

@@ -9,3 +9,4 @@ export {
   lowercase,
   isBuilderInputFocused,
 } from "./helpers"
+export * as featureFlag from "./featureFlags"

packages/builder/src/helpers/screen.ts

@@ -0,0 +1,46 @@
+import { Component, Screen, ScreenProps } from "@budibase/types"
+import clientManifest from "@budibase/client/manifest.json"
+
+export function findComponentsBySettingsType(
+  screen: Screen,
+  type: string | string[]
+) {
+  const typesArray = Array.isArray(type) ? type : [type]
+
+  const result: {
+    component: Component
+    setting: {
+      type: string
+      key: string
+    }
+  }[] = []
+  function recurseFieldComponentsInChildren(component: ScreenProps) {
+    if (!component) {
+      return
+    }
+
+    const definition = getManifestDefinition(component)
+    const setting =
+      "settings" in definition &&
+      definition.settings.find((s: any) => typesArray.includes(s.type))
+    if (setting && "type" in setting) {
+      result.push({
+        component,
+        setting: { type: setting.type!, key: setting.key! },
+      })
+    }
+    component._children?.forEach(child => {
+      recurseFieldComponentsInChildren(child)
+    })
+  }
+
+  recurseFieldComponentsInChildren(screen?.props)
+  return result
+}
+
+function getManifestDefinition(component: Component) {
+  const componentType = component._component.split("/").slice(-1)[0]
+  const definition =
+    clientManifest[componentType as keyof typeof clientManifest]
+  return definition
+}

packages/builder/src/pages/builder/_layout.svelte

@@ -18,7 +18,7 @@
 
   $: useAccountPortal = cloud && !$admin.disableAccountPortal
 
-  navigation.actions.init($redirect)
+  navigation.init($redirect)
 
   const validateTenantId = async () => {
     const host = window.location.host

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte

@@ -11,6 +11,7 @@
     selectedScreen,
     hoverStore,
     componentTreeNodesStore,
+    screenComponentErrors,
     snippets,
   } from "@/stores/builder"
   import ConfirmDialog from "@/components/common/ConfirmDialog.svelte"
@@ -68,6 +69,7 @@
       port: window.location.port,
     },
     snippets: $snippets,
+    componentErrors: $screenComponentErrors,
   }
 
   // Refresh the preview when required

packages/builder/src/pages/builder/apps/index.svelte

@@ -15,6 +15,7 @@
   import {
     appsStore,
     organisation,
+    admin,
     auth,
     groups,
     licensing,
@@ -42,6 +43,7 @@
     app => app.status === AppStatus.DEPLOYED
   )
   $: userApps = getUserApps(publishedApps, userGroups, $auth.user)
+  $: isOwner = $auth.accountPortalAccess && $admin.cloud
 
   function getUserApps(publishedApps, userGroups, user) {
     if (sdk.users.isAdmin(user)) {
@@ -111,7 +113,13 @@
               </MenuItem>
               <MenuItem
                 icon="LockClosed"
-                on:click={() => changePasswordModal.show()}
+                on:click={() => {
+                  if (isOwner) {
+                    window.location.href = `${$admin.accountPortalUrl}/portal/account`
+                  } else {
+                    changePasswordModal.show()
+                  }
+                }}
               >
                 Update password
               </MenuItem>

packages/builder/src/pages/builder/auth/reset.svelte

@@ -30,10 +30,16 @@
     try {
       loading = true
       if (forceResetPassword) {
+        const email = $auth.user.email
+        const tenantId = $auth.user.tenantId
         await auth.updateSelf({
           password,
           forceResetPassword: false,
         })
+        if (!$auth.user) {
+          // Update self will clear the platform user, so need to login
+          await auth.login(email, password, tenantId)
+        }
         $goto("../portal/")
       } else {
         await auth.resetPassword(password, resetCode)

packages/builder/src/pages/builder/portal/_components/UserDropdown.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { auth } from "@/stores/portal"
+  import { admin, auth } from "@/stores/portal"
   import { ActionMenu, MenuItem, Icon, Modal } from "@budibase/bbui"
   import { goto } from "@roxi/routify"
   import ProfileModal from "@/components/settings/ProfileModal.svelte"
@@ -13,6 +13,8 @@
   let updatePasswordModal
   let apiKeyModal
 
+  $: isOwner = $auth.accountPortalAccess && $admin.cloud
+
   const logout = async () => {
     try {
       await auth.logout()
@@ -32,7 +34,16 @@
   </MenuItem>
   <MenuItem icon="Moon" on:click={() => themeModal.show()}>Theme</MenuItem>
   {#if !$auth.isSSO}
-    <MenuItem icon="LockClosed" on:click={() => updatePasswordModal.show()}>
+    <MenuItem
+      icon="LockClosed"
+      on:click={() => {
+        if (isOwner) {
+          window.location.href = `${$admin.accountPortalUrl}/portal/account`
+        } else {
+          updatePasswordModal.show()
+        }
+      }}
+    >
       Update password
     </MenuItem>
   {/if}

packages/builder/src/stores/builder/index.js

@@ -16,6 +16,7 @@ import { userStore, userSelectedResourceMap, isOnlyUser } from "./users.js"
 import { deploymentStore } from "./deployments.js"
 import { contextMenuStore } from "./contextMenu.js"
 import { snippets } from "./snippets"
+import { screenComponentErrors } from "./screenComponent"
 
 // Backend
 import { tables } from "./tables"
@@ -67,6 +68,7 @@ export {
   snippets,
   rowActions,
   appPublished,
+  screenComponentErrors,
 }
 
 export const reset = () => {

packages/builder/src/stores/builder/screenComponent.ts

@@ -0,0 +1,83 @@
+import { derived } from "svelte/store"
+import { tables } from "./tables"
+import { selectedScreen } from "./screens"
+import { viewsV2 } from "./viewsV2"
+import { findComponentsBySettingsType } from "@/helpers/screen"
+import { UIDatasourceType, Screen } from "@budibase/types"
+import { queries } from "./queries"
+import { views } from "./views"
+import { featureFlag } from "@/helpers"
+
+function reduceBy<TItem extends {}, TKey extends keyof TItem>(
+  key: TKey,
+  list: TItem[]
+) {
+  return list.reduce(
+    (result, item) => ({
+      ...result,
+      [item[key] as string]: item,
+    }),
+    {}
+  )
+}
+
+const friendlyNameByType: Partial<Record<UIDatasourceType, string>> = {
+  viewV2: "view",
+}
+
+const validationKeyByType: Record<UIDatasourceType, string | null> = {
+  table: "tableId",
+  view: "name",
+  viewV2: "id",
+  query: "_id",
+  custom: null,
+}
+
+export const screenComponentErrors = derived(
+  [selectedScreen, tables, views, viewsV2, queries],
+  ([$selectedScreen, $tables, $views, $viewsV2, $queries]): Record<
+    string,
+    string[]
+  > => {
+    if (!featureFlag.isEnabled("CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS")) {
+      return {}
+    }
+    function getInvalidDatasources(
+      screen: Screen,
+      datasources: Record<string, any>
+    ) {
+      const result: Record<string, string[]> = {}
+      for (const { component, setting } of findComponentsBySettingsType(
+        screen,
+        ["table", "dataSource"]
+      )) {
+        const componentSettings = component[setting.key]
+        const { label } = componentSettings
+        const type = componentSettings.type as UIDatasourceType
+
+        const validationKey = validationKeyByType[type]
+        if (!validationKey) {
+          continue
+        }
+        const resourceId = componentSettings[validationKey]
+        if (!datasources[resourceId]) {
+          const friendlyTypeName = friendlyNameByType[type] ?? type
+          result[component._id!] = [
+            `The ${friendlyTypeName} named "${label}" could not be found`,
+          ]
+        }
+      }
+
+      return result
+    }
+
+    const datasources = {
+      ...reduceBy("_id", $tables.list),
+      ...reduceBy("name", $views.list),
+      ...reduceBy("id", $viewsV2.list),
+      ...reduceBy("_id", $queries.list),
+    }
+
+    return getInvalidDatasources($selectedScreen, datasources)
+  }
+)

packages/builder/src/stores/portal/admin.test.js

@@ -1,5 +1,5 @@
 import { it, expect, describe, beforeEach, vi } from "vitest"
-import { createAdminStore } from "./admin"
+import { AdminStore } from "./admin"
 import { writable, get } from "svelte/store"
 import { API } from "@/api"
 import { auth } from "@/stores/portal"
@@ -46,16 +46,7 @@ describe("admin store", () => {
     ctx.writableReturn = { update: vi.fn(), subscribe: vi.fn() }
     writable.mockReturnValue(ctx.writableReturn)
 
-    ctx.returnedStore = createAdminStore()
-  })
-
-  it("returns the created store", ctx => {
-    expect(ctx.returnedStore).toEqual({
-      subscribe: expect.toBe(ctx.writableReturn.subscribe),
-      init: expect.toBeFunc(),
-      unload: expect.toBeFunc(),
-      getChecklist: expect.toBeFunc(),
-    })
+    ctx.returnedStore = new AdminStore()
   })
 
   describe("init method", () => {

packages/builder/src/stores/portal/admin.ts

@@ -1,4 +1,4 @@
-import { writable, get } from "svelte/store"
+import { get } from "svelte/store"
 import { API } from "@/api"
 import { auth } from "@/stores/portal"
 import { banner } from "@budibase/bbui"
@@ -7,42 +7,44 @@ import {
   GetEnvironmentResponse,
   SystemStatusResponse,
 } from "@budibase/types"
+import { BudiStore } from "../BudiStore"
 
-interface PortalAdminStore extends GetEnvironmentResponse {
+interface AdminState extends GetEnvironmentResponse {
   loaded: boolean
   checklist?: ConfigChecklistResponse
   status?: SystemStatusResponse
 }
 
-export function createAdminStore() {
-  const admin = writable<PortalAdminStore>({
-    loaded: false,
-    multiTenancy: false,
-    cloud: false,
-    isDev: false,
-    disableAccountPortal: false,
-    offlineMode: false,
-    maintenance: [],
-  })
+export class AdminStore extends BudiStore<AdminState> {
+  constructor() {
+    super({
+      loaded: false,
+      multiTenancy: false,
+      cloud: false,
+      isDev: false,
+      disableAccountPortal: false,
+      offlineMode: false,
+      maintenance: [],
+    })
+  }
 
-  async function init() {
-    await getChecklist()
-    await getEnvironment()
+  async init() {
+    await this.getChecklist()
+    await this.getEnvironment()
     // enable system status checks in the cloud
-    if (get(admin).cloud) {
-      await getSystemStatus()
-      checkStatus()
+    if (get(this.store).cloud) {
+      await this.getSystemStatus()
+      this.checkStatus()
     }
-
-    admin.update(store => {
+    this.update(store => {
       store.loaded = true
       return store
     })
   }
 
-  async function getEnvironment() {
+  async getEnvironment() {
     const environment = await API.getEnvironment()
-    admin.update(store => {
+    this.update(store => {
       store.multiTenancy = environment.multiTenancy
       store.cloud = environment.cloud
       store.disableAccountPortal = environment.disableAccountPortal
@@ -56,43 +58,36 @@ export function createAdminStore() {
     })
   }
 
-  const checkStatus = async () => {
-    const health = get(admin)?.status?.health
+  async checkStatus() {
+    const health = get(this.store).status?.health
     if (!health?.passing) {
       await banner.showStatus()
     }
   }
 
-  async function getSystemStatus() {
+  async getSystemStatus() {
     const status = await API.getSystemStatus()
-    admin.update(store => {
+    this.update(store => {
       store.status = status
       return store
     })
   }
 
-  async function getChecklist() {
+  async getChecklist() {
     const tenantId = get(auth).tenantId
     const checklist = await API.getChecklist(tenantId)
-    admin.update(store => {
+    this.update(store => {
       store.checklist = checklist
       return store
     })
   }
 
-  function unload() {
-    admin.update(store => {
+  unload() {
+    this.update(store => {
       store.loaded = false
       return store
     })
   }
-
-  return {
-    subscribe: admin.subscribe,
-    init,
-    unload,
-    getChecklist,
-  }
 }
 
-export const admin = createAdminStore()
+export const admin = new AdminStore()

packages/builder/src/stores/portal/auditLogs.ts

@@ -13,7 +13,7 @@ interface PortalAuditLogsStore {
   logs?: SearchAuditLogsResponse
 }
 
-export class AuditLogsStore extends BudiStore<PortalAuditLogsStore> {
+class AuditLogsStore extends BudiStore<PortalAuditLogsStore> {
   constructor() {
     super({})
   }

packages/builder/src/stores/portal/auth.ts

@@ -121,8 +121,8 @@ class AuthStore extends BudiStore<PortalAuthStore> {
     }
   }
 
-  async login(username: string, password: string) {
-    const tenantId = get(this.store).tenantId
+  async login(username: string, password: string, targetTenantId?: string) {
+    const tenantId = targetTenantId || get(this.store).tenantId
     await API.logIn(tenantId, username, password)
     await this.getSelf()
   }

packages/builder/src/stores/portal/navigation.ts

@@ -1,38 +1,31 @@
-import { writable } from "svelte/store"
+import { BudiStore } from "../BudiStore"
 
 type GotoFuncType = (path: string) => void
 
-interface PortalNavigationStore {
+interface NavigationState {
   initialisated: boolean
   goto: GotoFuncType
 }
 
-export function createNavigationStore() {
-  const store = writable<PortalNavigationStore>({
-    initialisated: false,
-    goto: undefined as any,
-  })
-  const { set, subscribe } = store
+class NavigationStore extends BudiStore<NavigationState> {
+  constructor() {
+    super({
+      initialisated: false,
+      goto: undefined as any,
+    })
+  }
 
-  const init = (gotoFunc: GotoFuncType) => {
+  init(gotoFunc: GotoFuncType) {
     if (typeof gotoFunc !== "function") {
       throw new Error(
         `gotoFunc must be a function, found a "${typeof gotoFunc}" instead`
       )
     }
-
-    set({
+    this.set({
       initialisated: true,
       goto: gotoFunc,
     })
   }
-
-  return {
-    subscribe,
-    actions: {
-      init,
-    },
-  }
 }
 
-export const navigation = createNavigationStore()
+export const navigation = new NavigationStore()

packages/builder/src/stores/portal/templates.js

@@ -1,16 +0,0 @@
-import { writable } from "svelte/store"
-import { API } from "@/api"
-
-export function templatesStore() {
-  const { subscribe, set } = writable([])
-
-  return {
-    subscribe,
-    load: async () => {
-      const templates = await API.getAppTemplates()
-      set(templates)
-    },
-  }
-}
-
-export const templates = templatesStore()

packages/builder/src/stores/portal/templates.ts

@@ -0,0 +1,16 @@
+import { API } from "@/api"
+import { BudiStore } from "../BudiStore"
+import { TemplateMetadata } from "@budibase/types"
+
+class TemplateStore extends BudiStore<TemplateMetadata[]> {
+  constructor() {
+    super([])
+  }
+
+  async load() {
+    const templates = await API.getAppTemplates()
+    this.set(templates)
+  }
+}
+
+export const templates = new TemplateStore()

packages/builder/src/stores/portal/temporal.js

@@ -1,45 +0,0 @@
-import { createLocalStorageStore } from "@budibase/frontend-core"
-import { get } from "svelte/store"
-
-export const createTemporalStore = () => {
-  const initialValue = {}
-
-  const localStorageKey = `bb-temporal`
-  const store = createLocalStorageStore(localStorageKey, initialValue)
-
-  const setExpiring = (key, data, duration) => {
-    const updated = {
-      ...data,
-      expiry: Date.now() + duration * 1000,
-    }
-
-    store.update(state => ({
-      ...state,
-      [key]: updated,
-    }))
-  }
-
-  const getExpiring = key => {
-    const entry = get(store)[key]
-    if (!entry) {
-      return
-    }
-    const currentExpiry = entry.expiry
-    if (currentExpiry < Date.now()) {
-      store.update(state => {
-        delete state[key]
-        return state
-      })
-      return null
-    } else {
-      return entry
-    }
-  }
-
-  return {
-    subscribe: store.subscribe,
-    actions: { setExpiring, getExpiring },
-  }
-}
-
-export const temporalStore = createTemporalStore()

packages/builder/src/stores/portal/temporal.ts

@@ -0,0 +1,53 @@
+import { get } from "svelte/store"
+import { BudiStore, PersistenceType } from "../BudiStore"
+
+type TemporalItem = Record<string, any> & { expiry: number }
+type TemporalState = Record<string, TemporalItem>
+
+class TemporalStore extends BudiStore<TemporalState> {
+  constructor() {
+    super(
+      {},
+      {
+        persistence: {
+          key: "bb-temporal",
+          type: PersistenceType.LOCAL,
+        },
+      }
+    )
+  }
+
+  setExpiring = (
+    key: string,
+    data: Record<string, any>,
+    durationSeconds: number
+  ) => {
+    const updated: TemporalItem = {
+      ...data,
+      expiry: Date.now() + durationSeconds * 1000,
+    }
+    this.update(state => ({
+      ...state,
+      [key]: updated,
+    }))
+  }
+
+  getExpiring(key: string) {
+    const entry = get(this.store)[key]
+    if (!entry) {
+      return null
+    }
+    const currentExpiry = entry.expiry
+    if (currentExpiry < Date.now()) {
+      this.update(state => {
+        delete state[key]
+        return state
+      })
+      return null
+    } else {
+      return entry
+    }
+  }
+}
+
+export const temporalStore = new TemporalStore()

packages/builder/src/stores/portal/theme.js

@@ -1,37 +0,0 @@
-import { createLocalStorageStore } from "@budibase/frontend-core"
-import { derived } from "svelte/store"
-import {
-  DefaultBuilderTheme,
-  ensureValidTheme,
-  getThemeClassNames,
-  ThemeOptions,
-  ThemeClassPrefix,
-} from "@budibase/shared-core"
-
-export const getThemeStore = () => {
-  const themeElement = document.documentElement
-  const initialValue = {
-    theme: DefaultBuilderTheme,
-  }
-  const store = createLocalStorageStore("bb-theme", initialValue)
-  const derivedStore = derived(store, $store => ({
-    ...$store,
-    theme: ensureValidTheme($store.theme, DefaultBuilderTheme),
-  }))
-
-  // Update theme class when store changes
-  derivedStore.subscribe(({ theme }) => {
-    const classNames = getThemeClassNames(theme).split(" ")
-    ThemeOptions.forEach(option => {
-      const className = `${ThemeClassPrefix}${option.id}`
-      themeElement.classList.toggle(className, classNames.includes(className))
-    })
-  })
-
-  return {
-    ...store,
-    subscribe: derivedStore.subscribe,
-  }
-}
-
-export const themeStore = getThemeStore()

packages/builder/src/stores/portal/theme.ts

@@ -0,0 +1,45 @@
+import { derived, Writable } from "svelte/store"
+import {
+  DefaultBuilderTheme,
+  ensureValidTheme,
+  getThemeClassNames,
+  ThemeOptions,
+  ThemeClassPrefix,
+} from "@budibase/shared-core"
+import { Theme } from "@budibase/types"
+import { DerivedBudiStore, PersistenceType } from "../BudiStore"
+
+interface ThemeState {
+  theme: Theme
+}
+
+class ThemeStore extends DerivedBudiStore<ThemeState, ThemeState> {
+  constructor() {
+    const makeDerivedStore = (store: Writable<ThemeState>) => {
+      return derived(store, $store => ({
+        ...$store,
+        theme: ensureValidTheme($store.theme, DefaultBuilderTheme),
+      }))
+    }
+    super({ theme: DefaultBuilderTheme }, makeDerivedStore, {
+      persistence: {
+        key: "bb-theme",
+        type: PersistenceType.LOCAL,
+      },
+    })
+
+    // Update theme class when store changes
+    this.subscribe(({ theme }) => {
+      const classNames = getThemeClassNames(theme).split(" ")
+      ThemeOptions.forEach(option => {
+        const className = `${ThemeClassPrefix}${option.id}`
+        document.documentElement.classList.toggle(
+          className,
+          classNames.includes(className)
+        )
+      })
+    })
+  }
+}
+
+export const themeStore = new ThemeStore()

packages/client/src/components/Component.svelte

@@ -103,6 +103,7 @@
   let settingsDefinition
   let settingsDefinitionMap
   let missingRequiredSettings = false
+  let componentErrors = false
 
   // Temporary styles which can be added in the app preview for things like
   // DND. We clear these whenever a new instance is received.
@@ -137,16 +138,21 @@
 
   // Derive definition properties which can all be optional, so need to be
   // coerced to booleans
+  $: componentErrors = instance?._meta?.errors
   $: hasChildren = !!definition?.hasChildren
   $: showEmptyState = definition?.showEmptyState !== false
   $: hasMissingRequiredSettings = missingRequiredSettings?.length > 0
   $: editable = !!definition?.editable && !hasMissingRequiredSettings
+  $: hasComponentErrors = componentErrors?.length > 0
   $: requiredAncestors = definition?.requiredAncestors || []
   $: missingRequiredAncestors = requiredAncestors.filter(
     ancestor => !$component.ancestors.includes(`${BudibasePrefix}${ancestor}`)
   )
   $: hasMissingRequiredAncestors = missingRequiredAncestors?.length > 0
-  $: errorState = hasMissingRequiredSettings || hasMissingRequiredAncestors
+  $: errorState =
+    hasMissingRequiredSettings ||
+    hasMissingRequiredAncestors ||
+    hasComponentErrors
 
   // Interactive components can be selected, dragged and highlighted inside
   // the builder preview
@@ -692,6 +698,7 @@
       <ComponentErrorState
         {missingRequiredSettings}
         {missingRequiredAncestors}
+        {componentErrors}
       />
     {:else}
       <svelte:component this={constructor} bind:this={ref} {...initialSettings}>

packages/client/src/components/app/DataProvider.svelte

@@ -14,7 +14,6 @@
     GroupUserDatasource,
     DataFetchOptions,
   } from "@budibase/types"
-  import { SDK, Component } from "../../index"
 
   type ProviderDatasource = Exclude<
     DataFetchDatasource,
@@ -29,8 +28,8 @@
   export let paginate: boolean
   export let autoRefresh: number
 
-  const { styleable, Provider, ActionTypes, API } = getContext<SDK>("sdk")
-  const component = getContext<Component>("component")
+  const { styleable, Provider, ActionTypes, API } = getContext("sdk")
+  const component = getContext("component")
 
   let interval: ReturnType<typeof setInterval>
   let queryExtensions: Record<string, any> = {}

packages/client/src/components/app/blocks/form/FormBlock.svelte

@@ -1,37 +1,43 @@
-<script>
+<script lang="ts">
   import { getContext } from "svelte"
   import InnerFormBlock from "./InnerFormBlock.svelte"
   import { Utils } from "@budibase/frontend-core"
   import FormBlockWrapper from "./FormBlockWrapper.svelte"
   import { get } from "svelte/store"
+  import { TableSchema, UIDatasource } from "@budibase/types"
 
-  export let actionType
-  export let dataSource
-  export let size
-  export let disabled
-  export let fields
-  export let buttons
-  export let buttonPosition
-  export let title
-  export let description
-  export let rowId
-  export let actionUrl
-  export let noRowsMessage
-  export let notificationOverride
-  export let buttonsCollapsed
-  export let buttonsCollapsedText
+  type Field = { name: string; active: boolean }
+
+  export let actionType: string
+  export let dataSource: UIDatasource
+  export let size: string
+  export let disabled: boolean
+  export let fields: (Field | string)[]
+  export let buttons: {
+    "##eventHandlerType": string
+    parameters: Record<string, string>
+  }[]
+  export let buttonPosition: "top" | "bottom"
+  export let title: string
+  export let description: string
+  export let rowId: string
+  export let actionUrl: string
+  export let noRowsMessage: string
+  export let notificationOverride: boolean
+  export let buttonsCollapsed: boolean
+  export let buttonsCollapsedText: string
 
   // Legacy
-  export let showDeleteButton
-  export let showSaveButton
-  export let saveButtonLabel
-  export let deleteButtonLabel
+  export let showDeleteButton: boolean
+  export let showSaveButton: boolean
+  export let saveButtonLabel: boolean
+  export let deleteButtonLabel: boolean
 
   const { fetchDatasourceSchema, generateGoldenSample } = getContext("sdk")
   const component = getContext("component")
   const context = getContext("context")
 
-  let schema
+  let schema: TableSchema
 
   $: fetchSchema(dataSource)
   $: id = $component.id
@@ -61,7 +67,7 @@
     }
   }
 
-  const convertOldFieldFormat = fields => {
+  const convertOldFieldFormat = (fields: (Field | string)[]): Field[] => {
     if (!fields) {
       return []
     }
@@ -82,11 +88,11 @@
     })
   }
 
-  const getDefaultFields = (fields, schema) => {
+  const getDefaultFields = (fields: Field[], schema: TableSchema) => {
     if (!schema) {
       return []
     }
-    let defaultFields = []
+    let defaultFields: Field[] = []
 
     if (!fields || fields.length === 0) {
       Object.values(schema)
@@ -101,15 +107,14 @@
     return [...fields, ...defaultFields].filter(field => field.active)
   }
 
-  const fetchSchema = async () => {
-    schema = (await fetchDatasourceSchema(dataSource)) || {}
+  const fetchSchema = async (datasource: UIDatasource) => {
+    schema = (await fetchDatasourceSchema(datasource)) || {}
   }
 </script>
 
 <FormBlockWrapper {actionType} {dataSource} {rowId} {noRowsMessage}>
   <InnerFormBlock
     {dataSource}
-    {actionUrl}
     {actionType}
     {size}
     {disabled}
@@ -117,7 +122,6 @@
     {title}
     {description}
     {schema}
-    {notificationOverride}
     buttons={buttonsOrDefault}
     buttonPosition={buttons ? buttonPosition : "top"}
     {buttonsCollapsed}

packages/client/src/components/error-states/ComponentErrorState.svelte

@@ -1,11 +1,14 @@
-<script>
+<script lang="ts">
   import { getContext } from "svelte"
   import { Icon } from "@budibase/bbui"
   import MissingRequiredSetting from "./MissingRequiredSetting.svelte"
   import MissingRequiredAncestor from "./MissingRequiredAncestor.svelte"
 
-  export let missingRequiredSettings
-  export let missingRequiredAncestors
+  export let missingRequiredSettings:
+    | { key: string; label: string }[]
+    | undefined
+  export let missingRequiredAncestors: string[] | undefined
+  export let componentErrors: string[] | undefined
 
   const component = getContext("component")
   const { styleable, builderStore } = getContext("sdk")
@@ -13,6 +16,7 @@
   $: styles = { ...$component.styles, normal: {}, custom: null, empty: true }
   $: requiredSetting = missingRequiredSettings?.[0]
   $: requiredAncestor = missingRequiredAncestors?.[0]
+  $: errorMessage = componentErrors?.[0]
 </script>
 
 {#if $builderStore.inBuilder}
@@ -21,6 +25,8 @@
       <Icon name="Alert" color="var(--spectrum-global-color-static-red-600)" />
       {#if requiredAncestor}
         <MissingRequiredAncestor {requiredAncestor} />
+      {:else if errorMessage}
+        {errorMessage}
       {:else if requiredSetting}
         <MissingRequiredSetting {requiredSetting} />
       {/if}
@@ -32,7 +38,7 @@
   .component-placeholder {
     display: flex;
     flex-direction: row;
-    justify-content: flex-start;
+    justify-content: center;
     align-items: center;
     color: var(--spectrum-global-color-gray-600);
     font-size: var(--font-size-s);

packages/client/src/context.d.ts

@@ -0,0 +1,7 @@
+import { Component, Context, SDK } from "."
+
+declare module "svelte" {
+  export function getContext(key: "sdk"): SDK
+  export function getContext(key: "component"): Component
+  export function getContext(key: "context"): Context
+}

packages/client/src/index.js

@@ -43,6 +43,7 @@ const loadBudibase = async () => {
     usedPlugins: window["##BUDIBASE_USED_PLUGINS##"],
     location: window["##BUDIBASE_LOCATION##"],
     snippets: window["##BUDIBASE_SNIPPETS##"],
+    componentErrors: window["##BUDIBASE_COMPONENT_ERRORS##"],
   })
 
   // Set app ID - this window flag is set by both the preview and the real

packages/client/src/index.ts

@@ -7,9 +7,17 @@ export interface SDK {
   styleable: any
   Provider: any
   ActionTypes: typeof ActionTypes
+  fetchDatasourceSchema: any
+  generateGoldenSample: any
+  builderStore: Readable<{
+    inBuilder: boolean
+  }>
 }
 
 export type Component = Readable<{
   id: string
   styles: any
+  errorState: boolean
 }>
+
+export type Context = Readable<{}>

packages/client/src/stores/builder.js

@@ -19,6 +19,7 @@ const createBuilderStore = () => {
     eventResolvers: {},
     metadata: null,
     snippets: null,
+    componentErrors: {},
 
     // Legacy - allow the builder to specify a layout
     layout: null,

packages/client/src/stores/screens.js

@@ -42,6 +42,14 @@ const createScreenStore = () => {
         if ($builderStore.layout) {
           activeLayout = $builderStore.layout
         }
+
+        // Attach meta
+        const errors = $builderStore.componentErrors || {}
+        const attachComponentMeta = component => {
+          component._meta = { errors: errors[component._id] || [] }
+          component._children?.forEach(attachComponentMeta)
+        }
+        attachComponentMeta(activeScreen.props)
       } else {
         // Find the correct screen by matching the current route
         screens = $appStore.screens || []

packages/client/src/utils/schema.ts

@@ -1,5 +1,6 @@
 import { API } from "api"
 import { DataFetchMap, DataFetchType } from "@budibase/frontend-core"
+import { FieldType, TableSchema } from "@budibase/types"
 
 /**
  * Constructs a fetch instance for a given datasource.
@@ -42,14 +43,14 @@ export const fetchDatasourceSchema = async <
   }
 
   // Get the normal schema as long as we aren't wanting a form schema
-  let schema: any
+  let schema: TableSchema | undefined
   if (datasource?.type !== "query" || !options?.formSchema) {
-    schema = instance.getSchema(definition as any)
+    schema = instance.getSchema(definition as any) as TableSchema
   } else if ("parameters" in definition && definition.parameters?.length) {
     schema = {}
-    definition.parameters.forEach(param => {
-      schema[param.name] = { ...param, type: "string" }
-    })
+    for (const param of definition.parameters) {
+      schema[param.name] = { ...param, type: FieldType.STRING }
+    }
   }
   if (!schema) {
     return null
@@ -57,11 +58,11 @@ export const fetchDatasourceSchema = async <
 
   // Strip hidden fields from views
   if (datasource.type === "viewV2") {
-    Object.keys(schema).forEach(field => {
+    for (const field of Object.keys(schema)) {
       if (!schema[field].visible) {
         delete schema[field]
       }
-    })
+    }
   }
 
   // Enrich schema with relationships if required

packages/frontend-core/src/api/migrations.ts

@@ -1,8 +1,8 @@
-import { GetOldMigrationStatus } from "@budibase/types"
+import { GetMigrationStatus } from "@budibase/types"
 import { BaseAPIClient } from "./types"
 
 export interface MigrationEndpoints {
-  getMigrationStatus: () => Promise<GetOldMigrationStatus>
+  getMigrationStatus: () => Promise<GetMigrationStatus>
 }
 
 export const buildMigrationEndpoints = (

packages/frontend-core/src/components/grid/stores/datasource.ts

@@ -1,5 +1,3 @@
-// TODO: datasource and defitions are unions of the different implementations. At this point, the datasource does not know what type is being used, and the assignations will cause TS exceptions. Casting it "as any" for now. This should be fixed improving the type usages.
-
 import { derived, get, Readable, Writable } from "svelte/store"
 import {
   DataFetchDefinition,
@@ -10,12 +8,10 @@ import { enrichSchemaWithRelColumns, memo } from "../../../utils"
 import { cloneDeep } from "lodash"
 import {
   SaveRowRequest,
-  SaveTableRequest,
   UIDatasource,
   UIFieldMutation,
   UIFieldSchema,
   UIRow,
-  UpdateViewRequest,
   ViewV2Type,
 } from "@budibase/types"
 import { Store as StoreContext, BaseStoreProps } from "."
@@ -79,7 +75,7 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => {
   const schema = derived(definition, $definition => {
     const schema: Record<string, any> | undefined = getDatasourceSchema({
       API,
-      datasource: get(datasource) as any, // TODO: see line 1
+      datasource: get(datasource),
       definition: $definition ?? undefined,
     })
     if (!schema) {
@@ -137,7 +133,7 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => {
       let type = $datasource?.type
       // @ts-expect-error
       if (type === "provider") {
-        type = ($datasource as any).value?.datasource?.type // TODO: see line 1
+        type = ($datasource as any).value?.datasource?.type
       }
       // Handle calculation views
       if (
@@ -196,15 +192,13 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => {
   const refreshDefinition = async () => {
     const def = await getDatasourceDefinition({
       API,
-      datasource: get(datasource) as any, // TODO: see line 1
+      datasource: get(datasource),
     })
-    definition.set(def as any) // TODO: see line 1
+    definition.set(def ?? null)
   }
 
   // Saves the datasource definition
-  const saveDefinition = async (
-    newDefinition: SaveTableRequest | UpdateViewRequest
-  ) => {
+  const saveDefinition = async (newDefinition: DataFetchDefinition) => {
     // Update local state
     const originalDefinition = get(definition)
     definition.set(newDefinition)
@@ -245,7 +239,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => {
         delete newDefinition.schema[column].default
       }
     }
-    return await saveDefinition(newDefinition as any) // TODO: see line 1
+    return await saveDefinition(newDefinition)
   }
 
   // Adds a schema mutation for a single field
@@ -321,7 +315,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => {
     await saveDefinition({
       ...$definition,
       schema: newSchema,
-    } as any) // TODO: see line 1
+    })
     resetSchemaMutations()
   }
 

packages/frontend-core/src/fetch/ViewFetch.ts

@@ -21,7 +21,7 @@ export default class ViewFetch extends BaseDataFetch<ViewV1Datasource, Table> {
 
   getSchema(definition: Table) {
     const { datasource } = this.options
-    return definition?.views?.[datasource.name]?.schema
+    return definition?.views?.[datasource?.name]?.schema
   }
 
   async getData() {

packages/frontend-core/src/fetch/index.ts

@@ -101,12 +101,12 @@ export const fetchData = <
 
 // Creates an empty fetch instance with no datasource configured, so no data
 // will initially be loaded
-const createEmptyFetchInstance = ({
+const createEmptyFetchInstance = <T extends DataFetchDatasource>({
   API,
   datasource,
 }: {
   API: APIClient
-  datasource: DataFetchDatasource
+  datasource: T
 }) => {
   const handler = DataFetchMap[datasource?.type]
   if (!handler) {
@@ -114,7 +114,7 @@ const createEmptyFetchInstance = ({
   }
   return new handler({
     API,
-    datasource: null as never,
+    datasource: datasource as any,
     query: null as any,
   })
 }

packages/server/nodemon.json

@@ -7,7 +7,7 @@
     "../shared-core/src",
     "../string-templates/src"
   ],
-  "ext": "js,ts,json,svelte",
+  "ext": "js,ts,json,svelte,hbs",
   "ignore": [
     "**/*.spec.ts",
     "**/*.spec.js",

packages/server/scripts/dev/manage.js

@@ -43,7 +43,6 @@ async function init() {
     BB_ADMIN_USER_EMAIL: "",
     BB_ADMIN_USER_PASSWORD: "",
     PLUGINS_DIR: "",
-    HTTP_MIGRATIONS: "0",
     HTTP_LOGGING: "0",
     VERSION: "0.0.0+local",
     PASSWORD_MIN_LENGTH: "1",

packages/server/src/api/controllers/application.ts

@@ -27,7 +27,6 @@ import {
   env as envCore,
   ErrorCode,
   events,
-  migrations,
   objectStore,
   roles,
   tenancy,
@@ -43,7 +42,6 @@ import { groups, licensing, quotas } from "@budibase/pro"
 import {
   App,
   Layout,
-  MigrationType,
   PlanType,
   Screen,
   UserCtx,
@@ -488,13 +486,6 @@ async function creationEvents(request: BBRequest<CreateAppRequest>, app: App) {
 }
 
 async function appPostCreate(ctx: UserCtx<CreateAppRequest, App>, app: App) {
-  const tenantId = tenancy.getTenantId()
-  await migrations.backPopulateMigrations({
-    type: MigrationType.APP,
-    tenantId,
-    appId: app.appId,
-  })
-
   await creationEvents(ctx.request, app)
 
   // app import, template creation and duplication

packages/server/src/api/controllers/automation.ts

@@ -2,7 +2,7 @@ import * as triggers from "../../automations/triggers"
 import { sdk as coreSdk } from "@budibase/shared-core"
 import { DocumentType } from "../../db/utils"
 import { updateTestHistory, removeDeprecated } from "../../automations/utils"
-import { setTestFlag, clearTestFlag } from "../../utilities/redis"
+import { withTestFlag } from "../../utilities/redis"
 import { context, cache, events, db as dbCore } from "@budibase/backend-core"
 import { automations, features } from "@budibase/pro"
 import {
@@ -231,24 +231,25 @@ export async function test(
   ctx: UserCtx<TestAutomationRequest, TestAutomationResponse>
 ) {
   const db = context.getAppDB()
-  let automation = await db.get<Automation>(ctx.params.id)
-  await setTestFlag(automation._id!)
-  const testInput = prepareTestInput(ctx.request.body)
-  const response = await triggers.externalTrigger(
-    automation,
-    {
-      ...testInput,
-      appId: ctx.appId,
-      user: sdk.users.getUserContextBindings(ctx.user),
-    },
-    { getResponses: true }
-  )
-  // save a test history run
-  await updateTestHistory(ctx.appId, automation, {
-    ...ctx.request.body,
-    occurredAt: new Date().getTime(),
+  const automation = await db.tryGet<Automation>(ctx.params.id)
+  if (!automation) {
+    ctx.throw(404, `Automation ${ctx.params.id} not found`)
+  }
+
+  const { request, appId } = ctx
+  const { body } = request
+
+  ctx.body = await withTestFlag(automation._id!, async () => {
+    const occurredAt = new Date().getTime()
+    await updateTestHistory(appId, automation, { ...body, occurredAt })
+
+    const user = sdk.users.getUserContextBindings(ctx.user)
+    return await triggers.externalTrigger(
+      automation,
+      { ...prepareTestInput(body), appId, user },
+      { getResponses: true }
+    )
   })
-  await clearTestFlag(automation._id!)
-  ctx.body = response
+
   await events.automation.tested(automation)
 }

packages/server/src/api/controllers/migrations.ts

@@ -1,35 +1,11 @@
 import { context } from "@budibase/backend-core"
-import { migrate as migrationImpl, MIGRATIONS } from "../../migrations"
-import {
-  Ctx,
-  FetchOldMigrationResponse,
-  GetOldMigrationStatus,
-  RuneOldMigrationResponse,
-  RunOldMigrationRequest,
-} from "@budibase/types"
+import { Ctx, GetMigrationStatus } from "@budibase/types"
 import {
   getAppMigrationVersion,
   getLatestEnabledMigrationId,
 } from "../../appMigrations"
 
-export async function migrate(
-  ctx: Ctx<RunOldMigrationRequest, RuneOldMigrationResponse>
-) {
-  const options = ctx.request.body
-  // don't await as can take a while, just return
-  migrationImpl(options)
-  ctx.body = { message: "Migration started." }
-}
-
-export async function fetchDefinitions(
-  ctx: Ctx<void, FetchOldMigrationResponse>
-) {
-  ctx.body = MIGRATIONS
-}
-
-export async function getMigrationStatus(
-  ctx: Ctx<void, GetOldMigrationStatus>
-) {
+export async function getMigrationStatus(ctx: Ctx<void, GetMigrationStatus>) {
   const appId = context.getAppId()
 
   if (!appId) {

packages/server/src/api/controllers/static/templates/preview.hbs

@@ -73,7 +73,8 @@
         hiddenComponentIds,
         usedPlugins,
         location,
-        snippets
+        snippets,
+        componentErrors
       } = parsed
 
       // Set some flags so the app knows we're in the builder
@@ -91,6 +92,7 @@
       window["##BUDIBASE_USED_PLUGINS##"] = usedPlugins
       window["##BUDIBASE_LOCATION##"] = location
       window["##BUDIBASE_SNIPPETS##"] = snippets
+      window['##BUDIBASE_COMPONENT_ERRORS##'] = componentErrors
 
       // Initialise app
       try {

packages/server/src/api/controllers/view/viewsV2.ts

@@ -123,9 +123,11 @@ async function parseSchema(view: CreateViewRequest) {
 }
 
 export async function get(ctx: Ctx<void, ViewResponseEnriched>) {
-  ctx.body = {
-    data: await sdk.views.getEnriched(ctx.params.viewId),
+  const view = await sdk.views.getEnriched(ctx.params.viewId)
+  if (!view) {
+    ctx.throw(404)
   }
+  ctx.body = { data: view }
 }
 
 export async function fetch(ctx: Ctx<void, ViewFetchResponseEnriched>) {

packages/server/src/api/routes/migrations.ts

@@ -1,16 +1,8 @@
 import Router from "@koa/router"
 import * as migrationsController from "../controllers/migrations"
-import { auth } from "@budibase/backend-core"
 
 const router: Router = new Router()
 
-router
-  .post("/api/migrations/run", auth.internalApi, migrationsController.migrate)
-  .get(
-    "/api/migrations/definitions",
-    auth.internalApi,
-    migrationsController.fetchDefinitions
-  )
-  .get("/api/migrations/status", migrationsController.getMigrationStatus)
+router.get("/api/migrations/status", migrationsController.getMigrationStatus)
 
 export default router

packages/server/src/api/routes/tests/automation.spec.ts

@@ -19,9 +19,11 @@ import {
   Table,
 } from "@budibase/types"
 import { mocks } from "@budibase/backend-core/tests"
-import { FilterConditions } from "../../../automations/steps/filter"
 import { removeDeprecated } from "../../../automations/utils"
 import { createAutomationBuilder } from "../../../automations/tests/utilities/AutomationTestBuilder"
+import { automations } from "@budibase/shared-core"
+
+const FilterConditions = automations.steps.filter.FilterConditions
 
 const MAX_RETRIES = 4
 let {

packages/server/src/automations/actions.ts

@@ -1,3 +1,4 @@
+import { automations } from "@budibase/shared-core"
 import * as sendSmtpEmail from "./steps/sendSmtpEmail"
 import * as createRow from "./steps/createRow"
 import * as updateRow from "./steps/updateRow"
@@ -14,11 +15,10 @@ import * as make from "./steps/make"
 import * as filter from "./steps/filter"
 import * as delay from "./steps/delay"
 import * as queryRow from "./steps/queryRows"
-import * as loop from "./steps/loop"
 import * as collect from "./steps/collect"
-import * as branch from "./steps/branch"
 import * as triggerAutomationRun from "./steps/triggerAutomationRun"
 import * as openai from "./steps/openai"
+import * as bash from "./steps/bash"
 import env from "../environment"
 import {
   PluginType,
@@ -62,42 +62,39 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<
   string,
   AutomationStepDefinition
 > = {
-  SEND_EMAIL_SMTP: sendSmtpEmail.definition,
-  CREATE_ROW: createRow.definition,
-  UPDATE_ROW: updateRow.definition,
-  DELETE_ROW: deleteRow.definition,
-  OUTGOING_WEBHOOK: outgoingWebhook.definition,
-  EXECUTE_SCRIPT: executeScript.definition,
-  EXECUTE_QUERY: executeQuery.definition,
-  SERVER_LOG: serverLog.definition,
-  DELAY: delay.definition,
-  FILTER: filter.definition,
-  QUERY_ROWS: queryRow.definition,
-  LOOP: loop.definition,
-  COLLECT: collect.definition,
-  TRIGGER_AUTOMATION_RUN: triggerAutomationRun.definition,
-  BRANCH: branch.definition,
+  SEND_EMAIL_SMTP: automations.steps.sendSmtpEmail.definition,
+  CREATE_ROW: automations.steps.createRow.definition,
+  UPDATE_ROW: automations.steps.updateRow.definition,
+  DELETE_ROW: automations.steps.deleteRow.definition,
+  OUTGOING_WEBHOOK: automations.steps.outgoingWebhook.definition,
+  EXECUTE_SCRIPT: automations.steps.executeScript.definition,
+  EXECUTE_QUERY: automations.steps.executeQuery.definition,
+  SERVER_LOG: automations.steps.serverLog.definition,
+  DELAY: automations.steps.delay.definition,
+  FILTER: automations.steps.filter.definition,
+  QUERY_ROWS: automations.steps.queryRows.definition,
+  LOOP: automations.steps.loop.definition,
+  COLLECT: automations.steps.collect.definition,
+  TRIGGER_AUTOMATION_RUN: automations.steps.triggerAutomationRun.definition,
+  BRANCH: automations.steps.branch.definition,
   // these used to be lowercase step IDs, maintain for backwards compat
-  discord: discord.definition,
-  slack: slack.definition,
-  zapier: zapier.definition,
-  integromat: make.definition,
-  n8n: n8n.definition,
+  discord: automations.steps.discord.definition,
+  slack: automations.steps.slack.definition,
+  zapier: automations.steps.zapier.definition,
+  integromat: automations.steps.make.definition,
+  n8n: automations.steps.n8n.definition,
 }
 
 // don't add the bash script/definitions unless in self host
 // the fact this isn't included in any definitions means it cannot be
 // ran at all
 if (env.SELF_HOSTED) {
-  const bash = require("./steps/bash")
-
-  // @ts-ignore
+  // @ts-expect-error
   ACTION_IMPLS["EXECUTE_BASH"] = bash.run
-  // @ts-ignore
-  BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
+  BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = automations.steps.bash.definition
 
   if (env.isTest()) {
-    BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition
+    BUILTIN_ACTION_DEFINITIONS["OPENAI"] = automations.steps.openai.definition
   }
 }
 
@@ -105,7 +102,7 @@ export async function getActionDefinitions(): Promise<
   Record<keyof typeof AutomationActionStepId, AutomationStepDefinition>
 > {
   if (env.SELF_HOSTED) {
-    BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition
+    BUILTIN_ACTION_DEFINITIONS["OPENAI"] = automations.steps.openai.definition
   }
 
   const actionDefinitions = BUILTIN_ACTION_DEFINITIONS

packages/server/src/automations/steps/bash.ts

@@ -2,55 +2,7 @@ import { execSync } from "child_process"
 import { processStringSync } from "@budibase/string-templates"
 import * as automationUtils from "../automationUtils"
 import environment from "../../environment"
-import {
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
-  BashStepInputs,
-  BashStepOutputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  name: "Bash Scripting",
-  tagline: "Execute a bash command",
-  icon: "JourneyEvent",
-  description: "Run a bash script",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  stepId: AutomationActionStepId.EXECUTE_BASH,
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        code: {
-          type: AutomationIOType.STRING,
-          customType: AutomationCustomIOType.CODE,
-          title: "Code",
-        },
-      },
-      required: ["code"],
-    },
-    outputs: {
-      properties: {
-        stdout: {
-          type: AutomationIOType.STRING,
-          description: "Standard output of your bash command or script",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the command was successful",
-        },
-      },
-      required: ["stdout"],
-    },
-  },
-}
+import { BashStepInputs, BashStepOutputs } from "@budibase/types"
 
 export async function run({
   inputs,

packages/server/src/automations/steps/collect.ts

@@ -1,48 +1,4 @@
-import {
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  CollectStepInputs,
-  CollectStepOutputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  name: "Collect Data",
-  tagline: "Collect data to be sent to design",
-  icon: "Collection",
-  description:
-    "Collects specified data so it can be provided to the design section",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {},
-  stepId: AutomationActionStepId.COLLECT,
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        collection: {
-          type: AutomationIOType.STRING,
-          title: "What to Collect",
-        },
-      },
-      required: ["collection"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-        value: {
-          type: AutomationIOType.STRING,
-          description: "Collected data",
-        },
-      },
-      required: ["success", "value"],
-    },
-  },
-}
+import { CollectStepInputs, CollectStepOutputs } from "@budibase/types"
 
 export async function run({
   inputs,

packages/server/src/automations/steps/createRow.ts

@@ -6,75 +6,10 @@ import {
 } from "../automationUtils"
 import { buildCtx } from "./utils"
 import {
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
+  ContextEmitter,
   CreateRowStepInputs,
   CreateRowStepOutputs,
 } from "@budibase/types"
-import { EventEmitter } from "events"
-
-export const definition: AutomationStepDefinition = {
-  name: "Create Row",
-  tagline: "Create a {{inputs.enriched.table.name}} row",
-  icon: "TableRowAddBottom",
-  description: "Add a row to your database",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  stepId: AutomationActionStepId.CREATE_ROW,
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        row: {
-          type: AutomationIOType.OBJECT,
-          properties: {
-            tableId: {
-              type: AutomationIOType.STRING,
-              customType: AutomationCustomIOType.TABLE,
-            },
-          },
-          customType: AutomationCustomIOType.ROW,
-          title: "Table",
-          required: ["tableId"],
-        },
-      },
-      required: ["row"],
-    },
-    outputs: {
-      properties: {
-        row: {
-          type: AutomationIOType.OBJECT,
-          customType: AutomationCustomIOType.ROW,
-          description: "The new row",
-        },
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The response from the table",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the row creation was successful",
-        },
-        id: {
-          type: AutomationIOType.STRING,
-          description: "The identifier of the new row",
-        },
-        revision: {
-          type: AutomationIOType.STRING,
-          description: "The revision of the new row",
-        },
-      },
-      required: ["success", "id", "revision"],
-    },
-  },
-}
 
 export async function run({
   inputs,
@@ -83,7 +18,7 @@ export async function run({
 }: {
   inputs: CreateRowStepInputs
   appId: string
-  emitter: EventEmitter
+  emitter: ContextEmitter
 }): Promise<CreateRowStepOutputs> {
   if (inputs.row == null || inputs.row.tableId == null) {
     return {

packages/server/src/automations/steps/delay.ts

@@ -1,44 +1,5 @@
 import { wait } from "../../utilities"
-import {
-  AutomationActionStepId,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
-  DelayStepInputs,
-  DelayStepOutputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  name: "Delay",
-  icon: "Clock",
-  tagline: "Delay for {{inputs.time}} milliseconds",
-  description: "Delay the automation until an amount of time has passed",
-  stepId: AutomationActionStepId.DELAY,
-  internal: true,
-  features: {},
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        time: {
-          type: AutomationIOType.NUMBER,
-          title: "Delay in milliseconds",
-        },
-      },
-      required: ["time"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the delay was successful",
-        },
-      },
-      required: ["success"],
-    },
-  },
-  type: AutomationStepType.LOGIC,
-}
+import { DelayStepInputs, DelayStepOutputs } from "@budibase/types"
 
 export async function run({
   inputs,

packages/server/src/automations/steps/deleteRow.ts

@@ -1,66 +1,12 @@
-import { EventEmitter } from "events"
 import { destroy } from "../../api/controllers/row"
 import { buildCtx } from "./utils"
 import { getError } from "../automationUtils"
 import {
-  AutomationActionStepId,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationCustomIOType,
-  AutomationFeature,
+  ContextEmitter,
   DeleteRowStepInputs,
   DeleteRowStepOutputs,
-  AutomationStepDefinition,
 } from "@budibase/types"
 
-export const definition: AutomationStepDefinition = {
-  description: "Delete a row from your database",
-  icon: "TableRowRemoveCenter",
-  name: "Delete Row",
-  tagline: "Delete a {{inputs.enriched.table.name}} row",
-  type: AutomationStepType.ACTION,
-  stepId: AutomationActionStepId.DELETE_ROW,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        tableId: {
-          type: AutomationIOType.STRING,
-          customType: AutomationCustomIOType.TABLE,
-          title: "Table",
-        },
-        id: {
-          type: AutomationIOType.STRING,
-          title: "Row ID",
-        },
-      },
-      required: ["tableId", "id"],
-    },
-    outputs: {
-      properties: {
-        row: {
-          type: AutomationIOType.OBJECT,
-          customType: AutomationCustomIOType.ROW,
-          description: "The deleted row",
-        },
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The response from the table",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the deletion was successful",
-        },
-      },
-      required: ["row", "success"],
-    },
-  },
-}
-
 export async function run({
   inputs,
   appId,
@@ -68,7 +14,7 @@ export async function run({
 }: {
   inputs: DeleteRowStepInputs
   appId: string
-  emitter: EventEmitter
+  emitter: ContextEmitter
 }): Promise<DeleteRowStepOutputs> {
   if (inputs.id == null) {
     return {

packages/server/src/automations/steps/discord.ts

@@ -1,71 +1,10 @@
 import fetch from "node-fetch"
 import { getFetchResponse } from "./utils"
-import {
-  AutomationActionStepId,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
-  ExternalAppStepOutputs,
-  DiscordStepInputs,
-  AutomationStepDefinition,
-} from "@budibase/types"
+import { ExternalAppStepOutputs, DiscordStepInputs } from "@budibase/types"
 
 const DEFAULT_USERNAME = "Budibase Automate"
 const DEFAULT_AVATAR_URL = "https://i.imgur.com/a1cmTKM.png"
 
-export const definition: AutomationStepDefinition = {
-  name: "Discord Message",
-  tagline: "Send a message to a Discord server",
-  description: "Send a message to a Discord server",
-  icon: "ri-discord-line",
-  stepId: AutomationActionStepId.discord,
-  type: AutomationStepType.ACTION,
-  internal: false,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        url: {
-          type: AutomationIOType.STRING,
-          title: "Discord Webhook URL",
-        },
-        username: {
-          type: AutomationIOType.STRING,
-          title: "Bot Name",
-        },
-        avatar_url: {
-          type: AutomationIOType.STRING,
-          title: "Bot Avatar URL",
-        },
-        content: {
-          type: AutomationIOType.STRING,
-          title: "Message",
-        },
-      },
-      required: ["url", "content"],
-    },
-    outputs: {
-      properties: {
-        httpStatus: {
-          type: AutomationIOType.NUMBER,
-          description: "The HTTP status code of the request",
-        },
-        response: {
-          type: AutomationIOType.STRING,
-          description: "The response from the Discord Webhook",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the message sent successfully",
-        },
-      },
-    },
-  },
-}
-
 export async function run({
   inputs,
 }: {

packages/server/src/automations/steps/executeQuery.ts

@@ -1,69 +1,12 @@
-import { EventEmitter } from "events"
 import * as queryController from "../../api/controllers/query"
 import { buildCtx } from "./utils"
 import * as automationUtils from "../automationUtils"
 import {
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
+  ContextEmitter,
   ExecuteQueryStepInputs,
   ExecuteQueryStepOutputs,
 } from "@budibase/types"
 
-export const definition: AutomationStepDefinition = {
-  name: "External Data Connector",
-  tagline: "Execute Data Connector",
-  icon: "Data",
-  description: "Execute a query in an external data connector",
-  type: AutomationStepType.ACTION,
-  stepId: AutomationActionStepId.EXECUTE_QUERY,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        query: {
-          type: AutomationIOType.OBJECT,
-          properties: {
-            queryId: {
-              type: AutomationIOType.STRING,
-              customType: AutomationCustomIOType.QUERY,
-            },
-          },
-          customType: AutomationCustomIOType.QUERY_PARAMS,
-          title: "Parameters",
-          required: ["queryId"],
-        },
-      },
-      required: ["query"],
-    },
-    outputs: {
-      properties: {
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The response from the datasource execution",
-        },
-        info: {
-          type: AutomationIOType.OBJECT,
-          description:
-            "Some query types may return extra data, like headers from a REST query",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-      },
-      required: ["response", "success"],
-    },
-  },
-}
-
 export async function run({
   inputs,
   appId,
@@ -71,7 +14,7 @@ export async function run({
 }: {
   inputs: ExecuteQueryStepInputs
   appId: string
-  emitter: EventEmitter
+  emitter: ContextEmitter
 }): Promise<ExecuteQueryStepOutputs> {
   if (inputs.query == null) {
     return {

packages/server/src/automations/steps/executeScript.ts

@@ -2,55 +2,10 @@ import * as scriptController from "../../api/controllers/script"
 import { buildCtx } from "./utils"
 import * as automationUtils from "../automationUtils"
 import {
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
+  ContextEmitter,
   ExecuteScriptStepInputs,
   ExecuteScriptStepOutputs,
 } from "@budibase/types"
-import { EventEmitter } from "events"
-
-export const definition: AutomationStepDefinition = {
-  name: "JS Scripting",
-  tagline: "Execute JavaScript Code",
-  icon: "Code",
-  description: "Run a piece of JavaScript code in your automation",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  stepId: AutomationActionStepId.EXECUTE_SCRIPT,
-  inputs: {},
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  schema: {
-    inputs: {
-      properties: {
-        code: {
-          type: AutomationIOType.STRING,
-          customType: AutomationCustomIOType.CODE,
-          title: "Code",
-        },
-      },
-      required: ["code"],
-    },
-    outputs: {
-      properties: {
-        value: {
-          type: AutomationIOType.STRING,
-          description: "The result of the return statement",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-      },
-      required: ["success"],
-    },
-  },
-}
 
 export async function run({
   inputs,
@@ -61,7 +16,7 @@ export async function run({
   inputs: ExecuteScriptStepInputs
   appId: string
   context: object
-  emitter: EventEmitter
+  emitter: ContextEmitter
 }): Promise<ExecuteScriptStepOutputs> {
   if (inputs.code == null) {
     return {

packages/server/src/automations/steps/filter.ts

@@ -1,74 +1,7 @@
-import {
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  FilterStepInputs,
-  FilterStepOutputs,
-} from "@budibase/types"
+import { FilterStepInputs, FilterStepOutputs } from "@budibase/types"
+import { automations } from "@budibase/shared-core"
 
-export const FilterConditions = {
-  EQUAL: "EQUAL",
-  NOT_EQUAL: "NOT_EQUAL",
-  GREATER_THAN: "GREATER_THAN",
-  LESS_THAN: "LESS_THAN",
-}
-
-export const PrettyFilterConditions = {
-  [FilterConditions.EQUAL]: "Equals",
-  [FilterConditions.NOT_EQUAL]: "Not equals",
-  [FilterConditions.GREATER_THAN]: "Greater than",
-  [FilterConditions.LESS_THAN]: "Less than",
-}
-
-export const definition: AutomationStepDefinition = {
-  name: "Condition",
-  tagline: "{{inputs.field}} {{inputs.condition}} {{inputs.value}}",
-  icon: "Branch2",
-  description:
-    "Conditionally halt automations which do not meet certain conditions",
-  type: AutomationStepType.LOGIC,
-  internal: true,
-  features: {},
-  stepId: AutomationActionStepId.FILTER,
-  inputs: {
-    condition: FilterConditions.EQUAL,
-  },
-  schema: {
-    inputs: {
-      properties: {
-        field: {
-          type: AutomationIOType.STRING,
-          title: "Reference Value",
-        },
-        condition: {
-          type: AutomationIOType.STRING,
-          title: "Condition",
-          enum: Object.values(FilterConditions),
-          pretty: Object.values(PrettyFilterConditions),
-        },
-        value: {
-          type: AutomationIOType.STRING,
-          title: "Comparison Value",
-        },
-      },
-      required: ["field", "condition", "value"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-        result: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the logic block passed",
-        },
-      },
-      required: ["success", "result"],
-    },
-  },
-}
+const FilterConditions = automations.steps.filter.FilterConditions
 
 export async function run({
   inputs,

packages/server/src/automations/steps/make.ts

@@ -1,62 +1,6 @@
 import fetch from "node-fetch"
 import { getFetchResponse } from "./utils"
-import {
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
-  ExternalAppStepOutputs,
-  MakeIntegrationInputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  name: "Make Integration",
-  stepTitle: "Make",
-  tagline: "Trigger a Make scenario",
-  description:
-    "Performs a webhook call to Make and gets the response (if configured)",
-  icon: "ri-shut-down-line",
-  stepId: AutomationActionStepId.integromat,
-  type: AutomationStepType.ACTION,
-  internal: false,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        url: {
-          type: AutomationIOType.STRING,
-          title: "Webhook URL",
-        },
-        body: {
-          type: AutomationIOType.JSON,
-          title: "Payload",
-        },
-      },
-      required: ["url", "body"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether call was successful",
-        },
-        httpStatus: {
-          type: AutomationIOType.NUMBER,
-          description: "The HTTP status code returned",
-        },
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The webhook response - this can have properties",
-        },
-      },
-      required: ["success", "response"],
-    },
-  },
-}
+import { ExternalAppStepOutputs, MakeIntegrationInputs } from "@budibase/types"
 
 export async function run({
   inputs,

packages/server/src/automations/steps/n8n.ts

@@ -1,73 +1,11 @@
 import fetch, { HeadersInit } from "node-fetch"
 import { getFetchResponse } from "./utils"
 import {
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
   HttpMethod,
   ExternalAppStepOutputs,
   n8nStepInputs,
 } from "@budibase/types"
 
-export const definition: AutomationStepDefinition = {
-  name: "n8n Integration",
-  stepTitle: "n8n",
-  tagline: "Trigger an n8n workflow",
-  description:
-    "Performs a webhook call to n8n and gets the response (if configured)",
-  icon: "ri-shut-down-line",
-  stepId: AutomationActionStepId.n8n,
-  type: AutomationStepType.ACTION,
-  internal: false,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        url: {
-          type: AutomationIOType.STRING,
-          title: "Webhook URL",
-        },
-        method: {
-          type: AutomationIOType.STRING,
-          title: "Method",
-          enum: Object.values(HttpMethod),
-        },
-        authorization: {
-          type: AutomationIOType.STRING,
-          title: "Authorization",
-        },
-        body: {
-          type: AutomationIOType.JSON,
-          title: "Payload",
-        },
-      },
-      required: ["url", "method"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether call was successful",
-        },
-        httpStatus: {
-          type: AutomationIOType.NUMBER,
-          description: "The HTTP status code returned",
-        },
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The webhook response - this can have properties",
-        },
-      },
-      required: ["success", "response"],
-    },
-  },
-}
-
 export async function run({
   inputs,
 }: {

packages/server/src/automations/steps/openai.ts

@@ -1,67 +1,10 @@
 import { OpenAI } from "openai"
 
-import {
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  OpenAIStepInputs,
-  OpenAIStepOutputs,
-} from "@budibase/types"
+import { OpenAIStepInputs, OpenAIStepOutputs } from "@budibase/types"
 import { env } from "@budibase/backend-core"
 import * as automationUtils from "../automationUtils"
 import * as pro from "@budibase/pro"
 
-enum Model {
-  GPT_4O_MINI = "gpt-4o-mini",
-  GPT_4O = "gpt-4o",
-  GPT_4 = "gpt-4",
-  GPT_35_TURBO = "gpt-3.5-turbo",
-}
-
-export const definition: AutomationStepDefinition = {
-  name: "OpenAI",
-  tagline: "Send prompts to ChatGPT",
-  icon: "Algorithm",
-  description: "Interact with the OpenAI ChatGPT API.",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {},
-  stepId: AutomationActionStepId.OPENAI,
-  inputs: {
-    prompt: "",
-  },
-  schema: {
-    inputs: {
-      properties: {
-        prompt: {
-          type: AutomationIOType.STRING,
-          title: "Prompt",
-        },
-        model: {
-          type: AutomationIOType.STRING,
-          title: "Model",
-          enum: Object.values(Model),
-        },
-      },
-      required: ["prompt", "model"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-        response: {
-          type: AutomationIOType.STRING,
-          description: "What was output",
-        },
-      },
-      required: ["success", "response"],
-    },
-  },
-}
-
 /**
  * Maintains backward compatibility with automation steps created before the introduction
  * of custom configurations and Budibase AI

packages/server/src/automations/steps/outgoingWebhook.ts

@@ -2,12 +2,6 @@ import fetch from "node-fetch"
 import { getFetchResponse } from "./utils"
 import * as automationUtils from "../automationUtils"
 import {
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
   ExternalAppStepOutputs,
   OutgoingWebhookStepInputs,
 } from "@budibase/types"
@@ -26,69 +20,6 @@ const BODY_REQUESTS = [RequestType.POST, RequestType.PUT, RequestType.PATCH]
  * NOTE: this functionality is deprecated - it no longer should be used.
  */
 
-export const definition: AutomationStepDefinition = {
-  deprecated: true,
-  name: "Outgoing webhook",
-  tagline: "Send a {{inputs.requestMethod}} request",
-  icon: "Send",
-  description: "Send a request of specified method to a URL",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  stepId: AutomationActionStepId.OUTGOING_WEBHOOK,
-  inputs: {
-    requestMethod: "POST",
-    url: "http://",
-    requestBody: "{}",
-    headers: "{}",
-  },
-  schema: {
-    inputs: {
-      properties: {
-        requestMethod: {
-          type: AutomationIOType.STRING,
-          enum: Object.values(RequestType),
-          title: "Request method",
-        },
-        url: {
-          type: AutomationIOType.STRING,
-          title: "URL",
-        },
-        requestBody: {
-          type: AutomationIOType.STRING,
-          title: "JSON Body",
-          customType: AutomationCustomIOType.WIDE,
-        },
-        headers: {
-          type: AutomationIOType.STRING,
-          title: "Headers",
-          customType: AutomationCustomIOType.WIDE,
-        },
-      },
-      required: ["requestMethod", "url"],
-    },
-    outputs: {
-      properties: {
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The response from the webhook",
-        },
-        httpStatus: {
-          type: AutomationIOType.NUMBER,
-          description: "The HTTP status code returned",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-      },
-      required: ["response", "success"],
-    },
-  },
-}
-
 export async function run({
   inputs,
 }: {

packages/server/src/automations/steps/queryRows.ts

@@ -4,84 +4,12 @@ import { buildCtx } from "./utils"
 import * as automationUtils from "../automationUtils"
 import {
   FieldType,
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
   EmptyFilterOption,
   SortOrder,
   QueryRowsStepInputs,
   QueryRowsStepOutputs,
 } from "@budibase/types"
 
-const SortOrderPretty = {
-  [SortOrder.ASCENDING]: "Ascending",
-  [SortOrder.DESCENDING]: "Descending",
-}
-
-export const definition: AutomationStepDefinition = {
-  description: "Query rows from the database",
-  icon: "Search",
-  name: "Query rows",
-  tagline: "Query rows from {{inputs.enriched.table.name}} table",
-  type: AutomationStepType.ACTION,
-  stepId: AutomationActionStepId.QUERY_ROWS,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        tableId: {
-          type: AutomationIOType.STRING,
-          customType: AutomationCustomIOType.TABLE,
-          title: "Table",
-        },
-        filters: {
-          type: AutomationIOType.OBJECT,
-          customType: AutomationCustomIOType.FILTERS,
-          title: "Filtering",
-        },
-        sortColumn: {
-          type: AutomationIOType.STRING,
-          title: "Sort Column",
-          customType: AutomationCustomIOType.COLUMN,
-        },
-        sortOrder: {
-          type: AutomationIOType.STRING,
-          title: "Sort Order",
-          enum: Object.values(SortOrder),
-          pretty: Object.values(SortOrderPretty),
-        },
-        limit: {
-          type: AutomationIOType.NUMBER,
-          title: "Limit",
-          customType: AutomationCustomIOType.QUERY_LIMIT,
-        },
-      },
-      required: ["tableId"],
-    },
-    outputs: {
-      properties: {
-        rows: {
-          type: AutomationIOType.ARRAY,
-          customType: AutomationCustomIOType.ROWS,
-          description: "The rows that were found",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the query was successful",
-        },
-      },
-      required: ["rows", "success"],
-    },
-  },
-}
-
 async function getTable(appId: string, tableId: string) {
   const ctx: any = buildCtx(appId, null, {
     params: {

packages/server/src/automations/steps/sendSmtpEmail.ts

@@ -1,102 +1,6 @@
 import { sendSmtpEmail } from "../../utilities/workerRequests"
 import * as automationUtils from "../automationUtils"
-import {
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
-  AutomationCustomIOType,
-  SmtpEmailStepInputs,
-  BaseAutomationOutputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  description: "Send an email using SMTP",
-  tagline: "Send SMTP email to {{inputs.to}}",
-  icon: "Email",
-  name: "Send Email (SMTP)",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  stepId: AutomationActionStepId.SEND_EMAIL_SMTP,
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        to: {
-          type: AutomationIOType.STRING,
-          title: "Send To",
-        },
-        from: {
-          type: AutomationIOType.STRING,
-          title: "Send From",
-        },
-        cc: {
-          type: AutomationIOType.STRING,
-          title: "CC",
-        },
-        bcc: {
-          type: AutomationIOType.STRING,
-          title: "BCC",
-        },
-        subject: {
-          type: AutomationIOType.STRING,
-          title: "Email Subject",
-        },
-        contents: {
-          type: AutomationIOType.STRING,
-          title: "HTML Contents",
-        },
-        addInvite: {
-          type: AutomationIOType.BOOLEAN,
-          title: "Add calendar invite",
-        },
-        startTime: {
-          type: AutomationIOType.DATE,
-          title: "Start Time",
-          dependsOn: "addInvite",
-        },
-        endTime: {
-          type: AutomationIOType.DATE,
-          title: "End Time",
-          dependsOn: "addInvite",
-        },
-        summary: {
-          type: AutomationIOType.STRING,
-          title: "Meeting Summary",
-          dependsOn: "addInvite",
-        },
-        location: {
-          type: AutomationIOType.STRING,
-          title: "Location",
-          dependsOn: "addInvite",
-        },
-        attachments: {
-          type: AutomationIOType.ATTACHMENT,
-          customType: AutomationCustomIOType.MULTI_ATTACHMENTS,
-          title: "Attachments",
-        },
-      },
-      required: ["to", "from", "subject", "contents"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the email was sent",
-        },
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "A response from the email client, this may be an error",
-        },
-      },
-      required: ["success"],
-    },
-  },
-}
+import { SmtpEmailStepInputs, BaseAutomationOutputs } from "@budibase/types"
 
 export async function run({
   inputs,

packages/server/src/automations/steps/serverLog.ts

@@ -1,58 +1,4 @@
-import {
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
-  ServerLogStepInputs,
-  ServerLogStepOutputs,
-} from "@budibase/types"
-
-/**
- * Note, there is some functionality in this that is not currently exposed as it
- * is complex and maybe better to be opinionated here.
- * GET/DELETE requests cannot handle body elements so they will not be sent if configured.
- */
-
-export const definition: AutomationStepDefinition = {
-  name: "Backend log",
-  tagline: "Console log a value in the backend",
-  icon: "Monitoring",
-  description: "Logs the given text to the server (using console.log)",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  stepId: AutomationActionStepId.SERVER_LOG,
-  inputs: {
-    text: "",
-  },
-  schema: {
-    inputs: {
-      properties: {
-        text: {
-          type: AutomationIOType.STRING,
-          title: "Log",
-        },
-      },
-      required: ["text"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-        message: {
-          type: AutomationIOType.STRING,
-          description: "What was output",
-        },
-      },
-      required: ["success", "message"],
-    },
-  },
-}
+import { ServerLogStepInputs, ServerLogStepOutputs } from "@budibase/types"
 
 export async function run({
   inputs,

packages/server/src/automations/steps/slack.ts

@@ -1,59 +1,6 @@
 import fetch from "node-fetch"
 import { getFetchResponse } from "./utils"
-import {
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
-  ExternalAppStepOutputs,
-  SlackStepInputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  name: "Slack Message",
-  tagline: "Send a message to Slack",
-  description: "Send a message to Slack",
-  icon: "ri-slack-line",
-  stepId: AutomationActionStepId.slack,
-  type: AutomationStepType.ACTION,
-  internal: false,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        url: {
-          type: AutomationIOType.STRING,
-          title: "Incoming Webhook URL",
-        },
-        text: {
-          type: AutomationIOType.STRING,
-          title: "Message",
-        },
-      },
-      required: ["url", "text"],
-    },
-    outputs: {
-      properties: {
-        httpStatus: {
-          type: AutomationIOType.NUMBER,
-          description: "The HTTP status code of the request",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the message sent successfully",
-        },
-        response: {
-          type: AutomationIOType.STRING,
-          description: "The response from the Slack Webhook",
-        },
-      },
-    },
-  },
-}
+import { ExternalAppStepOutputs, SlackStepInputs } from "@budibase/types"
 
 export async function run({
   inputs,

packages/server/src/automations/steps/triggerAutomationRun.ts

@@ -1,10 +1,5 @@
 import {
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
   Automation,
-  AutomationCustomIOType,
   TriggerAutomationStepInputs,
   TriggerAutomationStepOutputs,
 } from "@budibase/types"
@@ -13,54 +8,6 @@ import { context } from "@budibase/backend-core"
 import { features } from "@budibase/pro"
 import env from "../../environment"
 
-export const definition: AutomationStepDefinition = {
-  name: "Trigger an automation",
-  tagline: "Triggers an automation synchronously",
-  icon: "Sync",
-  description: "Triggers an automation synchronously",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {},
-  stepId: AutomationActionStepId.TRIGGER_AUTOMATION_RUN,
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        automation: {
-          type: AutomationIOType.OBJECT,
-          properties: {
-            automationId: {
-              type: AutomationIOType.STRING,
-              customType: AutomationCustomIOType.AUTOMATION,
-            },
-          },
-          customType: AutomationCustomIOType.AUTOMATION_FIELDS,
-          title: "automatioFields",
-          required: ["automationId"],
-        },
-        timeout: {
-          type: AutomationIOType.NUMBER,
-          title: "Timeout (ms)",
-        },
-      },
-      required: ["automationId"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the automation was successful",
-        },
-        value: {
-          type: AutomationIOType.OBJECT,
-          description: "Automation Result",
-        },
-      },
-      required: ["success", "value"],
-    },
-  },
-}
-
 export async function run({
   inputs,
 }: {

packages/server/src/automations/steps/updateRow.ts

@@ -1,77 +1,12 @@
-import { EventEmitter } from "events"
 import * as rowController from "../../api/controllers/row"
 import * as automationUtils from "../automationUtils"
 import { buildCtx } from "./utils"
 import {
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
+  ContextEmitter,
   UpdateRowStepInputs,
   UpdateRowStepOutputs,
 } from "@budibase/types"
 
-export const definition: AutomationStepDefinition = {
-  name: "Update Row",
-  tagline: "Update a {{inputs.enriched.table.name}} row",
-  icon: "Refresh",
-  description: "Update a row in your database",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  stepId: AutomationActionStepId.UPDATE_ROW,
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        meta: {
-          type: AutomationIOType.OBJECT,
-          title: "Field settings",
-        },
-        row: {
-          type: AutomationIOType.OBJECT,
-          customType: AutomationCustomIOType.ROW,
-          title: "Table",
-        },
-        rowId: {
-          type: AutomationIOType.STRING,
-          title: "Row ID",
-        },
-      },
-      required: ["row", "rowId"],
-    },
-    outputs: {
-      properties: {
-        row: {
-          type: AutomationIOType.OBJECT,
-          customType: AutomationCustomIOType.ROW,
-          description: "The updated row",
-        },
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The response from the table",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-        id: {
-          type: AutomationIOType.STRING,
-          description: "The identifier of the updated row",
-        },
-        revision: {
-          type: AutomationIOType.STRING,
-          description: "The revision of the updated row",
-        },
-      },
-      required: ["success", "id", "revision"],
-    },
-  },
-}
 export async function run({
   inputs,
   appId,
@@ -79,7 +14,7 @@ export async function run({
 }: {
   inputs: UpdateRowStepInputs
   appId: string
-  emitter: EventEmitter
+  emitter: ContextEmitter
 }): Promise<UpdateRowStepOutputs> {
   if (inputs.rowId == null || inputs.row == null) {
     return {

packages/server/src/automations/steps/utils.ts

@@ -1,4 +1,4 @@
-import { EventEmitter } from "events"
+import { ContextEmitter } from "@budibase/types"
 
 export async function getFetchResponse(fetched: any) {
   let status = fetched.status,
@@ -22,7 +22,7 @@ export async function getFetchResponse(fetched: any) {
 // opts can contain, body, params and version
 export function buildCtx(
   appId: string,
-  emitter?: EventEmitter | null,
+  emitter?: ContextEmitter | null,
   opts: any = {}
 ) {
   const ctx: any = {

packages/server/src/automations/steps/zapier.ts

@@ -1,55 +1,6 @@
 import fetch from "node-fetch"
 import { getFetchResponse } from "./utils"
-import {
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
-  ZapierStepInputs,
-  ZapierStepOutputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  name: "Zapier Webhook",
-  stepId: AutomationActionStepId.zapier,
-  type: AutomationStepType.ACTION,
-  internal: false,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  description: "Trigger a Zapier Zap via webhooks",
-  tagline: "Trigger a Zapier Zap",
-  icon: "ri-flashlight-line",
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        url: {
-          type: AutomationIOType.STRING,
-          title: "Webhook URL",
-        },
-        body: {
-          type: AutomationIOType.JSON,
-          title: "Payload",
-        },
-      },
-      required: ["url"],
-    },
-    outputs: {
-      properties: {
-        httpStatus: {
-          type: AutomationIOType.NUMBER,
-          description: "The HTTP status code of the request",
-        },
-        response: {
-          type: AutomationIOType.STRING,
-          description: "The response from Zapier",
-        },
-      },
-    },
-  },
-}
+import { ZapierStepInputs, ZapierStepOutputs } from "@budibase/types"
 
 export async function run({
   inputs,

packages/server/src/automations/tests/filter.spec.ts

@@ -1,5 +1,7 @@
 import * as setup from "./utilities"
-import { FilterConditions } from "../steps/filter"
+import { automations } from "@budibase/shared-core"
+
+const FilterConditions = automations.steps.filter.FilterConditions
 
 describe("test the filter logic", () => {
   const config = setup.getConfig()

packages/server/src/automations/tests/scenarios/scenarios.spec.ts

@@ -6,9 +6,11 @@ import {
   DatabaseName,
   datasourceDescribe,
 } from "../../../integrations/tests/utils"
-import { FilterConditions } from "../../../automations/steps/filter"
 import { Knex } from "knex"
 import { generator } from "@budibase/backend-core/tests"
+import { automations } from "@budibase/shared-core"
+
+const FilterConditions = automations.steps.filter.FilterConditions
 
 describe("Automation Scenarios", () => {
   let config = setup.getConfig()

packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts

@@ -1,5 +1,4 @@
 import { v4 as uuidv4 } from "uuid"
-import { testAutomation } from "../../../api/routes/tests/utilities/TestFunctions"
 import { BUILTIN_ACTION_DEFINITIONS } from "../../actions"
 import { TRIGGER_DEFINITIONS } from "../../triggers"
 import {
@@ -7,7 +6,6 @@ import {
   AppActionTriggerOutputs,
   Automation,
   AutomationActionStepId,
-  AutomationResults,
   AutomationStep,
   AutomationStepInputs,
   AutomationTrigger,
@@ -24,6 +22,7 @@ import {
   ExecuteQueryStepInputs,
   ExecuteScriptStepInputs,
   FilterStepInputs,
+  isDidNotTriggerResponse,
   LoopStepInputs,
   OpenAIStepInputs,
   QueryRowsStepInputs,
@@ -36,13 +35,14 @@ import {
   SearchFilters,
   ServerLogStepInputs,
   SmtpEmailStepInputs,
+  TestAutomationRequest,
   UpdateRowStepInputs,
   WebhookTriggerInputs,
   WebhookTriggerOutputs,
 } from "@budibase/types"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 import * as setup from "../utilities"
-import { definition } from "../../../automations/steps/branch"
+import { automations } from "@budibase/shared-core"
 
 type TriggerOutputs =
   | RowCreatedTriggerOutputs
@@ -103,7 +103,7 @@ class BaseStepBuilder {
       branchStepInputs.children![branchId] = stepBuilder.build()
     })
     const branchStep: AutomationStep = {
-      ...definition,
+      ...automations.steps.branch.definition,
       id: uuidv4(),
       stepId: AutomationActionStepId.BRANCH,
       inputs: branchStepInputs,
@@ -279,7 +279,7 @@ class StepBuilder extends BaseStepBuilder {
 class AutomationBuilder extends BaseStepBuilder {
   private automationConfig: Automation
   private config: TestConfiguration
-  private triggerOutputs: any
+  private triggerOutputs: TriggerOutputs
   private triggerSet = false
 
   constructor(
@@ -398,21 +398,19 @@ class AutomationBuilder extends BaseStepBuilder {
 
   async run() {
     const automation = await this.save()
-    const results = await testAutomation(
-      this.config,
-      automation,
-      this.triggerOutputs
+    const response = await this.config.api.automation.test(
+      automation._id!,
+      this.triggerOutputs as TestAutomationRequest
     )
-    return this.processResults(results)
-  }
 
-  private processResults(results: {
-    body: AutomationResults
-  }): AutomationResults {
-    results.body.steps.shift()
+    if (isDidNotTriggerResponse(response)) {
+      throw new Error(response.message)
+    }
+
+    response.steps.shift()
     return {
-      trigger: results.body.trigger,
-      steps: results.body.steps,
+      trigger: response.trigger,
+      steps: response.steps,
     }
   }
 }

packages/server/src/automations/triggers.ts

@@ -21,6 +21,7 @@ import {
   AutomationRowEvent,
   UserBindings,
   AutomationResults,
+  DidNotTriggerResponse,
 } from "@budibase/types"
 import { executeInThread } from "../threads/automation"
 import { dataFilters, sdk } from "@budibase/shared-core"
@@ -33,14 +34,6 @@ const JOB_OPTS = {
 import * as automationUtils from "../automations/automationUtils"
 import { doesTableExist } from "../sdk/app/tables/getters"
 
-type DidNotTriggerResponse = {
-  outputs: {
-    success: false
-    status: AutomationStatus.STOPPED
-  }
-  message: AutomationStoppedReason.TRIGGER_FILTER_NOT_MET
-}
-
 async function getAllAutomations() {
   const db = context.getAppDB()
   let automations = await db.allDocs<Automation>(
@@ -156,14 +149,26 @@ export function isAutomationResults(
   )
 }
 
+interface AutomationTriggerParams {
+  fields: Record<string, any>
+  timeout?: number
+  appId?: string
+  user?: UserBindings
+}
+
 export async function externalTrigger(
   automation: Automation,
-  params: {
-    fields: Record<string, any>
-    timeout?: number
-    appId?: string
-    user?: UserBindings
-  },
+  params: AutomationTriggerParams,
+  options: { getResponses: true }
+): Promise<AutomationResults | DidNotTriggerResponse>
+export async function externalTrigger(
+  automation: Automation,
+  params: AutomationTriggerParams,
+  options?: { getResponses: false }
+): Promise<AutomationJob | DidNotTriggerResponse>
+export async function externalTrigger(
+  automation: Automation,
+  params: AutomationTriggerParams,
   { getResponses }: { getResponses?: boolean } = {}
 ): Promise<AutomationResults | DidNotTriggerResponse | AutomationJob> {
   if (automation.disabled) {

packages/server/src/environment.ts

@@ -54,7 +54,6 @@ const environment = {
   REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   REDIS_CLUSTERED: process.env.REDIS_CLUSTERED,
-  HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
   CLUSTER_MODE: process.env.CLUSTER_MODE,
   API_REQ_LIMIT_PER_SEC: process.env.API_REQ_LIMIT_PER_SEC,
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

packages/server/src/jsRunner/index.ts

@@ -4,12 +4,17 @@ import {
   JsTimeoutError,
   setJSRunner,
   setOnErrorLog,
+  setTestingBackendJS,
 } from "@budibase/string-templates"
 import { context, logging } from "@budibase/backend-core"
 import tracer from "dd-trace"
 import { IsolatedVM } from "./vm"
 
 export function init() {
+  // enforce that if we're using isolated-VM runner then we are running backend JS
+  if (env.isTest()) {
+    setTestingBackendJS()
+  }
   setJSRunner((js: string, ctx: Record<string, any>) => {
     return tracer.trace("runJS", {}, () => {
       try {

packages/server/src/migrations/functions/appUrls.ts

@@ -1,27 +0,0 @@
-import { db as dbCore } from "@budibase/backend-core"
-import sdk from "../../sdk"
-
-/**
- * Date:
- * January 2022
- *
- * Description:
- * Add the url to the app metadata if it doesn't exist
- */
-export const run = async (appDb: any) => {
-  let metadata
-  try {
-    metadata = await appDb.get(dbCore.DocumentType.APP_METADATA)
-  } catch (e) {
-    // sometimes the metadata document doesn't exist
-    // exit early instead of failing the migration
-    console.error("Error retrieving app metadata. Skipping", e)
-    return
-  }
-
-  if (!metadata.url) {
-    metadata.url = sdk.applications.getAppUrl({ name: metadata.name })
-    console.log(`Adding url to app: ${metadata.url}`)
-    await appDb.put(metadata)
-  }
-}

packages/server/src/migrations/functions/backfill/app.ts

@@ -1,149 +0,0 @@
-import * as automations from "./app/automations"
-import * as datasources from "./app/datasources"
-import * as layouts from "./app/layouts"
-import * as queries from "./app/queries"
-import * as roles from "./app/roles"
-import * as tables from "./app/tables"
-import * as screens from "./app/screens"
-import * as global from "./global"
-import { App, AppBackfillSucceededEvent, Event } from "@budibase/types"
-import { db as dbUtils, events } from "@budibase/backend-core"
-import env from "../../../environment"
-import { DEFAULT_TIMESTAMP } from "."
-
-const failGraceful = env.SELF_HOSTED && !env.isDev()
-
-const handleError = (e: any, errors?: any) => {
-  if (failGraceful) {
-    if (errors) {
-      errors.push(e)
-    }
-    return
-  }
-  console.trace(e)
-  throw e
-}
-
-const EVENTS = [
-  Event.AUTOMATION_CREATED,
-  Event.AUTOMATION_STEP_CREATED,
-  Event.DATASOURCE_CREATED,
-  Event.LAYOUT_CREATED,
-  Event.QUERY_CREATED,
-  Event.ROLE_CREATED,
-  Event.SCREEN_CREATED,
-  Event.TABLE_CREATED,
-  Event.VIEW_CREATED,
-  Event.VIEW_CALCULATION_CREATED,
-  Event.VIEW_FILTER_CREATED,
-  Event.APP_PUBLISHED,
-  Event.APP_CREATED,
-]
-
-/**
- * Date:
- * May 2022
- *
- * Description:
- * Backfill app events.
- */
-
-export const run = async (appDb: any) => {
-  try {
-    if (await global.isComplete()) {
-      // make sure new apps aren't backfilled
-      // return if the global migration for this tenant is complete
-      // which runs after the app migrations
-      return
-    }
-
-    // tell the event pipeline to start caching
-    // events for this tenant
-    await events.backfillCache.start(EVENTS)
-
-    let timestamp: string | number = DEFAULT_TIMESTAMP
-    const app: App = await appDb.get(dbUtils.DocumentType.APP_METADATA)
-    if (app.createdAt) {
-      timestamp = app.createdAt as string
-    }
-
-    if (dbUtils.isProdAppID(app.appId)) {
-      await events.app.published(app, timestamp)
-    }
-
-    const totals: any = {}
-    const errors: any = []
-
-    if (dbUtils.isDevAppID(app.appId)) {
-      await events.app.created(app, timestamp)
-      try {
-        totals.automations = await automations.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.datasources = await datasources.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.layouts = await layouts.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.queries = await queries.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.roles = await roles.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.screens = await screens.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.tables = await tables.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-    }
-
-    const properties: AppBackfillSucceededEvent = {
-      appId: app.appId,
-      automations: totals.automations,
-      datasources: totals.datasources,
-      layouts: totals.layouts,
-      queries: totals.queries,
-      roles: totals.roles,
-      tables: totals.tables,
-      screens: totals.screens,
-    }
-
-    if (errors.length) {
-      properties.errors = errors.map((e: any) =>
-        JSON.stringify(e, Object.getOwnPropertyNames(e))
-      )
-      properties.errorCount = errors.length
-    } else {
-      properties.errorCount = 0
-    }
-
-    await events.backfill.appSucceeded(properties)
-    // tell the event pipeline to stop caching events for this tenant
-    await events.backfillCache.end()
-  } catch (e) {
-    handleError(e)
-    await events.backfill.appFailed(e)
-  }
-}

packages/server/src/migrations/functions/backfill/app/automations.ts

@@ -1,26 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getAutomationParams } from "../../../../db/utils"
-import { Automation } from "@budibase/types"
-
-const getAutomations = async (appDb: any): Promise<Automation[]> => {
-  const response = await appDb.allDocs(
-    getAutomationParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
-  const automations = await getAutomations(appDb)
-
-  for (const automation of automations) {
-    await events.automation.created(automation, timestamp)
-
-    for (const step of automation.definition.steps) {
-      await events.automation.stepCreated(automation, step, timestamp)
-    }
-  }
-
-  return automations.length
-}

packages/server/src/migrations/functions/backfill/app/datasources.ts

@@ -1,22 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getDatasourceParams } from "../../../../db/utils"
-import { Datasource } from "@budibase/types"
-
-const getDatasources = async (appDb: any): Promise<Datasource[]> => {
-  const response = await appDb.allDocs(
-    getDatasourceParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
-  const datasources: Datasource[] = await getDatasources(appDb)
-
-  for (const datasource of datasources) {
-    await events.datasource.created(datasource, timestamp)
-  }
-
-  return datasources.length
-}

packages/server/src/migrations/functions/backfill/app/layouts.ts

@@ -1,29 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getLayoutParams } from "../../../../db/utils"
-import { Layout } from "@budibase/types"
-
-const getLayouts = async (appDb: any): Promise<Layout[]> => {
-  const response = await appDb.allDocs(
-    getLayoutParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
-  const layouts: Layout[] = await getLayouts(appDb)
-
-  for (const layout of layouts) {
-    // exclude default layouts
-    if (
-      layout._id === "layout_private_master" ||
-      layout._id === "layout_public_master"
-    ) {
-      continue
-    }
-    await events.layout.created(layout, timestamp)
-  }
-
-  return layouts.length
-}

packages/server/src/migrations/functions/backfill/app/queries.ts

@@ -1,47 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getQueryParams } from "../../../../db/utils"
-import { Query, Datasource, SourceName } from "@budibase/types"
-
-const getQueries = async (appDb: any): Promise<Query[]> => {
-  const response = await appDb.allDocs(
-    getQueryParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-const getDatasource = async (
-  appDb: any,
-  datasourceId: string
-): Promise<Datasource> => {
-  return appDb.get(datasourceId)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
-  const queries: Query[] = await getQueries(appDb)
-
-  for (const query of queries) {
-    let datasource: Datasource
-
-    try {
-      datasource = await getDatasource(appDb, query.datasourceId)
-    } catch (e: any) {
-      // handle known bug where a datasource has been deleted
-      // and the query has not
-      if (e.status === 404) {
-        datasource = {
-          type: "unknown",
-          _id: query.datasourceId,
-          source: "unknown" as SourceName,
-        }
-      } else {
-        throw e
-      }
-    }
-
-    await events.query.created(datasource, query, timestamp)
-  }
-
-  return queries.length
-}

packages/server/src/migrations/functions/backfill/app/roles.ts

@@ -1,22 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getRoleParams } from "../../../../db/utils"
-import { Role } from "@budibase/types"
-
-const getRoles = async (appDb: any): Promise<Role[]> => {
-  const response = await appDb.allDocs(
-    getRoleParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
-  const roles = await getRoles(appDb)
-
-  for (const role of roles) {
-    await events.role.created(role, timestamp)
-  }
-
-  return roles.length
-}

packages/server/src/migrations/functions/backfill/app/screens.ts

@@ -1,22 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getScreenParams } from "../../../../db/utils"
-import { Screen } from "@budibase/types"
-
-const getScreens = async (appDb: any): Promise<Screen[]> => {
-  const response = await appDb.allDocs(
-    getScreenParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
-  const screens = await getScreens(appDb)
-
-  for (const screen of screens) {
-    await events.screen.created(screen, timestamp)
-  }
-
-  return screens.length
-}