Merge branch 'master' into BUDI-8986/validate-datasource-setting-on-components

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.46",
+  "version": "3.2.47",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/src/index.ts

@@ -1,6 +1,5 @@
 export * as configs from "./configs"
 export * as events from "./events"
-export * as migrations from "./migrations"
 export * as users from "./users"
 export * as userUtils from "./users/utils"
 export * as roles from "./security/roles"

packages/backend-core/src/migrations/definitions.ts

@@ -1,40 +0,0 @@
-import {
-  MigrationType,
-  MigrationName,
-  MigrationDefinition,
-} from "@budibase/types"
-
-export const DEFINITIONS: MigrationDefinition[] = [
-  {
-    type: MigrationType.GLOBAL,
-    name: MigrationName.USER_EMAIL_VIEW_CASING,
-  },
-  {
-    type: MigrationType.GLOBAL,
-    name: MigrationName.SYNC_QUOTAS,
-  },
-  {
-    type: MigrationType.APP,
-    name: MigrationName.APP_URLS,
-  },
-  {
-    type: MigrationType.APP,
-    name: MigrationName.EVENT_APP_BACKFILL,
-  },
-  {
-    type: MigrationType.APP,
-    name: MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS,
-  },
-  {
-    type: MigrationType.GLOBAL,
-    name: MigrationName.EVENT_GLOBAL_BACKFILL,
-  },
-  {
-    type: MigrationType.INSTALLATION,
-    name: MigrationName.EVENT_INSTALLATION_BACKFILL,
-  },
-  {
-    type: MigrationType.GLOBAL,
-    name: MigrationName.GLOBAL_INFO_SYNC_USERS,
-  },
-]

packages/backend-core/src/migrations/index.ts

@@ -1,2 +0,0 @@
-export * from "./migrations"
-export * from "./definitions"

packages/backend-core/src/migrations/migrations.ts

@@ -1,186 +0,0 @@
-import { DEFAULT_TENANT_ID } from "../constants"
-import {
-  DocumentType,
-  StaticDatabases,
-  getAllApps,
-  getGlobalDBName,
-  getDB,
-} from "../db"
-import environment from "../environment"
-import * as platform from "../platform"
-import * as context from "../context"
-import { DEFINITIONS } from "."
-import {
-  Migration,
-  MigrationOptions,
-  MigrationType,
-  MigrationNoOpOptions,
-  App,
-} from "@budibase/types"
-
-export const getMigrationsDoc = async (db: any) => {
-  // get the migrations doc
-  try {
-    return await db.get(DocumentType.MIGRATIONS)
-  } catch (err: any) {
-    if (err.status && err.status === 404) {
-      return { _id: DocumentType.MIGRATIONS }
-    } else {
-      throw err
-    }
-  }
-}
-
-export const backPopulateMigrations = async (opts: MigrationNoOpOptions) => {
-  // filter migrations to the type and populate a no-op migration
-  const migrations: Migration[] = DEFINITIONS.filter(
-    def => def.type === opts.type
-  ).map(d => ({ ...d, fn: async () => {} }))
-  await runMigrations(migrations, { noOp: opts })
-}
-
-export const runMigration = async (
-  migration: Migration,
-  options: MigrationOptions = {}
-) => {
-  const migrationType = migration.type
-  const migrationName = migration.name
-  const silent = migration.silent
-
-  const log = (message: string) => {
-    if (!silent) {
-      console.log(message)
-    }
-  }
-
-  // get the db to store the migration in
-  let dbNames: string[]
-  if (migrationType === MigrationType.GLOBAL) {
-    dbNames = [getGlobalDBName()]
-  } else if (migrationType === MigrationType.APP) {
-    if (options.noOp) {
-      if (!options.noOp.appId) {
-        throw new Error("appId is required for noOp app migration")
-      }
-      dbNames = [options.noOp.appId]
-    } else {
-      const apps = (await getAllApps(migration.appOpts)) as App[]
-      dbNames = apps.map(app => app.appId)
-    }
-  } else if (migrationType === MigrationType.INSTALLATION) {
-    dbNames = [StaticDatabases.PLATFORM_INFO.name]
-  } else {
-    throw new Error(`Unrecognised migration type [${migrationType}]`)
-  }
-
-  const length = dbNames.length
-  let count = 0
-
-  // run the migration against each db
-  for (const dbName of dbNames) {
-    count++
-    const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
-
-    const db = getDB(dbName)
-
-    try {
-      const doc = await getMigrationsDoc(db)
-
-      // the migration has already been run
-      if (doc[migrationName]) {
-        // check for force
-        if (
-          options.force &&
-          options.force[migrationType] &&
-          options.force[migrationType].includes(migrationName)
-        ) {
-          log(`[Migration: ${migrationName}] [DB: ${dbName}] Forcing`)
-        } else {
-          // no force, exit
-          return
-        }
-      }
-
-      // check if the migration is not a no-op
-      if (!options.noOp) {
-        log(
-          `[Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
-        )
-
-        if (migration.preventRetry) {
-          // eagerly set the completion date
-          // so that we never run this migration twice even upon failure
-          doc[migrationName] = Date.now()
-          const response = await db.put(doc)
-          doc._rev = response.rev
-        }
-
-        // run the migration
-        if (migrationType === MigrationType.APP) {
-          await context.doInAppContext(db.name, async () => {
-            await migration.fn(db)
-          })
-        } else {
-          await migration.fn(db)
-        }
-
-        log(`[Migration: ${migrationName}] [DB: ${dbName}] Complete`)
-      }
-
-      // mark as complete
-      doc[migrationName] = Date.now()
-      await db.put(doc)
-    } catch (err) {
-      console.error(
-        `[Migration: ${migrationName}] [DB: ${dbName}] Error: `,
-        err
-      )
-      throw err
-    }
-  }
-}
-
-export const runMigrations = async (
-  migrations: Migration[],
-  options: MigrationOptions = {}
-) => {
-  let tenantIds
-
-  if (environment.MULTI_TENANCY) {
-    if (options.noOp) {
-      tenantIds = [options.noOp.tenantId]
-    } else if (!options.tenantIds || !options.tenantIds.length) {
-      // run for all tenants
-      tenantIds = await platform.tenants.getTenantIds()
-    } else {
-      tenantIds = options.tenantIds
-    }
-  } else {
-    // single tenancy
-    tenantIds = [DEFAULT_TENANT_ID]
-  }
-
-  if (tenantIds.length > 1) {
-    console.log(`Checking migrations for ${tenantIds.length} tenants`)
-  } else {
-    console.log("Checking migrations")
-  }
-
-  let count = 0
-  // for all tenants
-  for (const tenantId of tenantIds) {
-    count++
-    if (tenantIds.length > 1) {
-      console.log(`Progress [${count}/${tenantIds.length}]`)
-    }
-    // for all migrations
-    for (const migration of migrations) {
-      // run the migration
-      await context.doInTenant(
-        tenantId,
-        async () => await runMigration(migration, options)
-      )
-    }
-  }
-  console.log("Migrations complete")
-}

packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap

@@ -1,11 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`migrations should match snapshot 1`] = `
-{
-  "_id": "migrations",
-  "_rev": "1-2f64479842a0513aa8b97f356b0b9127",
-  "createdAt": "2020-01-01T00:00:00.000Z",
-  "test": 1577836800000,
-  "updatedAt": "2020-01-01T00:00:00.000Z",
-}
-`;

packages/backend-core/src/migrations/tests/migrations.spec.ts

@@ -1,64 +0,0 @@
-import { testEnv, DBTestConfiguration } from "../../../tests/extra"
-import * as migrations from "../index"
-import * as context from "../../context"
-import { MigrationType } from "@budibase/types"
-
-testEnv.multiTenant()
-
-describe("migrations", () => {
-  const config = new DBTestConfiguration()
-
-  const migrationFunction = jest.fn()
-
-  const MIGRATIONS = [
-    {
-      type: MigrationType.GLOBAL,
-      name: "test" as any,
-      fn: migrationFunction,
-    },
-  ]
-
-  beforeEach(() => {
-    config.newTenant()
-  })
-
-  afterEach(async () => {
-    jest.clearAllMocks()
-  })
-
-  const migrate = () => {
-    return migrations.runMigrations(MIGRATIONS, {
-      tenantIds: [config.tenantId],
-    })
-  }
-
-  it("should run a new migration", async () => {
-    await config.doInTenant(async () => {
-      await migrate()
-      expect(migrationFunction).toHaveBeenCalled()
-      const db = context.getGlobalDB()
-      const doc = await migrations.getMigrationsDoc(db)
-      expect(doc.test).toBeDefined()
-    })
-  })
-
-  it("should match snapshot", async () => {
-    await config.doInTenant(async () => {
-      await migrate()
-      const doc = await migrations.getMigrationsDoc(context.getGlobalDB())
-      expect(doc).toMatchSnapshot()
-    })
-  })
-
-  it("should skip a previously run migration", async () => {
-    await config.doInTenant(async () => {
-      const db = context.getGlobalDB()
-      await migrate()
-      const previousDoc = await migrations.getMigrationsDoc(db)
-      await migrate()
-      const currentDoc = await migrations.getMigrationsDoc(db)
-      expect(migrationFunction).toHaveBeenCalledTimes(1)
-      expect(currentDoc.test).toBe(previousDoc.test)
-    })
-  })
-})

packages/bbui/src/bbui.css

@@ -45,6 +45,11 @@
   --purple: #806fde;
   --purple-dark: #130080;
 
+  --error-bg: rgba(226, 109, 105, 0.3);
+  --warning-bg: rgba(255, 210, 106, 0.3);
+  --error-content: rgba(226, 109, 105, 0.6);
+  --warning-content: rgba(255, 210, 106, 0.6);
+
   --rounded-small: 4px;
   --rounded-medium: 8px;
   --rounded-large: 16px;

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -12,7 +12,7 @@
     decodeJSBinding,
     encodeJSBinding,
     processObjectSync,
-    processStringSync,
+    processStringWithLogsSync,
   } from "@budibase/string-templates"
   import { readableToRuntimeBinding } from "@/dataBinding"
   import CodeEditor from "../CodeEditor/CodeEditor.svelte"
@@ -41,6 +41,7 @@
     InsertAtPositionFn,
     JSONValue,
   } from "@budibase/types"
+  import type { Log } from "@budibase/string-templates"
   import type { CompletionContext } from "@codemirror/autocomplete"
 
   const dispatch = createEventDispatcher()
@@ -66,6 +67,7 @@
   let insertAtPos: InsertAtPositionFn | undefined
   let targetMode: BindingMode | null = null
   let expressionResult: string | undefined
+  let expressionLogs: Log[] | undefined
   let expressionError: string | undefined
   let evaluating = false
 
@@ -157,7 +159,7 @@
     (expression: string | null, context: any, snippets: Snippet[]) => {
       try {
         expressionError = undefined
-        expressionResult = processStringSync(
+        const output = processStringWithLogsSync(
           expression || "",
           {
             ...context,
@@ -167,6 +169,8 @@
             noThrow: false,
           }
         )
+        expressionResult = output.result
+        expressionLogs = output.logs
       } catch (err: any) {
         expressionResult = undefined
         expressionError = err
@@ -421,6 +425,7 @@
         <EvaluationSidePanel
           {expressionResult}
           {expressionError}
+          {expressionLogs}
           {evaluating}
           expression={editorValue ? editorValue : ""}
         />

packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte

@@ -4,11 +4,13 @@
   import { Helpers } from "@budibase/bbui"
   import { fade } from "svelte/transition"
   import { UserScriptError } from "@budibase/string-templates"
+  import type { Log } from "@budibase/string-templates"
   import type { JSONValue } from "@budibase/types"
 
   // this can be essentially any primitive response from the JS function
   export let expressionResult: JSONValue | undefined = undefined
   export let expressionError: string | undefined = undefined
+  export let expressionLogs: Log[] = []
   export let evaluating = false
   export let expression: string | null = null
 
@@ -16,6 +18,11 @@
   $: empty = expression == null || expression?.trim() === ""
   $: success = !error && !empty
   $: highlightedResult = highlight(expressionResult)
+  $: highlightedLogs = expressionLogs.map(l => ({
+    log: highlight(l.log.join(", ")),
+    line: l.line,
+    type: l.type,
+  }))
 
   const formatError = (err: any) => {
     if (err.code === UserScriptError.code) {
@@ -25,14 +32,14 @@
   }
 
   // json can be any primitive type
-  const highlight = (json?: any | null) => {
+  const highlight = (json?: JSONValue | null) => {
     if (json == null) {
       return ""
     }
 
     // Attempt to parse and then stringify, in case this is valid result
     try {
-      json = JSON.stringify(JSON.parse(json), null, 2)
+      json = JSON.stringify(JSON.parse(json as any), null, 2)
     } catch (err) {
       // couldn't parse/stringify, just treat it as the raw input
     }
@@ -61,7 +68,7 @@
   <div class="header" class:success class:error>
     <div class="header-content">
       {#if error}
-        <Icon name="Alert" color="var(--spectrum-global-color-red-600)" />
+        <Icon name="Alert" color="var(--error-content)" />
         <div>Error</div>
         {#if evaluating}
           <div transition:fade|local={{ duration: 130 }}>
@@ -90,8 +97,36 @@
     {:else if error}
       {formatError(expressionError)}
     {:else}
+      <div class="output-lines">
+        {#each highlightedLogs as logLine}
+          <div
+            class="line"
+            class:error-log={logLine.type === "error"}
+            class:warn-log={logLine.type === "warn"}
+          >
+            <div class="icon-log">
+              {#if logLine.type === "error"}
+                <Icon
+                  size="XS"
+                  name="CloseCircle"
+                  color="var(--error-content)"
+                />
+              {:else if logLine.type === "warn"}
+                <Icon size="XS" name="Alert" color="var(--warning-content)" />
+              {/if}
+              <!-- eslint-disable-next-line svelte/no-at-html-tags-->
+              <span>{@html logLine.log}</span>
+            </div>
+            {#if logLine.line}
+              <span style="color: var(--blue)">:{logLine.line}</span>
+            {/if}
+          </div>
+        {/each}
+        <div class="line">
           <!-- eslint-disable-next-line svelte/no-at-html-tags-->
           {@html highlightedResult}
+        </div>
+      </div>
     {/if}
   </div>
 </div>
@@ -130,20 +165,37 @@
     height: 100%;
     z-index: 1;
     position: absolute;
-    opacity: 10%;
   }
   .header.error::before {
-    background: var(--spectrum-global-color-red-400);
+    background: var(--error-bg);
   }
   .body {
     flex: 1 1 auto;
     padding: var(--spacing-m) var(--spacing-l);
     font-family: var(--font-mono);
     font-size: 12px;
-    overflow-y: scroll;
+    overflow-y: auto;
     overflow-x: hidden;
-    white-space: pre-wrap;
+    white-space: pre-line;
     word-wrap: break-word;
     height: 0;
   }
+  .output-lines {
+    display: flex;
+    flex-direction: column;
+    gap: var(--spacing-xs);
+  }
+  .line {
+    border-bottom: var(--border-light);
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+    align-items: end;
+    padding: var(--spacing-s);
+  }
+  .icon-log {
+    display: flex;
+    gap: var(--spacing-s);
+    align-items: start;
+  }
 </style>

packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte

@@ -1,4 +1,5 @@
 <script>
+  import { datasources } from "@/stores/builder"
   import { Divider, Heading } from "@budibase/bbui"
 
   export let dividerState
@@ -6,6 +7,8 @@
   export let dataSet
   export let value
   export let onSelect
+
+  $: displayDatasourceName = $datasources.list.length > 1
 </script>
 
 {#if dividerState}
@@ -21,7 +24,7 @@
   {#each dataSet as data}
     <li
       class="spectrum-Menu-item"
-      class:is-selected={value?.label === data.label &&
+      class:is-selected={value?.resourceId === data.resourceId &&
         value?.type === data.type}
       role="option"
       aria-selected="true"
@@ -29,7 +32,9 @@
       on:click={() => onSelect(data)}
     >
       <span class="spectrum-Menu-itemLabel">
-        {data.datasourceName ? `${data.datasourceName} - ` : ""}{data.label}
+        {data.datasourceName && displayDatasourceName
+          ? `${data.datasourceName} - `
+          : ""}{data.label}
       </span>
       <svg
         class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"

packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte

@@ -34,7 +34,7 @@
   import ClientBindingPanel from "@/components/common/bindings/ClientBindingPanel.svelte"
   import DataSourceCategory from "@/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte"
   import { API } from "@/api"
-  import { datasourceSelect as format } from "@/helpers/data/format"
+  import { sortAndFormat } from "@/helpers/data/format"
 
   export let value = {}
   export let otherSources
@@ -51,25 +51,13 @@
   let modal
 
   $: text = value?.label ?? "Choose an option"
-  $: tables = $tablesStore.list
+  $: tables = sortAndFormat.tables($tablesStore.list, $datasources.list)
-    .map(table => format.table(table, $datasources.list))
-    .sort((a, b) => {
-      // sort tables alphabetically, grouped by datasource
-      const dsA = a.datasourceName ?? ""
-      const dsB = b.datasourceName ?? ""
-
-      const dsComparison = dsA.localeCompare(dsB)
-      if (dsComparison !== 0) {
-        return dsComparison
-      }
-      return a.label.localeCompare(b.label)
-    })
   $: viewsV1 = $viewsStore.list.map(view => ({
     ...view,
     label: view.name,
     type: "view",
   }))
-  $: viewsV2 = $viewsV2Store.list.map(format.viewV2)
+  $: viewsV2 = sortAndFormat.viewsV2($viewsV2Store.list, $datasources.list)
   $: views = [...(viewsV1 || []), ...(viewsV2 || [])]
   $: queries = $queriesStore.list
     .filter(q => showAllQueries || q.queryVerb === "read" || q.readable)

packages/builder/src/components/design/settings/controls/TableSelect.svelte

@@ -1,22 +1,32 @@
 <script>
-  import { Select } from "@budibase/bbui"
+  import { Popover, Select } from "@budibase/bbui"
   import { createEventDispatcher, onMount } from "svelte"
-  import { tables as tablesStore, viewsV2 } from "@/stores/builder"
+  import {
-  import { tableSelect as format } from "@/helpers/data/format"
+    tables as tableStore,
+    datasources as datasourceStore,
+    viewsV2 as viewsV2Store,
+  } from "@/stores/builder"
+  import DataSourceCategory from "./DataSourceSelect/DataSourceCategory.svelte"
+  import { sortAndFormat } from "@/helpers/data/format"
 
   export let value
 
+  let anchorRight, dropdownRight
+
   const dispatch = createEventDispatcher()
 
-  $: tables = $tablesStore.list.map(format.table)
+  $: tables = sortAndFormat.tables($tableStore.list, $datasourceStore.list)
-  $: views = $viewsV2.list.map(format.viewV2)
+  $: views = sortAndFormat.viewsV2($viewsV2Store.list, $datasourceStore.list)
   $: options = [...(tables || []), ...(views || [])]
 
+  $: text = value?.label ?? "Choose an option"
+
   const onChange = e => {
     dispatch(
       "change",
-      options.find(x => x.resourceId === e.detail)
+      options.find(x => x.resourceId === e.resourceId)
     )
+    dropdownRight.hide()
   }
 
   onMount(() => {
@@ -29,10 +39,47 @@
   })
 </script>
 
+<div class="container" bind:this={anchorRight}>
   <Select
-  on:change={onChange}
+    readonly
-  value={value?.resourceId}
+    value={text}
-  {options}
+    options={[text]}
-  getOptionValue={x => x.resourceId}
+    on:click={dropdownRight.show}
-  getOptionLabel={x => x.label}
   />
+</div>
+<Popover bind:this={dropdownRight} anchor={anchorRight}>
+  <div class="dropdown">
+    <DataSourceCategory
+      heading="Tables"
+      dataSet={tables}
+      {value}
+      onSelect={onChange}
+    />
+    {#if views?.length}
+      <DataSourceCategory
+        dividerState={true}
+        heading="Views"
+        dataSet={views}
+        {value}
+        onSelect={onChange}
+      />
+    {/if}
+  </div>
+</Popover>
+
+<style>
+  .container {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+  }
+  .container :global(:first-child) {
+    flex: 1 1 auto;
+  }
+
+  .dropdown {
+    padding: var(--spacing-m) 0;
+    z-index: 99999999;
+  }
+</style>

packages/builder/src/helpers/data/format.js

@@ -9,11 +9,18 @@ export const datasourceSelect = {
       datasourceName: datasource?.name,
     }
   },
-  viewV2: view => ({
+  viewV2: (view, datasources) => {
+    const datasource = datasources
+      .filter(f => f.entities)
+      .flatMap(d => d.entities)
+      .find(ds => ds._id === view.tableId)
+    return {
       ...view,
       label: view.name,
       type: "viewV2",
-  }),
+      datasourceName: datasource?.name,
+    }
+  },
 }
 
 export const tableSelect = {
@@ -31,3 +38,36 @@ export const tableSelect = {
     resourceId: view.id,
   }),
 }
+
+export const sortAndFormat = {
+  tables: (tables, datasources) => {
+    return tables
+      .map(table => {
+        const formatted = datasourceSelect.table(table, datasources)
+        return {
+          ...formatted,
+          resourceId: table._id,
+        }
+      })
+      .sort((a, b) => {
+        // sort tables alphabetically, grouped by datasource
+        const dsA = a.datasourceName ?? ""
+        const dsB = b.datasourceName ?? ""
+
+        const dsComparison = dsA.localeCompare(dsB)
+        if (dsComparison !== 0) {
+          return dsComparison
+        }
+        return a.label.localeCompare(b.label)
+      })
+  },
+  viewsV2: (views, datasources) => {
+    return views.map(view => {
+      const formatted = datasourceSelect.viewV2(view, datasources)
+      return {
+        ...formatted,
+        resourceId: view.id,
+      }
+    })
+  },
+}

packages/builder/src/pages/builder/apps/index.svelte

@@ -15,6 +15,7 @@
   import {
     appsStore,
     organisation,
+    admin,
     auth,
     groups,
     licensing,
@@ -42,6 +43,7 @@
     app => app.status === AppStatus.DEPLOYED
   )
   $: userApps = getUserApps(publishedApps, userGroups, $auth.user)
+  $: isOwner = $auth.accountPortalAccess && $admin.cloud
 
   function getUserApps(publishedApps, userGroups, user) {
     if (sdk.users.isAdmin(user)) {
@@ -111,7 +113,13 @@
               </MenuItem>
               <MenuItem
                 icon="LockClosed"
-                on:click={() => changePasswordModal.show()}
+                on:click={() => {
+                  if (isOwner) {
+                    window.location.href = `${$admin.accountPortalUrl}/portal/account`
+                  } else {
+                    changePasswordModal.show()
+                  }
+                }}
               >
                 Update password
               </MenuItem>

packages/builder/src/pages/builder/auth/reset.svelte

@@ -30,10 +30,16 @@
     try {
       loading = true
       if (forceResetPassword) {
+        const email = $auth.user.email
+        const tenantId = $auth.user.tenantId
         await auth.updateSelf({
           password,
           forceResetPassword: false,
         })
+        if (!$auth.user) {
+          // Update self will clear the platform user, so need to login
+          await auth.login(email, password, tenantId)
+        }
         $goto("../portal/")
       } else {
         await auth.resetPassword(password, resetCode)

packages/builder/src/pages/builder/portal/_components/UserDropdown.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { auth } from "@/stores/portal"
+  import { admin, auth } from "@/stores/portal"
   import { ActionMenu, MenuItem, Icon, Modal } from "@budibase/bbui"
   import { goto } from "@roxi/routify"
   import ProfileModal from "@/components/settings/ProfileModal.svelte"
@@ -13,6 +13,8 @@
   let updatePasswordModal
   let apiKeyModal
 
+  $: isOwner = $auth.accountPortalAccess && $admin.cloud
+
   const logout = async () => {
     try {
       await auth.logout()
@@ -32,7 +34,16 @@
   </MenuItem>
   <MenuItem icon="Moon" on:click={() => themeModal.show()}>Theme</MenuItem>
   {#if !$auth.isSSO}
-    <MenuItem icon="LockClosed" on:click={() => updatePasswordModal.show()}>
+    <MenuItem
+      icon="LockClosed"
+      on:click={() => {
+        if (isOwner) {
+          window.location.href = `${$admin.accountPortalUrl}/portal/account`
+        } else {
+          updatePasswordModal.show()
+        }
+      }}
+    >
       Update password
     </MenuItem>
   {/if}

packages/builder/src/stores/portal/auth.ts

@@ -121,8 +121,8 @@ class AuthStore extends BudiStore<PortalAuthStore> {
     }
   }
 
-  async login(username: string, password: string) {
+  async login(username: string, password: string, targetTenantId?: string) {
-    const tenantId = get(this.store).tenantId
+    const tenantId = targetTenantId || get(this.store).tenantId
     await API.logIn(tenantId, username, password)
     await this.getSelf()
   }

packages/client/src/components/app/DataProvider.svelte

@@ -14,7 +14,6 @@
     GroupUserDatasource,
     DataFetchOptions,
   } from "@budibase/types"
-  import { SDK, Component } from "../../index"
 
   type ProviderDatasource = Exclude<
     DataFetchDatasource,
@@ -29,8 +28,8 @@
   export let paginate: boolean
   export let autoRefresh: number
 
-  const { styleable, Provider, ActionTypes, API } = getContext<SDK>("sdk")
+  const { styleable, Provider, ActionTypes, API } = getContext("sdk")
-  const component = getContext<Component>("component")
+  const component = getContext("component")
 
   let interval: ReturnType<typeof setInterval>
   let queryExtensions: Record<string, any> = {}

packages/client/src/components/app/blocks/form/FormBlock.svelte

@@ -4,7 +4,6 @@
   import { Utils } from "@budibase/frontend-core"
   import FormBlockWrapper from "./FormBlockWrapper.svelte"
   import { get } from "svelte/store"
-  import { Component, Context, SDK } from "../../../../index"
   import { TableSchema, UIDatasource } from "@budibase/types"
 
   type Field = { name: string; active: boolean }
@@ -34,9 +33,9 @@
   export let saveButtonLabel: boolean
   export let deleteButtonLabel: boolean
 
-  const { fetchDatasourceSchema, generateGoldenSample } = getContext<SDK>("sdk")
+  const { fetchDatasourceSchema, generateGoldenSample } = getContext("sdk")
-  const component = getContext<Component>("component")
+  const component = getContext("component")
-  const context = getContext<Context>("context")
+  const context = getContext("context")
 
   let schema: TableSchema
 

packages/client/src/components/error-states/ComponentErrorState.svelte

@@ -3,7 +3,6 @@
   import { Icon } from "@budibase/bbui"
   import MissingRequiredSetting from "./MissingRequiredSetting.svelte"
   import MissingRequiredAncestor from "./MissingRequiredAncestor.svelte"
-  import { Component, SDK } from "../../index"
 
   export let missingRequiredSettings:
     | { key: string; label: string }[]
@@ -11,8 +10,8 @@
   export let missingRequiredAncestors: string[] | undefined
   export let invalidSettings: string[] | undefined
 
-  const component = getContext<Component>("component")
+  const component = getContext("component")
-  const { styleable, builderStore } = getContext<SDK>("sdk")
+  const { styleable, builderStore } = getContext("sdk")
 
   $: styles = { ...$component.styles, normal: {}, custom: null, empty: true }
   $: requiredSetting = missingRequiredSettings?.[0]

packages/client/src/context.d.ts

@@ -0,0 +1,7 @@
+import { Component, Context, SDK } from "."
+
+declare module "svelte" {
+  export function getContext(key: "sdk"): SDK
+  export function getContext(key: "component"): Component
+  export function getContext(key: "context"): Context
+}

packages/client/src/index.ts

@@ -20,4 +20,4 @@ export type Component = Readable<{
   errorState: boolean
 }>
 
-export type Context = Readable<any>
+export type Context = Readable<{}>

packages/frontend-core/src/api/migrations.ts

@@ -1,8 +1,8 @@
-import { GetOldMigrationStatus } from "@budibase/types"
+import { GetMigrationStatus } from "@budibase/types"
 import { BaseAPIClient } from "./types"
 
 export interface MigrationEndpoints {
-  getMigrationStatus: () => Promise<GetOldMigrationStatus>
+  getMigrationStatus: () => Promise<GetMigrationStatus>
 }
 
 export const buildMigrationEndpoints = (

packages/server/scripts/dev/manage.js

@@ -43,7 +43,6 @@ async function init() {
     BB_ADMIN_USER_EMAIL: "",
     BB_ADMIN_USER_PASSWORD: "",
     PLUGINS_DIR: "",
-    HTTP_MIGRATIONS: "0",
     HTTP_LOGGING: "0",
     VERSION: "0.0.0+local",
     PASSWORD_MIN_LENGTH: "1",

packages/server/src/api/controllers/application.ts

@@ -27,7 +27,6 @@ import {
   env as envCore,
   ErrorCode,
   events,
-  migrations,
   objectStore,
   roles,
   tenancy,
@@ -43,7 +42,6 @@ import { groups, licensing, quotas } from "@budibase/pro"
 import {
   App,
   Layout,
-  MigrationType,
   PlanType,
   Screen,
   UserCtx,
@@ -488,13 +486,6 @@ async function creationEvents(request: BBRequest<CreateAppRequest>, app: App) {
 }
 
 async function appPostCreate(ctx: UserCtx<CreateAppRequest, App>, app: App) {
-  const tenantId = tenancy.getTenantId()
-  await migrations.backPopulateMigrations({
-    type: MigrationType.APP,
-    tenantId,
-    appId: app.appId,
-  })
-
   await creationEvents(ctx.request, app)
 
   // app import, template creation and duplication

packages/server/src/api/controllers/migrations.ts

@@ -1,35 +1,11 @@
 import { context } from "@budibase/backend-core"
-import { migrate as migrationImpl, MIGRATIONS } from "../../migrations"
+import { Ctx, GetMigrationStatus } from "@budibase/types"
-import {
-  Ctx,
-  FetchOldMigrationResponse,
-  GetOldMigrationStatus,
-  RuneOldMigrationResponse,
-  RunOldMigrationRequest,
-} from "@budibase/types"
 import {
   getAppMigrationVersion,
   getLatestEnabledMigrationId,
 } from "../../appMigrations"
 
-export async function migrate(
+export async function getMigrationStatus(ctx: Ctx<void, GetMigrationStatus>) {
-  ctx: Ctx<RunOldMigrationRequest, RuneOldMigrationResponse>
-) {
-  const options = ctx.request.body
-  // don't await as can take a while, just return
-  migrationImpl(options)
-  ctx.body = { message: "Migration started." }
-}
-
-export async function fetchDefinitions(
-  ctx: Ctx<void, FetchOldMigrationResponse>
-) {
-  ctx.body = MIGRATIONS
-}
-
-export async function getMigrationStatus(
-  ctx: Ctx<void, GetOldMigrationStatus>
-) {
   const appId = context.getAppId()
 
   if (!appId) {

packages/server/src/api/controllers/view/tests/__snapshots__/viewBuilder.spec.js.snap

@@ -1,147 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`viewBuilder Calculate and filter creates a view with the calculation statistics and filter schema 1`] = `
-{
-  "map": "function (doc) {
-      if ((doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && !(
-      doc["myField"] === undefined ||
-      doc["myField"] === null ||
-      doc["myField"] === "" ||
-      (Array.isArray(doc["myField"]) && doc["myField"].length === 0)
-  )) && (doc["age"] > 17)) {
-        emit(doc["_id"], doc["myField"]);
-      }
-    }",
-  "meta": {
-    "calculation": "stats",
-    "field": "myField",
-    "filters": [
-      {
-        "condition": "MT",
-        "key": "age",
-        "value": 17,
-      },
-    ],
-    "groupBy": undefined,
-    "schema": {
-      "avg": {
-        "type": "number",
-      },
-      "count": {
-        "type": "number",
-      },
-      "field": {
-        "type": "string",
-      },
-      "max": {
-        "type": "number",
-      },
-      "min": {
-        "type": "number",
-      },
-      "sum": {
-        "type": "number",
-      },
-      "sumsqr": {
-        "type": "number",
-      },
-    },
-    "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
-  },
-  "reduce": "_stats",
-}
-`;
-
-exports[`viewBuilder Calculate creates a view with the calculation statistics schema 1`] = `
-{
-  "map": "function (doc) {
-      if ((doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && !(
-      doc["myField"] === undefined ||
-      doc["myField"] === null ||
-      doc["myField"] === "" ||
-      (Array.isArray(doc["myField"]) && doc["myField"].length === 0)
-  )) ) {
-        emit(doc["_id"], doc["myField"]);
-      }
-    }",
-  "meta": {
-    "calculation": "stats",
-    "field": "myField",
-    "filters": [],
-    "groupBy": undefined,
-    "schema": {
-      "avg": {
-        "type": "number",
-      },
-      "count": {
-        "type": "number",
-      },
-      "field": {
-        "type": "string",
-      },
-      "max": {
-        "type": "number",
-      },
-      "min": {
-        "type": "number",
-      },
-      "sum": {
-        "type": "number",
-      },
-      "sumsqr": {
-        "type": "number",
-      },
-    },
-    "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
-  },
-  "reduce": "_stats",
-}
-`;
-
-exports[`viewBuilder Filter creates a view with multiple filters and conjunctions 1`] = `
-{
-  "map": "function (doc) {
-      if (doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && (doc["Name"] === "Test" || doc["Yes"] > "Value")) {
-        emit(doc["_id"], doc["undefined"]);
-      }
-    }",
-  "meta": {
-    "calculation": undefined,
-    "field": undefined,
-    "filters": [
-      {
-        "condition": "EQUALS",
-        "key": "Name",
-        "value": "Test",
-      },
-      {
-        "condition": "MT",
-        "conjunction": "OR",
-        "key": "Yes",
-        "value": "Value",
-      },
-    ],
-    "groupBy": undefined,
-    "schema": null,
-    "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
-  },
-}
-`;
-
-exports[`viewBuilder Group By creates a view emitting the group by field 1`] = `
-{
-  "map": "function (doc) {
-      if (doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" ) {
-        emit(doc["age"], doc["score"]);
-      }
-    }",
-  "meta": {
-    "calculation": undefined,
-    "field": "score",
-    "filters": [],
-    "groupBy": "age",
-    "schema": null,
-    "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
-  },
-}
-`;

packages/server/src/api/controllers/view/tests/viewBuilder.spec.js

@@ -1,75 +0,0 @@
-const viewTemplate = require("../viewBuilder").default
-
-describe("viewBuilder", () => {
-  describe("Filter", () => {
-    it("creates a view with multiple filters and conjunctions", () => {
-      expect(
-        viewTemplate({
-          name: "Test View",
-          tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
-          filters: [
-            {
-              value: "Test",
-              condition: "EQUALS",
-              key: "Name",
-            },
-            {
-              value: "Value",
-              condition: "MT",
-              key: "Yes",
-              conjunction: "OR",
-            },
-          ],
-        })
-      ).toMatchSnapshot()
-    })
-  })
-
-  describe("Calculate", () => {
-    it("creates a view with the calculation statistics schema", () => {
-      expect(
-        viewTemplate({
-          name: "Calculate View",
-          field: "myField",
-          calculation: "stats",
-          tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
-          filters: [],
-        })
-      ).toMatchSnapshot()
-    })
-  })
-
-  describe("Group By", () => {
-    it("creates a view emitting the group by field", () => {
-      expect(
-        viewTemplate({
-          name: "Test Scores Grouped By Age",
-          tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
-          groupBy: "age",
-          field: "score",
-          filters: [],
-        })
-      ).toMatchSnapshot()
-    })
-  })
-
-  describe("Calculate and filter", () => {
-    it("creates a view with the calculation statistics and filter schema", () => {
-      expect(
-        viewTemplate({
-          name: "Calculate View",
-          field: "myField",
-          calculation: "stats",
-          tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
-          filters: [
-            {
-              value: 17,
-              condition: "MT",
-              key: "age",
-            },
-          ],
-        })
-      ).toMatchSnapshot()
-    })
-  })
-})

packages/server/src/api/controllers/view/tests/viewBuilder.spec.ts

@@ -0,0 +1,174 @@
+import viewTemplate from "../viewBuilder"
+
+describe("viewBuilder", () => {
+  describe("Filter", () => {
+    it("creates a view with multiple filters and conjunctions", () => {
+      expect(
+        viewTemplate({
+          field: "myField",
+          tableId: "tableId",
+          filters: [
+            {
+              value: "Test",
+              condition: "EQUALS",
+              key: "Name",
+            },
+            {
+              value: "Value",
+              condition: "MT",
+              key: "Yes",
+              conjunction: "OR",
+            },
+          ],
+        })
+      ).toEqual({
+        map: `function (doc) {
+      if (doc.tableId === "tableId" && (doc["Name"] === "Test" || doc["Yes"] > "Value")) {
+        emit(doc["_id"], doc["myField"]);
+      }
+    }`,
+        meta: {
+          calculation: undefined,
+          field: "myField",
+          filters: [
+            {
+              condition: "EQUALS",
+              key: "Name",
+              value: "Test",
+            },
+            {
+              condition: "MT",
+              conjunction: "OR",
+              key: "Yes",
+              value: "Value",
+            },
+          ],
+          groupBy: undefined,
+          schema: null,
+          tableId: "tableId",
+        },
+      })
+    })
+  })
+
+  describe("Calculate", () => {
+    it("creates a view with the calculation statistics schema", () => {
+      expect(
+        viewTemplate({
+          field: "myField",
+          calculation: "stats",
+          tableId: "tableId",
+          filters: [],
+        })
+      ).toEqual({
+        map: `function (doc) {
+      if ((doc.tableId === "tableId" && !(
+      doc["myField"] === undefined ||
+      doc["myField"] === null ||
+      doc["myField"] === "" ||
+      (Array.isArray(doc["myField"]) && doc["myField"].length === 0)
+  )) ) {
+        emit(doc["_id"], doc["myField"]);
+      }
+    }`,
+        meta: {
+          calculation: "stats",
+          field: "myField",
+          filters: [],
+          groupBy: undefined,
+          schema: {
+            min: { type: "number" },
+            max: { type: "number" },
+            avg: { type: "number" },
+            count: { type: "number" },
+            sumsqr: { type: "number" },
+            sum: { type: "number" },
+            field: { type: "string" },
+          },
+          tableId: "tableId",
+        },
+        reduce: "_stats",
+      })
+    })
+  })
+
+  describe("Group By", () => {
+    it("creates a view emitting the group by field", () => {
+      expect(
+        viewTemplate({
+          tableId: "tableId",
+          groupBy: "age",
+          field: "score",
+          filters: [],
+        })
+      ).toEqual({
+        map: `function (doc) {
+      if (doc.tableId === "tableId" ) {
+        emit(doc["age"], doc["score"]);
+      }
+    }`,
+        meta: {
+          calculation: undefined,
+          field: "score",
+          filters: [],
+          groupBy: "age",
+          schema: null,
+          tableId: "tableId",
+        },
+      })
+    })
+  })
+
+  describe("Calculate and filter", () => {
+    it("creates a view with the calculation statistics and filter schema", () => {
+      expect(
+        viewTemplate({
+          field: "myField",
+          calculation: "stats",
+          tableId: "tableId",
+          filters: [
+            {
+              value: 17,
+              condition: "MT",
+              key: "age",
+            },
+          ],
+        })
+      ).toEqual({
+        map: `function (doc) {
+      if ((doc.tableId === "tableId" && !(
+      doc["myField"] === undefined ||
+      doc["myField"] === null ||
+      doc["myField"] === "" ||
+      (Array.isArray(doc["myField"]) && doc["myField"].length === 0)
+  )) && (doc["age"] > 17)) {
+        emit(doc["_id"], doc["myField"]);
+      }
+    }`,
+        meta: {
+          calculation: "stats",
+          field: "myField",
+          filters: [
+            {
+              condition: "MT",
+              key: "age",
+              value: 17,
+            },
+          ],
+          groupBy: undefined,
+          schema: {
+            min: { type: "number" },
+            max: { type: "number" },
+            avg: { type: "number" },
+            count: { type: "number" },
+            sumsqr: { type: "number" },
+            sum: { type: "number" },
+            field: { type: "string" },
+          },
+          tableId: "tableId",
+        },
+        reduce: "_stats",
+      })
+    })
+  })
+})

packages/server/src/api/controllers/view/viewBuilder.ts

@@ -1,4 +1,4 @@
-import { ViewFilter, ViewTemplateOpts, DBView } from "@budibase/types"
+import { ViewFilter, DBView } from "@budibase/types"
 
 const TOKEN_MAP: Record<string, string> = {
   EQUALS: "===",
@@ -120,7 +120,7 @@ function parseFilterExpression(filters: ViewFilter[]) {
  * @param groupBy - field to group calculation results on, if any
  */
 function parseEmitExpression(field: string, groupBy: string) {
-  return `emit(doc["${groupBy || "_id"}"], doc["${field}"]);`
+  return `emit(doc["${groupBy}"], doc["${field}"]);`
 }
 
 /**
@@ -135,7 +135,19 @@ function parseEmitExpression(field: string, groupBy: string) {
  * calculation: an optional calculation to be performed over the view data.
  */
 export default function (
-  { field, tableId, groupBy, filters = [], calculation }: ViewTemplateOpts,
+  {
+    field,
+    tableId,
+    groupBy,
+    filters = [],
+    calculation,
+  }: {
+    field: string
+    tableId: string
+    groupBy?: string
+    filters?: ViewFilter[]
+    calculation?: string
+  },
   groupByMulti?: boolean
 ): DBView {
   // first filter can't have a conjunction
@@ -168,7 +180,7 @@ export default function (
   const parsedFilters = parseFilterExpression(filters)
   const filterExpression = parsedFilters ? `&& (${parsedFilters})` : ""
 
-  const emitExpression = parseEmitExpression(field, groupBy)
+  const emitExpression = parseEmitExpression(field, groupBy || "_id")
   const tableExpression = `doc.tableId === "${tableId}"`
   const coreExpression = statFilter
     ? `(${tableExpression} && ${statFilter})`

packages/server/src/api/routes/migrations.ts

@@ -1,16 +1,8 @@
 import Router from "@koa/router"
 import * as migrationsController from "../controllers/migrations"
-import { auth } from "@budibase/backend-core"
 
 const router: Router = new Router()
 
-router
+router.get("/api/migrations/status", migrationsController.getMigrationStatus)
-  .post("/api/migrations/run", auth.internalApi, migrationsController.migrate)
-  .get(
-    "/api/migrations/definitions",
-    auth.internalApi,
-    migrationsController.fetchDefinitions
-  )
-  .get("/api/migrations/status", migrationsController.getMigrationStatus)
 
 export default router

packages/server/src/api/routes/tests/automation.spec.ts

@@ -19,9 +19,11 @@ import {
   Table,
 } from "@budibase/types"
 import { mocks } from "@budibase/backend-core/tests"
-import { FilterConditions } from "../../../automations/steps/filter"
 import { removeDeprecated } from "../../../automations/utils"
 import { createAutomationBuilder } from "../../../automations/tests/utilities/AutomationTestBuilder"
+import { automations } from "@budibase/shared-core"
+
+const FilterConditions = automations.steps.filter.FilterConditions
 
 const MAX_RETRIES = 4
 let {

packages/server/src/automations/actions.ts

@@ -1,3 +1,4 @@
+import { automations } from "@budibase/shared-core"
 import * as sendSmtpEmail from "./steps/sendSmtpEmail"
 import * as createRow from "./steps/createRow"
 import * as updateRow from "./steps/updateRow"
@@ -14,11 +15,10 @@ import * as make from "./steps/make"
 import * as filter from "./steps/filter"
 import * as delay from "./steps/delay"
 import * as queryRow from "./steps/queryRows"
-import * as loop from "./steps/loop"
 import * as collect from "./steps/collect"
-import * as branch from "./steps/branch"
 import * as triggerAutomationRun from "./steps/triggerAutomationRun"
 import * as openai from "./steps/openai"
+import * as bash from "./steps/bash"
 import env from "../environment"
 import {
   PluginType,
@@ -62,42 +62,39 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<
   string,
   AutomationStepDefinition
 > = {
-  SEND_EMAIL_SMTP: sendSmtpEmail.definition,
+  SEND_EMAIL_SMTP: automations.steps.sendSmtpEmail.definition,
-  CREATE_ROW: createRow.definition,
+  CREATE_ROW: automations.steps.createRow.definition,
-  UPDATE_ROW: updateRow.definition,
+  UPDATE_ROW: automations.steps.updateRow.definition,
-  DELETE_ROW: deleteRow.definition,
+  DELETE_ROW: automations.steps.deleteRow.definition,
-  OUTGOING_WEBHOOK: outgoingWebhook.definition,
+  OUTGOING_WEBHOOK: automations.steps.outgoingWebhook.definition,
-  EXECUTE_SCRIPT: executeScript.definition,
+  EXECUTE_SCRIPT: automations.steps.executeScript.definition,
-  EXECUTE_QUERY: executeQuery.definition,
+  EXECUTE_QUERY: automations.steps.executeQuery.definition,
-  SERVER_LOG: serverLog.definition,
+  SERVER_LOG: automations.steps.serverLog.definition,
-  DELAY: delay.definition,
+  DELAY: automations.steps.delay.definition,
-  FILTER: filter.definition,
+  FILTER: automations.steps.filter.definition,
-  QUERY_ROWS: queryRow.definition,
+  QUERY_ROWS: automations.steps.queryRows.definition,
-  LOOP: loop.definition,
+  LOOP: automations.steps.loop.definition,
-  COLLECT: collect.definition,
+  COLLECT: automations.steps.collect.definition,
-  TRIGGER_AUTOMATION_RUN: triggerAutomationRun.definition,
+  TRIGGER_AUTOMATION_RUN: automations.steps.triggerAutomationRun.definition,
-  BRANCH: branch.definition,
+  BRANCH: automations.steps.branch.definition,
   // these used to be lowercase step IDs, maintain for backwards compat
-  discord: discord.definition,
+  discord: automations.steps.discord.definition,
-  slack: slack.definition,
+  slack: automations.steps.slack.definition,
-  zapier: zapier.definition,
+  zapier: automations.steps.zapier.definition,
-  integromat: make.definition,
+  integromat: automations.steps.make.definition,
-  n8n: n8n.definition,
+  n8n: automations.steps.n8n.definition,
 }
 
 // don't add the bash script/definitions unless in self host
 // the fact this isn't included in any definitions means it cannot be
 // ran at all
 if (env.SELF_HOSTED) {
-  const bash = require("./steps/bash")
+  // @ts-expect-error
-
-  // @ts-ignore
   ACTION_IMPLS["EXECUTE_BASH"] = bash.run
-  // @ts-ignore
+  BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = automations.steps.bash.definition
-  BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
 
   if (env.isTest()) {
-    BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition
+    BUILTIN_ACTION_DEFINITIONS["OPENAI"] = automations.steps.openai.definition
   }
 }
 
@@ -105,7 +102,7 @@ export async function getActionDefinitions(): Promise<
   Record<keyof typeof AutomationActionStepId, AutomationStepDefinition>
 > {
   if (env.SELF_HOSTED) {
-    BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition
+    BUILTIN_ACTION_DEFINITIONS["OPENAI"] = automations.steps.openai.definition
   }
 
   const actionDefinitions = BUILTIN_ACTION_DEFINITIONS

packages/server/src/automations/steps/bash.ts

@@ -2,55 +2,7 @@ import { execSync } from "child_process"
 import { processStringSync } from "@budibase/string-templates"
 import * as automationUtils from "../automationUtils"
 import environment from "../../environment"
-import {
+import { BashStepInputs, BashStepOutputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
-  BashStepInputs,
-  BashStepOutputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  name: "Bash Scripting",
-  tagline: "Execute a bash command",
-  icon: "JourneyEvent",
-  description: "Run a bash script",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  stepId: AutomationActionStepId.EXECUTE_BASH,
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        code: {
-          type: AutomationIOType.STRING,
-          customType: AutomationCustomIOType.CODE,
-          title: "Code",
-        },
-      },
-      required: ["code"],
-    },
-    outputs: {
-      properties: {
-        stdout: {
-          type: AutomationIOType.STRING,
-          description: "Standard output of your bash command or script",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the command was successful",
-        },
-      },
-      required: ["stdout"],
-    },
-  },
-}
 
 export async function run({
   inputs,

packages/server/src/automations/steps/collect.ts

@@ -1,48 +1,4 @@
-import {
+import { CollectStepInputs, CollectStepOutputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  CollectStepInputs,
-  CollectStepOutputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  name: "Collect Data",
-  tagline: "Collect data to be sent to design",
-  icon: "Collection",
-  description:
-    "Collects specified data so it can be provided to the design section",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {},
-  stepId: AutomationActionStepId.COLLECT,
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        collection: {
-          type: AutomationIOType.STRING,
-          title: "What to Collect",
-        },
-      },
-      required: ["collection"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-        value: {
-          type: AutomationIOType.STRING,
-          description: "Collected data",
-        },
-      },
-      required: ["success", "value"],
-    },
-  },
-}
 
 export async function run({
   inputs,

packages/server/src/automations/steps/createRow.ts

@@ -5,77 +5,9 @@ import {
   sendAutomationAttachmentsToStorage,
 } from "../automationUtils"
 import { buildCtx } from "./utils"
-import {
+import { CreateRowStepInputs, CreateRowStepOutputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
-  CreateRowStepInputs,
-  CreateRowStepOutputs,
-} from "@budibase/types"
 import { EventEmitter } from "events"
 
-export const definition: AutomationStepDefinition = {
-  name: "Create Row",
-  tagline: "Create a {{inputs.enriched.table.name}} row",
-  icon: "TableRowAddBottom",
-  description: "Add a row to your database",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  stepId: AutomationActionStepId.CREATE_ROW,
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        row: {
-          type: AutomationIOType.OBJECT,
-          properties: {
-            tableId: {
-              type: AutomationIOType.STRING,
-              customType: AutomationCustomIOType.TABLE,
-            },
-          },
-          customType: AutomationCustomIOType.ROW,
-          title: "Table",
-          required: ["tableId"],
-        },
-      },
-      required: ["row"],
-    },
-    outputs: {
-      properties: {
-        row: {
-          type: AutomationIOType.OBJECT,
-          customType: AutomationCustomIOType.ROW,
-          description: "The new row",
-        },
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The response from the table",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the row creation was successful",
-        },
-        id: {
-          type: AutomationIOType.STRING,
-          description: "The identifier of the new row",
-        },
-        revision: {
-          type: AutomationIOType.STRING,
-          description: "The revision of the new row",
-        },
-      },
-      required: ["success", "id", "revision"],
-    },
-  },
-}
-
 export async function run({
   inputs,
   appId,

packages/server/src/automations/steps/delay.ts

@@ -1,44 +1,5 @@
 import { wait } from "../../utilities"
-import {
+import { DelayStepInputs, DelayStepOutputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
-  DelayStepInputs,
-  DelayStepOutputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  name: "Delay",
-  icon: "Clock",
-  tagline: "Delay for {{inputs.time}} milliseconds",
-  description: "Delay the automation until an amount of time has passed",
-  stepId: AutomationActionStepId.DELAY,
-  internal: true,
-  features: {},
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        time: {
-          type: AutomationIOType.NUMBER,
-          title: "Delay in milliseconds",
-        },
-      },
-      required: ["time"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the delay was successful",
-        },
-      },
-      required: ["success"],
-    },
-  },
-  type: AutomationStepType.LOGIC,
-}
 
 export async function run({
   inputs,

packages/server/src/automations/steps/deleteRow.ts

@@ -2,64 +2,7 @@ import { EventEmitter } from "events"
 import { destroy } from "../../api/controllers/row"
 import { buildCtx } from "./utils"
 import { getError } from "../automationUtils"
-import {
+import { DeleteRowStepInputs, DeleteRowStepOutputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationCustomIOType,
-  AutomationFeature,
-  DeleteRowStepInputs,
-  DeleteRowStepOutputs,
-  AutomationStepDefinition,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  description: "Delete a row from your database",
-  icon: "TableRowRemoveCenter",
-  name: "Delete Row",
-  tagline: "Delete a {{inputs.enriched.table.name}} row",
-  type: AutomationStepType.ACTION,
-  stepId: AutomationActionStepId.DELETE_ROW,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        tableId: {
-          type: AutomationIOType.STRING,
-          customType: AutomationCustomIOType.TABLE,
-          title: "Table",
-        },
-        id: {
-          type: AutomationIOType.STRING,
-          title: "Row ID",
-        },
-      },
-      required: ["tableId", "id"],
-    },
-    outputs: {
-      properties: {
-        row: {
-          type: AutomationIOType.OBJECT,
-          customType: AutomationCustomIOType.ROW,
-          description: "The deleted row",
-        },
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The response from the table",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the deletion was successful",
-        },
-      },
-      required: ["row", "success"],
-    },
-  },
-}
 
 export async function run({
   inputs,

packages/server/src/automations/steps/discord.ts

@@ -1,71 +1,10 @@
 import fetch from "node-fetch"
 import { getFetchResponse } from "./utils"
-import {
+import { ExternalAppStepOutputs, DiscordStepInputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
-  ExternalAppStepOutputs,
-  DiscordStepInputs,
-  AutomationStepDefinition,
-} from "@budibase/types"
 
 const DEFAULT_USERNAME = "Budibase Automate"
 const DEFAULT_AVATAR_URL = "https://i.imgur.com/a1cmTKM.png"
 
-export const definition: AutomationStepDefinition = {
-  name: "Discord Message",
-  tagline: "Send a message to a Discord server",
-  description: "Send a message to a Discord server",
-  icon: "ri-discord-line",
-  stepId: AutomationActionStepId.discord,
-  type: AutomationStepType.ACTION,
-  internal: false,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        url: {
-          type: AutomationIOType.STRING,
-          title: "Discord Webhook URL",
-        },
-        username: {
-          type: AutomationIOType.STRING,
-          title: "Bot Name",
-        },
-        avatar_url: {
-          type: AutomationIOType.STRING,
-          title: "Bot Avatar URL",
-        },
-        content: {
-          type: AutomationIOType.STRING,
-          title: "Message",
-        },
-      },
-      required: ["url", "content"],
-    },
-    outputs: {
-      properties: {
-        httpStatus: {
-          type: AutomationIOType.NUMBER,
-          description: "The HTTP status code of the request",
-        },
-        response: {
-          type: AutomationIOType.STRING,
-          description: "The response from the Discord Webhook",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the message sent successfully",
-        },
-      },
-    },
-  },
-}
-
 export async function run({
   inputs,
 }: {

packages/server/src/automations/steps/executeQuery.ts

@@ -3,67 +3,10 @@ import * as queryController from "../../api/controllers/query"
 import { buildCtx } from "./utils"
 import * as automationUtils from "../automationUtils"
 import {
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
   ExecuteQueryStepInputs,
   ExecuteQueryStepOutputs,
 } from "@budibase/types"
 
-export const definition: AutomationStepDefinition = {
-  name: "External Data Connector",
-  tagline: "Execute Data Connector",
-  icon: "Data",
-  description: "Execute a query in an external data connector",
-  type: AutomationStepType.ACTION,
-  stepId: AutomationActionStepId.EXECUTE_QUERY,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        query: {
-          type: AutomationIOType.OBJECT,
-          properties: {
-            queryId: {
-              type: AutomationIOType.STRING,
-              customType: AutomationCustomIOType.QUERY,
-            },
-          },
-          customType: AutomationCustomIOType.QUERY_PARAMS,
-          title: "Parameters",
-          required: ["queryId"],
-        },
-      },
-      required: ["query"],
-    },
-    outputs: {
-      properties: {
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The response from the datasource execution",
-        },
-        info: {
-          type: AutomationIOType.OBJECT,
-          description:
-            "Some query types may return extra data, like headers from a REST query",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-      },
-      required: ["response", "success"],
-    },
-  },
-}
-
 export async function run({
   inputs,
   appId,

packages/server/src/automations/steps/executeScript.ts

@@ -2,56 +2,11 @@ import * as scriptController from "../../api/controllers/script"
 import { buildCtx } from "./utils"
 import * as automationUtils from "../automationUtils"
 import {
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
   ExecuteScriptStepInputs,
   ExecuteScriptStepOutputs,
 } from "@budibase/types"
 import { EventEmitter } from "events"
 
-export const definition: AutomationStepDefinition = {
-  name: "JS Scripting",
-  tagline: "Execute JavaScript Code",
-  icon: "Code",
-  description: "Run a piece of JavaScript code in your automation",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  stepId: AutomationActionStepId.EXECUTE_SCRIPT,
-  inputs: {},
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  schema: {
-    inputs: {
-      properties: {
-        code: {
-          type: AutomationIOType.STRING,
-          customType: AutomationCustomIOType.CODE,
-          title: "Code",
-        },
-      },
-      required: ["code"],
-    },
-    outputs: {
-      properties: {
-        value: {
-          type: AutomationIOType.STRING,
-          description: "The result of the return statement",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-      },
-      required: ["success"],
-    },
-  },
-}
-
 export async function run({
   inputs,
   appId,

packages/server/src/automations/steps/filter.ts

@@ -1,74 +1,7 @@
-import {
+import { FilterStepInputs, FilterStepOutputs } from "@budibase/types"
-  AutomationActionStepId,
+import { automations } from "@budibase/shared-core"
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  FilterStepInputs,
-  FilterStepOutputs,
-} from "@budibase/types"
 
-export const FilterConditions = {
+const FilterConditions = automations.steps.filter.FilterConditions
-  EQUAL: "EQUAL",
-  NOT_EQUAL: "NOT_EQUAL",
-  GREATER_THAN: "GREATER_THAN",
-  LESS_THAN: "LESS_THAN",
-}
-
-export const PrettyFilterConditions = {
-  [FilterConditions.EQUAL]: "Equals",
-  [FilterConditions.NOT_EQUAL]: "Not equals",
-  [FilterConditions.GREATER_THAN]: "Greater than",
-  [FilterConditions.LESS_THAN]: "Less than",
-}
-
-export const definition: AutomationStepDefinition = {
-  name: "Condition",
-  tagline: "{{inputs.field}} {{inputs.condition}} {{inputs.value}}",
-  icon: "Branch2",
-  description:
-    "Conditionally halt automations which do not meet certain conditions",
-  type: AutomationStepType.LOGIC,
-  internal: true,
-  features: {},
-  stepId: AutomationActionStepId.FILTER,
-  inputs: {
-    condition: FilterConditions.EQUAL,
-  },
-  schema: {
-    inputs: {
-      properties: {
-        field: {
-          type: AutomationIOType.STRING,
-          title: "Reference Value",
-        },
-        condition: {
-          type: AutomationIOType.STRING,
-          title: "Condition",
-          enum: Object.values(FilterConditions),
-          pretty: Object.values(PrettyFilterConditions),
-        },
-        value: {
-          type: AutomationIOType.STRING,
-          title: "Comparison Value",
-        },
-      },
-      required: ["field", "condition", "value"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-        result: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the logic block passed",
-        },
-      },
-      required: ["success", "result"],
-    },
-  },
-}
 
 export async function run({
   inputs,

packages/server/src/automations/steps/make.ts

@@ -1,62 +1,6 @@
 import fetch from "node-fetch"
 import { getFetchResponse } from "./utils"
-import {
+import { ExternalAppStepOutputs, MakeIntegrationInputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
-  ExternalAppStepOutputs,
-  MakeIntegrationInputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  name: "Make Integration",
-  stepTitle: "Make",
-  tagline: "Trigger a Make scenario",
-  description:
-    "Performs a webhook call to Make and gets the response (if configured)",
-  icon: "ri-shut-down-line",
-  stepId: AutomationActionStepId.integromat,
-  type: AutomationStepType.ACTION,
-  internal: false,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        url: {
-          type: AutomationIOType.STRING,
-          title: "Webhook URL",
-        },
-        body: {
-          type: AutomationIOType.JSON,
-          title: "Payload",
-        },
-      },
-      required: ["url", "body"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether call was successful",
-        },
-        httpStatus: {
-          type: AutomationIOType.NUMBER,
-          description: "The HTTP status code returned",
-        },
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The webhook response - this can have properties",
-        },
-      },
-      required: ["success", "response"],
-    },
-  },
-}
 
 export async function run({
   inputs,

packages/server/src/automations/steps/n8n.ts

@@ -1,73 +1,11 @@
 import fetch, { HeadersInit } from "node-fetch"
 import { getFetchResponse } from "./utils"
 import {
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
   HttpMethod,
   ExternalAppStepOutputs,
   n8nStepInputs,
 } from "@budibase/types"
 
-export const definition: AutomationStepDefinition = {
-  name: "n8n Integration",
-  stepTitle: "n8n",
-  tagline: "Trigger an n8n workflow",
-  description:
-    "Performs a webhook call to n8n and gets the response (if configured)",
-  icon: "ri-shut-down-line",
-  stepId: AutomationActionStepId.n8n,
-  type: AutomationStepType.ACTION,
-  internal: false,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        url: {
-          type: AutomationIOType.STRING,
-          title: "Webhook URL",
-        },
-        method: {
-          type: AutomationIOType.STRING,
-          title: "Method",
-          enum: Object.values(HttpMethod),
-        },
-        authorization: {
-          type: AutomationIOType.STRING,
-          title: "Authorization",
-        },
-        body: {
-          type: AutomationIOType.JSON,
-          title: "Payload",
-        },
-      },
-      required: ["url", "method"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether call was successful",
-        },
-        httpStatus: {
-          type: AutomationIOType.NUMBER,
-          description: "The HTTP status code returned",
-        },
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The webhook response - this can have properties",
-        },
-      },
-      required: ["success", "response"],
-    },
-  },
-}
-
 export async function run({
   inputs,
 }: {

packages/server/src/automations/steps/openai.ts

@@ -1,67 +1,10 @@
 import { OpenAI } from "openai"
 
-import {
+import { OpenAIStepInputs, OpenAIStepOutputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  OpenAIStepInputs,
-  OpenAIStepOutputs,
-} from "@budibase/types"
 import { env } from "@budibase/backend-core"
 import * as automationUtils from "../automationUtils"
 import * as pro from "@budibase/pro"
 
-enum Model {
-  GPT_4O_MINI = "gpt-4o-mini",
-  GPT_4O = "gpt-4o",
-  GPT_4 = "gpt-4",
-  GPT_35_TURBO = "gpt-3.5-turbo",
-}
-
-export const definition: AutomationStepDefinition = {
-  name: "OpenAI",
-  tagline: "Send prompts to ChatGPT",
-  icon: "Algorithm",
-  description: "Interact with the OpenAI ChatGPT API.",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {},
-  stepId: AutomationActionStepId.OPENAI,
-  inputs: {
-    prompt: "",
-  },
-  schema: {
-    inputs: {
-      properties: {
-        prompt: {
-          type: AutomationIOType.STRING,
-          title: "Prompt",
-        },
-        model: {
-          type: AutomationIOType.STRING,
-          title: "Model",
-          enum: Object.values(Model),
-        },
-      },
-      required: ["prompt", "model"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-        response: {
-          type: AutomationIOType.STRING,
-          description: "What was output",
-        },
-      },
-      required: ["success", "response"],
-    },
-  },
-}
-
 /**
  * Maintains backward compatibility with automation steps created before the introduction
  * of custom configurations and Budibase AI

packages/server/src/automations/steps/outgoingWebhook.ts

@@ -2,12 +2,6 @@ import fetch from "node-fetch"
 import { getFetchResponse } from "./utils"
 import * as automationUtils from "../automationUtils"
 import {
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
   ExternalAppStepOutputs,
   OutgoingWebhookStepInputs,
 } from "@budibase/types"
@@ -26,69 +20,6 @@ const BODY_REQUESTS = [RequestType.POST, RequestType.PUT, RequestType.PATCH]
  * NOTE: this functionality is deprecated - it no longer should be used.
  */
 
-export const definition: AutomationStepDefinition = {
-  deprecated: true,
-  name: "Outgoing webhook",
-  tagline: "Send a {{inputs.requestMethod}} request",
-  icon: "Send",
-  description: "Send a request of specified method to a URL",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  stepId: AutomationActionStepId.OUTGOING_WEBHOOK,
-  inputs: {
-    requestMethod: "POST",
-    url: "http://",
-    requestBody: "{}",
-    headers: "{}",
-  },
-  schema: {
-    inputs: {
-      properties: {
-        requestMethod: {
-          type: AutomationIOType.STRING,
-          enum: Object.values(RequestType),
-          title: "Request method",
-        },
-        url: {
-          type: AutomationIOType.STRING,
-          title: "URL",
-        },
-        requestBody: {
-          type: AutomationIOType.STRING,
-          title: "JSON Body",
-          customType: AutomationCustomIOType.WIDE,
-        },
-        headers: {
-          type: AutomationIOType.STRING,
-          title: "Headers",
-          customType: AutomationCustomIOType.WIDE,
-        },
-      },
-      required: ["requestMethod", "url"],
-    },
-    outputs: {
-      properties: {
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The response from the webhook",
-        },
-        httpStatus: {
-          type: AutomationIOType.NUMBER,
-          description: "The HTTP status code returned",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-      },
-      required: ["response", "success"],
-    },
-  },
-}
-
 export async function run({
   inputs,
 }: {

packages/server/src/automations/steps/queryRows.ts

@@ -4,84 +4,12 @@ import { buildCtx } from "./utils"
 import * as automationUtils from "../automationUtils"
 import {
   FieldType,
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
   EmptyFilterOption,
   SortOrder,
   QueryRowsStepInputs,
   QueryRowsStepOutputs,
 } from "@budibase/types"
 
-const SortOrderPretty = {
-  [SortOrder.ASCENDING]: "Ascending",
-  [SortOrder.DESCENDING]: "Descending",
-}
-
-export const definition: AutomationStepDefinition = {
-  description: "Query rows from the database",
-  icon: "Search",
-  name: "Query rows",
-  tagline: "Query rows from {{inputs.enriched.table.name}} table",
-  type: AutomationStepType.ACTION,
-  stepId: AutomationActionStepId.QUERY_ROWS,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        tableId: {
-          type: AutomationIOType.STRING,
-          customType: AutomationCustomIOType.TABLE,
-          title: "Table",
-        },
-        filters: {
-          type: AutomationIOType.OBJECT,
-          customType: AutomationCustomIOType.FILTERS,
-          title: "Filtering",
-        },
-        sortColumn: {
-          type: AutomationIOType.STRING,
-          title: "Sort Column",
-          customType: AutomationCustomIOType.COLUMN,
-        },
-        sortOrder: {
-          type: AutomationIOType.STRING,
-          title: "Sort Order",
-          enum: Object.values(SortOrder),
-          pretty: Object.values(SortOrderPretty),
-        },
-        limit: {
-          type: AutomationIOType.NUMBER,
-          title: "Limit",
-          customType: AutomationCustomIOType.QUERY_LIMIT,
-        },
-      },
-      required: ["tableId"],
-    },
-    outputs: {
-      properties: {
-        rows: {
-          type: AutomationIOType.ARRAY,
-          customType: AutomationCustomIOType.ROWS,
-          description: "The rows that were found",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the query was successful",
-        },
-      },
-      required: ["rows", "success"],
-    },
-  },
-}
-
 async function getTable(appId: string, tableId: string) {
   const ctx: any = buildCtx(appId, null, {
     params: {

packages/server/src/automations/steps/sendSmtpEmail.ts

@@ -1,102 +1,6 @@
 import { sendSmtpEmail } from "../../utilities/workerRequests"
 import * as automationUtils from "../automationUtils"
-import {
+import { SmtpEmailStepInputs, BaseAutomationOutputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
-  AutomationCustomIOType,
-  SmtpEmailStepInputs,
-  BaseAutomationOutputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  description: "Send an email using SMTP",
-  tagline: "Send SMTP email to {{inputs.to}}",
-  icon: "Email",
-  name: "Send Email (SMTP)",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  stepId: AutomationActionStepId.SEND_EMAIL_SMTP,
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        to: {
-          type: AutomationIOType.STRING,
-          title: "Send To",
-        },
-        from: {
-          type: AutomationIOType.STRING,
-          title: "Send From",
-        },
-        cc: {
-          type: AutomationIOType.STRING,
-          title: "CC",
-        },
-        bcc: {
-          type: AutomationIOType.STRING,
-          title: "BCC",
-        },
-        subject: {
-          type: AutomationIOType.STRING,
-          title: "Email Subject",
-        },
-        contents: {
-          type: AutomationIOType.STRING,
-          title: "HTML Contents",
-        },
-        addInvite: {
-          type: AutomationIOType.BOOLEAN,
-          title: "Add calendar invite",
-        },
-        startTime: {
-          type: AutomationIOType.DATE,
-          title: "Start Time",
-          dependsOn: "addInvite",
-        },
-        endTime: {
-          type: AutomationIOType.DATE,
-          title: "End Time",
-          dependsOn: "addInvite",
-        },
-        summary: {
-          type: AutomationIOType.STRING,
-          title: "Meeting Summary",
-          dependsOn: "addInvite",
-        },
-        location: {
-          type: AutomationIOType.STRING,
-          title: "Location",
-          dependsOn: "addInvite",
-        },
-        attachments: {
-          type: AutomationIOType.ATTACHMENT,
-          customType: AutomationCustomIOType.MULTI_ATTACHMENTS,
-          title: "Attachments",
-        },
-      },
-      required: ["to", "from", "subject", "contents"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the email was sent",
-        },
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "A response from the email client, this may be an error",
-        },
-      },
-      required: ["success"],
-    },
-  },
-}
 
 export async function run({
   inputs,

packages/server/src/automations/steps/serverLog.ts

@@ -1,58 +1,4 @@
-import {
+import { ServerLogStepInputs, ServerLogStepOutputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
-  ServerLogStepInputs,
-  ServerLogStepOutputs,
-} from "@budibase/types"
-
-/**
- * Note, there is some functionality in this that is not currently exposed as it
- * is complex and maybe better to be opinionated here.
- * GET/DELETE requests cannot handle body elements so they will not be sent if configured.
- */
-
-export const definition: AutomationStepDefinition = {
-  name: "Backend log",
-  tagline: "Console log a value in the backend",
-  icon: "Monitoring",
-  description: "Logs the given text to the server (using console.log)",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  stepId: AutomationActionStepId.SERVER_LOG,
-  inputs: {
-    text: "",
-  },
-  schema: {
-    inputs: {
-      properties: {
-        text: {
-          type: AutomationIOType.STRING,
-          title: "Log",
-        },
-      },
-      required: ["text"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-        message: {
-          type: AutomationIOType.STRING,
-          description: "What was output",
-        },
-      },
-      required: ["success", "message"],
-    },
-  },
-}
 
 export async function run({
   inputs,

packages/server/src/automations/steps/slack.ts

@@ -1,59 +1,6 @@
 import fetch from "node-fetch"
 import { getFetchResponse } from "./utils"
-import {
+import { ExternalAppStepOutputs, SlackStepInputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
-  ExternalAppStepOutputs,
-  SlackStepInputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  name: "Slack Message",
-  tagline: "Send a message to Slack",
-  description: "Send a message to Slack",
-  icon: "ri-slack-line",
-  stepId: AutomationActionStepId.slack,
-  type: AutomationStepType.ACTION,
-  internal: false,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        url: {
-          type: AutomationIOType.STRING,
-          title: "Incoming Webhook URL",
-        },
-        text: {
-          type: AutomationIOType.STRING,
-          title: "Message",
-        },
-      },
-      required: ["url", "text"],
-    },
-    outputs: {
-      properties: {
-        httpStatus: {
-          type: AutomationIOType.NUMBER,
-          description: "The HTTP status code of the request",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the message sent successfully",
-        },
-        response: {
-          type: AutomationIOType.STRING,
-          description: "The response from the Slack Webhook",
-        },
-      },
-    },
-  },
-}
 
 export async function run({
   inputs,

packages/server/src/automations/steps/triggerAutomationRun.ts

@@ -1,10 +1,5 @@
 import {
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
   Automation,
-  AutomationCustomIOType,
   TriggerAutomationStepInputs,
   TriggerAutomationStepOutputs,
 } from "@budibase/types"
@@ -13,54 +8,6 @@ import { context } from "@budibase/backend-core"
 import { features } from "@budibase/pro"
 import env from "../../environment"
 
-export const definition: AutomationStepDefinition = {
-  name: "Trigger an automation",
-  tagline: "Triggers an automation synchronously",
-  icon: "Sync",
-  description: "Triggers an automation synchronously",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {},
-  stepId: AutomationActionStepId.TRIGGER_AUTOMATION_RUN,
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        automation: {
-          type: AutomationIOType.OBJECT,
-          properties: {
-            automationId: {
-              type: AutomationIOType.STRING,
-              customType: AutomationCustomIOType.AUTOMATION,
-            },
-          },
-          customType: AutomationCustomIOType.AUTOMATION_FIELDS,
-          title: "automatioFields",
-          required: ["automationId"],
-        },
-        timeout: {
-          type: AutomationIOType.NUMBER,
-          title: "Timeout (ms)",
-        },
-      },
-      required: ["automationId"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the automation was successful",
-        },
-        value: {
-          type: AutomationIOType.OBJECT,
-          description: "Automation Result",
-        },
-      },
-      required: ["success", "value"],
-    },
-  },
-}
-
 export async function run({
   inputs,
 }: {

packages/server/src/automations/steps/updateRow.ts

@@ -2,76 +2,8 @@ import { EventEmitter } from "events"
 import * as rowController from "../../api/controllers/row"
 import * as automationUtils from "../automationUtils"
 import { buildCtx } from "./utils"
-import {
+import { UpdateRowStepInputs, UpdateRowStepOutputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
-  UpdateRowStepInputs,
-  UpdateRowStepOutputs,
-} from "@budibase/types"
 
-export const definition: AutomationStepDefinition = {
-  name: "Update Row",
-  tagline: "Update a {{inputs.enriched.table.name}} row",
-  icon: "Refresh",
-  description: "Update a row in your database",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  stepId: AutomationActionStepId.UPDATE_ROW,
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        meta: {
-          type: AutomationIOType.OBJECT,
-          title: "Field settings",
-        },
-        row: {
-          type: AutomationIOType.OBJECT,
-          customType: AutomationCustomIOType.ROW,
-          title: "Table",
-        },
-        rowId: {
-          type: AutomationIOType.STRING,
-          title: "Row ID",
-        },
-      },
-      required: ["row", "rowId"],
-    },
-    outputs: {
-      properties: {
-        row: {
-          type: AutomationIOType.OBJECT,
-          customType: AutomationCustomIOType.ROW,
-          description: "The updated row",
-        },
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The response from the table",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-        id: {
-          type: AutomationIOType.STRING,
-          description: "The identifier of the updated row",
-        },
-        revision: {
-          type: AutomationIOType.STRING,
-          description: "The revision of the updated row",
-        },
-      },
-      required: ["success", "id", "revision"],
-    },
-  },
-}
 export async function run({
   inputs,
   appId,

packages/server/src/automations/steps/zapier.ts

@@ -1,55 +1,6 @@
 import fetch from "node-fetch"
 import { getFetchResponse } from "./utils"
-import {
+import { ZapierStepInputs, ZapierStepOutputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
-  ZapierStepInputs,
-  ZapierStepOutputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  name: "Zapier Webhook",
-  stepId: AutomationActionStepId.zapier,
-  type: AutomationStepType.ACTION,
-  internal: false,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  description: "Trigger a Zapier Zap via webhooks",
-  tagline: "Trigger a Zapier Zap",
-  icon: "ri-flashlight-line",
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        url: {
-          type: AutomationIOType.STRING,
-          title: "Webhook URL",
-        },
-        body: {
-          type: AutomationIOType.JSON,
-          title: "Payload",
-        },
-      },
-      required: ["url"],
-    },
-    outputs: {
-      properties: {
-        httpStatus: {
-          type: AutomationIOType.NUMBER,
-          description: "The HTTP status code of the request",
-        },
-        response: {
-          type: AutomationIOType.STRING,
-          description: "The response from Zapier",
-        },
-      },
-    },
-  },
-}
 
 export async function run({
   inputs,

packages/server/src/automations/tests/filter.spec.ts

@@ -1,5 +1,7 @@
 import * as setup from "./utilities"
-import { FilterConditions } from "../steps/filter"
+import { automations } from "@budibase/shared-core"
+
+const FilterConditions = automations.steps.filter.FilterConditions
 
 describe("test the filter logic", () => {
   const config = setup.getConfig()

packages/server/src/automations/tests/scenarios/scenarios.spec.ts

@@ -6,9 +6,11 @@ import {
   DatabaseName,
   datasourceDescribe,
 } from "../../../integrations/tests/utils"
-import { FilterConditions } from "../../../automations/steps/filter"
 import { Knex } from "knex"
 import { generator } from "@budibase/backend-core/tests"
+import { automations } from "@budibase/shared-core"
+
+const FilterConditions = automations.steps.filter.FilterConditions
 
 describe("Automation Scenarios", () => {
   let config = setup.getConfig()

packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts

@@ -42,7 +42,7 @@ import {
 } from "@budibase/types"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 import * as setup from "../utilities"
-import { definition } from "../../../automations/steps/branch"
+import { automations } from "@budibase/shared-core"
 
 type TriggerOutputs =
   | RowCreatedTriggerOutputs
@@ -103,7 +103,7 @@ class BaseStepBuilder {
       branchStepInputs.children![branchId] = stepBuilder.build()
     })
     const branchStep: AutomationStep = {
-      ...definition,
+      ...automations.steps.branch.definition,
       id: uuidv4(),
       stepId: AutomationActionStepId.BRANCH,
       inputs: branchStepInputs,

packages/server/src/environment.ts

@@ -54,7 +54,6 @@ const environment = {
   REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   REDIS_CLUSTERED: process.env.REDIS_CLUSTERED,
-  HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
   CLUSTER_MODE: process.env.CLUSTER_MODE,
   API_REQ_LIMIT_PER_SEC: process.env.API_REQ_LIMIT_PER_SEC,
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

packages/server/src/jsRunner/index.ts

@@ -4,12 +4,17 @@ import {
   JsTimeoutError,
   setJSRunner,
   setOnErrorLog,
+  setTestingBackendJS,
 } from "@budibase/string-templates"
 import { context, logging } from "@budibase/backend-core"
 import tracer from "dd-trace"
 import { IsolatedVM } from "./vm"
 
 export function init() {
+  // enforce that if we're using isolated-VM runner then we are running backend JS
+  if (env.isTest()) {
+    setTestingBackendJS()
+  }
   setJSRunner((js: string, ctx: Record<string, any>) => {
     return tracer.trace("runJS", {}, () => {
       try {

packages/server/src/migrations/functions/appUrls.ts

@@ -1,27 +0,0 @@
-import { db as dbCore } from "@budibase/backend-core"
-import sdk from "../../sdk"
-
-/**
- * Date:
- * January 2022
- *
- * Description:
- * Add the url to the app metadata if it doesn't exist
- */
-export const run = async (appDb: any) => {
-  let metadata
-  try {
-    metadata = await appDb.get(dbCore.DocumentType.APP_METADATA)
-  } catch (e) {
-    // sometimes the metadata document doesn't exist
-    // exit early instead of failing the migration
-    console.error("Error retrieving app metadata. Skipping", e)
-    return
-  }
-
-  if (!metadata.url) {
-    metadata.url = sdk.applications.getAppUrl({ name: metadata.name })
-    console.log(`Adding url to app: ${metadata.url}`)
-    await appDb.put(metadata)
-  }
-}

packages/server/src/migrations/functions/backfill/app.ts

@@ -1,149 +0,0 @@
-import * as automations from "./app/automations"
-import * as datasources from "./app/datasources"
-import * as layouts from "./app/layouts"
-import * as queries from "./app/queries"
-import * as roles from "./app/roles"
-import * as tables from "./app/tables"
-import * as screens from "./app/screens"
-import * as global from "./global"
-import { App, AppBackfillSucceededEvent, Event } from "@budibase/types"
-import { db as dbUtils, events } from "@budibase/backend-core"
-import env from "../../../environment"
-import { DEFAULT_TIMESTAMP } from "."
-
-const failGraceful = env.SELF_HOSTED && !env.isDev()
-
-const handleError = (e: any, errors?: any) => {
-  if (failGraceful) {
-    if (errors) {
-      errors.push(e)
-    }
-    return
-  }
-  console.trace(e)
-  throw e
-}
-
-const EVENTS = [
-  Event.AUTOMATION_CREATED,
-  Event.AUTOMATION_STEP_CREATED,
-  Event.DATASOURCE_CREATED,
-  Event.LAYOUT_CREATED,
-  Event.QUERY_CREATED,
-  Event.ROLE_CREATED,
-  Event.SCREEN_CREATED,
-  Event.TABLE_CREATED,
-  Event.VIEW_CREATED,
-  Event.VIEW_CALCULATION_CREATED,
-  Event.VIEW_FILTER_CREATED,
-  Event.APP_PUBLISHED,
-  Event.APP_CREATED,
-]
-
-/**
- * Date:
- * May 2022
- *
- * Description:
- * Backfill app events.
- */
-
-export const run = async (appDb: any) => {
-  try {
-    if (await global.isComplete()) {
-      // make sure new apps aren't backfilled
-      // return if the global migration for this tenant is complete
-      // which runs after the app migrations
-      return
-    }
-
-    // tell the event pipeline to start caching
-    // events for this tenant
-    await events.backfillCache.start(EVENTS)
-
-    let timestamp: string | number = DEFAULT_TIMESTAMP
-    const app: App = await appDb.get(dbUtils.DocumentType.APP_METADATA)
-    if (app.createdAt) {
-      timestamp = app.createdAt as string
-    }
-
-    if (dbUtils.isProdAppID(app.appId)) {
-      await events.app.published(app, timestamp)
-    }
-
-    const totals: any = {}
-    const errors: any = []
-
-    if (dbUtils.isDevAppID(app.appId)) {
-      await events.app.created(app, timestamp)
-      try {
-        totals.automations = await automations.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.datasources = await datasources.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.layouts = await layouts.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.queries = await queries.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.roles = await roles.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.screens = await screens.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.tables = await tables.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-    }
-
-    const properties: AppBackfillSucceededEvent = {
-      appId: app.appId,
-      automations: totals.automations,
-      datasources: totals.datasources,
-      layouts: totals.layouts,
-      queries: totals.queries,
-      roles: totals.roles,
-      tables: totals.tables,
-      screens: totals.screens,
-    }
-
-    if (errors.length) {
-      properties.errors = errors.map((e: any) =>
-        JSON.stringify(e, Object.getOwnPropertyNames(e))
-      )
-      properties.errorCount = errors.length
-    } else {
-      properties.errorCount = 0
-    }
-
-    await events.backfill.appSucceeded(properties)
-    // tell the event pipeline to stop caching events for this tenant
-    await events.backfillCache.end()
-  } catch (e) {
-    handleError(e)
-    await events.backfill.appFailed(e)
-  }
-}

packages/server/src/migrations/functions/backfill/app/automations.ts

@@ -1,26 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getAutomationParams } from "../../../../db/utils"
-import { Automation } from "@budibase/types"
-
-const getAutomations = async (appDb: any): Promise<Automation[]> => {
-  const response = await appDb.allDocs(
-    getAutomationParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
-  const automations = await getAutomations(appDb)
-
-  for (const automation of automations) {
-    await events.automation.created(automation, timestamp)
-
-    for (const step of automation.definition.steps) {
-      await events.automation.stepCreated(automation, step, timestamp)
-    }
-  }
-
-  return automations.length
-}

packages/server/src/migrations/functions/backfill/app/datasources.ts

@@ -1,22 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getDatasourceParams } from "../../../../db/utils"
-import { Datasource } from "@budibase/types"
-
-const getDatasources = async (appDb: any): Promise<Datasource[]> => {
-  const response = await appDb.allDocs(
-    getDatasourceParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
-  const datasources: Datasource[] = await getDatasources(appDb)
-
-  for (const datasource of datasources) {
-    await events.datasource.created(datasource, timestamp)
-  }
-
-  return datasources.length
-}

packages/server/src/migrations/functions/backfill/app/layouts.ts

@@ -1,29 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getLayoutParams } from "../../../../db/utils"
-import { Layout } from "@budibase/types"
-
-const getLayouts = async (appDb: any): Promise<Layout[]> => {
-  const response = await appDb.allDocs(
-    getLayoutParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
-  const layouts: Layout[] = await getLayouts(appDb)
-
-  for (const layout of layouts) {
-    // exclude default layouts
-    if (
-      layout._id === "layout_private_master" ||
-      layout._id === "layout_public_master"
-    ) {
-      continue
-    }
-    await events.layout.created(layout, timestamp)
-  }
-
-  return layouts.length
-}

packages/server/src/migrations/functions/backfill/app/queries.ts

@@ -1,47 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getQueryParams } from "../../../../db/utils"
-import { Query, Datasource, SourceName } from "@budibase/types"
-
-const getQueries = async (appDb: any): Promise<Query[]> => {
-  const response = await appDb.allDocs(
-    getQueryParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-const getDatasource = async (
-  appDb: any,
-  datasourceId: string
-): Promise<Datasource> => {
-  return appDb.get(datasourceId)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
-  const queries: Query[] = await getQueries(appDb)
-
-  for (const query of queries) {
-    let datasource: Datasource
-
-    try {
-      datasource = await getDatasource(appDb, query.datasourceId)
-    } catch (e: any) {
-      // handle known bug where a datasource has been deleted
-      // and the query has not
-      if (e.status === 404) {
-        datasource = {
-          type: "unknown",
-          _id: query.datasourceId,
-          source: "unknown" as SourceName,
-        }
-      } else {
-        throw e
-      }
-    }
-
-    await events.query.created(datasource, query, timestamp)
-  }
-
-  return queries.length
-}

packages/server/src/migrations/functions/backfill/app/roles.ts

@@ -1,22 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getRoleParams } from "../../../../db/utils"
-import { Role } from "@budibase/types"
-
-const getRoles = async (appDb: any): Promise<Role[]> => {
-  const response = await appDb.allDocs(
-    getRoleParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
-  const roles = await getRoles(appDb)
-
-  for (const role of roles) {
-    await events.role.created(role, timestamp)
-  }
-
-  return roles.length
-}

packages/server/src/migrations/functions/backfill/app/screens.ts

@@ -1,22 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getScreenParams } from "../../../../db/utils"
-import { Screen } from "@budibase/types"
-
-const getScreens = async (appDb: any): Promise<Screen[]> => {
-  const response = await appDb.allDocs(
-    getScreenParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
-  const screens = await getScreens(appDb)
-
-  for (const screen of screens) {
-    await events.screen.created(screen, timestamp)
-  }
-
-  return screens.length
-}

packages/server/src/migrations/functions/backfill/app/tables.ts

@@ -1,13 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { Database } from "@budibase/types"
-import sdk from "../../../../sdk"
-
-export const backfill = async (appDb: Database, timestamp: string | number) => {
-  const tables = await sdk.tables.getAllInternalTables(appDb)
-
-  for (const table of tables) {
-    await events.table.created(table, timestamp)
-  }
-
-  return tables.length
-}

packages/server/src/migrations/functions/backfill/global.ts

@@ -1,214 +0,0 @@
-import * as users from "./global/users"
-import * as configs from "./global/configs"
-import * as quotas from "./global/quotas"
-import {
-  tenancy,
-  events,
-  migrations,
-  accounts,
-  db as dbUtils,
-} from "@budibase/backend-core"
-import {
-  App,
-  CloudAccount,
-  Event,
-  Hosting,
-  QuotaUsage,
-  TenantBackfillSucceededEvent,
-  User,
-} from "@budibase/types"
-import env from "../../../environment"
-import { DEFAULT_TIMESTAMP } from "."
-
-const failGraceful = env.SELF_HOSTED && !env.isDev()
-
-const handleError = (e: any, errors?: any) => {
-  if (failGraceful) {
-    if (errors) {
-      errors.push(e)
-    }
-    return
-  }
-  throw e
-}
-
-const formatUsage = (usage: QuotaUsage) => {
-  let maxAutomations = 0
-  let maxQueries = 0
-  let rows = 0
-
-  if (usage) {
-    if (usage.usageQuota) {
-      rows = usage.usageQuota.rows
-    }
-
-    if (usage.monthly) {
-      for (const value of Object.values(usage.monthly)) {
-        if (value.automations > maxAutomations) {
-          maxAutomations = value.automations
-        }
-        if (value.queries > maxQueries) {
-          maxQueries = value.queries
-        }
-      }
-    }
-  }
-
-  return {
-    maxAutomations,
-    maxQueries,
-    rows,
-  }
-}
-
-const EVENTS = [
-  Event.EMAIL_SMTP_CREATED,
-  Event.AUTH_SSO_CREATED,
-  Event.AUTH_SSO_ACTIVATED,
-  Event.ORG_NAME_UPDATED,
-  Event.ORG_LOGO_UPDATED,
-  Event.ORG_PLATFORM_URL_UPDATED,
-  Event.USER_CREATED,
-  Event.USER_PERMISSION_ADMIN_ASSIGNED,
-  Event.USER_PERMISSION_BUILDER_ASSIGNED,
-  Event.ROLE_ASSIGNED,
-  Event.ROWS_CREATED,
-  Event.QUERIES_RUN,
-  Event.AUTOMATIONS_RUN,
-]
-
-/**
- * Date:
- * May 2022
- *
- * Description:
- * Backfill global events.
- */
-
-export const run = async (db: any) => {
-  try {
-    const tenantId = tenancy.getTenantId()
-    let timestamp: string | number = DEFAULT_TIMESTAMP
-
-    const totals: any = {}
-    const errors: any = []
-
-    let allUsers: User[] = []
-    try {
-      allUsers = await users.getUsers(db)
-    } catch (e: any) {
-      handleError(e, errors)
-    }
-
-    if (!allUsers || allUsers.length === 0) {
-      // first time startup - we don't need to backfill anything
-      // tenant will be identified when admin user is created
-      if (env.SELF_HOSTED) {
-        await events.installation.firstStartup()
-      }
-      return
-    }
-
-    try {
-      const installTimestamp = await getInstallTimestamp(db, allUsers)
-      if (installTimestamp) {
-        timestamp = installTimestamp
-      }
-    } catch (e) {
-      handleError(e, errors)
-    }
-
-    let account: CloudAccount | undefined
-    if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
-      account = await accounts.getAccountByTenantId(tenantId)
-    }
-
-    try {
-      await events.identification.identifyTenantGroup(
-        tenantId,
-        env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD,
-        timestamp
-      )
-    } catch (e) {
-      handleError(e, errors)
-    }
-
-    // tell the event pipeline to start caching
-    // events for this tenant
-    await events.backfillCache.start(EVENTS)
-
-    try {
-      await configs.backfill(db, timestamp)
-    } catch (e) {
-      handleError(e, errors)
-    }
-
-    try {
-      totals.users = await users.backfill(db, account)
-    } catch (e) {
-      handleError(e, errors)
-    }
-
-    try {
-      const allApps = (await dbUtils.getAllApps({ dev: true })) as App[]
-      totals.apps = allApps.length
-
-      totals.usage = await quotas.backfill(allApps)
-    } catch (e) {
-      handleError(e, errors)
-    }
-
-    const properties: TenantBackfillSucceededEvent = {
-      apps: totals.apps,
-      users: totals.users,
-      ...formatUsage(totals.usage),
-      usage: totals.usage,
-    }
-
-    if (errors.length) {
-      properties.errors = errors.map((e: any) =>
-        JSON.stringify(e, Object.getOwnPropertyNames(e))
-      )
-      properties.errorCount = errors.length
-    } else {
-      properties.errorCount = 0
-    }
-
-    await events.backfill.tenantSucceeded(properties)
-    // tell the event pipeline to stop caching events for this tenant
-    await events.backfillCache.end()
-  } catch (e) {
-    handleError(e)
-    await events.backfill.tenantFailed(e)
-  }
-}
-
-export const isComplete = async (): Promise<boolean> => {
-  const globalDb = tenancy.getGlobalDB()
-  const migrationsDoc = await migrations.getMigrationsDoc(globalDb)
-  return !!migrationsDoc.event_global_backfill
-}
-
-export const getInstallTimestamp = async (
-  globalDb: any,
-  allUsers?: User[]
-): Promise<number | undefined> => {
-  if (!allUsers) {
-    allUsers = await users.getUsers(globalDb)
-  }
-
-  // get the oldest user timestamp
-  if (allUsers) {
-    const timestamps = allUsers
-      .map(user => user.createdAt)
-      .filter(timestamp => !!timestamp)
-      .sort(
-        (a, b) =>
-          new Date(a as number).getTime() - new Date(b as number).getTime()
-      )
-
-    if (timestamps.length) {
-      return timestamps[0]
-    }
-  }
-}

packages/server/src/migrations/functions/backfill/global/configs.ts

@@ -1,74 +0,0 @@
-import {
-  events,
-  DocumentType,
-  SEPARATOR,
-  UNICODE_MAX,
-} from "@budibase/backend-core"
-import {
-  Config,
-  isSMTPConfig,
-  isGoogleConfig,
-  isOIDCConfig,
-  isSettingsConfig,
-  ConfigType,
-  DatabaseQueryOpts,
-} from "@budibase/types"
-import env from "./../../../../environment"
-
-export function getConfigParams(): DatabaseQueryOpts {
-  return {
-    include_docs: true,
-    startkey: `${DocumentType.CONFIG}${SEPARATOR}`,
-    endkey: `${DocumentType.CONFIG}${SEPARATOR}${UNICODE_MAX}`,
-  }
-}
-
-const getConfigs = async (globalDb: any): Promise<Config[]> => {
-  const response = await globalDb.allDocs(getConfigParams())
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (
-  globalDb: any,
-  timestamp: string | number | undefined
-) => {
-  const configs = await getConfigs(globalDb)
-
-  for (const config of configs) {
-    if (isSMTPConfig(config)) {
-      await events.email.SMTPCreated(timestamp)
-    }
-    if (isGoogleConfig(config)) {
-      await events.auth.SSOCreated(ConfigType.GOOGLE, timestamp)
-      if (config.config.activated) {
-        await events.auth.SSOActivated(ConfigType.GOOGLE, timestamp)
-      }
-    }
-    if (isOIDCConfig(config)) {
-      await events.auth.SSOCreated(ConfigType.OIDC, timestamp)
-      if (config.config.configs[0].activated) {
-        await events.auth.SSOActivated(ConfigType.OIDC, timestamp)
-      }
-    }
-    if (isSettingsConfig(config)) {
-      const company = config.config.company
-      if (company && company !== "Budibase") {
-        await events.org.nameUpdated(timestamp)
-      }
-
-      const logoUrl = config.config.logoUrl
-      if (logoUrl) {
-        await events.org.logoUpdated(timestamp)
-      }
-
-      const platformUrl = config.config.platformUrl
-      if (
-        platformUrl &&
-        platformUrl !== "http://localhost:10000" &&
-        env.SELF_HOSTED
-      ) {
-        await events.org.platformURLUpdated(timestamp)
-      }
-    }
-  }
-}

packages/server/src/migrations/functions/backfill/global/quotas.ts

@@ -1,60 +0,0 @@
-import { DEFAULT_TIMESTAMP } from "./../index"
-import { events } from "@budibase/backend-core"
-import { quotas } from "@budibase/pro"
-import { App } from "@budibase/types"
-
-const getOldestCreatedAt = (allApps: App[]): string | undefined => {
-  const timestamps = allApps
-    .filter(app => !!app.createdAt)
-    .map(app => app.createdAt as string)
-    .sort((a, b) => new Date(a).getTime() - new Date(b).getTime())
-
-  if (timestamps.length) {
-    return timestamps[0]
-  }
-}
-
-const getMonthTimestamp = (monthString: string): number => {
-  const parts = monthString.split("-")
-  const month = parseInt(parts[0]) - 1 // we already do +1 in month string calculation
-  const year = parseInt(parts[1])
-
-  // using 0 as the day in next month gives us last day in previous month
-  const date = new Date(year, month + 1, 0).getTime()
-  const now = new Date().getTime()
-
-  if (date > now) {
-    return now
-  } else {
-    return date
-  }
-}
-
-export const backfill = async (allApps: App[]) => {
-  const usage = await quotas.getQuotaUsage()
-
-  const rows = usage.usageQuota.rows
-  let timestamp: string | number = DEFAULT_TIMESTAMP
-
-  const oldestAppTimestamp = getOldestCreatedAt(allApps)
-  if (oldestAppTimestamp) {
-    timestamp = oldestAppTimestamp
-  }
-
-  await events.rows.created(rows, timestamp)
-
-  for (const [monthString, quotas] of Object.entries(usage.monthly)) {
-    if (monthString === "current") {
-      continue
-    }
-    const monthTimestamp = getMonthTimestamp(monthString)
-
-    const queries = quotas.queries
-    await events.query.run(queries, monthTimestamp)
-
-    const automations = quotas.automations
-    await events.automation.run(automations, monthTimestamp)
-  }
-
-  return usage
-}

packages/server/src/migrations/functions/backfill/global/users.ts

@@ -1,53 +0,0 @@
-import {
-  events,
-  db as dbUtils,
-  users as usersCore,
-} from "@budibase/backend-core"
-import { User, CloudAccount } from "@budibase/types"
-import { DEFAULT_TIMESTAMP } from ".."
-
-// manually define user doc params - normally server doesn't read users from the db
-const getUserParams = (props: any) => {
-  return dbUtils.getDocParams(dbUtils.DocumentType.USER, null, props)
-}
-
-export const getUsers = async (globalDb: any): Promise<User[]> => {
-  const response = await globalDb.allDocs(
-    getUserParams({
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (
-  globalDb: any,
-  account: CloudAccount | undefined
-) => {
-  const users = await getUsers(globalDb)
-
-  for (const user of users) {
-    let timestamp: string | number = DEFAULT_TIMESTAMP
-    if (user.createdAt) {
-      timestamp = user.createdAt
-    }
-    await events.identification.identifyUser(user, account, timestamp)
-    await events.user.created(user, timestamp)
-
-    if (usersCore.hasAdminPermissions(user)) {
-      await events.user.permissionAdminAssigned(user, timestamp)
-    }
-
-    if (usersCore.hasBuilderPermissions(user)) {
-      await events.user.permissionBuilderAssigned(user, timestamp)
-    }
-
-    if (user.roles) {
-      for (const [, role] of Object.entries(user.roles)) {
-        await events.role.assigned(user, role, timestamp)
-      }
-    }
-  }
-
-  return users.length
-}

packages/server/src/migrations/functions/backfill/index.ts

@@ -1,7 +0,0 @@
-export * as app from "./app"
-export * as global from "./global"
-export * as installation from "./installation"
-
-// historical events are free in posthog - make sure we default to a
-// historical time if no other can be found
-export const DEFAULT_TIMESTAMP = new Date(2022, 0, 1).getTime()

packages/server/src/migrations/functions/backfill/installation.ts

@@ -1,50 +0,0 @@
-import { DEFAULT_TIMESTAMP } from "./index"
-import { events, tenancy, installation } from "@budibase/backend-core"
-import { Installation } from "@budibase/types"
-import * as global from "./global"
-import env from "../../../environment"
-
-const failGraceful = env.SELF_HOSTED
-
-const handleError = (e: any, errors?: any) => {
-  if (failGraceful) {
-    if (errors) {
-      errors.push(e)
-    }
-    return
-  }
-  throw e
-}
-
-/**
- * Date:
- * May 2022
- *
- * Description:
- * Backfill installation events.
- */
-
-export const run = async () => {
-  try {
-    // need to use the default tenant to try to get the installation time
-    await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => {
-      const db = tenancy.getGlobalDB()
-      let timestamp: string | number = DEFAULT_TIMESTAMP
-
-      const installTimestamp = await global.getInstallTimestamp(db)
-      if (installTimestamp) {
-        timestamp = installTimestamp
-      }
-
-      const install: Installation = await installation.getInstall()
-      await events.identification.identifyInstallationGroup(
-        install.installId,
-        timestamp
-      )
-    })
-    await events.backfill.installationSucceeded()
-  } catch (e) {
-    handleError(e)
-    await events.backfill.installationFailed(e)
-  }
-}

packages/server/src/migrations/functions/syncQuotas.ts

@@ -1,19 +0,0 @@
-import { runQuotaMigration } from "./usageQuotas"
-import * as syncApps from "./usageQuotas/syncApps"
-import * as syncRows from "./usageQuotas/syncRows"
-import * as syncPlugins from "./usageQuotas/syncPlugins"
-import * as syncUsers from "./usageQuotas/syncUsers"
-import * as syncCreators from "./usageQuotas/syncCreators"
-
-/**
- * Synchronise quotas to the state of the db.
- */
-export const run = async () => {
-  await runQuotaMigration(async () => {
-    await syncApps.run()
-    await syncRows.run()
-    await syncPlugins.run()
-    await syncUsers.run()
-    await syncCreators.run()
-  })
-}

packages/server/src/migrations/functions/tableSettings.ts

@@ -1,145 +0,0 @@
-import { getScreenParams } from "../../db/utils"
-import { Screen } from "@budibase/types"
-import { makePropSafe as safe } from "@budibase/string-templates"
-/**
- * Date:
- * November 2022
- *
- * Description:
- * Update table settings to use actions instead of links. We do not remove the
- * legacy values here as we cannot guarantee that their apps are up-t-date.
- * It is safe to simply save both the new and old structure in the definition.
- *
- * Migration 1:
- * Legacy "linkRows", "linkURL", "linkPeek" and "linkColumn" settings on tables
- * and table blocks are migrated into a "Navigate To" action under the new
- * "onClick" setting.
- *
- * Migration 2:
- * Legacy "titleButtonURL" and "titleButtonPeek" settings on table blocks are
- * migrated into a "Navigate To" action under the new "onClickTitleButton"
- * setting.
- */
-export const run = async (appDb: any) => {
-  // Get all app screens
-  let screens: Screen[]
-  try {
-    screens = (
-      await appDb.allDocs(
-        getScreenParams(null, {
-          include_docs: true,
-        })
-      )
-    ).rows.map((row: any) => row.doc)
-  } catch (e) {
-    // sometimes the metadata document doesn't exist
-    // exit early instead of failing the migration
-    console.error("Error retrieving app metadata. Skipping", e)
-    return
-  }
-
-  // Recursively update any relevant components and mutate the screen docs
-  for (let screen of screens) {
-    const changed = migrateTableSettings(screen.props)
-
-    // Save screen if we updated it
-    if (changed) {
-      await appDb.put(screen)
-      console.log(
-        `Screen ${screen.routing?.route} contained table settings which were migrated`
-      )
-    }
-  }
-}
-
-// Recursively searches and mutates a screen doc to migrate table component
-// and table block settings
-const migrateTableSettings = (component: any) => {
-  let changed = false
-  if (!component) {
-    return changed
-  }
-
-  // Migration 1: migrate table row click settings
-  if (
-    component._component.endsWith("/table") ||
-    component._component.endsWith("/tableblock")
-  ) {
-    const { linkRows, linkURL, linkPeek, linkColumn, onClick } = component
-    if (linkRows && !onClick) {
-      const column = linkColumn || "_id"
-      const action = convertLinkSettingToAction(linkURL, !!linkPeek, column)
-      if (action) {
-        changed = true
-        component.onClick = action
-        if (component._component.endsWith("/tableblock")) {
-          component.clickBehaviour = "actions"
-        }
-      }
-    }
-  }
-
-  // Migration 2: migrate table block title button settings
-  if (component._component.endsWith("/tableblock")) {
-    const {
-      showTitleButton,
-      titleButtonURL,
-      titleButtonPeek,
-      onClickTitleButton,
-    } = component
-    if (showTitleButton && !onClickTitleButton) {
-      const action = convertLinkSettingToAction(
-        titleButtonURL,
-        !!titleButtonPeek
-      )
-      if (action) {
-        changed = true
-        component.onClickTitleButton = action
-        component.titleButtonClickBehaviour = "actions"
-      }
-    }
-  }
-
-  // Recurse down the tree as needed
-  component._children?.forEach((child: any) => {
-    const childChanged = migrateTableSettings(child)
-    changed = changed || childChanged
-  })
-  return changed
-}
-
-// Util ti convert the legacy settings into a navigation action structure
-const convertLinkSettingToAction = (
-  linkURL: string,
-  linkPeek: boolean,
-  linkColumn?: string
-) => {
-  // Sanity check we have a URL
-  if (!linkURL) {
-    return null
-  }
-
-  // Default URL to the old URL setting
-  let url = linkURL
-
-  // If we enriched the old URL with a column, update the url
-  if (linkColumn && linkURL.includes("/:")) {
-    // Convert old link URL setting, which is a screen URL, into a valid
-    // binding using the new clicked row binding
-    const split = linkURL.split("/:")
-    const col = linkColumn || "_id"
-    const binding = `{{ ${safe("eventContext")}.${safe("row")}.${safe(col)} }}`
-    url = `${split[0]}/${binding}`
-  }
-
-  // Create action structure
-  return [
-    {
-      "##eventHandlerType": "Navigate To",
-      parameters: {
-        url,
-        peek: linkPeek,
-      },
-    },
-  ]
-}

packages/server/src/migrations/functions/tests/appUrls.spec.js

@@ -1,26 +0,0 @@
-const { db: dbCore } = require("@budibase/backend-core")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
-
-const migration = require("../appUrls")
-
-describe("run", () => {
-  let config = new TestConfig(false)
-
-  beforeAll(async () => {
-    await config.init()
-  })
-
-  afterAll(config.end)
-
-  it("runs successfully", async () => {
-    const app = await config.createApp("testApp")
-    const metadata = await dbCore.doWithDB(app.appId, async db => {
-      const metadataDoc = await db.get(dbCore.DocumentType.APP_METADATA)
-      delete metadataDoc.url
-      await db.put(metadataDoc)
-      await migration.run(db)
-      return await db.get(dbCore.DocumentType.APP_METADATA)
-    })
-    expect(metadata.url).toEqual("/testapp")
-  })
-})

packages/server/src/migrations/functions/tests/tableSettings.spec.ts

@@ -1,144 +0,0 @@
-import { App, Screen } from "@budibase/types"
-
-import { db as dbCore } from "@budibase/backend-core"
-import TestConfig from "../../../tests/utilities/TestConfiguration"
-import { run as runMigration } from "../tableSettings"
-
-describe("run", () => {
-  const config = new TestConfig(false)
-  let app: App
-  let screen: Screen
-
-  beforeAll(async () => {
-    await config.init()
-    app = await config.createApp("testApp")
-    screen = await config.createScreen()
-  })
-
-  afterAll(config.end)
-
-  it("migrates table block row on click settings", async () => {
-    // Add legacy table block as first child
-    screen.props._children = [
-      {
-        _instanceName: "Table Block",
-        _styles: {},
-        _component: "@budibase/standard-components/tableblock",
-        _id: "foo",
-        linkRows: true,
-        linkURL: "/rows/:id",
-        linkPeek: true,
-        linkColumn: "name",
-      },
-    ]
-    await config.createScreen(screen)
-
-    // Run migration
-    screen = await dbCore.doWithDB(app.appId, async (db: any) => {
-      await runMigration(db)
-      return await db.get(screen._id)
-    })
-
-    // Verify new "onClick" setting
-    const onClick = screen.props._children?.[0].onClick
-    expect(onClick).toBeDefined()
-    expect(onClick.length).toBe(1)
-    expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To")
-    expect(onClick[0].parameters.url).toBe(
-      `/rows/{{ [eventContext].[row].[name] }}`
-    )
-    expect(onClick[0].parameters.peek).toBeTruthy()
-  })
-
-  it("migrates table row on click settings", async () => {
-    // Add legacy table block as first child
-    screen.props._children = [
-      {
-        _instanceName: "Table",
-        _styles: {},
-        _component: "@budibase/standard-components/table",
-        _id: "foo",
-        linkRows: true,
-        linkURL: "/rows/:id",
-        linkPeek: true,
-        linkColumn: "name",
-      },
-    ]
-    await config.createScreen(screen)
-
-    // Run migration
-    screen = await dbCore.doWithDB(app.appId, async (db: any) => {
-      await runMigration(db)
-      return await db.get(screen._id)
-    })
-
-    // Verify new "onClick" setting
-    const onClick = screen.props._children?.[0].onClick
-    expect(onClick).toBeDefined()
-    expect(onClick.length).toBe(1)
-    expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To")
-    expect(onClick[0].parameters.url).toBe(
-      `/rows/{{ [eventContext].[row].[name] }}`
-    )
-    expect(onClick[0].parameters.peek).toBeTruthy()
-  })
-
-  it("migrates table block title button settings", async () => {
-    // Add legacy table block as first child
-    screen.props._children = [
-      {
-        _instanceName: "Table Block",
-        _styles: {},
-        _component: "@budibase/standard-components/tableblock",
-        _id: "foo",
-        showTitleButton: true,
-        titleButtonURL: "/url",
-        titleButtonPeek: true,
-      },
-    ]
-    await config.createScreen(screen)
-
-    // Run migration
-    screen = await dbCore.doWithDB(app.appId, async (db: any) => {
-      await runMigration(db)
-      return await db.get(screen._id)
-    })
-
-    // Verify new "onClickTitleButton" setting
-    const onClick = screen.props._children?.[0].onClickTitleButton
-    expect(onClick).toBeDefined()
-    expect(onClick.length).toBe(1)
-    expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To")
-    expect(onClick[0].parameters.url).toBe("/url")
-    expect(onClick[0].parameters.peek).toBeTruthy()
-  })
-
-  it("ignores components that have already been migrated", async () => {
-    // Add legacy table block as first child
-    screen.props._children = [
-      {
-        _instanceName: "Table Block",
-        _styles: {},
-        _component: "@budibase/standard-components/tableblock",
-        _id: "foo",
-        linkRows: true,
-        linkURL: "/rows/:id",
-        linkPeek: true,
-        linkColumn: "name",
-        onClick: "foo",
-      },
-    ]
-    const initialDefinition = JSON.stringify(screen.props._children?.[0])
-    await config.createScreen(screen)
-
-    // Run migration
-    screen = await dbCore.doWithDB(app.appId, async (db: any) => {
-      await runMigration(db)
-      return await db.get(screen._id)
-    })
-
-    // Verify new "onClick" setting
-    const newDefinition = JSON.stringify(screen.props._children?.[0])
-    expect(initialDefinition).toEqual(newDefinition)
-  })
-})

packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js

@@ -1,34 +0,0 @@
-jest.mock("@budibase/backend-core", () => {
-  const core = jest.requireActual("@budibase/backend-core")
-  return {
-    ...core,
-    db: {
-      ...core.db,
-      createNewUserEmailView: jest.fn(),
-    },
-  }
-})
-const { context, db: dbCore } = require("@budibase/backend-core")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
-
-// mock email view creation
-
-const migration = require("../userEmailViewCasing")
-
-describe("run", () => {
-  let config = new TestConfig(false)
-
-  beforeAll(async () => {
-    await config.init()
-  })
-
-  afterAll(config.end)
-
-  it("runs successfully", async () => {
-    await config.doInTenant(async () => {
-      const globalDb = context.getGlobalDB()
-      await migration.run(globalDb)
-      expect(dbCore.createNewUserEmailView).toHaveBeenCalledTimes(1)
-    })
-  })
-})

packages/server/src/migrations/functions/usageQuotas/index.ts

@@ -1,3 +0,0 @@
-export const runQuotaMigration = async (migration: () => Promise<void>) => {
-  await migration()
-}

packages/server/src/migrations/functions/usageQuotas/syncApps.ts

@@ -1,13 +0,0 @@
-import { db as dbCore } from "@budibase/backend-core"
-import { quotas } from "@budibase/pro"
-import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-
-export const run = async () => {
-  // get app count
-  const devApps = await dbCore.getAllApps({ dev: true })
-  const appCount = devApps ? devApps.length : 0
-
-  // sync app count
-  console.log(`Syncing app count: ${appCount}`)
-  await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
-}

packages/server/src/migrations/functions/usageQuotas/syncCreators.ts

@@ -1,13 +0,0 @@
-import { users } from "@budibase/backend-core"
-import { quotas } from "@budibase/pro"
-import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-
-export const run = async () => {
-  const creatorCount = await users.getCreatorCount()
-  console.log(`Syncing creator count: ${creatorCount}`)
-  await quotas.setUsage(
-    creatorCount,
-    StaticQuotaName.CREATORS,
-    QuotaUsageType.STATIC
-  )
-}

packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts

@@ -1,10 +0,0 @@
-import { logging } from "@budibase/backend-core"
-import { plugins } from "@budibase/pro"
-
-export const run = async () => {
-  try {
-    await plugins.checkPluginQuotas()
-  } catch (err) {
-    logging.logAlert("Failed to update plugin quotas", err)
-  }
-}

packages/server/src/migrations/functions/usageQuotas/syncRows.ts

@@ -1,27 +0,0 @@
-import { db as dbCore } from "@budibase/backend-core"
-import { getUniqueRows } from "../../../utilities/usageQuota/rows"
-import { quotas } from "@budibase/pro"
-import { StaticQuotaName, QuotaUsageType, App } from "@budibase/types"
-
-export const run = async () => {
-  // get all rows in all apps
-  const allApps = (await dbCore.getAllApps({ all: true })) as App[]
-  const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : []
-  const { appRows } = await getUniqueRows(appIds)
-
-  // get the counts per app
-  const counts: { [key: string]: number } = {}
-  let rowCount = 0
-  Object.entries(appRows).forEach(([appId, rows]) => {
-    counts[appId] = rows.length
-    rowCount += rows.length
-  })
-
-  // sync row count
-  console.log(`Syncing row count: ${rowCount}`)
-  await quotas.setUsagePerApp(
-    counts,
-    StaticQuotaName.ROWS,
-    QuotaUsageType.STATIC
-  )
-}

packages/server/src/migrations/functions/usageQuotas/syncUsers.ts

@@ -1,9 +0,0 @@
-import { users } from "@budibase/backend-core"
-import { quotas } from "@budibase/pro"
-import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-
-export const run = async () => {
-  const userCount = await users.getUserCount()
-  console.log(`Syncing user count: ${userCount}`)
-  await quotas.setUsage(userCount, StaticQuotaName.USERS, QuotaUsageType.STATIC)
-}

packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts

@@ -1,35 +0,0 @@
-import TestConfig from "../../../../tests/utilities/TestConfiguration"
-import * as syncApps from "../syncApps"
-import { quotas } from "@budibase/pro"
-import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-
-describe("syncApps", () => {
-  let config = new TestConfig(false)
-
-  beforeEach(async () => {
-    await config.init()
-  })
-
-  afterAll(config.end)
-
-  it("runs successfully", async () => {
-    return config.doInContext(undefined, async () => {
-      // create the usage quota doc and mock usages
-      await quotas.getQuotaUsage()
-      await quotas.setUsage(3, StaticQuotaName.APPS, QuotaUsageType.STATIC)
-
-      let usageDoc = await quotas.getQuotaUsage()
-      expect(usageDoc.usageQuota.apps).toEqual(3)
-
-      // create an extra app to test the migration
-      await config.createApp("quota-test")
-
-      // migrate
-      await syncApps.run()
-
-      // assert the migration worked
-      usageDoc = await quotas.getQuotaUsage()
-      expect(usageDoc.usageQuota.apps).toEqual(2)
-    })
-  })
-})

packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts

@@ -1,26 +0,0 @@
-import TestConfig from "../../../../tests/utilities/TestConfiguration"
-import * as syncCreators from "../syncCreators"
-import { quotas } from "@budibase/pro"
-
-describe("syncCreators", () => {
-  let config = new TestConfig(false)
-
-  beforeEach(async () => {
-    await config.init()
-  })
-
-  afterAll(config.end)
-
-  it("syncs creators", async () => {
-    return config.doInContext(undefined, async () => {
-      await config.createUser({ admin: { global: true } })
-
-      await syncCreators.run()
-
-      const usageDoc = await quotas.getQuotaUsage()
-      // default + additional creator
-      const creatorsCount = 2
-      expect(usageDoc.usageQuota.creators).toBe(creatorsCount)
-    })
-  })
-})

packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -1,53 +0,0 @@
-import TestConfig from "../../../../tests/utilities/TestConfiguration"
-import * as syncRows from "../syncRows"
-import { quotas } from "@budibase/pro"
-import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-import { db as dbCore, context } from "@budibase/backend-core"
-
-describe("syncRows", () => {
-  const config = new TestConfig()
-
-  beforeEach(async () => {
-    await config.init()
-  })
-
-  afterAll(config.end)
-
-  it("runs successfully", async () => {
-    return config.doInContext(undefined, async () => {
-      // create the usage quota doc and mock usages
-      await quotas.getQuotaUsage()
-      await quotas.setUsage(300, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
-
-      let usageDoc = await quotas.getQuotaUsage()
-      expect(usageDoc.usageQuota.rows).toEqual(300)
-
-      // app 1
-      const app1 = config.app
-      await context.doInAppContext(app1!.appId, async () => {
-        await config.createTable()
-        await config.createRow()
-      })
-      // app 2
-      const app2 = await config.createApp("second-app")
-      await context.doInAppContext(app2.appId, async () => {
-        await config.createTable()
-        await config.createRow()
-        await config.createRow()
-      })
-
-      // migrate
-      await syncRows.run()
-
-      // assert the migration worked
-      usageDoc = await quotas.getQuotaUsage()
-      expect(usageDoc.usageQuota.rows).toEqual(3)
-      expect(
-        usageDoc.apps?.[dbCore.getProdAppID(app1!.appId)].usageQuota.rows
-      ).toEqual(1)
-      expect(
-        usageDoc.apps?.[dbCore.getProdAppID(app2.appId)].usageQuota.rows
-      ).toEqual(2)
-    })
-  })
-})

packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts

@@ -1,26 +0,0 @@
-import TestConfig from "../../../../tests/utilities/TestConfiguration"
-import * as syncUsers from "../syncUsers"
-import { quotas } from "@budibase/pro"
-
-describe("syncUsers", () => {
-  let config = new TestConfig(false)
-
-  beforeEach(async () => {
-    await config.init()
-  })
-
-  afterAll(config.end)
-
-  it("syncs users", async () => {
-    return config.doInContext(undefined, async () => {
-      await config.createUser()
-
-      await syncUsers.run()
-
-      const usageDoc = await quotas.getQuotaUsage()
-      // default + additional user
-      const userCount = 2
-      expect(usageDoc.usageQuota.users).toBe(userCount)
-    })
-  })
-})

packages/server/src/migrations/functions/userEmailViewCasing.ts

@@ -1,13 +0,0 @@
-import { db as dbCore } from "@budibase/backend-core"
-
-/**
- * Date:
- * October 2021
- *
- * Description:
- * Recreate the user email view to include latest changes i.e. lower casing the email address
- */
-
-export const run = async () => {
-  await dbCore.createNewUserEmailView()
-}

packages/server/src/migrations/index.ts

@@ -1,115 +0,0 @@
-import { locks, migrations } from "@budibase/backend-core"
-import {
-  Migration,
-  MigrationOptions,
-  MigrationName,
-  LockType,
-  LockName,
-} from "@budibase/types"
-import env from "../environment"
-
-// migration functions
-import * as userEmailViewCasing from "./functions/userEmailViewCasing"
-import * as syncQuotas from "./functions/syncQuotas"
-import * as appUrls from "./functions/appUrls"
-import * as tableSettings from "./functions/tableSettings"
-import * as backfill from "./functions/backfill"
-/**
- * Populate the migration function and additional configuration from
- * the static migration definitions.
- */
-export const buildMigrations = () => {
-  const definitions = migrations.DEFINITIONS
-  const serverMigrations: Migration[] = []
-
-  for (const definition of definitions) {
-    switch (definition.name) {
-      case MigrationName.USER_EMAIL_VIEW_CASING: {
-        serverMigrations.push({
-          ...definition,
-          fn: userEmailViewCasing.run,
-        })
-        break
-      }
-      case MigrationName.SYNC_QUOTAS: {
-        serverMigrations.push({
-          ...definition,
-          fn: syncQuotas.run,
-        })
-        break
-      }
-      case MigrationName.APP_URLS: {
-        serverMigrations.push({
-          ...definition,
-          appOpts: { all: true },
-          fn: appUrls.run,
-        })
-        break
-      }
-      case MigrationName.EVENT_APP_BACKFILL: {
-        serverMigrations.push({
-          ...definition,
-          appOpts: { all: true },
-          fn: backfill.app.run,
-          silent: !!env.SELF_HOSTED, // reduce noisy logging
-          preventRetry: !!env.SELF_HOSTED, // only ever run once
-        })
-        break
-      }
-      case MigrationName.EVENT_GLOBAL_BACKFILL: {
-        serverMigrations.push({
-          ...definition,
-          fn: backfill.global.run,
-          silent: !!env.SELF_HOSTED, // reduce noisy logging
-          preventRetry: !!env.SELF_HOSTED, // only ever run once
-        })
-        break
-      }
-      case MigrationName.EVENT_INSTALLATION_BACKFILL: {
-        serverMigrations.push({
-          ...definition,
-          fn: backfill.installation.run,
-          silent: !!env.SELF_HOSTED, // reduce noisy logging
-          preventRetry: !!env.SELF_HOSTED, // only ever run once
-        })
-        break
-      }
-      case MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS: {
-        serverMigrations.push({
-          ...definition,
-          appOpts: { dev: true },
-          fn: tableSettings.run,
-        })
-        break
-      }
-    }
-  }
-
-  return serverMigrations
-}
-
-export const MIGRATIONS = buildMigrations()
-
-export const migrate = async (options?: MigrationOptions) => {
-  if (env.SELF_HOSTED) {
-    // self host runs migrations on startup
-    // make sure only a single instance runs them
-    await migrateWithLock(options)
-  } else {
-    await migrations.runMigrations(MIGRATIONS, options)
-  }
-}
-
-const migrateWithLock = async (options?: MigrationOptions) => {
-  await locks.doWithLock(
-    {
-      type: LockType.TRY_ONCE,
-      name: LockName.MIGRATIONS,
-      ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes
-      systemLock: true,
-    },
-    async () => {
-      await migrations.runMigrations(MIGRATIONS, options)
-    }
-  )
-}

packages/server/src/migrations/tests/helpers.ts

@@ -1,40 +0,0 @@
-// Mimic configs test configuration from worker, creation configs directly in database
-
-import * as structures from "./structures"
-import { configs } from "@budibase/backend-core"
-import { Config } from "@budibase/types"
-
-export const saveSettingsConfig = async (globalDb: any) => {
-  const config = structures.settings()
-  await saveConfig(config, globalDb)
-}
-
-export const saveGoogleConfig = async (globalDb: any) => {
-  const config = structures.google()
-  await saveConfig(config, globalDb)
-}
-
-export const saveOIDCConfig = async (globalDb: any) => {
-  const config = structures.oidc()
-  await saveConfig(config, globalDb)
-}
-
-export const saveSmtpConfig = async (globalDb: any) => {
-  const config = structures.smtp()
-  await saveConfig(config, globalDb)
-}
-
-const saveConfig = async (config: Config, globalDb: any) => {
-  config._id = configs.generateConfigID(config.type)
-
-  let response
-  try {
-    response = await globalDb.get(config._id)
-    config._rev = response._rev
-    await globalDb.put(config)
-  } catch (e: any) {
-    if (e.status === 404) {
-      await globalDb.put(config)
-    }
-  }
-}

packages/server/src/migrations/tests/index.spec.ts

@@ -1,137 +0,0 @@
-import {
-  events,
-  migrations,
-  tenancy,
-  DocumentType,
-  context,
-} from "@budibase/backend-core"
-import TestConfig from "../../tests/utilities/TestConfiguration"
-import * as structures from "../../tests/utilities/structures"
-import { MIGRATIONS } from "../"
-import * as helpers from "./helpers"
-
-import tk from "timekeeper"
-import { View } from "@budibase/types"
-
-const timestamp = new Date().toISOString()
-tk.freeze(timestamp)
-
-const clearMigrations = async () => {
-  const dbs = [context.getDevAppDB(), context.getProdAppDB()]
-  for (const db of dbs) {
-    const doc = await db.get<any>(DocumentType.MIGRATIONS)
-    const newDoc = { _id: doc._id, _rev: doc._rev }
-    await db.put(newDoc)
-  }
-}
-
-describe("migrations", () => {
-  const config = new TestConfig()
-
-  beforeAll(async () => {
-    await config.init()
-  })
-
-  afterAll(() => {
-    config.end()
-  })
-
-  describe("backfill", () => {
-    it("runs app db migration", async () => {
-      await config.doInContext(undefined, async () => {
-        await clearMigrations()
-        await config.createAutomation()
-        await config.createAutomation(structures.newAutomation())
-        await config.createDatasource()
-        await config.createDatasource()
-        await config.createLayout()
-        await config.createQuery()
-        await config.createQuery()
-        await config.createRole()
-        await config.createRole()
-        await config.createTable()
-        await config.createLegacyView()
-        await config.createTable()
-        await config.createLegacyView(
-          structures.view(config.table!._id!) as View
-        )
-        await config.createScreen()
-        await config.createScreen()
-
-        jest.clearAllMocks()
-        const migration = MIGRATIONS.filter(
-          m => m.name === "event_app_backfill"
-        )[0]
-        await migrations.runMigration(migration)
-
-        expect(events.app.created).toHaveBeenCalledTimes(1)
-        expect(events.app.published).toHaveBeenCalledTimes(1)
-        expect(events.automation.created).toHaveBeenCalledTimes(2)
-        expect(events.automation.stepCreated).toHaveBeenCalledTimes(1)
-        expect(events.datasource.created).toHaveBeenCalledTimes(2)
-        expect(events.layout.created).toHaveBeenCalledTimes(1)
-        expect(events.query.created).toHaveBeenCalledTimes(2)
-        expect(events.role.created).toHaveBeenCalledTimes(3) // created roles + admin (created on table creation)
-        expect(events.table.created).toHaveBeenCalledTimes(3)
-        expect(events.backfill.appSucceeded).toHaveBeenCalledTimes(2)
-
-        // to make sure caching is working as expected
-        expect(
-          events.processors.analyticsProcessor.processEvent
-        ).toHaveBeenCalledTimes(20) // Addition of of the events above
-      })
-    })
-  })
-
-  it("runs global db migration", async () => {
-    await config.doInContext(undefined, async () => {
-      await clearMigrations()
-      const appId = config.getProdAppId()
-      const roles = { [appId]: "role_12345" }
-      await config.createUser({
-        builder: { global: false },
-        admin: { global: true },
-        roles,
-      }) // admin only
-      await config.createUser({
-        builder: { global: false },
-        admin: { global: false },
-        roles,
-      }) // non admin non builder
-      await config.createTable()
-      await config.createRow()
-      await config.createRow()
-
-      const db = tenancy.getGlobalDB()
-      await helpers.saveGoogleConfig(db)
-      await helpers.saveOIDCConfig(db)
-      await helpers.saveSettingsConfig(db)
-      await helpers.saveSmtpConfig(db)
-
-      jest.clearAllMocks()
-      const migration = MIGRATIONS.filter(
-        m => m.name === "event_global_backfill"
-      )[0]
-      await migrations.runMigration(migration)
-
-      expect(events.user.created).toHaveBeenCalledTimes(3)
-      expect(events.role.assigned).toHaveBeenCalledTimes(2)
-      expect(events.user.permissionBuilderAssigned).toHaveBeenCalledTimes(1) // default test user
-      expect(events.user.permissionAdminAssigned).toHaveBeenCalledTimes(1) // admin from above
-      expect(events.rows.created).toHaveBeenCalledTimes(1)
-      expect(events.rows.created).toHaveBeenCalledWith(2, timestamp)
-      expect(events.email.SMTPCreated).toHaveBeenCalledTimes(1)
-      expect(events.auth.SSOCreated).toHaveBeenCalledTimes(2)
-      expect(events.auth.SSOActivated).toHaveBeenCalledTimes(2)
-      expect(events.org.logoUpdated).toHaveBeenCalledTimes(1)
-      expect(events.org.nameUpdated).toHaveBeenCalledTimes(1)
-      expect(events.org.platformURLUpdated).toHaveBeenCalledTimes(1)
-      expect(events.backfill.tenantSucceeded).toHaveBeenCalledTimes(1)
-
-      // to make sure caching is working as expected
-      expect(
-        events.processors.analyticsProcessor.processEvent
-      ).toHaveBeenCalledTimes(19)
-    })
-  })
-})

packages/server/src/migrations/tests/structures.ts

@@ -1,67 +0,0 @@
-import { utils } from "@budibase/backend-core"
-import {
-  SMTPConfig,
-  OIDCConfig,
-  GoogleConfig,
-  SettingsConfig,
-  ConfigType,
-} from "@budibase/types"
-
-export const oidc = (conf?: OIDCConfig): OIDCConfig => {
-  return {
-    type: ConfigType.OIDC,
-    config: {
-      configs: [
-        {
-          configUrl: "http://someconfigurl",
-          clientID: "clientId",
-          clientSecret: "clientSecret",
-          logo: "Microsoft",
-          name: "Active Directory",
-          uuid: utils.newid(),
-          activated: true,
-          scopes: [],
-          ...conf,
-        },
-      ],
-    },
-  }
-}
-
-export const google = (conf?: GoogleConfig): GoogleConfig => {
-  return {
-    type: ConfigType.GOOGLE,
-    config: {
-      clientID: "clientId",
-      clientSecret: "clientSecret",
-      activated: true,
-      ...conf,
-    },
-  }
-}
-
-export const smtp = (conf?: SMTPConfig): SMTPConfig => {
-  return {
-    type: ConfigType.SMTP,
-    config: {
-      port: 12345,
-      host: "smtptesthost.com",
-      from: "testfrom@example.com",
-      subject: "Hello!",
-      secure: false,
-      ...conf,
-    },
-  }
-}
-
-export const settings = (conf?: SettingsConfig): SettingsConfig => {
-  return {
-    type: ConfigType.SETTINGS,
-    config: {
-      platformUrl: "http://mycustomdomain.com",
-      logoUrl: "http://mylogourl,com",
-      company: "mycompany",
-      ...conf,
-    },
-  }
-}

packages/server/src/startup/index.ts

@@ -15,7 +15,6 @@ import { watch } from "../watch"
 import * as automations from "../automations"
 import * as fileSystem from "../utilities/fileSystem"
 import { default as eventEmitter, init as eventInit } from "../events"
-import * as migrations from "../migrations"
 import * as bullboard from "../automations/bullboard"
 import * as appMigrations from "../appMigrations/queue"
 import * as pro from "@budibase/pro"
@@ -106,18 +105,6 @@ export async function startup(
     initialiseWebsockets(app, server)
   }
 
-  // run migrations on startup if not done via http
-  // not recommended in a clustered environment
-  if (!env.HTTP_MIGRATIONS && !env.isTest()) {
-    console.log("Running migrations")
-    try {
-      await migrations.migrate()
-    } catch (e) {
-      logging.logAlert("Error performing migrations. Exiting.", e)
-      shutdown(server)
-    }
-  }
-
   // monitor plugin directory if required
   if (
     env.SELF_HOSTED &&

packages/shared-core/src/automations/index.ts

@@ -0,0 +1 @@
+export * as steps from "./steps/index"

packages/shared-core/src/automations/steps/bash.ts

@@ -0,0 +1,47 @@
+import {
+  AutomationActionStepId,
+  AutomationCustomIOType,
+  AutomationFeature,
+  AutomationIOType,
+  AutomationStepDefinition,
+  AutomationStepType,
+} from "@budibase/types"
+
+export const definition: AutomationStepDefinition = {
+  name: "Bash Scripting",
+  tagline: "Execute a bash command",
+  icon: "JourneyEvent",
+  description: "Run a bash script",
+  type: AutomationStepType.ACTION,
+  internal: true,
+  features: {
+    [AutomationFeature.LOOPING]: true,
+  },
+  stepId: AutomationActionStepId.EXECUTE_BASH,
+  inputs: {},
+  schema: {
+    inputs: {
+      properties: {
+        code: {
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.CODE,
+          title: "Code",
+        },
+      },
+      required: ["code"],
+    },
+    outputs: {
+      properties: {
+        stdout: {
+          type: AutomationIOType.STRING,
+          description: "Standard output of your bash command or script",
+        },
+        success: {
+          type: AutomationIOType.BOOLEAN,
+          description: "Whether the command was successful",
+        },
+      },
+      required: ["stdout"],
+    },
+  },
+}

packages/shared-core/src/automations/steps/collect.ts

@@ -0,0 +1,43 @@
+import {
+  AutomationActionStepId,
+  AutomationStepDefinition,
+  AutomationStepType,
+  AutomationIOType,
+} from "@budibase/types"
+
+export const definition: AutomationStepDefinition = {
+  name: "Collect Data",
+  tagline: "Collect data to be sent to design",
+  icon: "Collection",
+  description:
+    "Collects specified data so it can be provided to the design section",
+  type: AutomationStepType.ACTION,
+  internal: true,
+  features: {},
+  stepId: AutomationActionStepId.COLLECT,
+  inputs: {},
+  schema: {
+    inputs: {
+      properties: {
+        collection: {
+          type: AutomationIOType.STRING,
+          title: "What to Collect",
+        },
+      },
+      required: ["collection"],
+    },
+    outputs: {
+      properties: {
+        success: {
+          type: AutomationIOType.BOOLEAN,
+          description: "Whether the action was successful",
+        },
+        value: {
+          type: AutomationIOType.STRING,
+          description: "Collected data",
+        },
+      },
+      required: ["success", "value"],
+    },
+  },
+}

packages/shared-core/src/automations/steps/createRow.ts

@@ -0,0 +1,67 @@
+import {
+  AutomationActionStepId,
+  AutomationCustomIOType,
+  AutomationFeature,
+  AutomationIOType,
+  AutomationStepDefinition,
+  AutomationStepType,
+} from "@budibase/types"
+
+export const definition: AutomationStepDefinition = {
+  name: "Create Row",
+  tagline: "Create a {{inputs.enriched.table.name}} row",
+  icon: "TableRowAddBottom",
+  description: "Add a row to your database",
+  type: AutomationStepType.ACTION,
+  internal: true,
+  features: {
+    [AutomationFeature.LOOPING]: true,
+  },
+  stepId: AutomationActionStepId.CREATE_ROW,
+  inputs: {},
+  schema: {
+    inputs: {
+      properties: {
+        row: {
+          type: AutomationIOType.OBJECT,
+          properties: {
+            tableId: {
+              type: AutomationIOType.STRING,
+              customType: AutomationCustomIOType.TABLE,
+            },
+          },
+          customType: AutomationCustomIOType.ROW,
+          title: "Table",
+          required: ["tableId"],
+        },
+      },
+      required: ["row"],
+    },
+    outputs: {
+      properties: {
+        row: {
+          type: AutomationIOType.OBJECT,
+          customType: AutomationCustomIOType.ROW,
+          description: "The new row",
+        },
+        response: {
+          type: AutomationIOType.OBJECT,
+          description: "The response from the table",
+        },
+        success: {
+          type: AutomationIOType.BOOLEAN,
+          description: "Whether the row creation was successful",
+        },
+        id: {
+          type: AutomationIOType.STRING,
+          description: "The identifier of the new row",
+        },
+        revision: {
+          type: AutomationIOType.STRING,
+          description: "The revision of the new row",
+        },
+      },
+      required: ["success", "id", "revision"],
+    },
+  },
+}