Merge master.

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.44",
+  "version": "3.2.47",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/src/db/couch/connections.ts

@@ -1,6 +1,6 @@
 import env from "../../environment"
 
-export const getCouchInfo = (connection?: string) => {
+export const getCouchInfo = (connection?: string | null) => {
   // clean out any auth credentials
   const urlInfo = getUrlInfo(connection)
   let username
@@ -45,7 +45,7 @@ export const getCouchInfo = (connection?: string) => {
   }
 }
 
-export const getUrlInfo = (url = env.COUCH_DB_URL) => {
+export const getUrlInfo = (url: string | null = env.COUCH_DB_URL) => {
   let cleanUrl, username, password, host
   if (url) {
     // Ensure the URL starts with a protocol

packages/backend-core/src/db/tests/pouch.spec.ts

@@ -1,5 +1,6 @@
 require("../../../tests")
-const getUrlInfo = require("../couch").getUrlInfo
+
+import { getUrlInfo } from "../couch"
 
 describe("pouch", () => {
   describe("Couch DB URL parsing", () => {

packages/backend-core/src/index.ts

@@ -1,6 +1,5 @@
 export * as configs from "./configs"
 export * as events from "./events"
-export * as migrations from "./migrations"
 export * as users from "./users"
 export * as userUtils from "./users/utils"
 export * as roles from "./security/roles"

packages/backend-core/src/migrations/definitions.ts

@@ -1,40 +0,0 @@
-import {
-  MigrationType,
-  MigrationName,
-  MigrationDefinition,
-} from "@budibase/types"
-
-export const DEFINITIONS: MigrationDefinition[] = [
-  {
-    type: MigrationType.GLOBAL,
-    name: MigrationName.USER_EMAIL_VIEW_CASING,
-  },
-  {
-    type: MigrationType.GLOBAL,
-    name: MigrationName.SYNC_QUOTAS,
-  },
-  {
-    type: MigrationType.APP,
-    name: MigrationName.APP_URLS,
-  },
-  {
-    type: MigrationType.APP,
-    name: MigrationName.EVENT_APP_BACKFILL,
-  },
-  {
-    type: MigrationType.APP,
-    name: MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS,
-  },
-  {
-    type: MigrationType.GLOBAL,
-    name: MigrationName.EVENT_GLOBAL_BACKFILL,
-  },
-  {
-    type: MigrationType.INSTALLATION,
-    name: MigrationName.EVENT_INSTALLATION_BACKFILL,
-  },
-  {
-    type: MigrationType.GLOBAL,
-    name: MigrationName.GLOBAL_INFO_SYNC_USERS,
-  },
-]

packages/backend-core/src/migrations/index.ts

@@ -1,2 +0,0 @@
-export * from "./migrations"
-export * from "./definitions"

packages/backend-core/src/migrations/migrations.ts

@@ -1,186 +0,0 @@
-import { DEFAULT_TENANT_ID } from "../constants"
-import {
-  DocumentType,
-  StaticDatabases,
-  getAllApps,
-  getGlobalDBName,
-  getDB,
-} from "../db"
-import environment from "../environment"
-import * as platform from "../platform"
-import * as context from "../context"
-import { DEFINITIONS } from "."
-import {
-  Migration,
-  MigrationOptions,
-  MigrationType,
-  MigrationNoOpOptions,
-  App,
-} from "@budibase/types"
-
-export const getMigrationsDoc = async (db: any) => {
-  // get the migrations doc
-  try {
-    return await db.get(DocumentType.MIGRATIONS)
-  } catch (err: any) {
-    if (err.status && err.status === 404) {
-      return { _id: DocumentType.MIGRATIONS }
-    } else {
-      throw err
-    }
-  }
-}
-
-export const backPopulateMigrations = async (opts: MigrationNoOpOptions) => {
-  // filter migrations to the type and populate a no-op migration
-  const migrations: Migration[] = DEFINITIONS.filter(
-    def => def.type === opts.type
-  ).map(d => ({ ...d, fn: async () => {} }))
-  await runMigrations(migrations, { noOp: opts })
-}
-
-export const runMigration = async (
-  migration: Migration,
-  options: MigrationOptions = {}
-) => {
-  const migrationType = migration.type
-  const migrationName = migration.name
-  const silent = migration.silent
-
-  const log = (message: string) => {
-    if (!silent) {
-      console.log(message)
-    }
-  }
-
-  // get the db to store the migration in
-  let dbNames: string[]
-  if (migrationType === MigrationType.GLOBAL) {
-    dbNames = [getGlobalDBName()]
-  } else if (migrationType === MigrationType.APP) {
-    if (options.noOp) {
-      if (!options.noOp.appId) {
-        throw new Error("appId is required for noOp app migration")
-      }
-      dbNames = [options.noOp.appId]
-    } else {
-      const apps = (await getAllApps(migration.appOpts)) as App[]
-      dbNames = apps.map(app => app.appId)
-    }
-  } else if (migrationType === MigrationType.INSTALLATION) {
-    dbNames = [StaticDatabases.PLATFORM_INFO.name]
-  } else {
-    throw new Error(`Unrecognised migration type [${migrationType}]`)
-  }
-
-  const length = dbNames.length
-  let count = 0
-
-  // run the migration against each db
-  for (const dbName of dbNames) {
-    count++
-    const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
-
-    const db = getDB(dbName)
-
-    try {
-      const doc = await getMigrationsDoc(db)
-
-      // the migration has already been run
-      if (doc[migrationName]) {
-        // check for force
-        if (
-          options.force &&
-          options.force[migrationType] &&
-          options.force[migrationType].includes(migrationName)
-        ) {
-          log(`[Migration: ${migrationName}] [DB: ${dbName}] Forcing`)
-        } else {
-          // no force, exit
-          return
-        }
-      }
-
-      // check if the migration is not a no-op
-      if (!options.noOp) {
-        log(
-          `[Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
-        )
-
-        if (migration.preventRetry) {
-          // eagerly set the completion date
-          // so that we never run this migration twice even upon failure
-          doc[migrationName] = Date.now()
-          const response = await db.put(doc)
-          doc._rev = response.rev
-        }
-
-        // run the migration
-        if (migrationType === MigrationType.APP) {
-          await context.doInAppContext(db.name, async () => {
-            await migration.fn(db)
-          })
-        } else {
-          await migration.fn(db)
-        }
-
-        log(`[Migration: ${migrationName}] [DB: ${dbName}] Complete`)
-      }
-
-      // mark as complete
-      doc[migrationName] = Date.now()
-      await db.put(doc)
-    } catch (err) {
-      console.error(
-        `[Migration: ${migrationName}] [DB: ${dbName}] Error: `,
-        err
-      )
-      throw err
-    }
-  }
-}
-
-export const runMigrations = async (
-  migrations: Migration[],
-  options: MigrationOptions = {}
-) => {
-  let tenantIds
-
-  if (environment.MULTI_TENANCY) {
-    if (options.noOp) {
-      tenantIds = [options.noOp.tenantId]
-    } else if (!options.tenantIds || !options.tenantIds.length) {
-      // run for all tenants
-      tenantIds = await platform.tenants.getTenantIds()
-    } else {
-      tenantIds = options.tenantIds
-    }
-  } else {
-    // single tenancy
-    tenantIds = [DEFAULT_TENANT_ID]
-  }
-
-  if (tenantIds.length > 1) {
-    console.log(`Checking migrations for ${tenantIds.length} tenants`)
-  } else {
-    console.log("Checking migrations")
-  }
-
-  let count = 0
-  // for all tenants
-  for (const tenantId of tenantIds) {
-    count++
-    if (tenantIds.length > 1) {
-      console.log(`Progress [${count}/${tenantIds.length}]`)
-    }
-    // for all migrations
-    for (const migration of migrations) {
-      // run the migration
-      await context.doInTenant(
-        tenantId,
-        async () => await runMigration(migration, options)
-      )
-    }
-  }
-  console.log("Migrations complete")
-}

packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap

@@ -1,11 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`migrations should match snapshot 1`] = `
-{
-  "_id": "migrations",
-  "_rev": "1-2f64479842a0513aa8b97f356b0b9127",
-  "createdAt": "2020-01-01T00:00:00.000Z",
-  "test": 1577836800000,
-  "updatedAt": "2020-01-01T00:00:00.000Z",
-}
-`;

packages/backend-core/src/migrations/tests/migrations.spec.ts

@@ -1,64 +0,0 @@
-import { testEnv, DBTestConfiguration } from "../../../tests/extra"
-import * as migrations from "../index"
-import * as context from "../../context"
-import { MigrationType } from "@budibase/types"
-
-testEnv.multiTenant()
-
-describe("migrations", () => {
-  const config = new DBTestConfiguration()
-
-  const migrationFunction = jest.fn()
-
-  const MIGRATIONS = [
-    {
-      type: MigrationType.GLOBAL,
-      name: "test" as any,
-      fn: migrationFunction,
-    },
-  ]
-
-  beforeEach(() => {
-    config.newTenant()
-  })
-
-  afterEach(async () => {
-    jest.clearAllMocks()
-  })
-
-  const migrate = () => {
-    return migrations.runMigrations(MIGRATIONS, {
-      tenantIds: [config.tenantId],
-    })
-  }
-
-  it("should run a new migration", async () => {
-    await config.doInTenant(async () => {
-      await migrate()
-      expect(migrationFunction).toHaveBeenCalled()
-      const db = context.getGlobalDB()
-      const doc = await migrations.getMigrationsDoc(db)
-      expect(doc.test).toBeDefined()
-    })
-  })
-
-  it("should match snapshot", async () => {
-    await config.doInTenant(async () => {
-      await migrate()
-      const doc = await migrations.getMigrationsDoc(context.getGlobalDB())
-      expect(doc).toMatchSnapshot()
-    })
-  })
-
-  it("should skip a previously run migration", async () => {
-    await config.doInTenant(async () => {
-      const db = context.getGlobalDB()
-      await migrate()
-      const previousDoc = await migrations.getMigrationsDoc(db)
-      await migrate()
-      const currentDoc = await migrations.getMigrationsDoc(db)
-      expect(migrationFunction).toHaveBeenCalledTimes(1)
-      expect(currentDoc.test).toBe(previousDoc.test)
-    })
-  })
-})

packages/backend-core/src/sql/sql.ts

@@ -1172,20 +1172,22 @@ class InternalBuilder {
           nulls = value.direction === SortOrder.ASCENDING ? "first" : "last"
         }
 
+        const composite = `${aliased}.${key}`
+        let identifier
+
         if (this.isAggregateField(key)) {
-          query = query.orderBy(key, direction, nulls)
+          identifier = this.rawQuotedIdentifier(key)
+        } else if (this.client === SqlClient.ORACLE) {
+          identifier = this.convertClobs(composite)
         } else {
-          let composite = `${aliased}.${key}`
+          identifier = this.rawQuotedIdentifier(composite)
-          if (this.client === SqlClient.ORACLE) {
+        }
-            query = query.orderByRaw(`?? ?? nulls ??`, [
+
-              this.convertClobs(composite),
+        query = query.orderByRaw(`?? ?? ${nulls ? "nulls ??" : ""}`, [
+          identifier,
           this.knex.raw(direction),
-              this.knex.raw(nulls as string),
+          ...(nulls ? [this.knex.raw(nulls as string)] : []),
         ])
-          } else {
-            query = query.orderBy(composite, direction, nulls)
-          }
-        }
       }
     }
 
@@ -1344,14 +1346,16 @@ class InternalBuilder {
 
       // add the correlation to the overall query
       subQuery = subQuery.where(
-        correlatedTo,
+        this.rawQuotedIdentifier(correlatedTo),
         "=",
         this.rawQuotedIdentifier(correlatedFrom)
       )
 
       const standardWrap = (select: Knex.Raw): Knex.QueryBuilder => {
         subQuery = subQuery
-          .select(relationshipFields)
+          .select(
+            relationshipFields.map(field => this.rawQuotedIdentifier(field))
+          )
           .limit(getRelationshipLimit())
         // @ts-ignore - the from alias syntax isn't in Knex typing
         return knex.select(select).from({

packages/backend-core/tests/core/users/users.spec.ts

@@ -1,17 +1,17 @@
-const _ = require("lodash/fp")
+import { range } from "lodash/fp"
-const { structures } = require("../../../tests")
+import { structures } from "../.."
 
 jest.mock("../../../src/context")
 jest.mock("../../../src/db")
 
-const context = require("../../../src/context")
+import * as context from "../../../src/context"
-const db = require("../../../src/db")
+import * as db from "../../../src/db"
 
-const { getCreatorCount } = require("../../../src/users/users")
+import { getCreatorCount } from "../../../src/users/users"
 
 describe("Users", () => {
-  let getGlobalDBMock
+  let getGlobalDBMock: jest.SpyInstance
-  let paginationMock
+  let paginationMock: jest.SpyInstance
 
   beforeEach(() => {
     jest.resetAllMocks()
@@ -22,11 +22,10 @@ describe("Users", () => {
     jest.spyOn(db, "getGlobalUserParams")
   })
 
-  it("Retrieves the number of creators", async () => {
+  it("retrieves the number of creators", async () => {
-    const getUsers = (offset, limit, creators = false) => {
+    const getUsers = (offset: number, limit: number, creators = false) => {
-      const range = _.range(offset, limit)
       const opts = creators ? { builder: { global: true } } : undefined
-      return range.map(() => structures.users.user(opts))
+      return range(offset, limit).map(() => structures.users.user(opts))
     }
     const page1Data = getUsers(0, 8)
     const page2Data = getUsers(8, 12, true)

packages/bbui/package.json

@@ -3,7 +3,7 @@
   "description": "A UI solution used in the different Budibase projects.",
   "version": "0.0.0",
   "license": "MPL-2.0",
-  "svelte": "src/index.js",
+  "svelte": "src/index.ts",
   "module": "dist/bbui.mjs",
   "exports": {
     ".": {
@@ -14,7 +14,8 @@
     "./spectrum-icons-vite.js": "./src/spectrum-icons-vite.js"
   },
   "scripts": {
-    "build": "vite build"
+    "build": "vite build",
+    "dev": "vite build --watch --mode=dev"
   },
   "devDependencies": {
     "@sveltejs/vite-plugin-svelte": "1.4.0",

packages/bbui/src/Icon/Icon.svelte

@@ -1,23 +1,23 @@
-<script>
+<script lang="ts">
   import {
     default as AbsTooltip,
     TooltipPosition,
     TooltipType,
   } from "../Tooltip/AbsTooltip.svelte"
 
-  export let name = "Add"
+  export let name: string = "Add"
-  export let hidden = false
+  export let hidden: boolean = false
   export let size = "M"
-  export let hoverable = false
+  export let hoverable: boolean = false
-  export let disabled = false
+  export let disabled: boolean = false
-  export let color
+  export let color: string | undefined = undefined
-  export let hoverColor
+  export let hoverColor: string | undefined = undefined
-  export let tooltip
+  export let tooltip: string | undefined = undefined
   export let tooltipPosition = TooltipPosition.Bottom
   export let tooltipType = TooltipType.Default
-  export let tooltipColor
+  export let tooltipColor: string | undefined = undefined
-  export let tooltipWrap = true
+  export let tooltipWrap: boolean = true
-  export let newStyles = false
+  export let newStyles: boolean = false
 </script>
 
 <AbsTooltip

packages/bbui/src/Label/Label.svelte

@@ -4,7 +4,7 @@
 
   export let size = "M"
   export let tooltip = ""
-  export let muted
+  export let muted = undefined
 </script>
 
 <TooltipWrapper {tooltip} {size}>

packages/bbui/src/Tooltip/AbsTooltip.svelte

@@ -23,7 +23,7 @@
   export let type = TooltipType.Default
   export let text = ""
   export let fixed = false
-  export let color = null
+  export let color = ""
   export let noWrap = false
 
   let wrapper

packages/bbui/src/Typography/Heading.svelte

@@ -2,10 +2,10 @@
   import "@spectrum-css/typography/dist/index-vars.css"
 
   // Sizes
-  export let size = "M"
+  export let size: "XS" | "S" | "M" | "L" = "M"
-  export let textAlign = undefined
+  export let textAlign: string | undefined = undefined
-  export let noPadding = false
+  export let noPadding: boolean = false
-  export let weight = "default" // light, heavy, default
+  export let weight: "light" | "heavy" | "default" = "default"
 </script>
 
 <h1

packages/bbui/src/bbui.css

@@ -45,6 +45,11 @@
   --purple: #806fde;
   --purple-dark: #130080;
 
+  --error-bg: rgba(226, 109, 105, 0.3);
+  --warning-bg: rgba(255, 210, 106, 0.3);
+  --error-content: rgba(226, 109, 105, 0.6);
+  --warning-content: rgba(255, 210, 106, 0.6);
+
   --rounded-small: 4px;
   --rounded-medium: 8px;
   --rounded-large: 16px;

packages/bbui/src/helpers.d.ts

@@ -1,3 +0,0 @@
-declare module "./helpers" {
-  export const cloneDeep: <T>(obj: T) => T
-}

packages/bbui/src/helpers.ts

@@ -6,9 +6,8 @@ export const deepGet = helpers.deepGet
 /**
  * Generates a DOM safe UUID.
  * Starting with a letter is important to make it DOM safe.
- * @return {string} a random DOM safe UUID
  */
-export function uuid() {
+export function uuid(): string {
   return "cxxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx".replace(/[xy]/g, c => {
     const r = (Math.random() * 16) | 0
     const v = c === "x" ? r : (r & 0x3) | 0x8
@@ -18,22 +17,18 @@ export function uuid() {
 
 /**
  * Capitalises a string
- * @param string the string to capitalise
- * @return {string} the capitalised string
  */
-export const capitalise = string => {
+export const capitalise = (string?: string | null): string => {
   if (!string) {
-    return string
+    return ""
   }
   return string.substring(0, 1).toUpperCase() + string.substring(1)
 }
 
 /**
  * Computes a short hash of a string
- * @param string the string to compute a hash of
- * @return {string} the hash string
  */
-export const hashString = string => {
+export const hashString = (string?: string | null): string => {
   if (!string) {
     return "0"
   }
@@ -54,11 +49,12 @@ export const hashString = string => {
  * will override the value "foo" rather than "bar".
  * If a deep path is specified and the parent keys don't exist then these will
  * be created.
- * @param obj the object
- * @param key the key
- * @param value the value
  */
-export const deepSet = (obj, key, value) => {
+export const deepSet = (
+  obj: Record<string, any> | null,
+  key: string | null,
+  value: any
+): void => {
   if (!obj || !key) {
     return
   }
@@ -82,9 +78,8 @@ export const deepSet = (obj, key, value) => {
 
 /**
  * Deeply clones an object. Functions are not supported.
- * @param obj the object to clone
  */
-export const cloneDeep = obj => {
+export const cloneDeep = <T>(obj: T): T => {
   if (!obj) {
     return obj
   }
@@ -93,9 +88,8 @@ export const cloneDeep = obj => {
 
 /**
  * Copies a value to the clipboard
- * @param value the value to copy
  */
-export const copyToClipboard = value => {
+export const copyToClipboard = (value: any): Promise<void> => {
   return new Promise(res => {
     if (navigator.clipboard && window.isSecureContext) {
       // Try using the clipboard API first
@@ -117,9 +111,12 @@ export const copyToClipboard = value => {
   })
 }
 
-// Parsed a date value. This is usually an ISO string, but can be a
+// Parse a date value. This is usually an ISO string, but can be a
 // bunch of different formats and shapes depending on schema flags.
-export const parseDate = (value, { enableTime = true }) => {
+export const parseDate = (
+  value: string | dayjs.Dayjs | null,
+  { enableTime = true }
+): dayjs.Dayjs | null => {
   // If empty then invalid
   if (!value) {
     return null
@@ -128,7 +125,7 @@ export const parseDate = (value, { enableTime = true }) => {
   // Certain string values need transformed
   if (typeof value === "string") {
     // Check for time only values
-    if (!isNaN(new Date(`0-${value}`))) {
+    if (!isNaN(new Date(`0-${value}`).valueOf())) {
       value = `0-${value}`
     }
 
@@ -153,9 +150,9 @@ export const parseDate = (value, { enableTime = true }) => {
 // Stringifies a dayjs object to create an ISO string that respects the various
 // schema flags
 export const stringifyDate = (
-  value,
+  value: null | dayjs.Dayjs,
   { enableTime = true, timeOnly = false, ignoreTimezones = false } = {}
-) => {
+): string | null => {
   if (!value) {
     return null
   }
@@ -192,7 +189,7 @@ export const stringifyDate = (
 }
 
 // Determine the dayjs-compatible format of the browser's default locale
-const getPatternForPart = part => {
+const getPatternForPart = (part: Intl.DateTimeFormatPart): string => {
   switch (part.type) {
     case "day":
       return "D".repeat(part.value.length)
@@ -214,9 +211,9 @@ const localeDateFormat = new Intl.DateTimeFormat()
 
 // Formats a dayjs date according to schema flags
 export const getDateDisplayValue = (
-  value,
+  value: dayjs.Dayjs | null,
   { enableTime = true, timeOnly = false } = {}
-) => {
+): string => {
   if (!value?.isValid()) {
     return ""
   }
@@ -229,7 +226,7 @@ export const getDateDisplayValue = (
   }
 }
 
-export const hexToRGBA = (color, opacity) => {
+export const hexToRGBA = (color: string, opacity: number): string => {
   if (color.includes("#")) {
     color = color.replace("#", "")
   }

packages/bbui/svelte.config.js

@@ -0,0 +1,7 @@
+const { vitePreprocess } = require("@sveltejs/vite-plugin-svelte")
+
+const config = {
+  preprocess: vitePreprocess(),
+}
+
+module.exports = config

packages/bbui/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "../../tsconfig.build.json",
+  "compilerOptions": {
+    "allowJs": true,
+    "outDir": "./dist",
+    "lib": ["ESNext"],
+    "baseUrl": ".",
+    "paths": {
+      "@budibase/*": [
+        "../*/src/index.ts",
+        "../*/src/index.js",
+        "../*",
+        "../../node_modules/@budibase/*"
+      ]
+    }
+  },
+  "include": ["./src/**/*"],
+  "exclude": ["node_modules", "**/*.json", "**/*.spec.ts", "**/*.spec.js"]
+}

packages/bbui/vite.config.js

@@ -9,7 +9,7 @@ export default defineConfig(({ mode }) => {
     build: {
       sourcemap: !isProduction,
       lib: {
-        entry: "src/index.js",
+        entry: "src/index.ts",
         formats: ["es"],
       },
     },

packages/builder/package.json

@@ -74,7 +74,6 @@
     "dayjs": "^1.10.8",
     "downloadjs": "1.4.7",
     "fast-json-patch": "^3.1.1",
-    "json-format-highlight": "^1.0.4",
     "lodash": "4.17.21",
     "posthog-js": "^1.118.0",
     "remixicon": "2.5.0",
@@ -94,6 +93,7 @@
     "@sveltejs/vite-plugin-svelte": "1.4.0",
     "@testing-library/jest-dom": "6.4.2",
     "@testing-library/svelte": "^4.1.0",
+    "@types/sanitize-html": "^2.13.0",
     "@types/shortid": "^2.2.0",
     "babel-jest": "^29.6.2",
     "identity-obj-proxy": "^3.0.0",

packages/builder/src/components/automation/SetupPanel/CronBuilder.svelte

@@ -9,7 +9,7 @@
   } from "@budibase/bbui"
   import { onMount, createEventDispatcher } from "svelte"
   import { flags } from "@/stores/builder"
-  import { featureFlags, licensing } from "@/stores/portal"
+  import { licensing } from "@/stores/portal"
   import { API } from "@/api"
   import MagicWand from "../../../../assets/MagicWand.svelte"
 
@@ -27,8 +27,7 @@
   let loadingAICronExpression = false
 
   $: aiEnabled =
-    ($featureFlags.AI_CUSTOM_CONFIGS && $licensing.customAIConfigsEnabled) ||
+    $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled
-    ($featureFlags.BUDIBASE_AI && $licensing.budibaseAIEnabled)
   $: {
     if (cronExpression) {
       try {

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -26,7 +26,7 @@
   import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
   import { tables, datasources } from "@/stores/builder"
-  import { featureFlags } from "@/stores/portal"
+  import { licensing } from "@/stores/portal"
   import { TableNames, UNEDITABLE_USER_FIELDS } from "@/constants"
   import {
     FIELDS,
@@ -100,7 +100,8 @@
   let optionsValid = true
 
   $: rowGoldenSample = RowUtils.generateGoldenSample($rows)
-  $: aiEnabled = $featureFlags.BUDIBASE_AI || $featureFlags.AI_CUSTOM_CONFIGS
+  $: aiEnabled =
+    $licensing.customAIConfigsEnabled || $licensing.budibaseAiEnabled
   $: if (primaryDisplay) {
     editableColumn.constraints.presence = { allowEmpty: false }
   }

packages/builder/src/components/common/CodeEditor/CodeEditor.svelte

@@ -1,4 +1,4 @@
-<script>
+<script lang="ts">
   import { Label } from "@budibase/bbui"
   import { onMount, createEventDispatcher, onDestroy } from "svelte"
   import { FIND_ANY_HBS_REGEX } from "@budibase/string-templates"
@@ -12,7 +12,6 @@
     completionStatus,
   } from "@codemirror/autocomplete"
   import {
-    EditorView,
     lineNumbers,
     keymap,
     highlightSpecialChars,
@@ -25,6 +24,7 @@
     MatchDecorator,
     ViewPlugin,
     Decoration,
+    EditorView,
   } from "@codemirror/view"
   import {
     bracketMatching,
@@ -44,12 +44,14 @@
   import { javascript } from "@codemirror/lang-javascript"
   import { EditorModes } from "./"
   import { themeStore } from "@/stores/portal"
+  import type { EditorMode } from "@budibase/types"
 
-  export let label
+  export let label: string | undefined = undefined
-  export let completions = []
+  // TODO: work out what best type fits this
-  export let mode = EditorModes.Handlebars
+  export let completions: any[] = []
-  export let value = ""
+  export let mode: EditorMode = EditorModes.Handlebars
-  export let placeholder = null
+  export let value: string | null = ""
+  export let placeholder: string | null = null
   export let autocompleteEnabled = true
   export let autofocus = false
   export let jsBindingWrapping = true
@@ -58,8 +60,8 @@
 
   const dispatch = createEventDispatcher()
 
-  let textarea
+  let textarea: HTMLDivElement
-  let editor
+  let editor: EditorView
   let mounted = false
   let isEditorInitialised = false
   let queuedRefresh = false
@@ -100,15 +102,22 @@
   /**
    * Will refresh the editor contents only after
    * it has been fully initialised
-   * @param value {string} the editor value
    */
-  const refresh = (value, initialised, mounted) => {
+  const refresh = (
+    value: string | null,
+    initialised?: boolean,
+    mounted?: boolean
+  ) => {
     if (!initialised || !mounted) {
       queuedRefresh = true
       return
     }
 
-    if (editor.state.doc.toString() !== value || queuedRefresh) {
+    if (
+      editor &&
+      value &&
+      (editor.state.doc.toString() !== value || queuedRefresh)
+    ) {
       editor.dispatch({
         changes: { from: 0, to: editor.state.doc.length, insert: value },
       })
@@ -120,12 +129,17 @@
   export const getCaretPosition = () => {
     const selection_range = editor.state.selection.ranges[0]
     return {
-      start: selection_range.from,
+      start: selection_range?.from,
-      end: selection_range.to,
+      end: selection_range?.to,
     }
   }
 
-  export const insertAtPos = opts => {
+  export const insertAtPos = (opts: {
+    start: number
+    end?: number
+    value: string
+    cursor: { anchor: number }
+  }) => {
     // Updating the value inside.
     // Retain focus
     editor.dispatch({
@@ -192,7 +206,7 @@
 
   const indentWithTabCustom = {
     key: "Tab",
-    run: view => {
+    run: (view: EditorView) => {
       if (completionStatus(view.state) === "active") {
         acceptCompletion(view)
         return true
@@ -200,7 +214,7 @@
       indentMore(view)
       return true
     },
-    shift: view => {
+    shift: (view: EditorView) => {
       indentLess(view)
       return true
     },
@@ -232,7 +246,8 @@
 
   // None of this is reactive, but it never has been, so we just assume most
   // config flags aren't changed at runtime
-  const buildExtensions = base => {
+  // TODO: work out type for base
+  const buildExtensions = (base: any[]) => {
     let complete = [...base]
 
     if (autocompleteEnabled) {
@@ -242,7 +257,7 @@
           closeOnBlur: true,
           icons: false,
           optionClass: completion =>
-            completion.simple
+            "simple" in completion && completion.simple
               ? "autocomplete-option-simple"
               : "autocomplete-option",
         })
@@ -347,7 +362,7 @@
 
 {#if label}
   <div>
-    <Label small>{label}</Label>
+    <Label size="S">{label}</Label>
   </div>
 {/if}
 

packages/builder/src/components/common/CodeEditor/index.ts

@@ -1,8 +1,15 @@
 import { getManifest } from "@budibase/string-templates"
 import sanitizeHtml from "sanitize-html"
 import { groupBy } from "lodash"
+import {
+  BindingCompletion,
+  EditorModesMap,
+  Helper,
+  Snippet,
+} from "@budibase/types"
+import { CompletionContext } from "@codemirror/autocomplete"
 
-export const EditorModes = {
+export const EditorModes: EditorModesMap = {
   JS: {
     name: "javascript",
     json: false,
@@ -26,7 +33,7 @@ export const SECTIONS = {
   },
 }
 
-export const buildHelperInfoNode = (completion, helper) => {
+export const buildHelperInfoNode = (completion: any, helper: Helper) => {
   const ele = document.createElement("div")
   ele.classList.add("info-bubble")
 
@@ -46,7 +53,7 @@ export const buildHelperInfoNode = (completion, helper) => {
   return ele
 }
 
-const toSpectrumIcon = name => {
+const toSpectrumIcon = (name: string) => {
   return `<svg
     class="spectrum-Icon spectrum-Icon--sizeS"
     focusable="false"
@@ -58,7 +65,12 @@ const toSpectrumIcon = name => {
   </svg>`
 }
 
-export const buildSectionHeader = (type, sectionName, icon, rank) => {
+export const buildSectionHeader = (
+  type: string,
+  sectionName: string,
+  icon: string,
+  rank: number
+) => {
   const ele = document.createElement("div")
   ele.classList.add("info-section")
   if (type) {
@@ -72,43 +84,52 @@ export const buildSectionHeader = (type, sectionName, icon, rank) => {
   }
 }
 
-export const helpersToCompletion = (helpers, mode) => {
+export const helpersToCompletion = (
+  helpers: Record<string, Helper>,
+  mode: { name: "javascript" | "handlebars" }
+) => {
   const { type, name: sectionName, icon } = SECTIONS.HB_HELPER
   const helperSection = buildSectionHeader(type, sectionName, icon, 99)
 
-  return Object.keys(helpers).reduce((acc, key) => {
+  return Object.keys(helpers).flatMap(helperName => {
-    let helper = helpers[key]
+    let helper = helpers[helperName]
-    acc.push({
+    return {
-      label: key,
+      label: helperName,
-      info: completion => {
+      info: (completion: BindingCompletion) => {
         return buildHelperInfoNode(completion, helper)
       },
       type: "helper",
       section: helperSection,
       detail: "Function",
-      apply: (view, completion, from, to) => {
+      apply: (
-        insertBinding(view, from, to, key, mode)
+        view: any,
+        completion: BindingCompletion,
+        from: number,
+        to: number
+      ) => {
+        insertBinding(view, from, to, helperName, mode)
       },
+    }
   })
-    return acc
-  }, [])
 }
 
-export const getHelperCompletions = mode => {
+export const getHelperCompletions = (mode: {
-  const manifest = getManifest()
+  name: "javascript" | "handlebars"
-  return Object.keys(manifest).reduce((acc, key) => {
+}) => {
-    acc = acc || []
+  // TODO: manifest needs to be properly typed
-    return [...acc, ...helpersToCompletion(manifest[key], mode)]
+  const manifest: any = getManifest()
-  }, [])
+  return Object.keys(manifest).flatMap(key => {
+    return helpersToCompletion(manifest[key], mode)
+  })
 }
 
-export const snippetAutoComplete = snippets => {
+export const snippetAutoComplete = (snippets: Snippet[]) => {
-  return function myCompletions(context) {
+  return function myCompletions(context: CompletionContext) {
     if (!snippets?.length) {
       return null
     }
     const word = context.matchBefore(/\w*/)
-    if (word.from == word.to && !context.explicit) {
+    if (!word || (word.from == word.to && !context.explicit)) {
       return null
     }
     return {
@@ -117,7 +138,12 @@ export const snippetAutoComplete = snippets => {
         label: `snippets.${snippet.name}`,
         type: "text",
         simple: true,
-        apply: (view, completion, from, to) => {
+        apply: (
+          view: any,
+          completion: BindingCompletion,
+          from: number,
+          to: number
+        ) => {
           insertSnippet(view, from, to, completion.label)
         },
       })),
@@ -125,7 +151,7 @@ export const snippetAutoComplete = snippets => {
   }
 }
 
-const bindingFilter = (options, query) => {
+const bindingFilter = (options: BindingCompletion[], query: string) => {
   return options.filter(completion => {
     const section_parsed = completion.section.name.toLowerCase()
     const label_parsed = completion.label.toLowerCase()
@@ -138,8 +164,8 @@ const bindingFilter = (options, query) => {
   })
 }
 
-export const hbAutocomplete = baseCompletions => {
+export const hbAutocomplete = (baseCompletions: BindingCompletion[]) => {
-  async function coreCompletion(context) {
+  async function coreCompletion(context: CompletionContext) {
     let bindingStart = context.matchBefore(EditorModes.Handlebars.match)
 
     let options = baseCompletions || []
@@ -149,6 +175,9 @@ export const hbAutocomplete = baseCompletions => {
     }
     // Accommodate spaces
     const match = bindingStart.text.match(/{{[\s]*/)
+    if (!match) {
+      return null
+    }
     const query = bindingStart.text.replace(match[0], "")
     let filtered = bindingFilter(options, query)
 
@@ -162,14 +191,17 @@ export const hbAutocomplete = baseCompletions => {
   return coreCompletion
 }
 
-export const jsAutocomplete = baseCompletions => {
+export const jsAutocomplete = (baseCompletions: BindingCompletion[]) => {
-  async function coreCompletion(context) {
+  async function coreCompletion(context: CompletionContext) {
     let jsBinding = context.matchBefore(/\$\("[\s\w]*/)
     let options = baseCompletions || []
 
     if (jsBinding) {
       // Accommodate spaces
       const match = jsBinding.text.match(/\$\("[\s]*/)
+      if (!match) {
+        return null
+      }
       const query = jsBinding.text.replace(match[0], "")
       let filtered = bindingFilter(options, query)
       return {
@@ -185,7 +217,10 @@ export const jsAutocomplete = baseCompletions => {
   return coreCompletion
 }
 
-export const buildBindingInfoNode = (completion, binding) => {
+export const buildBindingInfoNode = (
+  completion: BindingCompletion,
+  binding: any
+) => {
   if (!binding.valueHTML || binding.value == null) {
     return null
   }
@@ -196,7 +231,12 @@ export const buildBindingInfoNode = (completion, binding) => {
 }
 
 // Readdress these methods. They shouldn't be used
-export const hbInsert = (value, from, to, text) => {
+export const hbInsert = (
+  value: string,
+  from: number,
+  to: number,
+  text: string
+) => {
   let parsedInsert = ""
 
   const left = from ? value.substring(0, from) : ""
@@ -212,11 +252,14 @@ export const hbInsert = (value, from, to, text) => {
 }
 
 export function jsInsert(
-  value,
+  value: string,
-  from,
+  from: number,
-  to,
+  to: number,
-  text,
+  text: string,
-  { helper, disableWrapping } = {}
+  {
+    helper,
+    disableWrapping,
+  }: { helper?: boolean; disableWrapping?: boolean } = {}
 ) {
   let parsedInsert = ""
 
@@ -236,7 +279,13 @@ export function jsInsert(
 }
 
 // Autocomplete apply behaviour
-export const insertBinding = (view, from, to, text, mode) => {
+export const insertBinding = (
+  view: any,
+  from: number,
+  to: number,
+  text: string,
+  mode: { name: "javascript" | "handlebars" }
+) => {
   let parsedInsert
 
   if (mode.name == "javascript") {
@@ -270,7 +319,12 @@ export const insertBinding = (view, from, to, text, mode) => {
   })
 }
 
-export const insertSnippet = (view, from, to, text) => {
+export const insertSnippet = (
+  view: any,
+  from: number,
+  to: number,
+  text: string
+) => {
   let cursorPos = from + text.length
   view.dispatch({
     changes: {
@@ -284,9 +338,13 @@ export const insertSnippet = (view, from, to, text) => {
   })
 }
 
-export const bindingsToCompletions = (bindings, mode) => {
+// TODO: typing in this function isn't great
+export const bindingsToCompletions = (
+  bindings: any,
+  mode: { name: "javascript" | "handlebars" }
+) => {
   const bindingByCategory = groupBy(bindings, "category")
-  const categoryMeta = bindings?.reduce((acc, ele) => {
+  const categoryMeta = bindings?.reduce((acc: any, ele: any) => {
     acc[ele.category] = acc[ele.category] || {}
 
     if (ele.icon) {
@@ -298,10 +356,12 @@ export const bindingsToCompletions = (bindings, mode) => {
     return acc
   }, {})
 
-  const completions = Object.keys(bindingByCategory).reduce((comps, catKey) => {
+  const completions = Object.keys(bindingByCategory).reduce(
+    (comps: any, catKey: string) => {
       const { icon, rank } = categoryMeta[catKey] || {}
 
-    const bindindSectionHeader = buildSectionHeader(
+      const bindingSectionHeader = buildSectionHeader(
+        // @ts-ignore something wrong with this - logically this should be dictionary
         bindingByCategory.type,
         catKey,
         icon || "",
@@ -313,21 +373,29 @@ export const bindingsToCompletions = (bindings, mode) => {
         ...bindingByCategory[catKey].reduce((acc, binding) => {
           let displayType = binding.fieldSchema?.type || binding.display?.type
           acc.push({
-          label: binding.display?.name || binding.readableBinding || "NO NAME",
+            label:
-          info: completion => {
+              binding.display?.name || binding.readableBinding || "NO NAME",
+            info: (completion: BindingCompletion) => {
               return buildBindingInfoNode(completion, binding)
             },
             type: "binding",
             detail: displayType,
-          section: bindindSectionHeader,
+            section: bindingSectionHeader,
-          apply: (view, completion, from, to) => {
+            apply: (
+              view: any,
+              completion: BindingCompletion,
+              from: number,
+              to: number
+            ) => {
               insertBinding(view, from, to, binding.readableBinding, mode)
             },
           })
           return acc
         }, []),
       ]
-  }, [])
+    },
+    []
+  )
 
   return completions
 }

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -1,4 +1,4 @@
-<script>
+<script lang="ts">
   import {
     DrawerContent,
     ActionButton,
@@ -12,7 +12,7 @@
     decodeJSBinding,
     encodeJSBinding,
     processObjectSync,
-    processStringSync,
+    processStringWithLogsSync,
   } from "@budibase/string-templates"
   import { readableToRuntimeBinding } from "@/dataBinding"
   import CodeEditor from "../CodeEditor/CodeEditor.svelte"
@@ -28,45 +28,47 @@
   import EvaluationSidePanel from "./EvaluationSidePanel.svelte"
   import SnippetSidePanel from "./SnippetSidePanel.svelte"
   import { BindingHelpers } from "./utils"
-  import formatHighlight from "json-format-highlight"
   import { capitalise } from "@/helpers"
-  import { Utils } from "@budibase/frontend-core"
+  import { Utils, JsonFormatter } from "@budibase/frontend-core"
   import { licensing } from "@/stores/portal"
+  import { BindingMode, SidePanel } from "@budibase/types"
+  import type {
+    EnrichedBinding,
+    BindingCompletion,
+    Snippet,
+    Helper,
+    CaretPositionFn,
+    InsertAtPositionFn,
+    JSONValue,
+  } from "@budibase/types"
+  import type { Log } from "@budibase/string-templates"
+  import type { CompletionContext } from "@codemirror/autocomplete"
 
   const dispatch = createEventDispatcher()
 
-  export let bindings = []
+  export let bindings: EnrichedBinding[] = []
-  export let value = ""
+  export let value: string = ""
   export let allowHBS = true
   export let allowJS = false
   export let allowHelpers = true
   export let allowSnippets = true
   export let context = null
-  export let snippets = null
+  export let snippets: Snippet[] | null = null
   export let autofocusEditor = false
   export let placeholder = null
   export let showTabBar = true
 
-  const Modes = {
+  let mode: BindingMode | null
-    Text: "Text",
+  let sidePanel: SidePanel | null
-    JavaScript: "JavaScript",
-  }
-  const SidePanels = {
-    Bindings: "FlashOn",
-    Evaluation: "Play",
-    Snippets: "Code",
-  }
-
-  let mode
-  let sidePanel
   let initialValueJS = value?.startsWith?.("{{ js ")
-  let jsValue = initialValueJS ? value : null
+  let jsValue: string | null = initialValueJS ? value : null
-  let hbsValue = initialValueJS ? null : value
+  let hbsValue: string | null = initialValueJS ? null : value
-  let getCaretPosition
+  let getCaretPosition: CaretPositionFn | undefined
-  let insertAtPos
+  let insertAtPos: InsertAtPositionFn | undefined
-  let targetMode = null
+  let targetMode: BindingMode | null = null
-  let expressionResult
+  let expressionResult: string | undefined
-  let expressionError
+  let expressionLogs: Log[] | undefined
+  let expressionError: string | undefined
   let evaluating = false
 
   $: useSnippets = allowSnippets && !$licensing.isFreePlan
@@ -78,10 +80,12 @@
     mode
   )
   $: enrichedBindings = enrichBindings(bindings, context, snippets)
-  $: usingJS = mode === Modes.JavaScript
+  $: usingJS = mode === BindingMode.JavaScript
   $: editorMode =
-    mode === Modes.JavaScript ? EditorModes.JS : EditorModes.Handlebars
+    mode === BindingMode.JavaScript ? EditorModes.JS : EditorModes.Handlebars
-  $: editorValue = editorMode === EditorModes.JS ? jsValue : hbsValue
+  $: editorValue = (editorMode === EditorModes.JS ? jsValue : hbsValue) as
+    | string
+    | null
   $: runtimeExpression = readableToRuntimeBinding(enrichedBindings, value)
   $: requestEval(runtimeExpression, context, snippets)
   $: bindingCompletions = bindingsToCompletions(enrichedBindings, editorMode)
@@ -95,7 +99,7 @@
     }
   }
 
-  const getHBSCompletions = bindingCompletions => {
+  const getHBSCompletions = (bindingCompletions: BindingCompletion[]) => {
     return [
       hbAutocomplete([
         ...bindingCompletions,
@@ -104,48 +108,58 @@
     ]
   }
 
-  const getJSCompletions = (bindingCompletions, snippets, useSnippets) => {
+  const getJSCompletions = (
-    const completions = [
+    bindingCompletions: BindingCompletion[],
+    snippets: Snippet[] | null,
+    useSnippets?: boolean
+  ) => {
+    const completions: ((_: CompletionContext) => any)[] = [
       jsAutocomplete([
         ...bindingCompletions,
         ...getHelperCompletions(EditorModes.JS),
       ]),
     ]
-    if (useSnippets) {
+    if (useSnippets && snippets) {
       completions.push(snippetAutoComplete(snippets))
     }
     return completions
   }
 
-  const getModeOptions = (allowHBS, allowJS) => {
+  const getModeOptions = (allowHBS: boolean, allowJS: boolean) => {
     let options = []
     if (allowHBS) {
-      options.push(Modes.Text)
+      options.push(BindingMode.Text)
     }
     if (allowJS) {
-      options.push(Modes.JavaScript)
+      options.push(BindingMode.JavaScript)
     }
     return options
   }
 
-  const getSidePanelOptions = (bindings, context, useSnippets, mode) => {
+  const getSidePanelOptions = (
+    bindings: EnrichedBinding[],
+    context: any,
+    useSnippets: boolean,
+    mode: BindingMode | null
+  ) => {
     let options = []
     if (bindings?.length) {
-      options.push(SidePanels.Bindings)
+      options.push(SidePanel.Bindings)
     }
     if (context && Object.keys(context).length > 0) {
-      options.push(SidePanels.Evaluation)
+      options.push(SidePanel.Evaluation)
     }
-    if (useSnippets && mode === Modes.JavaScript) {
+    if (useSnippets && mode === BindingMode.JavaScript) {
-      options.push(SidePanels.Snippets)
+      options.push(SidePanel.Snippets)
     }
     return options
   }
 
-  const debouncedEval = Utils.debounce((expression, context, snippets) => {
+  const debouncedEval = Utils.debounce(
+    (expression: string | null, context: any, snippets: Snippet[]) => {
       try {
-      expressionError = null
+        expressionError = undefined
-      expressionResult = processStringSync(
+        const output = processStringWithLogsSync(
           expression || "",
           {
             ...context,
@@ -155,20 +169,28 @@
             noThrow: false,
           }
         )
-    } catch (err) {
+        expressionResult = output.result
-      expressionResult = null
+        expressionLogs = output.logs
+      } catch (err: any) {
+        expressionResult = undefined
         expressionError = err
       }
       evaluating = false
-  }, 260)
+    },
+    260
+  )
 
-  const requestEval = (expression, context, snippets) => {
+  const requestEval = (
+    expression: string | null,
+    context: any,
+    snippets: Snippet[] | null
+  ) => {
     evaluating = true
     debouncedEval(expression, context, snippets)
   }
 
-  const highlightJSON = json => {
+  const highlightJSON = (json: JSONValue) => {
-    return formatHighlight(json, {
+    return JsonFormatter.format(json, {
       keyColor: "#e06c75",
       numberColor: "#e5c07b",
       stringColor: "#98c379",
@@ -178,7 +200,11 @@
     })
   }
 
-  const enrichBindings = (bindings, context, snippets) => {
+  const enrichBindings = (
+    bindings: EnrichedBinding[],
+    context: any,
+    snippets: Snippet[] | null
+  ) => {
     // Create a single big array to enrich in one go
     const bindingStrings = bindings.map(binding => {
       if (binding.runtimeBinding.startsWith('trim "')) {
@@ -189,17 +215,18 @@
         return `{{ literal ${binding.runtimeBinding} }}`
       }
     })
-    const bindingEvauations = processObjectSync(bindingStrings, {
+    const bindingEvaluations = processObjectSync(bindingStrings, {
       ...context,
       snippets,
     })
 
     // Enrich bindings with evaluations and highlighted HTML
     return bindings.map((binding, idx) => {
-      if (!context) {
+      if (!context || typeof bindingEvaluations !== "object") {
         return binding
       }
-      const value = JSON.stringify(bindingEvauations[idx], null, 2)
+      const evalObj: Record<any, any> = bindingEvaluations
+      const value = JSON.stringify(evalObj[idx], null, 2)
       return {
         ...binding,
         value,
@@ -208,29 +235,38 @@
     })
   }
 
-  const updateValue = val => {
+  const updateValue = (val: any) => {
     const runtimeExpression = readableToRuntimeBinding(enrichedBindings, val)
     dispatch("change", val)
     requestEval(runtimeExpression, context, snippets)
   }
 
-  const onSelectHelper = (helper, js) => {
+  const onSelectHelper = (helper: Helper, js?: boolean) => {
-    bindingHelpers.onSelectHelper(js ? jsValue : hbsValue, helper, { js })
+    bindingHelpers.onSelectHelper(js ? jsValue : hbsValue, helper, {
+      js,
+      dontDecode: undefined,
+    })
   }
 
-  const onSelectBinding = (binding, { forceJS } = {}) => {
+  const onSelectBinding = (
+    binding: EnrichedBinding,
+    { forceJS }: { forceJS?: boolean } = {}
+  ) => {
     const js = usingJS || forceJS
-    bindingHelpers.onSelectBinding(js ? jsValue : hbsValue, binding, { js })
+    bindingHelpers.onSelectBinding(js ? jsValue : hbsValue, binding, {
+      js,
+      dontDecode: undefined,
+    })
   }
 
-  const changeMode = newMode => {
+  const changeMode = (newMode: BindingMode) => {
     if (targetMode || newMode === mode) {
       return
     }
 
     // Get the raw editor value to see if we are abandoning changes
     let rawValue = editorValue
-    if (mode === Modes.JavaScript) {
+    if (mode === BindingMode.JavaScript && rawValue) {
       rawValue = decodeJSBinding(rawValue)
     }
 
@@ -249,16 +285,16 @@
     targetMode = null
   }
 
-  const changeSidePanel = newSidePanel => {
+  const changeSidePanel = (newSidePanel: SidePanel) => {
     sidePanel = newSidePanel === sidePanel ? null : newSidePanel
   }
 
-  const onChangeHBSValue = e => {
+  const onChangeHBSValue = (e: { detail: string }) => {
     hbsValue = e.detail
     updateValue(hbsValue)
   }
 
-  const onChangeJSValue = e => {
+  const onChangeJSValue = (e: { detail: string }) => {
     jsValue = encodeJSBinding(e.detail)
     if (!e.detail?.trim()) {
       // Don't bother saving empty values as JS
@@ -268,9 +304,14 @@
     }
   }
 
+  const addSnippet = (snippet: Snippet) =>
+    bindingHelpers.onSelectSnippet(snippet)
+
   onMount(() => {
     // Set the initial mode appropriately
-    const initialValueMode = initialValueJS ? Modes.JavaScript : Modes.Text
+    const initialValueMode = initialValueJS
+      ? BindingMode.JavaScript
+      : BindingMode.Text
     if (editorModeOptions.includes(initialValueMode)) {
       mode = initialValueMode
     } else {
@@ -314,7 +355,7 @@
         </div>
       {/if}
       <div class="editor">
-        {#if mode === Modes.Text}
+        {#if mode === BindingMode.Text}
           {#key hbsCompletions}
             <CodeEditor
               value={hbsValue}
@@ -328,10 +369,10 @@
               jsBindingWrapping={false}
             />
           {/key}
-        {:else if mode === Modes.JavaScript}
+        {:else if mode === BindingMode.JavaScript}
           {#key jsCompletions}
             <CodeEditor
-              value={decodeJSBinding(jsValue)}
+              value={jsValue ? decodeJSBinding(jsValue) : jsValue}
               on:change={onChangeJSValue}
               completions={jsCompletions}
               mode={EditorModes.JS}
@@ -371,7 +412,7 @@
       </div>
     </div>
     <div class="side" class:visible={!!sidePanel}>
-      {#if sidePanel === SidePanels.Bindings}
+      {#if sidePanel === SidePanel.Bindings}
         <BindingSidePanel
           bindings={enrichedBindings}
           {allowHelpers}
@@ -380,18 +421,16 @@
           addBinding={onSelectBinding}
           mode={editorMode}
         />
-      {:else if sidePanel === SidePanels.Evaluation}
+      {:else if sidePanel === SidePanel.Evaluation}
         <EvaluationSidePanel
           {expressionResult}
           {expressionError}
+          {expressionLogs}
           {evaluating}
-          expression={editorValue}
+          expression={editorValue ? editorValue : ""}
-        />
-      {:else if sidePanel === SidePanels.Snippets}
-        <SnippetSidePanel
-          addSnippet={snippet => bindingHelpers.onSelectSnippet(snippet)}
-          {snippets}
         />
+      {:else if sidePanel === SidePanel.Snippets}
+        <SnippetSidePanel {addSnippet} {snippets} />
       {/if}
     </div>
   </div>

packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte

@@ -1,40 +1,50 @@
-<script>
+<script lang="ts">
-  import formatHighlight from "json-format-highlight"
+  import { JsonFormatter } from "@budibase/frontend-core"
   import { Icon, ProgressCircle, notifications } from "@budibase/bbui"
-  import { copyToClipboard } from "@budibase/bbui/helpers"
+  import { Helpers } from "@budibase/bbui"
   import { fade } from "svelte/transition"
   import { UserScriptError } from "@budibase/string-templates"
+  import type { Log } from "@budibase/string-templates"
+  import type { JSONValue } from "@budibase/types"
 
-  export let expressionResult
+  // this can be essentially any primitive response from the JS function
-  export let expressionError
+  export let expressionResult: JSONValue | undefined = undefined
+  export let expressionError: string | undefined = undefined
+  export let expressionLogs: Log[] = []
   export let evaluating = false
-  export let expression = null
+  export let expression: string | null = null
 
   $: error = expressionError != null
   $: empty = expression == null || expression?.trim() === ""
   $: success = !error && !empty
   $: highlightedResult = highlight(expressionResult)
+  $: highlightedLogs = expressionLogs.map(l => ({
+    log: highlight(l.log.join(", ")),
+    line: l.line,
+    type: l.type,
+  }))
 
-  const formatError = err => {
+  const formatError = (err: any) => {
     if (err.code === UserScriptError.code) {
       return err.userScriptError.toString()
     }
     return err.toString()
   }
 
-  const highlight = json => {
+  // json can be any primitive type
+  const highlight = (json?: JSONValue | null) => {
     if (json == null) {
       return ""
     }
 
     // Attempt to parse and then stringify, in case this is valid result
     try {
-      json = JSON.stringify(JSON.parse(json), null, 2)
+      json = JSON.stringify(JSON.parse(json as any), null, 2)
     } catch (err) {
-      // Ignore
+      // couldn't parse/stringify, just treat it as the raw input
     }
 
-    return formatHighlight(json, {
+    return JsonFormatter.format(json, {
       keyColor: "#e06c75",
       numberColor: "#e5c07b",
       stringColor: "#98c379",
@@ -45,11 +55,11 @@
   }
 
   const copy = () => {
-    let clipboardVal = expressionResult.result
+    let clipboardVal = expressionResult
     if (typeof clipboardVal === "object") {
       clipboardVal = JSON.stringify(clipboardVal, null, 2)
     }
-    copyToClipboard(clipboardVal)
+    Helpers.copyToClipboard(clipboardVal)
     notifications.success("Value copied to clipboard")
   }
 </script>
@@ -58,7 +68,7 @@
   <div class="header" class:success class:error>
     <div class="header-content">
       {#if error}
-        <Icon name="Alert" color="var(--spectrum-global-color-red-600)" />
+        <Icon name="Alert" color="var(--error-content)" />
         <div>Error</div>
         {#if evaluating}
           <div transition:fade|local={{ duration: 130 }}>
@@ -87,8 +97,36 @@
     {:else if error}
       {formatError(expressionError)}
     {:else}
+      <div class="output-lines">
+        {#each highlightedLogs as logLine}
+          <div
+            class="line"
+            class:error-log={logLine.type === "error"}
+            class:warn-log={logLine.type === "warn"}
+          >
+            <div class="icon-log">
+              {#if logLine.type === "error"}
+                <Icon
+                  size="XS"
+                  name="CloseCircle"
+                  color="var(--error-content)"
+                />
+              {:else if logLine.type === "warn"}
+                <Icon size="XS" name="Alert" color="var(--warning-content)" />
+              {/if}
+              <!-- eslint-disable-next-line svelte/no-at-html-tags-->
+              <span>{@html logLine.log}</span>
+            </div>
+            {#if logLine.line}
+              <span style="color: var(--blue)">:{logLine.line}</span>
+            {/if}
+          </div>
+        {/each}
+        <div class="line">
           <!-- eslint-disable-next-line svelte/no-at-html-tags-->
           {@html highlightedResult}
+        </div>
+      </div>
     {/if}
   </div>
 </div>
@@ -127,20 +165,37 @@
     height: 100%;
     z-index: 1;
     position: absolute;
-    opacity: 10%;
   }
   .header.error::before {
-    background: var(--spectrum-global-color-red-400);
+    background: var(--error-bg);
   }
   .body {
     flex: 1 1 auto;
     padding: var(--spacing-m) var(--spacing-l);
     font-family: var(--font-mono);
     font-size: 12px;
-    overflow-y: scroll;
+    overflow-y: auto;
     overflow-x: hidden;
-    white-space: pre-wrap;
+    white-space: pre-line;
     word-wrap: break-word;
     height: 0;
   }
+  .output-lines {
+    display: flex;
+    flex-direction: column;
+    gap: var(--spacing-xs);
+  }
+  .line {
+    border-bottom: var(--border-light);
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+    align-items: end;
+    padding: var(--spacing-s);
+  }
+  .icon-log {
+    display: flex;
+    gap: var(--spacing-s);
+    align-items: start;
+  }
 </style>

packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte

@@ -1,4 +1,5 @@
 <script>
+  import { datasources } from "@/stores/builder"
   import { Divider, Heading } from "@budibase/bbui"
 
   export let dividerState
@@ -6,6 +7,8 @@
   export let dataSet
   export let value
   export let onSelect
+
+  $: displayDatasourceName = $datasources.list.length > 1
 </script>
 
 {#if dividerState}
@@ -21,7 +24,7 @@
   {#each dataSet as data}
     <li
       class="spectrum-Menu-item"
-      class:is-selected={value?.label === data.label &&
+      class:is-selected={value?.resourceId === data.resourceId &&
         value?.type === data.type}
       role="option"
       aria-selected="true"
@@ -29,7 +32,9 @@
       on:click={() => onSelect(data)}
     >
       <span class="spectrum-Menu-itemLabel">
-        {data.datasourceName ? `${data.datasourceName} - ` : ""}{data.label}
+        {data.datasourceName && displayDatasourceName
+          ? `${data.datasourceName} - `
+          : ""}{data.label}
       </span>
       <svg
         class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"

packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte

@@ -34,7 +34,7 @@
   import ClientBindingPanel from "@/components/common/bindings/ClientBindingPanel.svelte"
   import DataSourceCategory from "@/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte"
   import { API } from "@/api"
-  import { datasourceSelect as format } from "@/helpers/data/format"
+  import { sortAndFormat } from "@/helpers/data/format"
 
   export let value = {}
   export let otherSources
@@ -51,25 +51,13 @@
   let modal
 
   $: text = value?.label ?? "Choose an option"
-  $: tables = $tablesStore.list
+  $: tables = sortAndFormat.tables($tablesStore.list, $datasources.list)
-    .map(table => format.table(table, $datasources.list))
-    .sort((a, b) => {
-      // sort tables alphabetically, grouped by datasource
-      const dsA = a.datasourceName ?? ""
-      const dsB = b.datasourceName ?? ""
-
-      const dsComparison = dsA.localeCompare(dsB)
-      if (dsComparison !== 0) {
-        return dsComparison
-      }
-      return a.label.localeCompare(b.label)
-    })
   $: viewsV1 = $viewsStore.list.map(view => ({
     ...view,
     label: view.name,
     type: "view",
   }))
-  $: viewsV2 = $viewsV2Store.list.map(format.viewV2)
+  $: viewsV2 = sortAndFormat.viewsV2($viewsV2Store.list, $datasources.list)
   $: views = [...(viewsV1 || []), ...(viewsV2 || [])]
   $: queries = $queriesStore.list
     .filter(q => showAllQueries || q.queryVerb === "read" || q.readable)

packages/builder/src/components/design/settings/controls/TableSelect.svelte

@@ -1,22 +1,32 @@
 <script>
-  import { Select } from "@budibase/bbui"
+  import { Popover, Select } from "@budibase/bbui"
   import { createEventDispatcher, onMount } from "svelte"
-  import { tables as tablesStore, viewsV2 } from "@/stores/builder"
+  import {
-  import { tableSelect as format } from "@/helpers/data/format"
+    tables as tableStore,
+    datasources as datasourceStore,
+    viewsV2 as viewsV2Store,
+  } from "@/stores/builder"
+  import DataSourceCategory from "./DataSourceSelect/DataSourceCategory.svelte"
+  import { sortAndFormat } from "@/helpers/data/format"
 
   export let value
 
+  let anchorRight, dropdownRight
+
   const dispatch = createEventDispatcher()
 
-  $: tables = $tablesStore.list.map(format.table)
+  $: tables = sortAndFormat.tables($tableStore.list, $datasourceStore.list)
-  $: views = $viewsV2.list.map(format.viewV2)
+  $: views = sortAndFormat.viewsV2($viewsV2Store.list, $datasourceStore.list)
   $: options = [...(tables || []), ...(views || [])]
 
+  $: text = value?.label ?? "Choose an option"
+
   const onChange = e => {
     dispatch(
       "change",
-      options.find(x => x.resourceId === e.detail)
+      options.find(x => x.resourceId === e.resourceId)
     )
+    dropdownRight.hide()
   }
 
   onMount(() => {
@@ -29,10 +39,47 @@
   })
 </script>
 
+<div class="container" bind:this={anchorRight}>
   <Select
-  on:change={onChange}
+    readonly
-  value={value?.resourceId}
+    value={text}
-  {options}
+    options={[text]}
-  getOptionValue={x => x.resourceId}
+    on:click={dropdownRight.show}
-  getOptionLabel={x => x.label}
   />
+</div>
+<Popover bind:this={dropdownRight} anchor={anchorRight}>
+  <div class="dropdown">
+    <DataSourceCategory
+      heading="Tables"
+      dataSet={tables}
+      {value}
+      onSelect={onChange}
+    />
+    {#if views?.length}
+      <DataSourceCategory
+        dividerState={true}
+        heading="Views"
+        dataSet={views}
+        {value}
+        onSelect={onChange}
+      />
+    {/if}
+  </div>
+</Popover>
+
+<style>
+  .container {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+  }
+  .container :global(:first-child) {
+    flex: 1 1 auto;
+  }
+
+  .dropdown {
+    padding: var(--spacing-m) 0;
+    z-index: 99999999;
+  }
+</style>

packages/builder/src/helpers/data/format.js

@@ -9,11 +9,18 @@ export const datasourceSelect = {
       datasourceName: datasource?.name,
     }
   },
-  viewV2: view => ({
+  viewV2: (view, datasources) => {
+    const datasource = datasources
+      .filter(f => f.entities)
+      .flatMap(d => d.entities)
+      .find(ds => ds._id === view.tableId)
+    return {
       ...view,
       label: view.name,
       type: "viewV2",
-  }),
+      datasourceName: datasource?.name,
+    }
+  },
 }
 
 export const tableSelect = {
@@ -31,3 +38,36 @@ export const tableSelect = {
     resourceId: view.id,
   }),
 }
+
+export const sortAndFormat = {
+  tables: (tables, datasources) => {
+    return tables
+      .map(table => {
+        const formatted = datasourceSelect.table(table, datasources)
+        return {
+          ...formatted,
+          resourceId: table._id,
+        }
+      })
+      .sort((a, b) => {
+        // sort tables alphabetically, grouped by datasource
+        const dsA = a.datasourceName ?? ""
+        const dsB = b.datasourceName ?? ""
+
+        const dsComparison = dsA.localeCompare(dsB)
+        if (dsComparison !== 0) {
+          return dsComparison
+        }
+        return a.label.localeCompare(b.label)
+      })
+  },
+  viewsV2: (views, datasources) => {
+    return views.map(view => {
+      const formatted = datasourceSelect.viewV2(view, datasources)
+      return {
+        ...formatted,
+        resourceId: view.id,
+      }
+    })
+  },
+}

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte

@@ -1,6 +1,6 @@
 <script>
   import { viewsV2, rowActions } from "@/stores/builder"
-  import { admin, themeStore, featureFlags } from "@/stores/portal"
+  import { admin, themeStore, licensing } from "@/stores/portal"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "@/api"
   import { notifications } from "@budibase/bbui"
@@ -53,7 +53,7 @@
   {buttons}
   allowAddRows
   allowDeleteRows
-  aiEnabled={$featureFlags.BUDIBASE_AI || $featureFlags.AI_CUSTOM_CONFIGS}
+  aiEnabled={$licensing.customAIConfigsEnabled || $licensing.budibaseAiEnabled}
   showAvatars={false}
   on:updatedatasource={handleGridViewUpdate}
   isCloud={$admin.cloud}

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte

@@ -8,7 +8,7 @@
     rowActions,
     roles,
   } from "@/stores/builder"
-  import { themeStore, admin, featureFlags } from "@/stores/portal"
+  import { themeStore, admin, licensing } from "@/stores/portal"
   import { TableNames } from "@/constants"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "@/api"
@@ -130,7 +130,8 @@
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
     isCloud={$admin.cloud}
-    aiEnabled={$featureFlags.BUDIBASE_AI || $featureFlags.AI_CUSTOM_CONFIGS}
+    aiEnabled={$licensing.customAIConfigsEnabled ||
+      $licensing.budibaseAIEnabled}
     {buttons}
     buttonsCollapsed
     on:updatedatasource={handleGridTableUpdate}

packages/builder/src/pages/builder/apps/index.svelte

@@ -15,6 +15,7 @@
   import {
     appsStore,
     organisation,
+    admin,
     auth,
     groups,
     licensing,
@@ -42,6 +43,7 @@
     app => app.status === AppStatus.DEPLOYED
   )
   $: userApps = getUserApps(publishedApps, userGroups, $auth.user)
+  $: isOwner = $auth.accountPortalAccess && $admin.cloud
 
   function getUserApps(publishedApps, userGroups, user) {
     if (sdk.users.isAdmin(user)) {
@@ -111,7 +113,13 @@
               </MenuItem>
               <MenuItem
                 icon="LockClosed"
-                on:click={() => changePasswordModal.show()}
+                on:click={() => {
+                  if (isOwner) {
+                    window.location.href = `${$admin.accountPortalUrl}/portal/account`
+                  } else {
+                    changePasswordModal.show()
+                  }
+                }}
               >
                 Update password
               </MenuItem>

packages/builder/src/pages/builder/auth/reset.svelte

@@ -30,10 +30,16 @@
     try {
       loading = true
       if (forceResetPassword) {
+        const email = $auth.user.email
+        const tenantId = $auth.user.tenantId
         await auth.updateSelf({
           password,
           forceResetPassword: false,
         })
+        if (!$auth.user) {
+          // Update self will clear the platform user, so need to login
+          await auth.login(email, password, tenantId)
+        }
         $goto("../portal/")
       } else {
         await auth.resetPassword(password, resetCode)

packages/builder/src/pages/builder/portal/_components/UserDropdown.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { auth } from "@/stores/portal"
+  import { admin, auth } from "@/stores/portal"
   import { ActionMenu, MenuItem, Icon, Modal } from "@budibase/bbui"
   import { goto } from "@roxi/routify"
   import ProfileModal from "@/components/settings/ProfileModal.svelte"
@@ -13,6 +13,8 @@
   let updatePasswordModal
   let apiKeyModal
 
+  $: isOwner = $auth.accountPortalAccess && $admin.cloud
+
   const logout = async () => {
     try {
       await auth.logout()
@@ -32,7 +34,16 @@
   </MenuItem>
   <MenuItem icon="Moon" on:click={() => themeModal.show()}>Theme</MenuItem>
   {#if !$auth.isSSO}
-    <MenuItem icon="LockClosed" on:click={() => updatePasswordModal.show()}>
+    <MenuItem
+      icon="LockClosed"
+      on:click={() => {
+        if (isOwner) {
+          window.location.href = `${$admin.accountPortalUrl}/portal/account`
+        } else {
+          updatePasswordModal.show()
+        }
+      }}
+    >
       Update password
     </MenuItem>
   {/if}

packages/builder/src/pages/builder/portal/settings/ai/index.svelte

@@ -12,7 +12,7 @@
     Tags,
     Tag,
   } from "@budibase/bbui"
-  import { admin, licensing, featureFlags } from "@/stores/portal"
+  import { admin, licensing } from "@/stores/portal"
   import { API } from "@/api"
   import AIConfigModal from "./ConfigModal.svelte"
   import AIConfigTile from "./AIConfigTile.svelte"
@@ -27,8 +27,7 @@
   let editingUuid
 
   $: isCloud = $admin.cloud
-  $: customAIConfigsEnabled =
+  $: customAIConfigsEnabled = $licensing.customAIConfigsEnabled
-    $featureFlags.AI_CUSTOM_CONFIGS && $licensing.customAIConfigsEnabled
 
   async function fetchAIConfig() {
     try {

packages/builder/src/pages/builder/portal/settings/index.svelte

@@ -1,10 +1,5 @@
 <script>
   import { redirect } from "@roxi/routify"
-  import { featureFlags } from "@/stores/portal"
 
-  if ($featureFlags.AI_CUSTOM_CONFIGS) {
   $redirect("./ai")
-  } else {
-    $redirect("./auth")
-  }
 </script>

packages/builder/src/stores/portal/auth.ts

@@ -121,8 +121,8 @@ class AuthStore extends BudiStore<PortalAuthStore> {
     }
   }
 
-  async login(username: string, password: string) {
+  async login(username: string, password: string, targetTenantId?: string) {
-    const tenantId = get(this.store).tenantId
+    const tenantId = targetTenantId || get(this.store).tenantId
     await API.logIn(tenantId, username, password)
     await this.getSelf()
   }

packages/client/src/components/app/DataProvider.svelte

@@ -1,22 +1,38 @@
-<script>
+<script lang="ts">
   import { getContext } from "svelte"
   import { Pagination, ProgressCircle } from "@budibase/bbui"
   import { fetchData, QueryUtils } from "@budibase/frontend-core"
-  import { LogicalOperator, EmptyFilterOption } from "@budibase/types"
+  import {
+    LogicalOperator,
+    EmptyFilterOption,
+    TableSchema,
+    SortOrder,
+    SearchFilters,
+    UISearchFilter,
+    DataFetchDatasource,
+    UserDatasource,
+    GroupUserDatasource,
+    DataFetchOptions,
+  } from "@budibase/types"
 
-  export let dataSource
+  type ProviderDatasource = Exclude<
-  export let filter
+    DataFetchDatasource,
-  export let sortColumn
+    UserDatasource | GroupUserDatasource
-  export let sortOrder
+  >
-  export let limit
+
-  export let paginate
+  export let dataSource: ProviderDatasource
-  export let autoRefresh
+  export let filter: UISearchFilter
+  export let sortColumn: string
+  export let sortOrder: SortOrder
+  export let limit: number
+  export let paginate: boolean
+  export let autoRefresh: number
 
   const { styleable, Provider, ActionTypes, API } = getContext("sdk")
   const component = getContext("component")
 
-  let interval
+  let interval: ReturnType<typeof setInterval>
-  let queryExtensions = {}
+  let queryExtensions: Record<string, any> = {}
 
   $: defaultQuery = QueryUtils.buildQuery(filter)
 
@@ -49,8 +65,14 @@
     },
     {
       type: ActionTypes.SetDataProviderSorting,
-      callback: ({ column, order }) => {
+      callback: ({
-        let newOptions = {}
+        column,
+        order,
+      }: {
+        column: string
+        order: SortOrder | undefined
+      }) => {
+        let newOptions: Partial<DataFetchOptions> = {}
         if (column) {
           newOptions.sortColumn = column
         }
@@ -63,6 +85,7 @@
       },
     },
   ]
+
   $: dataContext = {
     rows: $fetch.rows,
     info: $fetch.info,
@@ -75,14 +98,12 @@
     id: $component?.id,
     state: {
       query: $fetch.query,
-      sortColumn: $fetch.sortColumn,
-      sortOrder: $fetch.sortOrder,
     },
     limit,
-    primaryDisplay: $fetch.definition?.primaryDisplay,
+    primaryDisplay: ($fetch.definition as any)?.primaryDisplay,
   }
 
-  const createFetch = datasource => {
+  const createFetch = (datasource: ProviderDatasource) => {
     return fetchData({
       API,
       datasource,
@@ -96,7 +117,7 @@
     })
   }
 
-  const sanitizeSchema = schema => {
+  const sanitizeSchema = (schema: TableSchema | null) => {
     if (!schema) {
       return schema
     }
@@ -109,14 +130,14 @@
     return cloned
   }
 
-  const addQueryExtension = (key, extension) => {
+  const addQueryExtension = (key: string, extension: any) => {
     if (!key || !extension) {
       return
     }
     queryExtensions = { ...queryExtensions, [key]: extension }
   }
 
-  const removeQueryExtension = key => {
+  const removeQueryExtension = (key: string) => {
     if (!key) {
       return
     }
@@ -125,11 +146,14 @@
     queryExtensions = newQueryExtensions
   }
 
-  const extendQuery = (defaultQuery, extensions) => {
+  const extendQuery = (
+    defaultQuery: SearchFilters,
+    extensions: Record<string, any>
+  ): SearchFilters => {
     if (!Object.keys(extensions).length) {
       return defaultQuery
     }
-    const extended = {
+    const extended: SearchFilters = {
       [LogicalOperator.AND]: {
         conditions: [
           ...(defaultQuery ? [defaultQuery] : []),
@@ -140,12 +164,12 @@
     }
 
     // If there are no conditions applied at all, clear the request.
-    return extended[LogicalOperator.AND]?.conditions?.length > 0
+    return (extended[LogicalOperator.AND]?.conditions?.length ?? 0) > 0
       ? extended
-      : null
+      : {}
   }
 
-  const setUpAutoRefresh = autoRefresh => {
+  const setUpAutoRefresh = (autoRefresh: number) => {
     clearInterval(interval)
     if (autoRefresh) {
       interval = setInterval(fetch.refresh, Math.max(10000, autoRefresh * 1000))

packages/client/src/components/app/blocks/form/FormBlock.svelte

@@ -1,37 +1,43 @@
-<script>
+<script lang="ts">
   import { getContext } from "svelte"
   import InnerFormBlock from "./InnerFormBlock.svelte"
   import { Utils } from "@budibase/frontend-core"
   import FormBlockWrapper from "./FormBlockWrapper.svelte"
   import { get } from "svelte/store"
+  import { TableSchema, UIDatasource } from "@budibase/types"
 
-  export let actionType
+  type Field = { name: string; active: boolean }
-  export let dataSource
+
-  export let size
+  export let actionType: string
-  export let disabled
+  export let dataSource: UIDatasource
-  export let fields
+  export let size: string
-  export let buttons
+  export let disabled: boolean
-  export let buttonPosition
+  export let fields: (Field | string)[]
-  export let title
+  export let buttons: {
-  export let description
+    "##eventHandlerType": string
-  export let rowId
+    parameters: Record<string, string>
-  export let actionUrl
+  }[]
-  export let noRowsMessage
+  export let buttonPosition: "top" | "bottom"
-  export let notificationOverride
+  export let title: string
-  export let buttonsCollapsed
+  export let description: string
-  export let buttonsCollapsedText
+  export let rowId: string
+  export let actionUrl: string
+  export let noRowsMessage: string
+  export let notificationOverride: boolean
+  export let buttonsCollapsed: boolean
+  export let buttonsCollapsedText: string
 
   // Legacy
-  export let showDeleteButton
+  export let showDeleteButton: boolean
-  export let showSaveButton
+  export let showSaveButton: boolean
-  export let saveButtonLabel
+  export let saveButtonLabel: boolean
-  export let deleteButtonLabel
+  export let deleteButtonLabel: boolean
 
   const { fetchDatasourceSchema, generateGoldenSample } = getContext("sdk")
   const component = getContext("component")
   const context = getContext("context")
 
-  let schema
+  let schema: TableSchema
 
   $: fetchSchema(dataSource)
   $: id = $component.id
@@ -61,7 +67,7 @@
     }
   }
 
-  const convertOldFieldFormat = fields => {
+  const convertOldFieldFormat = (fields: (Field | string)[]): Field[] => {
     if (!fields) {
       return []
     }
@@ -82,11 +88,11 @@
     })
   }
 
-  const getDefaultFields = (fields, schema) => {
+  const getDefaultFields = (fields: Field[], schema: TableSchema) => {
     if (!schema) {
       return []
     }
-    let defaultFields = []
+    let defaultFields: Field[] = []
 
     if (!fields || fields.length === 0) {
       Object.values(schema)
@@ -101,15 +107,14 @@
     return [...fields, ...defaultFields].filter(field => field.active)
   }
 
-  const fetchSchema = async () => {
+  const fetchSchema = async (datasource: UIDatasource) => {
-    schema = (await fetchDatasourceSchema(dataSource)) || {}
+    schema = (await fetchDatasourceSchema(datasource)) || {}
   }
 </script>
 
 <FormBlockWrapper {actionType} {dataSource} {rowId} {noRowsMessage}>
   <InnerFormBlock
     {dataSource}
-    {actionUrl}
     {actionType}
     {size}
     {disabled}
@@ -117,7 +122,6 @@
     {title}
     {description}
     {schema}
-    {notificationOverride}
     buttons={buttonsOrDefault}
     buttonPosition={buttons ? buttonPosition : "top"}
     {buttonsCollapsed}

packages/client/src/components/error-states/ComponentErrorState.svelte

@@ -1,11 +1,13 @@
-<script>
+<script lang="ts">
   import { getContext } from "svelte"
   import { Icon } from "@budibase/bbui"
   import MissingRequiredSetting from "./MissingRequiredSetting.svelte"
   import MissingRequiredAncestor from "./MissingRequiredAncestor.svelte"
 
-  export let missingRequiredSettings
+  export let missingRequiredSettings:
-  export let missingRequiredAncestors
+    | { key: string; label: string }[]
+    | undefined
+  export let missingRequiredAncestors: string[] | undefined
 
   const component = getContext("component")
   const { styleable, builderStore } = getContext("sdk")

packages/client/src/context.d.ts

@@ -0,0 +1,7 @@
+import { Component, Context, SDK } from "."
+
+declare module "svelte" {
+  export function getContext(key: "sdk"): SDK
+  export function getContext(key: "component"): Component
+  export function getContext(key: "context"): Context
+}

packages/client/src/index.ts

@@ -0,0 +1,23 @@
+import { APIClient } from "@budibase/frontend-core"
+import type { ActionTypes } from "./constants"
+import { Readable } from "svelte/store"
+
+export interface SDK {
+  API: APIClient
+  styleable: any
+  Provider: any
+  ActionTypes: typeof ActionTypes
+  fetchDatasourceSchema: any
+  generateGoldenSample: any
+  builderStore: Readable<{
+    inBuilder: boolean
+  }>
+}
+
+export type Component = Readable<{
+  id: string
+  styles: any
+  errorState: boolean
+}>
+
+export type Context = Readable<{}>

packages/client/src/stores/components.js

@@ -6,7 +6,7 @@ import { screenStore } from "./screens"
 import { builderStore } from "./builder"
 import Router from "../components/Router.svelte"
 import * as AppComponents from "../components/app/index.js"
-import { ScreenslotType } from "../constants.js"
+import { ScreenslotType } from "../constants"
 
 export const BudibasePrefix = "@budibase/standard-components/"
 

packages/client/src/utils/schema.ts

@@ -1,5 +1,6 @@
 import { API } from "api"
 import { DataFetchMap, DataFetchType } from "@budibase/frontend-core"
+import { FieldType, TableSchema } from "@budibase/types"
 
 /**
  * Constructs a fetch instance for a given datasource.
@@ -42,14 +43,14 @@ export const fetchDatasourceSchema = async <
   }
 
   // Get the normal schema as long as we aren't wanting a form schema
-  let schema: any
+  let schema: TableSchema | undefined
   if (datasource?.type !== "query" || !options?.formSchema) {
-    schema = instance.getSchema(definition as any)
+    schema = instance.getSchema(definition as any) as TableSchema
   } else if ("parameters" in definition && definition.parameters?.length) {
     schema = {}
-    definition.parameters.forEach(param => {
+    for (const param of definition.parameters) {
-      schema[param.name] = { ...param, type: "string" }
+      schema[param.name] = { ...param, type: FieldType.STRING }
-    })
+    }
   }
   if (!schema) {
     return null
@@ -57,11 +58,11 @@ export const fetchDatasourceSchema = async <
 
   // Strip hidden fields from views
   if (datasource.type === "viewV2") {
-    Object.keys(schema).forEach(field => {
+    for (const field of Object.keys(schema)) {
       if (!schema[field].visible) {
         delete schema[field]
       }
-    })
+    }
   }
 
   // Enrich schema with relationships if required

packages/frontend-core/src/api/migrations.ts

@@ -1,8 +1,8 @@
-import { GetOldMigrationStatus } from "@budibase/types"
+import { GetMigrationStatus } from "@budibase/types"
 import { BaseAPIClient } from "./types"
 
 export interface MigrationEndpoints {
-  getMigrationStatus: () => Promise<GetOldMigrationStatus>
+  getMigrationStatus: () => Promise<GetMigrationStatus>
 }
 
 export const buildMigrationEndpoints = (

packages/frontend-core/src/components/grid/stores/datasource.ts

@@ -1,7 +1,11 @@
 // TODO: datasource and defitions are unions of the different implementations. At this point, the datasource does not know what type is being used, and the assignations will cause TS exceptions. Casting it "as any" for now. This should be fixed improving the type usages.
 
 import { derived, get, Readable, Writable } from "svelte/store"
-import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch"
+import {
+  DataFetchDefinition,
+  getDatasourceDefinition,
+  getDatasourceSchema,
+} from "../../../fetch"
 import { enrichSchemaWithRelColumns, memo } from "../../../utils"
 import { cloneDeep } from "lodash"
 import {
@@ -18,7 +22,7 @@ import { Store as StoreContext, BaseStoreProps } from "."
 import { DatasourceActions } from "./datasources"
 
 interface DatasourceStore {
-  definition: Writable<UIDatasource | null>
+  definition: Writable<DataFetchDefinition | null>
   schemaMutations: Writable<Record<string, UIFieldMutation>>
   subSchemaMutations: Writable<Record<string, Record<string, UIFieldMutation>>>
 }
@@ -131,11 +135,17 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => {
     [datasource, definition],
     ([$datasource, $definition]) => {
       let type = $datasource?.type
+      // @ts-expect-error
       if (type === "provider") {
         type = ($datasource as any).value?.datasource?.type // TODO: see line 1
       }
       // Handle calculation views
-      if (type === "viewV2" && $definition?.type === ViewV2Type.CALCULATION) {
+      if (
+        type === "viewV2" &&
+        $definition &&
+        "type" in $definition &&
+        $definition.type === ViewV2Type.CALCULATION
+      ) {
         return false
       }
       return !!type && ["table", "viewV2", "link"].includes(type)
@@ -197,7 +207,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => {
   ) => {
     // Update local state
     const originalDefinition = get(definition)
-    definition.set(newDefinition as UIDatasource)
+    definition.set(newDefinition)
 
     // Update server
     if (get(config).canSaveSchema) {
@@ -225,6 +235,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => {
     // Update primary display
     newDefinition.primaryDisplay = column
 
+    if (newDefinition.schema) {
       // Sanitise schema to ensure field is required and has no default value
       if (!newDefinition.schema[column].constraints) {
         newDefinition.schema[column].constraints = {}
@@ -233,6 +244,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => {
       if ("default" in newDefinition.schema[column]) {
         delete newDefinition.schema[column].default
       }
+    }
     return await saveDefinition(newDefinition as any) // TODO: see line 1
   }
 

packages/frontend-core/src/components/grid/stores/datasources/table.ts

@@ -8,6 +8,7 @@ import {
 import { get } from "svelte/store"
 import { Store as StoreContext } from ".."
 import { DatasourceTableActions } from "."
+import TableFetch from "../../../../fetch/TableFetch"
 
 const SuppressErrors = true
 
@@ -119,7 +120,7 @@ export const initialise = (context: StoreContext) => {
     unsubscribers.push(
       allFilters.subscribe($allFilters => {
         // Ensure we're updating the correct fetch
-        const $fetch = get(fetch)
+        const $fetch = get(fetch) as TableFetch | null
         if ($fetch?.options?.datasource?.tableId !== $datasource.tableId) {
           return
         }
@@ -133,7 +134,7 @@ export const initialise = (context: StoreContext) => {
     unsubscribers.push(
       sort.subscribe($sort => {
         // Ensure we're updating the correct fetch
-        const $fetch = get(fetch)
+        const $fetch = get(fetch) as TableFetch | null
         if ($fetch?.options?.datasource?.tableId !== $datasource.tableId) {
           return
         }

packages/frontend-core/src/components/grid/stores/datasources/viewV2.ts

@@ -4,11 +4,11 @@ import {
   SaveRowRequest,
   SortOrder,
   UIDatasource,
-  UIView,
   UpdateViewRequest,
 } from "@budibase/types"
 import { Store as StoreContext } from ".."
 import { DatasourceViewActions } from "."
+import ViewV2Fetch from "../../../../fetch/ViewV2Fetch"
 
 const SuppressErrors = true
 
@@ -134,6 +134,9 @@ export const initialise = (context: StoreContext) => {
         if (!get(config).canSaveSchema) {
           return
         }
+        if (!$definition || !("id" in $definition)) {
+          return
+        }
         if ($definition?.id !== $datasource.id) {
           return
         }
@@ -184,7 +187,10 @@ export const initialise = (context: StoreContext) => {
     unsubscribers.push(
       sort.subscribe(async $sort => {
         // Ensure we're updating the correct view
-        const $view = get(definition) as UIView
+        const $view = get(definition)
+        if (!$view || !("id" in $view)) {
+          return
+        }
         if ($view?.id !== $datasource.id) {
           return
         }
@@ -207,7 +213,7 @@ export const initialise = (context: StoreContext) => {
 
         // Also update the fetch to ensure the new sort is respected.
         // Ensure we're updating the correct fetch.
-        const $fetch = get(fetch)
+        const $fetch = get(fetch) as ViewV2Fetch | null
         if ($fetch?.options?.datasource?.id !== $datasource.id) {
           return
         }
@@ -225,6 +231,9 @@ export const initialise = (context: StoreContext) => {
           return
         }
         const $view = get(definition)
+        if (!$view || !("id" in $view)) {
+          return
+        }
         if ($view?.id !== $datasource.id) {
           return
         }
@@ -246,7 +255,7 @@ export const initialise = (context: StoreContext) => {
         if (!get(config).canSaveSchema) {
           return
         }
-        const $fetch = get(fetch)
+        const $fetch = get(fetch) as ViewV2Fetch | null
         if ($fetch?.options?.datasource?.id !== $datasource.id) {
           return
         }
@@ -262,7 +271,7 @@ export const initialise = (context: StoreContext) => {
         if (get(config).canSaveSchema) {
           return
         }
-        const $fetch = get(fetch)
+        const $fetch = get(fetch) as ViewV2Fetch | null
         if ($fetch?.options?.datasource?.id !== $datasource.id) {
           return
         }

packages/frontend-core/src/components/grid/stores/rows.ts

@@ -1,5 +1,5 @@
 import { writable, derived, get, Writable, Readable } from "svelte/store"
-import { fetchData } from "../../../fetch"
+import { DataFetch, fetchData } from "../../../fetch"
 import { NewRowID, RowPageSize } from "../lib/constants"
 import {
   generateRowID,
@@ -13,7 +13,6 @@ import { sleep } from "../../../utils/utils"
 import { FieldType, Row, UIRow } from "@budibase/types"
 import { getRelatedTableValues } from "../../../utils"
 import { Store as StoreContext } from "."
-import DataFetch from "../../../fetch/DataFetch"
 
 interface IndexedUIRow extends UIRow {
   __idx: number
@@ -21,7 +20,7 @@ interface IndexedUIRow extends UIRow {
 
 interface RowStore {
   rows: Writable<UIRow[]>
-  fetch: Writable<DataFetch<any, any, any> | null> // TODO: type this properly, having a union of all the possible options
+  fetch: Writable<DataFetch | null>
   loaded: Writable<boolean>
   refreshing: Writable<boolean>
   loading: Writable<boolean>
@@ -254,7 +253,7 @@ export const createActions = (context: StoreContext): RowActionStore => {
 
         // Reset state properties when dataset changes
         if (!$instanceLoaded || resetRows) {
-          definition.set($fetch.definition as any) // TODO: datasource and defitions are unions of the different implementations. At this point, the datasource does not know what type is being used, and the assignations will cause TS exceptions. Casting it "as any" for now. This should be fixed improving the type usages.
+          definition.set($fetch.definition ?? null)
         }
 
         // Reset scroll state when data changes

packages/frontend-core/src/fetch/CustomFetch.ts

@@ -1,13 +1,9 @@
-import DataFetch from "./DataFetch"
+import { CustomDatasource } from "@budibase/types"
-
+import BaseDataFetch from "./DataFetch"
-interface CustomDatasource {
-  type: "custom"
-  data: any
-}
 
 type CustomDefinition = Record<string, any>
 
-export default class CustomFetch extends DataFetch<
+export default class CustomFetch extends BaseDataFetch<
   CustomDatasource,
   CustomDefinition
 > {

packages/frontend-core/src/fetch/DataFetch.ts

@@ -3,14 +3,13 @@ import { cloneDeep } from "lodash/fp"
 import { QueryUtils } from "../utils"
 import { convertJSONSchemaToTableSchema } from "../utils/json"
 import {
+  DataFetchOptions,
   FieldType,
-  LegacyFilter,
   Row,
   SearchFilters,
   SortOrder,
   SortType,
   TableSchema,
-  UISearchFilter,
 } from "@budibase/types"
 import { APIClient } from "../api/types"
 import { DataFetchType } from "."
@@ -44,14 +43,11 @@ interface DataFetchDerivedStore<TDefinition, TQuery>
   supportsPagination: boolean
 }
 
-export interface DataFetchParams<
+export interface DataFetchParams<TDatasource, TQuery = SearchFilters> {
-  TDatasource,
-  TQuery = SearchFilters | undefined
-> {
   API: APIClient
   datasource: TDatasource
   query: TQuery
-  options?: {}
+  options?: Partial<DataFetchOptions<TQuery>>
 }
 
 /**
@@ -59,7 +55,7 @@ export interface DataFetchParams<
  * internal table or datasource plus.
  * For other types of datasource, this class is overridden and extended.
  */
-export default abstract class DataFetch<
+export default abstract class BaseDataFetch<
   TDatasource extends { type: DataFetchType },
   TDefinition extends {
     schema?: Record<string, any> | null
@@ -73,18 +69,11 @@ export default abstract class DataFetch<
     supportsSort: boolean
     supportsPagination: boolean
   }
-  options: {
+  options: DataFetchOptions<TQuery> & {
     datasource: TDatasource
-    limit: number
+
-    // Search config
-    filter: UISearchFilter | LegacyFilter[] | null
-    query: TQuery
-    // Sorting config
-    sortColumn: string | null
-    sortOrder: SortOrder
     sortType: SortType | null
-    // Pagination config
+
-    paginate: boolean
     // Client side feature customisation
     clientSideSearching: boolean
     clientSideSorting: boolean
@@ -267,6 +256,7 @@ export default abstract class DataFetch<
 
     // Build the query
     let query = this.options.query
+
     if (!query) {
       query = buildQuery(filter ?? undefined) as TQuery
     }
@@ -430,7 +420,7 @@ export default abstract class DataFetch<
    * Resets the data set and updates options
    * @param newOptions any new options
    */
-  async update(newOptions: any) {
+  async update(newOptions: Partial<DataFetchOptions<TQuery>>) {
     // Check if any settings have actually changed
     let refresh = false
     for (const [key, value] of Object.entries(newOptions || {})) {

packages/frontend-core/src/fetch/FieldFetch.ts

@@ -1,14 +1,10 @@
-import { Row } from "@budibase/types"
+import {
-import DataFetch from "./DataFetch"
+  FieldDatasource,
-
+  JSONArrayFieldDatasource,
-type Types = "field" | "queryarray" | "jsonarray"
+  QueryArrayFieldDatasource,
-
+  Row,
-export interface FieldDatasource<TType extends Types> {
+} from "@budibase/types"
-  type: TType
+import BaseDataFetch from "./DataFetch"
-  tableId: string
-  fieldType: "attachment" | "array"
-  value: string[] | Row[]
-}
 
 export interface FieldDefinition {
   schema?: Record<string, { type: string }> | null
@@ -18,10 +14,12 @@ function isArrayOfStrings(value: string[] | Row[]): value is string[] {
   return Array.isArray(value) && !!value[0] && typeof value[0] !== "object"
 }
 
-export default class FieldFetch<TType extends Types> extends DataFetch<
+export default class FieldFetch<
-  FieldDatasource<TType>,
+  TDatasource extends
-  FieldDefinition
+    | FieldDatasource
-> {
+    | QueryArrayFieldDatasource
+    | JSONArrayFieldDatasource = FieldDatasource
+> extends BaseDataFetch<TDatasource, FieldDefinition> {
   async getDefinition(): Promise<FieldDefinition | null> {
     const { datasource } = this.options
 

packages/frontend-core/src/fetch/GroupUserFetch.ts

@@ -1,20 +1,20 @@
 import { get } from "svelte/store"
-import DataFetch, { DataFetchParams } from "./DataFetch"
+import BaseDataFetch, { DataFetchParams } from "./DataFetch"
-import { TableNames } from "../constants"
+import { GroupUserDatasource, InternalTable } from "@budibase/types"
 
 interface GroupUserQuery {
   groupId: string
   emailSearch: string
 }
 
-interface GroupUserDatasource {
+interface GroupUserDefinition {
-  type: "groupUser"
+  schema?: Record<string, any> | null
-  tableId: TableNames.USERS
+  primaryDisplay?: string
 }
 
-export default class GroupUserFetch extends DataFetch<
+export default class GroupUserFetch extends BaseDataFetch<
   GroupUserDatasource,
-  {},
+  GroupUserDefinition,
   GroupUserQuery
 > {
   constructor(opts: DataFetchParams<GroupUserDatasource, GroupUserQuery>) {
@@ -22,7 +22,7 @@ export default class GroupUserFetch extends DataFetch<
       ...opts,
       datasource: {
         type: "groupUser",
-        tableId: TableNames.USERS,
+        tableId: InternalTable.USER_METADATA,
       },
     })
   }

packages/frontend-core/src/fetch/JSONArrayFetch.ts

@@ -1,7 +1,8 @@
 import FieldFetch from "./FieldFetch"
 import { getJSONArrayDatasourceSchema } from "../utils/json"
+import { JSONArrayFieldDatasource } from "@budibase/types"
 
-export default class JSONArrayFetch extends FieldFetch<"jsonarray"> {
+export default class JSONArrayFetch extends FieldFetch<JSONArrayFieldDatasource> {
   async getDefinition() {
     const { datasource } = this.options
 

packages/frontend-core/src/fetch/NestedProviderFetch.ts

@@ -1,20 +1,11 @@
-import { Row, TableSchema } from "@budibase/types"
+import { NestedProviderDatasource, TableSchema } from "@budibase/types"
-import DataFetch from "./DataFetch"
+import BaseDataFetch from "./DataFetch"
-
-interface NestedProviderDatasource {
-  type: "provider"
-  value?: {
-    schema: TableSchema
-    primaryDisplay: string
-    rows: Row[]
-  }
-}
 
 interface NestedProviderDefinition {
   schema?: TableSchema
   primaryDisplay?: string
 }
-export default class NestedProviderFetch extends DataFetch<
+export default class NestedProviderFetch extends BaseDataFetch<
   NestedProviderDatasource,
   NestedProviderDefinition
 > {

packages/frontend-core/src/fetch/QueryArrayFetch.ts

@@ -3,8 +3,9 @@ import {
   getJSONArrayDatasourceSchema,
   generateQueryArraySchemas,
 } from "../utils/json"
+import { QueryArrayFieldDatasource } from "@budibase/types"
 
-export default class QueryArrayFetch extends FieldFetch<"queryarray"> {
+export default class QueryArrayFetch extends FieldFetch<QueryArrayFieldDatasource> {
   async getDefinition() {
     const { datasource } = this.options
 

packages/frontend-core/src/fetch/QueryFetch.ts

@@ -1,23 +1,9 @@
-import DataFetch from "./DataFetch"
+import BaseDataFetch from "./DataFetch"
 import { Helpers } from "@budibase/bbui"
-import { ExecuteQueryRequest, Query } from "@budibase/types"
+import { ExecuteQueryRequest, Query, QueryDatasource } from "@budibase/types"
 import { get } from "svelte/store"
 
-interface QueryDatasource {
+export default class QueryFetch extends BaseDataFetch<QueryDatasource, Query> {
-  type: "query"
-  _id: string
-  fields: Record<string, any> & {
-    pagination?: {
-      type: string
-      location: string
-      pageParam: string
-    }
-  }
-  queryParams?: Record<string, string>
-  parameters: { name: string; default: string }[]
-}
-
-export default class QueryFetch extends DataFetch<QueryDatasource, Query> {
   async determineFeatureFlags() {
     const definition = await this.getDefinition()
     const supportsPagination =

packages/frontend-core/src/fetch/RelationshipFetch.ts

@@ -1,15 +1,7 @@
-import { Table } from "@budibase/types"
+import { RelationshipDatasource, Table } from "@budibase/types"
-import DataFetch from "./DataFetch"
+import BaseDataFetch from "./DataFetch"
 
-interface RelationshipDatasource {
+export default class RelationshipFetch extends BaseDataFetch<
-  type: "link"
-  tableId: string
-  rowId: string
-  rowTableId: string
-  fieldName: string
-}
-
-export default class RelationshipFetch extends DataFetch<
   RelationshipDatasource,
   Table
 > {

packages/frontend-core/src/fetch/TableFetch.ts

@@ -1,13 +1,8 @@
 import { get } from "svelte/store"
-import DataFetch from "./DataFetch"
+import BaseDataFetch from "./DataFetch"
-import { SortOrder, Table } from "@budibase/types"
+import { SortOrder, Table, TableDatasource } from "@budibase/types"
 
-interface TableDatasource {
+export default class TableFetch extends BaseDataFetch<TableDatasource, Table> {
-  type: "table"
-  tableId: string
-}
-
-export default class TableFetch extends DataFetch<TableDatasource, Table> {
   async determineFeatureFlags() {
     return {
       supportsSearch: true,

packages/frontend-core/src/fetch/UserFetch.ts

@@ -1,22 +1,24 @@
 import { get } from "svelte/store"
-import DataFetch, { DataFetchParams } from "./DataFetch"
+import BaseDataFetch, { DataFetchParams } from "./DataFetch"
-import { TableNames } from "../constants"
 import { utils } from "@budibase/shared-core"
-import { SearchFilters, SearchUsersRequest } from "@budibase/types"
+import {
+  InternalTable,
+  SearchFilters,
+  SearchUsersRequest,
+  UserDatasource,
+} from "@budibase/types"
 
 interface UserFetchQuery {
   appId: string
   paginated: boolean
 }
 
-interface UserDatasource {
+interface UserDefinition {
-  type: "user"
+  schema?: Record<string, any> | null
-  tableId: TableNames.USERS
+  primaryDisplay?: string
 }
 
-interface UserDefinition {}
+export default class UserFetch extends BaseDataFetch<
-
-export default class UserFetch extends DataFetch<
   UserDatasource,
   UserDefinition,
   UserFetchQuery
@@ -26,7 +28,7 @@ export default class UserFetch extends DataFetch<
       ...opts,
       datasource: {
         type: "user",
-        tableId: TableNames.USERS,
+        tableId: InternalTable.USER_METADATA,
       },
     })
   }

packages/frontend-core/src/fetch/ViewFetch.ts

@@ -1,16 +1,7 @@
-import { Table } from "@budibase/types"
+import { Table, ViewV1Datasource } from "@budibase/types"
-import DataFetch from "./DataFetch"
+import BaseDataFetch from "./DataFetch"
 
-type ViewV1Datasource = {
+export default class ViewFetch extends BaseDataFetch<ViewV1Datasource, Table> {
-  type: "view"
-  name: string
-  tableId: string
-  calculation: string
-  field: string
-  groupBy: string
-}
-
-export default class ViewFetch extends DataFetch<ViewV1Datasource, Table> {
   async getDefinition() {
     const { datasource } = this.options
 

packages/frontend-core/src/fetch/ViewV2Fetch.ts

@@ -1,14 +1,14 @@
-import { SortOrder, ViewV2Enriched, ViewV2Type } from "@budibase/types"
+import {
-import DataFetch from "./DataFetch"
+  SortOrder,
+  ViewDatasource,
+  ViewV2Enriched,
+  ViewV2Type,
+} from "@budibase/types"
+import BaseDataFetch from "./DataFetch"
 import { get } from "svelte/store"
 import { helpers } from "@budibase/shared-core"
 
-interface ViewDatasource {
+export default class ViewV2Fetch extends BaseDataFetch<
-  type: "viewV2"
-  id: string
-}
-
-export default class ViewV2Fetch extends DataFetch<
   ViewDatasource,
   ViewV2Enriched
 > {

packages/frontend-core/src/fetch/index.ts

@@ -11,6 +11,7 @@ import GroupUserFetch from "./GroupUserFetch"
 import CustomFetch from "./CustomFetch"
 import QueryArrayFetch from "./QueryArrayFetch"
 import { APIClient } from "../api/types"
+import { DataFetchDatasource, Table, ViewV2Enriched } from "@budibase/types"
 
 export type DataFetchType = keyof typeof DataFetchMap
 
@@ -26,32 +27,88 @@ export const DataFetchMap = {
 
   // Client specific datasource types
   provider: NestedProviderFetch,
-  field: FieldFetch<"field">,
+  field: FieldFetch,
   jsonarray: JSONArrayFetch,
   queryarray: QueryArrayFetch,
 }
 
+export interface DataFetchClassMap {
+  table: TableFetch
+  view: ViewFetch
+  viewV2: ViewV2Fetch
+  query: QueryFetch
+  link: RelationshipFetch
+  user: UserFetch
+  groupUser: GroupUserFetch
+  custom: CustomFetch
+
+  // Client specific datasource types
+  provider: NestedProviderFetch
+  field: FieldFetch
+  jsonarray: JSONArrayFetch
+  queryarray: QueryArrayFetch
+}
+
+export type DataFetch =
+  | TableFetch
+  | ViewFetch
+  | ViewV2Fetch
+  | QueryFetch
+  | RelationshipFetch
+  | UserFetch
+  | GroupUserFetch
+  | CustomFetch
+  | NestedProviderFetch
+  | FieldFetch
+  | JSONArrayFetch
+  | QueryArrayFetch
+
+export type DataFetchDefinition =
+  | Table
+  | ViewV2Enriched
+  | {
+      // These fields are added to allow checking these fields on definition usages without requiring constant castings
+      schema?: Record<string, any> | null
+      primaryDisplay?: string
+      rowHeight?: number
+      type?: string
+      name?: string
+    }
+
 // Constructs a new fetch model for a certain datasource
-export const fetchData = ({ API, datasource, options }: any) => {
+export const fetchData = <
-  const Fetch = DataFetchMap[datasource?.type as DataFetchType] || TableFetch
+  T extends DataFetchDatasource,
+  Type extends T["type"] = T["type"]
+>({
+  API,
+  datasource,
+  options,
+}: {
+  API: APIClient
+  datasource: T
+  options: any
+}): Type extends keyof DataFetchClassMap
+  ? DataFetchClassMap[Type]
+  : TableFetch => {
+  const Fetch = DataFetchMap[datasource?.type] || TableFetch
   const fetch = new Fetch({ API, datasource, ...options })
 
   // Initially fetch data but don't bother waiting for the result
   fetch.getInitialData()
 
-  return fetch
+  return fetch as any
 }
 
 // Creates an empty fetch instance with no datasource configured, so no data
 // will initially be loaded
-const createEmptyFetchInstance = <TDatasource extends { type: DataFetchType }>({
+const createEmptyFetchInstance = ({
   API,
   datasource,
 }: {
   API: APIClient
-  datasource: TDatasource
+  datasource: DataFetchDatasource
 }) => {
-  const handler = DataFetchMap[datasource?.type as DataFetchType]
+  const handler = DataFetchMap[datasource?.type]
   if (!handler) {
     return null
   }
@@ -63,29 +120,25 @@ const createEmptyFetchInstance = <TDatasource extends { type: DataFetchType }>({
 }
 
 // Fetches the definition of any type of datasource
-export const getDatasourceDefinition = async <
+export const getDatasourceDefinition = async ({
-  TDatasource extends { type: DataFetchType }
->({
   API,
   datasource,
 }: {
   API: APIClient
-  datasource: TDatasource
+  datasource: DataFetchDatasource
 }) => {
   const instance = createEmptyFetchInstance({ API, datasource })
   return await instance?.getDefinition()
 }
 
 // Fetches the schema of any type of datasource
-export const getDatasourceSchema = <
+export const getDatasourceSchema = ({
-  TDatasource extends { type: DataFetchType }
->({
   API,
   datasource,
   definition,
 }: {
   API: APIClient
-  datasource: TDatasource
+  datasource: DataFetchDatasource
   definition?: any
 }) => {
   const instance = createEmptyFetchInstance({ API, datasource })

packages/frontend-core/src/index.ts

@@ -1,7 +1,6 @@
 export { createAPIClient } from "./api"
 export type { APIClient } from "./api"
 export { fetchData, DataFetchMap } from "./fetch"
-export type { DataFetchType } from "./fetch"
 export * as Constants from "./constants"
 export * from "./stores"
 export * from "./utils"

packages/frontend-core/src/utils/index.ts

@@ -8,6 +8,7 @@ export * as search from "./searchFields"
 export * as SchemaUtils from "./schema"
 export { memo, derivedMemo } from "./memo"
 export { createWebsocket } from "./websocket"
+export * as JsonFormatter from "./jsonFormatter"
 export * from "./download"
 export * from "./settings"
 export * from "./relatedColumns"

packages/frontend-core/src/utils/jsonFormatter.ts

@@ -0,0 +1,71 @@
+import { JSONValue } from "@budibase/types"
+
+export type ColorsOptions = {
+  keyColor?: string
+  numberColor?: string
+  stringColor?: string
+  trueColor?: string
+  falseColor?: string
+  nullColor?: string
+}
+
+const defaultColors: ColorsOptions = {
+  keyColor: "dimgray",
+  numberColor: "lightskyblue",
+  stringColor: "lightcoral",
+  trueColor: "lightseagreen",
+  falseColor: "#f66578",
+  nullColor: "cornflowerblue",
+}
+
+const entityMap = {
+  "&": "&",
+  "<": "&lt;",
+  ">": "&gt;",
+  '"': "&quot;",
+  "'": "&#39;",
+  "`": "&#x60;",
+  "=": "&#x3D;",
+}
+
+function escapeHtml(html: string) {
+  return String(html).replace(/[&<>"'`=]/g, function (s) {
+    return entityMap[s as keyof typeof entityMap]
+  })
+}
+
+export function format(json: JSONValue, colorOptions: ColorsOptions = {}) {
+  const valueType = typeof json
+  let jsonString =
+    typeof json === "string" ? json : JSON.stringify(json, null, 2) || valueType
+  let colors = Object.assign({}, defaultColors, colorOptions)
+  jsonString = jsonString
+    .replace(/&/g, "&")
+    .replace(/</g, "<")
+    .replace(/>/g, ">")
+  return jsonString.replace(
+    /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g,
+    (match: string) => {
+      let color = colors.numberColor
+      let style = ""
+      if (/^"/.test(match)) {
+        if (/:$/.test(match)) {
+          color = colors.keyColor
+        } else {
+          color = colors.stringColor
+          match = '"' + escapeHtml(match.substr(1, match.length - 2)) + '"'
+          style = "word-wrap:break-word;white-space:pre-wrap;"
+        }
+      } else {
+        color = /true/.test(match)
+          ? colors.trueColor
+          : /false/.test(match)
+          ? colors.falseColor
+          : /null/.test(match)
+          ? colors.nullColor
+          : color
+      }
+      return `<span style="${style}color:${color}">${match}</span>`
+    }
+  )
+}

packages/frontend-core/src/utils/utils.js

@@ -43,7 +43,7 @@ export const sequential = fn => {
  * invocations is enforced.
  * @param callback an async function to run
  * @param minDelay the minimum delay between invocations
- * @returns {Promise} a debounced version of the callback
+ * @returns a debounced version of the callback
  */
 export const debounce = (callback, minDelay = 1000) => {
   let timeout

packages/server/scripts/dev/manage.js

@@ -43,7 +43,6 @@ async function init() {
     BB_ADMIN_USER_EMAIL: "",
     BB_ADMIN_USER_PASSWORD: "",
     PLUGINS_DIR: "",
-    HTTP_MIGRATIONS: "0",
     HTTP_LOGGING: "0",
     VERSION: "0.0.0+local",
     PASSWORD_MIN_LENGTH: "1",

packages/server/src/api/controllers/application.ts

@@ -27,7 +27,6 @@ import {
   env as envCore,
   ErrorCode,
   events,
-  migrations,
   objectStore,
   roles,
   tenancy,
@@ -43,7 +42,6 @@ import { groups, licensing, quotas } from "@budibase/pro"
 import {
   App,
   Layout,
-  MigrationType,
   PlanType,
   Screen,
   UserCtx,
@@ -488,13 +486,6 @@ async function creationEvents(request: BBRequest<CreateAppRequest>, app: App) {
 }
 
 async function appPostCreate(ctx: UserCtx<CreateAppRequest, App>, app: App) {
-  const tenantId = tenancy.getTenantId()
-  await migrations.backPopulateMigrations({
-    type: MigrationType.APP,
-    tenantId,
-    appId: app.appId,
-  })
-
   await creationEvents(ctx.request, app)
 
   // app import, template creation and duplication

packages/server/src/api/controllers/migrations.ts

@@ -1,35 +1,11 @@
 import { context } from "@budibase/backend-core"
-import { migrate as migrationImpl, MIGRATIONS } from "../../migrations"
+import { Ctx, GetMigrationStatus } from "@budibase/types"
-import {
-  Ctx,
-  FetchOldMigrationResponse,
-  GetOldMigrationStatus,
-  RuneOldMigrationResponse,
-  RunOldMigrationRequest,
-} from "@budibase/types"
 import {
   getAppMigrationVersion,
   getLatestEnabledMigrationId,
 } from "../../appMigrations"
 
-export async function migrate(
+export async function getMigrationStatus(ctx: Ctx<void, GetMigrationStatus>) {
-  ctx: Ctx<RunOldMigrationRequest, RuneOldMigrationResponse>
-) {
-  const options = ctx.request.body
-  // don't await as can take a while, just return
-  migrationImpl(options)
-  ctx.body = { message: "Migration started." }
-}
-
-export async function fetchDefinitions(
-  ctx: Ctx<void, FetchOldMigrationResponse>
-) {
-  ctx.body = MIGRATIONS
-}
-
-export async function getMigrationStatus(
-  ctx: Ctx<void, GetOldMigrationStatus>
-) {
   const appId = context.getAppId()
 
   if (!appId) {

packages/server/src/api/controllers/query/import/sources/tests/curl/curl.spec.ts

@@ -1,34 +1,24 @@
-const { Curl } = require("../../curl")
+import { Curl } from "../../curl"
-const fs = require("fs")
+import { readFileSync } from "fs"
-const path = require("path")
+import { join } from "path"
 
-const getData = file => {
+const getData = (file: string) => {
-  return fs.readFileSync(path.join(__dirname, `./data/${file}.txt`), "utf8")
+  return readFileSync(join(__dirname, `./data/${file}.txt`), "utf8")
 }
 
 describe("Curl Import", () => {
-  let curl
+  let curl: Curl
 
   beforeEach(() => {
     curl = new Curl()
   })
 
   it("validates unsupported data", async () => {
-    let data
+    expect(await curl.isSupported("{}")).toBe(false)
-    let supported
+    expect(await curl.isSupported("")).toBe(false)
-
-    // JSON
-    data = "{}"
-    supported = await curl.isSupported(data)
-    expect(supported).toBe(false)
-
-    // Empty
-    data = ""
-    supported = await curl.isSupported(data)
-    expect(supported).toBe(false)
   })
 
-  const init = async file => {
+  const init = async (file: string) => {
     await curl.isSupported(getData(file))
   }
 
@@ -39,14 +29,14 @@ describe("Curl Import", () => {
   })
 
   describe("Returns queries", () => {
-    const getQueries = async file => {
+    const getQueries = async (file: string) => {
       await init(file)
-      const queries = await curl.getQueries()
+      const queries = await curl.getQueries("fake_datasource_id")
       expect(queries.length).toBe(1)
       return queries
     }
 
-    const testVerb = async (file, verb) => {
+    const testVerb = async (file: string, verb: string) => {
       const queries = await getQueries(file)
       expect(queries[0].queryVerb).toBe(verb)
     }
@@ -59,7 +49,7 @@ describe("Curl Import", () => {
       await testVerb("patch", "patch")
     })
 
-    const testPath = async (file, urlPath) => {
+    const testPath = async (file: string, urlPath: string) => {
       const queries = await getQueries(file)
       expect(queries[0].fields.path).toBe(urlPath)
     }
@@ -69,7 +59,10 @@ describe("Curl Import", () => {
       await testPath("path", "http://example.com/paths/abc")
     })
 
-    const testHeaders = async (file, headers) => {
+    const testHeaders = async (
+      file: string,
+      headers: Record<string, string>
+    ) => {
       const queries = await getQueries(file)
       expect(queries[0].fields.headers).toStrictEqual(headers)
     }
@@ -82,7 +75,7 @@ describe("Curl Import", () => {
       })
     })
 
-    const testQuery = async (file, queryString) => {
+    const testQuery = async (file: string, queryString: string) => {
       const queries = await getQueries(file)
       expect(queries[0].fields.queryString).toBe(queryString)
     }
@@ -91,7 +84,7 @@ describe("Curl Import", () => {
       await testQuery("query", "q1=v1&q1=v2")
     })
 
-    const testBody = async (file, body) => {
+    const testBody = async (file: string, body?: Record<string, any>) => {
       const queries = await getQueries(file)
       expect(queries[0].fields.requestBody).toStrictEqual(
         JSON.stringify(body, null, 2)

packages/server/src/api/controllers/query/import/sources/tests/openapi2/openapi2.spec.js

@@ -1,243 +0,0 @@
-const { OpenAPI2 } = require("../../openapi2")
-const fs = require("fs")
-const path = require("path")
-
-const getData = (file, extension) => {
-  return fs.readFileSync(
-    path.join(__dirname, `./data/${file}/${file}.${extension}`),
-    "utf8"
-  )
-}
-
-describe("OpenAPI2 Import", () => {
-  let openapi2
-
-  beforeEach(() => {
-    openapi2 = new OpenAPI2()
-  })
-
-  it("validates unsupported data", async () => {
-    let data
-    let supported
-
-    // non json / yaml
-    data = "curl http://example.com"
-    supported = await openapi2.isSupported(data)
-    expect(supported).toBe(false)
-
-    // Empty
-    data = ""
-    supported = await openapi2.isSupported(data)
-    expect(supported).toBe(false)
-  })
-
-  const init = async (file, extension) => {
-    await openapi2.isSupported(getData(file, extension))
-  }
-
-  const runTests = async (filename, test, assertions) => {
-    for (let extension of ["json", "yaml"]) {
-      await test(filename, extension, assertions)
-    }
-  }
-
-  const testImportInfo = async (file, extension) => {
-    await init(file, extension)
-    const info = await openapi2.getInfo()
-    expect(info.name).toBe("Swagger Petstore")
-  }
-
-  it("returns import info", async () => {
-    await runTests("petstore", testImportInfo)
-  })
-
-  describe("Returns queries", () => {
-    const indexQueries = queries => {
-      return queries.reduce((acc, query) => {
-        acc[query.name] = query
-        return acc
-      }, {})
-    }
-
-    const getQueries = async (file, extension) => {
-      await init(file, extension)
-      const queries = await openapi2.getQueries()
-      expect(queries.length).toBe(6)
-      return indexQueries(queries)
-    }
-
-    const testVerb = async (file, extension, assertions) => {
-      const queries = await getQueries(file, extension)
-      for (let [operationId, method] of Object.entries(assertions)) {
-        expect(queries[operationId].queryVerb).toBe(method)
-      }
-    }
-
-    it("populates verb", async () => {
-      const assertions = {
-        createEntity: "create",
-        getEntities: "read",
-        getEntity: "read",
-        updateEntity: "update",
-        patchEntity: "patch",
-        deleteEntity: "delete",
-      }
-      await runTests("crud", testVerb, assertions)
-    })
-
-    const testPath = async (file, extension, assertions) => {
-      const queries = await getQueries(file, extension)
-      for (let [operationId, urlPath] of Object.entries(assertions)) {
-        expect(queries[operationId].fields.path).toBe(urlPath)
-      }
-    }
-
-    it("populates path", async () => {
-      const assertions = {
-        createEntity: "http://example.com/entities",
-        getEntities: "http://example.com/entities",
-        getEntity: "http://example.com/entities/{{entityId}}",
-        updateEntity: "http://example.com/entities/{{entityId}}",
-        patchEntity: "http://example.com/entities/{{entityId}}",
-        deleteEntity: "http://example.com/entities/{{entityId}}",
-      }
-      await runTests("crud", testPath, assertions)
-    })
-
-    const testHeaders = async (file, extension, assertions) => {
-      const queries = await getQueries(file, extension)
-      for (let [operationId, headers] of Object.entries(assertions)) {
-        expect(queries[operationId].fields.headers).toStrictEqual(headers)
-      }
-    }
-
-    const contentTypeHeader = {
-      "Content-Type": "application/json",
-    }
-
-    it("populates headers", async () => {
-      const assertions = {
-        createEntity: {
-          ...contentTypeHeader,
-        },
-        getEntities: {},
-        getEntity: {},
-        updateEntity: {
-          ...contentTypeHeader,
-        },
-        patchEntity: {
-          ...contentTypeHeader,
-        },
-        deleteEntity: {
-          "x-api-key": "{{x-api-key}}",
-        },
-      }
-
-      await runTests("crud", testHeaders, assertions)
-    })
-
-    const testQuery = async (file, extension, assertions) => {
-      const queries = await getQueries(file, extension)
-      for (let [operationId, queryString] of Object.entries(assertions)) {
-        expect(queries[operationId].fields.queryString).toStrictEqual(
-          queryString
-        )
-      }
-    }
-
-    it("populates query", async () => {
-      const assertions = {
-        createEntity: "",
-        getEntities: "page={{page}}&size={{size}}",
-        getEntity: "",
-        updateEntity: "",
-        patchEntity: "",
-        deleteEntity: "",
-      }
-      await runTests("crud", testQuery, assertions)
-    })
-
-    const testParameters = async (file, extension, assertions) => {
-      const queries = await getQueries(file, extension)
-      for (let [operationId, parameters] of Object.entries(assertions)) {
-        expect(queries[operationId].parameters).toStrictEqual(parameters)
-      }
-    }
-
-    it("populates parameters", async () => {
-      const assertions = {
-        createEntity: [],
-        getEntities: [
-          {
-            name: "page",
-            default: "",
-          },
-          {
-            name: "size",
-            default: "",
-          },
-        ],
-        getEntity: [
-          {
-            name: "entityId",
-            default: "",
-          },
-        ],
-        updateEntity: [
-          {
-            name: "entityId",
-            default: "",
-          },
-        ],
-        patchEntity: [
-          {
-            name: "entityId",
-            default: "",
-          },
-        ],
-        deleteEntity: [
-          {
-            name: "entityId",
-            default: "",
-          },
-          {
-            name: "x-api-key",
-            default: "",
-          },
-        ],
-      }
-      await runTests("crud", testParameters, assertions)
-    })
-
-    const testBody = async (file, extension, assertions) => {
-      const queries = await getQueries(file, extension)
-      for (let [operationId, body] of Object.entries(assertions)) {
-        expect(queries[operationId].fields.requestBody).toStrictEqual(
-          JSON.stringify(body, null, 2)
-        )
-      }
-    }
-    it("populates body", async () => {
-      const assertions = {
-        createEntity: {
-          name: "name",
-          type: "type",
-        },
-        getEntities: undefined,
-        getEntity: undefined,
-        updateEntity: {
-          id: 1,
-          name: "name",
-          type: "type",
-        },
-        patchEntity: {
-          id: 1,
-          name: "name",
-          type: "type",
-        },
-        deleteEntity: undefined,
-      }
-      await runTests("crud", testBody, assertions)
-    })
-  })
-})

packages/server/src/api/controllers/query/import/sources/tests/openapi2/openapi2.spec.ts

@@ -0,0 +1,135 @@
+import { OpenAPI2 } from "../../openapi2"
+import { readFileSync } from "fs"
+import { join } from "path"
+import { groupBy, mapValues } from "lodash"
+import { Query } from "@budibase/types"
+
+const getData = (file: string, extension: string) => {
+  return readFileSync(
+    join(__dirname, `./data/${file}/${file}.${extension}`),
+    "utf8"
+  )
+}
+
+describe("OpenAPI2 Import", () => {
+  let openapi2: OpenAPI2
+
+  beforeEach(() => {
+    openapi2 = new OpenAPI2()
+  })
+
+  it("validates unsupported data", async () => {
+    expect(await openapi2.isSupported("curl http://example.com")).toBe(false)
+    expect(await openapi2.isSupported("")).toBe(false)
+  })
+
+  describe.each(["json", "yaml"])("%s", extension => {
+    describe("petstore", () => {
+      beforeEach(async () => {
+        await openapi2.isSupported(getData("petstore", extension))
+      })
+
+      it("returns import info", async () => {
+        const { name } = await openapi2.getInfo()
+        expect(name).toBe("Swagger Petstore")
+      })
+    })
+
+    describe("crud", () => {
+      let queries: Record<string, Query>
+      beforeEach(async () => {
+        await openapi2.isSupported(getData("crud", extension))
+
+        const raw = await openapi2.getQueries("fake_datasource_id")
+        queries = mapValues(groupBy(raw, "name"), group => group[0])
+      })
+
+      it("should have 6 queries", () => {
+        expect(Object.keys(queries).length).toBe(6)
+      })
+
+      it.each([
+        ["createEntity", "create"],
+        ["getEntities", "read"],
+        ["getEntity", "read"],
+        ["updateEntity", "update"],
+        ["patchEntity", "patch"],
+        ["deleteEntity", "delete"],
+      ])("should have correct verb for %s", (operationId, method) => {
+        expect(queries[operationId].queryVerb).toBe(method)
+      })
+
+      it.each([
+        ["createEntity", "http://example.com/entities"],
+        ["getEntities", "http://example.com/entities"],
+        ["getEntity", "http://example.com/entities/{{entityId}}"],
+        ["updateEntity", "http://example.com/entities/{{entityId}}"],
+        ["patchEntity", "http://example.com/entities/{{entityId}}"],
+        ["deleteEntity", "http://example.com/entities/{{entityId}}"],
+      ])("should have correct path for %s", (operationId, urlPath) => {
+        expect(queries[operationId].fields.path).toBe(urlPath)
+      })
+
+      it.each([
+        ["createEntity", { "Content-Type": "application/json" }],
+        ["getEntities", {}],
+        ["getEntity", {}],
+        ["updateEntity", { "Content-Type": "application/json" }],
+        ["patchEntity", { "Content-Type": "application/json" }],
+        ["deleteEntity", { "x-api-key": "{{x-api-key}}" }],
+      ])(`should have correct headers for %s`, (operationId, headers) => {
+        expect(queries[operationId].fields.headers).toStrictEqual(headers)
+      })
+
+      it.each([
+        ["createEntity", ""],
+        ["getEntities", "page={{page}}&size={{size}}"],
+        ["getEntity", ""],
+        ["updateEntity", ""],
+        ["patchEntity", ""],
+        ["deleteEntity", ""],
+      ])(
+        `should have correct query string for %s`,
+        (operationId, queryString) => {
+          expect(queries[operationId].fields.queryString).toBe(queryString)
+        }
+      )
+
+      it.each([
+        ["createEntity", []],
+        [
+          "getEntities",
+          [
+            { name: "page", default: "" },
+            { name: "size", default: "" },
+          ],
+        ],
+        ["getEntity", [{ name: "entityId", default: "" }]],
+        ["updateEntity", [{ name: "entityId", default: "" }]],
+        ["patchEntity", [{ name: "entityId", default: "" }]],
+        [
+          "deleteEntity",
+          [
+            { name: "entityId", default: "" },
+            { name: "x-api-key", default: "" },
+          ],
+        ],
+      ])(`should have correct parameters for %s`, (operationId, parameters) => {
+        expect(queries[operationId].parameters).toStrictEqual(parameters)
+      })
+
+      it.each([
+        ["createEntity", { name: "name", type: "type" }],
+        ["getEntities", undefined],
+        ["getEntity", undefined],
+        ["updateEntity", { id: 1, name: "name", type: "type" }],
+        ["patchEntity", { id: 1, name: "name", type: "type" }],
+        ["deleteEntity", undefined],
+      ])(`should have correct body for %s`, (operationId, body) => {
+        expect(queries[operationId].fields.requestBody).toBe(
+          JSON.stringify(body, null, 2)
+        )
+      })
+    })
+  })
+})

packages/server/src/api/controllers/view/tests/__snapshots__/viewBuilder.spec.js.snap

@@ -1,147 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`viewBuilder Calculate and filter creates a view with the calculation statistics and filter schema 1`] = `
-{
-  "map": "function (doc) {
-      if ((doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && !(
-      doc["myField"] === undefined ||
-      doc["myField"] === null ||
-      doc["myField"] === "" ||
-      (Array.isArray(doc["myField"]) && doc["myField"].length === 0)
-  )) && (doc["age"] > 17)) {
-        emit(doc["_id"], doc["myField"]);
-      }
-    }",
-  "meta": {
-    "calculation": "stats",
-    "field": "myField",
-    "filters": [
-      {
-        "condition": "MT",
-        "key": "age",
-        "value": 17,
-      },
-    ],
-    "groupBy": undefined,
-    "schema": {
-      "avg": {
-        "type": "number",
-      },
-      "count": {
-        "type": "number",
-      },
-      "field": {
-        "type": "string",
-      },
-      "max": {
-        "type": "number",
-      },
-      "min": {
-        "type": "number",
-      },
-      "sum": {
-        "type": "number",
-      },
-      "sumsqr": {
-        "type": "number",
-      },
-    },
-    "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
-  },
-  "reduce": "_stats",
-}
-`;
-
-exports[`viewBuilder Calculate creates a view with the calculation statistics schema 1`] = `
-{
-  "map": "function (doc) {
-      if ((doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && !(
-      doc["myField"] === undefined ||
-      doc["myField"] === null ||
-      doc["myField"] === "" ||
-      (Array.isArray(doc["myField"]) && doc["myField"].length === 0)
-  )) ) {
-        emit(doc["_id"], doc["myField"]);
-      }
-    }",
-  "meta": {
-    "calculation": "stats",
-    "field": "myField",
-    "filters": [],
-    "groupBy": undefined,
-    "schema": {
-      "avg": {
-        "type": "number",
-      },
-      "count": {
-        "type": "number",
-      },
-      "field": {
-        "type": "string",
-      },
-      "max": {
-        "type": "number",
-      },
-      "min": {
-        "type": "number",
-      },
-      "sum": {
-        "type": "number",
-      },
-      "sumsqr": {
-        "type": "number",
-      },
-    },
-    "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
-  },
-  "reduce": "_stats",
-}
-`;
-
-exports[`viewBuilder Filter creates a view with multiple filters and conjunctions 1`] = `
-{
-  "map": "function (doc) {
-      if (doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && (doc["Name"] === "Test" || doc["Yes"] > "Value")) {
-        emit(doc["_id"], doc["undefined"]);
-      }
-    }",
-  "meta": {
-    "calculation": undefined,
-    "field": undefined,
-    "filters": [
-      {
-        "condition": "EQUALS",
-        "key": "Name",
-        "value": "Test",
-      },
-      {
-        "condition": "MT",
-        "conjunction": "OR",
-        "key": "Yes",
-        "value": "Value",
-      },
-    ],
-    "groupBy": undefined,
-    "schema": null,
-    "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
-  },
-}
-`;
-
-exports[`viewBuilder Group By creates a view emitting the group by field 1`] = `
-{
-  "map": "function (doc) {
-      if (doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" ) {
-        emit(doc["age"], doc["score"]);
-      }
-    }",
-  "meta": {
-    "calculation": undefined,
-    "field": "score",
-    "filters": [],
-    "groupBy": "age",
-    "schema": null,
-    "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
-  },
-}
-`;

packages/server/src/api/controllers/view/tests/viewBuilder.spec.js

@@ -1,75 +0,0 @@
-const viewTemplate = require("../viewBuilder").default
-
-describe("viewBuilder", () => {
-  describe("Filter", () => {
-    it("creates a view with multiple filters and conjunctions", () => {
-      expect(
-        viewTemplate({
-          name: "Test View",
-          tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
-          filters: [
-            {
-              value: "Test",
-              condition: "EQUALS",
-              key: "Name",
-            },
-            {
-              value: "Value",
-              condition: "MT",
-              key: "Yes",
-              conjunction: "OR",
-            },
-          ],
-        })
-      ).toMatchSnapshot()
-    })
-  })
-
-  describe("Calculate", () => {
-    it("creates a view with the calculation statistics schema", () => {
-      expect(
-        viewTemplate({
-          name: "Calculate View",
-          field: "myField",
-          calculation: "stats",
-          tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
-          filters: [],
-        })
-      ).toMatchSnapshot()
-    })
-  })
-
-  describe("Group By", () => {
-    it("creates a view emitting the group by field", () => {
-      expect(
-        viewTemplate({
-          name: "Test Scores Grouped By Age",
-          tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
-          groupBy: "age",
-          field: "score",
-          filters: [],
-        })
-      ).toMatchSnapshot()
-    })
-  })
-
-  describe("Calculate and filter", () => {
-    it("creates a view with the calculation statistics and filter schema", () => {
-      expect(
-        viewTemplate({
-          name: "Calculate View",
-          field: "myField",
-          calculation: "stats",
-          tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
-          filters: [
-            {
-              value: 17,
-              condition: "MT",
-              key: "age",
-            },
-          ],
-        })
-      ).toMatchSnapshot()
-    })
-  })
-})

packages/server/src/api/controllers/view/tests/viewBuilder.spec.ts

@@ -0,0 +1,174 @@
+import viewTemplate from "../viewBuilder"
+
+describe("viewBuilder", () => {
+  describe("Filter", () => {
+    it("creates a view with multiple filters and conjunctions", () => {
+      expect(
+        viewTemplate({
+          field: "myField",
+          tableId: "tableId",
+          filters: [
+            {
+              value: "Test",
+              condition: "EQUALS",
+              key: "Name",
+            },
+            {
+              value: "Value",
+              condition: "MT",
+              key: "Yes",
+              conjunction: "OR",
+            },
+          ],
+        })
+      ).toEqual({
+        map: `function (doc) {
+      if (doc.tableId === "tableId" && (doc["Name"] === "Test" || doc["Yes"] > "Value")) {
+        emit(doc["_id"], doc["myField"]);
+      }
+    }`,
+        meta: {
+          calculation: undefined,
+          field: "myField",
+          filters: [
+            {
+              condition: "EQUALS",
+              key: "Name",
+              value: "Test",
+            },
+            {
+              condition: "MT",
+              conjunction: "OR",
+              key: "Yes",
+              value: "Value",
+            },
+          ],
+          groupBy: undefined,
+          schema: null,
+          tableId: "tableId",
+        },
+      })
+    })
+  })
+
+  describe("Calculate", () => {
+    it("creates a view with the calculation statistics schema", () => {
+      expect(
+        viewTemplate({
+          field: "myField",
+          calculation: "stats",
+          tableId: "tableId",
+          filters: [],
+        })
+      ).toEqual({
+        map: `function (doc) {
+      if ((doc.tableId === "tableId" && !(
+      doc["myField"] === undefined ||
+      doc["myField"] === null ||
+      doc["myField"] === "" ||
+      (Array.isArray(doc["myField"]) && doc["myField"].length === 0)
+  )) ) {
+        emit(doc["_id"], doc["myField"]);
+      }
+    }`,
+        meta: {
+          calculation: "stats",
+          field: "myField",
+          filters: [],
+          groupBy: undefined,
+          schema: {
+            min: { type: "number" },
+            max: { type: "number" },
+            avg: { type: "number" },
+            count: { type: "number" },
+            sumsqr: { type: "number" },
+            sum: { type: "number" },
+            field: { type: "string" },
+          },
+          tableId: "tableId",
+        },
+        reduce: "_stats",
+      })
+    })
+  })
+
+  describe("Group By", () => {
+    it("creates a view emitting the group by field", () => {
+      expect(
+        viewTemplate({
+          tableId: "tableId",
+          groupBy: "age",
+          field: "score",
+          filters: [],
+        })
+      ).toEqual({
+        map: `function (doc) {
+      if (doc.tableId === "tableId" ) {
+        emit(doc["age"], doc["score"]);
+      }
+    }`,
+        meta: {
+          calculation: undefined,
+          field: "score",
+          filters: [],
+          groupBy: "age",
+          schema: null,
+          tableId: "tableId",
+        },
+      })
+    })
+  })
+
+  describe("Calculate and filter", () => {
+    it("creates a view with the calculation statistics and filter schema", () => {
+      expect(
+        viewTemplate({
+          field: "myField",
+          calculation: "stats",
+          tableId: "tableId",
+          filters: [
+            {
+              value: 17,
+              condition: "MT",
+              key: "age",
+            },
+          ],
+        })
+      ).toEqual({
+        map: `function (doc) {
+      if ((doc.tableId === "tableId" && !(
+      doc["myField"] === undefined ||
+      doc["myField"] === null ||
+      doc["myField"] === "" ||
+      (Array.isArray(doc["myField"]) && doc["myField"].length === 0)
+  )) && (doc["age"] > 17)) {
+        emit(doc["_id"], doc["myField"]);
+      }
+    }`,
+        meta: {
+          calculation: "stats",
+          field: "myField",
+          filters: [
+            {
+              condition: "MT",
+              key: "age",
+              value: 17,
+            },
+          ],
+          groupBy: undefined,
+          schema: {
+            min: { type: "number" },
+            max: { type: "number" },
+            avg: { type: "number" },
+            count: { type: "number" },
+            sumsqr: { type: "number" },
+            sum: { type: "number" },
+            field: { type: "string" },
+          },
+          tableId: "tableId",
+        },
+        reduce: "_stats",
+      })
+    })
+  })
+})

packages/server/src/api/controllers/view/viewBuilder.ts

@@ -1,4 +1,4 @@
-import { ViewFilter, ViewTemplateOpts, DBView } from "@budibase/types"
+import { ViewFilter, DBView } from "@budibase/types"
 
 const TOKEN_MAP: Record<string, string> = {
   EQUALS: "===",
@@ -120,7 +120,7 @@ function parseFilterExpression(filters: ViewFilter[]) {
  * @param groupBy - field to group calculation results on, if any
  */
 function parseEmitExpression(field: string, groupBy: string) {
-  return `emit(doc["${groupBy || "_id"}"], doc["${field}"]);`
+  return `emit(doc["${groupBy}"], doc["${field}"]);`
 }
 
 /**
@@ -135,7 +135,19 @@ function parseEmitExpression(field: string, groupBy: string) {
  * calculation: an optional calculation to be performed over the view data.
  */
 export default function (
-  { field, tableId, groupBy, filters = [], calculation }: ViewTemplateOpts,
+  {
+    field,
+    tableId,
+    groupBy,
+    filters = [],
+    calculation,
+  }: {
+    field: string
+    tableId: string
+    groupBy?: string
+    filters?: ViewFilter[]
+    calculation?: string
+  },
   groupByMulti?: boolean
 ): DBView {
   // first filter can't have a conjunction
@@ -168,7 +180,7 @@ export default function (
   const parsedFilters = parseFilterExpression(filters)
   const filterExpression = parsedFilters ? `&& (${parsedFilters})` : ""
 
-  const emitExpression = parseEmitExpression(field, groupBy)
+  const emitExpression = parseEmitExpression(field, groupBy || "_id")
   const tableExpression = `doc.tableId === "${tableId}"`
   const coreExpression = statFilter
     ? `(${tableExpression} && ${statFilter})`

packages/server/src/api/controllers/view/viewsV2.ts

@@ -123,9 +123,11 @@ async function parseSchema(view: CreateViewRequest) {
 }
 
 export async function get(ctx: Ctx<void, ViewResponseEnriched>) {
-  ctx.body = {
+  const view = await sdk.views.getEnriched(ctx.params.viewId)
-    data: await sdk.views.getEnriched(ctx.params.viewId),
+  if (!view) {
+    ctx.throw(404)
   }
+  ctx.body = { data: view }
 }
 
 export async function fetch(ctx: Ctx<void, ViewFetchResponseEnriched>) {

packages/server/src/api/routes/migrations.ts

@@ -1,16 +1,8 @@
 import Router from "@koa/router"
 import * as migrationsController from "../controllers/migrations"
-import { auth } from "@budibase/backend-core"
 
 const router: Router = new Router()
 
-router
+router.get("/api/migrations/status", migrationsController.getMigrationStatus)
-  .post("/api/migrations/run", auth.internalApi, migrationsController.migrate)
-  .get(
-    "/api/migrations/definitions",
-    auth.internalApi,
-    migrationsController.fetchDefinitions
-  )
-  .get("/api/migrations/status", migrationsController.getMigrationStatus)
 
 export default router

packages/server/src/api/routes/tests/automation.spec.ts

@@ -19,8 +19,10 @@ import {
   Table,
 } from "@budibase/types"
 import { mocks } from "@budibase/backend-core/tests"
-import { FilterConditions } from "../../../automations/steps/filter"
 import { createAutomationBuilder } from "../../../automations/tests/utilities/AutomationTestBuilder"
+import { automations } from "@budibase/shared-core"
+
+const FilterConditions = automations.steps.filter.FilterConditions
 
 const MAX_RETRIES = 4
 let {

packages/server/src/api/routes/tests/row.spec.ts

@@ -3650,6 +3650,51 @@ if (descriptions.length) {
         })
       })
 
+      if (isInternal || isMSSQL) {
+        describe("Fields with spaces", () => {
+          let table: Table
+          let otherTable: Table
+          let relatedRow: Row
+
+          beforeAll(async () => {
+            otherTable = await config.api.table.save(defaultTable())
+            table = await config.api.table.save(
+              saveTableRequest({
+                schema: {
+                  links: {
+                    name: "links",
+                    fieldName: "links",
+                    type: FieldType.LINK,
+                    tableId: otherTable._id!,
+                    relationshipType: RelationshipType.ONE_TO_MANY,
+                  },
+                  "nameWithSpace ": {
+                    name: "nameWithSpace ",
+                    type: FieldType.STRING,
+                  },
+                },
+              })
+            )
+            relatedRow = await config.api.row.save(otherTable._id!, {
+              name: generator.word(),
+              description: generator.paragraph(),
+            })
+            await config.api.row.save(table._id!, {
+              "nameWithSpace ": generator.word(),
+              tableId: table._id!,
+              links: [relatedRow._id],
+            })
+          })
+
+          it("Successfully returns rows that have spaces in their field names", async () => {
+            const { rows } = await config.api.row.search(table._id!)
+            expect(rows.length).toBe(1)
+            const row = rows[0]
+            expect(row["nameWithSpace "]).toBeDefined()
+          })
+        })
+      }
+
       if (!isInternal && !isOracle) {
         describe("bigint ids", () => {
           let table1: Table, table2: Table

packages/server/src/automations/actions.ts

@@ -1,3 +1,4 @@
+import { automations } from "@budibase/shared-core"
 import * as sendSmtpEmail from "./steps/sendSmtpEmail"
 import * as createRow from "./steps/createRow"
 import * as updateRow from "./steps/updateRow"
@@ -15,11 +16,10 @@ import * as make from "./steps/make"
 import * as filter from "./steps/filter"
 import * as delay from "./steps/delay"
 import * as queryRow from "./steps/queryRows"
-import * as loop from "./steps/loop"
 import * as collect from "./steps/collect"
-import * as branch from "./steps/branch"
 import * as triggerAutomationRun from "./steps/triggerAutomationRun"
 import * as openai from "./steps/openai"
+import * as bash from "./steps/bash"
 import env from "../environment"
 import {
   PluginType,
@@ -64,43 +64,40 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<
   string,
   AutomationStepDefinition
 > = {
-  SEND_EMAIL_SMTP: sendSmtpEmail.definition,
+  SEND_EMAIL_SMTP: automations.steps.sendSmtpEmail.definition,
-  CREATE_ROW: createRow.definition,
+  CREATE_ROW: automations.steps.createRow.definition,
-  UPDATE_ROW: updateRow.definition,
+  UPDATE_ROW: automations.steps.updateRow.definition,
-  DELETE_ROW: deleteRow.definition,
+  DELETE_ROW: automations.steps.deleteRow.definition,
-  OUTGOING_WEBHOOK: outgoingWebhook.definition,
+  OUTGOING_WEBHOOK: automations.steps.outgoingWebhook.definition,
-  EXECUTE_SCRIPT: executeScript.definition,
+  EXECUTE_SCRIPT: automations.steps.executeScript.definition,
-  EXECUTE_SCRIPT_V2: executeScriptV2.definition,
+  EXECUTE_SCRIPT_V2: automations.steps.executeScriptV2.definition,
-  EXECUTE_QUERY: executeQuery.definition,
+  EXECUTE_QUERY: automations.steps.executeQuery.definition,
-  SERVER_LOG: serverLog.definition,
+  SERVER_LOG: automations.steps.serverLog.definition,
-  DELAY: delay.definition,
+  DELAY: automations.steps.delay.definition,
-  FILTER: filter.definition,
+  FILTER: automations.steps.filter.definition,
-  QUERY_ROWS: queryRow.definition,
+  QUERY_ROWS: automations.steps.queryRows.definition,
-  LOOP: loop.definition,
+  LOOP: automations.steps.loop.definition,
-  COLLECT: collect.definition,
+  COLLECT: automations.steps.collect.definition,
-  TRIGGER_AUTOMATION_RUN: triggerAutomationRun.definition,
+  TRIGGER_AUTOMATION_RUN: automations.steps.triggerAutomationRun.definition,
-  BRANCH: branch.definition,
+  BRANCH: automations.steps.branch.definition,
   // these used to be lowercase step IDs, maintain for backwards compat
-  discord: discord.definition,
+  discord: automations.steps.discord.definition,
-  slack: slack.definition,
+  slack: automations.steps.slack.definition,
-  zapier: zapier.definition,
+  zapier: automations.steps.zapier.definition,
-  integromat: make.definition,
+  integromat: automations.steps.make.definition,
-  n8n: n8n.definition,
+  n8n: automations.steps.n8n.definition,
 }
 
 // don't add the bash script/definitions unless in self host
 // the fact this isn't included in any definitions means it cannot be
 // ran at all
 if (env.SELF_HOSTED) {
-  const bash = require("./steps/bash")
+  // @ts-expect-error
-
-  // @ts-ignore
   ACTION_IMPLS["EXECUTE_BASH"] = bash.run
-  // @ts-ignore
+  BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = automations.steps.bash.definition
-  BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
 
   if (env.isTest()) {
-    BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition
+    BUILTIN_ACTION_DEFINITIONS["OPENAI"] = automations.steps.openai.definition
   }
 }
 
@@ -108,7 +105,7 @@ export async function getActionDefinitions(): Promise<
   Record<keyof typeof AutomationActionStepId, AutomationStepDefinition>
 > {
   if (env.SELF_HOSTED) {
-    BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition
+    BUILTIN_ACTION_DEFINITIONS["OPENAI"] = automations.steps.openai.definition
   }
 
   const actionDefinitions = BUILTIN_ACTION_DEFINITIONS

packages/server/src/automations/steps/bash.ts

@@ -2,55 +2,7 @@ import { execSync } from "child_process"
 import { processStringSync } from "@budibase/string-templates"
 import * as automationUtils from "../automationUtils"
 import environment from "../../environment"
-import {
+import { BashStepInputs, BashStepOutputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
-  BashStepInputs,
-  BashStepOutputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  name: "Bash Scripting",
-  tagline: "Execute a bash command",
-  icon: "JourneyEvent",
-  description: "Run a bash script",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  stepId: AutomationActionStepId.EXECUTE_BASH,
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        code: {
-          type: AutomationIOType.STRING,
-          customType: AutomationCustomIOType.CODE,
-          title: "Code",
-        },
-      },
-      required: ["code"],
-    },
-    outputs: {
-      properties: {
-        stdout: {
-          type: AutomationIOType.STRING,
-          description: "Standard output of your bash command or script",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the command was successful",
-        },
-      },
-      required: ["stdout"],
-    },
-  },
-}
 
 export async function run({
   inputs,

packages/server/src/automations/steps/collect.ts

@@ -1,48 +1,4 @@
-import {
+import { CollectStepInputs, CollectStepOutputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  CollectStepInputs,
-  CollectStepOutputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  name: "Collect Data",
-  tagline: "Collect data to be sent to design",
-  icon: "Collection",
-  description:
-    "Collects specified data so it can be provided to the design section",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {},
-  stepId: AutomationActionStepId.COLLECT,
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        collection: {
-          type: AutomationIOType.STRING,
-          title: "What to Collect",
-        },
-      },
-      required: ["collection"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-        value: {
-          type: AutomationIOType.STRING,
-          description: "Collected data",
-        },
-      },
-      required: ["success", "value"],
-    },
-  },
-}
 
 export async function run({
   inputs,

packages/server/src/automations/steps/createRow.ts

@@ -5,77 +5,9 @@ import {
   sendAutomationAttachmentsToStorage,
 } from "../automationUtils"
 import { buildCtx } from "./utils"
-import {
+import { CreateRowStepInputs, CreateRowStepOutputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
-  CreateRowStepInputs,
-  CreateRowStepOutputs,
-} from "@budibase/types"
 import { EventEmitter } from "events"
 
-export const definition: AutomationStepDefinition = {
-  name: "Create Row",
-  tagline: "Create a {{inputs.enriched.table.name}} row",
-  icon: "TableRowAddBottom",
-  description: "Add a row to your database",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  stepId: AutomationActionStepId.CREATE_ROW,
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        row: {
-          type: AutomationIOType.OBJECT,
-          properties: {
-            tableId: {
-              type: AutomationIOType.STRING,
-              customType: AutomationCustomIOType.TABLE,
-            },
-          },
-          customType: AutomationCustomIOType.ROW,
-          title: "Table",
-          required: ["tableId"],
-        },
-      },
-      required: ["row"],
-    },
-    outputs: {
-      properties: {
-        row: {
-          type: AutomationIOType.OBJECT,
-          customType: AutomationCustomIOType.ROW,
-          description: "The new row",
-        },
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The response from the table",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the row creation was successful",
-        },
-        id: {
-          type: AutomationIOType.STRING,
-          description: "The identifier of the new row",
-        },
-        revision: {
-          type: AutomationIOType.STRING,
-          description: "The revision of the new row",
-        },
-      },
-      required: ["success", "id", "revision"],
-    },
-  },
-}
-
 export async function run({
   inputs,
   appId,

packages/server/src/automations/steps/delay.ts

@@ -1,44 +1,5 @@
 import { wait } from "../../utilities"
-import {
+import { DelayStepInputs, DelayStepOutputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
-  DelayStepInputs,
-  DelayStepOutputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  name: "Delay",
-  icon: "Clock",
-  tagline: "Delay for {{inputs.time}} milliseconds",
-  description: "Delay the automation until an amount of time has passed",
-  stepId: AutomationActionStepId.DELAY,
-  internal: true,
-  features: {},
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        time: {
-          type: AutomationIOType.NUMBER,
-          title: "Delay in milliseconds",
-        },
-      },
-      required: ["time"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the delay was successful",
-        },
-      },
-      required: ["success"],
-    },
-  },
-  type: AutomationStepType.LOGIC,
-}
 
 export async function run({
   inputs,

packages/server/src/automations/steps/deleteRow.ts

@@ -2,64 +2,7 @@ import { EventEmitter } from "events"
 import { destroy } from "../../api/controllers/row"
 import { buildCtx } from "./utils"
 import { getError } from "../automationUtils"
-import {
+import { DeleteRowStepInputs, DeleteRowStepOutputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationCustomIOType,
-  AutomationFeature,
-  DeleteRowStepInputs,
-  DeleteRowStepOutputs,
-  AutomationStepDefinition,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  description: "Delete a row from your database",
-  icon: "TableRowRemoveCenter",
-  name: "Delete Row",
-  tagline: "Delete a {{inputs.enriched.table.name}} row",
-  type: AutomationStepType.ACTION,
-  stepId: AutomationActionStepId.DELETE_ROW,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        tableId: {
-          type: AutomationIOType.STRING,
-          customType: AutomationCustomIOType.TABLE,
-          title: "Table",
-        },
-        id: {
-          type: AutomationIOType.STRING,
-          title: "Row ID",
-        },
-      },
-      required: ["tableId", "id"],
-    },
-    outputs: {
-      properties: {
-        row: {
-          type: AutomationIOType.OBJECT,
-          customType: AutomationCustomIOType.ROW,
-          description: "The deleted row",
-        },
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The response from the table",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the deletion was successful",
-        },
-      },
-      required: ["row", "success"],
-    },
-  },
-}
 
 export async function run({
   inputs,

packages/server/src/automations/steps/discord.ts

@@ -1,71 +1,10 @@
 import fetch from "node-fetch"
 import { getFetchResponse } from "./utils"
-import {
+import { ExternalAppStepOutputs, DiscordStepInputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
-  ExternalAppStepOutputs,
-  DiscordStepInputs,
-  AutomationStepDefinition,
-} from "@budibase/types"
 
 const DEFAULT_USERNAME = "Budibase Automate"
 const DEFAULT_AVATAR_URL = "https://i.imgur.com/a1cmTKM.png"
 
-export const definition: AutomationStepDefinition = {
-  name: "Discord Message",
-  tagline: "Send a message to a Discord server",
-  description: "Send a message to a Discord server",
-  icon: "ri-discord-line",
-  stepId: AutomationActionStepId.discord,
-  type: AutomationStepType.ACTION,
-  internal: false,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        url: {
-          type: AutomationIOType.STRING,
-          title: "Discord Webhook URL",
-        },
-        username: {
-          type: AutomationIOType.STRING,
-          title: "Bot Name",
-        },
-        avatar_url: {
-          type: AutomationIOType.STRING,
-          title: "Bot Avatar URL",
-        },
-        content: {
-          type: AutomationIOType.STRING,
-          title: "Message",
-        },
-      },
-      required: ["url", "content"],
-    },
-    outputs: {
-      properties: {
-        httpStatus: {
-          type: AutomationIOType.NUMBER,
-          description: "The HTTP status code of the request",
-        },
-        response: {
-          type: AutomationIOType.STRING,
-          description: "The response from the Discord Webhook",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the message sent successfully",
-        },
-      },
-    },
-  },
-}
-
 export async function run({
   inputs,
 }: {

packages/server/src/automations/steps/executeQuery.ts

@@ -3,67 +3,10 @@ import * as queryController from "../../api/controllers/query"
 import { buildCtx } from "./utils"
 import * as automationUtils from "../automationUtils"
 import {
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
   ExecuteQueryStepInputs,
   ExecuteQueryStepOutputs,
 } from "@budibase/types"
 
-export const definition: AutomationStepDefinition = {
-  name: "External Data Connector",
-  tagline: "Execute Data Connector",
-  icon: "Data",
-  description: "Execute a query in an external data connector",
-  type: AutomationStepType.ACTION,
-  stepId: AutomationActionStepId.EXECUTE_QUERY,
-  internal: true,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        query: {
-          type: AutomationIOType.OBJECT,
-          properties: {
-            queryId: {
-              type: AutomationIOType.STRING,
-              customType: AutomationCustomIOType.QUERY,
-            },
-          },
-          customType: AutomationCustomIOType.QUERY_PARAMS,
-          title: "Parameters",
-          required: ["queryId"],
-        },
-      },
-      required: ["query"],
-    },
-    outputs: {
-      properties: {
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The response from the datasource execution",
-        },
-        info: {
-          type: AutomationIOType.OBJECT,
-          description:
-            "Some query types may return extra data, like headers from a REST query",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-      },
-      required: ["response", "success"],
-    },
-  },
-}
-
 export async function run({
   inputs,
   appId,

packages/server/src/automations/steps/executeScript.ts

@@ -2,57 +2,11 @@ import * as scriptController from "../../api/controllers/script"
 import { buildCtx } from "./utils"
 import * as automationUtils from "../automationUtils"
 import {
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
   ExecuteScriptStepInputs,
   ExecuteScriptStepOutputs,
 } from "@budibase/types"
 import { EventEmitter } from "events"
 
-export const definition: AutomationStepDefinition = {
-  name: "JS Scripting",
-  deprecated: true,
-  tagline: "Execute JavaScript Code",
-  icon: "Code",
-  description: "Run a piece of JavaScript code in your automation",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  stepId: AutomationActionStepId.EXECUTE_SCRIPT,
-  inputs: {},
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  schema: {
-    inputs: {
-      properties: {
-        code: {
-          type: AutomationIOType.STRING,
-          customType: AutomationCustomIOType.CODE,
-          title: "Code",
-        },
-      },
-      required: ["code"],
-    },
-    outputs: {
-      properties: {
-        value: {
-          type: AutomationIOType.STRING,
-          description: "The result of the return statement",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-      },
-      required: ["success"],
-    },
-  },
-}
-
 export async function run({
   inputs,
   appId,

packages/server/src/automations/steps/executeScriptV2.ts

@@ -1,56 +1,10 @@
 import * as automationUtils from "../automationUtils"
 import {
-  AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationFeature,
-  AutomationIOType,
-  AutomationStepDefinition,
-  AutomationStepType,
   ExecuteScriptStepInputs,
   ExecuteScriptStepOutputs,
 } from "@budibase/types"
 import { processStringSync } from "@budibase/string-templates"
 
-export const definition: AutomationStepDefinition = {
-  name: "JavaScript",
-  tagline: "Execute JavaScript Code",
-  icon: "Brackets",
-  description: "Run a piece of JavaScript code in your automation",
-  type: AutomationStepType.ACTION,
-  internal: true,
-  new: true,
-  stepId: AutomationActionStepId.EXECUTE_SCRIPT_V2,
-  inputs: {},
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  schema: {
-    inputs: {
-      properties: {
-        code: {
-          type: AutomationIOType.STRING,
-          customType: AutomationCustomIOType.CODE,
-          title: "Code",
-        },
-      },
-      required: ["code"],
-    },
-    outputs: {
-      properties: {
-        value: {
-          type: AutomationIOType.STRING,
-          description: "The result of the return statement",
-        },
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-      },
-      required: ["success"],
-    },
-  },
-}
-
 export async function run({
   inputs,
   context,

packages/server/src/automations/steps/filter.ts

@@ -1,74 +1,7 @@
-import {
+import { FilterStepInputs, FilterStepOutputs } from "@budibase/types"
-  AutomationActionStepId,
+import { automations } from "@budibase/shared-core"
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  FilterStepInputs,
-  FilterStepOutputs,
-} from "@budibase/types"
 
-export const FilterConditions = {
+const FilterConditions = automations.steps.filter.FilterConditions
-  EQUAL: "EQUAL",
-  NOT_EQUAL: "NOT_EQUAL",
-  GREATER_THAN: "GREATER_THAN",
-  LESS_THAN: "LESS_THAN",
-}
-
-export const PrettyFilterConditions = {
-  [FilterConditions.EQUAL]: "Equals",
-  [FilterConditions.NOT_EQUAL]: "Not equals",
-  [FilterConditions.GREATER_THAN]: "Greater than",
-  [FilterConditions.LESS_THAN]: "Less than",
-}
-
-export const definition: AutomationStepDefinition = {
-  name: "Condition",
-  tagline: "{{inputs.field}} {{inputs.condition}} {{inputs.value}}",
-  icon: "Branch2",
-  description:
-    "Conditionally halt automations which do not meet certain conditions",
-  type: AutomationStepType.LOGIC,
-  internal: true,
-  features: {},
-  stepId: AutomationActionStepId.FILTER,
-  inputs: {
-    condition: FilterConditions.EQUAL,
-  },
-  schema: {
-    inputs: {
-      properties: {
-        field: {
-          type: AutomationIOType.STRING,
-          title: "Reference Value",
-        },
-        condition: {
-          type: AutomationIOType.STRING,
-          title: "Condition",
-          enum: Object.values(FilterConditions),
-          pretty: Object.values(PrettyFilterConditions),
-        },
-        value: {
-          type: AutomationIOType.STRING,
-          title: "Comparison Value",
-        },
-      },
-      required: ["field", "condition", "value"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the action was successful",
-        },
-        result: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether the logic block passed",
-        },
-      },
-      required: ["success", "result"],
-    },
-  },
-}
 
 export async function run({
   inputs,

packages/server/src/automations/steps/make.ts

@@ -1,62 +1,6 @@
 import fetch from "node-fetch"
 import { getFetchResponse } from "./utils"
-import {
+import { ExternalAppStepOutputs, MakeIntegrationInputs } from "@budibase/types"
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
-  ExternalAppStepOutputs,
-  MakeIntegrationInputs,
-} from "@budibase/types"
-
-export const definition: AutomationStepDefinition = {
-  name: "Make Integration",
-  stepTitle: "Make",
-  tagline: "Trigger a Make scenario",
-  description:
-    "Performs a webhook call to Make and gets the response (if configured)",
-  icon: "ri-shut-down-line",
-  stepId: AutomationActionStepId.integromat,
-  type: AutomationStepType.ACTION,
-  internal: false,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        url: {
-          type: AutomationIOType.STRING,
-          title: "Webhook URL",
-        },
-        body: {
-          type: AutomationIOType.JSON,
-          title: "Payload",
-        },
-      },
-      required: ["url", "body"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether call was successful",
-        },
-        httpStatus: {
-          type: AutomationIOType.NUMBER,
-          description: "The HTTP status code returned",
-        },
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The webhook response - this can have properties",
-        },
-      },
-      required: ["success", "response"],
-    },
-  },
-}
 
 export async function run({
   inputs,

packages/server/src/automations/steps/n8n.ts

@@ -1,73 +1,11 @@
 import fetch, { HeadersInit } from "node-fetch"
 import { getFetchResponse } from "./utils"
 import {
-  AutomationActionStepId,
-  AutomationStepDefinition,
-  AutomationStepType,
-  AutomationIOType,
-  AutomationFeature,
   HttpMethod,
   ExternalAppStepOutputs,
   n8nStepInputs,
 } from "@budibase/types"
 
-export const definition: AutomationStepDefinition = {
-  name: "n8n Integration",
-  stepTitle: "n8n",
-  tagline: "Trigger an n8n workflow",
-  description:
-    "Performs a webhook call to n8n and gets the response (if configured)",
-  icon: "ri-shut-down-line",
-  stepId: AutomationActionStepId.n8n,
-  type: AutomationStepType.ACTION,
-  internal: false,
-  features: {
-    [AutomationFeature.LOOPING]: true,
-  },
-  inputs: {},
-  schema: {
-    inputs: {
-      properties: {
-        url: {
-          type: AutomationIOType.STRING,
-          title: "Webhook URL",
-        },
-        method: {
-          type: AutomationIOType.STRING,
-          title: "Method",
-          enum: Object.values(HttpMethod),
-        },
-        authorization: {
-          type: AutomationIOType.STRING,
-          title: "Authorization",
-        },
-        body: {
-          type: AutomationIOType.JSON,
-          title: "Payload",
-        },
-      },
-      required: ["url", "method"],
-    },
-    outputs: {
-      properties: {
-        success: {
-          type: AutomationIOType.BOOLEAN,
-          description: "Whether call was successful",
-        },
-        httpStatus: {
-          type: AutomationIOType.NUMBER,
-          description: "The HTTP status code returned",
-        },
-        response: {
-          type: AutomationIOType.OBJECT,
-          description: "The webhook response - this can have properties",
-        },
-      },
-      required: ["success", "response"],
-    },
-  },
-}
-
 export async function run({
   inputs,
 }: {