Merge branch 'master' into BUDI-8885/only-select-required-columns-from-sql-databases

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.37",
+  "version": "3.2.39",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/src/middleware/errorHandling.ts

@@ -32,8 +32,12 @@ export async function errorHandling(ctx: any, next: any) {
     }
 
     if (environment.isTest() && ctx.headers["x-budibase-include-stacktrace"]) {
+      let rootErr = err
+      while (rootErr.cause) {
+        rootErr = rootErr.cause
+      }
       // @ts-ignore
-      error.stack = err.stack
+      error.stack = rootErr.stack
     }
 
     ctx.body = error

packages/backend-core/src/sql/sql.ts

@@ -802,14 +802,29 @@ class InternalBuilder {
         filters.oneOf,
         ArrayOperator.ONE_OF,
         (q, key: string, array) => {
+          const schema = this.getFieldSchema(key)
+          const values = Array.isArray(array) ? array : [array]
           if (shouldOr) {
             q = q.or
           }
           if (this.client === SqlClient.ORACLE) {
             // @ts-ignore
             key = this.convertClobs(key)
+          } else if (
+            this.client === SqlClient.SQL_LITE &&
+            schema?.type === FieldType.DATETIME &&
+            schema.dateOnly
+          ) {
+            for (const value of values) {
+              if (value != null) {
+                q = q.or.whereLike(key, `${value.toISOString().slice(0, 10)}%`)
+              } else {
+                q = q.or.whereNull(key)
+              }
+            }
+            return q
           }
-          return q.whereIn(key, Array.isArray(array) ? array : [array])
+          return q.whereIn(key, values)
         },
         (q, key: string[], array) => {
           if (shouldOr) {
@@ -868,6 +883,19 @@ class InternalBuilder {
         let high = value.high
         let low = value.low
 
+        if (
+          this.client === SqlClient.SQL_LITE &&
+          schema?.type === FieldType.DATETIME &&
+          schema.dateOnly
+        ) {
+          if (high != null) {
+            high = `${high.toISOString().slice(0, 10)}T23:59:59.999Z`
+          }
+          if (low != null) {
+            low = low.toISOString().slice(0, 10)
+          }
+        }
+
         if (this.client === SqlClient.ORACLE) {
           rawKey = this.convertClobs(key)
         } else if (
@@ -900,6 +928,7 @@ class InternalBuilder {
     }
     if (filters.equal) {
       iterate(filters.equal, BasicOperator.EQUAL, (q, key, value) => {
+        const schema = this.getFieldSchema(key)
         if (shouldOr) {
           q = q.or
         }
@@ -914,6 +943,16 @@ class InternalBuilder {
             // @ts-expect-error knex types are wrong, raw is fine here
             subq.whereNotNull(identifier).andWhere(identifier, value)
           )
+        } else if (
+          this.client === SqlClient.SQL_LITE &&
+          schema?.type === FieldType.DATETIME &&
+          schema.dateOnly
+        ) {
+          if (value != null) {
+            return q.whereLike(key, `${value.toISOString().slice(0, 10)}%`)
+          } else {
+            return q.whereNull(key)
+          }
         } else {
           return q.whereRaw(`COALESCE(?? = ?, FALSE)`, [
             this.rawQuotedIdentifier(key),
@@ -924,6 +963,7 @@ class InternalBuilder {
     }
     if (filters.notEqual) {
       iterate(filters.notEqual, BasicOperator.NOT_EQUAL, (q, key, value) => {
+        const schema = this.getFieldSchema(key)
         if (shouldOr) {
           q = q.or
         }
@@ -945,6 +985,18 @@ class InternalBuilder {
               // @ts-expect-error knex types are wrong, raw is fine here
               .or.whereNull(identifier)
           )
+        } else if (
+          this.client === SqlClient.SQL_LITE &&
+          schema?.type === FieldType.DATETIME &&
+          schema.dateOnly
+        ) {
+          if (value != null) {
+            return q.not
+              .whereLike(key, `${value.toISOString().slice(0, 10)}%`)
+              .or.whereNull(key)
+          } else {
+            return q.not.whereNull(key)
+          }
         } else {
           return q.whereRaw(`COALESCE(?? != ?, TRUE)`, [
             this.rawQuotedIdentifier(key),

packages/backend-core/src/sql/utils.ts

@@ -14,7 +14,7 @@ import environment from "../environment"
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
 const ENCODED_SPACE = encodeURIComponent(" ")
-const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/
+const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}.\d{3}Z)?$/
 const TIME_REGEX = /^(?:\d{2}:)?(?:\d{2}:)(?:\d{2})$/
 
 export function isExternalTableID(tableId: string) {
@@ -149,15 +149,7 @@ export function isInvalidISODateString(str: string) {
 }
 
 export function isValidISODateString(str: string) {
-  const trimmedValue = str.trim()
+  return ISO_DATE_REGEX.test(str.trim())
-  if (!ISO_DATE_REGEX.test(trimmedValue)) {
-    return false
-  }
-  let d = new Date(trimmedValue)
-  if (isNaN(d.getTime())) {
-    return false
-  }
-  return d.toISOString() === trimmedValue
 }
 
 export function isValidFilter(value: any) {

packages/bbui/src/helpers.d.ts

@@ -0,0 +1,3 @@
+declare module "./helpers" {
+  export const cloneDeep: <T>(obj: T) => T
+}

packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte

@@ -43,7 +43,6 @@
   export let showDataProviders = true
 
   const dispatch = createEventDispatcher()
-  const arrayTypes = ["attachment", "array"]
 
   let anchorRight, dropdownRight
   let drawer
@@ -116,8 +115,11 @@
       }
     })
   $: fields = bindings
-    .filter(x => arrayTypes.includes(x.fieldSchema?.type))
+    .filter(
-    .filter(x => x.fieldSchema?.tableId != null)
+      x =>
+        x.fieldSchema?.type === "attachment" ||
+        (x.fieldSchema?.type === "array" && x.tableId)
+    )
     .map(binding => {
       const { providerId, readableBinding, runtimeBinding } = binding
       const { name, type, tableId } = binding.fieldSchema

packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte

@@ -442,13 +442,11 @@
 
   const onUpdateUserInvite = async (invite, role) => {
     let updateBody = {
-      code: invite.code,
       apps: {
         ...invite.apps,
         [prodAppId]: role,
       },
     }
-
     if (role === Constants.Roles.CREATOR) {
       updateBody.builder = updateBody.builder || {}
       updateBody.builder.apps = [...(updateBody.builder.apps ?? []), prodAppId]
@@ -456,7 +454,7 @@
     } else if (role !== Constants.Roles.CREATOR && invite?.builder?.apps) {
       invite.builder.apps = []
     }
-    await users.updateInvite(updateBody)
+    await users.updateInvite(invite.code, updateBody)
     await filterInvites(query)
   }
 
@@ -470,8 +468,7 @@
     let updated = { ...invite }
     delete updated.info.apps[prodAppId]
 
-    return await users.updateInvite({
+    return await users.updateInvite(updated.code, {
-      code: updated.code,
       apps: updated.apps,
     })
   }

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -191,8 +191,14 @@
           ? "View errors"
           : "View error"}
         on:dismiss={async () => {
-          await automationStore.actions.clearLogErrors({ appId })
+          const automationId = Object.keys(automationErrors[appId] || {})[0]
-          await appsStore.load()
+          if (automationId) {
+            await automationStore.actions.clearLogErrors({
+              appId,
+              automationId,
+            })
+            await appsStore.load()
+          }
         }}
         message={automationErrorMessage(appId)}
       />

packages/builder/src/pages/builder/portal/users/groups/_components/GroupUsers.svelte

@@ -52,7 +52,7 @@
   ]
 
   const removeUser = async id => {
-    await groups.actions.removeUser(groupId, id)
+    await groups.removeUser(groupId, id)
     fetchGroupUsers.refresh()
   }
 

packages/builder/src/pages/builder/portal/users/users/index.svelte

@@ -251,6 +251,7 @@
       passwordModal.show()
       await fetch.refresh()
     } catch (error) {
+      console.error(error)
       notifications.error("Error creating user")
     }
   }

packages/builder/src/stores/builder/queries.js

@@ -1,130 +0,0 @@
-import { writable, get, derived } from "svelte/store"
-import { datasources } from "./datasources"
-import { integrations } from "./integrations"
-import { API } from "@/api"
-import { duplicateName } from "@/helpers/duplicate"
-
-const sortQueries = queryList => {
-  queryList.sort((q1, q2) => {
-    return q1.name.localeCompare(q2.name)
-  })
-}
-
-export function createQueriesStore() {
-  const store = writable({
-    list: [],
-    selectedQueryId: null,
-  })
-  const derivedStore = derived(store, $store => ({
-    ...$store,
-    selected: $store.list?.find(q => q._id === $store.selectedQueryId),
-  }))
-
-  const fetch = async () => {
-    const queries = await API.getQueries()
-    sortQueries(queries)
-    store.update(state => ({
-      ...state,
-      list: queries,
-    }))
-  }
-
-  const save = async (datasourceId, query) => {
-    const _integrations = get(integrations)
-    const dataSource = get(datasources).list.filter(
-      ds => ds._id === datasourceId
-    )
-    // Check if readable attribute is found
-    if (dataSource.length !== 0) {
-      const integration = _integrations[dataSource[0].source]
-      const readable = integration.query[query.queryVerb].readable
-      if (readable) {
-        query.readable = readable
-      }
-    }
-    query.datasourceId = datasourceId
-    const savedQuery = await API.saveQuery(query)
-    store.update(state => {
-      const idx = state.list.findIndex(query => query._id === savedQuery._id)
-      const queries = state.list
-      if (idx >= 0) {
-        queries.splice(idx, 1, savedQuery)
-      } else {
-        queries.push(savedQuery)
-      }
-      sortQueries(queries)
-      return {
-        list: queries,
-        selectedQueryId: savedQuery._id,
-      }
-    })
-    return savedQuery
-  }
-
-  const importQueries = async ({ data, datasourceId }) => {
-    return await API.importQueries(datasourceId, data)
-  }
-
-  const select = id => {
-    store.update(state => ({
-      ...state,
-      selectedQueryId: id,
-    }))
-  }
-
-  const preview = async query => {
-    const result = await API.previewQuery(query)
-    // Assume all the fields are strings and create a basic schema from the
-    // unique fields returned by the server
-    const schema = {}
-    for (let [field, metadata] of Object.entries(result.schema)) {
-      schema[field] = metadata || { type: "string" }
-    }
-    return { ...result, schema, rows: result.rows || [] }
-  }
-
-  const deleteQuery = async query => {
-    await API.deleteQuery(query._id, query._rev)
-    store.update(state => {
-      state.list = state.list.filter(existing => existing._id !== query._id)
-      return state
-    })
-  }
-
-  const duplicate = async query => {
-    let list = get(store).list
-    const newQuery = { ...query }
-    const datasourceId = query.datasourceId
-
-    delete newQuery._id
-    delete newQuery._rev
-    newQuery.name = duplicateName(
-      query.name,
-      list.map(q => q.name)
-    )
-
-    return await save(datasourceId, newQuery)
-  }
-
-  const removeDatasourceQueries = datasourceId => {
-    store.update(state => ({
-      ...state,
-      list: state.list.filter(table => table.datasourceId !== datasourceId),
-    }))
-  }
-
-  return {
-    subscribe: derivedStore.subscribe,
-    fetch,
-    init: fetch,
-    select,
-    save,
-    import: importQueries,
-    delete: deleteQuery,
-    preview,
-    duplicate,
-    removeDatasourceQueries,
-  }
-}
-
-export const queries = createQueriesStore()

packages/builder/src/stores/builder/queries.ts

@@ -0,0 +1,156 @@
+import { derived, get, Writable } from "svelte/store"
+import { datasources } from "./datasources"
+import { integrations } from "./integrations"
+import { API } from "@/api"
+import { duplicateName } from "@/helpers/duplicate"
+import { DerivedBudiStore } from "@/stores/BudiStore"
+import {
+  Query,
+  QueryPreview,
+  PreviewQueryResponse,
+  SaveQueryRequest,
+  ImportRestQueryRequest,
+  QuerySchema,
+} from "@budibase/types"
+
+const sortQueries = (queryList: Query[]) => {
+  queryList.sort((q1, q2) => {
+    return q1.name.localeCompare(q2.name)
+  })
+}
+
+interface BuilderQueryStore {
+  list: Query[]
+  selectedQueryId: string | null
+}
+
+interface DerivedQueryStore extends BuilderQueryStore {
+  selected?: Query
+}
+
+export class QueryStore extends DerivedBudiStore<
+  BuilderQueryStore,
+  DerivedQueryStore
+> {
+  constructor() {
+    const makeDerivedStore = (store: Writable<BuilderQueryStore>) => {
+      return derived(store, ($store): DerivedQueryStore => {
+        return {
+          list: $store.list,
+          selectedQueryId: $store.selectedQueryId,
+          selected: $store.list?.find(q => q._id === $store.selectedQueryId),
+        }
+      })
+    }
+
+    super(
+      {
+        list: [],
+        selectedQueryId: null,
+      },
+      makeDerivedStore
+    )
+
+    this.select = this.select.bind(this)
+  }
+
+  async fetch() {
+    const queries = await API.getQueries()
+    sortQueries(queries)
+    this.store.update(state => ({
+      ...state,
+      list: queries,
+    }))
+  }
+
+  async save(datasourceId: string, query: SaveQueryRequest) {
+    const _integrations = get(integrations)
+    const dataSource = get(datasources).list.filter(
+      ds => ds._id === datasourceId
+    )
+    // Check if readable attribute is found
+    if (dataSource.length !== 0) {
+      const integration = _integrations[dataSource[0].source]
+      const readable = integration.query[query.queryVerb].readable
+      if (readable) {
+        query.readable = readable
+      }
+    }
+    query.datasourceId = datasourceId
+    const savedQuery = await API.saveQuery(query)
+    this.store.update(state => {
+      const idx = state.list.findIndex(query => query._id === savedQuery._id)
+      const queries = state.list
+      if (idx >= 0) {
+        queries.splice(idx, 1, savedQuery)
+      } else {
+        queries.push(savedQuery)
+      }
+      sortQueries(queries)
+      return {
+        list: queries,
+        selectedQueryId: savedQuery._id || null,
+      }
+    })
+    return savedQuery
+  }
+
+  async importQueries(data: ImportRestQueryRequest) {
+    return await API.importQueries(data)
+  }
+
+  select(id: string | null) {
+    this.store.update(state => ({
+      ...state,
+      selectedQueryId: id,
+    }))
+  }
+
+  async preview(query: QueryPreview): Promise<PreviewQueryResponse> {
+    const result = await API.previewQuery(query)
+    // Assume all the fields are strings and create a basic schema from the
+    // unique fields returned by the server
+    const schema: Record<string, QuerySchema> = {}
+    for (let [field, metadata] of Object.entries(result.schema)) {
+      schema[field] = (metadata as QuerySchema) || { type: "string" }
+    }
+    return { ...result, schema, rows: result.rows || [] }
+  }
+
+  async delete(query: Query) {
+    if (!query._id || !query._rev) {
+      throw new Error("Query ID or Revision is missing")
+    }
+    await API.deleteQuery(query._id, query._rev)
+    this.store.update(state => ({
+      ...state,
+      list: state.list.filter(existing => existing._id !== query._id),
+    }))
+  }
+
+  async duplicate(query: Query) {
+    let list = get(this.store).list
+    const newQuery = { ...query }
+    const datasourceId = query.datasourceId
+
+    delete newQuery._id
+    delete newQuery._rev
+    newQuery.name = duplicateName(
+      query.name,
+      list.map(q => q.name)
+    )
+
+    return await this.save(datasourceId, newQuery)
+  }
+
+  removeDatasourceQueries(datasourceId: string) {
+    this.store.update(state => ({
+      ...state,
+      list: state.list.filter(table => table.datasourceId !== datasourceId),
+    }))
+  }
+
+  init = this.fetch
+}
+
+export const queries = new QueryStore()

packages/builder/src/stores/portal/menu.js

@@ -1,138 +0,0 @@
-import { derived } from "svelte/store"
-import { admin } from "./admin"
-import { auth } from "./auth"
-import { isEnabled } from "@/helpers/featureFlags"
-import { sdk } from "@budibase/shared-core"
-import { FeatureFlag } from "@budibase/types"
-
-export const menu = derived([admin, auth], ([$admin, $auth]) => {
-  const user = $auth?.user
-  const isAdmin = sdk.users.isAdmin(user)
-  const cloud = $admin?.cloud
-  // Determine user sub pages
-  let userSubPages = [
-    {
-      title: "Users",
-      href: "/builder/portal/users/users",
-    },
-  ]
-  userSubPages.push({
-    title: "Groups",
-    href: "/builder/portal/users/groups",
-  })
-
-  // Pages that all devs and admins can access
-  let menu = [
-    {
-      title: "Apps",
-      href: "/builder/portal/apps",
-    },
-  ]
-  if (sdk.users.isGlobalBuilder(user)) {
-    menu.push({
-      title: "Users",
-      href: "/builder/portal/users",
-      subPages: userSubPages,
-    })
-    menu.push({
-      title: "Plugins",
-      href: "/builder/portal/plugins",
-    })
-  }
-
-  // Add settings page for admins
-  if (isAdmin) {
-    let settingsSubPages = [
-      {
-        title: "Auth",
-        href: "/builder/portal/settings/auth",
-      },
-      {
-        title: "Email",
-        href: "/builder/portal/settings/email",
-      },
-      {
-        title: "Organisation",
-        href: "/builder/portal/settings/organisation",
-      },
-      {
-        title: "Branding",
-        href: "/builder/portal/settings/branding",
-      },
-      {
-        title: "Environment",
-        href: "/builder/portal/settings/environment",
-      },
-    ]
-    if (isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) {
-      settingsSubPages.push({
-        title: "AI",
-        href: "/builder/portal/settings/ai",
-      })
-    }
-
-    if (!cloud) {
-      settingsSubPages.push({
-        title: "Version",
-        href: "/builder/portal/settings/version",
-      })
-      settingsSubPages.push({
-        title: "Diagnostics",
-        href: "/builder/portal/settings/diagnostics",
-      })
-    }
-    menu.push({
-      title: "Settings",
-      href: "/builder/portal/settings",
-      subPages: [...settingsSubPages].sort((a, b) =>
-        a.title.localeCompare(b.title)
-      ),
-    })
-  }
-
-  // Add account page
-  let accountSubPages = [
-    {
-      title: "Usage",
-      href: "/builder/portal/account/usage",
-    },
-  ]
-  if (isAdmin) {
-    accountSubPages.push({
-      title: "Audit Logs",
-      href: "/builder/portal/account/auditLogs",
-    })
-
-    if (!cloud) {
-      accountSubPages.push({
-        title: "System Logs",
-        href: "/builder/portal/account/systemLogs",
-      })
-    }
-  }
-  if (cloud && user?.accountPortalAccess) {
-    accountSubPages.push({
-      title: "Upgrade",
-      href: $admin?.accountPortalUrl + "/portal/upgrade",
-    })
-  } else if (!cloud && isAdmin) {
-    accountSubPages.push({
-      title: "Upgrade",
-      href: "/builder/portal/account/upgrade",
-    })
-  }
-  // add license check here
-  if (user?.accountPortalAccess && user.account.stripeCustomerId) {
-    accountSubPages.push({
-      title: "Billing",
-      href: $admin?.accountPortalUrl + "/portal/billing",
-    })
-  }
-  menu.push({
-    title: "Account",
-    href: "/builder/portal/account",
-    subPages: accountSubPages,
-  })
-
-  return menu
-})

packages/builder/src/stores/portal/menu.ts

@@ -0,0 +1,145 @@
+import { derived, Readable } from "svelte/store"
+import { admin } from "./admin"
+import { auth } from "./auth"
+import { sdk } from "@budibase/shared-core"
+
+interface MenuItem {
+  title: string
+  href: string
+  subPages?: MenuItem[]
+}
+
+export const menu: Readable<MenuItem[]> = derived(
+  [admin, auth],
+  ([$admin, $auth]) => {
+    const user = $auth?.user
+    const isAdmin = user != null && sdk.users.isAdmin(user)
+    const isGlobalBuilder = user != null && sdk.users.isGlobalBuilder(user)
+    const cloud = $admin?.cloud
+
+    // Determine user sub pages
+    let userSubPages: MenuItem[] = [
+      {
+        title: "Users",
+        href: "/builder/portal/users/users",
+      },
+    ]
+    userSubPages.push({
+      title: "Groups",
+      href: "/builder/portal/users/groups",
+    })
+
+    // Pages that all devs and admins can access
+    let menu: MenuItem[] = [
+      {
+        title: "Apps",
+        href: "/builder/portal/apps",
+      },
+    ]
+    if (isGlobalBuilder) {
+      menu.push({
+        title: "Users",
+        href: "/builder/portal/users",
+        subPages: userSubPages,
+      })
+      menu.push({
+        title: "Plugins",
+        href: "/builder/portal/plugins",
+      })
+    }
+
+    // Add settings page for admins
+    if (isAdmin) {
+      let settingsSubPages: MenuItem[] = [
+        {
+          title: "Auth",
+          href: "/builder/portal/settings/auth",
+        },
+        {
+          title: "Email",
+          href: "/builder/portal/settings/email",
+        },
+        {
+          title: "Organisation",
+          href: "/builder/portal/settings/organisation",
+        },
+        {
+          title: "Branding",
+          href: "/builder/portal/settings/branding",
+        },
+        {
+          title: "Environment",
+          href: "/builder/portal/settings/environment",
+        },
+        {
+          title: "AI",
+          href: "/builder/portal/settings/ai",
+        },
+      ]
+
+      if (!cloud) {
+        settingsSubPages.push({
+          title: "Version",
+          href: "/builder/portal/settings/version",
+        })
+        settingsSubPages.push({
+          title: "Diagnostics",
+          href: "/builder/portal/settings/diagnostics",
+        })
+      }
+      menu.push({
+        title: "Settings",
+        href: "/builder/portal/settings",
+        subPages: [...settingsSubPages].sort((a, b) =>
+          a.title.localeCompare(b.title)
+        ),
+      })
+    }
+
+    // Add account page
+    let accountSubPages: MenuItem[] = [
+      {
+        title: "Usage",
+        href: "/builder/portal/account/usage",
+      },
+    ]
+    if (isAdmin) {
+      accountSubPages.push({
+        title: "Audit Logs",
+        href: "/builder/portal/account/auditLogs",
+      })
+
+      if (!cloud) {
+        accountSubPages.push({
+          title: "System Logs",
+          href: "/builder/portal/account/systemLogs",
+        })
+      }
+    }
+    if (cloud && user?.accountPortalAccess) {
+      accountSubPages.push({
+        title: "Upgrade",
+        href: $admin?.accountPortalUrl + "/portal/upgrade",
+      })
+    } else if (!cloud && isAdmin) {
+      accountSubPages.push({
+        title: "Upgrade",
+        href: "/builder/portal/account/upgrade",
+      })
+    }
+    // add license check here
+    if (user?.accountPortalAccess && user?.account?.stripeCustomerId) {
+      accountSubPages.push({
+        title: "Billing",
+        href: $admin?.accountPortalUrl + "/portal/billing",
+      })
+    }
+    menu.push({
+      title: "Account",
+      href: "/builder/portal/account",
+      subPages: accountSubPages,
+    })
+
+    return menu
+  }
+)

packages/builder/src/stores/portal/oidc.js

@@ -1,31 +0,0 @@
-import { writable, get } from "svelte/store"
-import { API } from "@/api"
-import { auth } from "@/stores/portal"
-
-const OIDC_CONFIG = {
-  logo: undefined,
-  name: undefined,
-  uuid: undefined,
-}
-
-export function createOidcStore() {
-  const store = writable(OIDC_CONFIG)
-  const { set, subscribe } = store
-  return {
-    subscribe,
-    set,
-    init: async () => {
-      const tenantId = get(auth).tenantId
-      const config = await API.getOIDCConfig(tenantId)
-      if (Object.keys(config || {}).length) {
-        // Just use the first config for now.
-        // We will be support multiple logins buttons later on.
-        set(...config)
-      } else {
-        set(OIDC_CONFIG)
-      }
-    },
-  }
-}
-
-export const oidc = createOidcStore()

packages/builder/src/stores/portal/oidc.ts

@@ -0,0 +1,21 @@
+import { get } from "svelte/store"
+import { API } from "@/api"
+import { auth } from "@/stores/portal"
+import { BudiStore } from "../BudiStore"
+import { PublicOIDCConfig } from "@budibase/types"
+
+class OIDCStore extends BudiStore<PublicOIDCConfig> {
+  constructor() {
+    super({})
+  }
+
+  async init() {
+    const tenantId = get(auth).tenantId
+    const configs = await API.getOIDCConfigs(tenantId)
+    // Just use the first config for now.
+    // We will be support multiple logins buttons later on.
+    this.set(configs[0] || {})
+  }
+}
+
+export const oidc = new OIDCStore()

packages/builder/src/stores/portal/organisation.js

@@ -1,66 +0,0 @@
-import { writable, get } from "svelte/store"
-import { API } from "@/api"
-import { auth } from "@/stores/portal"
-import _ from "lodash"
-
-const DEFAULT_CONFIG = {
-  platformUrl: "",
-  logoUrl: undefined,
-  faviconUrl: undefined,
-  emailBrandingEnabled: true,
-  testimonialsEnabled: true,
-  platformTitle: "Budibase",
-  loginHeading: undefined,
-  loginButton: undefined,
-  metaDescription: undefined,
-  metaImageUrl: undefined,
-  metaTitle: undefined,
-  docsUrl: undefined,
-  company: "Budibase",
-  oidc: undefined,
-  google: undefined,
-  googleDatasourceConfigured: undefined,
-  oidcCallbackUrl: "",
-  googleCallbackUrl: "",
-  isSSOEnforced: false,
-  loaded: false,
-}
-
-export function createOrganisationStore() {
-  const store = writable(DEFAULT_CONFIG)
-  const { subscribe, set } = store
-
-  async function init() {
-    const tenantId = get(auth).tenantId
-    const settingsConfigDoc = await API.getTenantConfig(tenantId)
-    set({ ...DEFAULT_CONFIG, ...settingsConfigDoc.config, loaded: true })
-  }
-
-  async function save(config) {
-    // Delete non-persisted fields
-    const storeConfig = _.cloneDeep(get(store))
-    delete storeConfig.oidc
-    delete storeConfig.google
-    delete storeConfig.googleDatasourceConfigured
-    delete storeConfig.oidcCallbackUrl
-    delete storeConfig.googleCallbackUrl
-
-    // delete internal store field
-    delete storeConfig.loaded
-
-    await API.saveConfig({
-      type: "settings",
-      config: { ...storeConfig, ...config },
-    })
-    await init()
-  }
-
-  return {
-    subscribe,
-    set,
-    save,
-    init,
-  }
-}
-
-export const organisation = createOrganisationStore()

packages/builder/src/stores/portal/organisation.ts

@@ -0,0 +1,71 @@
+import { get } from "svelte/store"
+import { API } from "@/api"
+import { auth } from "@/stores/portal"
+import {
+  ConfigType,
+  PublicSettingsInnerConfig,
+  SettingsBrandingConfig,
+  SettingsInnerConfig,
+} from "@budibase/types"
+import { BudiStore } from "../BudiStore"
+
+interface LocalOrganisationState {
+  loaded: boolean
+}
+
+type SavedOrganisationState = SettingsInnerConfig & SettingsBrandingConfig
+type OrganisationState = SavedOrganisationState &
+  PublicSettingsInnerConfig &
+  LocalOrganisationState
+
+const DEFAULT_STATE: OrganisationState = {
+  platformUrl: "",
+  emailBrandingEnabled: true,
+  testimonialsEnabled: true,
+  platformTitle: "Budibase",
+  company: "Budibase",
+  google: false,
+  googleDatasourceConfigured: false,
+  oidc: false,
+  oidcCallbackUrl: "",
+  googleCallbackUrl: "",
+  loaded: false,
+}
+
+class OrganisationStore extends BudiStore<OrganisationState> {
+  constructor() {
+    super(DEFAULT_STATE)
+  }
+
+  async init() {
+    const tenantId = get(auth).tenantId
+    const settingsConfigDoc = await API.getTenantConfig(tenantId)
+    this.set({ ...DEFAULT_STATE, ...settingsConfigDoc.config, loaded: true })
+  }
+
+  async save(changes: Partial<SavedOrganisationState>) {
+    // Strip non persisted fields
+    const {
+      oidc,
+      google,
+      googleDatasourceConfigured,
+      oidcCallbackUrl,
+      googleCallbackUrl,
+      loaded,
+      ...config
+    } = get(this.store)
+
+    // Save new config
+    const newConfig: SavedOrganisationState = {
+      ...config,
+      ...changes,
+    }
+    await API.saveConfig({
+      type: ConfigType.SETTINGS,
+      config: newConfig,
+    })
+    await this.init()
+  }
+}
+
+export const organisation = new OrganisationStore()

packages/builder/src/stores/portal/users.ts

@@ -1,41 +1,71 @@
-import { writable } from "svelte/store"
 import { API } from "@/api"
-import { update } from "lodash"
 import { licensing } from "."
 import { sdk } from "@budibase/shared-core"
 import { Constants } from "@budibase/frontend-core"
+import {
+  DeleteInviteUsersRequest,
+  InviteUsersRequest,
+  SearchUsersRequest,
+  SearchUsersResponse,
+  UpdateInviteRequest,
+  User,
+  UserIdentifier,
+  UnsavedUser,
+} from "@budibase/types"
+import { BudiStore } from "../BudiStore"
 
-export function createUsersStore() {
+interface UserInfo {
-  const { subscribe, set } = writable({})
+  email: string
+  password: string
+  forceResetPassword?: boolean
+  role: keyof typeof Constants.BudibaseRoles
+}
 
-  // opts can contain page and search params
+type UserState = SearchUsersResponse & SearchUsersRequest
-  async function search(opts = {}) {
+
+class UserStore extends BudiStore<UserState> {
+  constructor() {
+    super({
+      data: [],
+    })
+  }
+
+  async search(opts: SearchUsersRequest = {}) {
     const paged = await API.searchUsers(opts)
-    set({
+    this.set({
       ...paged,
       ...opts,
     })
     return paged
   }
 
-  async function get(userId) {
+  async get(userId: string) {
     try {
       return await API.getUser(userId)
     } catch (err) {
       return null
     }
   }
-  const fetch = async () => {
+
+  async fetch() {
     return await API.getUsers()
   }
 
-  // One or more users.
+  async onboard(payload: InviteUsersRequest) {
-  async function onboard(payload) {
     return await API.onboardUsers(payload)
   }
 
-  async function invite(payload) {
+  async invite(
-    const users = payload.map(user => {
+    payload: {
+      admin?: boolean
+      builder?: boolean
+      creator?: boolean
+      email: string
+      apps?: any[]
+      groups?: any[]
+    }[]
+  ) {
+    const users: InviteUsersRequest = payload.map(user => {
       let builder = undefined
       if (user.admin || user.builder) {
         builder = { global: true }
@@ -55,11 +85,16 @@ export function createUsersStore() {
     return API.inviteUsers(users)
   }
 
-  async function removeInvites(payload) {
+  async removeInvites(payload: DeleteInviteUsersRequest) {
     return API.removeUserInvites(payload)
   }
 
-  async function acceptInvite(inviteCode, password, firstName, lastName) {
+  async acceptInvite(
+    inviteCode: string,
+    password: string,
+    firstName: string,
+    lastName?: string
+  ) {
     return API.acceptInvite({
       inviteCode,
       password,
@@ -68,21 +103,25 @@ export function createUsersStore() {
     })
   }
 
-  async function fetchInvite(inviteCode) {
+  async fetchInvite(inviteCode: string) {
     return API.getUserInvite(inviteCode)
   }
 
-  async function getInvites() {
+  async getInvites() {
     return API.getUserInvites()
   }
 
-  async function updateInvite(invite) {
+  async updateInvite(code: string, invite: UpdateInviteRequest) {
-    return API.updateUserInvite(invite.code, invite)
+    return API.updateUserInvite(code, invite)
   }
 
-  async function create(data) {
+  async getUserCountByApp(appId: string) {
-    let mappedUsers = data.users.map(user => {
+    return await API.getUserCountByApp(appId)
-      const body = {
+  }
+
+  async create(data: { users: UserInfo[]; groups: any[] }) {
+    let mappedUsers: UnsavedUser[] = data.users.map((user: any) => {
+      const body: UnsavedUser = {
         email: user.email,
         password: user.password,
         roles: {},
@@ -92,17 +131,17 @@ export function createUsersStore() {
       }
 
       switch (user.role) {
-        case "appUser":
+        case Constants.BudibaseRoles.AppUser:
           body.builder = { global: false }
           body.admin = { global: false }
           break
-        case "developer":
+        case Constants.BudibaseRoles.Developer:
           body.builder = { global: true }
           break
-        case "creator":
+        case Constants.BudibaseRoles.Creator:
           body.builder = { creator: true, global: false }
           break
-        case "admin":
+        case Constants.BudibaseRoles.Admin:
           body.admin = { global: true }
           body.builder = { global: true }
           break
@@ -111,43 +150,47 @@ export function createUsersStore() {
       return body
     })
     const response = await API.createUsers(mappedUsers, data.groups)
+    licensing.setQuotaUsage()
 
     // re-search from first page
-    await search()
+    await this.search()
     return response
   }
 
-  async function del(id) {
+  async delete(id: string) {
     await API.deleteUser(id)
-    update(users => users.filter(user => user._id !== id))
+    licensing.setQuotaUsage()
   }
 
-  async function getUserCountByApp(appId) {
+  async bulkDelete(users: UserIdentifier[]) {
-    return await API.getUserCountByApp(appId)
+    const res = API.deleteUsers(users)
+    licensing.setQuotaUsage()
+    return res
   }
 
-  async function bulkDelete(users) {
+  async save(user: User) {
-    return API.deleteUsers(users)
+    const res = await API.saveUser(user)
+    licensing.setQuotaUsage()
+    return res
   }
 
-  async function save(user) {
+  async addAppBuilder(userId: string, appId: string) {
-    return await API.saveUser(user)
-  }
-
-  async function addAppBuilder(userId, appId) {
     return await API.addAppBuilder(userId, appId)
   }
 
-  async function removeAppBuilder(userId, appId) {
+  async removeAppBuilder(userId: string, appId: string) {
     return await API.removeAppBuilder(userId, appId)
   }
 
-  async function getAccountHolder() {
+  async getAccountHolder() {
     return await API.getAccountHolder()
   }
 
-  const getUserRole = user => {
+  getUserRole(user?: User & { tenantOwnerEmail?: string }) {
-    if (user && user.email === user.tenantOwnerEmail) {
+    if (!user) {
+      return Constants.BudibaseRoles.AppUser
+    }
+    if (user.email === user.tenantOwnerEmail) {
       return Constants.BudibaseRoles.Owner
     } else if (sdk.users.isAdmin(user)) {
       return Constants.BudibaseRoles.Admin
@@ -159,38 +202,6 @@ export function createUsersStore() {
       return Constants.BudibaseRoles.AppUser
     }
   }
-
-  const refreshUsage =
-    fn =>
-    async (...args) => {
-      const response = await fn(...args)
-      await licensing.setQuotaUsage()
-      return response
-    }
-
-  return {
-    subscribe,
-    search,
-    get,
-    getUserRole,
-    fetch,
-    invite,
-    onboard,
-    fetchInvite,
-    getInvites,
-    removeInvites,
-    updateInvite,
-    getUserCountByApp,
-    addAppBuilder,
-    removeAppBuilder,
-    // any operation that adds or deletes users
-    acceptInvite,
-    create: refreshUsage(create),
-    save: refreshUsage(save),
-    bulkDelete: refreshUsage(bulkDelete),
-    delete: refreshUsage(del),
-    getAccountHolder,
-  }
 }
 
-export const users = createUsersStore()
+export const users = new UserStore()

packages/client/src/api/api.ts

@@ -1,6 +1,10 @@
 import { createAPIClient } from "@budibase/frontend-core"
-import { authStore } from "../stores/auth.js"
+import { authStore } from "../stores/auth"
-import { notificationStore, devToolsEnabled, devToolsStore } from "../stores/"
+import {
+  notificationStore,
+  devToolsEnabled,
+  devToolsStore,
+} from "../stores/index"
 import { get } from "svelte/store"
 
 export const API = createAPIClient({

packages/client/src/api/index.js

@@ -1,5 +1,5 @@
-import { API } from "./api.js"
+import { API } from "./api"
-import { patchAPI } from "./patches.js"
+import { patchAPI } from "./patches"
 
 // Certain endpoints which return rows need patched so that they transform
 // and enrich the row docs, so that they can be correctly handled by the

packages/client/src/index.d.ts

@@ -0,0 +1,5 @@
+interface Window {
+  "##BUDIBASE_APP_ID##": string
+  "##BUDIBASE_IN_BUILDER##": string
+  MIGRATING_APP: boolean
+}

packages/client/src/sdk.js

@@ -29,7 +29,7 @@ import { ActionTypes } from "./constants"
 import {
   fetchDatasourceSchema,
   fetchDatasourceDefinition,
-} from "./utils/schema.js"
+} from "./utils/schema"
 import { getAPIKey } from "./utils/api.js"
 import { enrichButtonActions } from "./utils/buttonActions.js"
 import { processStringSync, makePropSafe } from "@budibase/string-templates"

packages/client/src/stores/auth.ts

@@ -2,7 +2,9 @@ import { API } from "api"
 import { writable } from "svelte/store"
 
 const createAuthStore = () => {
-  const store = writable(null)
+  const store = writable<{
+    csrfToken?: string
+  } | null>(null)
 
   // Fetches the user object if someone is logged in and has reloaded the page
   const fetchUser = async () => {

packages/client/src/stores/derived/currentRole.js

@@ -1,7 +1,7 @@
 import { derived } from "svelte/store"
 import { Constants } from "@budibase/frontend-core"
 import { devToolsStore } from "../devTools.js"
-import { authStore } from "../auth.js"
+import { authStore } from "../auth"
 import { devToolsEnabled } from "./devToolsEnabled.js"
 
 // Derive the current role of the logged-in user

packages/client/src/stores/notification.ts

@@ -6,7 +6,7 @@ const DEFAULT_NOTIFICATION_TIMEOUT = 3000
 const createNotificationStore = () => {
   let block = false
 
-  const store = writable([])
+  const store = writable<{ id: string; message: string; count: number }[]>([])
 
   const blockNotifications = (timeout = 1000) => {
     block = true
@@ -14,11 +14,11 @@ const createNotificationStore = () => {
   }
 
   const send = (
-    message,
+    message: string,
     type = "info",
-    icon,
+    icon: string,
     autoDismiss = true,
-    duration,
+    duration?: number,
     count = 1
   ) => {
     if (block) {
@@ -66,7 +66,7 @@ const createNotificationStore = () => {
     }
   }
 
-  const dismiss = id => {
+  const dismiss = (id: string) => {
     store.update(state => {
       return state.filter(n => n.id !== id)
     })
@@ -76,13 +76,13 @@ const createNotificationStore = () => {
     subscribe: store.subscribe,
     actions: {
       send,
-      info: (msg, autoDismiss, duration) =>
+      info: (msg: string, autoDismiss?: boolean, duration?: number) =>
         send(msg, "info", "Info", autoDismiss ?? true, duration),
-      success: (msg, autoDismiss, duration) =>
+      success: (msg: string, autoDismiss?: boolean, duration?: number) =>
         send(msg, "success", "CheckmarkCircle", autoDismiss ?? true, duration),
-      warning: (msg, autoDismiss, duration) =>
+      warning: (msg: string, autoDismiss?: boolean, duration?: number) =>
         send(msg, "warning", "Alert", autoDismiss ?? true, duration),
-      error: (msg, autoDismiss, duration) =>
+      error: (msg: string, autoDismiss?: boolean, duration?: number) =>
         send(msg, "error", "Alert", autoDismiss ?? false, duration),
       blockNotifications,
       dismiss,

packages/client/src/stores/routes.ts

@@ -4,8 +4,24 @@ import { API } from "api"
 import { peekStore } from "./peek"
 import { builderStore } from "./builder"
 
+interface Route {
+  path: string
+  screenId: string
+}
+
+interface StoreType {
+  routes: Route[]
+  routeParams: {}
+  activeRoute?: Route | null
+  routeSessionId: number
+  routerLoaded: boolean
+  queryParams?: {
+    peek?: boolean
+  }
+}
+
 const createRouteStore = () => {
-  const initialState = {
+  const initialState: StoreType = {
     routes: [],
     routeParams: {},
     activeRoute: null,
@@ -22,7 +38,7 @@ const createRouteStore = () => {
     } catch (error) {
       routeConfig = null
     }
-    let routes = []
+    const routes: Route[] = []
     Object.values(routeConfig?.routes || {}).forEach(route => {
       Object.entries(route.subpaths || {}).forEach(([path, config]) => {
         routes.push({
@@ -43,13 +59,13 @@ const createRouteStore = () => {
       return state
     })
   }
-  const setRouteParams = routeParams => {
+  const setRouteParams = (routeParams: StoreType["routeParams"]) => {
     store.update(state => {
       state.routeParams = routeParams
       return state
     })
   }
-  const setQueryParams = queryParams => {
+  const setQueryParams = (queryParams: { peek?: boolean }) => {
     store.update(state => {
       state.queryParams = {
         ...queryParams,
@@ -60,13 +76,13 @@ const createRouteStore = () => {
       return state
     })
   }
-  const setActiveRoute = route => {
+  const setActiveRoute = (route: string) => {
     store.update(state => {
       state.activeRoute = state.routes.find(x => x.path === route)
       return state
     })
   }
-  const navigate = (url, peek, externalNewTab) => {
+  const navigate = (url: string, peek: boolean, externalNewTab: boolean) => {
     if (get(builderStore).inBuilder) {
       return
     }
@@ -93,7 +109,7 @@ const createRouteStore = () => {
   const setRouterLoaded = () => {
     store.update(state => ({ ...state, routerLoaded: true }))
   }
-  const createFullURL = relativeURL => {
+  const createFullURL = (relativeURL: string) => {
     if (!relativeURL?.startsWith("/")) {
       return relativeURL
     }

packages/client/src/utils/schema.ts

@@ -1,13 +1,5 @@
 import { API } from "api"
-import TableFetch from "@budibase/frontend-core/src/fetch/TableFetch.js"
+import { DataFetchMap, DataFetchType } from "@budibase/frontend-core"
-import ViewFetch from "@budibase/frontend-core/src/fetch/ViewFetch.js"
-import QueryFetch from "@budibase/frontend-core/src/fetch/QueryFetch.js"
-import RelationshipFetch from "@budibase/frontend-core/src/fetch/RelationshipFetch.js"
-import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProviderFetch.js"
-import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch.js"
-import JSONArrayFetch from "@budibase/frontend-core/src/fetch/JSONArrayFetch.js"
-import ViewV2Fetch from "@budibase/frontend-core/src/fetch/ViewV2Fetch.js"
-import QueryArrayFetch from "@budibase/frontend-core/src/fetch/QueryArrayFetch"
 
 /**
  * Constructs a fetch instance for a given datasource.
@@ -16,22 +8,20 @@ import QueryArrayFetch from "@budibase/frontend-core/src/fetch/QueryArrayFetch"
  * @param datasource the datasource
  * @returns
  */
-const getDatasourceFetchInstance = datasource => {
+const getDatasourceFetchInstance = <
-  const handler = {
+  TDatasource extends { type: DataFetchType }
-    table: TableFetch,
+>(
-    view: ViewFetch,
+  datasource: TDatasource
-    viewV2: ViewV2Fetch,
+) => {
-    query: QueryFetch,
+  const handler = DataFetchMap[datasource?.type]
-    link: RelationshipFetch,
-    provider: NestedProviderFetch,
-    field: FieldFetch,
-    jsonarray: JSONArrayFetch,
-    queryarray: QueryArrayFetch,
-  }[datasource?.type]
   if (!handler) {
     return null
   }
-  return new handler({ API })
+  return new handler({
+    API,
+    datasource: datasource as never,
+    query: null as any,
+  })
 }
 
 /**
@@ -39,21 +29,23 @@ const getDatasourceFetchInstance = datasource => {
  * @param datasource the datasource to fetch the schema for
  * @param options options for enriching the schema
  */
-export const fetchDatasourceSchema = async (
+export const fetchDatasourceSchema = async <
-  datasource,
+  TDatasource extends { type: DataFetchType }
+>(
+  datasource: TDatasource,
   options = { enrichRelationships: false, formSchema: false }
 ) => {
   const instance = getDatasourceFetchInstance(datasource)
-  const definition = await instance?.getDefinition(datasource)
+  const definition = await instance?.getDefinition()
-  if (!definition) {
+  if (!instance || !definition) {
     return null
   }
 
   // Get the normal schema as long as we aren't wanting a form schema
-  let schema
+  let schema: any
   if (datasource?.type !== "query" || !options?.formSchema) {
-    schema = instance.getSchema(datasource, definition)
+    schema = instance.getSchema(definition as any)
-  } else if (definition.parameters?.length) {
+  } else if ("parameters" in definition && definition.parameters?.length) {
     schema = {}
     definition.parameters.forEach(param => {
       schema[param.name] = { ...param, type: "string" }
@@ -73,7 +65,12 @@ export const fetchDatasourceSchema = async (
   }
 
   // Enrich schema with relationships if required
-  if (definition?.sql && options?.enrichRelationships) {
+  if (
+    definition &&
+    "sql" in definition &&
+    definition.sql &&
+    options?.enrichRelationships
+  ) {
     const relationshipAdditions = await getRelationshipSchemaAdditions(schema)
     schema = {
       ...schema,
@@ -89,20 +86,26 @@ export const fetchDatasourceSchema = async (
  * Fetches the definition of any kind of datasource.
  * @param datasource the datasource to fetch the schema for
  */
-export const fetchDatasourceDefinition = async datasource => {
+export const fetchDatasourceDefinition = async <
+  TDatasource extends { type: DataFetchType }
+>(
+  datasource: TDatasource
+) => {
   const instance = getDatasourceFetchInstance(datasource)
-  return await instance?.getDefinition(datasource)
+  return await instance?.getDefinition()
 }
 
 /**
  * Fetches the schema of relationship fields for a SQL table schema
  * @param schema the schema to enrich
  */
-export const getRelationshipSchemaAdditions = async schema => {
+export const getRelationshipSchemaAdditions = async (
+  schema: Record<string, any>
+) => {
   if (!schema) {
     return null
   }
-  let relationshipAdditions = {}
+  let relationshipAdditions: Record<string, any> = {}
   for (let fieldKey of Object.keys(schema)) {
     const fieldSchema = schema[fieldKey]
     if (fieldSchema?.type === "link") {
@@ -110,7 +113,10 @@ export const getRelationshipSchemaAdditions = async schema => {
         type: "table",
         tableId: fieldSchema?.tableId,
       })
-      Object.keys(linkSchema || {}).forEach(linkKey => {
+      if (!linkSchema) {
+        continue
+      }
+      Object.keys(linkSchema).forEach(linkKey => {
         relationshipAdditions[`${fieldKey}.${linkKey}`] = {
           type: linkSchema[linkKey].type,
           externalType: linkSchema[linkKey].externalType,

packages/frontend-core/src/api/configs.ts

@@ -16,7 +16,7 @@ import { BaseAPIClient } from "./types"
 export interface ConfigEndpoints {
   getConfig: (type: ConfigType) => Promise<FindConfigResponse>
   getTenantConfig: (tentantId: string) => Promise<GetPublicSettingsResponse>
-  getOIDCConfig: (tenantId: string) => Promise<GetPublicOIDCConfigResponse>
+  getOIDCConfigs: (tenantId: string) => Promise<GetPublicOIDCConfigResponse>
   getOIDCLogos: () => Promise<Config<OIDCLogosConfig>>
   saveConfig: (config: SaveConfigRequest) => Promise<SaveConfigResponse>
   deleteConfig: (id: string, rev: string) => Promise<DeleteConfigResponse>
@@ -73,7 +73,7 @@ export const buildConfigEndpoints = (API: BaseAPIClient): ConfigEndpoints => ({
    * Gets the OIDC config for a certain tenant.
    * @param tenantId the tenant ID to get the config for
    */
-  getOIDCConfig: async tenantId => {
+  getOIDCConfigs: async tenantId => {
     return await API.get({
       url: `/api/global/configs/public/oidc?tenantId=${tenantId}`,
     })

packages/frontend-core/src/api/index.ts

@@ -68,13 +68,13 @@ export const createAPIClient = (config: APIClientConfig = {}): APIClient => {
   ): Promise<APIError> => {
     // Try to read a message from the error
     let message = response.statusText
-    let json: any = null
+    let json = null
     try {
       json = await response.json()
       if (json?.message) {
         message = json.message
       } else if (json?.error) {
-        message = json.error
+        message = JSON.stringify(json.error)
       }
     } catch (error) {
       // Do nothing
@@ -93,7 +93,7 @@ export const createAPIClient = (config: APIClientConfig = {}): APIClient => {
   // Generates an error object from a string
   const makeError = (
     message: string,
-    url?: string,
+    url: string,
     method?: HTTPMethod
   ): APIError => {
     return {
@@ -226,7 +226,7 @@ export const createAPIClient = (config: APIClientConfig = {}): APIClient => {
         return await handler(callConfig)
       } catch (error) {
         if (config?.onError) {
-          config.onError(error)
+          config.onError(error as APIError)
         }
         throw error
       }
@@ -239,13 +239,9 @@ export const createAPIClient = (config: APIClientConfig = {}): APIClient => {
     patch: requestApiCall(HTTPMethod.PATCH),
     delete: requestApiCall(HTTPMethod.DELETE),
     put: requestApiCall(HTTPMethod.PUT),
-    error: (message: string) => {
-      throw makeError(message)
-    },
     invalidateCache: () => {
       cache = {}
     },
-
     // Generic utility to extract the current app ID. Assumes that any client
     // that exists in an app context will be attaching our app ID header.
     getAppID: (): string => {

packages/frontend-core/src/api/types.ts

@@ -46,7 +46,7 @@ export type Headers = Record<string, string>
 export type APIClientConfig = {
   enableCaching?: boolean
   attachHeaders?: (headers: Headers) => void
-  onError?: (error: any) => void
+  onError?: (error: APIError) => void
   onMigrationDetected?: (migration: string) => void
 }
 
@@ -86,14 +86,13 @@ export type BaseAPIClient = {
   patch: <RequestT = null, ResponseT = void>(
     params: APICallParams<RequestT, ResponseT>
   ) => Promise<ResponseT>
-  error: (message: string) => void
   invalidateCache: () => void
   getAppID: () => string
 }
 
 export type APIError = {
   message?: string
-  url?: string
+  url: string
   method?: HTTPMethod
   json: any
   status: number

packages/frontend-core/src/api/user.ts

@@ -21,11 +21,12 @@ import {
   SaveUserResponse,
   SearchUsersRequest,
   SearchUsersResponse,
+  UnsavedUser,
   UpdateInviteRequest,
   UpdateInviteResponse,
   UpdateSelfMetadataRequest,
   UpdateSelfMetadataResponse,
-  User,
+  UserIdentifier,
 } from "@budibase/types"
 import { BaseAPIClient } from "./types"
 
@@ -38,14 +39,9 @@ export interface UserEndpoints {
   createAdminUser: (
     user: CreateAdminUserRequest
   ) => Promise<CreateAdminUserResponse>
-  saveUser: (user: User) => Promise<SaveUserResponse>
+  saveUser: (user: UnsavedUser) => Promise<SaveUserResponse>
   deleteUser: (userId: string) => Promise<DeleteUserResponse>
-  deleteUsers: (
+  deleteUsers: (users: UserIdentifier[]) => Promise<BulkUserDeleted | undefined>
-    users: Array<{
-      userId: string
-      email: string
-    }>
-  ) => Promise<BulkUserDeleted | undefined>
   onboardUsers: (data: InviteUsersRequest) => Promise<InviteUsersResponse>
   getUserInvite: (code: string) => Promise<CheckInviteResponse>
   getUserInvites: () => Promise<GetUserInvitesResponse>
@@ -60,7 +56,7 @@ export interface UserEndpoints {
   getAccountHolder: () => Promise<LookupAccountHolderResponse>
   searchUsers: (data: SearchUsersRequest) => Promise<SearchUsersResponse>
   createUsers: (
-    users: User[],
+    users: UnsavedUser[],
     groups: any[]
   ) => Promise<BulkUserCreated | undefined>
   updateUserInvite: (

packages/frontend-core/src/api/views.ts

@@ -3,7 +3,15 @@ import { BaseAPIClient } from "./types"
 
 export interface ViewEndpoints {
   // Missing request or response types
-  fetchViewData: (name: string, opts: any) => Promise<Row[]>
+  fetchViewData: (
+    name: string,
+    opts: {
+      calculation?: string
+      field?: string
+      groupBy?: string
+      tableId: string
+    }
+  ) => Promise<Row[]>
   exportView: (name: string, format: string) => Promise<any>
   saveView: (view: any) => Promise<any>
   deleteView: (name: string) => Promise<any>
@@ -20,7 +28,9 @@ export const buildViewEndpoints = (API: BaseAPIClient): ViewEndpoints => ({
   fetchViewData: async (name, { field, groupBy, calculation }) => {
     const params = new URLSearchParams()
     if (calculation) {
-      params.set("field", field)
+      if (field) {
+        params.set("field", field)
+      }
       params.set("calculation", calculation)
     }
     if (groupBy) {

packages/frontend-core/src/api/viewsV2.ts

@@ -1,6 +1,7 @@
 import {
   CreateViewRequest,
   CreateViewResponse,
+  PaginatedSearchRowResponse,
   SearchRowResponse,
   SearchViewRowRequest,
   UpdateViewRequest,
@@ -13,10 +14,14 @@ export interface ViewV2Endpoints {
   fetchDefinition: (viewId: string) => Promise<ViewResponseEnriched>
   create: (view: CreateViewRequest) => Promise<CreateViewResponse>
   update: (view: UpdateViewRequest) => Promise<UpdateViewResponse>
-  fetch: (
+  fetch: <T extends SearchViewRowRequest>(
     viewId: string,
-    opts: SearchViewRowRequest
+    opts: T
-  ) => Promise<SearchRowResponse>
+  ) => Promise<
+    T extends { paginate: true }
+      ? PaginatedSearchRowResponse
+      : SearchRowResponse
+  >
   delete: (viewId: string) => Promise<void>
 }
 
@@ -59,7 +64,7 @@ export const buildViewV2Endpoints = (API: BaseAPIClient): ViewV2Endpoints => ({
    * @param viewId the id of the view
    * @param opts the search options
    */
-  fetch: async (viewId, opts) => {
+  fetch: async (viewId, opts: SearchViewRowRequest) => {
     return await API.post({
       url: `/api/v2/views/${encodeURIComponent(viewId)}/search`,
       body: opts,

packages/frontend-core/src/components/grid/lib/renderers.ts

@@ -1,4 +1,4 @@
-import { FieldType } from "@budibase/types"
+import { FieldType, UIColumn } from "@budibase/types"
 
 import OptionsCell from "../cells/OptionsCell.svelte"
 import DateCell from "../cells/DateCell.svelte"
@@ -40,13 +40,23 @@ const TypeComponentMap = {
   // Custom types for UI only
   role: RoleCell,
 }
-export const getCellRenderer = column => {
+
+function getCellRendererByType(type: FieldType | "role" | undefined) {
+  if (!type) {
+    return
+  }
+
+  return TypeComponentMap[type as keyof typeof TypeComponentMap]
+}
+
+export const getCellRenderer = (column: UIColumn) => {
   if (column.calculationType) {
     return NumberCell
   }
+
   return (
-    TypeComponentMap[column?.schema?.cellRenderType] ||
+    getCellRendererByType(column.schema?.cellRenderType) ||
-    TypeComponentMap[column?.schema?.type] ||
+    getCellRendererByType(column.schema?.type) ||
     TextCell
   )
 }

packages/frontend-core/src/components/grid/lib/utils.js

@@ -1,32 +0,0 @@
-// TODO: remove when all stores are typed
-
-import { GeneratedIDPrefix, CellIDSeparator } from "./constants"
-import { Helpers } from "@budibase/bbui"
-
-export const parseCellID = cellId => {
-  if (!cellId) {
-    return { rowId: undefined, field: undefined }
-  }
-  const parts = cellId.split(CellIDSeparator)
-  const field = parts.pop()
-  return { rowId: parts.join(CellIDSeparator), field }
-}
-
-export const getCellID = (rowId, fieldName) => {
-  return `${rowId}${CellIDSeparator}${fieldName}`
-}
-
-export const parseEventLocation = e => {
-  return {
-    x: e.clientX ?? e.touches?.[0]?.clientX,
-    y: e.clientY ?? e.touches?.[0]?.clientY,
-  }
-}
-
-export const generateRowID = () => {
-  return `${GeneratedIDPrefix}${Helpers.uuid()}`
-}
-
-export const isGeneratedRowID = id => {
-  return id?.startsWith(GeneratedIDPrefix)
-}

packages/frontend-core/src/components/grid/lib/websocket.ts

@@ -1,12 +1,14 @@
 import { get } from "svelte/store"
 import { createWebsocket } from "../../../utils"
 import { SocketEvent, GridSocketEvent } from "@budibase/shared-core"
+import { Store } from "../stores"
+import { UIDatasource, UIUser } from "@budibase/types"
 
-export const createGridWebsocket = context => {
+export const createGridWebsocket = (context: Store) => {
   const { rows, datasource, users, focusedCellId, definition, API } = context
   const socket = createWebsocket("/socket/grid")
 
-  const connectToDatasource = datasource => {
+  const connectToDatasource = (datasource: UIDatasource) => {
     if (!socket.connected) {
       return
     }
@@ -18,7 +20,7 @@ export const createGridWebsocket = context => {
         datasource,
         appId,
       },
-      ({ users: gridUsers }) => {
+      ({ users: gridUsers }: { users: UIUser[] }) => {
         users.set(gridUsers)
       }
     )
@@ -65,7 +67,7 @@ export const createGridWebsocket = context => {
     GridSocketEvent.DatasourceChange,
     ({ datasource: newDatasource }) => {
       // Listen builder renames, as these aren't handled otherwise
-      if (newDatasource?.name !== get(definition).name) {
+      if (newDatasource?.name !== get(definition)?.name) {
         definition.set(newDatasource)
       }
     }

packages/frontend-core/src/components/grid/stores/config.ts

@@ -69,7 +69,7 @@ export const deriveStores = (context: StoreContext): ConfigDerivedStore => {
       }
 
       // Disable features for non DS+
-      if (!["table", "viewV2"].includes(type)) {
+      if (type && !["table", "viewV2"].includes(type)) {
         config.canAddRows = false
         config.canEditRows = false
         config.canDeleteRows = false

packages/frontend-core/src/components/grid/stores/datasource.ts

@@ -1,3 +1,5 @@
+// TODO: datasource and defitions are unions of the different implementations. At this point, the datasource does not know what type is being used, and the assignations will cause TS exceptions. Casting it "as any" for now. This should be fixed improving the type usages.
+
 import { derived, get, Readable, Writable } from "svelte/store"
 import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch"
 import { enrichSchemaWithRelColumns, memo } from "../../../utils"
@@ -71,10 +73,10 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => {
   } = context
 
   const schema = derived(definition, $definition => {
-    let schema: Record<string, UIFieldSchema> = getDatasourceSchema({
+    const schema: Record<string, any> | undefined = getDatasourceSchema({
       API,
-      datasource: get(datasource),
+      datasource: get(datasource) as any, // TODO: see line 1
-      definition: $definition,
+      definition: $definition ?? undefined,
     })
     if (!schema) {
       return null
@@ -82,7 +84,7 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => {
 
     // Ensure schema is configured as objects.
     // Certain datasources like queries use primitives.
-    Object.keys(schema || {}).forEach(key => {
+    Object.keys(schema).forEach(key => {
       if (typeof schema[key] !== "object") {
         schema[key] = { name: key, type: schema[key] }
       }
@@ -130,13 +132,13 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => {
     ([$datasource, $definition]) => {
       let type = $datasource?.type
       if (type === "provider") {
-        type = ($datasource as any).value?.datasource?.type
+        type = ($datasource as any).value?.datasource?.type // TODO: see line 1
       }
       // Handle calculation views
       if (type === "viewV2" && $definition?.type === ViewV2Type.CALCULATION) {
         return false
       }
-      return ["table", "viewV2", "link"].includes(type)
+      return !!type && ["table", "viewV2", "link"].includes(type)
     }
   )
 
@@ -184,9 +186,9 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => {
   const refreshDefinition = async () => {
     const def = await getDatasourceDefinition({
       API,
-      datasource: get(datasource),
+      datasource: get(datasource) as any, // TODO: see line 1
     })
-    definition.set(def)
+    definition.set(def as any) // TODO: see line 1
   }
 
   // Saves the datasource definition
@@ -231,7 +233,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => {
     if ("default" in newDefinition.schema[column]) {
       delete newDefinition.schema[column].default
     }
-    return await saveDefinition(newDefinition as any)
+    return await saveDefinition(newDefinition as any) // TODO: see line 1
   }
 
   // Adds a schema mutation for a single field
@@ -307,7 +309,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => {
     await saveDefinition({
       ...$definition,
       schema: newSchema,
-    } as any)
+    } as any) // TODO: see line 1
     resetSchemaMutations()
   }
 

packages/frontend-core/src/components/grid/stores/rows.ts

@@ -10,9 +10,10 @@ import {
 import { tick } from "svelte"
 import { Helpers } from "@budibase/bbui"
 import { sleep } from "../../../utils/utils"
-import { FieldType, Row, UIFetchAPI, UIRow } from "@budibase/types"
+import { FieldType, Row, UIRow } from "@budibase/types"
 import { getRelatedTableValues } from "../../../utils"
 import { Store as StoreContext } from "."
+import DataFetch from "../../../fetch/DataFetch"
 
 interface IndexedUIRow extends UIRow {
   __idx: number
@@ -20,7 +21,7 @@ interface IndexedUIRow extends UIRow {
 
 interface RowStore {
   rows: Writable<UIRow[]>
-  fetch: Writable<UIFetchAPI | null>
+  fetch: Writable<DataFetch<any, any, any> | null> // TODO: type this properly, having a union of all the possible options
   loaded: Writable<boolean>
   refreshing: Writable<boolean>
   loading: Writable<boolean>
@@ -225,7 +226,7 @@ export const createActions = (context: StoreContext): RowActionStore => {
     })
 
     // Subscribe to changes of this fetch model
-    unsubscribe = newFetch.subscribe(async ($fetch: UIFetchAPI) => {
+    unsubscribe = newFetch.subscribe(async $fetch => {
       if ($fetch.error) {
         // Present a helpful error to the user
         let message = "An unknown error occurred"
@@ -253,7 +254,7 @@ export const createActions = (context: StoreContext): RowActionStore => {
 
         // Reset state properties when dataset changes
         if (!$instanceLoaded || resetRows) {
-          definition.set($fetch.definition)
+          definition.set($fetch.definition as any) // TODO: datasource and defitions are unions of the different implementations. At this point, the datasource does not know what type is being used, and the assignations will cause TS exceptions. Casting it "as any" for now. This should be fixed improving the type usages.
         }
 
         // Reset scroll state when data changes

packages/frontend-core/src/constants.ts

@@ -32,8 +32,8 @@ export const Cookies = {
 }
 
 // Table names
-export const TableNames = {
+export const enum TableNames {
-  USERS: "ta_users",
+  USERS = "ta_users",
 }
 
 export const BudibaseRoles = {

packages/frontend-core/src/fetch/CustomFetch.ts

@@ -1,8 +1,18 @@
-import DataFetch from "./DataFetch.js"
+import DataFetch from "./DataFetch"
 
-export default class CustomFetch extends DataFetch {
+interface CustomDatasource {
+  type: "custom"
+  data: any
+}
+
+type CustomDefinition = Record<string, any>
+
+export default class CustomFetch extends DataFetch<
+  CustomDatasource,
+  CustomDefinition
+> {
   // Gets the correct Budibase type for a JS value
-  getType(value) {
+  getType(value: any) {
     if (value == null) {
       return "string"
     }
@@ -22,7 +32,7 @@ export default class CustomFetch extends DataFetch {
   }
 
   // Parses the custom data into an array format
-  parseCustomData(data) {
+  parseCustomData(data: any) {
     if (!data) {
       return []
     }
@@ -55,7 +65,7 @@ export default class CustomFetch extends DataFetch {
   }
 
   // Enriches the custom data to ensure the structure and format is usable
-  enrichCustomData(data) {
+  enrichCustomData(data: (string | any)[]) {
     if (!data?.length) {
       return []
     }
@@ -72,7 +82,7 @@ export default class CustomFetch extends DataFetch {
       // Try parsing strings
       if (typeof value === "string") {
         const split = value.split(",").map(x => x.trim())
-        let obj = {}
+        const obj: Record<string, string> = {}
         for (let i = 0; i < split.length; i++) {
           const suffix = i === 0 ? "" : ` ${i + 1}`
           const key = `Value${suffix}`
@@ -87,27 +97,29 @@ export default class CustomFetch extends DataFetch {
   }
 
   // Extracts and parses the custom data from the datasource definition
-  getCustomData(datasource) {
+  getCustomData(datasource: CustomDatasource) {
     return this.enrichCustomData(this.parseCustomData(datasource?.data))
   }
 
-  async getDefinition(datasource) {
+  async getDefinition() {
+    const { datasource } = this.options
+
     // Try and work out the schema from the array provided
-    let schema = {}
+    const schema: CustomDefinition = {}
     const data = this.getCustomData(datasource)
     if (!data?.length) {
       return { schema }
     }
 
     // Go through every object and extract all valid keys
-    for (let datum of data) {
+    for (const datum of data) {
-      for (let key of Object.keys(datum)) {
+      for (const key of Object.keys(datum)) {
         if (key === "_id") {
           continue
         }
         if (!schema[key]) {
           let type = this.getType(datum[key])
-          let constraints = {}
+          const constraints: any = {}
 
           // Determine whether we should render text columns as options instead
           if (type === "string") {

packages/frontend-core/src/fetch/DataFetch.ts

@@ -1,25 +1,103 @@
-import { writable, derived, get } from "svelte/store"
+import { writable, derived, get, Writable, Readable } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
 import { QueryUtils } from "../utils"
 import { convertJSONSchemaToTableSchema } from "../utils/json"
-import { FieldType, SortOrder, SortType } from "@budibase/types"
+import {
+  FieldType,
+  LegacyFilter,
+  Row,
+  SearchFilters,
+  SortOrder,
+  SortType,
+  TableSchema,
+  UISearchFilter,
+} from "@budibase/types"
+import { APIClient } from "../api/types"
+import { DataFetchType } from "."
 
 const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils
 
+interface DataFetchStore<TDefinition, TQuery> {
+  rows: Row[]
+  info: any
+  schema: TableSchema | null
+  loading: boolean
+  loaded: boolean
+  query: TQuery
+  pageNumber: number
+  cursor: string | null
+  cursors: string[]
+  resetKey: string
+  error: {
+    message: string
+    status: number
+  } | null
+  definition?: TDefinition | null
+}
+
+interface DataFetchDerivedStore<TDefinition, TQuery>
+  extends DataFetchStore<TDefinition, TQuery> {
+  hasNextPage: boolean
+  hasPrevPage: boolean
+  supportsSearch: boolean
+  supportsSort: boolean
+  supportsPagination: boolean
+}
+
+export interface DataFetchParams<
+  TDatasource,
+  TQuery = SearchFilters | undefined
+> {
+  API: APIClient
+  datasource: TDatasource
+  query: TQuery
+  options?: {}
+}
+
 /**
  * Parent class which handles the implementation of fetching data from an
  * internal table or datasource plus.
  * For other types of datasource, this class is overridden and extended.
  */
-export default class DataFetch {
+export default abstract class DataFetch<
+  TDatasource extends { type: DataFetchType },
+  TDefinition extends {
+    schema?: Record<string, any> | null
+    primaryDisplay?: string
+  },
+  TQuery extends {} = SearchFilters
+> {
+  API: APIClient
+  features: {
+    supportsSearch: boolean
+    supportsSort: boolean
+    supportsPagination: boolean
+  }
+  options: {
+    datasource: TDatasource
+    limit: number
+    // Search config
+    filter: UISearchFilter | LegacyFilter[] | null
+    query: TQuery
+    // Sorting config
+    sortColumn: string | null
+    sortOrder: SortOrder
+    sortType: SortType | null
+    // Pagination config
+    paginate: boolean
+    // Client side feature customisation
+    clientSideSearching: boolean
+    clientSideSorting: boolean
+    clientSideLimiting: boolean
+  }
+  store: Writable<DataFetchStore<TDefinition, TQuery>>
+  derivedStore: Readable<DataFetchDerivedStore<TDefinition, TQuery>>
+
   /**
    * Constructs a new DataFetch instance.
    * @param opts the fetch options
    */
-  constructor(opts) {
+  constructor(opts: DataFetchParams<TDatasource, TQuery>) {
-    // API client
-    this.API = null
-
     // Feature flags
     this.features = {
       supportsSearch: false,
@@ -29,12 +107,12 @@ export default class DataFetch {
 
     // Config
     this.options = {
-      datasource: null,
+      datasource: opts.datasource,
       limit: 10,
 
       // Search config
       filter: null,
-      query: null,
+      query: opts.query,
 
       // Sorting config
       sortColumn: null,
@@ -57,11 +135,11 @@ export default class DataFetch {
       schema: null,
       loading: false,
       loaded: false,
-      query: null,
+      query: opts.query,
       pageNumber: 0,
       cursor: null,
       cursors: [],
-      resetKey: Math.random(),
+      resetKey: Math.random().toString(),
       error: null,
     })
 
@@ -102,9 +180,6 @@ export default class DataFetch {
       this.store.update($store => ({ ...$store, loaded: true }))
       return
     }
-
-    // Initially fetch data but don't bother waiting for the result
-    this.getInitialData()
   }
 
   /**
@@ -118,7 +193,10 @@ export default class DataFetch {
   /**
    * Gets the default sort column for this datasource
    */
-  getDefaultSortColumn(definition, schema) {
+  getDefaultSortColumn(
+    definition: { primaryDisplay?: string } | null,
+    schema: Record<string, any>
+  ): string | null {
     if (definition?.primaryDisplay && schema[definition.primaryDisplay]) {
       return definition.primaryDisplay
     } else {
@@ -130,13 +208,13 @@ export default class DataFetch {
    * Fetches a fresh set of data from the server, resetting pagination
    */
   async getInitialData() {
-    const { datasource, filter, paginate } = this.options
+    const { filter, paginate } = this.options
 
     // Fetch datasource definition and extract sort properties if configured
-    const definition = await this.getDefinition(datasource)
+    const definition = await this.getDefinition()
 
     // Determine feature flags
-    const features = this.determineFeatureFlags(definition)
+    const features = await this.determineFeatureFlags()
     this.features = {
       supportsSearch: !!features?.supportsSearch,
       supportsSort: !!features?.supportsSort,
@@ -144,11 +222,11 @@ export default class DataFetch {
     }
 
     // Fetch and enrich schema
-    let schema = this.getSchema(datasource, definition)
+    let schema = this.getSchema(definition)
-    schema = this.enrichSchema(schema)
     if (!schema) {
       return
     }
+    schema = this.enrichSchema(schema)
 
     // If an invalid sort column is specified, delete it
     if (this.options.sortColumn && !schema[this.options.sortColumn]) {
@@ -172,20 +250,25 @@ export default class DataFetch {
       if (
         fieldSchema?.type === FieldType.NUMBER ||
         fieldSchema?.type === FieldType.BIGINT ||
-        fieldSchema?.calculationType
+        ("calculationType" in fieldSchema && fieldSchema?.calculationType)
       ) {
         this.options.sortType = SortType.NUMBER
       }
+
       // If no sort order, default to ascending
       if (!this.options.sortOrder) {
         this.options.sortOrder = SortOrder.ASCENDING
+      } else {
+        // Ensure sortOrder matches the enum
+        this.options.sortOrder =
+          this.options.sortOrder.toLowerCase() as SortOrder
       }
     }
 
     // Build the query
     let query = this.options.query
     if (!query) {
-      query = buildQuery(filter)
+      query = buildQuery(filter ?? undefined) as TQuery
     }
 
     // Update store
@@ -210,7 +293,7 @@ export default class DataFetch {
       info: page.info,
       cursors: paginate && page.hasNextPage ? [null, page.cursor] : [null],
       error: page.error,
-      resetKey: Math.random(),
+      resetKey: Math.random().toString(),
     }))
   }
 
@@ -238,8 +321,8 @@ export default class DataFetch {
     }
 
     // If we don't support sorting, do a client-side sort
-    if (!this.features.supportsSort && clientSideSorting) {
+    if (!this.features.supportsSort && clientSideSorting && sortType) {
-      rows = sort(rows, sortColumn, sortOrder, sortType)
+      rows = sort(rows, sortColumn as any, sortOrder, sortType)
     }
 
     // If we don't support pagination, do a client-side limit
@@ -256,49 +339,28 @@ export default class DataFetch {
     }
   }
 
-  /**
+  abstract getData(): Promise<{
-   * Fetches a single page of data from the remote resource.
+    rows: Row[]
-   * Must be overridden by a datasource specific child class.
+    info?: any
-   */
+    hasNextPage?: boolean
-  async getData() {
+    cursor?: any
-    return {
+    error?: any
-      rows: [],
+  }>
-      info: null,
-      hasNextPage: false,
-      cursor: null,
-    }
-  }
 
   /**
    * Gets the definition for this datasource.
-   * Defaults to fetching a table definition.
+
-   * @param datasource
    * @return {object} the definition
    */
-  async getDefinition(datasource) {
+  abstract getDefinition(): Promise<TDefinition | null>
-    if (!datasource?.tableId) {
-      return null
-    }
-    try {
-      return await this.API.fetchTableDefinition(datasource.tableId)
-    } catch (error) {
-      this.store.update(state => ({
-        ...state,
-        error,
-      }))
-      return null
-    }
-  }
 
   /**
    * Gets the schema definition for a datasource.
-   * Defaults to getting the "schema" property of the definition.
-   * @param datasource the datasource
    * @param definition the datasource definition
    * @return {object} the schema
    */
-  getSchema(datasource, definition) {
+  getSchema(definition: TDefinition | null): Record<string, any> | undefined {
-    return definition?.schema
+    return definition?.schema ?? undefined
   }
 
   /**
@@ -307,53 +369,56 @@ export default class DataFetch {
    * @param schema the datasource schema
    * @return {object} the enriched datasource schema
    */
-  enrichSchema(schema) {
+  enrichSchema(schema: TableSchema): TableSchema {
-    if (schema == null) {
-      return null
-    }
-
     // Check for any JSON fields so we can add any top level properties
-    let jsonAdditions = {}
+    let jsonAdditions: Record<string, { type: string; nestedJSON: true }> = {}
-    Object.keys(schema).forEach(fieldKey => {
+    for (const fieldKey of Object.keys(schema)) {
       const fieldSchema = schema[fieldKey]
-      if (fieldSchema?.type === FieldType.JSON) {
+      if (fieldSchema.type === FieldType.JSON) {
         const jsonSchema = convertJSONSchemaToTableSchema(fieldSchema, {
           squashObjects: true,
-        })
+        }) as Record<string, { type: string }> | null // TODO: remove when convertJSONSchemaToTableSchema is typed
-        Object.keys(jsonSchema).forEach(jsonKey => {
+        if (jsonSchema) {
-          jsonAdditions[`${fieldKey}.${jsonKey}`] = {
+          for (const jsonKey of Object.keys(jsonSchema)) {
-            type: jsonSchema[jsonKey].type,
+            jsonAdditions[`${fieldKey}.${jsonKey}`] = {
-            nestedJSON: true,
+              type: jsonSchema[jsonKey].type,
+              nestedJSON: true,
+            }
           }
-        })
+        }
       }
-    })
+    }
-    schema = { ...schema, ...jsonAdditions }
 
     // Ensure schema is in the correct structure
-    let enrichedSchema = {}
+    let enrichedSchema: TableSchema = {}
-    Object.entries(schema).forEach(([fieldName, fieldSchema]) => {
+    Object.entries({ ...schema, ...jsonAdditions }).forEach(
-      if (typeof fieldSchema === "string") {
+      ([fieldName, fieldSchema]) => {
-        enrichedSchema[fieldName] = {
+        if (typeof fieldSchema === "string") {
-          type: fieldSchema,
+          enrichedSchema[fieldName] = {
-          name: fieldName,
+            type: fieldSchema,
-        }
+            name: fieldName,
-      } else {
+          }
-        enrichedSchema[fieldName] = {
+        } else {
-          ...fieldSchema,
+          enrichedSchema[fieldName] = {
-          name: fieldName,
+            ...fieldSchema,
+            type: fieldSchema.type as any, // TODO: check type union definition conflicts
+            name: fieldName,
+          }
         }
       }
-    })
+    )
 
     return enrichedSchema
   }
 
   /**
-   * Determine the feature flag for this datasource definition
+   * Determine the feature flag for this datasource
-   * @param definition
    */
-  determineFeatureFlags(_definition) {
+  async determineFeatureFlags(): Promise<{
+    supportsPagination: boolean
+    supportsSearch?: boolean
+    supportsSort?: boolean
+  }> {
     return {
       supportsSearch: false,
       supportsSort: false,
@@ -365,12 +430,11 @@ export default class DataFetch {
    * Resets the data set and updates options
    * @param newOptions any new options
    */
-  async update(newOptions) {
+  async update(newOptions: any) {
     // Check if any settings have actually changed
     let refresh = false
-    const entries = Object.entries(newOptions || {})
+    for (const [key, value] of Object.entries(newOptions || {})) {
-    for (let [key, value] of entries) {
+      const oldVal = this.options[key as keyof typeof this.options] ?? null
-      const oldVal = this.options[key] == null ? null : this.options[key]
       const newVal = value == null ? null : value
       if (JSON.stringify(newVal) !== JSON.stringify(oldVal)) {
         refresh = true
@@ -437,7 +501,7 @@ export default class DataFetch {
    * @param state the current store state
    * @return {boolean} whether there is a next page of data or not
    */
-  hasNextPage(state) {
+  private hasNextPage(state: DataFetchStore<TDefinition, TQuery>): boolean {
     return state.cursors[state.pageNumber + 1] != null
   }
 
@@ -447,7 +511,7 @@ export default class DataFetch {
    * @param state the current store state
    * @return {boolean} whether there is a previous page of data or not
    */
-  hasPrevPage(state) {
+  private hasPrevPage(state: { pageNumber: number }): boolean {
     return state.pageNumber > 0
   }
 

packages/frontend-core/src/fetch/FieldFetch.js

@@ -1,44 +0,0 @@
-import DataFetch from "./DataFetch.js"
-
-export default class FieldFetch extends DataFetch {
-  async getDefinition(datasource) {
-    // Field sources have their schema statically defined
-    let schema
-    if (datasource.fieldType === "attachment") {
-      schema = {
-        url: {
-          type: "string",
-        },
-        name: {
-          type: "string",
-        },
-      }
-    } else if (datasource.fieldType === "array") {
-      schema = {
-        value: {
-          type: "string",
-        },
-      }
-    }
-    return { schema }
-  }
-
-  async getData() {
-    const { datasource } = this.options
-
-    // These sources will be available directly from context
-    const data = datasource?.value || []
-    let rows
-    if (Array.isArray(data) && data[0] && typeof data[0] !== "object") {
-      rows = data.map(value => ({ value }))
-    } else {
-      rows = data
-    }
-
-    return {
-      rows: rows || [],
-      hasNextPage: false,
-      cursor: null,
-    }
-  }
-}

packages/frontend-core/src/fetch/FieldFetch.ts

@@ -0,0 +1,67 @@
+import { Row } from "@budibase/types"
+import DataFetch from "./DataFetch"
+
+type Types = "field" | "queryarray" | "jsonarray"
+
+export interface FieldDatasource<TType extends Types> {
+  type: TType
+  tableId: string
+  fieldType: "attachment" | "array"
+  value: string[] | Row[]
+}
+
+export interface FieldDefinition {
+  schema?: Record<string, { type: string }> | null
+}
+
+function isArrayOfStrings(value: string[] | Row[]): value is string[] {
+  return Array.isArray(value) && !!value[0] && typeof value[0] !== "object"
+}
+
+export default class FieldFetch<TType extends Types> extends DataFetch<
+  FieldDatasource<TType>,
+  FieldDefinition
+> {
+  async getDefinition(): Promise<FieldDefinition | null> {
+    const { datasource } = this.options
+
+    // Field sources have their schema statically defined
+    let schema
+    if (datasource.fieldType === "attachment") {
+      schema = {
+        url: {
+          type: "string",
+        },
+        name: {
+          type: "string",
+        },
+      }
+    } else if (datasource.fieldType === "array") {
+      schema = {
+        value: {
+          type: "string",
+        },
+      }
+    }
+    return { schema }
+  }
+
+  async getData() {
+    const { datasource } = this.options
+
+    // These sources will be available directly from context
+    const data = datasource?.value || []
+    let rows: Row[]
+    if (isArrayOfStrings(data)) {
+      rows = data.map(value => ({ value }))
+    } else {
+      rows = data
+    }
+
+    return {
+      rows: rows || [],
+      hasNextPage: false,
+      cursor: null,
+    }
+  }
+}

packages/frontend-core/src/fetch/GroupUserFetch.ts

@@ -1,18 +1,33 @@
 import { get } from "svelte/store"
-import DataFetch from "./DataFetch.js"
+import DataFetch, { DataFetchParams } from "./DataFetch"
 import { TableNames } from "../constants"
 
-export default class GroupUserFetch extends DataFetch {
+interface GroupUserQuery {
-  constructor(opts) {
+  groupId: string
+  emailSearch: string
+}
+
+interface GroupUserDatasource {
+  type: "groupUser"
+  tableId: TableNames.USERS
+}
+
+export default class GroupUserFetch extends DataFetch<
+  GroupUserDatasource,
+  {},
+  GroupUserQuery
+> {
+  constructor(opts: DataFetchParams<GroupUserDatasource, GroupUserQuery>) {
     super({
       ...opts,
       datasource: {
+        type: "groupUser",
         tableId: TableNames.USERS,
       },
     })
   }
 
-  determineFeatureFlags() {
+  async determineFeatureFlags() {
     return {
       supportsSearch: true,
       supportsSort: false,
@@ -28,11 +43,12 @@ export default class GroupUserFetch extends DataFetch {
 
   async getData() {
     const { query, cursor } = get(this.store)
+
     try {
       const res = await this.API.getGroupUsers({
         id: query.groupId,
         emailSearch: query.emailSearch,
-        bookmark: cursor,
+        bookmark: cursor ?? undefined,
       })
 
       return {

packages/frontend-core/src/fetch/JSONArrayFetch.ts

@@ -1,8 +1,10 @@
-import FieldFetch from "./FieldFetch.js"
+import FieldFetch from "./FieldFetch"
 import { getJSONArrayDatasourceSchema } from "../utils/json"
 
-export default class JSONArrayFetch extends FieldFetch {
+export default class JSONArrayFetch extends FieldFetch<"jsonarray"> {
-  async getDefinition(datasource) {
+  async getDefinition() {
+    const { datasource } = this.options
+
     // JSON arrays need their table definitions fetched.
     // We can then extract their schema as a subset of the table schema.
     try {

packages/frontend-core/src/fetch/NestedProviderFetch.js

@@ -1,21 +0,0 @@
-import DataFetch from "./DataFetch.js"
-
-export default class NestedProviderFetch extends DataFetch {
-  async getDefinition(datasource) {
-    // Nested providers should already have exposed their own schema
-    return {
-      schema: datasource?.value?.schema,
-      primaryDisplay: datasource?.value?.primaryDisplay,
-    }
-  }
-
-  async getData() {
-    const { datasource } = this.options
-    // Pull the rows from the existing data provider
-    return {
-      rows: datasource?.value?.rows || [],
-      hasNextPage: false,
-      cursor: null,
-    }
-  }
-}

packages/frontend-core/src/fetch/NestedProviderFetch.ts

@@ -0,0 +1,40 @@
+import { Row, TableSchema } from "@budibase/types"
+import DataFetch from "./DataFetch"
+
+interface NestedProviderDatasource {
+  type: "provider"
+  value?: {
+    schema: TableSchema
+    primaryDisplay: string
+    rows: Row[]
+  }
+}
+
+interface NestedProviderDefinition {
+  schema?: TableSchema
+  primaryDisplay?: string
+}
+export default class NestedProviderFetch extends DataFetch<
+  NestedProviderDatasource,
+  NestedProviderDefinition
+> {
+  async getDefinition() {
+    const { datasource } = this.options
+
+    // Nested providers should already have exposed their own schema
+    return {
+      schema: datasource?.value?.schema,
+      primaryDisplay: datasource?.value?.primaryDisplay,
+    }
+  }
+
+  async getData() {
+    const { datasource } = this.options
+    // Pull the rows from the existing data provider
+    return {
+      rows: datasource?.value?.rows || [],
+      hasNextPage: false,
+      cursor: null,
+    }
+  }
+}

packages/frontend-core/src/fetch/QueryArrayFetch.ts

@@ -1,11 +1,13 @@
-import FieldFetch from "./FieldFetch.js"
+import FieldFetch from "./FieldFetch"
 import {
   getJSONArrayDatasourceSchema,
   generateQueryArraySchemas,
 } from "../utils/json"
 
-export default class QueryArrayFetch extends FieldFetch {
+export default class QueryArrayFetch extends FieldFetch<"queryarray"> {
-  async getDefinition(datasource) {
+  async getDefinition() {
+    const { datasource } = this.options
+
     if (!datasource?.tableId) {
       return null
     }
@@ -14,10 +16,14 @@ export default class QueryArrayFetch extends FieldFetch {
     try {
       const table = await this.API.fetchQueryDefinition(datasource.tableId)
       const schema = generateQueryArraySchemas(
-        table?.schema,
+        table.schema,
-        table?.nestedSchemaFields
+        table.nestedSchemaFields
       )
-      return { schema: getJSONArrayDatasourceSchema(schema, datasource) }
+      const result = {
+        schema: getJSONArrayDatasourceSchema(schema, datasource),
+      }
+
+      return result
     } catch (error) {
       return null
     }

packages/frontend-core/src/fetch/QueryFetch.ts

@@ -1,9 +1,25 @@
-import DataFetch from "./DataFetch.js"
+import DataFetch from "./DataFetch"
 import { Helpers } from "@budibase/bbui"
+import { ExecuteQueryRequest, Query } from "@budibase/types"
 import { get } from "svelte/store"
 
-export default class QueryFetch extends DataFetch {
+interface QueryDatasource {
-  determineFeatureFlags(definition) {
+  type: "query"
+  _id: string
+  fields: Record<string, any> & {
+    pagination?: {
+      type: string
+      location: string
+      pageParam: string
+    }
+  }
+  queryParams?: Record<string, string>
+  parameters: { name: string; default: string }[]
+}
+
+export default class QueryFetch extends DataFetch<QueryDatasource, Query> {
+  async determineFeatureFlags() {
+    const definition = await this.getDefinition()
     const supportsPagination =
       !!definition?.fields?.pagination?.type &&
       !!definition?.fields?.pagination?.location &&
@@ -11,7 +27,9 @@ export default class QueryFetch extends DataFetch {
     return { supportsPagination }
   }
 
-  async getDefinition(datasource) {
+  async getDefinition() {
+    const { datasource } = this.options
+
     if (!datasource?._id) {
       return null
     }
@@ -40,17 +58,17 @@ export default class QueryFetch extends DataFetch {
     const type = definition?.fields?.pagination?.type
 
     // Set the default query params
-    let parameters = Helpers.cloneDeep(datasource?.queryParams || {})
+    const parameters = Helpers.cloneDeep(datasource.queryParams || {})
-    for (let param of datasource?.parameters || {}) {
+    for (const param of datasource?.parameters || []) {
       if (!parameters[param.name]) {
         parameters[param.name] = param.default
       }
     }
 
     // Add pagination to query if supported
-    let queryPayload = { parameters }
+    const queryPayload: ExecuteQueryRequest = { parameters }
     if (paginate && supportsPagination) {
-      const requestCursor = type === "page" ? parseInt(cursor || 1) : cursor
+      const requestCursor = type === "page" ? parseInt(cursor || "1") : cursor
       queryPayload.pagination = { page: requestCursor, limit }
     }
 
@@ -65,7 +83,7 @@ export default class QueryFetch extends DataFetch {
       if (paginate && supportsPagination) {
         if (type === "page") {
           // For "page number" pagination, increment the existing page number
-          nextCursor = queryPayload.pagination.page + 1
+          nextCursor = queryPayload.pagination!.page! + 1
           hasNextPage = data?.length === limit && limit > 0
         } else {
           // For "cursor" pagination, the cursor should be in the response

packages/frontend-core/src/fetch/RelationshipFetch.js

@@ -1,20 +0,0 @@
-import DataFetch from "./DataFetch.js"
-
-export default class RelationshipFetch extends DataFetch {
-  async getData() {
-    const { datasource } = this.options
-    if (!datasource?.rowId || !datasource?.rowTableId) {
-      return { rows: [] }
-    }
-    try {
-      const res = await this.API.fetchRelationshipData(
-        datasource.rowTableId,
-        datasource.rowId,
-        datasource.fieldName
-      )
-      return { rows: res }
-    } catch (error) {
-      return { rows: [] }
-    }
-  }
-}

packages/frontend-core/src/fetch/RelationshipFetch.ts

@@ -0,0 +1,49 @@
+import { Table } from "@budibase/types"
+import DataFetch from "./DataFetch"
+
+interface RelationshipDatasource {
+  type: "link"
+  tableId: string
+  rowId: string
+  rowTableId: string
+  fieldName: string
+}
+
+export default class RelationshipFetch extends DataFetch<
+  RelationshipDatasource,
+  Table
+> {
+  async getDefinition() {
+    const { datasource } = this.options
+
+    if (!datasource?.tableId) {
+      return null
+    }
+    try {
+      return await this.API.fetchTableDefinition(datasource.tableId)
+    } catch (error: any) {
+      this.store.update(state => ({
+        ...state,
+        error,
+      }))
+      return null
+    }
+  }
+
+  async getData() {
+    const { datasource } = this.options
+    if (!datasource?.rowId || !datasource?.rowTableId) {
+      return { rows: [] }
+    }
+    try {
+      const res = await this.API.fetchRelationshipData(
+        datasource.rowTableId,
+        datasource.rowId,
+        datasource.fieldName
+      )
+      return { rows: res }
+    } catch (error) {
+      return { rows: [] }
+    }
+  }
+}

packages/frontend-core/src/fetch/TableFetch.ts

@@ -1,9 +1,14 @@
 import { get } from "svelte/store"
-import DataFetch from "./DataFetch.js"
+import DataFetch from "./DataFetch"
-import { SortOrder } from "@budibase/types"
+import { SortOrder, Table } from "@budibase/types"
 
-export default class TableFetch extends DataFetch {
+interface TableDatasource {
-  determineFeatureFlags() {
+  type: "table"
+  tableId: string
+}
+
+export default class TableFetch extends DataFetch<TableDatasource, Table> {
+  async determineFeatureFlags() {
     return {
       supportsSearch: true,
       supportsSort: true,
@@ -11,6 +16,23 @@ export default class TableFetch extends DataFetch {
     }
   }
 
+  async getDefinition() {
+    const { datasource } = this.options
+
+    if (!datasource?.tableId) {
+      return null
+    }
+    try {
+      return await this.API.fetchTableDefinition(datasource.tableId)
+    } catch (error: any) {
+      this.store.update(state => ({
+        ...state,
+        error,
+      }))
+      return null
+    }
+  }
+
   async getData() {
     const { datasource, limit, sortColumn, sortOrder, sortType, paginate } =
       this.options
@@ -23,7 +45,7 @@ export default class TableFetch extends DataFetch {
         query,
         limit,
         sort: sortColumn,
-        sortOrder: sortOrder?.toLowerCase() ?? SortOrder.ASCENDING,
+        sortOrder: sortOrder ?? SortOrder.ASCENDING,
         sortType,
         paginate,
         bookmark: cursor,

packages/frontend-core/src/fetch/UserFetch.ts

@@ -1,19 +1,37 @@
 import { get } from "svelte/store"
-import DataFetch from "./DataFetch.js"
+import DataFetch, { DataFetchParams } from "./DataFetch"
 import { TableNames } from "../constants"
 import { utils } from "@budibase/shared-core"
+import { SearchFilters, SearchUsersRequest } from "@budibase/types"
 
-export default class UserFetch extends DataFetch {
+interface UserFetchQuery {
-  constructor(opts) {
+  appId: string
+  paginated: boolean
+}
+
+interface UserDatasource {
+  type: "user"
+  tableId: TableNames.USERS
+}
+
+interface UserDefinition {}
+
+export default class UserFetch extends DataFetch<
+  UserDatasource,
+  UserDefinition,
+  UserFetchQuery
+> {
+  constructor(opts: DataFetchParams<UserDatasource, UserFetchQuery>) {
     super({
       ...opts,
       datasource: {
+        type: "user",
         tableId: TableNames.USERS,
       },
     })
   }
 
-  determineFeatureFlags() {
+  async determineFeatureFlags() {
     return {
       supportsSearch: true,
       supportsSort: false,
@@ -22,9 +40,7 @@ export default class UserFetch extends DataFetch {
   }
 
   async getDefinition() {
-    return {
+    return { schema: {} }
-      schema: {},
-    }
   }
 
   async getData() {
@@ -32,15 +48,16 @@ export default class UserFetch extends DataFetch {
     const { cursor, query } = get(this.store)
 
     // Convert old format to new one - we now allow use of the lucene format
-    const { appId, paginated, ...rest } = query || {}
+    const { appId, paginated, ...rest } = query
-    const finalQuery = utils.isSupportedUserSearch(rest)
+
-      ? query
+    const finalQuery: SearchFilters = utils.isSupportedUserSearch(rest)
-      : { string: { email: null } }
+      ? rest
+      : {}
 
     try {
-      const opts = {
+      const opts: SearchUsersRequest = {
-        bookmark: cursor,
+        bookmark: cursor ?? undefined,
-        query: finalQuery,
+        query: finalQuery ?? undefined,
         appId: appId,
         paginate: paginated || paginate,
         limit,

packages/frontend-core/src/fetch/ViewFetch.js

@@ -1,23 +0,0 @@
-import DataFetch from "./DataFetch.js"
-
-export default class ViewFetch extends DataFetch {
-  getSchema(datasource, definition) {
-    return definition?.views?.[datasource.name]?.schema
-  }
-
-  async getData() {
-    const { datasource } = this.options
-    try {
-      const res = await this.API.fetchViewData(datasource.name, {
-        calculation: datasource.calculation,
-        field: datasource.field,
-        groupBy: datasource.groupBy,
-        tableId: datasource.tableId,
-      })
-      return { rows: res || [] }
-    } catch (error) {
-      console.error(error)
-      return { rows: [] }
-    }
-  }
-}

packages/frontend-core/src/fetch/ViewFetch.ts

@@ -0,0 +1,51 @@
+import { Table } from "@budibase/types"
+import DataFetch from "./DataFetch"
+
+type ViewV1Datasource = {
+  type: "view"
+  name: string
+  tableId: string
+  calculation: string
+  field: string
+  groupBy: string
+}
+
+export default class ViewFetch extends DataFetch<ViewV1Datasource, Table> {
+  async getDefinition() {
+    const { datasource } = this.options
+
+    if (!datasource?.tableId) {
+      return null
+    }
+    try {
+      return await this.API.fetchTableDefinition(datasource.tableId)
+    } catch (error: any) {
+      this.store.update(state => ({
+        ...state,
+        error,
+      }))
+      return null
+    }
+  }
+
+  getSchema(definition: Table) {
+    const { datasource } = this.options
+    return definition?.views?.[datasource.name]?.schema
+  }
+
+  async getData() {
+    const { datasource } = this.options
+    try {
+      const res = await this.API.fetchViewData(datasource.name, {
+        calculation: datasource.calculation,
+        field: datasource.field,
+        groupBy: datasource.groupBy,
+        tableId: datasource.tableId,
+      })
+      return { rows: res || [] }
+    } catch (error) {
+      console.error(error, { datasource })
+      return { rows: [] }
+    }
+  }
+}

packages/frontend-core/src/fetch/ViewV2Fetch.ts

@@ -1,9 +1,18 @@
-import { ViewV2Type } from "@budibase/types"
+import { SortOrder, ViewV2Enriched, ViewV2Type } from "@budibase/types"
-import DataFetch from "./DataFetch.js"
+import DataFetch from "./DataFetch"
 import { get } from "svelte/store"
+import { helpers } from "@budibase/shared-core"
 
-export default class ViewV2Fetch extends DataFetch {
+interface ViewDatasource {
-  determineFeatureFlags() {
+  type: "viewV2"
+  id: string
+}
+
+export default class ViewV2Fetch extends DataFetch<
+  ViewDatasource,
+  ViewV2Enriched
+> {
+  async determineFeatureFlags() {
     return {
       supportsSearch: true,
       supportsSort: true,
@@ -11,18 +20,13 @@ export default class ViewV2Fetch extends DataFetch {
     }
   }
 
-  getSchema(datasource, definition) {
+  async getDefinition() {
-    return definition?.schema
+    const { datasource } = this.options
-  }
 
-  async getDefinition(datasource) {
-    if (!datasource?.id) {
-      return null
-    }
     try {
       const res = await this.API.viewV2.fetchDefinition(datasource.id)
       return res?.data
-    } catch (error) {
+    } catch (error: any) {
       this.store.update(state => ({
         ...state,
         error,
@@ -42,8 +46,10 @@ export default class ViewV2Fetch extends DataFetch {
 
     // If this is a calculation view and we have no calculations, return nothing
     if (
-      definition.type === ViewV2Type.CALCULATION &&
+      definition?.type === ViewV2Type.CALCULATION &&
-      !Object.values(definition.schema || {}).some(x => x.calculationType)
+      !Object.values(definition.schema || {}).some(
+        helpers.views.isCalculationField
+      )
     ) {
       return {
         rows: [],
@@ -56,25 +62,41 @@ export default class ViewV2Fetch extends DataFetch {
     // If sort/filter params are not defined, update options to store the
     // params built in to this view. This ensures that we can accurately
     // compare old and new params and skip a redundant API call.
-    if (!sortColumn && definition.sort?.field) {
+    if (!sortColumn && definition?.sort?.field) {
       this.options.sortColumn = definition.sort.field
-      this.options.sortOrder = definition.sort.order
+      this.options.sortOrder = definition.sort.order || SortOrder.ASCENDING
     }
 
     try {
-      const res = await this.API.viewV2.fetch(datasource.id, {
+      const request = {
-        ...(query ? { query } : {}),
+        query,
         paginate,
         limit,
         bookmark: cursor,
         sort: sortColumn,
-        sortOrder: sortOrder?.toLowerCase(),
+        sortOrder: sortOrder,
         sortType,
-      })
+      }
-      return {
+      if (paginate) {
-        rows: res?.rows || [],
+        const res = await this.API.viewV2.fetch(datasource.id, {
-        hasNextPage: res?.hasNextPage || false,
+          ...request,
-        cursor: res?.bookmark || null,
+          paginate,
+        })
+        return {
+          rows: res?.rows || [],
+          hasNextPage: res?.hasNextPage || false,
+          cursor: res?.bookmark || null,
+        }
+      } else {
+        const res = await this.API.viewV2.fetch(datasource.id, {
+          ...request,
+          paginate,
+        })
+        return {
+          rows: res?.rows || [],
+          hasNextPage: false,
+          cursor: null,
+        }
       }
     } catch (error) {
       return {

packages/frontend-core/src/fetch/index.js

@@ -1,57 +0,0 @@
-import TableFetch from "./TableFetch.js"
-import ViewFetch from "./ViewFetch.js"
-import ViewV2Fetch from "./ViewV2Fetch.js"
-import QueryFetch from "./QueryFetch.js"
-import RelationshipFetch from "./RelationshipFetch.js"
-import NestedProviderFetch from "./NestedProviderFetch.js"
-import FieldFetch from "./FieldFetch.js"
-import JSONArrayFetch from "./JSONArrayFetch.js"
-import UserFetch from "./UserFetch.js"
-import GroupUserFetch from "./GroupUserFetch.js"
-import CustomFetch from "./CustomFetch.js"
-import QueryArrayFetch from "./QueryArrayFetch.js"
-
-const DataFetchMap = {
-  table: TableFetch,
-  view: ViewFetch,
-  viewV2: ViewV2Fetch,
-  query: QueryFetch,
-  link: RelationshipFetch,
-  user: UserFetch,
-  groupUser: GroupUserFetch,
-  custom: CustomFetch,
-
-  // Client specific datasource types
-  provider: NestedProviderFetch,
-  field: FieldFetch,
-  jsonarray: JSONArrayFetch,
-  queryarray: QueryArrayFetch,
-}
-
-// Constructs a new fetch model for a certain datasource
-export const fetchData = ({ API, datasource, options }) => {
-  const Fetch = DataFetchMap[datasource?.type] || TableFetch
-  return new Fetch({ API, datasource, ...options })
-}
-
-// Creates an empty fetch instance with no datasource configured, so no data
-// will initially be loaded
-const createEmptyFetchInstance = ({ API, datasource }) => {
-  const handler = DataFetchMap[datasource?.type]
-  if (!handler) {
-    return null
-  }
-  return new handler({ API })
-}
-
-// Fetches the definition of any type of datasource
-export const getDatasourceDefinition = async ({ API, datasource }) => {
-  const instance = createEmptyFetchInstance({ API, datasource })
-  return await instance?.getDefinition(datasource)
-}
-
-// Fetches the schema of any type of datasource
-export const getDatasourceSchema = ({ API, datasource, definition }) => {
-  const instance = createEmptyFetchInstance({ API, datasource })
-  return instance?.getSchema(datasource, definition)
-}

packages/frontend-core/src/fetch/index.ts

@@ -0,0 +1,93 @@
+import TableFetch from "./TableFetch"
+import ViewFetch from "./ViewFetch"
+import ViewV2Fetch from "./ViewV2Fetch"
+import QueryFetch from "./QueryFetch"
+import RelationshipFetch from "./RelationshipFetch"
+import NestedProviderFetch from "./NestedProviderFetch"
+import FieldFetch from "./FieldFetch"
+import JSONArrayFetch from "./JSONArrayFetch"
+import UserFetch from "./UserFetch"
+import GroupUserFetch from "./GroupUserFetch"
+import CustomFetch from "./CustomFetch"
+import QueryArrayFetch from "./QueryArrayFetch"
+import { APIClient } from "../api/types"
+
+export type DataFetchType = keyof typeof DataFetchMap
+
+export const DataFetchMap = {
+  table: TableFetch,
+  view: ViewFetch,
+  viewV2: ViewV2Fetch,
+  query: QueryFetch,
+  link: RelationshipFetch,
+  user: UserFetch,
+  groupUser: GroupUserFetch,
+  custom: CustomFetch,
+
+  // Client specific datasource types
+  provider: NestedProviderFetch,
+  field: FieldFetch<"field">,
+  jsonarray: JSONArrayFetch,
+  queryarray: QueryArrayFetch,
+}
+
+// Constructs a new fetch model for a certain datasource
+export const fetchData = ({ API, datasource, options }: any) => {
+  const Fetch = DataFetchMap[datasource?.type as DataFetchType] || TableFetch
+  const fetch = new Fetch({ API, datasource, ...options })
+
+  // Initially fetch data but don't bother waiting for the result
+  fetch.getInitialData()
+
+  return fetch
+}
+
+// Creates an empty fetch instance with no datasource configured, so no data
+// will initially be loaded
+const createEmptyFetchInstance = <TDatasource extends { type: DataFetchType }>({
+  API,
+  datasource,
+}: {
+  API: APIClient
+  datasource: TDatasource
+}) => {
+  const handler = DataFetchMap[datasource?.type as DataFetchType]
+  if (!handler) {
+    return null
+  }
+  return new handler({
+    API,
+    datasource: null as never,
+    query: null as any,
+  })
+}
+
+// Fetches the definition of any type of datasource
+export const getDatasourceDefinition = async <
+  TDatasource extends { type: DataFetchType }
+>({
+  API,
+  datasource,
+}: {
+  API: APIClient
+  datasource: TDatasource
+}) => {
+  const instance = createEmptyFetchInstance({ API, datasource })
+  return await instance?.getDefinition()
+}
+
+// Fetches the schema of any type of datasource
+export const getDatasourceSchema = <
+  TDatasource extends { type: DataFetchType }
+>({
+  API,
+  datasource,
+  definition,
+}: {
+  API: APIClient
+  datasource: TDatasource
+  definition?: any
+}) => {
+  const instance = createEmptyFetchInstance({ API, datasource })
+  return instance?.getSchema(definition)
+}

packages/frontend-core/src/index.ts

@@ -1,5 +1,6 @@
 export { createAPIClient } from "./api"
-export { fetchData } from "./fetch"
+export { fetchData, DataFetchMap } from "./fetch"
+export type { DataFetchType } from "./fetch"
 export * as Constants from "./constants"
 export * from "./stores"
 export * from "./utils"

packages/frontend-core/src/utils/json.d.ts

@@ -0,0 +1,23 @@
+import { JsonFieldMetadata, QuerySchema } from "@budibase/types"
+
+type Schema = Record<string, QuerySchema | string>
+
+declare module "./json" {
+  export const getJSONArrayDatasourceSchema: (
+    tableSchema: Schema,
+    datasource: any
+  ) => Record<string, { type: string; name: string; prefixKeys: string }>
+
+  export const generateQueryArraySchemas: (
+    schema: Schema,
+    nestedSchemaFields?: Record<string, Schema>
+  ) => Schema
+
+  export const convertJSONSchemaToTableSchema: (
+    jsonSchema: JsonFieldMetadata,
+    options: {
+      squashObjects?: boolean
+      prefixKeys?: string
+    }
+  ) => Record<string, { type: string; name: string; prefixKeys: string }>
+}

packages/frontend-core/src/utils/utils.js

@@ -209,6 +209,9 @@ export const buildFormBlockButtonConfig = props => {
     {
       "##eventHandlerType": "Close Side Panel",
     },
+    {
+      "##eventHandlerType": "Close Modal",
+    },
 
     ...(actionUrl
       ? [

packages/pro

@@ -1 +1 @@
-Subproject commit 32d84f109d4edc526145472a7446327312151442
+Subproject commit 193476cdfade6d3c613e6972f16ee0c527e01ff6

packages/server/src/api/controllers/query/index.ts

@@ -355,7 +355,7 @@ async function execute(
     ExecuteQueryRequest,
     ExecuteV2QueryResponse | ExecuteV1QueryResponse
   >,
-  opts: any = { rowsOnly: false, isAutomation: false }
+  opts = { rowsOnly: false, isAutomation: false }
 ) {
   const db = context.getAppDB()
 
@@ -416,7 +416,7 @@ export async function executeV1(
 export async function executeV2(
   ctx: UserCtx<ExecuteQueryRequest, ExecuteV2QueryResponse>
 ) {
-  return execute(ctx, { rowsOnly: false })
+  return execute(ctx, { rowsOnly: false, isAutomation: false })
 }
 
 export async function executeV2AsAutomation(

packages/server/src/api/controllers/row/staticFormula.ts

@@ -4,15 +4,8 @@ import {
   processAIColumns,
   processFormulas,
 } from "../../../utilities/rowProcessor"
-import { context, features } from "@budibase/backend-core"
+import { context } from "@budibase/backend-core"
-import {
+import { Table, Row, FormulaType, FieldType, ViewV2 } from "@budibase/types"
-  Table,
-  Row,
-  FeatureFlag,
-  FormulaType,
-  FieldType,
-  ViewV2,
-} from "@budibase/types"
 import * as linkRows from "../../../db/linkedRows"
 import isEqual from "lodash/isEqual"
 import { cloneDeep, merge } from "lodash/fp"
@@ -162,11 +155,10 @@ export async function finaliseRow(
     dynamic: false,
     contextRows: [enrichedRow],
   })
+
   const aiEnabled =
-    ((await features.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
+    (await pro.features.isBudibaseAIEnabled()) ||
-      (await pro.features.isBudibaseAIEnabled())) ||
+    (await pro.features.isAICustomConfigsEnabled())
-    ((await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) &&
-      (await pro.features.isAICustomConfigsEnabled()))
   if (aiEnabled) {
     row = await processAIColumns(table, row, {
       contextRows: [enrichedRow],
@@ -184,11 +176,6 @@ export async function finaliseRow(
   enrichedRow = await processFormulas(table, enrichedRow, {
     dynamic: false,
   })
-  if (aiEnabled) {
-    enrichedRow = await processAIColumns(table, enrichedRow, {
-      contextRows: [enrichedRow],
-    })
-  }
 
   // this updates the related formulas in other rows based on the relations to this row
   if (updateFormula) {

packages/server/src/api/controllers/row/views.ts

@@ -1,16 +1,16 @@
 import {
   UserCtx,
   ViewV2,
-  SearchRowResponse,
   SearchViewRowRequest,
   RequiredKeys,
   RowSearchParams,
+  PaginatedSearchRowResponse,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { context } from "@budibase/backend-core"
 
 export async function searchView(
-  ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
+  ctx: UserCtx<SearchViewRowRequest, PaginatedSearchRowResponse>
 ) {
   const { viewId } = ctx.params
 
@@ -49,7 +49,13 @@ export async function searchView(
     user: sdk.users.getUserContextBindings(ctx.user),
   })
   result.rows.forEach(r => (r._viewId = view.id))
-  ctx.body = result
+
+  ctx.body = {
+    rows: result.rows,
+    bookmark: result.bookmark,
+    hasNextPage: result.hasNextPage,
+    totalRows: result.totalRows,
+  }
 }
 
 function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {

packages/server/src/api/routes/tests/row.spec.ts

@@ -8,7 +8,13 @@ import {
 import tk from "timekeeper"
 import emitter from "../../../../src/events"
 import { outputProcessing } from "../../../utilities/rowProcessor"
-import { context, InternalTable, tenancy, utils } from "@budibase/backend-core"
+import {
+  context,
+  setEnv,
+  InternalTable,
+  tenancy,
+  utils,
+} from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
   AIOperationEnum,
@@ -42,19 +48,8 @@ import { InternalTables } from "../../../db/utils"
 import { withEnv } from "../../../environment"
 import { JsTimeoutError } from "@budibase/string-templates"
 import { isDate } from "../../../utilities"
-
+import nock from "nock"
-jest.mock("@budibase/pro", () => ({
+import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai"
-  ...jest.requireActual("@budibase/pro"),
-  ai: {
-    LargeLanguageModel: {
-      forCurrentTenant: async () => ({
-        llm: {},
-        run: jest.fn(() => `Mock LLM Response`),
-        buildPromptFromAIOperation: jest.fn(),
-      }),
-    },
-  },
-}))
 
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
@@ -99,6 +94,8 @@ if (descriptions.length) {
         const ds = await dsProvider()
         datasource = ds.datasource
         client = ds.client
+
+        mocks.licenses.useCloudFree()
       })
 
       afterAll(async () => {
@@ -172,10 +169,6 @@ if (descriptions.length) {
         )
       }
 
-      beforeEach(async () => {
-        mocks.licenses.useCloudFree()
-      })
-
       const getRowUsage = async () => {
         const { total } = await config.doInContext(undefined, () =>
           quotas.getCurrentUsageValues(
@@ -2348,7 +2341,7 @@ if (descriptions.length) {
               [FieldType.ARRAY]: ["options 2", "options 4"],
               [FieldType.NUMBER]: generator.natural(),
               [FieldType.BOOLEAN]: generator.bool(),
-              [FieldType.DATETIME]: generator.date().toISOString(),
+              [FieldType.DATETIME]: generator.date().toISOString().slice(0, 10),
               [FieldType.ATTACHMENTS]: [setup.structures.basicAttachment()],
               [FieldType.ATTACHMENT_SINGLE]: setup.structures.basicAttachment(),
               [FieldType.FORMULA]: undefined, // generated field
@@ -3224,10 +3217,17 @@ if (descriptions.length) {
       isInternal &&
         describe("AI fields", () => {
           let table: Table
+          let envCleanup: () => void
 
           beforeAll(async () => {
             mocks.licenses.useBudibaseAI()
             mocks.licenses.useAICustomConfigs()
+            envCleanup = setEnv({
+              OPENAI_API_KEY: "sk-abcdefghijklmnopqrstuvwxyz1234567890abcd",
+            })
+
+            mockChatGPTResponse("Mock LLM Response")
+
             table = await config.api.table.save(
               saveTableRequest({
                 schema: {
@@ -3251,7 +3251,9 @@ if (descriptions.length) {
           })
 
           afterAll(() => {
-            jest.unmock("@budibase/pro")
+            nock.cleanAll()
+            envCleanup()
+            mocks.licenses.useCloudFree()
           })
 
           it("should be able to save a row with an AI column", async () => {

packages/server/src/api/routes/tests/search.spec.ts

@@ -1683,6 +1683,151 @@ if (descriptions.length) {
                 })
               })
 
+            describe("datetime - date only", () => {
+              describe.each([true, false])(
+                "saved with timestamp: %s",
+                saveWithTimestamp => {
+                  describe.each([true, false])(
+                    "search with timestamp: %s",
+                    searchWithTimestamp => {
+                      const SAVE_SUFFIX = saveWithTimestamp
+                        ? "T00:00:00.000Z"
+                        : ""
+                      const SEARCH_SUFFIX = searchWithTimestamp
+                        ? "T00:00:00.000Z"
+                        : ""
+
+                      const JAN_1ST = `2020-01-01`
+                      const JAN_10TH = `2020-01-10`
+                      const JAN_30TH = `2020-01-30`
+                      const UNEXISTING_DATE = `2020-01-03`
+                      const NULL_DATE__ID = `null_date__id`
+
+                      beforeAll(async () => {
+                        tableOrViewId = await createTableOrView({
+                          dateid: { name: "dateid", type: FieldType.STRING },
+                          date: {
+                            name: "date",
+                            type: FieldType.DATETIME,
+                            dateOnly: true,
+                          },
+                        })
+
+                        await createRows([
+                          { dateid: NULL_DATE__ID, date: null },
+                          { date: `${JAN_1ST}${SAVE_SUFFIX}` },
+                          { date: `${JAN_10TH}${SAVE_SUFFIX}` },
+                        ])
+                      })
+
+                      describe("equal", () => {
+                        it("successfully finds a row", async () => {
+                          await expectQuery({
+                            equal: { date: `${JAN_1ST}${SEARCH_SUFFIX}` },
+                          }).toContainExactly([{ date: JAN_1ST }])
+                        })
+
+                        it("successfully finds an ISO8601 row", async () => {
+                          await expectQuery({
+                            equal: { date: `${JAN_10TH}${SEARCH_SUFFIX}` },
+                          }).toContainExactly([{ date: JAN_10TH }])
+                        })
+
+                        it("finds a row with ISO8601 timestamp", async () => {
+                          await expectQuery({
+                            equal: { date: `${JAN_1ST}${SEARCH_SUFFIX}` },
+                          }).toContainExactly([{ date: JAN_1ST }])
+                        })
+
+                        it("fails to find nonexistent row", async () => {
+                          await expectQuery({
+                            equal: {
+                              date: `${UNEXISTING_DATE}${SEARCH_SUFFIX}`,
+                            },
+                          }).toFindNothing()
+                        })
+                      })
+
+                      describe("notEqual", () => {
+                        it("successfully finds a row", async () => {
+                          await expectQuery({
+                            notEqual: { date: `${JAN_1ST}${SEARCH_SUFFIX}` },
+                          }).toContainExactly([
+                            { date: JAN_10TH },
+                            { dateid: NULL_DATE__ID },
+                          ])
+                        })
+
+                        it("fails to find nonexistent row", async () => {
+                          await expectQuery({
+                            notEqual: { date: `${JAN_30TH}${SEARCH_SUFFIX}` },
+                          }).toContainExactly([
+                            { date: JAN_1ST },
+                            { date: JAN_10TH },
+                            { dateid: NULL_DATE__ID },
+                          ])
+                        })
+                      })
+
+                      describe("oneOf", () => {
+                        it("successfully finds a row", async () => {
+                          await expectQuery({
+                            oneOf: { date: [`${JAN_1ST}${SEARCH_SUFFIX}`] },
+                          }).toContainExactly([{ date: JAN_1ST }])
+                        })
+
+                        it("fails to find nonexistent row", async () => {
+                          await expectQuery({
+                            oneOf: {
+                              date: [`${UNEXISTING_DATE}${SEARCH_SUFFIX}`],
+                            },
+                          }).toFindNothing()
+                        })
+                      })
+
+                      describe("range", () => {
+                        it("successfully finds a row", async () => {
+                          await expectQuery({
+                            range: {
+                              date: {
+                                low: `${JAN_1ST}${SEARCH_SUFFIX}`,
+                                high: `${JAN_1ST}${SEARCH_SUFFIX}`,
+                              },
+                            },
+                          }).toContainExactly([{ date: JAN_1ST }])
+                        })
+
+                        it("successfully finds multiple rows", async () => {
+                          await expectQuery({
+                            range: {
+                              date: {
+                                low: `${JAN_1ST}${SEARCH_SUFFIX}`,
+                                high: `${JAN_10TH}${SEARCH_SUFFIX}`,
+                              },
+                            },
+                          }).toContainExactly([
+                            { date: JAN_1ST },
+                            { date: JAN_10TH },
+                          ])
+                        })
+
+                        it("successfully finds no rows", async () => {
+                          await expectQuery({
+                            range: {
+                              date: {
+                                low: `${JAN_30TH}${SEARCH_SUFFIX}`,
+                                high: `${JAN_30TH}${SEARCH_SUFFIX}`,
+                              },
+                            },
+                          }).toFindNothing()
+                        })
+                      })
+                    }
+                  )
+                }
+              )
+            })
+
             isInternal &&
               !isInMemory &&
               describe("AI Column", () => {

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -1,4 +1,5 @@
 import {
+  AIOperationEnum,
   ArrayOperator,
   BasicOperator,
   BBReferenceFieldSubType,
@@ -42,7 +43,9 @@ import {
 } from "../../../integrations/tests/utils"
 import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
-import { context, db, events, roles } from "@budibase/backend-core"
+import { context, db, events, roles, setEnv } from "@budibase/backend-core"
+import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai"
+import nock from "nock"
 
 const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
 
@@ -100,6 +103,7 @@ if (descriptions.length) {
 
       beforeAll(async () => {
         await config.init()
+        mocks.licenses.useCloudFree()
 
         const ds = await dsProvider()
         rawDatasource = ds.rawDatasource
@@ -109,7 +113,6 @@ if (descriptions.length) {
 
       beforeEach(() => {
         jest.clearAllMocks()
-        mocks.licenses.useCloudFree()
       })
 
       describe("view crud", () => {
@@ -507,7 +510,6 @@ if (descriptions.length) {
             })
 
             it("readonly fields can be used on free license", async () => {
-              mocks.licenses.useCloudFree()
               const table = await config.api.table.save(
                 saveTableRequest({
                   schema: {
@@ -933,6 +935,95 @@ if (descriptions.length) {
                 }
               )
             })
+
+          isInternal &&
+            describe("AI fields", () => {
+              let envCleanup: () => void
+              beforeAll(() => {
+                mocks.licenses.useBudibaseAI()
+                mocks.licenses.useAICustomConfigs()
+                envCleanup = setEnv({
+                  OPENAI_API_KEY: "sk-abcdefghijklmnopqrstuvwxyz1234567890abcd",
+                })
+
+                mockChatGPTResponse(prompt => {
+                  if (prompt.includes("elephant")) {
+                    return "big"
+                  }
+                  if (prompt.includes("mouse")) {
+                    return "small"
+                  }
+                  if (prompt.includes("whale")) {
+                    return "big"
+                  }
+                  return "unknown"
+                })
+              })
+
+              afterAll(() => {
+                nock.cleanAll()
+                envCleanup()
+                mocks.licenses.useCloudFree()
+              })
+
+              it("can use AI fields in view calculations", async () => {
+                const table = await config.api.table.save(
+                  saveTableRequest({
+                    schema: {
+                      animal: {
+                        name: "animal",
+                        type: FieldType.STRING,
+                      },
+                      bigOrSmall: {
+                        name: "bigOrSmall",
+                        type: FieldType.AI,
+                        operation: AIOperationEnum.CATEGORISE_TEXT,
+                        categories: "big,small",
+                        columns: ["animal"],
+                      },
+                    },
+                  })
+                )
+
+                const view = await config.api.viewV2.create({
+                  tableId: table._id!,
+                  name: generator.guid(),
+                  type: ViewV2Type.CALCULATION,
+                  schema: {
+                    bigOrSmall: {
+                      visible: true,
+                    },
+                    count: {
+                      visible: true,
+                      calculationType: CalculationType.COUNT,
+                      field: "animal",
+                    },
+                  },
+                })
+
+                await config.api.row.save(table._id!, {
+                  animal: "elephant",
+                })
+
+                await config.api.row.save(table._id!, {
+                  animal: "mouse",
+                })
+
+                await config.api.row.save(table._id!, {
+                  animal: "whale",
+                })
+
+                const { rows } = await config.api.row.search(view.id, {
+                  sort: "bigOrSmall",
+                  sortOrder: SortOrder.ASCENDING,
+                })
+                expect(rows).toHaveLength(2)
+                expect(rows[0].bigOrSmall).toEqual("big")
+                expect(rows[1].bigOrSmall).toEqual("small")
+                expect(rows[0].count).toEqual(2)
+                expect(rows[1].count).toEqual(1)
+              })
+            })
         })
 
         describe("update", () => {
@@ -1836,7 +1927,6 @@ if (descriptions.length) {
               },
             })
 
-            mocks.licenses.useCloudFree()
             const view = await getDelegate(res)
             expect(view.schema?.one).toEqual(
               expect.objectContaining({ visible: true, readonly: true })

packages/server/src/automations/actions.ts

@@ -27,11 +27,9 @@ import {
   Hosting,
   ActionImplementation,
   AutomationStepDefinition,
-  FeatureFlag,
 } from "@budibase/types"
 import sdk from "../sdk"
 import { getAutomationPlugin } from "../utilities/fileSystem"
-import { features } from "@budibase/backend-core"
 
 type ActionImplType = ActionImplementations<
   typeof env.SELF_HOSTED extends "true" ? Hosting.SELF : Hosting.CLOUD
@@ -78,6 +76,7 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<
   LOOP: loop.definition,
   COLLECT: collect.definition,
   TRIGGER_AUTOMATION_RUN: triggerAutomationRun.definition,
+  BRANCH: branch.definition,
   // these used to be lowercase step IDs, maintain for backwards compat
   discord: discord.definition,
   slack: slack.definition,
@@ -105,14 +104,7 @@ if (env.SELF_HOSTED) {
 export async function getActionDefinitions(): Promise<
   Record<keyof typeof AutomationActionStepId, AutomationStepDefinition>
 > {
-  if (await features.isEnabled(FeatureFlag.AUTOMATION_BRANCHING)) {
+  if (env.SELF_HOSTED) {
-    BUILTIN_ACTION_DEFINITIONS["BRANCH"] = branch.definition
-  }
-  if (
-    env.SELF_HOSTED ||
-    (await features.isEnabled(FeatureFlag.BUDIBASE_AI)) ||
-    (await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS))
-  ) {
     BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition
   }
 

packages/server/src/automations/steps/openai.ts

@@ -7,9 +7,8 @@ import {
   AutomationIOType,
   OpenAIStepInputs,
   OpenAIStepOutputs,
-  FeatureFlag,
 } from "@budibase/types"
-import { env, features } from "@budibase/backend-core"
+import { env } from "@budibase/backend-core"
 import * as automationUtils from "../automationUtils"
 import * as pro from "@budibase/pro"
 
@@ -99,12 +98,8 @@ export async function run({
 
   try {
     let response
-    const customConfigsEnabled =
+    const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled()
-      (await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) &&
+    const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
-      (await pro.features.isAICustomConfigsEnabled())
-    const budibaseAIEnabled =
-      (await features.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
-      (await pro.features.isBudibaseAIEnabled())
 
     let llmWrapper
     if (budibaseAIEnabled || customConfigsEnabled) {

packages/server/src/sdk/app/views/index.ts

@@ -432,6 +432,21 @@ export async function enrichSchema(
       ...tableSchema[key],
       ...ui,
       order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key]?.order,
+      // When this was written, the only column types in FieldSchema to have columns
+      // field were the relationship columns. We blank this out here to make sure it's
+      // not set on non-relationship columns, then below we populate it by calling
+      // populateRelSchema.
+      //
+      // For Budibase 3.0 we introduced the FieldType.AI fields. Some of these fields
+      // have `columns: string[]` and it flew under the radar here because the
+      // AIFieldMetadata type isn't a union on its subtypes, it has a collection of
+      // optional fields. So columns is `columns?: string[]` which allows undefined,
+      // and doesn't fail this type check.
+      //
+      // What this means in practice is when FieldType.AI fields get enriched, we
+      // delete their `columns`. At the time of writing, I don't believe anything in
+      // the frontend depends on this, but it is odd and will probably bite us at
+      // some point.
       columns: undefined,
     }
 

packages/server/src/tests/utilities/mocks/openai.ts

@@ -0,0 +1,46 @@
+import nock from "nock"
+
+let chatID = 1
+
+export function mockChatGPTResponse(
+  response: string | ((prompt: string) => string)
+) {
+  return nock("https://api.openai.com")
+    .post("/v1/chat/completions")
+    .reply(200, (uri, requestBody) => {
+      let content = response
+      if (typeof response === "function") {
+        const messages = (requestBody as any).messages
+        content = response(messages[0].content)
+      }
+
+      chatID++
+
+      return {
+        id: `chatcmpl-${chatID}`,
+        object: "chat.completion",
+        created: Math.floor(Date.now() / 1000),
+        model: "gpt-4o-mini",
+        system_fingerprint: `fp_${chatID}`,
+        choices: [
+          {
+            index: 0,
+            message: { role: "assistant", content },
+            logprobs: null,
+            finish_reason: "stop",
+          },
+        ],
+        usage: {
+          prompt_tokens: 0,
+          completion_tokens: 0,
+          total_tokens: 0,
+          completion_tokens_details: {
+            reasoning_tokens: 0,
+            accepted_prediction_tokens: 0,
+            rejected_prediction_tokens: 0,
+          },
+        },
+      }
+    })
+    .persist()
+}

packages/server/src/threads/definitions.ts

@@ -3,7 +3,10 @@ import { Datasource, Row, Query } from "@budibase/types"
 export type WorkerCallback = (error: any, response?: any) => void
 
 export interface QueryEvent
-  extends Omit<Query, "datasourceId" | "name" | "parameters" | "readable"> {
+  extends Omit<
+    Query,
+    "datasourceId" | "name" | "parameters" | "readable" | "nestedSchemaFields"
+  > {
   appId?: string
   datasource: Datasource
   pagination?: any

packages/server/src/utilities/rowProcessor/index.ts

@@ -411,6 +411,15 @@ export async function coreOutputProcessing(
           row[property] = `${hours}:${minutes}:${seconds}`
         }
       }
+    } else if (column.type === FieldType.DATETIME && column.dateOnly) {
+      for (const row of rows) {
+        if (typeof row[property] === "string") {
+          row[property] = new Date(row[property])
+        }
+        if (row[property] instanceof Date) {
+          row[property] = row[property].toISOString().slice(0, 10)
+        }
+      }
     } else if (column.type === FieldType.LINK) {
       for (let row of rows) {
         // if relationship is empty - remove the array, this has been part of the API for some time

packages/server/src/utilities/rowProcessor/utils.ts

@@ -160,7 +160,7 @@ export async function processAIColumns<T extends Row | Row[]>(
 
           return tracer.trace("processAIColumn", {}, async span => {
             span?.addTags({ table_id: table._id, column })
-            const llmResponse = await llmWrapper.run(prompt!)
+            const llmResponse = await llmWrapper.run(prompt)
             return {
               ...row,
               [column]: llmResponse,

packages/shared-core/src/filters.ts

@@ -699,7 +699,27 @@ export function runQuery<T extends Record<string, any>>(
       return docValue._id === testValue
     }
 
-    return docValue === testValue
+    if (docValue === testValue) {
+      return true
+    }
+
+    if (docValue == null && testValue != null) {
+      return false
+    }
+
+    if (docValue != null && testValue == null) {
+      return false
+    }
+
+    const leftDate = dayjs(docValue)
+    if (leftDate.isValid()) {
+      const rightDate = dayjs(testValue)
+      if (rightDate.isValid()) {
+        return leftDate.isSame(rightDate)
+      }
+    }
+
+    return false
   }
 
   const not =
@@ -911,8 +931,8 @@ export function sort<T extends Record<string, any>>(
  * @param docs the data
  * @param limit the number of docs to limit to
  */
-export function limit<T>(docs: T[], limit: string): T[] {
+export function limit<T>(docs: T[], limit: string | number): T[] {
-  const numLimit = parseFloat(limit)
+  const numLimit = typeof limit === "number" ? limit : parseFloat(limit)
   if (isNaN(numLimit)) {
     return docs
   }

packages/shared-core/src/utils.ts

@@ -109,7 +109,9 @@ export function trimOtherProps(object: any, allowedProps: string[]) {
   return result
 }
 
-export function isSupportedUserSearch(query: SearchFilters) {
+export function isSupportedUserSearch(
+  query: SearchFilters
+): query is SearchFilters {
   const allowed = [
     { op: BasicOperator.STRING, key: "email" },
     { op: BasicOperator.EQUAL, key: "_id" },

packages/types/src/api/web/app/query.ts

@@ -40,6 +40,10 @@ export interface ExecuteQueryRequest {
 export type ExecuteV1QueryResponse = Record<string, any>[]
 export interface ExecuteV2QueryResponse {
   data: Record<string, any>[]
+  pagination?: {
+    page: number
+    cursor: string
+  }
 }
 
 export interface DeleteQueryResponse {

packages/types/src/api/web/global/self.ts

@@ -15,5 +15,5 @@ export interface GetGlobalSelfResponse extends User {
   license: License
   budibaseAccess: boolean
   accountPortalAccess: boolean
-  csrfToken: boolean
+  csrfToken: string
 }

packages/types/src/api/web/pagination.ts

@@ -24,4 +24,5 @@ export interface PaginationRequest extends BasicPaginationRequest {
 export interface PaginationResponse {
   bookmark: string | number | undefined
   hasNextPage?: boolean
+  totalRows?: number
 }

packages/types/src/api/web/user.ts

@@ -22,6 +22,8 @@ export interface UserDetails {
   password?: string
 }
 
+export type UnsavedUser = Omit<User, "tenantId">
+
 export interface BulkUserRequest {
   delete?: {
     users: Array<{
@@ -31,7 +33,7 @@ export interface BulkUserRequest {
   }
   create?: {
     roles?: any[]
-    users: User[]
+    users: UnsavedUser[]
     groups: any[]
   }
 }
@@ -124,7 +126,7 @@ export interface AcceptUserInviteRequest {
   inviteCode: string
   password: string
   firstName: string
-  lastName: string
+  lastName?: string
 }
 
 export interface AcceptUserInviteResponse {

packages/types/src/documents/app/query.ts

@@ -1,4 +1,5 @@
 import { Document } from "../document"
+import { Row } from "./row"
 
 export interface QuerySchema {
   name?: string
@@ -13,6 +14,7 @@ export interface Query extends Document {
   fields: RestQueryFields | any
   transformer: string | null
   schema: Record<string, QuerySchema | string>
+  nestedSchemaFields?: Record<string, Record<string, QuerySchema | string>>
   readable: boolean
   queryVerb: string
   // flag to state whether the default bindings are empty strings (old behaviour) or null
@@ -29,7 +31,7 @@ export interface QueryParameter {
 }
 
 export interface QueryResponse {
-  rows: any[]
+  rows: Row[]
   keys: string[]
   info: any
   extra: any

packages/types/src/documents/app/row.ts

@@ -154,6 +154,7 @@ export const GroupByTypes = [
   FieldType.BOOLEAN,
   FieldType.DATETIME,
   FieldType.BIGINT,
+  FieldType.AI,
 ]
 
 export function canGroupBy(type: FieldType) {

packages/types/src/documents/app/screen.ts

@@ -33,11 +33,6 @@ export interface ScreenRoutesViewOutput extends Document {
 export type ScreenRoutingJson = Record<
   string,
   {
-    subpaths: Record<
+    subpaths: Record<string, any>
-      string,
-      {
-        screens: Record<string, string>
-      }
-    >
   }
 >

packages/types/src/documents/app/table/schema.ts

@@ -123,7 +123,7 @@ export interface AIFieldMetadata extends BaseFieldSchema {
   operation: AIOperationEnum
   columns?: string[]
   column?: string
-  categories?: string[]
+  categories?: string
   prompt?: string
   language?: string
 }
@@ -227,6 +227,7 @@ interface OtherFieldMetadata extends BaseFieldSchema {
     | FieldType.OPTIONS
     | FieldType.BOOLEAN
     | FieldType.BIGINT
+    | FieldType.JSON
   >
 }
 

packages/types/src/documents/global/config.ts

@@ -26,13 +26,11 @@ export interface SMTPConfig extends Config<SMTPInnerConfig> {}
 export interface SettingsBrandingConfig {
   faviconUrl?: string
   faviconUrlEtag?: string
-
   emailBrandingEnabled?: boolean
   testimonialsEnabled?: boolean
   platformTitle?: string
   loginHeading?: string
   loginButton?: string
-
   metaDescription?: string
   metaImageUrl?: string
   metaTitle?: string
@@ -42,6 +40,7 @@ export interface SettingsInnerConfig {
   platformUrl?: string
   company?: string
   logoUrl?: string // Populated on read
+  docsUrl?: string
   logoUrlEtag?: string
   uniqueTenantId?: string
   analyticsEnabled?: boolean

packages/types/src/sdk/featureFlag.ts

@@ -1,17 +1,15 @@
 export enum FeatureFlag {
-  AUTOMATION_BRANCHING = "AUTOMATION_BRANCHING",
-  AI_CUSTOM_CONFIGS = "AI_CUSTOM_CONFIGS",
-  DEFAULT_VALUES = "DEFAULT_VALUES",
-  BUDIBASE_AI = "BUDIBASE_AI",
   USE_ZOD_VALIDATOR = "USE_ZOD_VALIDATOR",
+
+  // Account-portal
+  DIRECT_LOGIN_TO_ACCOUNT_PORTAL = "DIRECT_LOGIN_TO_ACCOUNT_PORTAL",
 }
 
 export const FeatureFlagDefaults = {
-  [FeatureFlag.DEFAULT_VALUES]: true,
-  [FeatureFlag.AUTOMATION_BRANCHING]: true,
-  [FeatureFlag.AI_CUSTOM_CONFIGS]: true,
-  [FeatureFlag.BUDIBASE_AI]: true,
   [FeatureFlag.USE_ZOD_VALIDATOR]: false,
+
+  // Account-portal
+  [FeatureFlag.DIRECT_LOGIN_TO_ACCOUNT_PORTAL]: false,
 }
 
 export type FeatureFlags = typeof FeatureFlagDefaults

packages/types/src/ui/stores/grid/columns.ts

@@ -14,6 +14,7 @@ export type UIColumn = FieldSchema & {
     type: FieldType
     readonly: boolean
     autocolumn: boolean
+    cellRenderType?: FieldType | "role"
   }
   calculationType: CalculationType
   __idx: number

packages/types/src/ui/stores/grid/datasource.ts

@@ -1,8 +1,6 @@
 import { UITable, UIView } from "@budibase/types"
 
-export type UIDatasource = (UITable | UIView) & {
+export type UIDatasource = UITable | UIView
-  type: string
-}
 
 export interface UIFieldMutation {
   visible?: boolean

packages/types/src/ui/stores/grid/fetch.ts

@@ -1,38 +0,0 @@
-import {
-  Row,
-  SortOrder,
-  UIDatasource,
-  UILegacyFilter,
-  UISearchFilter,
-} from "@budibase/types"
-
-export interface UIFetchAPI {
-  definition: UIDatasource
-
-  getInitialData: () => Promise<void>
-  loading: any
-  loaded: boolean
-
-  resetKey: string | null
-  error: any
-
-  hasNextPage: boolean
-  nextPage: () => Promise<void>
-
-  rows: Row[]
-
-  options?: {
-    datasource?: {
-      tableId: string
-      id: string
-    }
-  }
-  update: ({
-    sortOrder,
-    sortColumn,
-  }: {
-    sortOrder?: SortOrder
-    sortColumn?: string
-    filter?: UILegacyFilter[] | UISearchFilter
-  }) => any
-}

packages/types/src/ui/stores/grid/index.ts

@@ -6,4 +6,3 @@ export * from "./view"
 export * from "./user"
 export * from "./filters"
 export * from "./rows"
-export * from "./fetch"

packages/worker/src/api/controllers/global/users.ts

@@ -33,6 +33,7 @@ import {
   SaveUserResponse,
   SearchUsersRequest,
   SearchUsersResponse,
+  UnsavedUser,
   UpdateInviteRequest,
   UpdateInviteResponse,
   User,
@@ -49,6 +50,7 @@ import {
   tenancy,
   db,
   locks,
+  context,
 } from "@budibase/backend-core"
 import { checkAnyUserExists } from "../../../utilities/users"
 import { isEmailConfigured } from "../../../utilities/email"
@@ -66,10 +68,11 @@ const generatePassword = (length: number) => {
     .slice(0, length)
 }
 
-export const save = async (ctx: UserCtx<User, SaveUserResponse>) => {
+export const save = async (ctx: UserCtx<UnsavedUser, SaveUserResponse>) => {
   try {
     const currentUserId = ctx.user?._id
-    const requestUser = ctx.request.body
+    const tenantId = context.getTenantId()
+    const requestUser: User = { ...ctx.request.body, tenantId }
 
     // Do not allow the account holder role to be changed
     if (
@@ -151,7 +154,12 @@ export const bulkUpdate = async (
   let created, deleted
   try {
     if (input.create) {
-      created = await bulkCreate(input.create.users, input.create.groups)
+      const tenantId = context.getTenantId()
+      const users: User[] = input.create.users.map(user => ({
+        ...user,
+        tenantId,
+      }))
+      created = await bulkCreate(users, input.create.groups)
     }
     if (input.delete) {
       deleted = await bulkDelete(input.delete.users, currentUserId)