Merge branch 'master' of github.com:Budibase/budibase into binding-ts-improvements

packages/client/src/components/app/DataProvider.svelte

@@ -1,22 +1,39 @@
-<script>
+<script lang="ts">
   import { getContext } from "svelte"
   import { Pagination, ProgressCircle } from "@budibase/bbui"
   import { fetchData, QueryUtils } from "@budibase/frontend-core"
-  import { LogicalOperator, EmptyFilterOption } from "@budibase/types"
+  import {
+    LogicalOperator,
+    EmptyFilterOption,
+    TableSchema,
+    SortOrder,
+    SearchFilters,
+    UISearchFilter,
+    DataFetchDatasource,
+    UserDatasource,
+    GroupUserDatasource,
+    DataFetchOptions,
+  } from "@budibase/types"
+  import { SDK, Component } from "../../index"
 
-  export let dataSource
-  export let filter
-  export let sortColumn
-  export let sortOrder
-  export let limit
-  export let paginate
-  export let autoRefresh
+  type ProviderDatasource = Exclude<
+    DataFetchDatasource,
+    UserDatasource | GroupUserDatasource
+  >
 
-  const { styleable, Provider, ActionTypes, API } = getContext("sdk")
-  const component = getContext("component")
+  export let dataSource: ProviderDatasource
+  export let filter: UISearchFilter
+  export let sortColumn: string
+  export let sortOrder: SortOrder
+  export let limit: number
+  export let paginate: boolean
+  export let autoRefresh: number
 
-  let interval
-  let queryExtensions = {}
+  const { styleable, Provider, ActionTypes, API } = getContext<SDK>("sdk")
+  const component = getContext<Component>("component")
+
+  let interval: ReturnType<typeof setInterval>
+  let queryExtensions: Record<string, any> = {}
 
   $: defaultQuery = QueryUtils.buildQuery(filter)
 
@@ -49,8 +66,14 @@
     },
     {
       type: ActionTypes.SetDataProviderSorting,
-      callback: ({ column, order }) => {
-        let newOptions = {}
+      callback: ({
+        column,
+        order,
+      }: {
+        column: string
+        order: SortOrder | undefined
+      }) => {
+        let newOptions: Partial<DataFetchOptions> = {}
         if (column) {
           newOptions.sortColumn = column
         }
@@ -63,6 +86,7 @@
       },
     },
   ]
+
   $: dataContext = {
     rows: $fetch.rows,
     info: $fetch.info,
@@ -75,14 +99,12 @@
     id: $component?.id,
     state: {
       query: $fetch.query,
-      sortColumn: $fetch.sortColumn,
-      sortOrder: $fetch.sortOrder,
     },
     limit,
-    primaryDisplay: $fetch.definition?.primaryDisplay,
+    primaryDisplay: ($fetch.definition as any)?.primaryDisplay,
   }
 
-  const createFetch = datasource => {
+  const createFetch = (datasource: ProviderDatasource) => {
     return fetchData({
       API,
       datasource,
@@ -96,7 +118,7 @@
     })
   }
 
-  const sanitizeSchema = schema => {
+  const sanitizeSchema = (schema: TableSchema | null) => {
     if (!schema) {
       return schema
     }
@@ -109,14 +131,14 @@
     return cloned
   }
 
-  const addQueryExtension = (key, extension) => {
+  const addQueryExtension = (key: string, extension: any) => {
     if (!key || !extension) {
       return
     }
     queryExtensions = { ...queryExtensions, [key]: extension }
   }
 
-  const removeQueryExtension = key => {
+  const removeQueryExtension = (key: string) => {
     if (!key) {
       return
     }
@@ -125,11 +147,14 @@
     queryExtensions = newQueryExtensions
   }
 
-  const extendQuery = (defaultQuery, extensions) => {
+  const extendQuery = (
+    defaultQuery: SearchFilters,
+    extensions: Record<string, any>
+  ): SearchFilters => {
     if (!Object.keys(extensions).length) {
       return defaultQuery
     }
-    const extended = {
+    const extended: SearchFilters = {
       [LogicalOperator.AND]: {
         conditions: [
           ...(defaultQuery ? [defaultQuery] : []),
@@ -140,12 +165,12 @@
     }
 
     // If there are no conditions applied at all, clear the request.
-    return extended[LogicalOperator.AND]?.conditions?.length > 0
+    return (extended[LogicalOperator.AND]?.conditions?.length ?? 0) > 0
       ? extended
-      : null
+      : {}
   }
 
-  const setUpAutoRefresh = autoRefresh => {
+  const setUpAutoRefresh = (autoRefresh: number) => {
     clearInterval(interval)
     if (autoRefresh) {
       interval = setInterval(fetch.refresh, Math.max(10000, autoRefresh * 1000))

packages/client/src/index.ts

@@ -0,0 +1,15 @@
+import { APIClient } from "@budibase/frontend-core"
+import type { ActionTypes } from "./constants"
+import { Readable } from "svelte/store"
+
+export interface SDK {
+  API: APIClient
+  styleable: any
+  Provider: any
+  ActionTypes: typeof ActionTypes
+}
+
+export type Component = Readable<{
+  id: string
+  styles: any
+}>

packages/client/src/stores/components.js

@@ -6,7 +6,7 @@ import { screenStore } from "./screens"
 import { builderStore } from "./builder"
 import Router from "../components/Router.svelte"
 import * as AppComponents from "../components/app/index.js"
-import { ScreenslotType } from "../constants.js"
+import { ScreenslotType } from "../constants"
 
 export const BudibasePrefix = "@budibase/standard-components/"
 

packages/frontend-core/src/components/grid/stores/datasource.ts

@@ -1,7 +1,11 @@
 // TODO: datasource and defitions are unions of the different implementations. At this point, the datasource does not know what type is being used, and the assignations will cause TS exceptions. Casting it "as any" for now. This should be fixed improving the type usages.
 
 import { derived, get, Readable, Writable } from "svelte/store"
-import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch"
+import {
+  DataFetchDefinition,
+  getDatasourceDefinition,
+  getDatasourceSchema,
+} from "../../../fetch"
 import { enrichSchemaWithRelColumns, memo } from "../../../utils"
 import { cloneDeep } from "lodash"
 import {
@@ -18,7 +22,7 @@ import { Store as StoreContext, BaseStoreProps } from "."
 import { DatasourceActions } from "./datasources"
 
 interface DatasourceStore {
-  definition: Writable<UIDatasource | null>
+  definition: Writable<DataFetchDefinition | null>
   schemaMutations: Writable<Record<string, UIFieldMutation>>
   subSchemaMutations: Writable<Record<string, Record<string, UIFieldMutation>>>
 }
@@ -131,11 +135,17 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => {
     [datasource, definition],
     ([$datasource, $definition]) => {
       let type = $datasource?.type
+      // @ts-expect-error
       if (type === "provider") {
         type = ($datasource as any).value?.datasource?.type // TODO: see line 1
       }
       // Handle calculation views
-      if (type === "viewV2" && $definition?.type === ViewV2Type.CALCULATION) {
+      if (
+        type === "viewV2" &&
+        $definition &&
+        "type" in $definition &&
+        $definition.type === ViewV2Type.CALCULATION
+      ) {
         return false
       }
       return !!type && ["table", "viewV2", "link"].includes(type)
@@ -197,7 +207,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => {
   ) => {
     // Update local state
     const originalDefinition = get(definition)
-    definition.set(newDefinition as UIDatasource)
+    definition.set(newDefinition)
 
     // Update server
     if (get(config).canSaveSchema) {
@@ -225,13 +235,15 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => {
     // Update primary display
     newDefinition.primaryDisplay = column
 
-    // Sanitise schema to ensure field is required and has no default value
-    if (!newDefinition.schema[column].constraints) {
-      newDefinition.schema[column].constraints = {}
-    }
-    newDefinition.schema[column].constraints.presence = { allowEmpty: false }
-    if ("default" in newDefinition.schema[column]) {
-      delete newDefinition.schema[column].default
+    if (newDefinition.schema) {
+      // Sanitise schema to ensure field is required and has no default value
+      if (!newDefinition.schema[column].constraints) {
+        newDefinition.schema[column].constraints = {}
+      }
+      newDefinition.schema[column].constraints.presence = { allowEmpty: false }
+      if ("default" in newDefinition.schema[column]) {
+        delete newDefinition.schema[column].default
+      }
     }
     return await saveDefinition(newDefinition as any) // TODO: see line 1
   }

packages/frontend-core/src/components/grid/stores/datasources/table.ts

@@ -8,6 +8,7 @@ import {
 import { get } from "svelte/store"
 import { Store as StoreContext } from ".."
 import { DatasourceTableActions } from "."
+import TableFetch from "../../../../fetch/TableFetch"
 
 const SuppressErrors = true
 
@@ -119,7 +120,7 @@ export const initialise = (context: StoreContext) => {
     unsubscribers.push(
       allFilters.subscribe($allFilters => {
         // Ensure we're updating the correct fetch
-        const $fetch = get(fetch)
+        const $fetch = get(fetch) as TableFetch | null
         if ($fetch?.options?.datasource?.tableId !== $datasource.tableId) {
           return
         }
@@ -133,7 +134,7 @@ export const initialise = (context: StoreContext) => {
     unsubscribers.push(
       sort.subscribe($sort => {
         // Ensure we're updating the correct fetch
-        const $fetch = get(fetch)
+        const $fetch = get(fetch) as TableFetch | null
         if ($fetch?.options?.datasource?.tableId !== $datasource.tableId) {
           return
         }

packages/frontend-core/src/components/grid/stores/datasources/viewV2.ts

@@ -4,11 +4,11 @@ import {
   SaveRowRequest,
   SortOrder,
   UIDatasource,
-  UIView,
   UpdateViewRequest,
 } from "@budibase/types"
 import { Store as StoreContext } from ".."
 import { DatasourceViewActions } from "."
+import ViewV2Fetch from "../../../../fetch/ViewV2Fetch"
 
 const SuppressErrors = true
 
@@ -134,6 +134,9 @@ export const initialise = (context: StoreContext) => {
         if (!get(config).canSaveSchema) {
           return
         }
+        if (!$definition || !("id" in $definition)) {
+          return
+        }
         if ($definition?.id !== $datasource.id) {
           return
         }
@@ -184,7 +187,10 @@ export const initialise = (context: StoreContext) => {
     unsubscribers.push(
       sort.subscribe(async $sort => {
         // Ensure we're updating the correct view
-        const $view = get(definition) as UIView
+        const $view = get(definition)
+        if (!$view || !("id" in $view)) {
+          return
+        }
         if ($view?.id !== $datasource.id) {
           return
         }
@@ -207,7 +213,7 @@ export const initialise = (context: StoreContext) => {
 
         // Also update the fetch to ensure the new sort is respected.
         // Ensure we're updating the correct fetch.
-        const $fetch = get(fetch)
+        const $fetch = get(fetch) as ViewV2Fetch | null
         if ($fetch?.options?.datasource?.id !== $datasource.id) {
           return
         }
@@ -225,6 +231,9 @@ export const initialise = (context: StoreContext) => {
           return
         }
         const $view = get(definition)
+        if (!$view || !("id" in $view)) {
+          return
+        }
         if ($view?.id !== $datasource.id) {
           return
         }
@@ -246,7 +255,7 @@ export const initialise = (context: StoreContext) => {
         if (!get(config).canSaveSchema) {
           return
         }
-        const $fetch = get(fetch)
+        const $fetch = get(fetch) as ViewV2Fetch | null
         if ($fetch?.options?.datasource?.id !== $datasource.id) {
           return
         }
@@ -262,7 +271,7 @@ export const initialise = (context: StoreContext) => {
         if (get(config).canSaveSchema) {
           return
         }
-        const $fetch = get(fetch)
+        const $fetch = get(fetch) as ViewV2Fetch | null
         if ($fetch?.options?.datasource?.id !== $datasource.id) {
           return
         }

packages/frontend-core/src/components/grid/stores/rows.ts

@@ -1,5 +1,5 @@
 import { writable, derived, get, Writable, Readable } from "svelte/store"
-import { fetchData } from "../../../fetch"
+import { DataFetch, fetchData } from "../../../fetch"
 import { NewRowID, RowPageSize } from "../lib/constants"
 import {
   generateRowID,
@@ -13,7 +13,6 @@ import { sleep } from "../../../utils/utils"
 import { FieldType, Row, UIRow } from "@budibase/types"
 import { getRelatedTableValues } from "../../../utils"
 import { Store as StoreContext } from "."
-import DataFetch from "../../../fetch/DataFetch"
 
 interface IndexedUIRow extends UIRow {
   __idx: number
@@ -21,7 +20,7 @@ interface IndexedUIRow extends UIRow {
 
 interface RowStore {
   rows: Writable<UIRow[]>
-  fetch: Writable<DataFetch<any, any, any> | null> // TODO: type this properly, having a union of all the possible options
+  fetch: Writable<DataFetch | null>
   loaded: Writable<boolean>
   refreshing: Writable<boolean>
   loading: Writable<boolean>
@@ -254,7 +253,7 @@ export const createActions = (context: StoreContext): RowActionStore => {
 
         // Reset state properties when dataset changes
         if (!$instanceLoaded || resetRows) {
-          definition.set($fetch.definition as any) // TODO: datasource and defitions are unions of the different implementations. At this point, the datasource does not know what type is being used, and the assignations will cause TS exceptions. Casting it "as any" for now. This should be fixed improving the type usages.
+          definition.set($fetch.definition ?? null)
         }
 
         // Reset scroll state when data changes

packages/frontend-core/src/fetch/CustomFetch.ts

@@ -1,13 +1,9 @@
-import DataFetch from "./DataFetch"
-
-interface CustomDatasource {
-  type: "custom"
-  data: any
-}
+import { CustomDatasource } from "@budibase/types"
+import BaseDataFetch from "./DataFetch"
 
 type CustomDefinition = Record<string, any>
 
-export default class CustomFetch extends DataFetch<
+export default class CustomFetch extends BaseDataFetch<
   CustomDatasource,
   CustomDefinition
 > {

packages/frontend-core/src/fetch/DataFetch.ts

@@ -3,14 +3,13 @@ import { cloneDeep } from "lodash/fp"
 import { QueryUtils } from "../utils"
 import { convertJSONSchemaToTableSchema } from "../utils/json"
 import {
+  DataFetchOptions,
   FieldType,
-  LegacyFilter,
   Row,
   SearchFilters,
   SortOrder,
   SortType,
   TableSchema,
-  UISearchFilter,
 } from "@budibase/types"
 import { APIClient } from "../api/types"
 import { DataFetchType } from "."
@@ -44,14 +43,11 @@ interface DataFetchDerivedStore<TDefinition, TQuery>
   supportsPagination: boolean
 }
 
-export interface DataFetchParams<
-  TDatasource,
-  TQuery = SearchFilters | undefined
-> {
+export interface DataFetchParams<TDatasource, TQuery = SearchFilters> {
   API: APIClient
   datasource: TDatasource
   query: TQuery
-  options?: {}
+  options?: Partial<DataFetchOptions<TQuery>>
 }
 
 /**
@@ -59,7 +55,7 @@ export interface DataFetchParams<
  * internal table or datasource plus.
  * For other types of datasource, this class is overridden and extended.
  */
-export default abstract class DataFetch<
+export default abstract class BaseDataFetch<
   TDatasource extends { type: DataFetchType },
   TDefinition extends {
     schema?: Record<string, any> | null
@@ -73,18 +69,11 @@ export default abstract class DataFetch<
     supportsSort: boolean
     supportsPagination: boolean
   }
-  options: {
+  options: DataFetchOptions<TQuery> & {
     datasource: TDatasource
-    limit: number
-    // Search config
-    filter: UISearchFilter | LegacyFilter[] | null
-    query: TQuery
-    // Sorting config
-    sortColumn: string | null
-    sortOrder: SortOrder
+
     sortType: SortType | null
-    // Pagination config
-    paginate: boolean
+
     // Client side feature customisation
     clientSideSearching: boolean
     clientSideSorting: boolean
@@ -267,6 +256,7 @@ export default abstract class DataFetch<
 
     // Build the query
     let query = this.options.query
+
     if (!query) {
       query = buildQuery(filter ?? undefined) as TQuery
     }
@@ -430,7 +420,7 @@ export default abstract class DataFetch<
    * Resets the data set and updates options
    * @param newOptions any new options
    */
-  async update(newOptions: any) {
+  async update(newOptions: Partial<DataFetchOptions<TQuery>>) {
     // Check if any settings have actually changed
     let refresh = false
     for (const [key, value] of Object.entries(newOptions || {})) {

packages/frontend-core/src/fetch/FieldFetch.ts

@@ -1,14 +1,10 @@
-import { Row } from "@budibase/types"
-import DataFetch from "./DataFetch"
-
-type Types = "field" | "queryarray" | "jsonarray"
-
-export interface FieldDatasource<TType extends Types> {
-  type: TType
-  tableId: string
-  fieldType: "attachment" | "array"
-  value: string[] | Row[]
-}
+import {
+  FieldDatasource,
+  JSONArrayFieldDatasource,
+  QueryArrayFieldDatasource,
+  Row,
+} from "@budibase/types"
+import BaseDataFetch from "./DataFetch"
 
 export interface FieldDefinition {
   schema?: Record<string, { type: string }> | null
@@ -18,10 +14,12 @@ function isArrayOfStrings(value: string[] | Row[]): value is string[] {
   return Array.isArray(value) && !!value[0] && typeof value[0] !== "object"
 }
 
-export default class FieldFetch<TType extends Types> extends DataFetch<
-  FieldDatasource<TType>,
-  FieldDefinition
-> {
+export default class FieldFetch<
+  TDatasource extends
+    | FieldDatasource
+    | QueryArrayFieldDatasource
+    | JSONArrayFieldDatasource = FieldDatasource
+> extends BaseDataFetch<TDatasource, FieldDefinition> {
   async getDefinition(): Promise<FieldDefinition | null> {
     const { datasource } = this.options
 

packages/frontend-core/src/fetch/GroupUserFetch.ts

@@ -1,20 +1,20 @@
 import { get } from "svelte/store"
-import DataFetch, { DataFetchParams } from "./DataFetch"
-import { TableNames } from "../constants"
+import BaseDataFetch, { DataFetchParams } from "./DataFetch"
+import { GroupUserDatasource, InternalTable } from "@budibase/types"
 
 interface GroupUserQuery {
   groupId: string
   emailSearch: string
 }
 
-interface GroupUserDatasource {
-  type: "groupUser"
-  tableId: TableNames.USERS
+interface GroupUserDefinition {
+  schema?: Record<string, any> | null
+  primaryDisplay?: string
 }
 
-export default class GroupUserFetch extends DataFetch<
+export default class GroupUserFetch extends BaseDataFetch<
   GroupUserDatasource,
-  {},
+  GroupUserDefinition,
   GroupUserQuery
 > {
   constructor(opts: DataFetchParams<GroupUserDatasource, GroupUserQuery>) {
@@ -22,7 +22,7 @@ export default class GroupUserFetch extends DataFetch<
       ...opts,
       datasource: {
         type: "groupUser",
-        tableId: TableNames.USERS,
+        tableId: InternalTable.USER_METADATA,
       },
     })
   }

packages/frontend-core/src/fetch/JSONArrayFetch.ts

@@ -1,7 +1,8 @@
 import FieldFetch from "./FieldFetch"
 import { getJSONArrayDatasourceSchema } from "../utils/json"
+import { JSONArrayFieldDatasource } from "@budibase/types"
 
-export default class JSONArrayFetch extends FieldFetch<"jsonarray"> {
+export default class JSONArrayFetch extends FieldFetch<JSONArrayFieldDatasource> {
   async getDefinition() {
     const { datasource } = this.options
 

packages/frontend-core/src/fetch/NestedProviderFetch.ts

@@ -1,20 +1,11 @@
-import { Row, TableSchema } from "@budibase/types"
-import DataFetch from "./DataFetch"
-
-interface NestedProviderDatasource {
-  type: "provider"
-  value?: {
-    schema: TableSchema
-    primaryDisplay: string
-    rows: Row[]
-  }
-}
+import { NestedProviderDatasource, TableSchema } from "@budibase/types"
+import BaseDataFetch from "./DataFetch"
 
 interface NestedProviderDefinition {
   schema?: TableSchema
   primaryDisplay?: string
 }
-export default class NestedProviderFetch extends DataFetch<
+export default class NestedProviderFetch extends BaseDataFetch<
   NestedProviderDatasource,
   NestedProviderDefinition
 > {

packages/frontend-core/src/fetch/QueryArrayFetch.ts

@@ -3,8 +3,9 @@ import {
   getJSONArrayDatasourceSchema,
   generateQueryArraySchemas,
 } from "../utils/json"
+import { QueryArrayFieldDatasource } from "@budibase/types"
 
-export default class QueryArrayFetch extends FieldFetch<"queryarray"> {
+export default class QueryArrayFetch extends FieldFetch<QueryArrayFieldDatasource> {
   async getDefinition() {
     const { datasource } = this.options
 

packages/frontend-core/src/fetch/QueryFetch.ts

@@ -1,23 +1,9 @@
-import DataFetch from "./DataFetch"
+import BaseDataFetch from "./DataFetch"
 import { Helpers } from "@budibase/bbui"
-import { ExecuteQueryRequest, Query } from "@budibase/types"
+import { ExecuteQueryRequest, Query, QueryDatasource } from "@budibase/types"
 import { get } from "svelte/store"
 
-interface QueryDatasource {
-  type: "query"
-  _id: string
-  fields: Record<string, any> & {
-    pagination?: {
-      type: string
-      location: string
-      pageParam: string
-    }
-  }
-  queryParams?: Record<string, string>
-  parameters: { name: string; default: string }[]
-}
-
-export default class QueryFetch extends DataFetch<QueryDatasource, Query> {
+export default class QueryFetch extends BaseDataFetch<QueryDatasource, Query> {
   async determineFeatureFlags() {
     const definition = await this.getDefinition()
     const supportsPagination =

packages/frontend-core/src/fetch/RelationshipFetch.ts

@@ -1,15 +1,7 @@
-import { Table } from "@budibase/types"
-import DataFetch from "./DataFetch"
+import { RelationshipDatasource, Table } from "@budibase/types"
+import BaseDataFetch from "./DataFetch"
 
-interface RelationshipDatasource {
-  type: "link"
-  tableId: string
-  rowId: string
-  rowTableId: string
-  fieldName: string
-}
-
-export default class RelationshipFetch extends DataFetch<
+export default class RelationshipFetch extends BaseDataFetch<
   RelationshipDatasource,
   Table
 > {

packages/frontend-core/src/fetch/TableFetch.ts

@@ -1,13 +1,8 @@
 import { get } from "svelte/store"
-import DataFetch from "./DataFetch"
-import { SortOrder, Table } from "@budibase/types"
+import BaseDataFetch from "./DataFetch"
+import { SortOrder, Table, TableDatasource } from "@budibase/types"
 
-interface TableDatasource {
-  type: "table"
-  tableId: string
-}
-
-export default class TableFetch extends DataFetch<TableDatasource, Table> {
+export default class TableFetch extends BaseDataFetch<TableDatasource, Table> {
   async determineFeatureFlags() {
     return {
       supportsSearch: true,

packages/frontend-core/src/fetch/UserFetch.ts

@@ -1,22 +1,24 @@
 import { get } from "svelte/store"
-import DataFetch, { DataFetchParams } from "./DataFetch"
-import { TableNames } from "../constants"
+import BaseDataFetch, { DataFetchParams } from "./DataFetch"
 import { utils } from "@budibase/shared-core"
-import { SearchFilters, SearchUsersRequest } from "@budibase/types"
+import {
+  InternalTable,
+  SearchFilters,
+  SearchUsersRequest,
+  UserDatasource,
+} from "@budibase/types"
 
 interface UserFetchQuery {
   appId: string
   paginated: boolean
 }
 
-interface UserDatasource {
-  type: "user"
-  tableId: TableNames.USERS
+interface UserDefinition {
+  schema?: Record<string, any> | null
+  primaryDisplay?: string
 }
 
-interface UserDefinition {}
-
-export default class UserFetch extends DataFetch<
+export default class UserFetch extends BaseDataFetch<
   UserDatasource,
   UserDefinition,
   UserFetchQuery
@@ -26,7 +28,7 @@ export default class UserFetch extends DataFetch<
       ...opts,
       datasource: {
         type: "user",
-        tableId: TableNames.USERS,
+        tableId: InternalTable.USER_METADATA,
       },
     })
   }

packages/frontend-core/src/fetch/ViewFetch.ts

@@ -1,16 +1,7 @@
-import { Table } from "@budibase/types"
-import DataFetch from "./DataFetch"
+import { Table, ViewV1Datasource } from "@budibase/types"
+import BaseDataFetch from "./DataFetch"
 
-type ViewV1Datasource = {
-  type: "view"
-  name: string
-  tableId: string
-  calculation: string
-  field: string
-  groupBy: string
-}
-
-export default class ViewFetch extends DataFetch<ViewV1Datasource, Table> {
+export default class ViewFetch extends BaseDataFetch<ViewV1Datasource, Table> {
   async getDefinition() {
     const { datasource } = this.options
 

packages/frontend-core/src/fetch/ViewV2Fetch.ts

@@ -1,14 +1,14 @@
-import { SortOrder, ViewV2Enriched, ViewV2Type } from "@budibase/types"
-import DataFetch from "./DataFetch"
+import {
+  SortOrder,
+  ViewDatasource,
+  ViewV2Enriched,
+  ViewV2Type,
+} from "@budibase/types"
+import BaseDataFetch from "./DataFetch"
 import { get } from "svelte/store"
 import { helpers } from "@budibase/shared-core"
 
-interface ViewDatasource {
-  type: "viewV2"
-  id: string
-}
-
-export default class ViewV2Fetch extends DataFetch<
+export default class ViewV2Fetch extends BaseDataFetch<
   ViewDatasource,
   ViewV2Enriched
 > {

packages/frontend-core/src/fetch/index.ts

@@ -11,6 +11,7 @@ import GroupUserFetch from "./GroupUserFetch"
 import CustomFetch from "./CustomFetch"
 import QueryArrayFetch from "./QueryArrayFetch"
 import { APIClient } from "../api/types"
+import { DataFetchDatasource, Table, ViewV2Enriched } from "@budibase/types"
 
 export type DataFetchType = keyof typeof DataFetchMap
 
@@ -26,32 +27,88 @@ export const DataFetchMap = {
 
   // Client specific datasource types
   provider: NestedProviderFetch,
-  field: FieldFetch<"field">,
+  field: FieldFetch,
   jsonarray: JSONArrayFetch,
   queryarray: QueryArrayFetch,
 }
 
+export interface DataFetchClassMap {
+  table: TableFetch
+  view: ViewFetch
+  viewV2: ViewV2Fetch
+  query: QueryFetch
+  link: RelationshipFetch
+  user: UserFetch
+  groupUser: GroupUserFetch
+  custom: CustomFetch
+
+  // Client specific datasource types
+  provider: NestedProviderFetch
+  field: FieldFetch
+  jsonarray: JSONArrayFetch
+  queryarray: QueryArrayFetch
+}
+
+export type DataFetch =
+  | TableFetch
+  | ViewFetch
+  | ViewV2Fetch
+  | QueryFetch
+  | RelationshipFetch
+  | UserFetch
+  | GroupUserFetch
+  | CustomFetch
+  | NestedProviderFetch
+  | FieldFetch
+  | JSONArrayFetch
+  | QueryArrayFetch
+
+export type DataFetchDefinition =
+  | Table
+  | ViewV2Enriched
+  | {
+      // These fields are added to allow checking these fields on definition usages without requiring constant castings
+      schema?: Record<string, any> | null
+      primaryDisplay?: string
+      rowHeight?: number
+      type?: string
+      name?: string
+    }
+
 // Constructs a new fetch model for a certain datasource
-export const fetchData = ({ API, datasource, options }: any) => {
-  const Fetch = DataFetchMap[datasource?.type as DataFetchType] || TableFetch
+export const fetchData = <
+  T extends DataFetchDatasource,
+  Type extends T["type"] = T["type"]
+>({
+  API,
+  datasource,
+  options,
+}: {
+  API: APIClient
+  datasource: T
+  options: any
+}): Type extends keyof DataFetchClassMap
+  ? DataFetchClassMap[Type]
+  : TableFetch => {
+  const Fetch = DataFetchMap[datasource?.type] || TableFetch
   const fetch = new Fetch({ API, datasource, ...options })
 
   // Initially fetch data but don't bother waiting for the result
   fetch.getInitialData()
 
-  return fetch
+  return fetch as any
 }
 
 // Creates an empty fetch instance with no datasource configured, so no data
 // will initially be loaded
-const createEmptyFetchInstance = <TDatasource extends { type: DataFetchType }>({
+const createEmptyFetchInstance = ({
   API,
   datasource,
 }: {
   API: APIClient
-  datasource: TDatasource
+  datasource: DataFetchDatasource
 }) => {
-  const handler = DataFetchMap[datasource?.type as DataFetchType]
+  const handler = DataFetchMap[datasource?.type]
   if (!handler) {
     return null
   }
@@ -63,29 +120,25 @@ const createEmptyFetchInstance = <TDatasource extends { type: DataFetchType }>({
 }
 
 // Fetches the definition of any type of datasource
-export const getDatasourceDefinition = async <
-  TDatasource extends { type: DataFetchType }
->({
+export const getDatasourceDefinition = async ({
   API,
   datasource,
 }: {
   API: APIClient
-  datasource: TDatasource
+  datasource: DataFetchDatasource
 }) => {
   const instance = createEmptyFetchInstance({ API, datasource })
   return await instance?.getDefinition()
 }
 
 // Fetches the schema of any type of datasource
-export const getDatasourceSchema = <
-  TDatasource extends { type: DataFetchType }
->({
+export const getDatasourceSchema = ({
   API,
   datasource,
   definition,
 }: {
   API: APIClient
-  datasource: TDatasource
+  datasource: DataFetchDatasource
   definition?: any
 }) => {
   const instance = createEmptyFetchInstance({ API, datasource })

packages/frontend-core/src/index.ts

@@ -1,7 +1,6 @@
 export { createAPIClient } from "./api"
 export type { APIClient } from "./api"
 export { fetchData, DataFetchMap } from "./fetch"
-export type { DataFetchType } from "./fetch"
 export * as Constants from "./constants"
 export * from "./stores"
 export * from "./utils"

packages/server/src/api/controllers/query/import/sources/tests/curl/curl.spec.ts

@@ -1,34 +1,24 @@
-const { Curl } = require("../../curl")
-const fs = require("fs")
-const path = require("path")
+import { Curl } from "../../curl"
+import { readFileSync } from "fs"
+import { join } from "path"
 
-const getData = file => {
-  return fs.readFileSync(path.join(__dirname, `./data/${file}.txt`), "utf8")
+const getData = (file: string) => {
+  return readFileSync(join(__dirname, `./data/${file}.txt`), "utf8")
 }
 
 describe("Curl Import", () => {
-  let curl
+  let curl: Curl
 
   beforeEach(() => {
     curl = new Curl()
   })
 
   it("validates unsupported data", async () => {
-    let data
-    let supported
-
-    // JSON
-    data = "{}"
-    supported = await curl.isSupported(data)
-    expect(supported).toBe(false)
-
-    // Empty
-    data = ""
-    supported = await curl.isSupported(data)
-    expect(supported).toBe(false)
+    expect(await curl.isSupported("{}")).toBe(false)
+    expect(await curl.isSupported("")).toBe(false)
   })
 
-  const init = async file => {
+  const init = async (file: string) => {
     await curl.isSupported(getData(file))
   }
 
@@ -39,14 +29,14 @@ describe("Curl Import", () => {
   })
 
   describe("Returns queries", () => {
-    const getQueries = async file => {
+    const getQueries = async (file: string) => {
       await init(file)
-      const queries = await curl.getQueries()
+      const queries = await curl.getQueries("fake_datasource_id")
       expect(queries.length).toBe(1)
       return queries
     }
 
-    const testVerb = async (file, verb) => {
+    const testVerb = async (file: string, verb: string) => {
       const queries = await getQueries(file)
       expect(queries[0].queryVerb).toBe(verb)
     }
@@ -59,7 +49,7 @@ describe("Curl Import", () => {
       await testVerb("patch", "patch")
     })
 
-    const testPath = async (file, urlPath) => {
+    const testPath = async (file: string, urlPath: string) => {
       const queries = await getQueries(file)
       expect(queries[0].fields.path).toBe(urlPath)
     }
@@ -69,7 +59,10 @@ describe("Curl Import", () => {
       await testPath("path", "http://example.com/paths/abc")
     })
 
-    const testHeaders = async (file, headers) => {
+    const testHeaders = async (
+      file: string,
+      headers: Record<string, string>
+    ) => {
       const queries = await getQueries(file)
       expect(queries[0].fields.headers).toStrictEqual(headers)
     }
@@ -82,7 +75,7 @@ describe("Curl Import", () => {
       })
     })
 
-    const testQuery = async (file, queryString) => {
+    const testQuery = async (file: string, queryString: string) => {
       const queries = await getQueries(file)
       expect(queries[0].fields.queryString).toBe(queryString)
     }
@@ -91,7 +84,7 @@ describe("Curl Import", () => {
       await testQuery("query", "q1=v1&q1=v2")
     })
 
-    const testBody = async (file, body) => {
+    const testBody = async (file: string, body?: Record<string, any>) => {
       const queries = await getQueries(file)
       expect(queries[0].fields.requestBody).toStrictEqual(
         JSON.stringify(body, null, 2)

packages/server/src/api/controllers/query/import/sources/tests/openapi2/openapi2.spec.js

@@ -1,243 +0,0 @@
-const { OpenAPI2 } = require("../../openapi2")
-const fs = require("fs")
-const path = require("path")
-
-const getData = (file, extension) => {
-  return fs.readFileSync(
-    path.join(__dirname, `./data/${file}/${file}.${extension}`),
-    "utf8"
-  )
-}
-
-describe("OpenAPI2 Import", () => {
-  let openapi2
-
-  beforeEach(() => {
-    openapi2 = new OpenAPI2()
-  })
-
-  it("validates unsupported data", async () => {
-    let data
-    let supported
-
-    // non json / yaml
-    data = "curl http://example.com"
-    supported = await openapi2.isSupported(data)
-    expect(supported).toBe(false)
-
-    // Empty
-    data = ""
-    supported = await openapi2.isSupported(data)
-    expect(supported).toBe(false)
-  })
-
-  const init = async (file, extension) => {
-    await openapi2.isSupported(getData(file, extension))
-  }
-
-  const runTests = async (filename, test, assertions) => {
-    for (let extension of ["json", "yaml"]) {
-      await test(filename, extension, assertions)
-    }
-  }
-
-  const testImportInfo = async (file, extension) => {
-    await init(file, extension)
-    const info = await openapi2.getInfo()
-    expect(info.name).toBe("Swagger Petstore")
-  }
-
-  it("returns import info", async () => {
-    await runTests("petstore", testImportInfo)
-  })
-
-  describe("Returns queries", () => {
-    const indexQueries = queries => {
-      return queries.reduce((acc, query) => {
-        acc[query.name] = query
-        return acc
-      }, {})
-    }
-
-    const getQueries = async (file, extension) => {
-      await init(file, extension)
-      const queries = await openapi2.getQueries()
-      expect(queries.length).toBe(6)
-      return indexQueries(queries)
-    }
-
-    const testVerb = async (file, extension, assertions) => {
-      const queries = await getQueries(file, extension)
-      for (let [operationId, method] of Object.entries(assertions)) {
-        expect(queries[operationId].queryVerb).toBe(method)
-      }
-    }
-
-    it("populates verb", async () => {
-      const assertions = {
-        createEntity: "create",
-        getEntities: "read",
-        getEntity: "read",
-        updateEntity: "update",
-        patchEntity: "patch",
-        deleteEntity: "delete",
-      }
-      await runTests("crud", testVerb, assertions)
-    })
-
-    const testPath = async (file, extension, assertions) => {
-      const queries = await getQueries(file, extension)
-      for (let [operationId, urlPath] of Object.entries(assertions)) {
-        expect(queries[operationId].fields.path).toBe(urlPath)
-      }
-    }
-
-    it("populates path", async () => {
-      const assertions = {
-        createEntity: "http://example.com/entities",
-        getEntities: "http://example.com/entities",
-        getEntity: "http://example.com/entities/{{entityId}}",
-        updateEntity: "http://example.com/entities/{{entityId}}",
-        patchEntity: "http://example.com/entities/{{entityId}}",
-        deleteEntity: "http://example.com/entities/{{entityId}}",
-      }
-      await runTests("crud", testPath, assertions)
-    })
-
-    const testHeaders = async (file, extension, assertions) => {
-      const queries = await getQueries(file, extension)
-      for (let [operationId, headers] of Object.entries(assertions)) {
-        expect(queries[operationId].fields.headers).toStrictEqual(headers)
-      }
-    }
-
-    const contentTypeHeader = {
-      "Content-Type": "application/json",
-    }
-
-    it("populates headers", async () => {
-      const assertions = {
-        createEntity: {
-          ...contentTypeHeader,
-        },
-        getEntities: {},
-        getEntity: {},
-        updateEntity: {
-          ...contentTypeHeader,
-        },
-        patchEntity: {
-          ...contentTypeHeader,
-        },
-        deleteEntity: {
-          "x-api-key": "{{x-api-key}}",
-        },
-      }
-
-      await runTests("crud", testHeaders, assertions)
-    })
-
-    const testQuery = async (file, extension, assertions) => {
-      const queries = await getQueries(file, extension)
-      for (let [operationId, queryString] of Object.entries(assertions)) {
-        expect(queries[operationId].fields.queryString).toStrictEqual(
-          queryString
-        )
-      }
-    }
-
-    it("populates query", async () => {
-      const assertions = {
-        createEntity: "",
-        getEntities: "page={{page}}&size={{size}}",
-        getEntity: "",
-        updateEntity: "",
-        patchEntity: "",
-        deleteEntity: "",
-      }
-      await runTests("crud", testQuery, assertions)
-    })
-
-    const testParameters = async (file, extension, assertions) => {
-      const queries = await getQueries(file, extension)
-      for (let [operationId, parameters] of Object.entries(assertions)) {
-        expect(queries[operationId].parameters).toStrictEqual(parameters)
-      }
-    }
-
-    it("populates parameters", async () => {
-      const assertions = {
-        createEntity: [],
-        getEntities: [
-          {
-            name: "page",
-            default: "",
-          },
-          {
-            name: "size",
-            default: "",
-          },
-        ],
-        getEntity: [
-          {
-            name: "entityId",
-            default: "",
-          },
-        ],
-        updateEntity: [
-          {
-            name: "entityId",
-            default: "",
-          },
-        ],
-        patchEntity: [
-          {
-            name: "entityId",
-            default: "",
-          },
-        ],
-        deleteEntity: [
-          {
-            name: "entityId",
-            default: "",
-          },
-          {
-            name: "x-api-key",
-            default: "",
-          },
-        ],
-      }
-      await runTests("crud", testParameters, assertions)
-    })
-
-    const testBody = async (file, extension, assertions) => {
-      const queries = await getQueries(file, extension)
-      for (let [operationId, body] of Object.entries(assertions)) {
-        expect(queries[operationId].fields.requestBody).toStrictEqual(
-          JSON.stringify(body, null, 2)
-        )
-      }
-    }
-    it("populates body", async () => {
-      const assertions = {
-        createEntity: {
-          name: "name",
-          type: "type",
-        },
-        getEntities: undefined,
-        getEntity: undefined,
-        updateEntity: {
-          id: 1,
-          name: "name",
-          type: "type",
-        },
-        patchEntity: {
-          id: 1,
-          name: "name",
-          type: "type",
-        },
-        deleteEntity: undefined,
-      }
-      await runTests("crud", testBody, assertions)
-    })
-  })
-})

packages/server/src/api/controllers/query/import/sources/tests/openapi2/openapi2.spec.ts

@@ -0,0 +1,135 @@
+import { OpenAPI2 } from "../../openapi2"
+import { readFileSync } from "fs"
+import { join } from "path"
+import { groupBy, mapValues } from "lodash"
+import { Query } from "@budibase/types"
+
+const getData = (file: string, extension: string) => {
+  return readFileSync(
+    join(__dirname, `./data/${file}/${file}.${extension}`),
+    "utf8"
+  )
+}
+
+describe("OpenAPI2 Import", () => {
+  let openapi2: OpenAPI2
+
+  beforeEach(() => {
+    openapi2 = new OpenAPI2()
+  })
+
+  it("validates unsupported data", async () => {
+    expect(await openapi2.isSupported("curl http://example.com")).toBe(false)
+    expect(await openapi2.isSupported("")).toBe(false)
+  })
+
+  describe.each(["json", "yaml"])("%s", extension => {
+    describe("petstore", () => {
+      beforeEach(async () => {
+        await openapi2.isSupported(getData("petstore", extension))
+      })
+
+      it("returns import info", async () => {
+        const { name } = await openapi2.getInfo()
+        expect(name).toBe("Swagger Petstore")
+      })
+    })
+
+    describe("crud", () => {
+      let queries: Record<string, Query>
+      beforeEach(async () => {
+        await openapi2.isSupported(getData("crud", extension))
+
+        const raw = await openapi2.getQueries("fake_datasource_id")
+        queries = mapValues(groupBy(raw, "name"), group => group[0])
+      })
+
+      it("should have 6 queries", () => {
+        expect(Object.keys(queries).length).toBe(6)
+      })
+
+      it.each([
+        ["createEntity", "create"],
+        ["getEntities", "read"],
+        ["getEntity", "read"],
+        ["updateEntity", "update"],
+        ["patchEntity", "patch"],
+        ["deleteEntity", "delete"],
+      ])("should have correct verb for %s", (operationId, method) => {
+        expect(queries[operationId].queryVerb).toBe(method)
+      })
+
+      it.each([
+        ["createEntity", "http://example.com/entities"],
+        ["getEntities", "http://example.com/entities"],
+        ["getEntity", "http://example.com/entities/{{entityId}}"],
+        ["updateEntity", "http://example.com/entities/{{entityId}}"],
+        ["patchEntity", "http://example.com/entities/{{entityId}}"],
+        ["deleteEntity", "http://example.com/entities/{{entityId}}"],
+      ])("should have correct path for %s", (operationId, urlPath) => {
+        expect(queries[operationId].fields.path).toBe(urlPath)
+      })
+
+      it.each([
+        ["createEntity", { "Content-Type": "application/json" }],
+        ["getEntities", {}],
+        ["getEntity", {}],
+        ["updateEntity", { "Content-Type": "application/json" }],
+        ["patchEntity", { "Content-Type": "application/json" }],
+        ["deleteEntity", { "x-api-key": "{{x-api-key}}" }],
+      ])(`should have correct headers for %s`, (operationId, headers) => {
+        expect(queries[operationId].fields.headers).toStrictEqual(headers)
+      })
+
+      it.each([
+        ["createEntity", ""],
+        ["getEntities", "page={{page}}&size={{size}}"],
+        ["getEntity", ""],
+        ["updateEntity", ""],
+        ["patchEntity", ""],
+        ["deleteEntity", ""],
+      ])(
+        `should have correct query string for %s`,
+        (operationId, queryString) => {
+          expect(queries[operationId].fields.queryString).toBe(queryString)
+        }
+      )
+
+      it.each([
+        ["createEntity", []],
+        [
+          "getEntities",
+          [
+            { name: "page", default: "" },
+            { name: "size", default: "" },
+          ],
+        ],
+        ["getEntity", [{ name: "entityId", default: "" }]],
+        ["updateEntity", [{ name: "entityId", default: "" }]],
+        ["patchEntity", [{ name: "entityId", default: "" }]],
+        [
+          "deleteEntity",
+          [
+            { name: "entityId", default: "" },
+            { name: "x-api-key", default: "" },
+          ],
+        ],
+      ])(`should have correct parameters for %s`, (operationId, parameters) => {
+        expect(queries[operationId].parameters).toStrictEqual(parameters)
+      })
+
+      it.each([
+        ["createEntity", { name: "name", type: "type" }],
+        ["getEntities", undefined],
+        ["getEntity", undefined],
+        ["updateEntity", { id: 1, name: "name", type: "type" }],
+        ["patchEntity", { id: 1, name: "name", type: "type" }],
+        ["deleteEntity", undefined],
+      ])(`should have correct body for %s`, (operationId, body) => {
+        expect(queries[operationId].fields.requestBody).toBe(
+          JSON.stringify(body, null, 2)
+        )
+      })
+    })
+  })
+})

packages/types/src/ui/dataFetch/datasources.ts

@@ -0,0 +1,93 @@
+import { InternalTable, Row, TableSchema } from "../../documents"
+
+export type DataFetchDatasource =
+  | TableDatasource
+  | ViewV1Datasource
+  | ViewDatasource
+  | QueryDatasource
+  | RelationshipDatasource
+  | UserDatasource
+  | GroupUserDatasource
+  | CustomDatasource
+  | NestedProviderDatasource
+  | FieldDatasource
+  | QueryArrayFieldDatasource
+  | JSONArrayFieldDatasource
+
+export interface TableDatasource {
+  type: "table"
+  tableId: string
+}
+
+export type ViewV1Datasource = {
+  type: "view"
+  name: string
+  tableId: string
+  calculation: string
+  field: string
+  groupBy: string
+}
+
+export interface ViewDatasource {
+  type: "viewV2"
+  id: string
+}
+
+export interface QueryDatasource {
+  type: "query"
+  _id: string
+  fields: Record<string, any> & {
+    pagination?: {
+      type: string
+      location: string
+      pageParam: string
+    }
+  }
+  queryParams?: Record<string, string>
+  parameters: { name: string; default: string }[]
+}
+
+export interface RelationshipDatasource {
+  type: "link"
+  tableId: string
+  rowId: string
+  rowTableId: string
+  fieldName: string
+}
+
+export interface UserDatasource {
+  type: "user"
+  tableId: InternalTable.USER_METADATA
+}
+
+export interface GroupUserDatasource {
+  type: "groupUser"
+  tableId: InternalTable.USER_METADATA
+}
+
+export interface CustomDatasource {
+  type: "custom"
+  data: any
+}
+
+export interface NestedProviderDatasource {
+  type: "provider"
+  value?: {
+    schema: TableSchema
+    primaryDisplay: string
+    rows: Row[]
+  }
+}
+
+interface BaseFieldDatasource<
+  TType extends "field" | "queryarray" | "jsonarray"
+> {
+  type: TType
+  tableId: string
+  fieldType: "attachment" | "array"
+  value: string[] | Row[]
+}
+
+export type FieldDatasource = BaseFieldDatasource<"field">
+export type QueryArrayFieldDatasource = BaseFieldDatasource<"queryarray">
+export type JSONArrayFieldDatasource = BaseFieldDatasource<"jsonarray">

packages/types/src/ui/dataFetch/index.ts

@@ -0,0 +1,16 @@
+import { LegacyFilter, SortOrder, UISearchFilter } from "../../api"
+import { SearchFilters } from "../../sdk"
+
+export * from "./datasources"
+
+export interface DataFetchOptions<TQuery = SearchFilters> {
+  // Search config
+  filter: UISearchFilter | LegacyFilter[] | null
+  query: TQuery
+  // Sorting config
+  sortColumn: string | null
+  sortOrder: SortOrder
+  // Pagination config
+  limit: number
+  paginate: boolean
+}

packages/types/src/ui/index.ts

@@ -1,3 +1,5 @@
 export * from "./stores"
 export * from "./bindings"
 export * from "./components"
+export * from "./dataFetch"
+

packages/types/src/ui/stores/grid/datasource.ts

@@ -1,6 +1,6 @@
 import { UITable, UIView } from "@budibase/types"
 
-export type UIDatasource = UITable | UIView
+export type UIDatasource = UITable | (Omit<UIView, "type"> & { type: "viewV2" })
 
 export interface UIFieldMutation {
   visible?: boolean

packages/types/src/ui/stores/grid/table.ts

@@ -8,10 +8,9 @@ import {
   UISearchFilter,
 } from "@budibase/types"
 
-export interface UITable extends Omit<Table, "type"> {
+export interface UITable extends Table {
   name: string
   id: string
-  type: string
   tableId: string
   primaryDisplay?: string
   sort?: {