Add new core implementations for fetching data and schema from datasources

packages/client/src/api/datasources.js

@@ -1,94 +0,0 @@
-import { cloneDeep } from "lodash/fp"
-import { fetchTableData, fetchTableDefinition } from "./tables"
-import { fetchViewData } from "./views"
-import { fetchRelationshipData } from "./relationships"
-import { FieldTypes } from "../constants"
-import { executeQuery, fetchQueryDefinition } from "./queries"
-
-/**
- * Fetches all rows for a particular Budibase data source.
- */
-export const fetchDatasource = async dataSource => {
-  if (!dataSource || !dataSource.type) {
-    return []
-  }
-
-  // Fetch all rows in data source
-  const { type, tableId, fieldName } = dataSource
-  let rows = []
-  if (type === "table") {
-    rows = await fetchTableData(tableId)
-  } else if (type === "view") {
-    rows = await fetchViewData(dataSource)
-  } else if (type === "query") {
-    // Set the default query params
-    let parameters = cloneDeep(dataSource.queryParams || {})
-    for (let param of dataSource.parameters) {
-      if (!parameters[param.name]) {
-        parameters[param.name] = param.default
-      }
-    }
-    rows = await executeQuery({ queryId: dataSource._id, parameters })
-  } else if (type === FieldTypes.LINK) {
-    rows = await fetchRelationshipData({
-      rowId: dataSource.rowId,
-      tableId: dataSource.rowTableId,
-      fieldName,
-    })
-  }
-
-  // Enrich the result is always an array
-  return Array.isArray(rows) ? rows : []
-}
-
-/**
- * Fetches the schema of any kind of datasource.
- */
-export const fetchDatasourceSchema = async dataSource => {
-  if (!dataSource) {
-    return null
-  }
-  const { type } = dataSource
-
-  // Nested providers should already have exposed their own schema
-  if (type === "provider") {
-    return dataSource.value?.schema
-  }
-
-  // Field sources have their schema statically defined
-  if (type === "field") {
-    if (dataSource.fieldType === "attachment") {
-      return {
-        url: {
-          type: "string",
-        },
-        name: {
-          type: "string",
-        },
-      }
-    } else if (dataSource.fieldType === "array") {
-      return {
-        value: {
-          type: "string",
-        },
-      }
-    }
-  }
-
-  // Tables, views and links can be fetched by table ID
-  if (
-    (type === "table" || type === "view" || type === "link") &&
-    dataSource.tableId
-  ) {
-    const table = await fetchTableDefinition(dataSource.tableId)
-    return table?.schema
-  }
-
-  // Queries can be fetched by query ID
-  if (type === "query" && dataSource._id) {
-    const definition = await fetchQueryDefinition(dataSource._id)
-    return definition?.schema
-  }
-
-  return null
-}

packages/client/src/api/index.js

@@ -1,6 +1,5 @@
 export * from "./rows"
 export * from "./auth"
-export * from "./datasources"
 export * from "./tables"
 export * from "./attachments"
 export * from "./views"

packages/client/src/components/app/DataProvider.svelte

@@ -8,6 +8,8 @@
     luceneLimit,
   } from "builder/src/helpers/lucene"
   import Placeholder from "./Placeholder.svelte"
+  import { fetchData } from "utils/fetch/fetchData.js"
+  import { cloneDeep } from "lodash/fp"
 
   export let dataSource
   export let filter
@@ -16,102 +18,62 @@
   export let limit
   export let paginate
 
-  const { API, styleable, Provider, ActionTypes } = getContext("sdk")
+  const { styleable, Provider, ActionTypes } = getContext("sdk")
   const component = getContext("component")
 
-  // Loading flag every time data is being fetched
+  const createFetch = datasource => {
-  let loading = false
+    return fetchData(datasource, {
-
+      filter,
-  // Loading flag for the initial load
+      sortColumn,
-  // Mark as loaded if we have no datasource so we don't stall forever
+      sortOrder,
-  let loaded = !dataSource
-  let schemaLoaded = false
-
-  // Provider state
-  let rows = []
-  let allRows = []
-  let schema = {}
-  let bookmarks = [null]
-  let pageNumber = 0
-  let query = null
-  let queryExtensions = {}
-
-  // Sorting can be overridden at run time, so we can't use the prop directly
-  let currentSortColumn = sortColumn
-  let currentSortOrder = sortOrder
-
-  // Reset the current sort state to props if props change
-  $: currentSortColumn = sortColumn
-  $: currentSortOrder = sortOrder
-
-  $: defaultQuery = buildLuceneQuery(filter)
-  $: extendQuery(defaultQuery, queryExtensions)
-  $: internalTable = dataSource?.type === "table"
-  $: nestedProvider = dataSource?.type === "provider"
-  $: hasNextPage = bookmarks[pageNumber + 1] != null
-  $: hasPrevPage = pageNumber > 0
-  $: getSchema(dataSource)
-  $: sortType = getSortType(schema, currentSortColumn)
-
-  // Wait until schema loads before loading data, so that we can determine
-  // the correct sort type first time
-  $: {
-    if (schemaLoaded) {
-      fetchData(
-        dataSource,
-        schema,
-        query,
       limit,
-        currentSortColumn,
+      paginate,
-        currentSortOrder,
+    })
-        sortType,
-        paginate
-      )
-    }
   }
 
-  // Reactively filter and sort rows if required
+  let fetch = fetchData()
+
+  $: id = $component.id
   $: {
-    if (internalTable) {
+    console.log("new datasource", id, dataSource)
-      // Internal tables are already processed server-side
+    fetch = createFetch(dataSource)
-      rows = allRows
-    } else {
-      // For anything else we use client-side implementations to filter, sort
-      // and limit
-      const filtered = luceneQuery(allRows, query)
-      const sorted = luceneSort(
-        filtered,
-        currentSortColumn,
-        currentSortOrder,
-        sortType
-      )
-      rows = luceneLimit(sorted, limit)
-    }
   }
 
+  $: fetch.update({
+    filter,
+    sortColumn,
+    sortOrder,
+    limit,
+    paginate,
+  })
+
   // Build our action context
   $: actions = [
     {
       type: ActionTypes.RefreshDatasource,
-      callback: () => refresh(),
+      callback: () => fetch.refresh(),
       metadata: { dataSource },
     },
-    {
+    // {
-      type: ActionTypes.AddDataProviderQueryExtension,
+    //   type: ActionTypes.AddDataProviderQueryExtension,
-      callback: addQueryExtension,
+    //   callback: addQueryExtension,
-    },
+    // },
-    {
+    // {
-      type: ActionTypes.RemoveDataProviderQueryExtension,
+    //   type: ActionTypes.RemoveDataProviderQueryExtension,
-      callback: removeQueryExtension,
+    //   callback: removeQueryExtension,
-    },
+    // },
     {
       type: ActionTypes.SetDataProviderSorting,
       callback: ({ column, order }) => {
+        let newOptions = {}
         if (column) {
-          currentSortColumn = column
+          newOptions.sortColumn = column
         }
         if (order) {
-          currentSortOrder = order
+          newOptions.sortOrder = order
+        }
+        if (Object.keys(newOptions)?.length) {
+          fetch.update(newOptions)
         }
       },
     },
@@ -119,199 +81,25 @@
 
   // Build our data context
   $: dataContext = {
-    rows,
+    rows: $fetch.rows,
-    schema,
+    schema: $fetch.schema,
-    rowsLength: rows.length,
+    rowsLength: $fetch.rows.length,
 
     // Undocumented properties. These aren't supposed to be used in builder
     // bindings, but are used internally by other components
     id: $component?.id,
     state: {
-      query,
+      query: $fetch.query,
-      sortColumn: currentSortColumn,
+      sortColumn: $fetch.sortColumn,
-      sortOrder: currentSortOrder,
+      sortOrder: $fetch.sortOrder,
     },
-    loaded,
+    loaded: $fetch.loaded,
-  }
-
-  const getSortType = (schema, sortColumn) => {
-    if (!schema || !sortColumn || !schema[sortColumn]) {
-      return "string"
-    }
-    const type = schema?.[sortColumn]?.type
-    return type === "number" ? "number" : "string"
-  }
-
-  const refresh = async () => {
-    if (schemaLoaded && !nestedProvider) {
-      fetchData(
-        dataSource,
-        schema,
-        query,
-        limit,
-        currentSortColumn,
-        currentSortOrder,
-        sortType,
-        paginate
-      )
-    }
-  }
-
-  const fetchData = async (
-    dataSource,
-    schema,
-    query,
-    limit,
-    sortColumn,
-    sortOrder,
-    sortType,
-    paginate
-  ) => {
-    loading = true
-    if (dataSource?.type === "table") {
-      // Sanity check sort column, as using a non-existant column will prevent
-      // results coming back at all
-      const sort = schema?.[sortColumn] ? sortColumn : undefined
-
-      // For internal tables we use server-side processing
-      const res = await API.searchTable({
-        tableId: dataSource.tableId,
-        query,
-        limit,
-        sort,
-        sortOrder: sortOrder?.toLowerCase() ?? "ascending",
-        sortType,
-        paginate,
-      })
-      pageNumber = 0
-      allRows = res.rows
-      if (res.hasNextPage) {
-        bookmarks = [null, res.bookmark]
-      } else {
-        bookmarks = [null]
-      }
-    } else if (dataSource?.type === "provider") {
-      // For providers referencing another provider, just use the rows it
-      // provides
-      allRows = dataSource?.value?.rows || []
-    } else if (dataSource?.type === "field") {
-      // Field sources will be available from context.
-      // Enrich non object elements into object to ensure a valid schema.
-      const data = dataSource?.value || []
-      if (Array.isArray(data) && data[0] && typeof data[0] !== "object") {
-        allRows = data.map(value => ({ value }))
-      } else {
-        allRows = data
-      }
-    } else {
-      // For other data sources like queries or views, fetch all rows from the
-      // server
-      allRows = await API.fetchDatasource(dataSource)
-    }
-    loading = false
-    loaded = true
-  }
-
-  const getSchema = async dataSource => {
-    let newSchema = (await API.fetchDatasourceSchema(dataSource)) || {}
-
-    // Ensure there are "name" properties for all fields and that field schema
-    // are objects
-    Object.entries(newSchema).forEach(([fieldName, fieldSchema]) => {
-      if (typeof fieldSchema === "string") {
-        newSchema[fieldName] = {
-          type: fieldSchema,
-          name: fieldName,
-        }
-      } else {
-        newSchema[fieldName] = {
-          ...fieldSchema,
-          name: fieldName,
-        }
-      }
-    })
-    schema = newSchema
-    schemaLoaded = true
-  }
-
-  const nextPage = async () => {
-    if (!hasNextPage || !internalTable) {
-      return
-    }
-    const sort = schema?.[currentSortColumn] ? currentSortColumn : undefined
-    const res = await API.searchTable({
-      tableId: dataSource?.tableId,
-      query,
-      bookmark: bookmarks[pageNumber + 1],
-      limit,
-      sort,
-      sortOrder: currentSortOrder?.toLowerCase() ?? "ascending",
-      sortType,
-      paginate: true,
-    })
-    pageNumber++
-    allRows = res.rows
-    if (res.hasNextPage) {
-      bookmarks[pageNumber + 1] = res.bookmark
-    }
-  }
-
-  const prevPage = async () => {
-    if (!hasPrevPage || !internalTable) {
-      return
-    }
-    const sort = schema?.[currentSortColumn] ? currentSortColumn : undefined
-    const res = await API.searchTable({
-      tableId: dataSource?.tableId,
-      query,
-      bookmark: bookmarks[pageNumber - 1],
-      limit,
-      sort,
-      sortOrder: currentSortOrder?.toLowerCase() ?? "ascending",
-      sortType,
-      paginate: true,
-    })
-    pageNumber--
-    allRows = res.rows
-  }
-
-  const addQueryExtension = (key, extension) => {
-    if (!key || !extension) {
-      return
-    }
-    queryExtensions = { ...queryExtensions, [key]: extension }
-  }
-
-  const removeQueryExtension = key => {
-    if (!key) {
-      return
-    }
-    const newQueryExtensions = { ...queryExtensions }
-    delete newQueryExtensions[key]
-    queryExtensions = newQueryExtensions
-  }
-
-  const extendQuery = (defaultQuery, extensions) => {
-    const extensionValues = Object.values(extensions || {})
-    let extendedQuery = { ...defaultQuery }
-    extensionValues.forEach(extension => {
-      Object.entries(extension || {}).forEach(([operator, fields]) => {
-        extendedQuery[operator] = {
-          ...extendedQuery[operator],
-          ...fields,
-        }
-      })
-    })
-
-    if (JSON.stringify(query) !== JSON.stringify(extendedQuery)) {
-      query = extendedQuery
-    }
   }
 </script>
 
 <div use:styleable={$component.styles} class="container">
   <Provider {actions} data={dataContext}>
-    {#if !loaded}
+    {#if !$fetch.loaded}
       <div class="loading">
         <ProgressCircle />
       </div>
@@ -321,14 +109,14 @@
       {:else}
         <slot />
       {/if}
-      {#if paginate && internalTable}
+      {#if paginate && dataSource?.type === "table"}
         <div class="pagination">
           <Pagination
-            page={pageNumber + 1}
+            page={$fetch.pageNumber + 1}
-            {hasPrevPage}
+            hasPrevPage={$fetch.hasPrevPage}
-            {hasNextPage}
+            hasNextPage={$fetch.hasNextPage}
-            goToPrevPage={prevPage}
+            goToPrevPage={fetch.prevPage}
-            goToNextPage={nextPage}
+            goToNextPage={fetch.nextPage}
           />
         </div>
       {/if}

packages/client/src/components/app/DataProvider2.svelte

@@ -0,0 +1,361 @@
+<script>
+  import { getContext } from "svelte"
+  import { ProgressCircle, Pagination } from "@budibase/bbui"
+  import {
+    buildLuceneQuery,
+    luceneQuery,
+    luceneSort,
+    luceneLimit,
+  } from "builder/src/helpers/lucene"
+  import Placeholder from "./Placeholder.svelte"
+  import { fetchData } from "utils/fetch/fetchData.js"
+
+  export let dataSource
+  export let filter
+  export let sortColumn
+  export let sortOrder
+  export let limit
+  export let paginate
+
+  const { API, styleable, Provider, ActionTypes } = getContext("sdk")
+  const component = getContext("component")
+
+  // Loading flag every time data is being fetched
+  let loading = false
+
+  // Loading flag for the initial load
+  // Mark as loaded if we have no datasource so we don't stall forever
+  let loaded = !dataSource
+  let schemaLoaded = false
+
+  // Provider state
+  let rows = []
+  let allRows = []
+  let schema = {}
+  let bookmarks = [null]
+  let pageNumber = 0
+  let query = null
+  let queryExtensions = {}
+
+  // Sorting can be overridden at run time, so we can't use the prop directly
+  let currentSortColumn = sortColumn
+  let currentSortOrder = sortOrder
+
+  // Reset the current sort state to props if props change
+  $: currentSortColumn = sortColumn
+  $: currentSortOrder = sortOrder
+
+  $: defaultQuery = buildLuceneQuery(filter)
+  $: extendQuery(defaultQuery, queryExtensions)
+  $: internalTable = dataSource?.type === "table"
+  $: nestedProvider = dataSource?.type === "provider"
+  $: hasNextPage = bookmarks[pageNumber + 1] != null
+  $: hasPrevPage = pageNumber > 0
+  $: getSchema(dataSource)
+  $: sortType = getSortType(schema, currentSortColumn)
+
+  // Wait until schema loads before loading data, so that we can determine
+  // the correct sort type first time
+  $: {
+    if (schemaLoaded) {
+      fetchData(
+        dataSource,
+        schema,
+        query,
+        limit,
+        currentSortColumn,
+        currentSortOrder,
+        sortType,
+        paginate
+      )
+    }
+  }
+
+  // Reactively filter and sort rows if required
+  $: {
+    if (internalTable) {
+      // Internal tables are already processed server-side
+      rows = allRows
+    } else {
+      // For anything else we use client-side implementations to filter, sort
+      // and limit
+      const filtered = luceneQuery(allRows, query)
+      const sorted = luceneSort(
+        filtered,
+        currentSortColumn,
+        currentSortOrder,
+        sortType
+      )
+      rows = luceneLimit(sorted, limit)
+    }
+  }
+
+  // Build our action context
+  $: actions = [
+    {
+      type: ActionTypes.RefreshDatasource,
+      callback: () => refresh(),
+      metadata: { dataSource },
+    },
+    {
+      type: ActionTypes.AddDataProviderQueryExtension,
+      callback: addQueryExtension,
+    },
+    {
+      type: ActionTypes.RemoveDataProviderQueryExtension,
+      callback: removeQueryExtension,
+    },
+    {
+      type: ActionTypes.SetDataProviderSorting,
+      callback: ({ column, order }) => {
+        if (column) {
+          currentSortColumn = column
+        }
+        if (order) {
+          currentSortOrder = order
+        }
+      },
+    },
+  ]
+
+  // Build our data context
+  $: dataContext = {
+    rows,
+    schema,
+    rowsLength: rows.length,
+
+    // Undocumented properties. These aren't supposed to be used in builder
+    // bindings, but are used internally by other components
+    id: $component?.id,
+    state: {
+      query,
+      sortColumn: currentSortColumn,
+      sortOrder: currentSortOrder,
+    },
+    loaded,
+  }
+
+  const getSortType = (schema, sortColumn) => {
+    if (!schema || !sortColumn || !schema[sortColumn]) {
+      return "string"
+    }
+    const type = schema?.[sortColumn]?.type
+    return type === "number" ? "number" : "string"
+  }
+
+  const refresh = async () => {
+    if (schemaLoaded && !nestedProvider) {
+      fetchData(
+        dataSource,
+        schema,
+        query,
+        limit,
+        currentSortColumn,
+        currentSortOrder,
+        sortType,
+        paginate
+      )
+    }
+  }
+
+  const fetchData = async (
+    dataSource,
+    schema,
+    query,
+    limit,
+    sortColumn,
+    sortOrder,
+    sortType,
+    paginate
+  ) => {
+    loading = true
+    if (dataSource?.type === "table") {
+      // Sanity check sort column, as using a non-existant column will prevent
+      // results coming back at all
+      const sort = schema?.[sortColumn] ? sortColumn : undefined
+
+      // For internal tables we use server-side processing
+      const res = await API.searchTable({
+        tableId: dataSource.tableId,
+        query,
+        limit,
+        sort,
+        sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+        sortType,
+        paginate,
+      })
+      pageNumber = 0
+      allRows = res.rows
+      if (res.hasNextPage) {
+        bookmarks = [null, res.bookmark]
+      } else {
+        bookmarks = [null]
+      }
+    } else if (dataSource?.type === "provider") {
+      // For providers referencing another provider, just use the rows it
+      // provides
+      allRows = dataSource?.value?.rows || []
+    } else if (dataSource?.type === "field") {
+      // Field sources will be available from context.
+      // Enrich non object elements into object to ensure a valid schema.
+      const data = dataSource?.value || []
+      if (Array.isArray(data) && data[0] && typeof data[0] !== "object") {
+        allRows = data.map(value => ({ value }))
+      } else {
+        allRows = data
+      }
+    } else {
+      // For other data sources like queries or views, fetch all rows from the
+      // server
+      allRows = await API.fetchDatasource(dataSource)
+    }
+    loading = false
+    loaded = true
+  }
+
+  const getSchema = async dataSource => {
+    let newSchema = (await API.fetchDatasourceSchema(dataSource)) || {}
+
+    // Ensure there are "name" properties for all fields and that field schema
+    // are objects
+    Object.entries(newSchema).forEach(([fieldName, fieldSchema]) => {
+      if (typeof fieldSchema === "string") {
+        newSchema[fieldName] = {
+          type: fieldSchema,
+          name: fieldName,
+        }
+      } else {
+        newSchema[fieldName] = {
+          ...fieldSchema,
+          name: fieldName,
+        }
+      }
+    })
+    schema = newSchema
+    schemaLoaded = true
+  }
+
+  const nextPage = async () => {
+    if (!hasNextPage || !internalTable) {
+      return
+    }
+    const sort = schema?.[currentSortColumn] ? currentSortColumn : undefined
+    const res = await API.searchTable({
+      tableId: dataSource?.tableId,
+      query,
+      bookmark: bookmarks[pageNumber + 1],
+      limit,
+      sort,
+      sortOrder: currentSortOrder?.toLowerCase() ?? "ascending",
+      sortType,
+      paginate: true,
+    })
+    pageNumber++
+    allRows = res.rows
+    if (res.hasNextPage) {
+      bookmarks[pageNumber + 1] = res.bookmark
+    }
+  }
+
+  const prevPage = async () => {
+    if (!hasPrevPage || !internalTable) {
+      return
+    }
+    const sort = schema?.[currentSortColumn] ? currentSortColumn : undefined
+    const res = await API.searchTable({
+      tableId: dataSource?.tableId,
+      query,
+      bookmark: bookmarks[pageNumber - 1],
+      limit,
+      sort,
+      sortOrder: currentSortOrder?.toLowerCase() ?? "ascending",
+      sortType,
+      paginate: true,
+    })
+    pageNumber--
+    allRows = res.rows
+  }
+
+  const addQueryExtension = (key, extension) => {
+    if (!key || !extension) {
+      return
+    }
+    queryExtensions = { ...queryExtensions, [key]: extension }
+  }
+
+  const removeQueryExtension = key => {
+    if (!key) {
+      return
+    }
+    const newQueryExtensions = { ...queryExtensions }
+    delete newQueryExtensions[key]
+    queryExtensions = newQueryExtensions
+  }
+
+  const extendQuery = (defaultQuery, extensions) => {
+    const extensionValues = Object.values(extensions || {})
+    let extendedQuery = { ...defaultQuery }
+    extensionValues.forEach(extension => {
+      Object.entries(extension || {}).forEach(([operator, fields]) => {
+        extendedQuery[operator] = {
+          ...extendedQuery[operator],
+          ...fields,
+        }
+      })
+    })
+
+    if (JSON.stringify(query) !== JSON.stringify(extendedQuery)) {
+      query = extendedQuery
+    }
+  }
+</script>
+
+<div use:styleable={$component.styles} class="container">
+  <Provider {actions} data={dataContext}>
+    {#if !loaded}
+      <div class="loading">
+        <ProgressCircle />
+      </div>
+    {:else}
+      {#if $component.emptyState}
+        <Placeholder />
+      {:else}
+        <slot />
+      {/if}
+      {#if paginate && internalTable}
+        <div class="pagination">
+          <Pagination
+            page={pageNumber + 1}
+            {hasPrevPage}
+            {hasNextPage}
+            goToPrevPage={prevPage}
+            goToNextPage={nextPage}
+          />
+        </div>
+      {/if}
+    {/if}
+  </Provider>
+</div>
+
+<style>
+  .container {
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: stretch;
+  }
+  .loading {
+    display: flex;
+    flex-direction: row;
+    justify-content: center;
+    align-items: center;
+    height: 100px;
+  }
+  .pagination {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-end;
+    align-items: center;
+    margin-top: var(--spacing-xl);
+  }
+</style>

packages/client/src/components/app/blocks/CardsBlock.svelte

@@ -30,7 +30,7 @@
   export let cardButtonOnClick
   export let linkColumn
 
-  const { API, styleable } = getContext("sdk")
+  const { fetchDatasourceSchema, styleable } = getContext("sdk")
   const context = getContext("context")
   const component = getContext("component")
   const schemaComponentMap = {
@@ -111,7 +111,7 @@
   // Load the datasource schema so we can determine column types
   const fetchSchema = async dataSource => {
     if (dataSource) {
-      schema = await API.fetchDatasourceSchema(dataSource)
+      schema = await fetchDatasourceSchema(dataSource)
     }
   }
 </script>

packages/client/src/components/app/blocks/TableBlock.svelte

@@ -26,7 +26,7 @@
   export let titleButtonURL
   export let titleButtonPeek
 
-  const { API, styleable } = getContext("sdk")
+  const { fetchDatasourceSchema, styleable } = getContext("sdk")
   const context = getContext("context")
   const component = getContext("component")
   const schemaComponentMap = {
@@ -40,6 +40,7 @@
   let formId
   let dataProviderId
   let schema
+  let schemaLoaded = false
 
   $: fetchSchema(dataSource)
   $: enrichedSearchColumns = enrichSearchColumns(searchColumns, schema)
@@ -89,8 +90,9 @@
   // Load the datasource schema so we can determine column types
   const fetchSchema = async dataSource => {
     if (dataSource) {
-      schema = await API.fetchDatasourceSchema(dataSource)
+      schema = await fetchDatasourceSchema(dataSource)
     }
+    schemaLoaded = true
   }
 </script>
 
@@ -134,6 +136,7 @@
           </div>
         </div>
       {/if}
+      {#if schema}
         <BlockComponent
           type="dataprovider"
           bind:id={dataProviderId}
@@ -162,6 +165,7 @@
             }}
           />
         </BlockComponent>
+      {/if}
     </BlockComponent>
   </div>
 </Block>

packages/client/src/components/app/forms/Form.svelte

@@ -10,7 +10,7 @@
   export let actionType = "Create"
 
   const context = getContext("context")
-  const { API } = getContext("sdk")
+  const { API, fetchDatasourceSchema } = getContext("sdk")
 
   let loaded = false
   let schema
@@ -61,7 +61,7 @@
 
     // For all other cases, just grab the normal schema
     else {
-      const dataSourceSchema = await API.fetchDatasourceSchema(dataSource)
+      const dataSourceSchema = await fetchDatasourceSchema(dataSource)
       schema = dataSourceSchema || {}
     }
 

packages/client/src/sdk.js

@@ -9,6 +9,7 @@ import {
 import { styleable } from "utils/styleable"
 import { linkable } from "utils/linkable"
 import { getAction } from "utils/getAction"
+import { fetchDatasourceSchema } from "utils/schema.js"
 import Provider from "components/context/Provider.svelte"
 import { ActionTypes } from "constants"
 
@@ -22,6 +23,7 @@ export default {
   styleable,
   linkable,
   getAction,
+  fetchDatasourceSchema,
   Provider,
   ActionTypes,
 }

packages/client/src/utils/fetch/QueryFetch.js

@@ -0,0 +1,42 @@
+import TableFetch from "./TableFetch.js"
+import { executeQuery, fetchQueryDefinition } from "api"
+import { cloneDeep } from "lodash/fp.js"
+
+export default class ViewFetch extends TableFetch {
+  SupportsSearch = false
+  SupportsSort = false
+  SupportsPagination = false
+
+  /**
+   * Fetches the schema for a view
+   * @param datasource the view datasource config
+   * @return {object} the view schema
+   */
+  static async getSchema(datasource) {
+    if (!datasource?._id) {
+      return null
+    }
+    const definition = await fetchQueryDefinition(datasource._id)
+    return this.enrichSchema(definition?.schema)
+  }
+
+  /**
+   * Fetches a single page of data from the remote resource
+   */
+  async getData() {
+    const { datasource } = this.options
+
+    // Set the default query params
+    let parameters = cloneDeep(datasource?.queryParams || {})
+    for (let param of datasource?.parameters || {}) {
+      if (!parameters[param.name]) {
+        parameters[param.name] = param.default
+      }
+    }
+
+    const res = await executeQuery({ queryId: datasource?._id, parameters })
+    return {
+      rows: res || [],
+    }
+  }
+}

packages/client/src/utils/fetch/RelationshipFetch.js

@@ -0,0 +1,23 @@
+import TableFetch from "./TableFetch.js"
+import { fetchRelationshipData } from "api"
+
+export default class ViewFetch extends TableFetch {
+  SupportsSearch = false
+  SupportsSort = false
+  SupportsPagination = false
+
+  /**
+   * Fetches a single page of data from the remote resource
+   */
+  async getData() {
+    const { datasource } = this.options
+    const res = await fetchRelationshipData({
+      rowId: datasource?.rowId,
+      tableId: datasource?.rowTableId,
+      fieldName: datasource?.fieldName,
+    })
+    return {
+      rows: res || [],
+    }
+  }
+}

packages/client/src/utils/fetch/TableFetch.js

@@ -0,0 +1,324 @@
+import { writable, derived, get } from "svelte/store"
+import * as API from "api"
+import { buildLuceneQuery } from "builder/src/helpers/lucene"
+import { fetchTableDefinition } from "api"
+
+/**
+ * Parent class which handles the implementation of fetching data from an
+ * internal table or datasource plus.
+ * For other types of datasource, this class is overridden and extended.
+ */
+export default class TableFetch {
+  SupportsSearch = true
+  SupportsSort = true
+  SupportsPagination = true
+
+  // Config
+  options = {
+    datasource: null,
+    schema: null,
+    limit: 10,
+
+    // Search config
+    filter: null,
+    query: null,
+
+    // Sorting config
+    sortColumn: null,
+    sortOrder: "ascending",
+
+    // Pagination config
+    paginate: true,
+  }
+
+  // State of the fetch
+  store = writable({
+    rows: [],
+    schema: null,
+    loading: false,
+    loaded: false,
+    query: null,
+    pageNumber: 0,
+    cursor: null,
+    cursors: [],
+  })
+
+  /**
+   * Constructs a new DataFetch instance.
+   * @param opts the fetch options
+   */
+  constructor(opts) {
+    this.options = {
+      ...this.options,
+      ...opts,
+    }
+
+    // Bind all functions to properly scope "this"
+    this.getData = this.getData.bind(this)
+    this.getInitialData = this.getInitialData.bind(this)
+    this.refresh = this.refresh.bind(this)
+    this.update = this.update.bind(this)
+    this.hasNextPage = this.hasNextPage.bind(this)
+    this.hasPrevPage = this.hasPrevPage.bind(this)
+    this.nextPage = this.nextPage.bind(this)
+    this.prevPage = this.prevPage.bind(this)
+
+    // Derive certain properties to return
+    this.derivedStore = derived(this.store, $store => {
+      return {
+        ...$store,
+        hasNextPage: this.hasNextPage($store),
+        hasPrevPage: this.hasPrevPage($store),
+      }
+    })
+
+    // Mark as loaded if we have no datasource
+    if (!this.options.datasource) {
+      this.store.update($store => ({ ...$store, loaded: true }))
+      return
+    }
+
+    // Initially fetch data but don't bother waiting for the result
+    this.getInitialData()
+  }
+
+  /**
+   * Extend the svelte store subscribe method to that instances of this class
+   * can be treated like stores
+   */
+  get subscribe() {
+    return this.derivedStore.subscribe
+  }
+
+  /**
+   * Fetches a fresh set of data from the server, resetting pagination
+   */
+  async getInitialData() {
+    const { datasource, filter } = this.options
+    const tableId = datasource?.tableId
+
+    // Ensure table ID exists
+    if (!tableId) {
+      return
+    }
+
+    // Ensure schema exists and enrich it
+    let schema = this.options.schema
+    if (!schema) {
+      schema = await this.constructor.getSchema(datasource)
+    }
+    if (!schema) {
+      return
+    }
+
+    // Build the lucene query
+    let query = this.options.query
+    if (!query) {
+      query = buildLuceneQuery(filter)
+    }
+
+    // Update store
+    this.store.update($store => ({ ...$store, schema, query, loading: true }))
+
+    // Actually fetch data
+    const page = await this.getData()
+    this.store.update($store => ({
+      ...$store,
+      loading: false,
+      loaded: true,
+      pageNumber: 0,
+      rows: page.rows,
+      cursors: page.hasNextPage ? [null, page.cursor] : [null],
+    }))
+  }
+
+  /**
+   * Fetches a single page of data from the remote resource
+   */
+  async getData() {
+    const { datasource, limit, sortColumn, sortOrder, paginate } = this.options
+    const { tableId } = datasource
+    const { cursor, schema, query } = get(this.store)
+
+    // Work out what sort type to use
+    const type = schema?.[sortColumn]?.type
+    const sortType = type === "number" ? "number" : "string"
+
+    // Search table
+    const res = await API.searchTable({
+      tableId,
+      query,
+      limit,
+      sort: sortColumn,
+      sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+      sortType,
+      paginate,
+      bookmark: cursor,
+    })
+    return {
+      rows: res?.rows || [],
+      hasNextPage: res?.hasNextPage || false,
+      cursor: res?.bookmark || null,
+    }
+  }
+
+  /**
+   * Gets the schema definition for a table
+   * @param datasource the datasource definition
+   * @return {object} the schema
+   */
+  static async getSchema(datasource) {
+    if (!datasource?.tableId) {
+      return null
+    }
+    const table = await fetchTableDefinition(datasource.tableId)
+    return this.enrichSchema(table?.schema)
+  }
+
+  /**
+   * Enriches the schema and ensures that entries are objects with names
+   * @param schema the datasource schema
+   * @return {object} the enriched datasource schema
+   */
+  static enrichSchema(schema) {
+    if (schema == null) {
+      return null
+    }
+    let enrichedSchema = {}
+    Object.entries(schema).forEach(([fieldName, fieldSchema]) => {
+      if (typeof fieldSchema === "string") {
+        enrichedSchema[fieldName] = {
+          type: fieldSchema,
+          name: fieldName,
+        }
+      } else {
+        enrichedSchema[fieldName] = {
+          ...fieldSchema,
+          name: fieldName,
+        }
+      }
+    })
+    return enrichedSchema
+  }
+
+  /**
+   * Resets the data set and updates options
+   * @param newOptions any new options
+   */
+  async update(newOptions) {
+    // Check if any settings have actually changed
+    let refresh = false
+    const entries = Object.entries(newOptions || {})
+    for (let [key, value] of entries) {
+      if (JSON.stringify(value) !== JSON.stringify(this.options[key])) {
+        refresh = true
+        break
+      }
+    }
+    if (!refresh) {
+      return
+    }
+
+    // Assign new options and reload data
+    this.options = {
+      ...this.options,
+      ...newOptions,
+    }
+    await this.getInitialData()
+  }
+
+  /**
+   * Loads the same page again
+   */
+  async refresh() {
+    if (get(this.store).loading) {
+      return
+    }
+    const { rows } = await this.getData()
+    this.store.update($store => ({ ...$store, loading: true }))
+    this.store.update($store => ({ ...$store, rows, loading: false }))
+  }
+
+  /**
+   * Determines whether there is a next page of data based on the state of the
+   * store
+   * @param state the current store state
+   * @return {boolean} whether there is a next page of data or not
+   */
+  hasNextPage(state) {
+    return state.cursors[state.pageNumber + 1] != null
+  }
+
+  /**
+   * Determines whether there is a previous page of data based on the state of
+   * the store
+   * @param state the current store state
+   * @return {boolean} whether there is a previous page of data or not
+   */
+  hasPrevPage(state) {
+    return state.pageNumber > 0
+  }
+
+  /**
+   * Fetches the next page of data
+   */
+  async nextPage() {
+    const state = get(this.derivedStore)
+    if (state.loading || !this.options.paginate || !state.hasNextPage) {
+      return
+    }
+
+    // Fetch next page
+    const nextCursor = state.cursors[state.pageNumber + 1]
+    this.store.update($store => ({
+      ...$store,
+      loading: true,
+      cursor: nextCursor,
+    }))
+    const { rows, hasNextPage, cursor } = await this.getData()
+
+    // Update state
+    this.store.update($store => {
+      let { cursors, pageNumber } = $store
+      if (hasNextPage) {
+        cursors[pageNumber + 2] = cursor
+      }
+      return {
+        ...$store,
+        pageNumber: pageNumber + 1,
+        rows,
+        cursors,
+        loading: false,
+      }
+    })
+  }
+
+  /**
+   * Fetches the previous page of data
+   */
+  async prevPage() {
+    const state = get(this.derivedStore)
+    if (state.loading || !this.options.paginate || !state.hasPrevPage) {
+      return
+    }
+
+    // Fetch previous page
+    const prevCursor = state.cursors[state.pageNumber - 1]
+    this.store.update($store => ({
+      ...$store,
+      loading: true,
+      cursor: prevCursor,
+    }))
+    const { rows } = await this.getData()
+
+    // Update state
+    this.store.update($store => {
+      return {
+        ...$store,
+        pageNumber: $store.pageNumber - 1,
+        rows,
+        loading: false,
+      }
+    })
+  }
+}

packages/client/src/utils/fetch/ViewFetch.js

@@ -0,0 +1,34 @@
+import TableFetch from "./TableFetch.js"
+import { fetchTableDefinition, fetchViewData } from "api"
+
+export default class ViewFetch extends TableFetch {
+  SupportsSearch = false
+  SupportsSort = false
+  SupportsPagination = false
+
+  /**
+   * Fetches the schema for a view
+   * @param datasource the view datasource config
+   * @return {object} the view schema
+   */
+  static async getSchema(datasource) {
+    if (!datasource?.tableId) {
+      return null
+    }
+    const table = await fetchTableDefinition(datasource.tableId)
+    return this.enrichSchema(table?.views?.[datasource.name]?.schema)
+  }
+
+  /**
+   * Fetches a single page of data from the remote resource
+   */
+  async getData() {
+    const { datasource } = this.options
+    const res = await fetchViewData(datasource)
+    return {
+      rows: res || [],
+    }
+  }
+}
+
+ViewFetch.getSchema()

packages/client/src/utils/fetch/fetchData.js

@@ -0,0 +1,16 @@
+import TableFetch from "./TableFetch.js"
+import ViewFetch from "./ViewFetch.js"
+import QueryFetch from "./QueryFetch.js"
+import RelationshipFetch from "./RelationshipFetch.js"
+
+const DataFetchMap = {
+  table: TableFetch,
+  view: ViewFetch,
+  query: QueryFetch,
+  link: RelationshipFetch,
+}
+
+export const fetchData = (datasource, options) => {
+  const Fetch = DataFetchMap[datasource?.type] || TableFetch
+  return new Fetch({ datasource, ...options })
+}

packages/client/src/utils/schema.js

@@ -0,0 +1,49 @@
+import TableFetch from "./fetch/TableFetch.js"
+import ViewFetch from "./fetch/ViewFetch.js"
+import QueryFetch from "./fetch/QueryFetch.js"
+
+/**
+ * Fetches the schema of any kind of datasource.
+ */
+export const fetchDatasourceSchema = async datasource => {
+  const type = datasource?.type
+
+  // Nested providers should already have exposed their own schema
+  if (type === "provider") {
+    return datasource.value?.schema
+  }
+
+  // Field sources have their schema statically defined
+  if (type === "field") {
+    if (datasource.fieldType === "attachment") {
+      return {
+        url: {
+          type: "string",
+        },
+        name: {
+          type: "string",
+        },
+      }
+    } else if (datasource.fieldType === "array") {
+      return {
+        value: {
+          type: "string",
+        },
+      }
+    }
+  }
+
+  // All normal datasource schema can use their corresponsing implementations
+  // in the data fetch classes
+  if (type === "table" || type === "link") {
+    return TableFetch.getSchema(datasource)
+  }
+  if (type === "view") {
+    return ViewFetch.getSchema(datasource)
+  }
+  if (type === "query") {
+    return QueryFetch.getSchema(datasource)
+  }
+
+  return null
+}