Add client-side searching, sorting and limiting for incompatible datasources to new DataFetch model

packages/client/src/components/app/DataProvider.svelte

@@ -1,15 +1,8 @@
 <script>
   import { getContext } from "svelte"
   import { ProgressCircle, Pagination } from "@budibase/bbui"
-  import {
-    buildLuceneQuery,
-    luceneQuery,
-    luceneSort,
-    luceneLimit,
-  } from "builder/src/helpers/lucene"
   import Placeholder from "./Placeholder.svelte"
   import { fetchData } from "utils/fetch/fetchData.js"
-  import { cloneDeep } from "lodash/fp"
 
   export let dataSource
   export let filter
@@ -21,24 +14,7 @@
   const { styleable, Provider, ActionTypes } = getContext("sdk")
   const component = getContext("component")
 
-  const createFetch = datasource => {
+  $: fetch = createFetch(dataSource)
-    return fetchData(datasource, {
-      filter,
-      sortColumn,
-      sortOrder,
-      limit,
-      paginate,
-    })
-  }
-
-  let fetch = fetchData()
-
-  $: id = $component.id
-  $: {
-    console.log("new datasource", id, dataSource)
-    fetch = createFetch(dataSource)
-  }
-
   $: fetch.update({
     filter,
     sortColumn,
@@ -95,6 +71,16 @@
     },
     loaded: $fetch.loaded,
   }
+
+  const createFetch = datasource => {
+    return fetchData(datasource, {
+      filter,
+      sortColumn,
+      sortOrder,
+      limit,
+      paginate,
+    })
+  }
 </script>
 
 <div use:styleable={$component.styles} class="container">

packages/client/src/utils/fetch/DataFetch.js

@@ -0,0 +1,354 @@
+import { writable, derived, get } from "svelte/store"
+import {
+  buildLuceneQuery,
+  luceneLimit,
+  luceneQuery,
+  luceneSort,
+} from "builder/src/helpers/lucene"
+import { fetchTableDefinition } from "api"
+
+/**
+ * Parent class which handles the implementation of fetching data from an
+ * internal table or datasource plus.
+ * For other types of datasource, this class is overridden and extended.
+ */
+export default class TableFetch {
+  SupportsSearch = false
+  SupportsSort = false
+  SupportsPagination = false
+
+  // Config
+  options = {
+    datasource: null,
+    schema: null,
+    limit: 10,
+
+    // Search config
+    filter: null,
+    query: null,
+
+    // Sorting config
+    sortColumn: null,
+    sortOrder: "ascending",
+    sortType: null,
+
+    // Pagination config
+    paginate: true,
+  }
+
+  // State of the fetch
+  store = writable({
+    rows: [],
+    schema: null,
+    loading: false,
+    loaded: false,
+    query: null,
+    pageNumber: 0,
+    cursor: null,
+    cursors: [],
+  })
+
+  /**
+   * Constructs a new DataFetch instance.
+   * @param opts the fetch options
+   */
+  constructor(opts) {
+    this.options = {
+      ...this.options,
+      ...opts,
+    }
+
+    // Bind all functions to properly scope "this"
+    this.getData = this.getData.bind(this)
+    this.getInitialData = this.getInitialData.bind(this)
+    this.refresh = this.refresh.bind(this)
+    this.update = this.update.bind(this)
+    this.hasNextPage = this.hasNextPage.bind(this)
+    this.hasPrevPage = this.hasPrevPage.bind(this)
+    this.nextPage = this.nextPage.bind(this)
+    this.prevPage = this.prevPage.bind(this)
+
+    // Derive certain properties to return
+    this.derivedStore = derived(this.store, $store => {
+      return {
+        ...$store,
+        hasNextPage: this.hasNextPage($store),
+        hasPrevPage: this.hasPrevPage($store),
+      }
+    })
+
+    // Mark as loaded if we have no datasource
+    if (!this.options.datasource) {
+      this.store.update($store => ({ ...$store, loaded: true }))
+      return
+    }
+
+    // Initially fetch data but don't bother waiting for the result
+    this.getInitialData()
+  }
+
+  /**
+   * Extend the svelte store subscribe method to that instances of this class
+   * can be treated like stores
+   */
+  get subscribe() {
+    return this.derivedStore.subscribe
+  }
+
+  /**
+   * Fetches a fresh set of data from the server, resetting pagination
+   */
+  async getInitialData() {
+    const { datasource, filter, sortColumn } = this.options
+    const tableId = datasource?.tableId
+
+    // Ensure table ID exists
+    if (!tableId) {
+      return
+    }
+
+    // Ensure schema exists and enrich it
+    let schema = this.options.schema
+    if (!schema) {
+      schema = await this.constructor.getSchema(datasource)
+    }
+    if (!schema) {
+      return
+    }
+
+    // Determine what sort type to use
+    if (!this.options.sortType) {
+      let sortType = "string"
+      if (sortColumn) {
+        const type = schema?.[sortColumn]?.type
+        sortType = type === "number" ? "number" : "string"
+      }
+      this.options.sortType = sortType
+    }
+
+    // Build the lucene query
+    let query = this.options.query
+    if (!query) {
+      query = buildLuceneQuery(filter)
+    }
+
+    // Update store
+    this.store.update($store => ({ ...$store, schema, query, loading: true }))
+
+    // Actually fetch data
+    const page = await this.getPage()
+    this.store.update($store => ({
+      ...$store,
+      loading: false,
+      loaded: true,
+      pageNumber: 0,
+      rows: page.rows,
+      cursors: page.hasNextPage ? [null, page.cursor] : [null],
+    }))
+  }
+
+  /**
+   * Fetches some filtered, sorted and paginated data
+   */
+  async getPage() {
+    const { sortColumn, sortOrder, sortType, limit } = this.options
+    const { query } = get(this.store)
+
+    // Get the actual data
+    let { rows, hasNextPage, cursor } = await this.getData()
+
+    // If we don't support searching, do a client search
+    if (!this.SupportsSearch) {
+      rows = luceneQuery(rows, query)
+    }
+
+    // If we don't support sorting, do a client-side sort
+    if (!this.SupportsSort) {
+      rows = luceneSort(rows, sortColumn, sortOrder, sortType)
+    }
+
+    // If we don't support pagination, do a client-side limit
+    if (!this.SupportsPagination) {
+      rows = luceneLimit(rows, limit)
+    }
+
+    return {
+      rows,
+      hasNextPage,
+      cursor,
+    }
+  }
+
+  /**
+   * Fetches a single page of data from the remote resource.
+   * Must be overridden by a datasource specific child class.
+   */
+  async getData() {
+    return {
+      rows: [],
+      hasNextPage: false,
+      cursor: null,
+    }
+  }
+
+  /**
+   * Gets the schema definition for a datasource.
+   * Defaults to fetching a table definition.
+   * @param datasource the datasource definition
+   * @return {object} the schema
+   */
+  static async getSchema(datasource) {
+    if (!datasource?.tableId) {
+      return null
+    }
+    const table = await fetchTableDefinition(datasource.tableId)
+    return this.enrichSchema(table?.schema)
+  }
+
+  /**
+   * Enriches the schema and ensures that entries are objects with names
+   * @param schema the datasource schema
+   * @return {object} the enriched datasource schema
+   */
+  static enrichSchema(schema) {
+    if (schema == null) {
+      return null
+    }
+    let enrichedSchema = {}
+    Object.entries(schema).forEach(([fieldName, fieldSchema]) => {
+      if (typeof fieldSchema === "string") {
+        enrichedSchema[fieldName] = {
+          type: fieldSchema,
+          name: fieldName,
+        }
+      } else {
+        enrichedSchema[fieldName] = {
+          ...fieldSchema,
+          name: fieldName,
+        }
+      }
+    })
+    return enrichedSchema
+  }
+
+  /**
+   * Resets the data set and updates options
+   * @param newOptions any new options
+   */
+  async update(newOptions) {
+    // Check if any settings have actually changed
+    let refresh = false
+    const entries = Object.entries(newOptions || {})
+    for (let [key, value] of entries) {
+      if (JSON.stringify(value) !== JSON.stringify(this.options[key])) {
+        refresh = true
+        break
+      }
+    }
+    if (!refresh) {
+      return
+    }
+
+    // Assign new options and reload data
+    this.options = {
+      ...this.options,
+      ...newOptions,
+    }
+    await this.getInitialData()
+  }
+
+  /**
+   * Loads the same page again
+   */
+  async refresh() {
+    if (get(this.store).loading) {
+      return
+    }
+    const { rows } = await this.getPage()
+    this.store.update($store => ({ ...$store, loading: true }))
+    this.store.update($store => ({ ...$store, rows, loading: false }))
+  }
+
+  /**
+   * Determines whether there is a next page of data based on the state of the
+   * store
+   * @param state the current store state
+   * @return {boolean} whether there is a next page of data or not
+   */
+  hasNextPage(state) {
+    return state.cursors[state.pageNumber + 1] != null
+  }
+
+  /**
+   * Determines whether there is a previous page of data based on the state of
+   * the store
+   * @param state the current store state
+   * @return {boolean} whether there is a previous page of data or not
+   */
+  hasPrevPage(state) {
+    return state.pageNumber > 0
+  }
+
+  /**
+   * Fetches the next page of data
+   */
+  async nextPage() {
+    const state = get(this.derivedStore)
+    if (state.loading || !this.options.paginate || !state.hasNextPage) {
+      return
+    }
+
+    // Fetch next page
+    const nextCursor = state.cursors[state.pageNumber + 1]
+    this.store.update($store => ({
+      ...$store,
+      loading: true,
+      cursor: nextCursor,
+    }))
+    const { rows, hasNextPage, cursor } = await this.getPage()
+
+    // Update state
+    this.store.update($store => {
+      let { cursors, pageNumber } = $store
+      if (hasNextPage) {
+        cursors[pageNumber + 2] = cursor
+      }
+      return {
+        ...$store,
+        pageNumber: pageNumber + 1,
+        rows,
+        cursors,
+        loading: false,
+      }
+    })
+  }
+
+  /**
+   * Fetches the previous page of data
+   */
+  async prevPage() {
+    const state = get(this.derivedStore)
+    if (state.loading || !this.options.paginate || !state.hasPrevPage) {
+      return
+    }
+
+    // Fetch previous page
+    const prevCursor = state.cursors[state.pageNumber - 1]
+    this.store.update($store => ({
+      ...$store,
+      loading: true,
+      cursor: prevCursor,
+    }))
+    const { rows } = await this.getPage()
+
+    // Update state
+    this.store.update($store => {
+      return {
+        ...$store,
+        pageNumber: $store.pageNumber - 1,
+        rows,
+        loading: false,
+      }
+    })
+  }
+}

packages/client/src/utils/fetch/QueryFetch.js

@@ -1,17 +1,8 @@
-import TableFetch from "./TableFetch.js"
+import DataFetch from "./DataFetch.js"
 import { executeQuery, fetchQueryDefinition } from "api"
-import { cloneDeep } from "lodash/fp.js"
+import { cloneDeep } from "lodash/fp"
 
-export default class ViewFetch extends TableFetch {
+export default class ViewFetch extends DataFetch {
-  SupportsSearch = false
-  SupportsSort = false
-  SupportsPagination = false
-
-  /**
-   * Fetches the schema for a view
-   * @param datasource the view datasource config
-   * @return {object} the view schema
-   */
   static async getSchema(datasource) {
     if (!datasource?._id) {
       return null
@@ -20,9 +11,6 @@ export default class ViewFetch extends TableFetch {
     return this.enrichSchema(definition?.schema)
   }
 
-  /**
-   * Fetches a single page of data from the remote resource
-   */
   async getData() {
     const { datasource } = this.options
 

packages/client/src/utils/fetch/RelationshipFetch.js

@@ -1,14 +1,7 @@
-import TableFetch from "./TableFetch.js"
+import DataFetch from "./DataFetch.js"
 import { fetchRelationshipData } from "api"
 
-export default class ViewFetch extends TableFetch {
+export default class ViewFetch extends DataFetch {
-  SupportsSearch = false
-  SupportsSort = false
-  SupportsPagination = false
-
-  /**
-   * Fetches a single page of data from the remote resource
-   */
   async getData() {
     const { datasource } = this.options
     const res = await fetchRelationshipData({

packages/client/src/utils/fetch/TableFetch.js

@@ -1,151 +1,20 @@
-import { writable, derived, get } from "svelte/store"
+import { get } from "svelte/store"
-import * as API from "api"
+import DataFetch from "./DataFetch.js"
-import { buildLuceneQuery } from "builder/src/helpers/lucene"
+import { searchTable } from "api"
-import { fetchTableDefinition } from "api"
 
-/**
+export default class TableFetch extends DataFetch {
- * Parent class which handles the implementation of fetching data from an
- * internal table or datasource plus.
- * For other types of datasource, this class is overridden and extended.
- */
-export default class TableFetch {
   SupportsSearch = true
   SupportsSort = true
   SupportsPagination = true
 
-  // Config
-  options = {
-    datasource: null,
-    schema: null,
-    limit: 10,
-
-    // Search config
-    filter: null,
-    query: null,
-
-    // Sorting config
-    sortColumn: null,
-    sortOrder: "ascending",
-
-    // Pagination config
-    paginate: true,
-  }
-
-  // State of the fetch
-  store = writable({
-    rows: [],
-    schema: null,
-    loading: false,
-    loaded: false,
-    query: null,
-    pageNumber: 0,
-    cursor: null,
-    cursors: [],
-  })
-
-  /**
-   * Constructs a new DataFetch instance.
-   * @param opts the fetch options
-   */
-  constructor(opts) {
-    this.options = {
-      ...this.options,
-      ...opts,
-    }
-
-    // Bind all functions to properly scope "this"
-    this.getData = this.getData.bind(this)
-    this.getInitialData = this.getInitialData.bind(this)
-    this.refresh = this.refresh.bind(this)
-    this.update = this.update.bind(this)
-    this.hasNextPage = this.hasNextPage.bind(this)
-    this.hasPrevPage = this.hasPrevPage.bind(this)
-    this.nextPage = this.nextPage.bind(this)
-    this.prevPage = this.prevPage.bind(this)
-
-    // Derive certain properties to return
-    this.derivedStore = derived(this.store, $store => {
-      return {
-        ...$store,
-        hasNextPage: this.hasNextPage($store),
-        hasPrevPage: this.hasPrevPage($store),
-      }
-    })
-
-    // Mark as loaded if we have no datasource
-    if (!this.options.datasource) {
-      this.store.update($store => ({ ...$store, loaded: true }))
-      return
-    }
-
-    // Initially fetch data but don't bother waiting for the result
-    this.getInitialData()
-  }
-
-  /**
-   * Extend the svelte store subscribe method to that instances of this class
-   * can be treated like stores
-   */
-  get subscribe() {
-    return this.derivedStore.subscribe
-  }
-
-  /**
-   * Fetches a fresh set of data from the server, resetting pagination
-   */
-  async getInitialData() {
-    const { datasource, filter } = this.options
-    const tableId = datasource?.tableId
-
-    // Ensure table ID exists
-    if (!tableId) {
-      return
-    }
-
-    // Ensure schema exists and enrich it
-    let schema = this.options.schema
-    if (!schema) {
-      schema = await this.constructor.getSchema(datasource)
-    }
-    if (!schema) {
-      return
-    }
-
-    // Build the lucene query
-    let query = this.options.query
-    if (!query) {
-      query = buildLuceneQuery(filter)
-    }
-
-    // Update store
-    this.store.update($store => ({ ...$store, schema, query, loading: true }))
-
-    // Actually fetch data
-    const page = await this.getData()
-    this.store.update($store => ({
-      ...$store,
-      loading: false,
-      loaded: true,
-      pageNumber: 0,
-      rows: page.rows,
-      cursors: page.hasNextPage ? [null, page.cursor] : [null],
-    }))
-  }
-
-  /**
-   * Fetches a single page of data from the remote resource
-   */
   async getData() {
-    const { datasource, limit, sortColumn, sortOrder, paginate } = this.options
+    const { datasource, limit, sortColumn, sortOrder, sortType, paginate } =
+      this.options
     const { tableId } = datasource
-    const { cursor, schema, query } = get(this.store)
+    const { cursor, query } = get(this.store)
-
-    // Work out what sort type to use
-    const type = schema?.[sortColumn]?.type
-    const sortType = type === "number" ? "number" : "string"
 
     // Search table
-    const res = await API.searchTable({
+    const res = await searchTable({
       tableId,
       query,
       limit,
@@ -161,164 +30,4 @@ export default class TableFetch {
       cursor: res?.bookmark || null,
     }
   }
-
-  /**
-   * Gets the schema definition for a table
-   * @param datasource the datasource definition
-   * @return {object} the schema
-   */
-  static async getSchema(datasource) {
-    if (!datasource?.tableId) {
-      return null
-    }
-    const table = await fetchTableDefinition(datasource.tableId)
-    return this.enrichSchema(table?.schema)
-  }
-
-  /**
-   * Enriches the schema and ensures that entries are objects with names
-   * @param schema the datasource schema
-   * @return {object} the enriched datasource schema
-   */
-  static enrichSchema(schema) {
-    if (schema == null) {
-      return null
-    }
-    let enrichedSchema = {}
-    Object.entries(schema).forEach(([fieldName, fieldSchema]) => {
-      if (typeof fieldSchema === "string") {
-        enrichedSchema[fieldName] = {
-          type: fieldSchema,
-          name: fieldName,
-        }
-      } else {
-        enrichedSchema[fieldName] = {
-          ...fieldSchema,
-          name: fieldName,
-        }
-      }
-    })
-    return enrichedSchema
-  }
-
-  /**
-   * Resets the data set and updates options
-   * @param newOptions any new options
-   */
-  async update(newOptions) {
-    // Check if any settings have actually changed
-    let refresh = false
-    const entries = Object.entries(newOptions || {})
-    for (let [key, value] of entries) {
-      if (JSON.stringify(value) !== JSON.stringify(this.options[key])) {
-        refresh = true
-        break
-      }
-    }
-    if (!refresh) {
-      return
-    }
-
-    // Assign new options and reload data
-    this.options = {
-      ...this.options,
-      ...newOptions,
-    }
-    await this.getInitialData()
-  }
-
-  /**
-   * Loads the same page again
-   */
-  async refresh() {
-    if (get(this.store).loading) {
-      return
-    }
-    const { rows } = await this.getData()
-    this.store.update($store => ({ ...$store, loading: true }))
-    this.store.update($store => ({ ...$store, rows, loading: false }))
-  }
-
-  /**
-   * Determines whether there is a next page of data based on the state of the
-   * store
-   * @param state the current store state
-   * @return {boolean} whether there is a next page of data or not
-   */
-  hasNextPage(state) {
-    return state.cursors[state.pageNumber + 1] != null
-  }
-
-  /**
-   * Determines whether there is a previous page of data based on the state of
-   * the store
-   * @param state the current store state
-   * @return {boolean} whether there is a previous page of data or not
-   */
-  hasPrevPage(state) {
-    return state.pageNumber > 0
-  }
-
-  /**
-   * Fetches the next page of data
-   */
-  async nextPage() {
-    const state = get(this.derivedStore)
-    if (state.loading || !this.options.paginate || !state.hasNextPage) {
-      return
-    }
-
-    // Fetch next page
-    const nextCursor = state.cursors[state.pageNumber + 1]
-    this.store.update($store => ({
-      ...$store,
-      loading: true,
-      cursor: nextCursor,
-    }))
-    const { rows, hasNextPage, cursor } = await this.getData()
-
-    // Update state
-    this.store.update($store => {
-      let { cursors, pageNumber } = $store
-      if (hasNextPage) {
-        cursors[pageNumber + 2] = cursor
-      }
-      return {
-        ...$store,
-        pageNumber: pageNumber + 1,
-        rows,
-        cursors,
-        loading: false,
-      }
-    })
-  }
-
-  /**
-   * Fetches the previous page of data
-   */
-  async prevPage() {
-    const state = get(this.derivedStore)
-    if (state.loading || !this.options.paginate || !state.hasPrevPage) {
-      return
-    }
-
-    // Fetch previous page
-    const prevCursor = state.cursors[state.pageNumber - 1]
-    this.store.update($store => ({
-      ...$store,
-      loading: true,
-      cursor: prevCursor,
-    }))
-    const { rows } = await this.getData()
-
-    // Update state
-    this.store.update($store => {
-      return {
-        ...$store,
-        pageNumber: $store.pageNumber - 1,
-        rows,
-        loading: false,
-      }
-    })
-  }
 }

packages/client/src/utils/fetch/ViewFetch.js

@@ -1,16 +1,7 @@
-import TableFetch from "./TableFetch.js"
+import DataFetch from "./DataFetch.js"
 import { fetchTableDefinition, fetchViewData } from "api"
 
-export default class ViewFetch extends TableFetch {
+export default class ViewFetch extends DataFetch {
-  SupportsSearch = false
-  SupportsSort = false
-  SupportsPagination = false
-
-  /**
-   * Fetches the schema for a view
-   * @param datasource the view datasource config
-   * @return {object} the view schema
-   */
   static async getSchema(datasource) {
     if (!datasource?.tableId) {
       return null
@@ -19,9 +10,6 @@ export default class ViewFetch extends TableFetch {
     return this.enrichSchema(table?.views?.[datasource.name]?.schema)
   }
 
-  /**
-   * Fetches a single page of data from the remote resource
-   */
   async getData() {
     const { datasource } = this.options
     const res = await fetchViewData(datasource)
@@ -30,5 +18,3 @@ export default class ViewFetch extends TableFetch {
     }
   }
 }
-
-ViewFetch.getSchema()