Add new core implementation of fetching paginated table data and enable pagination in backend UI for tables

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -1,6 +1,5 @@
 <script>
-  import { tables, views } from "stores/backend"
-
+  import { tables } from "stores/backend"
   import CreateRowButton from "./buttons/CreateRowButton.svelte"
   import CreateColumnButton from "./buttons/CreateColumnButton.svelte"
   import CreateViewButton from "./buttons/CreateViewButton.svelte"
@@ -8,53 +7,51 @@
   import EditRolesButton from "./buttons/EditRolesButton.svelte"
   import ManageAccessButton from "./buttons/ManageAccessButton.svelte"
   import HideAutocolumnButton from "./buttons/HideAutocolumnButton.svelte"
-  import * as api from "./api"
   import Table from "./Table.svelte"
   import { TableNames } from "constants"
   import CreateEditRow from "./modals/CreateEditRow.svelte"
+  import { fetchTableData } from "helpers/fetchTableData"
+  import { Pagination } from "@budibase/bbui"
 
   let hideAutocolumns = true
-  let data = []
-  let loading = false
+  const data = fetchTableData()
+
   $: isUsersTable = $tables.selected?._id === TableNames.USERS
   $: title = $tables.selected?.name
   $: schema = $tables.selected?.schema
-  $: tableView = {
-    schema,
-    name: $views.selected?.name,
-  }
   $: type = $tables.selected?.type
   $: isInternal = type !== "external"
 
-  // Fetch rows for specified table
-  $: {
-    loading = true
-    const loadingTableId = $tables.selected?._id
-    api.fetchDataForTable($tables.selected?._id).then(rows => {
-      loading = false
+  // Fetch data whenever table changes
+  $: data.update({
+    tableId: $tables.selected?._id,
+    schema,
+    limit: 10,
+    paginate: true,
+  })
 
-      // If we started a slow request then quickly change table, sometimes
-      // the old data overwrites the new data.
-      // This check ensures that we don't do that.
-      if (loadingTableId !== $tables.selected?._id) {
-        return
-      }
-
-      data = rows || []
+  // Fetch data whenever sorting option changes
+  const onSort = e => {
+    data.update({
+      sortColumn: e.detail.column,
+      sortOrder: e.detail.order,
     })
   }
 </script>
 
-<Table
+<div>
+  <Table
     {title}
     {schema}
-  tableId={$tables.selected?._id}
-  {data}
     {type}
-  allowEditing={true}
+    tableId={$tables.selected?._id}
+    data={$data.rows}
     bind:hideAutocolumns
-  {loading}
->
+    loading={$data.loading}
+    on:sort={onSort}
+    allowEditing
+    disableSorting
+  >
     {#if isInternal}
       <CreateColumnButton />
     {/if}
@@ -76,4 +73,24 @@
       <!-- always have the export last -->
       <ExportButton view={$tables.selected?._id} />
     {/if}
-</Table>
+  </Table>
+  <div class="pagination">
+    <Pagination
+      page={$data.pageNumber + 1}
+      hasPrevPage={$data.hasPrevPage}
+      hasNextPage={$data.hasNextPage}
+      goToPrevPage={$data.loading ? null : data.prevPage}
+      goToNextPage={$data.loading ? null : data.nextPage}
+    />
+  </div>
+</div>
+
+<style>
+  .pagination {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-end;
+    align-items: center;
+    margin-top: var(--spacing-xl);
+  }
+</style>

packages/builder/src/components/backend/DataTable/Table.svelte

@@ -1,8 +1,7 @@
 <script>
   import { fade } from "svelte/transition"
   import { goto, params } from "@roxi/routify"
-  import { Table, Modal, Heading, notifications } from "@budibase/bbui"
-
+  import { Table, Modal, Heading, notifications, Layout } from "@budibase/bbui"
   import api from "builderStore/api"
   import Spinner from "components/common/Spinner.svelte"
   import DeleteRowsButton from "./buttons/DeleteRowsButton.svelte"
@@ -21,6 +20,7 @@
   export let hideAutocolumns
   export let rowCount
   export let type
+  export let disableSorting = false
 
   let selectedRows = []
   let editableColumn
@@ -98,7 +98,8 @@
   }
 </script>
 
-<div>
+<Layout noPadding gap="S">
+  <div>
     <div class="table-title">
       {#if title}
         <Heading size="S">{title}</Heading>
@@ -115,14 +116,15 @@
         <DeleteRowsButton {selectedRows} {deleteRows} />
       {/if}
     </div>
-</div>
-{#key tableId}
+  </div>
+  {#key tableId}
     <Table
       {data}
       {schema}
       {loading}
       {customRenderers}
       {rowCount}
+      {disableSorting}
       bind:selectedRows
       allowSelectRows={allowEditing && !isUsersTable}
       allowEditRows={allowEditing}
@@ -131,8 +133,10 @@
       on:editcolumn={e => editColumn(e.detail)}
       on:editrow={e => editRow(e.detail)}
       on:clickrelationship={e => selectRelationship(e.detail)}
+      on:sort
     />
-{/key}
+  {/key}
+</Layout>
 
 <Modal bind:this={editRowModal}>
   <svelte:component this={editRowComponent} row={editableRow} />

packages/builder/src/helpers/fetchTableData.js

@@ -0,0 +1,192 @@
+import { writable, derived, get } from "svelte/store"
+import * as API from "builderStore/api"
+import { buildLuceneQuery } from "../../../client/src/utils/lucene"
+
+const defaultOptions = {
+  tableId: null,
+  filter: null,
+  limit: 10,
+  sortColumn: null,
+  sortOrder: "ascending",
+  paginate: true,
+  schema: null,
+}
+
+export const fetchTableData = opts => {
+  // Save option set so we can override it later rather than relying on params
+  let options = {
+    ...defaultOptions,
+    ...opts,
+  }
+
+  // Local non-observable state
+  let query
+  let sortType
+
+  // Local observable state
+  const store = writable({
+    rows: [],
+    schema: null,
+    loading: false,
+    loaded: false,
+    bookmarks: [],
+    pageNumber: 0,
+  })
+
+  // Derive certain properties to return
+  const derivedStore = derived(store, $store => {
+    return {
+      ...$store,
+      hasNextPage: $store.bookmarks[$store.pageNumber + 1] != null,
+      hasPrevPage: $store.pageNumber > 0,
+    }
+  })
+
+  const fetchPage = async bookmark => {
+    const { tableId, limit, sortColumn, sortOrder, paginate } = options
+    store.update($store => ({ ...$store, loading: true }))
+    const res = await API.post(`/api/${options.tableId}/search`, {
+      tableId,
+      query,
+      limit,
+      sort: sortColumn,
+      sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+      sortType,
+      paginate,
+      bookmark,
+    })
+    store.update($store => ({ ...$store, loading: false, loaded: true }))
+    return await res.json()
+  }
+
+  // Fetches a fresh set of results from the server
+  const fetchData = async () => {
+    const { tableId, schema, sortColumn, filter } = options
+
+    // Ensure table ID exists
+    if (!tableId) {
+      return
+    }
+
+    // Get and enrich schema.
+    // Ensure there are "name" properties for all fields and that field schema
+    // are objects
+    let enrichedSchema = schema
+    if (!enrichedSchema) {
+      const definition = await API.get(`/api/tables/${tableId}`)
+      enrichedSchema = definition?.schema ?? null
+    }
+    if (enrichedSchema) {
+      Object.entries(schema).forEach(([fieldName, fieldSchema]) => {
+        if (typeof fieldSchema === "string") {
+          enrichedSchema[fieldName] = {
+            type: fieldSchema,
+            name: fieldName,
+          }
+        } else {
+          enrichedSchema[fieldName] = {
+            ...fieldSchema,
+            name: fieldName,
+          }
+        }
+      })
+
+      // Save fixed schema so we can provide it later
+      options.schema = enrichedSchema
+    }
+
+    // Ensure schema exists
+    if (!schema) {
+      return
+    }
+    store.update($store => ({ ...$store, schema }))
+
+    // Work out what sort type to use
+    if (!sortColumn || !schema[sortColumn]) {
+      sortType = "string"
+    }
+    const type = schema?.[sortColumn]?.type
+    sortType = type === "number" ? "number" : "string"
+
+    // Build the lucene query
+    query = buildLuceneQuery(filter)
+
+    // Actually fetch data
+    const page = await fetchPage()
+    store.update($store => ({
+      ...$store,
+      loading: false,
+      loaded: true,
+      pageNumber: 0,
+      rows: page.rows,
+      bookmarks: page.hasNextPage ? [null, page.bookmark] : [null],
+    }))
+  }
+
+  // Fetches the next page of data
+  const nextPage = async () => {
+    const state = get(derivedStore)
+    if (!options.paginate || !state.hasNextPage) {
+      return
+    }
+
+    // Fetch next page
+    const page = await fetchPage(state.bookmarks[state.pageNumber + 1])
+
+    // Update state
+    store.update($store => {
+      let { bookmarks, pageNumber } = $store
+      if (page.hasNextPage) {
+        bookmarks[pageNumber + 2] = page.bookmark
+      }
+      return {
+        ...$store,
+        pageNumber: pageNumber + 1,
+        rows: page.rows,
+        bookmarks,
+      }
+    })
+  }
+
+  // Fetches the previous page of data
+  const prevPage = async () => {
+    const state = get(derivedStore)
+    if (!options.paginate || !state.hasPrevPage) {
+      return
+    }
+
+    // Fetch previous page
+    const page = await fetchPage(state.bookmarks[state.pageNumber - 1])
+
+    // Update state
+    store.update($store => {
+      return {
+        ...$store,
+        pageNumber: $store.pageNumber - 1,
+        rows: page.rows,
+      }
+    })
+  }
+
+  // Resets the data set and updates options
+  const update = async newOptions => {
+    if (newOptions) {
+      options = {
+        ...options,
+        ...newOptions,
+      }
+    }
+    await fetchData()
+  }
+
+  // Initially fetch data but don't bother waiting for the result
+  fetchData()
+
+  // Return our derived store which will be updated over time
+  return {
+    subscribe: derivedStore.subscribe,
+    nextPage,
+    prevPage,
+    update,
+  }
+}