Add new core implementation of fetching paginated table data and enable pagination in backend UI for tables

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -1,6 +1,5 @@
 <script>
-  import { tables, views } from "stores/backend"
+  import { tables } from "stores/backend"
-
   import CreateRowButton from "./buttons/CreateRowButton.svelte"
   import CreateColumnButton from "./buttons/CreateColumnButton.svelte"
   import CreateViewButton from "./buttons/CreateViewButton.svelte"
@@ -8,72 +7,90 @@
   import EditRolesButton from "./buttons/EditRolesButton.svelte"
   import ManageAccessButton from "./buttons/ManageAccessButton.svelte"
   import HideAutocolumnButton from "./buttons/HideAutocolumnButton.svelte"
-  import * as api from "./api"
   import Table from "./Table.svelte"
   import { TableNames } from "constants"
   import CreateEditRow from "./modals/CreateEditRow.svelte"
+  import { fetchTableData } from "helpers/fetchTableData"
+  import { Pagination } from "@budibase/bbui"
 
   let hideAutocolumns = true
-  let data = []
+  const data = fetchTableData()
-  let loading = false
+
   $: isUsersTable = $tables.selected?._id === TableNames.USERS
   $: title = $tables.selected?.name
   $: schema = $tables.selected?.schema
-  $: tableView = {
-    schema,
-    name: $views.selected?.name,
-  }
   $: type = $tables.selected?.type
   $: isInternal = type !== "external"
 
-  // Fetch rows for specified table
+  // Fetch data whenever table changes
-  $: {
+  $: data.update({
-    loading = true
+    tableId: $tables.selected?._id,
-    const loadingTableId = $tables.selected?._id
+    schema,
-    api.fetchDataForTable($tables.selected?._id).then(rows => {
+    limit: 10,
-      loading = false
+    paginate: true,
+  })
 
-      // If we started a slow request then quickly change table, sometimes
+  // Fetch data whenever sorting option changes
-      // the old data overwrites the new data.
+  const onSort = e => {
-      // This check ensures that we don't do that.
+    data.update({
-      if (loadingTableId !== $tables.selected?._id) {
+      sortColumn: e.detail.column,
-        return
+      sortOrder: e.detail.order,
-      }
-
-      data = rows || []
     })
   }
 </script>
 
-<Table
+<div>
-  {title}
+  <Table
-  {schema}
+    {title}
-  tableId={$tables.selected?._id}
+    {schema}
-  {data}
+    {type}
-  {type}
+    tableId={$tables.selected?._id}
-  allowEditing={true}
+    data={$data.rows}
-  bind:hideAutocolumns
+    bind:hideAutocolumns
-  {loading}
+    loading={$data.loading}
->
+    on:sort={onSort}
-  {#if isInternal}
+    allowEditing
-    <CreateColumnButton />
+    disableSorting
-  {/if}
+  >
-  {#if schema && Object.keys(schema).length > 0}
-    {#if !isUsersTable}
-      <CreateRowButton
-        title={"Create row"}
-        modalContentComponent={CreateEditRow}
-      />
-    {/if}
     {#if isInternal}
-      <CreateViewButton />
+      <CreateColumnButton />
     {/if}
-    <ManageAccessButton resourceId={$tables.selected?._id} />
+    {#if schema && Object.keys(schema).length > 0}
-    {#if isUsersTable}
+      {#if !isUsersTable}
-      <EditRolesButton />
+        <CreateRowButton
+          title={"Create row"}
+          modalContentComponent={CreateEditRow}
+        />
+      {/if}
+      {#if isInternal}
+        <CreateViewButton />
+      {/if}
+      <ManageAccessButton resourceId={$tables.selected?._id} />
+      {#if isUsersTable}
+        <EditRolesButton />
+      {/if}
+      <HideAutocolumnButton bind:hideAutocolumns />
+      <!-- always have the export last -->
+      <ExportButton view={$tables.selected?._id} />
     {/if}
-    <HideAutocolumnButton bind:hideAutocolumns />
+  </Table>
-    <!-- always have the export last -->
+  <div class="pagination">
-    <ExportButton view={$tables.selected?._id} />
+    <Pagination
-  {/if}
+      page={$data.pageNumber + 1}
-</Table>
+      hasPrevPage={$data.hasPrevPage}
+      hasNextPage={$data.hasNextPage}
+      goToPrevPage={$data.loading ? null : data.prevPage}
+      goToNextPage={$data.loading ? null : data.nextPage}
+    />
+  </div>
+</div>
+
+<style>
+  .pagination {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-end;
+    align-items: center;
+    margin-top: var(--spacing-xl);
+  }
+</style>

packages/builder/src/components/backend/DataTable/Table.svelte

@@ -1,8 +1,7 @@
 <script>
   import { fade } from "svelte/transition"
   import { goto, params } from "@roxi/routify"
-  import { Table, Modal, Heading, notifications } from "@budibase/bbui"
+  import { Table, Modal, Heading, notifications, Layout } from "@budibase/bbui"
-
   import api from "builderStore/api"
   import Spinner from "components/common/Spinner.svelte"
   import DeleteRowsButton from "./buttons/DeleteRowsButton.svelte"
@@ -21,6 +20,7 @@
   export let hideAutocolumns
   export let rowCount
   export let type
+  export let disableSorting = false
 
   let selectedRows = []
   let editableColumn
@@ -98,41 +98,45 @@
   }
 </script>
 
-<div>
+<Layout noPadding gap="S">
-  <div class="table-title">
+  <div>
-    {#if title}
+    <div class="table-title">
-      <Heading size="S">{title}</Heading>
+      {#if title}
-    {/if}
+        <Heading size="S">{title}</Heading>
-    {#if loading}
+      {/if}
-      <div transition:fade>
+      {#if loading}
-        <Spinner size="10" />
+        <div transition:fade>
-      </div>
+          <Spinner size="10" />
-    {/if}
+        </div>
+      {/if}
+    </div>
+    <div class="popovers">
+      <slot />
+      {#if !isUsersTable && selectedRows.length > 0}
+        <DeleteRowsButton {selectedRows} {deleteRows} />
+      {/if}
+    </div>
   </div>
-  <div class="popovers">
+  {#key tableId}
-    <slot />
+    <Table
-    {#if !isUsersTable && selectedRows.length > 0}
+      {data}
-      <DeleteRowsButton {selectedRows} {deleteRows} />
+      {schema}
-    {/if}
+      {loading}
-  </div>
+      {customRenderers}
-</div>
+      {rowCount}
-{#key tableId}
+      {disableSorting}
-  <Table
+      bind:selectedRows
-    {data}
+      allowSelectRows={allowEditing && !isUsersTable}
-    {schema}
+      allowEditRows={allowEditing}
-    {loading}
+      allowEditColumns={allowEditing && isInternal}
-    {customRenderers}
+      showAutoColumns={!hideAutocolumns}
-    {rowCount}
+      on:editcolumn={e => editColumn(e.detail)}
-    bind:selectedRows
+      on:editrow={e => editRow(e.detail)}
-    allowSelectRows={allowEditing && !isUsersTable}
+      on:clickrelationship={e => selectRelationship(e.detail)}
-    allowEditRows={allowEditing}
+      on:sort
-    allowEditColumns={allowEditing && isInternal}
+    />
-    showAutoColumns={!hideAutocolumns}
+  {/key}
-    on:editcolumn={e => editColumn(e.detail)}
+</Layout>
-    on:editrow={e => editRow(e.detail)}
-    on:clickrelationship={e => selectRelationship(e.detail)}
-  />
-{/key}
 
 <Modal bind:this={editRowModal}>
   <svelte:component this={editRowComponent} row={editableRow} />

packages/builder/src/helpers/fetchTableData.js

@@ -0,0 +1,192 @@
+import { writable, derived, get } from "svelte/store"
+import * as API from "builderStore/api"
+import { buildLuceneQuery } from "../../../client/src/utils/lucene"
+
+const defaultOptions = {
+  tableId: null,
+  filter: null,
+  limit: 10,
+  sortColumn: null,
+  sortOrder: "ascending",
+  paginate: true,
+  schema: null,
+}
+
+export const fetchTableData = opts => {
+  // Save option set so we can override it later rather than relying on params
+  let options = {
+    ...defaultOptions,
+    ...opts,
+  }
+
+  // Local non-observable state
+  let query
+  let sortType
+
+  // Local observable state
+  const store = writable({
+    rows: [],
+    schema: null,
+    loading: false,
+    loaded: false,
+    bookmarks: [],
+    pageNumber: 0,
+  })
+
+  // Derive certain properties to return
+  const derivedStore = derived(store, $store => {
+    return {
+      ...$store,
+      hasNextPage: $store.bookmarks[$store.pageNumber + 1] != null,
+      hasPrevPage: $store.pageNumber > 0,
+    }
+  })
+
+  const fetchPage = async bookmark => {
+    const { tableId, limit, sortColumn, sortOrder, paginate } = options
+    store.update($store => ({ ...$store, loading: true }))
+    const res = await API.post(`/api/${options.tableId}/search`, {
+      tableId,
+      query,
+      limit,
+      sort: sortColumn,
+      sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+      sortType,
+      paginate,
+      bookmark,
+    })
+    store.update($store => ({ ...$store, loading: false, loaded: true }))
+    return await res.json()
+  }
+
+  // Fetches a fresh set of results from the server
+  const fetchData = async () => {
+    const { tableId, schema, sortColumn, filter } = options
+
+    // Ensure table ID exists
+    if (!tableId) {
+      return
+    }
+
+    // Get and enrich schema.
+    // Ensure there are "name" properties for all fields and that field schema
+    // are objects
+    let enrichedSchema = schema
+    if (!enrichedSchema) {
+      const definition = await API.get(`/api/tables/${tableId}`)
+      enrichedSchema = definition?.schema ?? null
+    }
+    if (enrichedSchema) {
+      Object.entries(schema).forEach(([fieldName, fieldSchema]) => {
+        if (typeof fieldSchema === "string") {
+          enrichedSchema[fieldName] = {
+            type: fieldSchema,
+            name: fieldName,
+          }
+        } else {
+          enrichedSchema[fieldName] = {
+            ...fieldSchema,
+            name: fieldName,
+          }
+        }
+      })
+
+      // Save fixed schema so we can provide it later
+      options.schema = enrichedSchema
+    }
+
+    // Ensure schema exists
+    if (!schema) {
+      return
+    }
+    store.update($store => ({ ...$store, schema }))
+
+    // Work out what sort type to use
+    if (!sortColumn || !schema[sortColumn]) {
+      sortType = "string"
+    }
+    const type = schema?.[sortColumn]?.type
+    sortType = type === "number" ? "number" : "string"
+
+    // Build the lucene query
+    query = buildLuceneQuery(filter)
+
+    // Actually fetch data
+    const page = await fetchPage()
+    store.update($store => ({
+      ...$store,
+      loading: false,
+      loaded: true,
+      pageNumber: 0,
+      rows: page.rows,
+      bookmarks: page.hasNextPage ? [null, page.bookmark] : [null],
+    }))
+  }
+
+  // Fetches the next page of data
+  const nextPage = async () => {
+    const state = get(derivedStore)
+    if (!options.paginate || !state.hasNextPage) {
+      return
+    }
+
+    // Fetch next page
+    const page = await fetchPage(state.bookmarks[state.pageNumber + 1])
+
+    // Update state
+    store.update($store => {
+      let { bookmarks, pageNumber } = $store
+      if (page.hasNextPage) {
+        bookmarks[pageNumber + 2] = page.bookmark
+      }
+      return {
+        ...$store,
+        pageNumber: pageNumber + 1,
+        rows: page.rows,
+        bookmarks,
+      }
+    })
+  }
+
+  // Fetches the previous page of data
+  const prevPage = async () => {
+    const state = get(derivedStore)
+    if (!options.paginate || !state.hasPrevPage) {
+      return
+    }
+
+    // Fetch previous page
+    const page = await fetchPage(state.bookmarks[state.pageNumber - 1])
+
+    // Update state
+    store.update($store => {
+      return {
+        ...$store,
+        pageNumber: $store.pageNumber - 1,
+        rows: page.rows,
+      }
+    })
+  }
+
+  // Resets the data set and updates options
+  const update = async newOptions => {
+    if (newOptions) {
+      options = {
+        ...options,
+        ...newOptions,
+      }
+    }
+    await fetchData()
+  }
+
+  // Initially fetch data but don't bother waiting for the result
+  fetchData()
+
+  // Return our derived store which will be updated over time
+  return {
+    subscribe: derivedStore.subscribe,
+    nextPage,
+    prevPage,
+    update,
+  }
+}