Allow multipage searches and implement optional pagination to data providers

packages/client/src/api/tables.js

@@ -17,24 +17,6 @@ export const fetchTableData = async tableId => {
   return await enrichRows(rows, tableId)
 }
 
-/**
- * Perform a mango query against an internal table
- * @param {String} tableId - id of the table to search
- * @param {Object} search - Mango Compliant search object
- * @param {Object} pagination - the pagination controls
- */
-export const searchTableData = async ({ tableId, search, pagination }) => {
-  const output = await API.post({
-    url: `/api/${tableId}/rows/search`,
-    body: {
-      query: search,
-      pagination,
-    },
-  })
-  output.rows = await enrichRows(output.rows, tableId)
-  return output
-}
-
 /**
  * Searches a table using Lucene.
  */
@@ -47,6 +29,7 @@ export const searchTable = async ({
   sort,
   sortOrder,
   sortType,
+  paginate,
 }) => {
   if (!tableId || (!query && !raw)) {
     return
@@ -61,10 +44,11 @@ export const searchTable = async ({
       sort,
       sortOrder,
       sortType,
+      paginate,
     },
   })
   return {
+    ...res,
     rows: await enrichRows(res?.rows, tableId),
-    bookmark: res.bookmark,
   }
 }

packages/server/src/api/controllers/row.js

@@ -16,7 +16,6 @@ const {
 const { FieldTypes } = require("../../constants")
 const { isEqual } = require("lodash")
 const { cloneDeep } = require("lodash/fp")
-const { QueryBuilder, search } = require("./search/utils")
 
 const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}`
 
@@ -248,45 +247,6 @@ exports.fetchView = async function (ctx) {
   }
 }
 
-exports.search = async function (ctx) {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
-  const {
-    query,
-    pagination: { pageSize = 10, bookmark },
-  } = ctx.request.body
-  const tableId = ctx.params.tableId
-
-  const queryBuilder = new QueryBuilder(appId)
-    .setLimit(pageSize)
-    .addTable(tableId)
-  if (bookmark) {
-    queryBuilder.setBookmark(bookmark)
-  }
-
-  let searchString
-  if (ctx.query && ctx.query.raw && ctx.query.raw !== "") {
-    searchString = queryBuilder.complete(query["RAW"])
-  } else {
-    // make all strings a starts with operation rather than pure equality
-    for (const [key, queryVal] of Object.entries(query)) {
-      if (typeof queryVal === "string") {
-        queryBuilder.addString(key, queryVal)
-      } else {
-        queryBuilder.addEqual(key, queryVal)
-      }
-    }
-    searchString = queryBuilder.complete()
-  }
-
-  const response = await search(searchString)
-  const table = await db.get(tableId)
-  ctx.body = {
-    rows: await outputProcessing(appId, table, response.rows),
-    bookmark: response.bookmark,
-  }
-}
-
 exports.fetchTableRows = async function (ctx) {
   const appId = ctx.appId
   const db = new CouchDB(appId)

packages/server/src/api/controllers/search/index.js

@@ -1,4 +1,4 @@
-const { QueryBuilder, buildSearchUrl, search } = require("./utils")
+const { fullSearch, paginatedSearch } = require("./utils")
 const CouchDB = require("../../../db")
 const { outputProcessing } = require("../../../utilities/rowProcessor")
 
@@ -8,38 +8,45 @@ exports.rowSearch = async ctx => {
   const {
     bookmark,
     query,
-    raw,
     limit,
     sort,
     sortOrder,
     sortType,
+    paginate,
   } = ctx.request.body
   const db = new CouchDB(appId)
 
-  let url
+  let response
-  if (query) {
+  const start = Date.now()
-    url = new QueryBuilder(
+  if (paginate) {
+    response = await paginatedSearch(
       appId,
       query,
-      bookmark,
+      tableId,
-      limit,
       sort,
       sortOrder,
-      sortType
+      sortType,
+      limit,
+      bookmark
     )
-      .addTable(tableId)
+  } else {
-      .complete()
+    response = await fullSearch(
-  } else if (raw) {
-    url = buildSearchUrl({
       appId,
-      query: raw,
+      query,
-      bookmark,
+      tableId,
-    })
+      sort,
+      sortOrder,
+      sortType,
+      limit
+    )
   }
-  const response = await search(url)
+  const end = Date.now()
+  console.log("Time: " + (end - start) / 1000 + " ms")
+
+  if (response.rows && response.rows.length) {
     const table = await db.get(tableId)
-  ctx.body = {
+    response.rows = await outputProcessing(appId, table, response.rows)
-    rows: await outputProcessing(appId, table, response.rows),
-    bookmark: response.bookmark,
   }
+
+  ctx.body = response
 }

packages/server/src/api/controllers/search/utils.js

@@ -3,51 +3,12 @@ const { checkSlashesInUrl } = require("../../../utilities")
 const env = require("../../../environment")
 const fetch = require("node-fetch")
 
-/**
- * Given a set of inputs this will generate the URL which is to be sent to the search proxy in CouchDB.
- * @param {string} appId The ID of the app which we will be searching within.
- * @param {string} query The lucene query string which is to be used for searching.
- * @param {string|null} bookmark If there were more than the limit specified can send the bookmark that was
- * returned with query for next set of search results.
- * @param {number} limit The number of entries to return per query.
- * @param {string} sort The column to sort by.
- * @param {string} sortOrder The order to sort by. "ascending" or "descending".
- * @param {string} sortType The type of sort to perform. "string" or "number".
- * @param {boolean} excludeDocs By default full rows are returned, if required this can be disabled.
- * @return {string} The URL which a GET can be performed on to receive results.
- */
-function buildSearchUrl({
-  appId,
-  query,
-  bookmark,
-  sort,
-  sortOrder,
-  sortType,
-  excludeDocs,
-  limit = 50,
-}) {
-  let url = `${env.COUCH_DB_URL}/${appId}/_design/database/_search`
-  url += `/${SearchIndexes.ROWS}?q=${query}`
-  url += `&limit=${Math.min(limit, 200)}`
-  if (!excludeDocs) {
-    url += "&include_docs=true"
-  }
-  if (sort) {
-    const orderChar = sortOrder === "descending" ? "-" : ""
-    url += `&sort="${orderChar}${sort.replace(/ /, "_")}<${sortType}>"`
-  }
-  if (bookmark) {
-    url += `&bookmark=${bookmark}`
-  }
-  return checkSlashesInUrl(url)
-}
-
 const luceneEscape = value => {
   return `${value}`.replace(/[ #+\-&|!(){}\[\]^"~*?:\\]/g, "\\$&")
 }
 
 class QueryBuilder {
-  constructor(appId, base, bookmark, limit, sort, sortOrder, sortType) {
+  constructor(appId, base) {
     this.appId = appId
     this.query = {
       string: {},
@@ -59,11 +20,14 @@ class QueryBuilder {
       notEmpty: {},
       ...base,
     }
-    this.bookmark = bookmark
+    this.limit = 50
-    this.limit = limit || 50
+    this.sortOrder = "ascending"
-    this.sort = sort
+    this.sortType = "string"
-    this.sortOrder = sortOrder || "ascending"
+  }
-    this.sortType = sortType || "string"
+
+  setTable(tableId) {
+    this.query.equal.tableId = tableId
+    return this
   }
 
   setLimit(limit) {
@@ -71,6 +35,21 @@ class QueryBuilder {
     return this
   }
 
+  setSort(sort) {
+    this.sort = sort
+    return this
+  }
+
+  setSortOrder(sortOrder) {
+    this.sortOrder = sortOrder
+    return this
+  }
+
+  setSortType(sortType) {
+    this.sortType = sortType
+    return this
+  }
+
   setBookmark(bookmark) {
     this.bookmark = bookmark
     return this
@@ -114,12 +93,7 @@ class QueryBuilder {
     return this
   }
 
-  addTable(tableId) {
+  buildSearchURL(excludeDocs = false) {
-    this.query.equal.tableId = tableId
-    return this
-  }
-
-  complete(rawQuery = null) {
     let output = "*:*"
     function build(structure, queryFn) {
       for (let [key, value] of Object.entries(structure)) {
@@ -171,22 +145,28 @@ class QueryBuilder {
     if (this.query.notEmpty) {
       build(this.query.notEmpty, key => `${key}:["" TO *]`)
     }
-    if (rawQuery) {
+
-      output = output.length === 0 ? rawQuery : `&${rawQuery}`
+    let url = `${env.COUCH_DB_URL}/${this.appId}/_design/database/_search`
+    url += `/${SearchIndexes.ROWS}?q=${output}`
+    url += `&limit=${Math.min(this.limit, 200)}`
+    if (!excludeDocs) {
+      url += "&include_docs=true"
     }
-    return buildSearchUrl({
+    if (this.sort) {
-      appId: this.appId,
+      const orderChar = this.sortOrder === "descending" ? "-" : ""
-      query: output,
+      url += `&sort="${orderChar}${this.sort.replace(/ /, "_")}<${
-      bookmark: this.bookmark,
+        this.sortType
-      limit: this.limit,
+      }>"`
-      sort: this.sort,
+    }
-      sortOrder: this.sortOrder,
+    if (this.bookmark) {
-      sortType: this.sortType,
+      url += `&bookmark=${this.bookmark}`
-    })
+    }
+    console.log(url)
+    return checkSlashesInUrl(url)
   }
 }
 
-exports.search = async query => {
+const runQuery = async query => {
   const response = await fetch(query, {
     method: "GET",
   })
@@ -203,5 +183,101 @@ exports.search = async query => {
   return output
 }
 
-exports.QueryBuilder = QueryBuilder
+const recursiveSearch = async (
-exports.buildSearchUrl = buildSearchUrl
+  appId,
+  query,
+  tableId,
+  sort,
+  sortOrder,
+  sortType,
+  limit,
+  bookmark,
+  rows
+) => {
+  if (rows.length >= limit) {
+    return rows
+  }
+  const pageSize = rows.length > limit - 200 ? limit - rows.length : 200
+  const url = new QueryBuilder(appId, query)
+    .setTable(tableId)
+    .setBookmark(bookmark)
+    .setLimit(pageSize)
+    .setSort(sort)
+    .setSortOrder(sortOrder)
+    .setSortType(sortType)
+    .buildSearchURL()
+  const page = await runQuery(url)
+  if (!page.rows.length) {
+    return rows
+  }
+  if (page.rows.length < 200) {
+    return [...rows, ...page.rows]
+  }
+  return await recursiveSearch(
+    appId,
+    query,
+    tableId,
+    sort,
+    sortOrder,
+    sortType,
+    limit,
+    page.bookmark,
+    [...rows, ...page.rows]
+  )
+}
+
+exports.paginatedSearch = async (
+  appId,
+  query,
+  tableId,
+  sort,
+  sortOrder,
+  sortType,
+  limit,
+  bookmark
+) => {
+  if (limit == null || isNaN(limit) || limit < 0) {
+    limit = 50
+  }
+  const builder = new QueryBuilder(appId, query)
+    .setTable(tableId)
+    .setSort(sort)
+    .setSortOrder(sortOrder)
+    .setSortType(sortType)
+    .setBookmark(bookmark)
+    .setLimit(limit)
+  const searchUrl = builder.buildSearchURL()
+  const nextUrl = builder.setLimit(1).buildSearchURL()
+  const searchResults = await runQuery(searchUrl)
+  const nextResults = await runQuery(nextUrl)
+  return {
+    ...searchResults,
+    hasNextPage: nextResults.rows && nextResults.rows.length > 0,
+  }
+}
+
+exports.fullSearch = async (
+  appId,
+  query,
+  tableId,
+  sort,
+  sortOrder,
+  sortType,
+  limit
+) => {
+  if (limit == null || isNaN(limit) || limit < 0) {
+    limit = 1000
+  }
+  const rows = await recursiveSearch(
+    appId,
+    query,
+    tableId,
+    sort,
+    sortOrder,
+    sortType,
+    Math.min(limit, 1000),
+    null,
+    []
+  )
+  return { rows }
+}

packages/server/src/api/routes/row.js

@@ -39,12 +39,6 @@ router
     usage,
     rowController.save
   )
-  .post(
-    "/api/:tableId/rows/search",
-    paramResource("tableId"),
-    authorized(PermissionTypes.TABLE, PermissionLevels.READ),
-    rowController.search
-  )
   .patch(
     "/api/:tableId/rows/:rowId",
     paramSubResource("tableId", "rowId"),

packages/standard-components/manifest.json

@@ -65,41 +65,6 @@
       "type": "schema"
     }
   },
-  "search": {
-    "name": "Search",
-    "description": "A searchable list of items.",
-    "icon": "Search",
-    "styleable": true,
-    "hasChildren": true,
-    "settings": [
-      {
-        "type": "table",
-        "label": "Table",
-        "key": "table"
-      },
-      {
-        "type": "multifield",
-        "label": "Columns",
-        "key": "columns",
-        "dependsOn": "table"
-      },
-      {
-        "type": "number",
-        "label": "Rows/Page",
-        "defaultValue": 25,
-        "key": "pageSize"
-      },
-      {
-        "type": "text",
-        "label": "Empty Text",
-        "key": "noRowsMessage",
-        "defaultValue": "No rows found."
-      }
-    ],
-    "context": {
-      "type": "schema"
-    }
-  },
   "stackedlist": {
     "name": "Stacked List",
     "icon": "TaskList",
@@ -1446,7 +1411,14 @@
       {
         "type": "number",
         "label": "Limit",
-        "key": "limit"
+        "key": "limit",
+        "defaultValue": 50
+      },
+      {
+        "type": "boolean",
+        "label": "Paginate",
+        "key": "paginate",
+        "defaultValue": true
       }
     ],
     "context": {
@@ -1464,14 +1436,6 @@
           "label": "Schema",
           "key": "schema"
         },
-        {
-          "label": "Loading",
-          "key": "loading"
-        },
-        {
-          "label": "Loaded",
-          "key": "loaded"
-        },
         {
           "label": "Page Number",
           "key": "pageNumber"

packages/standard-components/src/DataProvider.svelte

@@ -1,11 +1,13 @@
 <script>
   import { getContext } from "svelte"
+  import { ProgressCircle, Pagination } from "@budibase/bbui"
 
   export let dataSource
   export let filter
   export let sortColumn
   export let sortOrder
-  export let limit = 50
+  export let limit
+  export let paginate
 
   const { API, styleable, Provider, ActionTypes } = getContext("sdk")
   const component = getContext("component")
@@ -29,7 +31,15 @@
   $: hasPrevPage = pageNumber > 0
   $: getSchema(dataSource)
   $: sortType = getSortType(schema, sortColumn)
-  $: fetchData(dataSource, query, limit, sortColumn, sortOrder, sortType)
+  $: fetchData(
+    dataSource,
+    query,
+    limit,
+    sortColumn,
+    sortOrder,
+    sortType,
+    paginate
+  )
   $: {
     if (internalTable) {
       rows = allRows
@@ -110,8 +120,10 @@
     limit,
     sortColumn,
     sortOrder,
-    sortType
+    sortType,
+    paginate
   ) => {
+    console.log("FETCH")
     loading = true
     if (dataSource?.type === "table") {
       const res = await API.searchTable({
@@ -121,21 +133,11 @@
         sort: sortColumn,
         sortOrder: sortOrder?.toLowerCase() ?? "ascending",
         sortType,
+        paginate,
       })
       pageNumber = 0
       allRows = res.rows
-
+      if (res.hasNextPage) {
-      // Check we have next data
-      const next = await API.searchTable({
-        tableId: dataSource.tableId,
-        query,
-        limit: 1,
-        bookmark: res.bookmark,
-        sort: sortColumn,
-        sortOrder: sortOrder?.toLowerCase() ?? "ascending",
-        sortType,
-      })
-      if (next.rows?.length) {
         bookmarks = [null, res.bookmark]
       } else {
         bookmarks = [null]
@@ -224,21 +226,11 @@
       sort: sortColumn,
       sortOrder: sortOrder?.toLowerCase() ?? "ascending",
       sortType,
+      paginate: true,
     })
     pageNumber++
     allRows = res.rows
-
+    if (res.hasNextPage) {
-    // Check we have next data
-    const next = await API.searchTable({
-      tableId: dataSource.tableId,
-      query,
-      limit: 1,
-      bookmark: res.bookmark,
-      sort: sortColumn,
-      sortOrder: sortOrder?.toLowerCase() ?? "ascending",
-      sortType,
-    })
-    if (next.rows?.length) {
       bookmarks[pageNumber + 1] = res.bookmark
     }
   }
@@ -255,14 +247,55 @@
       sort: sortColumn,
       sortOrder: sortOrder?.toLowerCase() ?? "ascending",
       sortType,
+      paginate: true,
     })
     pageNumber--
     allRows = res.rows
   }
 </script>
 
-<div use:styleable={$component.styles}>
+<div use:styleable={$component.styles} class="container">
   <Provider {actions} data={dataContext}>
+    {#if !loaded && loading}
+      <div class="loading">
+        <ProgressCircle />
+      </div>
+    {:else}
       <slot />
+      {#if paginate}
+        <div class="pagination">
+          <Pagination
+            page={pageNumber + 1}
+            {hasPrevPage}
+            {hasNextPage}
+            goToPrevPage={prevPage}
+            goToNextPage={nextPage}
+          />
+        </div>
+      {/if}
+    {/if}
   </Provider>
 </div>
+
+<style>
+  .container {
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: stretch;
+  }
+  .loading {
+    display: flex;
+    flex-direction: row;
+    justify-content: center;
+    align-items: center;
+    height: 100px;
+  }
+  .pagination {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-end;
+    align-items: center;
+    margin-top: var(--spacing-xl);
+  }
+</style>

packages/standard-components/src/Search.svelte

@@ -1,195 +0,0 @@
-<script>
-  import { getContext } from "svelte"
-  import {
-    Button,
-    DatePicker,
-    Label,
-    Select,
-    Toggle,
-    Input,
-  } from "@budibase/bbui"
-
-  const { API, styleable, Provider, builderStore, ActionTypes } = getContext(
-    "sdk"
-  )
-  const component = getContext("component")
-
-  export let table
-  export let columns = []
-  export let pageSize
-  export let noRowsMessage
-
-  let rows = []
-  let loaded = false
-  let search = {}
-  let tableDefinition
-  let schema
-
-  let nextBookmark = null
-  let bookmark = null
-  let lastBookmark = null
-
-  $: fetchData(table, bookmark)
-  // omit empty strings
-  $: parsedSearch = Object.keys(search).reduce(
-    (acc, next) =>
-      search[next] === "" ? acc : { ...acc, [next]: search[next] },
-    {}
-  )
-  $: actions = [
-    {
-      type: ActionTypes.RefreshDatasource,
-      callback: () => fetchData(table, bookmark),
-      metadata: { datasource: { type: "table", tableId: table } },
-    },
-  ]
-
-  async function fetchData(table, mark) {
-    if (table) {
-      const tableDef = await API.fetchTableDefinition(table)
-      schema = tableDef.schema
-      const output = await API.searchTableData({
-        tableId: table,
-        search: parsedSearch,
-        pagination: {
-          pageSize,
-          bookmark: mark,
-        },
-      })
-      rows = output.rows
-      nextBookmark = output.bookmark
-    }
-    loaded = true
-  }
-
-  function nextPage() {
-    lastBookmark = bookmark
-    bookmark = nextBookmark
-  }
-
-  function previousPage() {
-    nextBookmark = bookmark
-    if (lastBookmark !== bookmark) {
-      bookmark = lastBookmark
-    } else {
-      // special case for going back to beginning
-      bookmark = null
-      lastBookmark = null
-    }
-  }
-</script>
-
-<Provider {actions}>
-  <div use:styleable={$component.styles}>
-    <div class="query-builder">
-      {#if schema}
-        {#each columns as field}
-          <div class="form-field">
-            <Label extraSmall grey>{schema[field].name}</Label>
-            {#if schema[field].type === "options"}
-              <Select secondary bind:value={search[field]}>
-                <option value="">Choose an option</option>
-                {#each schema[field].constraints.inclusion as opt}
-                  <option>{opt}</option>
-                {/each}
-              </Select>
-            {:else if schema[field].type === "datetime"}
-              <DatePicker bind:value={search[field]} />
-            {:else if schema[field].type === "boolean"}
-              <Toggle text={schema[field].name} bind:checked={search[field]} />
-            {:else if schema[field].type === "number"}
-              <Input type="number" bind:value={search[field]} />
-            {:else if schema[field].type === "string"}
-              <Input bind:value={search[field]} />
-            {/if}
-          </div>
-        {/each}
-      {/if}
-      <div class="actions">
-        <Button
-          secondary
-          on:click={() => {
-            search = {}
-            bookmark = null
-          }}
-        >
-          Reset
-        </Button>
-        <Button
-          primary
-          on:click={() => {
-            bookmark = null
-            fetchData(table, bookmark)
-          }}
-        >
-          Search
-        </Button>
-      </div>
-    </div>
-    {#if loaded}
-      {#if rows.length > 0}
-        {#if $component.children === 0 && $builderStore.inBuilder}
-          <p><i class="ri-image-line" />Add some components to display.</p>
-        {:else}
-          {#each rows as row}
-            <Provider data={row}>
-              <slot />
-            </Provider>
-          {/each}
-        {/if}
-      {:else if noRowsMessage}
-        <p><i class="ri-search-2-line" />{noRowsMessage}</p>
-      {/if}
-    {/if}
-    <div class="pagination">
-      {#if lastBookmark != null || bookmark != null}
-        <Button primary on:click={previousPage}>Back</Button>
-      {/if}
-      {#if nextBookmark != null && rows.length !== 0}
-        <Button primary on:click={nextPage}>Next</Button>
-      {/if}
-    </div>
-  </div>
-</Provider>
-
-<style>
-  p {
-    margin: 0 var(--spacing-m);
-    background-color: var(--grey-2);
-    color: var(--grey-6);
-    font-size: var(--font-size-s);
-    padding: var(--spacing-l);
-    border-radius: var(--border-radius-s);
-    display: grid;
-    place-items: center;
-  }
-  p i {
-    margin-bottom: var(--spacing-m);
-    font-size: 1.5rem;
-    color: var(--grey-5);
-  }
-
-  .query-builder {
-    padding: var(--spacing-m);
-    border-radius: var(--border-radius-s);
-  }
-
-  .actions {
-    display: grid;
-    grid-gap: var(--spacing-s);
-    justify-content: flex-end;
-    grid-auto-flow: column;
-  }
-
-  .form-field {
-    margin-bottom: var(--spacing-m);
-  }
-
-  .pagination {
-    display: grid;
-    grid-gap: var(--spacing-s);
-    justify-content: flex-end;
-    margin-top: var(--spacing-m);
-    grid-auto-flow: column;
-  }
-</style>