Add server-side searching and pagination to data providers using internal tables

packages/builder/src/components/common/DrawerBindableInput.svelte

@@ -13,10 +13,11 @@
   export let title = "Bindings"
   export let placeholder
   export let label
+  export let disabled = false
 
   const dispatch = createEventDispatcher()
   let bindingDrawer
-  $: tempValue = value
+  $: tempValue = Array.isArray(value) ? value : []
   $: readableValue = runtimeToReadableBinding(bindings, value)
 
   const handleClose = () => {
@@ -32,12 +33,15 @@
 <div class="control">
   <Input
     {label}
+    {disabled}
     value={readableValue}
     on:change={event => onChange(event.detail)}
     {placeholder} />
+  {#if !disabled}
     <div class="icon" on:click={bindingDrawer.show}>
       <Icon size="S" name="FlashOn" />
     </div>
+  {/if}
 </div>
 <Drawer bind:this={bindingDrawer} {title}>
   <svelte:fragment slot="description">

packages/builder/src/components/common/ValuesList.svelte

@@ -7,7 +7,7 @@
 
   const inputChanged = ev => {
     try {
-      values = ev.target.value.split("\n")
+      values = ev.detail.split("\n")
     } catch (_) {
       values = []
     }

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/NextPage.svelte

@@ -0,0 +1,38 @@
+<script>
+  import { Select, Label } from "@budibase/bbui"
+  import { currentAsset, store } from "builderStore"
+  import { getActionProviderComponents } from "builderStore/dataBinding"
+
+  export let parameters
+
+  $: actionProviders = getActionProviderComponents(
+    $currentAsset,
+    $store.selectedComponentId,
+    "NextPage"
+  )
+</script>
+
+<div class="root">
+  <Label small>Data Provider</Label>
+  <Select
+    bind:value={parameters.componentId}
+    options={actionProviders}
+    getOptionLabel={(x) => x._instanceName}
+    getOptionValue={(x) => x._id}
+  />
+</div>
+
+<style>
+  .root {
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+    max-width: 800px;
+    margin: 0 auto;
+  }
+
+  .root :global(> div) {
+    flex: 1;
+    margin-left: var(--spacing-l);
+  }
+</style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/PrevPage.svelte

@@ -0,0 +1,38 @@
+<script>
+  import { Select, Label } from "@budibase/bbui"
+  import { currentAsset, store } from "builderStore"
+  import { getActionProviderComponents } from "builderStore/dataBinding"
+
+  export let parameters
+
+  $: actionProviders = getActionProviderComponents(
+    $currentAsset,
+    $store.selectedComponentId,
+    "PrevPage"
+  )
+</script>
+
+<div class="root">
+  <Label small>Data Provider</Label>
+  <Select
+    bind:value={parameters.componentId}
+    options={actionProviders}
+    getOptionLabel={(x) => x._instanceName}
+    getOptionValue={(x) => x._id}
+  />
+</div>
+
+<style>
+  .root {
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+    max-width: 800px;
+    margin: 0 auto;
+  }
+
+  .root :global(> div) {
+    flex: 1;
+    margin-left: var(--spacing-l);
+  }
+</style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/index.js

@@ -6,6 +6,8 @@ import TriggerAutomation from "./TriggerAutomation.svelte"
 import ValidateForm from "./ValidateForm.svelte"
 import LogIn from "./LogIn.svelte"
 import LogOut from "./LogOut.svelte"
+import NextPage from "./NextPage.svelte"
+import PrevPage from "./PrevPage.svelte"
 
 // defines what actions are available, when adding a new one
 // the component is the setup panel for the action
@@ -45,4 +47,12 @@ export default [
     name: "Log Out",
     component: LogOut,
   },
+  {
+    name: "Next Page",
+    component: NextPage,
+  },
+  {
+    name: "Previous Page",
+    component: PrevPage,
+  },
 ]

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte

@@ -0,0 +1,217 @@
+<script>
+  import {
+    DatePicker,
+    ActionButton,
+    Button,
+    Select,
+    Combobox,
+  } from "@budibase/bbui"
+  import { store, currentAsset } from "builderStore"
+  import { getBindableProperties } from "builderStore/dataBinding"
+  import { createEventDispatcher } from "svelte"
+  import DrawerBindableInput from "components/common/DrawerBindableInput.svelte"
+  import { generate } from "shortid"
+
+  const dispatch = createEventDispatcher()
+
+  export let schemaFields
+  export let value
+
+  const OperatorOptions = {
+    Equals: {
+      value: "equal",
+      label: "Equals",
+    },
+    NotEquals: {
+      value: "notEqual",
+      label: "Not equals",
+    },
+    Empty: {
+      value: "empty",
+      label: "Is empty",
+    },
+    NotEmpty: {
+      value: "notEmpty",
+      label: "Is not empty",
+    },
+    StartsWith: {
+      value: "string",
+      label: "Starts with",
+    },
+    Like: {
+      value: "fuzzy",
+      label: "Like",
+    },
+    MoreThan: {
+      value: "rangeLow",
+      label: "More than",
+    },
+    LessThan: {
+      value: "rangeHigh",
+      label: "Less than",
+    },
+  }
+  const BannedTypes = ["link", "attachment"]
+  $: bindableProperties = getBindableProperties(
+    $currentAsset,
+    $store.selectedComponentId
+  )
+  $: fieldOptions = (schemaFields ?? [])
+    .filter(field => !BannedTypes.includes(field.type))
+    .map(field => field.name)
+
+  const addField = () => {
+    value = [
+      ...value,
+      {
+        id: generate(),
+        field: null,
+        operator: OperatorOptions.Equals.value,
+        value: null,
+      },
+    ]
+  }
+
+  const removeField = id => {
+    value = value.filter(field => field.id !== id)
+  }
+
+  const getValidOperatorsForType = type => {
+    const Op = OperatorOptions
+    if (type === "string") {
+      return [
+        Op.Equals,
+        Op.NotEquals,
+        Op.StartsWith,
+        Op.Like,
+        Op.Empty,
+        Op.NotEmpty,
+      ]
+    } else if (type === "number") {
+      return [
+        Op.Equals,
+        Op.NotEquals,
+        Op.MoreThan,
+        Op.LessThan,
+        Op.Empty,
+        Op.NotEmpty,
+      ]
+    } else if (type === "options") {
+      return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
+    } else if (type === "boolean") {
+      return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
+    } else if (type === "longform") {
+      return [
+        Op.Equals,
+        Op.NotEquals,
+        Op.StartsWith,
+        Op.Like,
+        Op.Empty,
+        Op.NotEmpty,
+      ]
+    } else if (type === "datetime") {
+      return [
+        Op.Equals,
+        Op.NotEquals,
+        Op.MoreThan,
+        Op.LessThan,
+        Op.Empty,
+        Op.NotEmpty,
+      ]
+    }
+    return []
+  }
+
+  const onFieldChange = (expression, field) => {
+    // Update the field type
+    expression.type = schemaFields.find(x => x.name === field)?.type
+
+    // Ensure a valid operator is set
+    const validOperators = getValidOperatorsForType(expression.type)
+    if (!validOperators.includes(expression.operator)) {
+      expression.operator =
+        validOperators[0]?.value ?? OperatorOptions.Equals.value
+      onOperatorChange(expression, expression.operator)
+    }
+  }
+
+  const onOperatorChange = (expression, operator) => {
+    const noValueOptions = [
+      OperatorOptions.Empty.value,
+      OperatorOptions.NotEmpty.value,
+    ]
+    expression.noValue = noValueOptions.includes(operator)
+    if (expression.noValue) {
+      expression.value = null
+    }
+  }
+
+  const getFieldOptions = field => {
+    const schema = schemaFields.find(x => x.name === field)
+    return schema?.constraints?.inclusion || []
+  }
+</script>
+
+{#if value?.length}
+  <div class="fields">
+    {#each value as expression, idx}
+      <Select
+        bind:value={expression.field}
+        options={fieldOptions}
+        on:change={e => onFieldChange(expression, e.detail)}
+        placeholder="Column" />
+      <Select
+        disabled={!expression.field}
+        options={getValidOperatorsForType(expression.type)}
+        bind:value={expression.operator}
+        on:change={e => onOperatorChange(expression, e.detail)}
+        placeholder={null} />
+      {#if ['string', 'longform', 'number'].includes(expression.type)}
+        <DrawerBindableInput
+          disabled={expression.noValue}
+          title={`Value for "${expression.field}"`}
+          value={expression.value}
+          placeholder="Value"
+          bindings={bindableProperties}
+          on:change={event => (expression.value = event.detail)} />
+      {:else if expression.type === 'options'}
+        <Combobox
+          disabled={expression.noValue}
+          options={getFieldOptions(expression.field)}
+          bind:value={expression.value} />
+      {:else if expression.type === 'boolean'}
+        <Combobox
+          disabled
+          options={[{ label: 'True', value: true }, { label: 'False', value: false }]}
+          bind:value={expression.value} />
+      {:else if expression.type === 'datetime'}
+        <DatePicker
+          disabled={expression.noValue}
+          bind:value={expression.value} />
+      {:else}
+        <DrawerBindableInput disabled />
+      {/if}
+
+      <ActionButton
+        size="S"
+        quiet
+        icon="Close"
+        on:click={() => removeField(expression.id)} />
+    {/each}
+  </div>
+{/if}
+<div>
+  <Button icon="AddCircle" size="M" secondary on:click={addField}>
+    Add expression
+  </Button>
+</div>
+
+<style>
+  .fields {
+    display: grid;
+    column-gap: var(--spacing-l);
+    row-gap: var(--spacing-s);
+    align-items: center;
+    grid-template-columns: 1fr 120px 1fr auto;
+  }
+</style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte

@@ -1,12 +1,19 @@
 <script>
-  import { Button, Drawer, Body, DrawerContent, Layout } from "@budibase/bbui"
+  import {
+    notifications,
+    Button,
+    Drawer,
+    Body,
+    DrawerContent,
+    Layout,
+  } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
-  import { notifications } from "@budibase/bbui"
+  import {} from "@budibase/bbui"
   import {
     getDatasourceForProvider,
     getSchemaForDatasource,
   } from "builderStore/dataBinding"
-  import SaveFields from "./EventsEditor/actions/SaveFields.svelte"
+  import FilterBuilder from "./FilterBuilder.svelte"
   import { currentAsset } from "builderStore"
 
   const dispatch = createEventDispatcher()
@@ -14,11 +21,11 @@
   export let value = {}
   export let componentInstance
   let drawer
-  let tempValue = value
+  let tempValue = Array.isArray(value) ? value : []
 
   $: schemaFields = getSchemaFields(componentInstance)
 
-  const getSchemaFields = (component) => {
+  const getSchemaFields = component => {
     const datasource = getDatasourceForProvider($currentAsset, component)
     const { schema } = getSchemaForDatasource(datasource)
     return Object.values(schema || {})
@@ -29,10 +36,6 @@
     notifications.success("Filters saved.")
     drawer.hide()
   }
-
-  const onFieldsChanged = (event) => {
-    tempValue = event.detail
-  }
 </script>
 
 <Button secondary wide on:click={drawer.show}>Define Filters</Button>
@@ -48,24 +51,7 @@
           constaints.
         {/if}
       </Body>
-      <div class="fields">
-        <SaveFields
-          parameterFields={value}
-          {schemaFields}
-          valueLabel="Equals"
-          on:change={onFieldsChanged}
-        />
-      </div>
+      <FilterBuilder bind:value={tempValue} {schemaFields} />
     </Layout>
   </DrawerContent>
 </Drawer>
-
-<style>
-  .fields {
-    display: grid;
-    column-gap: var(--spacing-l);
-    row-gap: var(--spacing-s);
-    align-items: center;
-    grid-template-columns: auto 1fr auto 1fr auto;
-  }
-</style>

packages/builder/src/components/design/PropertiesPanel/SettingsView.svelte

@@ -16,7 +16,7 @@
   import MultiFieldSelect from "./PropertyControls/MultiFieldSelect.svelte"
   import SchemaSelect from "./PropertyControls/SchemaSelect.svelte"
   import EventsEditor from "./PropertyControls/EventsEditor"
-  import FilterEditor from "./PropertyControls/FilterEditor.svelte"
+  import FilterEditor from "./PropertyControls/FilterEditor/FilterEditor.svelte"
   import { IconSelect } from "./PropertyControls/IconSelect"
   import ColorPicker from "./PropertyControls/ColorPicker.svelte"
   import StringFieldSelect from "./PropertyControls/StringFieldSelect.svelte"

packages/client/src/api/tables.js

@@ -5,14 +5,14 @@ import { enrichRows } from "./rows"
  * Fetches a table definition.
  * Since definitions cannot change at runtime, the result is cached.
  */
-export const fetchTableDefinition = async tableId => {
+export const fetchTableDefinition = async (tableId) => {
   return await API.get({ url: `/api/tables/${tableId}`, cache: true })
 }
 
 /**
  * Fetches all rows from a table.
  */
-export const fetchTableData = async tableId => {
+export const fetchTableData = async (tableId) => {
   const rows = await API.get({ url: `/api/${tableId}/rows` })
   return await enrichRows(rows, tableId)
 }
@@ -34,3 +34,35 @@ export const searchTableData = async ({ tableId, search, pagination }) => {
   output.rows = await enrichRows(output.rows, tableId)
   return output
 }
+
+/**
+ * Searches a table using Lucene.
+ */
+export const searchTable = async ({
+  tableId,
+  query,
+  raw,
+  bookmark,
+  limit,
+  sort,
+  sortOrder,
+}) => {
+  if (!tableId || (!query && !raw)) {
+    return
+  }
+  const res = await API.post({
+    url: `/api/search/${tableId}/rows`,
+    body: {
+      query,
+      raw,
+      bookmark,
+      limit,
+      sort,
+      sortOrder,
+    },
+  })
+  return {
+    rows: await enrichRows(res?.rows, tableId),
+    bookmark: res.bookmark,
+  }
+}

packages/client/src/constants.js

@@ -5,4 +5,6 @@ export const TableNames = {
 export const ActionTypes = {
   ValidateForm: "ValidateForm",
   RefreshDatasource: "RefreshDatasource",
+  NextPage: "NextPage",
+  PrevPage: "PrevPage",
 }

packages/client/src/utils/buttonActions.js

@@ -16,27 +16,27 @@ const saveRowHandler = async (action, context) => {
   }
 }
 
-const deleteRowHandler = async action => {
+const deleteRowHandler = async (action) => {
   const { tableId, revId, rowId } = action.parameters
   if (tableId && revId && rowId) {
     await deleteRow({ tableId, rowId, revId })
   }
 }
 
-const triggerAutomationHandler = async action => {
+const triggerAutomationHandler = async (action) => {
   const { fields } = action.parameters
   if (fields) {
     await triggerAutomation(action.parameters.automationId, fields)
   }
 }
 
-const navigationHandler = action => {
+const navigationHandler = (action) => {
   if (action.parameters.url) {
     routeStore.actions.navigate(action.parameters.url)
   }
 }
 
-const queryExecutionHandler = async action => {
+const queryExecutionHandler = async (action) => {
   const { datasourceId, queryId, queryParams } = action.parameters
   await executeQuery({
     datasourceId,
@@ -68,7 +68,23 @@ const refreshDatasourceHandler = async (action, context) => {
   )
 }
 
-const loginHandler = async action => {
+const nextPageHandler = async (action, context) => {
+  return await executeActionHandler(
+    context,
+    action.parameters.componentId,
+    ActionTypes.NextPage
+  )
+}
+
+const prevPageHandler = async (action, context) => {
+  return await executeActionHandler(
+    context,
+    action.parameters.componentId,
+    ActionTypes.PrevPage
+  )
+}
+
+const loginHandler = async (action) => {
   const { email, password } = action.parameters
   await authStore.actions.logIn({ email, password })
 }
@@ -87,6 +103,8 @@ const handlerMap = {
   ["Refresh Datasource"]: refreshDatasourceHandler,
   ["Log In"]: loginHandler,
   ["Log Out"]: logoutHandler,
+  ["Next Page"]: nextPageHandler,
+  ["Previous Page"]: prevPageHandler,
 }
 
 /**
@@ -96,9 +114,10 @@ const handlerMap = {
 export const enrichButtonActions = (actions, context) => {
   // Prevent button actions in the builder preview
   if (get(builderStore).inBuilder) {
-    return () => {}
+    // TODO uncomment
+    // return () => {}
   }
-  const handlers = actions.map(def => handlerMap[def["##eventHandlerType"]])
+  const handlers = actions.map((def) => handlerMap[def["##eventHandlerType"]])
   return async () => {
     for (let i = 0; i < handlers.length; i++) {
       try {

packages/server/src/api/controllers/search/index.js

@@ -1,12 +1,14 @@
 const { QueryBuilder, buildSearchUrl, search } = require("./utils")
 
-exports.rowSearch = async ctx => {
+exports.rowSearch = async (ctx) => {
   const appId = ctx.appId
   const { tableId } = ctx.params
-  const { bookmark, query, raw } = ctx.request.body
+  const { bookmark, query, raw, limit, sort, sortOrder } = ctx.request.body
   let url
   if (query) {
-    url = new QueryBuilder(appId, query, bookmark).addTable(tableId).complete()
+    url = new QueryBuilder(appId, query, bookmark, limit, sort, sortOrder)
+      .addTable(tableId)
+      .complete()
   } else if (raw) {
     url = buildSearchUrl({
       appId,

packages/server/src/api/controllers/search/utils.js

@@ -10,24 +10,43 @@ const fetch = require("node-fetch")
  * @param {string|null} bookmark If there were more than the limit specified can send the bookmark that was
  * returned with query for next set of search results.
  * @param {number} limit The number of entries to return per query.
+ * @param {string} sort The column to sort by.
+ * @param {string} sortOrder The order to sort by. "ascending" or "descending".
  * @param {boolean} excludeDocs By default full rows are returned, if required this can be disabled.
  * @return {string} The URL which a GET can be performed on to receive results.
  */
-function buildSearchUrl({ appId, query, bookmark, excludeDocs, limit = 50 }) {
+function buildSearchUrl({
+  appId,
+  query,
+  bookmark,
+  sort,
+  sortOrder,
+  excludeDocs,
+  limit = 50,
+}) {
   let url = `${env.COUCH_DB_URL}/${appId}/_design/database/_search`
   url += `/${SearchIndexes.ROWS}?q=${query}`
   url += `&limit=${limit}`
   if (!excludeDocs) {
     url += "&include_docs=true"
   }
+  if (sort) {
+    const orderChar = sortOrder === "descending" ? "-" : ""
+    url += `&sort="${orderChar}${sort.replace(/ /, "_")}<string>"`
+  }
   if (bookmark) {
     url += `&bookmark=${bookmark}`
   }
+  console.log(url)
   return checkSlashesInUrl(url)
 }
 
+const luceneEscape = (value) => {
+  return `${value}`.replace(/[ #+\-&|!(){}\[\]^"~*?:\\]/g, "\\$&")
+}
+
 class QueryBuilder {
-  constructor(appId, base) {
+  constructor(appId, base, bookmark, limit, sort, sortOrder) {
     this.appId = appId
     this.query = {
       string: {},
@@ -35,10 +54,14 @@ class QueryBuilder {
       range: {},
       equal: {},
       notEqual: {},
+      empty: {},
+      notEmpty: {},
       ...base,
     }
-    this.limit = 50
-    this.bookmark = null
+    this.bookmark = bookmark
+    this.limit = limit || 50
+    this.sort = sort
+    this.sortOrder = sortOrder || "ascending"
   }
 
   setLimit(limit) {
@@ -79,39 +102,73 @@ class QueryBuilder {
     return this
   }
 
+  addEmpty(key, value) {
+    this.query.empty[key] = value
+    return this
+  }
+
+  addNotEmpty(key, value) {
+    this.query.notEmpty[key] = value
+    return this
+  }
+
   addTable(tableId) {
     this.query.equal.tableId = tableId
     return this
   }
 
   complete(rawQuery = null) {
-    let output = ""
+    let output = "*:*"
     function build(structure, queryFn) {
       for (let [key, value] of Object.entries(structure)) {
-        if (output.length !== 0) {
-          output += " AND "
+        const expression = queryFn(luceneEscape(key.replace(/ /, "_")), value)
+        if (expression == null) {
+          continue
         }
-        output += queryFn(key, value).replace(/ /, "\\ ")
+        output += ` AND ${expression}`
       }
     }
 
     if (this.query.string) {
-      build(this.query.string, (key, value) => `${key}:${value}*`)
+      build(this.query.string, (key, value) => {
+        return value ? `${key}:${luceneEscape(value.toLowerCase())}*` : null
+      })
     }
     if (this.query.range) {
-      build(
-        this.query.range,
-        (key, value) => `${key}:[${value.low} TO ${value.high}]`
-      )
+      build(this.query.range, (key, value) => {
+        if (!value) {
+          return null
+        }
+        if (isNaN(value.low) || value.low == null || value.low === "") {
+          return null
+        }
+        if (isNaN(value.high) || value.high == null || value.high === "") {
+          return null
+        }
+        console.log(value)
+        return `${key}:[${value.low} TO ${value.high}]`
+      })
     }
     if (this.query.fuzzy) {
-      build(this.query.fuzzy, (key, value) => `${key}:${value}~`)
+      build(this.query.fuzzy, (key, value) => {
+        return value ? `${key}:${luceneEscape(value.toLowerCase())}~` : null
+      })
     }
     if (this.query.equal) {
-      build(this.query.equal, (key, value) => `${key}:${value}`)
+      build(this.query.equal, (key, value) => {
+        return value ? `${key}:${luceneEscape(value.toLowerCase())}` : null
+      })
     }
     if (this.query.notEqual) {
-      build(this.query.notEqual, (key, value) => `!${key}:${value}`)
+      build(this.query.notEqual, (key, value) => {
+        return value ? `!${key}:${luceneEscape(value.toLowerCase())}` : null
+      })
+    }
+    if (this.query.empty) {
+      build(this.query.empty, (key) => `!${key}:["" TO *]`)
+    }
+    if (this.query.notEmpty) {
+      build(this.query.notEmpty, (key) => `${key}:["" TO *]`)
     }
     if (rawQuery) {
       output = output.length === 0 ? rawQuery : `&${rawQuery}`
@@ -121,11 +178,13 @@ class QueryBuilder {
       query: output,
       bookmark: this.bookmark,
       limit: this.limit,
+      sort: this.sort,
+      sortOrder: this.sortOrder,
     })
   }
 }
 
-exports.search = async query => {
+exports.search = async (query) => {
   const response = await fetch(query, {
     method: "GET",
   })
@@ -134,7 +193,7 @@ exports.search = async query => {
     rows: [],
   }
   if (json.rows != null && json.rows.length > 0) {
-    output.rows = json.rows.map(row => row.doc)
+    output.rows = json.rows.map((row) => row.doc)
   }
   if (json.bookmark) {
     output.bookmark = json.bookmark

packages/server/src/db/views/staticViews.js

@@ -25,11 +25,11 @@ const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR
  * @returns {Promise<void>} The view now exists, please note that the next view of this query will actually build it,
  * so it may be slow.
  */
-exports.createLinkView = async appId => {
+exports.createLinkView = async (appId) => {
   const db = new CouchDB(appId)
   const designDoc = await db.get("_design/database")
   const view = {
-    map: function(doc) {
+    map: function (doc) {
       // everything in this must remain constant as its going to Pouch, no external variables
       if (doc.type === "link") {
         let doc1 = doc.doc1
@@ -57,7 +57,7 @@ exports.createLinkView = async appId => {
   await db.put(designDoc)
 }
 
-exports.createRoutingView = async appId => {
+exports.createRoutingView = async (appId) => {
   const db = new CouchDB(appId)
   const designDoc = await db.get("_design/database")
   const view = {
@@ -84,23 +84,28 @@ async function searchIndex(appId, indexName, fnString) {
   designDoc.indexes = {
     [indexName]: {
       index: fnString,
+      analyzer: "keyword",
     },
   }
   await db.put(designDoc)
 }
 
-exports.createAllSearchIndex = async appId => {
+exports.createAllSearchIndex = async (appId) => {
   await searchIndex(
     appId,
     SearchIndexes.ROWS,
-    function(doc) {
+    function (doc) {
       function idx(input, prev) {
         for (let key of Object.keys(input)) {
-          const idxKey = prev != null ? `${prev}.${key}` : key
-          if (key === "_id" || key === "_rev") {
+          let idxKey = prev != null ? `${prev}.${key}` : key
+          idxKey = idxKey.replace(/ /, "_")
+          if (key === "_id" || key === "_rev" || input[key] == null) {
             continue
           }
-          if (typeof input[key] !== "object") {
+          if (typeof input[key] === "string") {
+            // eslint-disable-next-line no-undef
+            index(idxKey, input[key].toLowerCase(), { store: true })
+          } else if (typeof input[key] !== "object") {
             // eslint-disable-next-line no-undef
             index(idxKey, input[key], { store: true })
           } else {

packages/standard-components/manifest.json

@@ -1419,6 +1419,7 @@
     "icon": "Data",
     "styleable": false,
     "hasChildren": true,
+    "actions": ["NextPage", "PrevPage"],
     "settings": [
       {
         "type": "dataSource",
@@ -1470,6 +1471,10 @@
         {
           "label": "Loaded",
           "key": "loaded"
+        },
+        {
+          "label": "Page Number",
+          "key": "pageNumber"
         }
       ]
     }

packages/standard-components/src/DataProvider.svelte

@@ -5,7 +5,7 @@
   export let filter
   export let sortColumn
   export let sortOrder
-  export let limit
+  export let limit = 50
 
   const { API, styleable, Provider, ActionTypes } = getContext("sdk")
   const component = getContext("component")
@@ -16,13 +16,18 @@
   // Loading flag for the initial load
   let loaded = false
 
-  let allRows = []
+  // Provider state
+  let rows = []
   let schema = {}
+  let bookmarks = [null]
+  let pageNumber = 0
 
-  $: fetchData(dataSource)
-  $: filteredRows = filterRows(allRows, filter)
-  $: sortedRows = sortRows(filteredRows, sortColumn, sortOrder)
-  $: rows = limitRows(sortedRows, limit)
+  $: query = dataSource?.type === "table" ? buildLuceneQuery(filter) : null
+  $: hasNextPage = bookmarks[pageNumber + 1] != null
+  $: hasPrevPage = pageNumber > 0
+  $: fetchData(dataSource, query, limit, sortColumn, sortOrder)
+  // $: sortedRows = sortRows(filteredRows, sortColumn, sortOrder)
+  // $: rows = limitRows(sortedRows, limit)
   $: getSchema(dataSource)
   $: actions = [
     {
@@ -30,6 +35,14 @@
       callback: () => fetchData(dataSource),
       metadata: { dataSource },
     },
+    {
+      type: ActionTypes.NextPage,
+      callback: () => nextPage(),
+    },
+    {
+      type: ActionTypes.PrevPage,
+      callback: () => prevPage(),
+    },
   ]
   $: dataContext = {
     rows,
@@ -37,23 +50,82 @@
     rowsLength: rows.length,
     loading,
     loaded,
+    pageNumber: pageNumber + 1,
+    hasNextPage,
+    hasPrevPage,
   }
 
-  const fetchData = async dataSource => {
+  const buildLuceneQuery = (filter) => {
+    let query = {
+      string: {},
+      fuzzy: {},
+      range: {},
+      equal: {},
+      notEqual: {},
+      empty: {},
+      notEmpty: {},
+    }
+    if (Array.isArray(filter)) {
+      filter.forEach((expression) => {
+        if (expression.operator.startsWith("range")) {
+          let range = {
+            low: Number.MIN_SAFE_INTEGER,
+            high: Number.MAX_SAFE_INTEGER,
+          }
+          if (expression.operator === "rangeLow") {
+            range.low = expression.value
+          } else if (expression.operator === "rangeHigh") {
+            range.high = expression.value
+          }
+          query.range[expression.field] = range
+        } else if (query[expression.operator]) {
+          query[expression.operator][expression.field] = expression.value
+        }
+      })
+    }
+    return query
+  }
+
+  const fetchData = async (dataSource, query, limit, sortColumn, sortOrder) => {
     loading = true
-    allRows = await API.fetchDatasource(dataSource)
+    if (dataSource?.type === "table") {
+      const res = await API.searchTable({
+        tableId: dataSource.tableId,
+        query,
+        limit,
+        sort: sortColumn,
+        sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+      })
+      pageNumber = 0
+      rows = res.rows
+
+      // Check we have next data
+      const next = await API.searchTable({
+        tableId: dataSource.tableId,
+        query,
+        limit: 1,
+        bookmark: res.bookmark,
+        sort: sortColumn,
+        sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+      })
+      if (next.rows?.length) {
+        bookmarks = [null, res.bookmark]
+      } else {
+        bookmarks = [null]
+      }
+    } else {
+      const rows = await API.fetchDatasource(dataSource)
+      rows = inMemoryFilterRows(rows, filter)
+    }
     loading = false
     loaded = true
   }
 
-  const filterRows = (rows, filter) => {
-    if (!Object.keys(filter || {}).length) {
-      return rows
-    }
+  const inMemoryFilterRows = (rows, filter) => {
     let filteredData = [...rows]
     Object.entries(filter).forEach(([field, value]) => {
       if (value != null && value !== "") {
-        filteredData = filteredData.filter(row => {
+        filteredData = filteredData.filter((row) => {
           return row[field] === value
         })
       }
@@ -84,7 +156,7 @@
     return rows.slice(0, numLimit)
   }
 
-  const getSchema = async dataSource => {
+  const getSchema = async (dataSource) => {
     if (dataSource?.schema) {
       schema = dataSource.schema
     } else if (dataSource?.tableId) {
@@ -101,6 +173,51 @@
       }
     })
   }
+
+  const nextPage = async () => {
+    if (!hasNextPage) {
+      return
+    }
+    const res = await API.searchTable({
+      tableId: dataSource?.tableId,
+      query,
+      bookmark: bookmarks[pageNumber + 1],
+      limit,
+      sort: sortColumn,
+      sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+    })
+    pageNumber++
+    rows = res.rows
+
+    // Check we have next data
+    const next = await API.searchTable({
+      tableId: dataSource.tableId,
+      query,
+      limit: 1,
+      bookmark: res.bookmark,
+      sort: sortColumn,
+      sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+    })
+    if (next.rows?.length) {
+      bookmarks[pageNumber + 1] = res.bookmark
+    }
+  }
+
+  const prevPage = async () => {
+    if (!hasPrevPage) {
+      return
+    }
+    const res = await API.searchTable({
+      tableId: dataSource?.tableId,
+      query,
+      bookmark: bookmarks[pageNumber - 1],
+      limit,
+      sort: sortColumn,
+      sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+    })
+    pageNumber--
+    rows = res.rows
+  }
 </script>
 
 <div use:styleable={$component.styles}>