From 1a2e17ff17af6fa154ff238daefd6dd6c37dae40 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <aptkingston@ssl-mail.com>
Date: Tue, 11 May 2021 11:24:16 +0100
Subject: [PATCH] Add support for numerical sorting

---
 .../popovers/EditTablePopover.svelte          |  2 -
 .../FilterEditor/FilterBuilder.svelte         | 36 +++++++-----
 .../FilterEditor/FilterEditor.svelte          |  1 -
 packages/client/src/api/tables.js             |  6 +-
 .../src/api/controllers/search/index.js       | 22 +++++++-
 .../src/api/controllers/search/utils.js       | 20 ++++---
 .../src/DataProvider.svelte                   | 55 ++++++++++++++-----
 yarn.lock                                     |  2 +-
 8 files changed, 101 insertions(+), 43 deletions(-)

diff --git a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte
index 2d419f18aa..6cae231be9 100644
--- a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte
@@ -40,13 +40,11 @@
     if (wasSelectedTable._id === table._id) {
       $goto("./table")
     }
-    editorModal.hide()
   }
 
   async function save() {
     await tables.save(table)
     notifications.success("Table renamed successfully")
-    editorModal.hide()
   }
 
   function checkValid(evt) {
diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte
index a5c192cace..7afbaa5b7d 100644
--- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte
+++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterBuilder.svelte
@@ -9,7 +9,7 @@
   import { store, currentAsset } from "builderStore"
   import { getBindableProperties } from "builderStore/dataBinding"
   import { createEventDispatcher } from "svelte"
-  import DrawerBindableInput from "components/common/DrawerBindableInput.svelte"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
   import { generate } from "shortid"
 
   const dispatch = createEventDispatcher()
@@ -159,35 +159,44 @@
         bind:value={expression.field}
         options={fieldOptions}
         on:change={e => onFieldChange(expression, e.detail)}
-        placeholder="Column" />
+        placeholder="Column"
+      />
       <Select
         disabled={!expression.field}
         options={getValidOperatorsForType(expression.type)}
         bind:value={expression.operator}
         on:change={e => onOperatorChange(expression, e.detail)}
-        placeholder={null} />
-      {#if ['string', 'longform', 'number'].includes(expression.type)}
+        placeholder={null}
+      />
+      {#if ["string", "longform", "number"].includes(expression.type)}
         <DrawerBindableInput
           disabled={expression.noValue}
           title={`Value for "${expression.field}"`}
           value={expression.value}
           placeholder="Value"
           bindings={bindableProperties}
-          on:change={event => (expression.value = event.detail)} />
-      {:else if expression.type === 'options'}
+          on:change={event => (expression.value = event.detail)}
+        />
+      {:else if expression.type === "options"}
         <Combobox
           disabled={expression.noValue}
           options={getFieldOptions(expression.field)}
-          bind:value={expression.value} />
-      {:else if expression.type === 'boolean'}
+          bind:value={expression.value}
+        />
+      {:else if expression.type === "boolean"}
         <Combobox
           disabled
-          options={[{ label: 'True', value: true }, { label: 'False', value: false }]}
-          bind:value={expression.value} />
-      {:else if expression.type === 'datetime'}
+          options={[
+            { label: "True", value: true },
+            { label: "False", value: false },
+          ]}
+          bind:value={expression.value}
+        />
+      {:else if expression.type === "datetime"}
         <DatePicker
           disabled={expression.noValue}
-          bind:value={expression.value} />
+          bind:value={expression.value}
+        />
       {:else}
         <DrawerBindableInput disabled />
       {/if}
@@ -196,7 +205,8 @@
         size="S"
         quiet
         icon="Close"
-        on:click={() => removeField(expression.id)} />
+        on:click={() => removeField(expression.id)}
+      />
     {/each}
   </div>
 {/if}
diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte
index c37fc64a8b..d90097939a 100644
--- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte
+++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte
@@ -8,7 +8,6 @@
     Layout,
   } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
-  import {} from "@budibase/bbui"
   import {
     getDatasourceForProvider,
     getSchemaForDatasource,
diff --git a/packages/client/src/api/tables.js b/packages/client/src/api/tables.js
index 632f24e4a4..693ce8f013 100644
--- a/packages/client/src/api/tables.js
+++ b/packages/client/src/api/tables.js
@@ -5,14 +5,14 @@ import { enrichRows } from "./rows"
  * Fetches a table definition.
  * Since definitions cannot change at runtime, the result is cached.
  */
-export const fetchTableDefinition = async (tableId) => {
+export const fetchTableDefinition = async tableId => {
   return await API.get({ url: `/api/tables/${tableId}`, cache: true })
 }
 
 /**
  * Fetches all rows from a table.
  */
-export const fetchTableData = async (tableId) => {
+export const fetchTableData = async tableId => {
   const rows = await API.get({ url: `/api/${tableId}/rows` })
   return await enrichRows(rows, tableId)
 }
@@ -46,6 +46,7 @@ export const searchTable = async ({
   limit,
   sort,
   sortOrder,
+  sortType,
 }) => {
   if (!tableId || (!query && !raw)) {
     return
@@ -59,6 +60,7 @@ export const searchTable = async ({
       limit,
       sort,
       sortOrder,
+      sortType,
     },
   })
   return {
diff --git a/packages/server/src/api/controllers/search/index.js b/packages/server/src/api/controllers/search/index.js
index 648a66c742..2a309cc7e6 100644
--- a/packages/server/src/api/controllers/search/index.js
+++ b/packages/server/src/api/controllers/search/index.js
@@ -1,12 +1,28 @@
 const { QueryBuilder, buildSearchUrl, search } = require("./utils")
 
-exports.rowSearch = async (ctx) => {
+exports.rowSearch = async ctx => {
   const appId = ctx.appId
   const { tableId } = ctx.params
-  const { bookmark, query, raw, limit, sort, sortOrder } = ctx.request.body
+  const {
+    bookmark,
+    query,
+    raw,
+    limit,
+    sort,
+    sortOrder,
+    sortType,
+  } = ctx.request.body
   let url
   if (query) {
-    url = new QueryBuilder(appId, query, bookmark, limit, sort, sortOrder)
+    url = new QueryBuilder(
+      appId,
+      query,
+      bookmark,
+      limit,
+      sort,
+      sortOrder,
+      sortType
+    )
       .addTable(tableId)
       .complete()
   } else if (raw) {
diff --git a/packages/server/src/api/controllers/search/utils.js b/packages/server/src/api/controllers/search/utils.js
index d18f48533e..7827850428 100644
--- a/packages/server/src/api/controllers/search/utils.js
+++ b/packages/server/src/api/controllers/search/utils.js
@@ -12,6 +12,7 @@ const fetch = require("node-fetch")
  * @param {number} limit The number of entries to return per query.
  * @param {string} sort The column to sort by.
  * @param {string} sortOrder The order to sort by. "ascending" or "descending".
+ * @param {string} sortType The type of sort to perform. "string" or "number".
  * @param {boolean} excludeDocs By default full rows are returned, if required this can be disabled.
  * @return {string} The URL which a GET can be performed on to receive results.
  */
@@ -21,18 +22,19 @@ function buildSearchUrl({
   bookmark,
   sort,
   sortOrder,
+  sortType,
   excludeDocs,
   limit = 50,
 }) {
   let url = `${env.COUCH_DB_URL}/${appId}/_design/database/_search`
   url += `/${SearchIndexes.ROWS}?q=${query}`
-  url += `&limit=${limit}`
+  url += `&limit=${Math.min(limit, 200)}`
   if (!excludeDocs) {
     url += "&include_docs=true"
   }
   if (sort) {
     const orderChar = sortOrder === "descending" ? "-" : ""
-    url += `&sort="${orderChar}${sort.replace(/ /, "_")}<string>"`
+    url += `&sort="${orderChar}${sort.replace(/ /, "_")}<${sortType}>"`
   }
   if (bookmark) {
     url += `&bookmark=${bookmark}`
@@ -41,12 +43,12 @@ function buildSearchUrl({
   return checkSlashesInUrl(url)
 }
 
-const luceneEscape = (value) => {
+const luceneEscape = value => {
   return `${value}`.replace(/[ #+\-&|!(){}\[\]^"~*?:\\]/g, "\\$&")
 }
 
 class QueryBuilder {
-  constructor(appId, base, bookmark, limit, sort, sortOrder) {
+  constructor(appId, base, bookmark, limit, sort, sortOrder, sortType) {
     this.appId = appId
     this.query = {
       string: {},
@@ -62,6 +64,7 @@ class QueryBuilder {
     this.limit = limit || 50
     this.sort = sort
     this.sortOrder = sortOrder || "ascending"
+    this.sortType = sortType || "string"
   }
 
   setLimit(limit) {
@@ -165,10 +168,10 @@ class QueryBuilder {
       })
     }
     if (this.query.empty) {
-      build(this.query.empty, (key) => `!${key}:["" TO *]`)
+      build(this.query.empty, key => `!${key}:["" TO *]`)
     }
     if (this.query.notEmpty) {
-      build(this.query.notEmpty, (key) => `${key}:["" TO *]`)
+      build(this.query.notEmpty, key => `${key}:["" TO *]`)
     }
     if (rawQuery) {
       output = output.length === 0 ? rawQuery : `&${rawQuery}`
@@ -180,11 +183,12 @@ class QueryBuilder {
       limit: this.limit,
       sort: this.sort,
       sortOrder: this.sortOrder,
+      sortType: this.sortType,
     })
   }
 }
 
-exports.search = async (query) => {
+exports.search = async query => {
   const response = await fetch(query, {
     method: "GET",
   })
@@ -193,7 +197,7 @@ exports.search = async (query) => {
     rows: [],
   }
   if (json.rows != null && json.rows.length > 0) {
-    output.rows = json.rows.map((row) => row.doc)
+    output.rows = json.rows.map(row => row.doc)
   }
   if (json.bookmark) {
     output.bookmark = json.bookmark
diff --git a/packages/standard-components/src/DataProvider.svelte b/packages/standard-components/src/DataProvider.svelte
index bc73a9290c..38dd9f3955 100644
--- a/packages/standard-components/src/DataProvider.svelte
+++ b/packages/standard-components/src/DataProvider.svelte
@@ -18,17 +18,26 @@
 
   // Provider state
   let rows = []
+  let allRows = []
   let schema = {}
   let bookmarks = [null]
   let pageNumber = 0
 
+  $: internalTable = dataSource?.type === "table"
   $: query = dataSource?.type === "table" ? buildLuceneQuery(filter) : null
   $: hasNextPage = bookmarks[pageNumber + 1] != null
   $: hasPrevPage = pageNumber > 0
-  $: fetchData(dataSource, query, limit, sortColumn, sortOrder)
-  // $: sortedRows = sortRows(filteredRows, sortColumn, sortOrder)
-  // $: rows = limitRows(sortedRows, limit)
   $: getSchema(dataSource)
+  $: sortType = getSortType(schema, sortColumn)
+  $: fetchData(dataSource, query, limit, sortColumn, sortOrder, sortType)
+  $: {
+    if (internalTable) {
+      rows = allRows
+    } else {
+      rows = sortRows(allRows, sortColumn, sortOrder)
+      rows = limitRows(rows, limit)
+    }
+  }
   $: actions = [
     {
       type: ActionTypes.RefreshDatasource,
@@ -55,7 +64,15 @@
     hasPrevPage,
   }
 
-  const buildLuceneQuery = (filter) => {
+  const getSortType = (schema, sortColumn) => {
+    if (!schema || !sortColumn || !schema[sortColumn]) {
+      return "string"
+    }
+    const type = schema?.[sortColumn]?.type
+    return type === "number" ? "number" : "string"
+  }
+
+  const buildLuceneQuery = filter => {
     let query = {
       string: {},
       fuzzy: {},
@@ -66,7 +83,7 @@
       notEmpty: {},
     }
     if (Array.isArray(filter)) {
-      filter.forEach((expression) => {
+      filter.forEach(expression => {
         if (expression.operator.startsWith("range")) {
           let range = {
             low: Number.MIN_SAFE_INTEGER,
@@ -86,7 +103,14 @@
     return query
   }
 
-  const fetchData = async (dataSource, query, limit, sortColumn, sortOrder) => {
+  const fetchData = async (
+    dataSource,
+    query,
+    limit,
+    sortColumn,
+    sortOrder,
+    sortType
+  ) => {
     loading = true
     if (dataSource?.type === "table") {
       const res = await API.searchTable({
@@ -95,9 +119,10 @@
         limit,
         sort: sortColumn,
         sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+        sortType,
       })
       pageNumber = 0
-      rows = res.rows
+      allRows = res.rows
 
       // Check we have next data
       const next = await API.searchTable({
@@ -107,6 +132,7 @@
         bookmark: res.bookmark,
         sort: sortColumn,
         sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+        sortType,
       })
       if (next.rows?.length) {
         bookmarks = [null, res.bookmark]
@@ -115,7 +141,7 @@
       }
     } else {
       const rows = await API.fetchDatasource(dataSource)
-      rows = inMemoryFilterRows(rows, filter)
+      allRows = inMemoryFilterRows(rows, filter)
     }
     loading = false
     loaded = true
@@ -123,9 +149,9 @@
 
   const inMemoryFilterRows = (rows, filter) => {
     let filteredData = [...rows]
-    Object.entries(filter).forEach(([field, value]) => {
+    Object.entries(filter || {}).forEach(([field, value]) => {
       if (value != null && value !== "") {
-        filteredData = filteredData.filter((row) => {
+        filteredData = filteredData.filter(row => {
           return row[field] === value
         })
       }
@@ -156,7 +182,7 @@
     return rows.slice(0, numLimit)
   }
 
-  const getSchema = async (dataSource) => {
+  const getSchema = async dataSource => {
     if (dataSource?.schema) {
       schema = dataSource.schema
     } else if (dataSource?.tableId) {
@@ -196,9 +222,10 @@
       limit,
       sort: sortColumn,
       sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+      sortType,
     })
     pageNumber++
-    rows = res.rows
+    allRows = res.rows
 
     // Check we have next data
     const next = await API.searchTable({
@@ -208,6 +235,7 @@
       bookmark: res.bookmark,
       sort: sortColumn,
       sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+      sortType,
     })
     if (next.rows?.length) {
       bookmarks[pageNumber + 1] = res.bookmark
@@ -225,9 +253,10 @@
       limit,
       sort: sortColumn,
       sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+      sortType,
     })
     pageNumber--
-    rows = res.rows
+    allRows = res.rows
   }
 </script>
 
diff --git a/yarn.lock b/yarn.lock
index c8c4430e09..f344bd8338 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4559,7 +4559,7 @@ supports-color@^7.1.0:
   dependencies:
     has-flag "^4.0.0"
 
-svelte@^3.37.0:
+svelte@^3.38.2:
   version "3.38.2"
   resolved "https://registry.yarnpkg.com/svelte/-/svelte-3.38.2.tgz#55e5c681f793ae349b5cc2fe58e5782af4275ef5"
   integrity sha512-q5Dq0/QHh4BLJyEVWGe7Cej5NWs040LWjMbicBGZ+3qpFWJ1YObRmUDZKbbovddLC9WW7THTj3kYbTOFmU9fbg==