From d6f92c7039ca4499343c618ab61b6b4d25ca29ed Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 13 Jul 2022 10:53:50 +0100
Subject: [PATCH] Allowing query backend to attempt to convert types in
 returned JSON to build a better base schema.

---
 .../builder/src/stores/backend/queries.js     |  4 +--
 packages/frontend-core/src/utils/lucene.js    |  5 +--
 .../server/src/api/controllers/query/index.ts | 31 +++++++++++++++++--
 .../server/src/api/routes/tests/query.spec.js | 23 +++++++++++---
 4 files changed, 51 insertions(+), 12 deletions(-)

diff --git a/packages/builder/src/stores/backend/queries.js b/packages/builder/src/stores/backend/queries.js
index e5b79fb165..bb456ce405 100644
--- a/packages/builder/src/stores/backend/queries.js
+++ b/packages/builder/src/stores/backend/queries.js
@@ -90,8 +90,8 @@ export function createQueriesStore() {
       // Assume all the fields are strings and create a basic schema from the
       // unique fields returned by the server
       const schema = {}
-      for (let field of result.schemaFields) {
-        schema[field] = "string"
+      for (let [field, type] of Object.entries(result.schemaFields)) {
+        schema[field] = type || "string"
       }
       return { ...result, schema, rows: result.rows || [] }
     },
diff --git a/packages/frontend-core/src/utils/lucene.js b/packages/frontend-core/src/utils/lucene.js
index d02d6e60a3..1001ec26a8 100644
--- a/packages/frontend-core/src/utils/lucene.js
+++ b/packages/frontend-core/src/utils/lucene.js
@@ -170,10 +170,7 @@ export const runLuceneQuery = (docs, query) => {
     const filters = Object.entries(query[type] || {})
     for (let i = 0; i < filters.length; i++) {
       const [key, testValue] = filters[i]
-      let docValue = Helpers.deepGet(doc, key)
-      if (typeof docValue !== "string" && typeof testValue === "string") {
-        docValue = docValue.toString()
-      }
+      const docValue = Helpers.deepGet(doc, key)
       if (failFn(docValue, testValue)) {
         return false
       }
diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts
index 8dcf570343..412bda62d3 100644
--- a/packages/server/src/api/controllers/query/index.ts
+++ b/packages/server/src/api/controllers/query/index.ts
@@ -1,5 +1,5 @@
 import { generateQueryID, getQueryParams, isProdAppID } from "../../../db/utils"
-import { BaseQueryVerbs } from "../../../constants"
+import { BaseQueryVerbs, FieldTypes } from "../../../constants"
 import { Thread, ThreadType } from "../../../threads"
 import { save as saveDatasource } from "../datasource"
 import { RestImporter } from "./import"
@@ -154,10 +154,37 @@ export async function preview(ctx: any) {
         },
       })
     const { rows, keys, info, extra } = await quotas.addQuery(runFn)
+    const schemaFields: any = {}
+    if (rows?.length > 0) {
+      for (let key of [...new Set(keys)] as string[]) {
+        const field = rows[0][key]
+        let type = typeof field,
+          fieldType = FieldTypes.STRING
+        if (field)
+          switch (type) {
+            case "boolean":
+              schemaFields[key] = FieldTypes.BOOLEAN
+              break
+            case "object":
+              if (field instanceof Date) {
+                fieldType = FieldTypes.DATETIME
+              } else if (Array.isArray(field)) {
+                fieldType = FieldTypes.ARRAY
+              } else {
+                fieldType = FieldTypes.JSON
+              }
+              break
+            case "number":
+              fieldType = FieldTypes.NUMBER
+              break
+          }
+        schemaFields[key] = fieldType
+      }
+    }
     await events.query.previewed(datasource, query)
     ctx.body = {
       rows,
-      schemaFields: [...new Set(keys)],
+      schemaFields,
       info,
       extra,
     }
diff --git a/packages/server/src/api/routes/tests/query.spec.js b/packages/server/src/api/routes/tests/query.spec.js
index 00cae3a9c9..45f69de5e9 100644
--- a/packages/server/src/api/routes/tests/query.spec.js
+++ b/packages/server/src/api/routes/tests/query.spec.js
@@ -215,7 +215,10 @@ describe("/queries", () => {
         .expect("Content-Type", /json/)
         .expect(200)
       // these responses come from the mock
-      expect(res.body.schemaFields).toEqual(["a", "b"])
+      expect(res.body.schemaFields).toEqual({
+        "a": "string",
+        "b": "number",
+      })
       expect(res.body.rows.length).toEqual(1)
       expect(events.query.previewed).toBeCalledTimes(1)
       expect(events.query.previewed).toBeCalledWith(datasource, query)
@@ -289,7 +292,11 @@ describe("/queries", () => {
         queryString: "test={{ variable2 }}",
       })
       // these responses come from the mock
-      expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
+      expect(res.body.schemaFields).toEqual({
+        "opts": "json",
+        "url": "string",
+        "value": "string",
+      })
       expect(res.body.rows[0].url).toEqual("http://www.google.com?test=1")
     })
 
@@ -299,7 +306,11 @@ describe("/queries", () => {
         path: "www.google.com",
         queryString: "test={{ variable3 }}",
       })
-      expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
+      expect(res.body.schemaFields).toEqual({
+        "opts": "json",
+        "url": "string",
+        "value": "string"
+      })
       expect(res.body.rows[0].url).toContain("doctype html")
     })
 
@@ -318,7 +329,11 @@ describe("/queries", () => {
         path: "www.failonce.com",
         queryString: "test={{ variable3 }}",
       })
-      expect(res.body.schemaFields).toEqual(["fails", "url", "opts"])
+      expect(res.body.schemaFields).toEqual({
+        "fails": "number",
+        "opts": "json",
+        "url": "string"
+      })
       expect(res.body.rows[0].fails).toEqual(1)
     })