From c0e0b48a805708d1dbad59bef118b61c5c0c142e Mon Sep 17 00:00:00 2001
From: Andrew Kingston <aptkingston@ssl-mail.com>
Date: Mon, 19 Oct 2020 19:24:05 +0100
Subject: [PATCH] Allow CSV upload in browser and add display column option

---
 .../TableNavigator/TableDataImport.svelte     | 64 +++++++++++++------
 .../modals/CreateTableModal.svelte            | 13 +++-
 packages/server/src/api/controllers/table.js  | 11 ++--
 packages/server/src/utilities/csvParser.js    |  8 +--
 4 files changed, 62 insertions(+), 34 deletions(-)

diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
index 07eccebf99..1e766072f2 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { Heading, Body, Button, Select } from "@budibase/bbui"
+  import { Heading, Body, Button, Select, Label } from "@budibase/bbui"
   import { notifier } from "builderStore/store/notifications"
   import { FIELDS } from "constants/backend"
   import api from "builderStore/api"
@@ -14,15 +14,17 @@
     schema: {},
   }
 
-  let parseResult
+  let csvString
+  let primaryDisplay
+  let schema = {}
+  let fields = []
 
-  $: schema = parseResult && parseResult.schema
-  $: valid =
-    !schema || Object.keys(schema).every(column => schema[column].success)
+  $: valid = !schema || fields.every(column => schema[column].success)
   $: dataImport = {
     valid,
     schema: buildTableSchema(schema),
-    path: files[0] && files[0].path,
+    csvString,
+    primaryDisplay,
   }
 
   function buildTableSchema(schema) {
@@ -43,11 +45,20 @@
 
   async function validateCSV() {
     const response = await api.post("/api/tables/csv/validate", {
-      file: files[0],
+      csvString,
       schema: schema || {},
     })
 
-    parseResult = await response.json()
+    const parseResult = await response.json()
+    schema = parseResult && parseResult.schema
+    fields = Object.keys(schema || {}).filter(
+      key => schema[key].type !== "omit"
+    )
+
+    // Check primary display is valid
+    if (!primaryDisplay || fields.indexOf(primaryDisplay) === -1) {
+      primaryDisplay = fields[0]
+    }
 
     if (response.status !== 200) {
       notifier.danger("CSV Invalid, please try another CSV file")
@@ -57,13 +68,7 @@
 
   async function handleFile(evt) {
     const fileArray = Array.from(evt.target.files)
-    const filesToProcess = fileArray.map(({ name, path, size }) => ({
-      name,
-      path,
-      size,
-    }))
-
-    if (filesToProcess.some(file => file.size >= FILE_SIZE_LIMIT)) {
+    if (fileArray.some(file => file.size >= FILE_SIZE_LIMIT)) {
       notifier.danger(
         `Files cannot exceed ${FILE_SIZE_LIMIT /
           BYTES_IN_MB}MB. Please try again with smaller files.`
@@ -71,9 +76,14 @@
       return
     }
 
-    files = filesToProcess
-
-    await validateCSV()
+    // Read CSV as plain text to upload alongside schema
+    let reader = new FileReader()
+    reader.addEventListener("load", function(e) {
+      csvString = e.target.result
+      files = fileArray
+      validateCSV()
+    })
+    reader.readAsBinaryString(fileArray[0])
   }
 
   async function omitColumn(columnName) {
@@ -94,8 +104,8 @@
   </label>
 </div>
 <div class="schema-fields">
-  {#if schema}
-    {#each Object.keys(schema).filter(key => schema[key].type !== 'omit') as columnName}
+  {#if fields.length}
+    {#each fields as columnName}
       <div class="field">
         <span>{columnName}</span>
         <Select
@@ -117,6 +127,16 @@
     {/each}
   {/if}
 </div>
+{#if fields.length}
+  <div class="display-column">
+    <Label extraSmall grey>Display Column</Label>
+    <Select thin secondary bind:value={primaryDisplay}>
+      {#each fields as field}
+        <option value={field}>{field}</option>
+      {/each}
+    </Select>
+  </div>
+{/if}
 
 <style>
   .dropzone {
@@ -188,4 +208,8 @@
     grid-gap: var(--spacing-m);
     font-size: var(--font-size-xs);
   }
+
+  .display-column {
+    margin-top: var(--spacing-xl);
+  }
 </style>
diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
index bba3342686..dd691e9e80 100644
--- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
@@ -38,12 +38,19 @@
   }
 
   async function saveTable() {
-    // Create table
-    const table = await backendUiStore.actions.tables.save({
+    let newTable = {
       name,
       schema: dataImport.schema || {},
       dataImport,
-    })
+    }
+
+    // Only set primary display if defined
+    if (dataImport.primaryDisplay && dataImport.primaryDisplay.length) {
+      newTable.primaryDisplay = dataImport.primaryDisplay
+    }
+
+    // Create table
+    const table = await backendUiStore.actions.tables.save(newTable)
     notifier.success(`Table ${name} created successfully.`)
     analytics.captureEvent("Table Created", { name })
 
diff --git a/packages/server/src/api/controllers/table.js b/packages/server/src/api/controllers/table.js
index 3030274016..61726d591a 100644
--- a/packages/server/src/api/controllers/table.js
+++ b/packages/server/src/api/controllers/table.js
@@ -109,7 +109,7 @@ exports.save = async function(ctx) {
   ctx.eventEmitter &&
     ctx.eventEmitter.emitTable(`table:save`, instanceId, tableToSave)
 
-  if (dataImport && dataImport.path) {
+  if (dataImport && dataImport.csvString) {
     // Populate the table with rows imported from CSV in a bulk update
     const data = await csvParser.transform(dataImport)
 
@@ -156,10 +156,7 @@ exports.destroy = async function(ctx) {
 }
 
 exports.validateCSVSchema = async function(ctx) {
-  const { file, schema = {} } = ctx.request.body
-  const result = await csvParser.parse(file.path, schema)
-  ctx.body = {
-    schema: result,
-    path: file.path,
-  }
+  const { csvString, schema = {} } = ctx.request.body
+  const result = await csvParser.parse(csvString, schema)
+  ctx.body = { schema: result }
 }
diff --git a/packages/server/src/utilities/csvParser.js b/packages/server/src/utilities/csvParser.js
index 222ff6523f..3ae020eb65 100644
--- a/packages/server/src/utilities/csvParser.js
+++ b/packages/server/src/utilities/csvParser.js
@@ -11,8 +11,8 @@ const PARSERS = {
   datetime: attribute => new Date(attribute).toISOString(),
 }
 
-function parse(path, parsers) {
-  const result = csv().fromFile(path)
+function parse(csvString, parsers) {
+  const result = csv().fromString(csvString)
 
   const schema = {}
 
@@ -52,7 +52,7 @@ function parse(path, parsers) {
   })
 }
 
-async function transform({ schema, path }) {
+async function transform({ schema, csvString }) {
   const colParser = {}
 
   for (let key in schema) {
@@ -60,7 +60,7 @@ async function transform({ schema, path }) {
   }
 
   try {
-    const json = await csv({ colParser }).fromFile(path)
+    const json = await csv({ colParser }).fromString(csvString)
     return json
   } catch (err) {
     console.error(`Error transforming CSV to JSON for data import`, err)