account for omitted rows

packages/builder/src/components/nav/ModelNavigator/TableDataImport.svelte

@@ -2,6 +2,7 @@
   import { Heading, Body, Button, Select } from "@budibase/bbui"
   import { notifier } from "builderStore/store/notifications"
   import { FIELDS } from "constants/backend"
+  import api from "builderStore/api"
 
   const BYTES_IN_KB = 1000
   const BYTES_IN_MB = 1000000
@@ -28,6 +29,9 @@
     const modelSchema = {}
     for (let key in schema) {
       const type = schema[key].type
+
+      if (type === "omit") continue
+
       modelSchema[key] = {
         name: key,
         type,
@@ -38,16 +42,9 @@
   }
 
   async function validateCSV() {
-    const response = await fetch("/api/models/csv/validate", {
+    const response = await api.post("/api/models/csv/validate", {
-      method: "POST",
-      body: JSON.stringify({
       file: files[0],
       schema: schema || {},
-      }),
-      headers: {
-        "Content-Type": "application/json",
-        Accept: "application/json",
-      },
     })
 
     parseResult = await response.json()
@@ -79,8 +76,9 @@
     await validateCSV()
   }
 
-  function omitColumn(columnName) {
+  async function omitColumn(columnName) {
-    parsers[columnName] = PARSERS.omit
+    schema[columnName].type = "omit"
+    await validateCSV()
   }
 
   const handleTypeChange = column => evt => {
@@ -97,7 +95,7 @@
 </div>
 <div class="schema-fields">
   {#if schema}
-    {#each Object.keys(schema) as columnName}
+    {#each Object.keys(schema).filter(key => schema[key].type !== 'omit') as columnName}
       <div class="field">
         <span>{columnName}</span>
         <Select
@@ -109,9 +107,7 @@
           <option value={'number'}>Number</option>
           <option value={'datetime'}>Date</option>
         </Select>
-        <span
+        <span class="field-status" class:error={!schema[columnName].success}>
-          class:success={schema[columnName].success}
-          class:error={!schema[columnName].success}>
           {schema[columnName].success ? 'Success' : 'Failure'}
         </span>
         <i
@@ -132,7 +128,7 @@
     transition: all 0.3s;
   }
 
-  .success {
+  .field-status {
     color: green;
   }
 
@@ -151,7 +147,7 @@
     box-sizing: border-box;
     overflow: hidden;
     border-radius: var(--border-radius-s);
-    color: var(--white);
+    color: var(--ink);
     padding: var(--spacing-s) var(--spacing-l);
     transition: all 0.2s ease 0s;
     display: inline-flex;
@@ -166,13 +162,9 @@
     justify-content: center;
     margin-top: 10px;
     width: 100%;
-    border: solid 1.5px var(--ink);
+    background-color: var(--grey-2);
-    background-color: var(--ink);
   }
 
-  /* .schema-fields {
-  } */
-
   .omit-button {
     font-size: 1.2em;
     color: var(--grey-7);

packages/server/src/utilities/csvParser.js

@@ -7,13 +7,11 @@ const VALIDATORS = {
 }
 
 const PARSERS = {
-  string: attribute => attribute.toString(),
-  number: attribute => Number(attribute),
   datetime: attribute => new Date(attribute).toISOString(),
 }
 
 function parse(path, parsers) {
-  const result = csv().fromFile(path)
+  const result = csv({ parsers }).fromFile(path)
 
   const schema = {}
 
@@ -30,10 +28,17 @@ function parse(path, parsers) {
       // For each CSV row
       // parse all the columns that need parsed
       for (let key in parsers) {
-        // if the schema has already borked for a parser, skip this column
+        // if the parsing has already failed for a column
-        if (!schema[key] || !schema[key].success) continue
+        // skip that column
+        if (
+          !schema[key] ||
+          !schema[key].success ||
+          schema[key].type === "omit"
+        ) {
+          continue
+        }
 
-        // get the validator
+        // get the validator for the column type
         const validator = VALIDATORS[parsers[key].type]
 
         try {
@@ -54,12 +59,11 @@ function parse(path, parsers) {
   })
 }
 
-// TODO: significant refactor
 async function transform({ schema, path }) {
   const colParser = {}
 
   for (let key in schema) {
-    colParser[key] = PARSERS[schema[key].type]
+    colParser[key] = PARSERS[schema[key].type] || schema[key].type
   }
 
   try {