From 6a94aada1932b5dc350bc2f6db0205ff161cbb8e Mon Sep 17 00:00:00 2001
From: mikesealey <mikesealey@hotmail.com>
Date: Wed, 29 Jan 2025 17:52:14 +0000
Subject: [PATCH] adds more robust testing for complex cases

---
 packages/server/src/utilities/csv.ts          | 22 +++++---------
 .../server/src/utilities/tests/csv.spec.ts    | 30 +++++++++++++++++++
 2 files changed, 38 insertions(+), 14 deletions(-)

diff --git a/packages/server/src/utilities/csv.ts b/packages/server/src/utilities/csv.ts
index 1e139f36fa..880a078a2a 100644
--- a/packages/server/src/utilities/csv.ts
+++ b/packages/server/src/utilities/csv.ts
@@ -4,15 +4,10 @@ export async function jsonFromCsvString(csvString: string) {
   const possibleDelimeters = [",", ";", ":", "|", "~", "\t", " "]
 
   for (let i = 0; i < possibleDelimeters.length; i++) {
-    let numOfHeaders: number | undefined = undefined
+    let headers: string[] | undefined = undefined
     let headerMismatch = false
 
     try {
-      const castedWithEmptyValues = await csv({
-        ignoreEmpty: true,
-        delimiter: possibleDelimeters[i],
-      }).fromString(csvString)
-
       // By default the csvtojson library casts empty values as empty strings. This
       // is causing issues on conversion.  ignoreEmpty will remove the key completly
       // if empty, so creating this empty object will ensure we return the values
@@ -28,18 +23,17 @@ export async function jsonFromCsvString(csvString: string) {
         // If the number of columms in each row is different to
         // the number of headers, this isn't the right delimiter
         const columns = Object.keys(r)
-        if (numOfHeaders == null) {
-          numOfHeaders = columns.length
+        if (headers == null) {
+          headers = columns
         }
-        if (numOfHeaders === 1 || numOfHeaders !== columns.length) {
+        if (headers.length === 1 || headers.length !== columns.length) {
           headerMismatch = true
           break
         }
-        for (const [key] of Object.entries(r).filter(
-          ([, value]) => value === ""
-        )) {
-          if (castedWithEmptyValues[i][key] === undefined) {
-            r[key] = null
+
+        for (const header of headers) {
+          if (r[header] === undefined || r[header] === "") {
+            r[header] = null
           }
         }
       }
diff --git a/packages/server/src/utilities/tests/csv.spec.ts b/packages/server/src/utilities/tests/csv.spec.ts
index 14063d0e8e..85796307f9 100644
--- a/packages/server/src/utilities/tests/csv.spec.ts
+++ b/packages/server/src/utilities/tests/csv.spec.ts
@@ -1,3 +1,4 @@
+import { delimiter } from "path"
 import { jsonFromCsvString } from "../csv"
 
 describe("csv", () => {
@@ -29,5 +30,34 @@ describe("csv", () => {
         expect(Object.keys(r)).toEqual(["id", "optional", "title"])
       )
     })
+
+    const possibleDelimeters = [",", ";", ":", "|", "~", "\t", " "]
+
+    const csvArray = [
+      ["id", "title"],
+      ["1", "aaa"],
+      ["2", "bbb"],
+      ["3", "c ccc"],
+      ["", ""],
+      [":5", "eee5:e"],
+    ]
+
+    test.each(possibleDelimeters)(
+      "Should parse with delimiter %s",
+      async delimiter => {
+        const csvString = csvArray
+          .map(row => row.map(col => `"${col}"`).join(delimiter))
+          .join("\n")
+        const result = await jsonFromCsvString(csvString)
+
+        expect(result).toEqual([
+          { id: "1", title: "aaa" },
+          { id: "2", title: "bbb" },
+          { id: "3", title: "c ccc" },
+          { id: null, title: null },
+          { id: ":5", title: "eee5:e" },
+        ])
+      }
+    )
   })
 })