diff --git a/packages/server/src/jsRunner/tests/isolatedVM.spec.ts b/packages/server/src/jsRunner/tests/isolatedVM.spec.ts index 5a9bc05d76..63a1a52b09 100644 --- a/packages/server/src/jsRunner/tests/isolatedVM.spec.ts +++ b/packages/server/src/jsRunner/tests/isolatedVM.spec.ts @@ -107,4 +107,15 @@ describe("Test isolated vm directly", () => { ) expect(result).toEqual([]) }) + + it("should ensure error results are cleared between runs", () => { + const context = {} + // throw error + // Ensure the first execution throws an error + expect(() => runJSWithIsolatedVM(`test.foo.bar = 123`, context)).toThrow() + + // Ensure the error is not persisted across VMs + const secondResult = runJSWithIsolatedVM(`return {}`, context) + expect(secondResult).toEqual({}) + }) }) diff --git a/packages/server/src/jsRunner/vm/isolated-vm.ts b/packages/server/src/jsRunner/vm/isolated-vm.ts index 37ee048dc2..b47137bcba 100644 --- a/packages/server/src/jsRunner/vm/isolated-vm.ts +++ b/packages/server/src/jsRunner/vm/isolated-vm.ts @@ -186,6 +186,7 @@ export class IsolatedVM implements VM { code = ` try { + results = {} results['${this.runResultKey}']=${this.codeWrapper(code)} } catch (e) { results['${this.runErrorKey}']=e diff --git a/packages/server/src/utilities/csv.ts b/packages/server/src/utilities/csv.ts index 43d712165a..ac9304b12a 100644 --- a/packages/server/src/utilities/csv.ts +++ b/packages/server/src/utilities/csv.ts @@ -1,22 +1,52 @@ import csv from "csvtojson" export async function jsonFromCsvString(csvString: string) { - const castedWithEmptyValues = await csv({ ignoreEmpty: true }).fromString( - csvString - ) + const possibleDelimiters = [",", ";", ":", "|", "~", "\t", " "] - // By default the csvtojson library casts empty values as empty strings. This - // is causing issues on conversion. ignoreEmpty will remove the key completly - // if empty, so creating this empty object will ensure we return the values - // with the keys but empty values - const result = await csv({ ignoreEmpty: false }).fromString(csvString) - result.forEach((r, i) => { - for (const [key] of Object.entries(r).filter(([, value]) => value === "")) { - if (castedWithEmptyValues[i][key] === undefined) { - r[key] = null + for (let i = 0; i < possibleDelimiters.length; i++) { + let headers: string[] | undefined = undefined + let headerMismatch = false + + try { + // By default the csvtojson library casts empty values as empty strings. This + // is causing issues on conversion. ignoreEmpty will remove the key completly + // if empty, so creating this empty object will ensure we return the values + // with the keys but empty values + const result = await csv({ + ignoreEmpty: false, + delimiter: possibleDelimiters[i], + }).fromString(csvString) + for (const [, row] of result.entries()) { + // The purpose of this is to find rows that have been split + // into the wrong number of columns - Any valid .CSV file will have + // the same number of colums in each row + // If the number of columms in each row is different to + // the number of headers, this isn't the right delimiter + const columns = Object.keys(row) + if (headers == null) { + headers = columns + } + if (headers.length === 1 || headers.length !== columns.length) { + headerMismatch = true + break + } + + for (const header of headers) { + if (row[header] === undefined || row[header] === "") { + row[header] = null + } + } } + if (headerMismatch) { + continue + } else { + return result + } + } catch (err) { + // Splitting on the wrong delimiter sometimes throws CSV parsing error + // (eg unterminated strings), which tells us we've picked the wrong delimiter + continue } - }) - - return result + } + throw new Error("Unable to determine delimiter") } diff --git a/packages/server/src/utilities/tests/csv.spec.ts b/packages/server/src/utilities/tests/csv.spec.ts index 14063d0e8e..b1dc192bf0 100644 --- a/packages/server/src/utilities/tests/csv.spec.ts +++ b/packages/server/src/utilities/tests/csv.spec.ts @@ -29,5 +29,34 @@ describe("csv", () => { expect(Object.keys(r)).toEqual(["id", "optional", "title"]) ) }) + + const possibleDelimeters = [",", ";", ":", "|", "~", "\t", " "] + + const csvArray = [ + ["id", "title"], + ["1", "aaa"], + ["2", "bbb"], + ["3", "c ccc"], + ["", ""], + [":5", "eee5:e"], + ] + + test.each(possibleDelimeters)( + "Should parse with delimiter %s", + async delimiter => { + const csvString = csvArray + .map(row => row.map(col => `"${col}"`).join(delimiter)) + .join("\n") + const result = await jsonFromCsvString(csvString) + + expect(result).toEqual([ + { id: "1", title: "aaa" }, + { id: "2", title: "bbb" }, + { id: "3", title: "c ccc" }, + { id: null, title: null }, + { id: ":5", title: "eee5:e" }, + ]) + } + ) }) })