Add and fix table tests

packages/server/src/api/routes/tests/table.spec.ts

@@ -17,8 +17,10 @@ import {
   TableSchema,
   TableSourceType,
   User,
+  ValidateTableImportResponse,
   ViewCalculation,
   ViewV2Enriched,
+  RowExportFormat,
 } from "@budibase/types"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
@@ -1086,7 +1088,10 @@ describe.each([
     })
   })
 
-  describe("import validation", () => {
+  describe.each([
+    [RowExportFormat.CSV, (val: any) => JSON.stringify(val).replace(/"/g, "'")],
+    [RowExportFormat.JSON, (val: any) => val],
+  ])("import validation (%s)", (_, userParser) => {
     const basicSchema: TableSchema = {
       id: {
         type: FieldType.NUMBER,
@@ -1098,9 +1103,41 @@ describe.each([
       },
     }
 
-    describe("validateNewTableImport", () => {
+    const importCases: [
-      it("can validate basic imports", async () => {
+      string,
-        const result = await config.api.table.validateNewTableImport(
+      (rows: Row[], schema: TableSchema) => Promise<ValidateTableImportResponse>
+    ][] = [
+      [
+        "validateNewTableImport",
+        async (rows: Row[], schema: TableSchema) => {
+          const result = await config.api.table.validateNewTableImport({
+            rows,
+            schema,
+          })
+          return result
+        },
+      ],
+      [
+        "validateExistingTableImport",
+        async (rows: Row[], schema: TableSchema) => {
+          const table = await config.api.table.save(
+            tableForDatasource(datasource, {
+              primary: ["id"],
+              schema,
+            })
+          )
+          const result = await config.api.table.validateExistingTableImport({
+            tableId: table._id,
+            rows,
+          })
+          return result
+        },
+      ],
+    ]
+
+    describe.each(importCases)("%s", (_, testDelegate) => {
+      it("validates basic imports", async () => {
+        const result = await testDelegate(
           [{ id: generator.natural(), name: generator.first() }],
           basicSchema
         )
@@ -1119,18 +1156,18 @@ describe.each([
       it.each(
         isInternal ? PROTECTED_INTERNAL_COLUMNS : PROTECTED_EXTERNAL_COLUMNS
       )("don't allow protected names in schema (%s)", async columnName => {
-        const result = await config.api.table.validateNewTableImport(
+        const result = await config.api.table.validateNewTableImport({
-          [
+          rows: [
             {
               id: generator.natural(),
               name: generator.first(),
               [columnName]: generator.word(),
             },
           ],
-          {
+          schema: {
             ...basicSchema,
-          }
+          },
-        )
+        })
 
         expect(result).toEqual({
           allValid: false,
@@ -1146,25 +1183,53 @@ describe.each([
         })
       })
 
+      it("does not allow imports without rows", async () => {
+        const result = await testDelegate([], basicSchema)
+
+        expect(result).toEqual({
+          allValid: false,
+          errors: {},
+          invalidColumns: [],
+          schemaValidation: {},
+        })
+      })
+
+      it("validates imports with some empty rows", async () => {
+        const result = await testDelegate(
+          [{}, { id: generator.natural(), name: generator.first() }, {}],
+          basicSchema
+        )
+
+        expect(result).toEqual({
+          allValid: true,
+          errors: {},
+          invalidColumns: [],
+          schemaValidation: {
+            id: true,
+            name: true,
+          },
+        })
+      })
+
       isInternal &&
         it.each(
           isInternal ? PROTECTED_INTERNAL_COLUMNS : PROTECTED_EXTERNAL_COLUMNS
         )("don't allow protected names in the rows (%s)", async columnName => {
-          const result = await config.api.table.validateNewTableImport(
+          const result = await config.api.table.validateNewTableImport({
-            [
+            rows: [
               {
                 id: generator.natural(),
                 name: generator.first(),
               },
             ],
-            {
+            schema: {
               ...basicSchema,
               [columnName]: {
                 name: columnName,
                 type: FieldType.STRING,
               },
-            }
+            },
-          )
+          })
 
           expect(result).toEqual({
             allValid: false,
@@ -1179,20 +1244,24 @@ describe.each([
             },
           })
         })
-    })
 
-    describe("validateExistingTableImport", () => {
+      it("validates required fields and valid rows", async () => {
-      it("can validate basic imports", async () => {
+        const schema: TableSchema = {
-        const table = await config.api.table.save(
+          ...basicSchema,
-          tableForDatasource(datasource, {
+          name: {
-            primary: ["id"],
+            type: FieldType.STRING,
-            schema: basicSchema,
+            name: "name",
-          })
+            constraints: { presence: true },
+          },
+        }
+
+        const result = await testDelegate(
+          [
+            { id: generator.natural(), name: generator.first() },
+            { id: generator.natural(), name: generator.first() },
+          ],
+          schema
         )
-        const result = await config.api.table.validateExistingTableImport({
-          tableId: table._id,
-          rows: [{ id: generator.natural(), name: generator.first() }],
-        })
 
         expect(result).toEqual({
           allValid: true,
@@ -1205,6 +1274,154 @@ describe.each([
         })
       })
 
+      it("validates required fields and non-valid rows", async () => {
+        const schema: TableSchema = {
+          ...basicSchema,
+          name: {
+            type: FieldType.STRING,
+            name: "name",
+            constraints: { presence: true },
+          },
+        }
+
+        const result = await testDelegate(
+          [
+            { id: generator.natural(), name: generator.first() },
+            { id: generator.natural(), name: "" },
+          ],
+          schema
+        )
+
+        expect(result).toEqual({
+          allValid: false,
+          errors: {},
+          invalidColumns: [],
+          schemaValidation: {
+            id: true,
+            name: false,
+          },
+        })
+      })
+
+      describe("bb references", () => {
+        const getUserValues = () => ({
+          _id: docIds.generateGlobalUserID(),
+          primaryDisplay: generator.first(),
+          email: generator.email({}),
+        })
+
+        it("can validate user column imports", async () => {
+          const schema: TableSchema = {
+            ...basicSchema,
+            user: {
+              type: FieldType.BB_REFERENCE_SINGLE,
+              subtype: BBReferenceFieldSubType.USER,
+              name: "user",
+            },
+          }
+
+          const result = await testDelegate(
+            [
+              {
+                id: generator.natural(),
+                name: generator.first(),
+                user: userParser(getUserValues()),
+              },
+            ],
+            schema
+          )
+
+          expect(result).toEqual({
+            allValid: true,
+            errors: {},
+            invalidColumns: [],
+            schemaValidation: {
+              id: true,
+              name: true,
+              user: true,
+            },
+          })
+        })
+
+        it("can validate user column imports with invalid data", async () => {
+          const schema: TableSchema = {
+            ...basicSchema,
+            user: {
+              type: FieldType.BB_REFERENCE_SINGLE,
+              subtype: BBReferenceFieldSubType.USER,
+              name: "user",
+            },
+          }
+
+          const result = await testDelegate(
+            [
+              {
+                id: generator.natural(),
+                name: generator.first(),
+                user: userParser(getUserValues()),
+              },
+              {
+                id: generator.natural(),
+                name: generator.first(),
+                user: "no valid user data",
+              },
+            ],
+            schema
+          )
+
+          expect(result).toEqual({
+            allValid: false,
+            errors: {},
+            invalidColumns: [],
+            schemaValidation: {
+              id: true,
+              name: true,
+              user: false,
+            },
+          })
+        })
+
+        it("can validate users column imports", async () => {
+          const schema: TableSchema = {
+            ...basicSchema,
+            user: {
+              type: FieldType.BB_REFERENCE,
+              subtype: BBReferenceFieldSubType.USER,
+              name: "user",
+              externalType: "array",
+            },
+          }
+
+          const result = await testDelegate(
+            [
+              {
+                id: generator.natural(),
+                name: generator.first(),
+                user: userParser([
+                  getUserValues(),
+                  getUserValues(),
+                  getUserValues(),
+                ]),
+              },
+            ],
+            schema
+          )
+
+          expect(result).toEqual({
+            allValid: true,
+            errors: {},
+            invalidColumns: [],
+            schemaValidation: {
+              id: true,
+              name: true,
+              user: true,
+            },
+          })
+        })
+      })
+    })
+
+    describe("validateExistingTableImport", () => {
       isInternal &&
         it("can reimport _id fields for internal tables", async () => {
           const table = await config.api.table.save(

packages/server/src/tests/utilities/api/table.ts

@@ -5,11 +5,10 @@ import {
   CsvToJsonResponse,
   MigrateRequest,
   MigrateResponse,
-  Row,
   SaveTableRequest,
   SaveTableResponse,
   Table,
-  TableSchema,
+  ValidateNewTableImportRequest,
   ValidateTableImportRequest,
   ValidateTableImportResponse,
 } from "@budibase/types"
@@ -73,17 +72,13 @@ export class TableAPI extends TestAPI {
   }
 
   validateNewTableImport = async (
-    rows: Row[],
+    body: ValidateNewTableImportRequest,
-    schema: TableSchema,
     expectations?: Expectations
   ): Promise<ValidateTableImportResponse> => {
     return await this._post<ValidateTableImportResponse>(
       `/api/tables/validateNewTableImport`,
       {
-        body: {
+        body,
-          rows,
-          schema,
-        },
         expectations,
       }
     )

packages/server/src/utilities/schema.ts

@@ -210,10 +210,6 @@ function isValidBBReference(
   subtype: BBReferenceFieldSubType,
   isRequired: boolean
 ): boolean {
-  if (typeof data !== "string") {
-    return false
-  }
-
   if (type === FieldType.BB_REFERENCE_SINGLE) {
     if (!data) {
       return !isRequired
@@ -240,7 +236,10 @@ function isValidBBReference(
   }
 }
 
-function parseJsonExport<T>(value: string) {
+function parseJsonExport<T>(value: any) {
+  if (typeof value !== "string") {
+    return value
+  }
   try {
     const parsed = JSON.parse(value)