From 5d0918a6cb1ceaf468b59da90e2b3ed884182f3a Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 20 Jul 2023 12:21:09 +0100
Subject: [PATCH 1/4] Prevent invalid characters in column names when importing
 tables

---
 packages/bbui/src/Icon/Icon.svelte            | 12 ++---
 .../DataTable/modals/CreateEditColumn.svelte  |  3 +-
 .../TableNavigator/TableDataImport.svelte     | 46 ++++++++++---------
 packages/server/src/utilities/schema.ts       |  8 ++++
 packages/shared-core/src/constants.ts         |  1 +
 5 files changed, 38 insertions(+), 32 deletions(-)

diff --git a/packages/bbui/src/Icon/Icon.svelte b/packages/bbui/src/Icon/Icon.svelte
index 452a8c74a1..a2cf8a1b3a 100644
--- a/packages/bbui/src/Icon/Icon.svelte
+++ b/packages/bbui/src/Icon/Icon.svelte
@@ -47,7 +47,7 @@
   </svg>
   {#if tooltip && showTooltip}
     <div class="tooltip" in:fade={{ duration: 130, delay: 250 }}>
-      <Tooltip textWrapping direction="bottom" text={tooltip} />
+      <Tooltip textWrapping direction="top" text={tooltip} />
     </div>
   {/if}
 </div>
@@ -80,15 +80,9 @@
     position: absolute;
     pointer-events: none;
     left: 50%;
-    top: calc(100% + 4px);
-    width: 100vw;
-    max-width: 150px;
+    bottom: calc(100% + 4px);
     transform: translateX(-50%);
     text-align: center;
-  }
-
-  .spectrum-Icon--sizeXS {
-    width: 10px;
-    height: 10px;
+    z-index: 1;
   }
 </style>
diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 6ffd7f8bb3..f319f09b16 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -33,6 +33,7 @@
   import { getBindings } from "components/backend/DataTable/formula"
   import { getContext } from "svelte"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
+  import { ValidColumnNameRegex } from "@budibase/shared-core"
 
   const AUTO_TYPE = "auto"
   const FORMULA_TYPE = FIELDS.FORMULA.type
@@ -379,7 +380,7 @@
     const newError = {}
     if (!external && fieldInfo.name?.startsWith("_")) {
       newError.name = `Column name cannot start with an underscore.`
-    } else if (fieldInfo.name && !fieldInfo.name.match(/^[_a-zA-Z0-9\s]*$/g)) {
+    } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) {
       newError.name = `Illegal character; must be alpha-numeric.`
     } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) {
       newError.name = `${PROHIBITED_COLUMN_NAMES.join(
diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
index f34a3e9c98..1191f92b31 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { Select } from "@budibase/bbui"
+  import { Select, Icon } from "@budibase/bbui"
   import { FIELDS } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
@@ -12,6 +12,7 @@
   let loading = false
   let validation = {}
   let validateHash = ""
+  let errors = {}
 
   export let rows = []
   export let schema = {}
@@ -69,18 +70,20 @@
 
   async function validate(rows, schema) {
     loading = true
-    error = null
-    validation = {}
-    allValid = false
 
     try {
       if (rows.length > 0) {
         const response = await API.validateNewTableImport({ rows, schema })
         validation = response.schemaValidation
         allValid = response.allValid
+        errors = response.errors
+        error = null
       }
     } catch (e) {
       error = e.message
+      validation = {}
+      allValid = false
+      errors = {}
     }
 
     loading = false
@@ -147,16 +150,22 @@
           disabled={loading}
         />
         <span
-          class={loading || validation[column.name]
+          class={validation[column.name]
             ? "fieldStatusSuccess"
             : "fieldStatusFailure"}
         >
-          {validation[column.name] ? "Success" : "Failure"}
+          {#if validation[column.name]}
+            Success
+          {:else}
+            Failure
+            <Icon name="Help" tooltip={errors[column.name]} />
+          {/if}
         </span>
-        <i
-          class={`omit-button ri-close-circle-fill ${
-            loading ? "omit-button-disabled" : ""
-          }`}
+        <Icon
+          size="S"
+          name="Close"
+          disabled={loading}
+          hoverable
           on:click={() => {
             delete schema[column.name]
             schema = schema
@@ -237,23 +246,16 @@
     justify-self: center;
     font-weight: 600;
   }
-
   .fieldStatusFailure {
     color: var(--red);
     justify-self: center;
     font-weight: 600;
+    display: flex;
+    align-items: center;
+    gap: 4px;
   }
-
-  .omit-button {
-    font-size: 1.2em;
-    color: var(--grey-7);
-    cursor: pointer;
-    justify-self: flex-end;
-  }
-
-  .omit-button-disabled {
-    pointer-events: none;
-    opacity: 70%;
+  .fieldStatusFailure :global(.spectrum-Icon) {
+    width: 12px;
   }
 
   .display-column {
diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts
index 7b5de7898a..785840b23e 100644
--- a/packages/server/src/utilities/schema.ts
+++ b/packages/server/src/utilities/schema.ts
@@ -1,4 +1,5 @@
 import { FieldTypes } from "../constants"
+import { ValidColumnNameRegex } from "@budibase/shared-core"
 
 interface SchemaColumn {
   readonly name: string
@@ -27,6 +28,7 @@ interface ValidationResults {
   schemaValidation: SchemaValidation
   allValid: boolean
   invalidColumns: Array<string>
+  errors: Record<string, string>
 }
 
 const PARSERS: any = {
@@ -69,6 +71,7 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
     schemaValidation: {},
     allValid: false,
     invalidColumns: [],
+    errors: {},
   }
 
   rows.forEach(row => {
@@ -79,6 +82,11 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
       // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array
       if (typeof columnType !== "string") {
         results.invalidColumns.push(columnName)
+      } else if (!columnName.match(ValidColumnNameRegex)) {
+        // Check for special characters in column names
+        results.invalidColumns.push(columnName)
+        results.errors[columnName] =
+          "Column names can't contain special characters"
       } else if (
         columnData == null &&
         !schema[columnName].constraints?.presence
diff --git a/packages/shared-core/src/constants.ts b/packages/shared-core/src/constants.ts
index 307285012b..d9aacfcecf 100644
--- a/packages/shared-core/src/constants.ts
+++ b/packages/shared-core/src/constants.ts
@@ -89,3 +89,4 @@ export enum BuilderSocketEvent {
 }
 
 export const SocketSessionTTL = 60
+export const ValidColumnNameRegex = /^[_a-zA-Z0-9\s]*$/g

From 08920f8015cfe80fc63d32eaee6d9816b12bdc36 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 20 Jul 2023 14:06:31 +0100
Subject: [PATCH 2/4] Remove invalid columns from display column selection and
 fix schema failure handling

---
 .../TableNavigator/TableDataImport.svelte     | 25 +++++++++++--------
 packages/server/src/utilities/schema.ts       |  2 +-
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
index 1191f92b31..6de069bab4 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
@@ -4,16 +4,6 @@
   import { API } from "api"
   import { parseFile } from "./utils"
 
-  let fileInput
-  let error = null
-  let fileName = null
-  let fileType = null
-
-  let loading = false
-  let validation = {}
-  let validateHash = ""
-  let errors = {}
-
   export let rows = []
   export let schema = {}
   export let allValid = true
@@ -51,6 +41,19 @@
     },
   ]
 
+  let fileInput
+  let error = null
+  let fileName = null
+  let fileType = null
+  let loading = false
+  let validation = {}
+  let validateHash = ""
+  let errors = {}
+
+  $: displayColumnOptions = Object.keys(schema || {}).filter(column => {
+    return validation[column]
+  })
+
   async function handleFile(e) {
     loading = true
     error = null
@@ -178,7 +181,7 @@
     <Select
       label="Display Column"
       bind:value={displayColumn}
-      options={Object.keys(schema)}
+      options={displayColumnOptions}
       sort
     />
   </div>
diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts
index 785840b23e..5ced82d8cf 100644
--- a/packages/server/src/utilities/schema.ts
+++ b/packages/server/src/utilities/schema.ts
@@ -84,7 +84,7 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
         results.invalidColumns.push(columnName)
       } else if (!columnName.match(ValidColumnNameRegex)) {
         // Check for special characters in column names
-        results.invalidColumns.push(columnName)
+        results.schemaValidation[columnName] = false
         results.errors[columnName] =
           "Column names can't contain special characters"
       } else if (

From 1f6466f0ebe42fb5e199ee4f36ad7566fecb6b1c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 20 Jul 2023 14:31:20 +0100
Subject: [PATCH 3/4] Only show icon if an error message is present

---
 .../components/backend/TableNavigator/TableDataImport.svelte  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
index 6de069bab4..5b43d6e2ed 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
@@ -161,7 +161,9 @@
             Success
           {:else}
             Failure
-            <Icon name="Help" tooltip={errors[column.name]} />
+            {#if errors[column.name]}
+              <Icon name="Help" tooltip={errors[column.name]} />
+            {/if}
           {/if}
         </span>
         <Icon

From bac7a253c2dfcd7ef58be8baf4a8a4cfe3631685 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 20 Jul 2023 14:45:53 +0100
Subject: [PATCH 4/4] Fix the rest of the nasty flashing when working with
 importing CSVs

---
 .../TableNavigator/TableDataImport.svelte     | 43 ++++++++-----------
 1 file changed, 19 insertions(+), 24 deletions(-)

diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
index 5b43d6e2ed..0877ac201c 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
@@ -53,6 +53,15 @@
   $: displayColumnOptions = Object.keys(schema || {}).filter(column => {
     return validation[column]
   })
+  $: {
+    // binding in consumer is causing double renders here
+    const newValidateHash = JSON.stringify(rows) + JSON.stringify(schema)
+    if (newValidateHash !== validateHash) {
+      validate(rows, schema)
+    }
+    validateHash = newValidateHash
+  }
+  $: openFileUpload(promptUpload, fileInput)
 
   async function handleFile(e) {
     loading = true
@@ -73,7 +82,6 @@
 
   async function validate(rows, schema) {
     loading = true
-
     try {
       if (rows.length > 0) {
         const response = await API.validateNewTableImport({ rows, schema })
@@ -88,21 +96,9 @@
       allValid = false
       errors = {}
     }
-
     loading = false
   }
 
-  $: {
-    // binding in consumer is causing double renders here
-    const newValidateHash = JSON.stringify(rows) + JSON.stringify(schema)
-
-    if (newValidateHash !== validateHash) {
-      validate(rows, schema)
-    }
-
-    validateHash = newValidateHash
-  }
-
   const handleChange = (name, e) => {
     schema[name].type = e.detail
     schema[name].constraints = FIELDS[e.detail.toUpperCase()].constraints
@@ -114,7 +110,13 @@
     }
   }
 
-  $: openFileUpload(promptUpload, fileInput)
+  const deleteColumn = name => {
+    if (loading) {
+      return
+    }
+    delete schema[name]
+    schema = schema
+  }
 </script>
 
 <div class="dropzone">
@@ -127,10 +129,8 @@
     on:change={handleFile}
   />
   <label for="file-upload" class:uploaded={rows.length > 0}>
-    {#if loading}
-      loading...
-    {:else if error}
-      error: {error}
+    {#if error}
+      Error: {error}
     {:else if fileName}
       {fileName}
     {:else}
@@ -150,7 +150,6 @@
           placeholder={null}
           getOptionLabel={option => option.label}
           getOptionValue={option => option.value}
-          disabled={loading}
         />
         <span
           class={validation[column.name]
@@ -169,12 +168,8 @@
         <Icon
           size="S"
           name="Close"
-          disabled={loading}
           hoverable
-          on:click={() => {
-            delete schema[column.name]
-            schema = schema
-          }}
+          on:click={() => deleteColumn(column.name)}
         />
       </div>
     {/each}