Merge pull request #11299 from Budibase/fix/import-special-characters

Fix CSV imports with special characters breaking lucene

packages/bbui/src/Icon/Icon.svelte

@@ -47,7 +47,7 @@
   </svg>
   {#if tooltip && showTooltip}
     <div class="tooltip" in:fade={{ duration: 130, delay: 250 }}>
-      <Tooltip textWrapping direction="bottom" text={tooltip} />
+      <Tooltip textWrapping direction="top" text={tooltip} />
     </div>
   {/if}
 </div>
@@ -80,15 +80,9 @@
     position: absolute;
     pointer-events: none;
     left: 50%;
-    top: calc(100% + 4px);
-    width: 100vw;
-    max-width: 150px;
+    bottom: calc(100% + 4px);
     transform: translateX(-50%);
     text-align: center;
-  }
-
-  .spectrum-Icon--sizeXS {
-    width: 10px;
-    height: 10px;
+    z-index: 1;
   }
 </style>

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -33,6 +33,7 @@
   import { getBindings } from "components/backend/DataTable/formula"
   import { getContext } from "svelte"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
+  import { ValidColumnNameRegex } from "@budibase/shared-core"
 
   const AUTO_TYPE = "auto"
   const FORMULA_TYPE = FIELDS.FORMULA.type
@@ -379,7 +380,7 @@
     const newError = {}
     if (!external && fieldInfo.name?.startsWith("_")) {
       newError.name = `Column name cannot start with an underscore.`
-    } else if (fieldInfo.name && !fieldInfo.name.match(/^[_a-zA-Z0-9\s]*$/g)) {
+    } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) {
       newError.name = `Illegal character; must be alpha-numeric.`
     } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) {
       newError.name = `${PROHIBITED_COLUMN_NAMES.join(

packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte

@@ -1,18 +1,9 @@
 <script>
-  import { Select } from "@budibase/bbui"
+  import { Select, Icon } from "@budibase/bbui"
   import { FIELDS } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
 
-  let fileInput
-  let error = null
-  let fileName = null
-  let fileType = null
-
-  let loading = false
-  let validation = {}
-  let validateHash = ""
-
   export let rows = []
   export let schema = {}
   export let allValid = true
@@ -50,6 +41,28 @@
     },
   ]
 
+  let fileInput
+  let error = null
+  let fileName = null
+  let fileType = null
+  let loading = false
+  let validation = {}
+  let validateHash = ""
+  let errors = {}
+
+  $: displayColumnOptions = Object.keys(schema || {}).filter(column => {
+    return validation[column]
+  })
+  $: {
+    // binding in consumer is causing double renders here
+    const newValidateHash = JSON.stringify(rows) + JSON.stringify(schema)
+    if (newValidateHash !== validateHash) {
+      validate(rows, schema)
+    }
+    validateHash = newValidateHash
+  }
+  $: openFileUpload(promptUpload, fileInput)
+
   async function handleFile(e) {
     loading = true
     error = null
@@ -69,34 +82,23 @@
 
   async function validate(rows, schema) {
     loading = true
-    error = null
-    validation = {}
-    allValid = false
-
     try {
       if (rows.length > 0) {
         const response = await API.validateNewTableImport({ rows, schema })
         validation = response.schemaValidation
         allValid = response.allValid
+        errors = response.errors
+        error = null
       }
     } catch (e) {
       error = e.message
+      validation = {}
+      allValid = false
+      errors = {}
     }
-
     loading = false
   }
 
-  $: {
-    // binding in consumer is causing double renders here
-    const newValidateHash = JSON.stringify(rows) + JSON.stringify(schema)
-
-    if (newValidateHash !== validateHash) {
-      validate(rows, schema)
-    }
-
-    validateHash = newValidateHash
-  }
-
   const handleChange = (name, e) => {
     schema[name].type = e.detail
     schema[name].constraints = FIELDS[e.detail.toUpperCase()].constraints
@@ -108,7 +110,13 @@
     }
   }
 
-  $: openFileUpload(promptUpload, fileInput)
+  const deleteColumn = name => {
+    if (loading) {
+      return
+    }
+    delete schema[name]
+    schema = schema
+  }
 </script>
 
 <div class="dropzone">
@@ -121,10 +129,8 @@
     on:change={handleFile}
   />
   <label for="file-upload" class:uploaded={rows.length > 0}>
-    {#if loading}
-      loading...
-    {:else if error}
-      error: {error}
+    {#if error}
+      Error: {error}
     {:else if fileName}
       {fileName}
     {:else}
@@ -144,23 +150,26 @@
           placeholder={null}
           getOptionLabel={option => option.label}
           getOptionValue={option => option.value}
-          disabled={loading}
         />
         <span
-          class={loading || validation[column.name]
+          class={validation[column.name]
             ? "fieldStatusSuccess"
             : "fieldStatusFailure"}
         >
-          {validation[column.name] ? "Success" : "Failure"}
+          {#if validation[column.name]}
+            Success
+          {:else}
+            Failure
+            {#if errors[column.name]}
+              <Icon name="Help" tooltip={errors[column.name]} />
+            {/if}
+          {/if}
         </span>
-        <i
-          class={`omit-button ri-close-circle-fill ${
-            loading ? "omit-button-disabled" : ""
-          }`}
-          on:click={() => {
-            delete schema[column.name]
-            schema = schema
-          }}
+        <Icon
+          size="S"
+          name="Close"
+          hoverable
+          on:click={() => deleteColumn(column.name)}
         />
       </div>
     {/each}
@@ -169,7 +178,7 @@
     <Select
       label="Display Column"
       bind:value={displayColumn}
-      options={Object.keys(schema)}
+      options={displayColumnOptions}
       sort
     />
   </div>
@@ -237,23 +246,16 @@
     justify-self: center;
     font-weight: 600;
   }
-
   .fieldStatusFailure {
     color: var(--red);
     justify-self: center;
     font-weight: 600;
+    display: flex;
+    align-items: center;
+    gap: 4px;
   }
-
-  .omit-button {
-    font-size: 1.2em;
-    color: var(--grey-7);
-    cursor: pointer;
-    justify-self: flex-end;
-  }
-
-  .omit-button-disabled {
-    pointer-events: none;
-    opacity: 70%;
+  .fieldStatusFailure :global(.spectrum-Icon) {
+    width: 12px;
   }
 
   .display-column {

packages/server/src/utilities/schema.ts

@@ -1,4 +1,5 @@
 import { FieldTypes } from "../constants"
+import { ValidColumnNameRegex } from "@budibase/shared-core"
 
 interface SchemaColumn {
   readonly name: string
@@ -27,6 +28,7 @@ interface ValidationResults {
   schemaValidation: SchemaValidation
   allValid: boolean
   invalidColumns: Array<string>
+  errors: Record<string, string>
 }
 
 const PARSERS: any = {
@@ -69,6 +71,7 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
     schemaValidation: {},
     allValid: false,
     invalidColumns: [],
+    errors: {},
   }
 
   rows.forEach(row => {
@@ -79,6 +82,11 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
       // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array
       if (typeof columnType !== "string") {
         results.invalidColumns.push(columnName)
+      } else if (!columnName.match(ValidColumnNameRegex)) {
+        // Check for special characters in column names
+        results.schemaValidation[columnName] = false
+        results.errors[columnName] =
+          "Column names can't contain special characters"
       } else if (
         columnData == null &&
         !schema[columnName].constraints?.presence

packages/shared-core/src/constants.ts

@@ -89,3 +89,4 @@ export enum BuilderSocketEvent {
 }
 
 export const SocketSessionTTL = 60
+export const ValidColumnNameRegex = /^[_a-zA-Z0-9\s]*$/g