Merge branch 'master' into patch-1

eslint-local-rules/index.js

@@ -41,12 +41,11 @@ module.exports = {
           if (
             /^@budibase\/[^/]+\/.*$/.test(importPath) &&
             importPath !== "@budibase/backend-core/tests" &&
-            importPath !== "@budibase/string-templates/test/utils" &&
-            importPath !== "@budibase/client/manifest.json"
+            importPath !== "@budibase/string-templates/test/utils"
           ) {
             context.report({
               node,
-              message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests, @budibase/string-templates/test/utils and @budibase/client/manifest.json.`,
+              message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests and @budibase/string-templates/test/utils.`,
             })
           }
         },

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.3.5",
+  "version": "3.3.6",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/builder/src/helpers/screen.ts

@@ -1,46 +0,0 @@
-import { Component, Screen, ScreenProps } from "@budibase/types"
-import clientManifest from "@budibase/client/manifest.json"
-
-export function findComponentsBySettingsType(
-  screen: Screen,
-  type: string | string[]
-) {
-  const typesArray = Array.isArray(type) ? type : [type]
-
-  const result: {
-    component: Component
-    setting: {
-      type: string
-      key: string
-    }
-  }[] = []
-  function recurseFieldComponentsInChildren(component: ScreenProps) {
-    if (!component) {
-      return
-    }
-
-    const definition = getManifestDefinition(component)
-    const setting =
-      "settings" in definition &&
-      definition.settings.find((s: any) => typesArray.includes(s.type))
-    if (setting && "type" in setting) {
-      result.push({
-        component,
-        setting: { type: setting.type!, key: setting.key! },
-      })
-    }
-    component._children?.forEach(child => {
-      recurseFieldComponentsInChildren(child)
-    })
-  }
-
-  recurseFieldComponentsInChildren(screen?.props)
-  return result
-}
-
-function getManifestDefinition(component: Component) {
-  const componentType = component._component.split("/").slice(-1)[0]
-  const definition =
-    clientManifest[componentType as keyof typeof clientManifest]
-  return definition
-}

packages/builder/src/stores/builder/screenComponent.ts

@@ -2,12 +2,17 @@ import { derived } from "svelte/store"
 import { tables } from "./tables"
 import { selectedScreen } from "./screens"
 import { viewsV2 } from "./viewsV2"
-import { findComponentsBySettingsType } from "@/helpers/screen"
-import { UIDatasourceType, Screen, Component } from "@budibase/types"
+import {
+  UIDatasourceType,
+  Screen,
+  Component,
+  ScreenProps,
+} from "@budibase/types"
 import { queries } from "./queries"
 import { views } from "./views"
 import { bindings, featureFlag } from "@/helpers"
 import { getBindableProperties } from "@/dataBinding"
+import { componentStore, ComponentDefinition } from "./components"
 import { findAllComponents } from "@/helpers/components"
 
 function reduceBy<TItem extends {}, TKey extends keyof TItem>(
@@ -39,12 +44,16 @@ const validationKeyByType: Record<UIDatasourceType, string | null> = {
 }
 
 export const screenComponentErrors = derived(
-  [selectedScreen, tables, views, viewsV2, queries],
-  ([$selectedScreen, $tables, $views, $viewsV2, $queries]): Record<
-    string,
-    string[]
-  > => {
-    if (!featureFlag.isEnabled("CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS")) {
+  [selectedScreen, tables, views, viewsV2, queries, componentStore],
+  ([
+    $selectedScreen,
+    $tables,
+    $views,
+    $viewsV2,
+    $queries,
+    $componentStore,
+  ]): Record<string, string[]> => {
+    if (!featureFlag.isEnabled("CHECK_COMPONENT_SETTINGS_ERRORS")) {
       return {}
     }
     function getInvalidDatasources(
@@ -52,9 +61,11 @@ export const screenComponentErrors = derived(
       datasources: Record<string, any>
     ) {
       const result: Record<string, string[]> = {}
+
       for (const { component, setting } of findComponentsBySettingsType(
         screen,
-        ["table", "dataSource"]
+        ["table", "dataSource"],
+        $componentStore.components
       )) {
         const componentSettings = component[setting.key]
         if (!componentSettings) {
@@ -113,15 +124,52 @@ export const screenComponentErrors = derived(
   }
 )
 
+function findComponentsBySettingsType(
+  screen: Screen,
+  type: string | string[],
+  definitions: Record<string, ComponentDefinition>
+) {
+  const typesArray = Array.isArray(type) ? type : [type]
+
+  const result: {
+    component: Component
+    setting: {
+      type: string
+      key: string
+    }
+  }[] = []
+
+  function recurseFieldComponentsInChildren(component: ScreenProps) {
+    if (!component) {
+      return
+    }
+
+    const definition = definitions[component._component]
+
+    const setting = definition?.settings?.find((s: any) =>
+      typesArray.includes(s.type)
+    )
+    if (setting && "type" in setting) {
+      result.push({
+        component,
+        setting: { type: setting.type!, key: setting.key! },
+      })
+    }
+    component._children?.forEach(child => {
+      recurseFieldComponentsInChildren(child)
+    })
+  }
+
+  recurseFieldComponentsInChildren(screen?.props)
+  return result
+}
+
 export const screenComponents = derived(
   [selectedScreen],
   ([$selectedScreen]) => {
     if (!$selectedScreen) {
       return []
     }
-    const allComponents = findAllComponents(
-      $selectedScreen.props
-    ) as Component[]
-    return allComponents
+    return findAllComponents($selectedScreen.props) as Component[]
   }
 )

packages/server/src/utilities/csv.ts

@@ -1,22 +1,52 @@
 import csv from "csvtojson"
 
 export async function jsonFromCsvString(csvString: string) {
-  const castedWithEmptyValues = await csv({ ignoreEmpty: true }).fromString(
-    csvString
-  )
+  const possibleDelimiters = [",", ";", ":", "|", "~", "\t", " "]
 
-  // By default the csvtojson library casts empty values as empty strings. This
-  // is causing issues on conversion.  ignoreEmpty will remove the key completly
-  // if empty, so creating this empty object will ensure we return the values
-  // with the keys but empty values
-  const result = await csv({ ignoreEmpty: false }).fromString(csvString)
-  result.forEach((r, i) => {
-    for (const [key] of Object.entries(r).filter(([, value]) => value === "")) {
-      if (castedWithEmptyValues[i][key] === undefined) {
-        r[key] = null
+  for (let i = 0; i < possibleDelimiters.length; i++) {
+    let headers: string[] | undefined = undefined
+    let headerMismatch = false
+
+    try {
+      // By default the csvtojson library casts empty values as empty strings. This
+      // is causing issues on conversion.  ignoreEmpty will remove the key completly
+      // if empty, so creating this empty object will ensure we return the values
+      // with the keys but empty values
+      const result = await csv({
+        ignoreEmpty: false,
+        delimiter: possibleDelimiters[i],
+      }).fromString(csvString)
+      for (const [, row] of result.entries()) {
+        // The purpose of this is to find rows that have been split
+        // into the wrong number of columns - Any valid .CSV file will have
+        // the same number of colums in each row
+        // If the number of columms in each row is different to
+        // the number of headers, this isn't the right delimiter
+        const columns = Object.keys(row)
+        if (headers == null) {
+          headers = columns
+        }
+        if (headers.length === 1 || headers.length !== columns.length) {
+          headerMismatch = true
+          break
+        }
+
+        for (const header of headers) {
+          if (row[header] === undefined || row[header] === "") {
+            row[header] = null
+          }
+        }
       }
+      if (headerMismatch) {
+        continue
+      } else {
+        return result
+      }
+    } catch (err) {
+      // Splitting on the wrong delimiter sometimes throws CSV parsing error
+      // (eg unterminated strings), which tells us we've picked the wrong delimiter
+      continue
     }
-  })
-
-  return result
+  }
+  throw new Error("Unable to determine delimiter")
 }

packages/server/src/utilities/tests/csv.spec.ts

@@ -29,5 +29,34 @@ describe("csv", () => {
         expect(Object.keys(r)).toEqual(["id", "optional", "title"])
       )
     })
+
+    const possibleDelimeters = [",", ";", ":", "|", "~", "\t", " "]
+
+    const csvArray = [
+      ["id", "title"],
+      ["1", "aaa"],
+      ["2", "bbb"],
+      ["3", "c ccc"],
+      ["", ""],
+      [":5", "eee5:e"],
+    ]
+
+    test.each(possibleDelimeters)(
+      "Should parse with delimiter %s",
+      async delimiter => {
+        const csvString = csvArray
+          .map(row => row.map(col => `"${col}"`).join(delimiter))
+          .join("\n")
+        const result = await jsonFromCsvString(csvString)
+
+        expect(result).toEqual([
+          { id: "1", title: "aaa" },
+          { id: "2", title: "bbb" },
+          { id: "3", title: "c ccc" },
+          { id: null, title: null },
+          { id: ":5", title: "eee5:e" },
+        ])
+      }
+    )
   })
 })

packages/types/src/sdk/featureFlag.ts

@@ -1,6 +1,6 @@
 export enum FeatureFlag {
   USE_ZOD_VALIDATOR = "USE_ZOD_VALIDATOR",
-  CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS = "CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS",
+  CHECK_COMPONENT_SETTINGS_ERRORS = "CHECK_COMPONENT_SETTINGS_ERRORS",
 
   // Account-portal
   DIRECT_LOGIN_TO_ACCOUNT_PORTAL = "DIRECT_LOGIN_TO_ACCOUNT_PORTAL",
@@ -8,7 +8,7 @@ export enum FeatureFlag {
 
 export const FeatureFlagDefaults = {
   [FeatureFlag.USE_ZOD_VALIDATOR]: false,
-  [FeatureFlag.CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS]: false,
+  [FeatureFlag.CHECK_COMPONENT_SETTINGS_ERRORS]: false,
 
   // Account-portal
   [FeatureFlag.DIRECT_LOGIN_TO_ACCOUNT_PORTAL]: false,