diff --git a/eslint-local-rules/index.js b/eslint-local-rules/index.js index 9348706399..d9d894c33e 100644 --- a/eslint-local-rules/index.js +++ b/eslint-local-rules/index.js @@ -41,12 +41,11 @@ module.exports = { if ( /^@budibase\/[^/]+\/.*$/.test(importPath) && importPath !== "@budibase/backend-core/tests" && - importPath !== "@budibase/string-templates/test/utils" && - importPath !== "@budibase/client/manifest.json" + importPath !== "@budibase/string-templates/test/utils" ) { context.report({ node, - message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests, @budibase/string-templates/test/utils and @budibase/client/manifest.json.`, + message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests and @budibase/string-templates/test/utils.`, }) } }, diff --git a/lerna.json b/lerna.json index c16b958d24..15f405c847 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.3.5", + "version": "3.3.6", "npmClient": "yarn", "concurrency": 20, "command": { diff --git a/packages/builder/src/helpers/screen.ts b/packages/builder/src/helpers/screen.ts deleted file mode 100644 index 296a597adb..0000000000 --- a/packages/builder/src/helpers/screen.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { Component, Screen, ScreenProps } from "@budibase/types" -import clientManifest from "@budibase/client/manifest.json" - -export function findComponentsBySettingsType( - screen: Screen, - type: string | string[] -) { - const typesArray = Array.isArray(type) ? type : [type] - - const result: { - component: Component - setting: { - type: string - key: string - } - }[] = [] - function recurseFieldComponentsInChildren(component: ScreenProps) { - if (!component) { - return - } - - const definition = getManifestDefinition(component) - const setting = - "settings" in definition && - definition.settings.find((s: any) => typesArray.includes(s.type)) - if (setting && "type" in setting) { - result.push({ - component, - setting: { type: setting.type!, key: setting.key! }, - }) - } - component._children?.forEach(child => { - recurseFieldComponentsInChildren(child) - }) - } - - recurseFieldComponentsInChildren(screen?.props) - return result -} - -function getManifestDefinition(component: Component) { - const componentType = component._component.split("/").slice(-1)[0] - const definition = - clientManifest[componentType as keyof typeof clientManifest] - return definition -} diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 56e9d311a4..f1a9440c02 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -2,12 +2,17 @@ import { derived } from "svelte/store" import { tables } from "./tables" import { selectedScreen } from "./screens" import { viewsV2 } from "./viewsV2" -import { findComponentsBySettingsType } from "@/helpers/screen" -import { UIDatasourceType, Screen, Component } from "@budibase/types" +import { + UIDatasourceType, + Screen, + Component, + ScreenProps, +} from "@budibase/types" import { queries } from "./queries" import { views } from "./views" import { bindings, featureFlag } from "@/helpers" import { getBindableProperties } from "@/dataBinding" +import { componentStore, ComponentDefinition } from "./components" import { findAllComponents } from "@/helpers/components" function reduceBy( @@ -39,12 +44,16 @@ const validationKeyByType: Record = { } export const screenComponentErrors = derived( - [selectedScreen, tables, views, viewsV2, queries], - ([$selectedScreen, $tables, $views, $viewsV2, $queries]): Record< - string, - string[] - > => { - if (!featureFlag.isEnabled("CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS")) { + [selectedScreen, tables, views, viewsV2, queries, componentStore], + ([ + $selectedScreen, + $tables, + $views, + $viewsV2, + $queries, + $componentStore, + ]): Record => { + if (!featureFlag.isEnabled("CHECK_COMPONENT_SETTINGS_ERRORS")) { return {} } function getInvalidDatasources( @@ -52,9 +61,11 @@ export const screenComponentErrors = derived( datasources: Record ) { const result: Record = {} + for (const { component, setting } of findComponentsBySettingsType( screen, - ["table", "dataSource"] + ["table", "dataSource"], + $componentStore.components )) { const componentSettings = component[setting.key] if (!componentSettings) { @@ -113,15 +124,52 @@ export const screenComponentErrors = derived( } ) +function findComponentsBySettingsType( + screen: Screen, + type: string | string[], + definitions: Record +) { + const typesArray = Array.isArray(type) ? type : [type] + + const result: { + component: Component + setting: { + type: string + key: string + } + }[] = [] + + function recurseFieldComponentsInChildren(component: ScreenProps) { + if (!component) { + return + } + + const definition = definitions[component._component] + + const setting = definition?.settings?.find((s: any) => + typesArray.includes(s.type) + ) + if (setting && "type" in setting) { + result.push({ + component, + setting: { type: setting.type!, key: setting.key! }, + }) + } + component._children?.forEach(child => { + recurseFieldComponentsInChildren(child) + }) + } + + recurseFieldComponentsInChildren(screen?.props) + return result +} + export const screenComponents = derived( [selectedScreen], ([$selectedScreen]) => { if (!$selectedScreen) { return [] } - const allComponents = findAllComponents( - $selectedScreen.props - ) as Component[] - return allComponents + return findAllComponents($selectedScreen.props) as Component[] } ) diff --git a/packages/server/src/utilities/csv.ts b/packages/server/src/utilities/csv.ts index 43d712165a..ac9304b12a 100644 --- a/packages/server/src/utilities/csv.ts +++ b/packages/server/src/utilities/csv.ts @@ -1,22 +1,52 @@ import csv from "csvtojson" export async function jsonFromCsvString(csvString: string) { - const castedWithEmptyValues = await csv({ ignoreEmpty: true }).fromString( - csvString - ) + const possibleDelimiters = [",", ";", ":", "|", "~", "\t", " "] - // By default the csvtojson library casts empty values as empty strings. This - // is causing issues on conversion. ignoreEmpty will remove the key completly - // if empty, so creating this empty object will ensure we return the values - // with the keys but empty values - const result = await csv({ ignoreEmpty: false }).fromString(csvString) - result.forEach((r, i) => { - for (const [key] of Object.entries(r).filter(([, value]) => value === "")) { - if (castedWithEmptyValues[i][key] === undefined) { - r[key] = null + for (let i = 0; i < possibleDelimiters.length; i++) { + let headers: string[] | undefined = undefined + let headerMismatch = false + + try { + // By default the csvtojson library casts empty values as empty strings. This + // is causing issues on conversion. ignoreEmpty will remove the key completly + // if empty, so creating this empty object will ensure we return the values + // with the keys but empty values + const result = await csv({ + ignoreEmpty: false, + delimiter: possibleDelimiters[i], + }).fromString(csvString) + for (const [, row] of result.entries()) { + // The purpose of this is to find rows that have been split + // into the wrong number of columns - Any valid .CSV file will have + // the same number of colums in each row + // If the number of columms in each row is different to + // the number of headers, this isn't the right delimiter + const columns = Object.keys(row) + if (headers == null) { + headers = columns + } + if (headers.length === 1 || headers.length !== columns.length) { + headerMismatch = true + break + } + + for (const header of headers) { + if (row[header] === undefined || row[header] === "") { + row[header] = null + } + } } + if (headerMismatch) { + continue + } else { + return result + } + } catch (err) { + // Splitting on the wrong delimiter sometimes throws CSV parsing error + // (eg unterminated strings), which tells us we've picked the wrong delimiter + continue } - }) - - return result + } + throw new Error("Unable to determine delimiter") } diff --git a/packages/server/src/utilities/tests/csv.spec.ts b/packages/server/src/utilities/tests/csv.spec.ts index 14063d0e8e..b1dc192bf0 100644 --- a/packages/server/src/utilities/tests/csv.spec.ts +++ b/packages/server/src/utilities/tests/csv.spec.ts @@ -29,5 +29,34 @@ describe("csv", () => { expect(Object.keys(r)).toEqual(["id", "optional", "title"]) ) }) + + const possibleDelimeters = [",", ";", ":", "|", "~", "\t", " "] + + const csvArray = [ + ["id", "title"], + ["1", "aaa"], + ["2", "bbb"], + ["3", "c ccc"], + ["", ""], + [":5", "eee5:e"], + ] + + test.each(possibleDelimeters)( + "Should parse with delimiter %s", + async delimiter => { + const csvString = csvArray + .map(row => row.map(col => `"${col}"`).join(delimiter)) + .join("\n") + const result = await jsonFromCsvString(csvString) + + expect(result).toEqual([ + { id: "1", title: "aaa" }, + { id: "2", title: "bbb" }, + { id: "3", title: "c ccc" }, + { id: null, title: null }, + { id: ":5", title: "eee5:e" }, + ]) + } + ) }) }) diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts index d9f092c80a..97893a1b5e 100644 --- a/packages/types/src/sdk/featureFlag.ts +++ b/packages/types/src/sdk/featureFlag.ts @@ -1,6 +1,6 @@ export enum FeatureFlag { USE_ZOD_VALIDATOR = "USE_ZOD_VALIDATOR", - CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS = "CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS", + CHECK_COMPONENT_SETTINGS_ERRORS = "CHECK_COMPONENT_SETTINGS_ERRORS", // Account-portal DIRECT_LOGIN_TO_ACCOUNT_PORTAL = "DIRECT_LOGIN_TO_ACCOUNT_PORTAL", @@ -8,7 +8,7 @@ export enum FeatureFlag { export const FeatureFlagDefaults = { [FeatureFlag.USE_ZOD_VALIDATOR]: false, - [FeatureFlag.CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS]: false, + [FeatureFlag.CHECK_COMPONENT_SETTINGS_ERRORS]: false, // Account-portal [FeatureFlag.DIRECT_LOGIN_TO_ACCOUNT_PORTAL]: false,