Export data make CSV delimiter configurable (#13028)

* Add delimiter option

* Add custom delimiter

* external export delimiter

* Custom headers for row export

* External export rows custom headers

* Support custom JSON export labels

* Handle export table source switch

* update account portal

* Add space as delimiter

* Refactor

* update account portal

packages/account-portal

@@ -1 +1 @@
-Subproject commit ab324e35d855012bd0f49caa53c6dd765223c6fa
+Subproject commit de6d44c372a7f48ca0ce8c6c0c19311d4bc21646

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExportData.svelte

@@ -1,9 +1,9 @@
 <script>
-  import { Label, Select, Body, Multiselect } from "@budibase/bbui"
-  import { findAllMatchingComponents, findComponent } from "helpers/components"
-  import { selectedScreen } from "stores/builder"
+  import { Label, Select, Body } from "@budibase/bbui"
   import { onMount } from "svelte"
-  import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding"
+  import ColumnEditor from "../../ColumnEditor/ColumnEditor.svelte"
+  import { findAllMatchingComponents } from "helpers/components"
+  import { selectedScreen } from "stores/builder"
 
   export let parameters
 
@@ -18,37 +18,65 @@
     },
   ]
 
+  const DELIMITERS = [
+    {
+      label: ",",
+      value: ",",
+    },
+    {
+      label: ";",
+      value: ";",
+    },
+    {
+      label: ":",
+      value: ":",
+    },
+    {
+      label: "|",
+      value: "|",
+    },
+    {
+      label: "~",
+      value: "~",
+    },
+    {
+      label: "[tab]",
+      value: "\t",
+    },
+    {
+      label: "[space]",
+      value: " ",
+    },
+  ]
+
   $: tables = findAllMatchingComponents($selectedScreen?.props, component =>
     component._component.endsWith("table")
-  ).map(table => ({
-    label: table._instanceName,
-    value: table._id,
-  }))
+  )
   $: tableBlocks = findAllMatchingComponents(
     $selectedScreen?.props,
     component => component._component.endsWith("tableblock")
-  ).map(block => ({
-    label: block._instanceName,
-    value: `${block._id}-table`,
+  )
+  $: components = tables.concat(tableBlocks)
+  $: componentOptions = components.map(table => ({
+    label: table._instanceName,
+    value: table._component.includes("tableblock")
+      ? `${table._id}-table`
+      : table._id,
   }))
-  $: componentOptions = tables.concat(tableBlocks)
-  $: columnOptions = getColumnOptions(parameters.tableComponentId)
-
-  const getColumnOptions = tableId => {
-    // Strip block suffix if block component
-    if (tableId?.includes("-")) {
-      tableId = tableId.split("-")[0]
-    }
-    const selectedTable = findComponent($selectedScreen?.props, tableId)
-    const datasource = getDatasourceForProvider($selectedScreen, selectedTable)
-    const { schema } = getSchemaForDatasource($selectedScreen, datasource)
-    return Object.keys(schema || {})
-  }
+  $: selectedTableId = parameters.tableComponentId?.includes("-")
+    ? parameters.tableComponentId.split("-")[0]
+    : parameters.tableComponentId
+  $: selectedTable = components.find(
+    component => component._id === selectedTableId
+  )
 
   onMount(() => {
     if (!parameters.type) {
       parameters.type = "csv"
     }
+    if (!parameters.delimiter) {
+      parameters.delimiter = ","
+    }
   })
 </script>
 
@@ -67,13 +95,29 @@
       options={componentOptions}
       on:change={() => (parameters.columns = [])}
     />
+    <span />
     <Label small>Export as</Label>
     <Select bind:value={parameters.type} options={FORMATS} />
+    <Select
+      bind:value={parameters.delimiter}
+      placeholder={null}
+      options={DELIMITERS}
+      disabled={parameters.type !== "csv"}
+    />
     <Label small>Export columns</Label>
-    <Multiselect
-      placeholder="All columns"
+    <ColumnEditor
       bind:value={parameters.columns}
-      options={columnOptions}
+      allowCellEditing={false}
+      componentInstance={selectedTable}
+      on:change={e => {
+        const columns = e.detail
+        parameters.customHeaders = columns.reduce((headerMap, column) => {
+          return {
+            [column.name]: column.displayName,
+            ...headerMap,
+          }
+        }, {})
+      }}
     />
   </div>
 </div>
@@ -97,8 +141,8 @@
   .params {
     display: grid;
     column-gap: var(--spacing-xs);
-    row-gap: var(--spacing-s);
-    grid-template-columns: 90px 1fr;
+    row-gap: var(--spacing-m);
+    grid-template-columns: 90px 1fr 90px;
     align-items: center;
   }
 </style>

packages/builder/src/components/design/settings/controls/ColumnEditor/ColumnEditor.svelte

@@ -29,6 +29,12 @@
     allowLinks: true,
   })
 
+  $: {
+    value = (value || []).filter(
+      column => (schema || {})[column.name || column] !== undefined
+    )
+  }
+
   const getText = value => {
     if (!value?.length) {
       return "All columns"

packages/client/src/utils/buttonActions.js

@@ -341,7 +341,11 @@ const exportDataHandler = async action => {
         tableId: selection.tableId,
         rows: selection.selectedRows,
         format: action.parameters.type,
-        columns: action.parameters.columns,
+        columns: action.parameters.columns?.map(
+          column => column.name || column
+        ),
+        delimiter: action.parameters.delimiter,
+        customHeaders: action.parameters.customHeaders,
       })
       download(
         new Blob([data], { type: "text/plain" }),

packages/frontend-core/src/api/rows.js

@@ -89,13 +89,24 @@ export const buildRowEndpoints = API => ({
    * @param rows the array of rows to export
    * @param format the format to export (csv or json)
    * @param columns which columns to export (all if undefined)
+   * @param delimiter how values should be separated in a CSV (default is comma)
    */
-  exportRows: async ({ tableId, rows, format, columns, search }) => {
+  exportRows: async ({
+    tableId,
+    rows,
+    format,
+    columns,
+    search,
+    delimiter,
+    customHeaders,
+  }) => {
     return await API.post({
       url: `/api/${tableId}/rows/exportRows?format=${format}`,
       body: {
         rows,
         columns,
+        delimiter,
+        customHeaders,
         ...search,
       },
       parseResponse: async response => {

packages/server/src/api/controllers/row/index.ts

@@ -223,7 +223,8 @@ export const exportRows = async (
 
   const format = ctx.query.format
 
-  const { rows, columns, query, sort, sortOrder } = ctx.request.body
+  const { rows, columns, query, sort, sortOrder, delimiter, customHeaders } =
+    ctx.request.body
   if (typeof format !== "string" || !exporters.isFormat(format)) {
     ctx.throw(
       400,
@@ -241,6 +242,8 @@ export const exportRows = async (
     query,
     sort,
     sortOrder,
+    delimiter,
+    customHeaders,
   })
   ctx.attachment(fileName)
   ctx.body = apiFileReturn(content)

packages/server/src/api/controllers/view/exporters.ts

@@ -1,7 +1,19 @@
 import { Row, TableSchema } from "@budibase/types"
 
-export function csv(headers: string[], rows: Row[]) {
-  let csv = headers.map(key => `"${key}"`).join(",")
+function getHeaders(
+  headers: string[],
+  customHeaders: { [key: string]: string }
+) {
+  return headers.map(header => `"${customHeaders[header] || header}"`)
+}
+
+export function csv(
+  headers: string[],
+  rows: Row[],
+  delimiter: string = ",",
+  customHeaders: { [key: string]: string } = {}
+) {
+  let csv = getHeaders(headers, customHeaders).join(delimiter)
 
   for (let row of rows) {
     csv = `${csv}\n${headers
@@ -15,7 +27,7 @@ export function csv(headers: string[], rows: Row[]) {
             : ""
         return val.trim()
       })
-      .join(",")}`
+      .join(delimiter)}`
   }
   return csv
 }

packages/server/src/sdk/app/rows/search.ts

@@ -36,11 +36,13 @@ export async function search(options: SearchParams): Promise<{
 export interface ExportRowsParams {
   tableId: string
   format: Format
+  delimiter?: string
   rowIds?: string[]
   columns?: string[]
   query?: SearchFilters
   sort?: string
   sortOrder?: SortOrder
+  customHeaders?: { [key: string]: string }
 }
 
 export interface ExportRowsResult {

packages/server/src/sdk/app/rows/search/external.ts

@@ -101,7 +101,17 @@ export async function search(options: SearchParams) {
 export async function exportRows(
   options: ExportRowsParams
 ): Promise<ExportRowsResult> {
-  const { tableId, format, columns, rowIds, query, sort, sortOrder } = options
+  const {
+    tableId,
+    format,
+    columns,
+    rowIds,
+    query,
+    sort,
+    sortOrder,
+    delimiter,
+    customHeaders,
+  } = options
   const { datasourceId, tableName } = breakExternalTableId(tableId)
 
   let requestQuery: SearchFilters = {}
@@ -153,12 +163,17 @@ export async function exportRows(
     rows = result.rows
   }
 
-  let exportRows = cleanExportRows(rows, schema, format, columns)
+  let exportRows = cleanExportRows(rows, schema, format, columns, customHeaders)
 
   let content: string
   switch (format) {
     case exporters.Format.CSV:
-      content = exporters.csv(headers ?? Object.keys(schema), exportRows)
+      content = exporters.csv(
+        headers ?? Object.keys(schema),
+        exportRows,
+        delimiter,
+        customHeaders
+      )
       break
     case exporters.Format.JSON:
       content = exporters.json(exportRows)

packages/server/src/sdk/app/rows/search/internal.ts

@@ -84,7 +84,17 @@ export async function search(options: SearchParams) {
 export async function exportRows(
   options: ExportRowsParams
 ): Promise<ExportRowsResult> {
-  const { tableId, format, rowIds, columns, query, sort, sortOrder } = options
+  const {
+    tableId,
+    format,
+    rowIds,
+    columns,
+    query,
+    sort,
+    sortOrder,
+    delimiter,
+    customHeaders,
+  } = options
   const db = context.getAppDB()
   const table = await sdk.tables.getTable(tableId)
 
@@ -124,11 +134,16 @@ export async function exportRows(
     rows = result
   }
 
-  let exportRows = cleanExportRows(rows, schema, format, columns)
+  let exportRows = cleanExportRows(rows, schema, format, columns, customHeaders)
   if (format === Format.CSV) {
     return {
       fileName: "export.csv",
-      content: csv(headers ?? Object.keys(rows[0]), exportRows),
+      content: csv(
+        headers ?? Object.keys(rows[0]),
+        exportRows,
+        delimiter,
+        customHeaders
+      ),
     }
   } else if (format === Format.JSON) {
     return {

packages/server/src/sdk/app/rows/utils.ts

@@ -16,7 +16,8 @@ export function cleanExportRows(
   rows: any[],
   schema: TableSchema,
   format: string,
-  columns?: string[]
+  columns?: string[],
+  customHeaders: { [key: string]: string } = {}
 ) {
   let cleanRows = [...rows]
 
@@ -44,11 +45,27 @@ export function cleanExportRows(
         }
       }
     }
+  } else if (format === Format.JSON) {
+    // Replace row keys with custom headers
+    for (let row of cleanRows) {
+      renameKeys(customHeaders, row)
+    }
   }
 
   return cleanRows
 }
 
+function renameKeys(keysMap: { [key: string]: any }, row: any) {
+  for (const key in keysMap) {
+    Object.defineProperty(
+      row,
+      keysMap[key],
+      Object.getOwnPropertyDescriptor(row, key) || {}
+    )
+    delete row[key]
+  }
+}
+
 function isForeignKey(key: string, table: Table) {
   const relationships = Object.values(table.schema).filter(isRelationshipColumn)
   return relationships.some(

packages/types/src/api/web/app/rows.ts

@@ -37,6 +37,8 @@ export interface ExportRowsRequest {
   query?: SearchFilters
   sort?: string
   sortOrder?: SortOrder
+  delimiter?: string
+  customHeaders?: { [key: string]: string }
 }
 
 export type ExportRowsResponse = ReadStream