From b5c98871adeb7af58ba35d58a3be92aaab306526 Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Thu, 25 May 2023 18:05:07 +0100
Subject: [PATCH] Update table data via CSV import  (#10313)

* Add identifierFields select for import

* Update rows on import (Internal DB)

* Only allow internal DB to upsert import CSV

* Clear identifierFields when turning off update

* Passing table instead of tableId

* Pass table

* Pass tableType
---
 .../backend/DataTable/DataTable.svelte        |  1 +
 .../DataTable/buttons/ImportButton.svelte     |  3 +-
 .../buttons/grid/GridImportButton.svelte      |  3 +-
 .../DataTable/modals/ImportModal.svelte       | 12 ++++++-
 .../ExistingTableDataImport.svelte            | 21 +++++++++++-
 packages/frontend-core/src/api/tables.js      |  6 ++--
 .../src/components/grid/layout/Grid.svelte    |  2 ++
 .../src/api/controllers/table/internal.ts     |  8 ++---
 .../server/src/api/controllers/table/utils.ts | 33 ++++++++++++++++++-
 9 files changed, 76 insertions(+), 13 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte
index dfe30a3711..83e115ebf2 100644
--- a/packages/builder/src/components/backend/DataTable/DataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte
@@ -32,6 +32,7 @@
   <Grid
     {API}
     tableId={id}
+    tableType={$tables.selected?.type}
     allowAddRows={!isUsersTable}
     allowDeleteRows={!isUsersTable}
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
diff --git a/packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte
index 88ef30210a..88c57daa21 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte
@@ -3,6 +3,7 @@
   import ImportModal from "../modals/ImportModal.svelte"
 
   export let tableId
+  export let tableType
   export let disabled
 
   let modal
@@ -12,5 +13,5 @@
   Import
 </ActionButton>
 <Modal bind:this={modal}>
-  <ImportModal {tableId} on:importrows />
+  <ImportModal {tableId} {tableType} on:importrows />
 </Modal>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte
index 5aa530b028..5b89d92438 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte
@@ -4,11 +4,12 @@
 
   export let disabled = false
 
-  const { rows, tableId } = getContext("grid")
+  const { rows, tableId, tableType } = getContext("grid")
 </script>
 
 <ImportButton
   {disabled}
   tableId={$tableId}
+  {tableType}
   on:importrows={rows.actions.refreshData}
 />
diff --git a/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte
index 173fb88829..c020d1a7ac 100644
--- a/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte
@@ -13,15 +13,18 @@
   const dispatch = createEventDispatcher()
 
   export let tableId
+  export let tableType
   let rows = []
   let allValid = false
   let displayColumn = null
+  let identifierFields = []
 
   async function importData() {
     try {
       await API.importTableData({
         tableId,
         rows,
+        identifierFields,
       })
       notifications.success("Rows successfully imported")
     } catch (error) {
@@ -45,6 +48,13 @@
   </Body>
   <Layout gap="XS" noPadding>
     <Label grey extraSmall>CSV or JSON file to import</Label>
-    <TableDataImport {tableId} bind:rows bind:allValid bind:displayColumn />
+    <TableDataImport
+      {tableId}
+      {tableType}
+      bind:rows
+      bind:allValid
+      bind:displayColumn
+      bind:identifierFields
+    />
   </Layout>
 </ModalContent>
diff --git a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte
index cb150f71fc..eba17e8ab9 100644
--- a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { Select } from "@budibase/bbui"
+  import { Select, Toggle, Multiselect } from "@budibase/bbui"
   import { FIELDS } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
@@ -9,14 +9,17 @@
   let fileType = null
 
   let loading = false
+  let updateExistingRows = false
   let validation = {}
   let validateHash = ""
   let schema = null
   let invalidColumns = []
 
   export let tableId = null
+  export let tableType
   export let rows = []
   export let allValid = false
+  export let identifierFields = []
 
   const typeOptions = [
     {
@@ -159,6 +162,22 @@
       </div>
     {/each}
   </div>
+  {#if tableType === "internal"}
+    <br />
+    <Toggle
+      bind:value={updateExistingRows}
+      on:change={() => (identifierFields = [])}
+      thin
+      text="Update existing rows"
+    />
+    {#if updateExistingRows}
+      <Multiselect
+        label="Identifier field(s)"
+        options={Object.keys(validation)}
+        bind:value={identifierFields}
+      />
+    {/if}
+  {/if}
   {#if invalidColumns.length > 0}
     <p class="spectrum-FieldLabel spectrum-FieldLabel--sizeM">
       The following columns are present in the data you wish to import, but do
diff --git a/packages/frontend-core/src/api/tables.js b/packages/frontend-core/src/api/tables.js
index 00f0e9ff2e..b84bd3076a 100644
--- a/packages/frontend-core/src/api/tables.js
+++ b/packages/frontend-core/src/api/tables.js
@@ -62,13 +62,15 @@ export const buildTableEndpoints = API => ({
   /**
    * Imports data into an existing table
    * @param tableId the table ID to import to
-   * @param data the data import object
+   * @param rows the data import object
+   * @param identifierFields column names to be used as keys for overwriting existing rows
    */
-  importTableData: async ({ tableId, rows }) => {
+  importTableData: async ({ tableId, rows, identifierFields }) => {
     return await API.post({
       url: `/api/tables/${tableId}/import`,
       body: {
         rows,
+        identifierFields,
       },
     })
   },
diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte
index 2035ec4d39..cb02263be3 100644
--- a/packages/frontend-core/src/components/grid/layout/Grid.svelte
+++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte
@@ -33,6 +33,7 @@
 
   export let API = null
   export let tableId = null
+  export let tableType = null
   export let schemaOverrides = null
   export let allowAddRows = true
   export let allowAddColumns = true
@@ -62,6 +63,7 @@
     rand,
     config,
     tableId: tableIdStore,
+    tableType,
     schemaOverrides: schemaOverridesStore,
   }
   context = { ...context, ...createEventManagers() }
diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts
index d2a4de575e..a688befcbe 100644
--- a/packages/server/src/api/controllers/table/internal.ts
+++ b/packages/server/src/api/controllers/table/internal.ts
@@ -186,11 +186,7 @@ export async function destroy(ctx: any) {
 export async function bulkImport(ctx: any) {
   const db = context.getAppDB()
   const table = await sdk.tables.getTable(ctx.params.tableId)
-  const { rows } = ctx.request.body
-  await handleDataImport(ctx.user, table, rows)
-
-  // Ensure auto id and other table updates are persisted
-  await db.put(table)
-
+  const { rows, identifierFields } = ctx.request.body
+  await handleDataImport(ctx.user, table, rows, identifierFields)
   return table
 }
diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts
index f088dbaa8e..923608e9a2 100644
--- a/packages/server/src/api/controllers/table/utils.ts
+++ b/packages/server/src/api/controllers/table/utils.ts
@@ -149,7 +149,12 @@ export function importToRows(
   return finalData
 }
 
-export async function handleDataImport(user: any, table: any, rows: any) {
+export async function handleDataImport(
+  user: any,
+  table: any,
+  rows: any,
+  identifierFields: Array<string> = []
+) {
   const schema: unknown = table.schema
 
   if (!rows || !isRows(rows) || !isSchema(schema)) {
@@ -161,6 +166,32 @@ export async function handleDataImport(user: any, table: any, rows: any) {
 
   let finalData: any = importToRows(data, table, user)
 
+  //Set IDs of finalData to match existing row if an update is expected
+  if (identifierFields.length > 0) {
+    const allDocs = await db.allDocs(
+      getRowParams(table._id, null, {
+        include_docs: true,
+      })
+    )
+    allDocs.rows
+      .map(existingRow => existingRow.doc)
+      .forEach((doc: any) => {
+        finalData.forEach((finalItem: any) => {
+          let match = true
+          for (const field of identifierFields) {
+            if (finalItem[field] !== doc[field]) {
+              match = false
+              break
+            }
+          }
+          if (match) {
+            finalItem._id = doc._id
+            finalItem._rev = doc._rev
+          }
+        })
+      })
+  }
+
   await quotas.addRows(finalData.length, () => db.bulkDocs(finalData), {
     tableId: table._id,
   })