MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3356 from Budibase/feature/existing-table-import 

Import CSV to existing table from builder
This commit is contained in:
Michael Drury 2021-11-15 14:32:41 +00:00 committed by GitHub
parent 04cfca8b8e 72a3532cc8
commit 716864b38f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 277 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/bbui/src/InlineAlert/InlineAlert.svelte
View File

@ -47,5 +47,6 @@
--spectrum-semantic-positive-border-color: #2d9d78;
--spectrum-semantic-positive-icon-color: #2d9d78;
--spectrum-semantic-negative-icon-color: #e34850;
min-width: 100px;
}
</style>

5
packages/builder/src/components/backend/DataTable/DataTable.svelte
View File

@ -6,6 +6,7 @@
import CreateViewButton from "./buttons/CreateViewButton.svelte"
import ExistingRelationshipButton from "./buttons/ExistingRelationshipButton.svelte"
import ExportButton from "./buttons/ExportButton.svelte"
import ImportButton from "./buttons/ImportButton.svelte"
import EditRolesButton from "./buttons/EditRolesButton.svelte"
import ManageAccessButton from "./buttons/ManageAccessButton.svelte"
import HideAutocolumnButton from "./buttons/HideAutocolumnButton.svelte"
@ -124,6 +125,10 @@
<HideAutocolumnButton bind:hideAutocolumns />
<!-- always have the export last -->
<ExportButton view={$tables.selected?._id} />
<ImportButton
tableId={$tables.selected?._id}
on:updaterows={onUpdateRows}
/>
{#key id}
<TableFilterButton {schema} on:change={onFilter} />
{/key}

2
packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte
View File

@ -7,7 +7,7 @@
let modal
</script>
<ActionButton icon="Download" size="S" quiet on:click={modal.show}>
<ActionButton icon="DataDownload" size="S" quiet on:click={modal.show}>
Export
</ActionButton>
<Modal bind:this={modal}>

15
packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte Normal file
View File

@ -0,0 +1,15 @@
<script>
import { ActionButton, Modal } from "@budibase/bbui"
import ImportModal from "../modals/ImportModal.svelte"
export let tableId
let modal
</script>
<ActionButton icon="DataUpload" size="S" quiet on:click={modal.show}>
Import
</ActionButton>
<Modal bind:this={modal}>
<ImportModal {tableId} on:updaterows />
</Modal>

43
packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte Normal file
View File

@ -0,0 +1,43 @@
<script>
import { ModalContent, Label, notifications, Body } from "@budibase/bbui"
import TableDataImport from "../../TableNavigator/TableDataImport.svelte"
import api from "builderStore/api"
import { createEventDispatcher } from "svelte"
const dispatch = createEventDispatcher()
export let tableId
let dataImport
$: valid = dataImport?.csvString != null && dataImport?.valid
async function importData() {
const response = await api.post(`/api/tables/${tableId}/import`, {
dataImport,
})
if (response.status !== 200) {
const error = await response.text()
notifications.error(`Unable to import data - ${error}`)
} else {
notifications.success("Rows successfully imported.")
}
dispatch("updaterows")
}
</script>
<ModalContent
title="Import Data"
confirmText="Import"
onConfirm={importData}
disabled={!valid}
>
<Body
>Import rows to an existing table from a CSV. Only columns from the CSV
which exist in the table will be imported.</Body
>
<Label grey extraSmall>CSV to import</Label>
<TableDataImport bind:dataImport bind:existingTableId={tableId} />
</ModalContent>
<style>
</style>

40
packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
View File

@ -1,6 +1,5 @@
<script>
import { Select } from "@budibase/bbui"
import { notifications } from "@budibase/bbui"
import { Select, InlineAlert, notifications } from "@budibase/bbui"
import { FIELDS } from "constants/backend"
import api from "builderStore/api"
@ -12,11 +11,13 @@
valid: true,
schema: {},
}
export let existingTableId
let csvString
let primaryDisplay
let csvString = undefined
let primaryDisplay = undefined
let schema = {}
let fields = []
let hasValidated = false
$: valid = !schema || fields.every(column => schema[column].success)
$: dataImport = {
@ -25,6 +26,9 @@
csvString,
primaryDisplay,
}
$: noFieldsError = existingTableId
? "No columns in CSV match existing table schema"
: "Could not find any columns to import"
function buildTableSchema(schema) {
const tableSchema = {}
@ -46,6 +50,7 @@
const response = await api.post("/api/tables/csv/validate", {
csvString,
schema: schema || {},
tableId: existingTableId,
})
const parseResult = await response.json()
@ -63,6 +68,7 @@
notifications.error("CSV Invalid, please try another CSV file")
return []
}
hasValidated = true
}
async function handleFile(evt) {
@ -138,6 +144,7 @@
placeholder={null}
getOptionLabel={option => option.label}
getOptionValue={option => option.value}
disabled={!!existingTableId}
/>
<span class="field-status" class:error={!schema[columnName].success}>
{schema[columnName].success ? "Success" : "Failure"}
@ -149,15 +156,22 @@
</div>
{/each}
</div>
{/if}
{#if fields.length}
<div class="display-column">
<Select
label="Display Column"
bind:value={primaryDisplay}
options={fields}
sort
{#if !existingTableId}
<div class="display-column">
<Select
label="Display Column"
bind:value={primaryDisplay}
options={fields}
sort
/>
</div>
{/if}
{:else if hasValidated}
<div>
<InlineAlert
header="Invalid CSV"
bind:message={noFieldsError}
type="error"
/>
</div>
{/if}

8
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -33,6 +33,7 @@ interface RunConfig {
sort?: SortJson
paginate?: PaginationJson
row?: Row
rows?: Row[]
}
module External {
@ -600,7 +601,10 @@ module External {
throw `Unable to process query, table "${tableName}" not defined.`
}
// look for specific components of config which may not be considered acceptable
let { id, row, filters, sort, paginate } = cleanupConfig(config, table)
let { id, row, filters, sort, paginate, rows } = cleanupConfig(
config,
table
)
filters = buildFilters(id, filters || {}, table)
const relationships = this.buildRelationships(table)
// clean up row on ingress using schema
@ -626,7 +630,7 @@ module External {
sort,
paginate,
relationships,
body: row,
body: row || rows,
// pass an id filter into extra, purely for mysql/returning
extra: {
idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),

2
packages/server/src/api/controllers/row/external.js
View File

@ -30,6 +30,8 @@ async function handleRequest(appId, operation, tableId, opts = {}) {
)
}
exports.handleRequest = handleRequest
exports.patch = async ctx => {
const appId = ctx.appId
const inputs = ctx.request.body

19
packages/server/src/api/controllers/table/external.js
View File

@ -17,6 +17,8 @@ const {
} = require("../../../constants")
const { makeExternalQuery } = require("../../../integrations/base/utils")
const { cloneDeep } = require("lodash/fp")
const csvParser = require("../../../utilities/csvParser")
const { handleRequest } = require("../row/external")
async function makeTableRequest(
datasource,
@ -279,3 +281,20 @@ exports.destroy = async function (ctx) {
return tableToDelete
}
exports.bulkImport = async function (ctx) {
const appId = ctx.appId
const table = await getTable(appId, ctx.params.tableId)
const { dataImport } = ctx.request.body
if (!dataImport || !dataImport.schema || !dataImport.csvString) {
ctx.throw(400, "Provided data import information is invalid.")
}
const rows = await csvParser.transform({
...dataImport,
existingTable: table,
})
await handleRequest(appId, DataSourceOperation.BULK_CREATE, table._id, {
rows,
})
return table
}

22
packages/server/src/api/controllers/table/index.js
View File

@ -81,8 +81,26 @@ exports.destroy = async function (ctx) {
ctx.body = { message: `Table ${tableId} deleted.` }
}
exports.bulkImport = async function (ctx) {
const tableId = ctx.params.tableId
await pickApi({ tableId }).bulkImport(ctx)
// right now we don't trigger anything for bulk import because it
// can only be done in the builder, but in the future we may need to
// think about events for bulk items
ctx.status = 200
ctx.body = { message: `Bulk rows created.` }
}
exports.validateCSVSchema = async function (ctx) {
const { csvString, schema = {} } = ctx.request.body
const result = await csvParser.parse(csvString, schema)
// tableId being specified means its an import to an existing table
const { csvString, schema = {}, tableId } = ctx.request.body
let existingTable
if (tableId) {
existingTable = await getTable(ctx.appId, tableId)
}
let result = await csvParser.parse(csvString, schema)
if (existingTable) {
result = csvParser.updateSchema({ schema: result, existingTable })
}
ctx.body = { schema: result }
}

15
packages/server/src/api/controllers/table/internal.js
View File

@ -2,7 +2,12 @@ const CouchDB = require("../../../db")
const linkRows = require("../../../db/linkedRows")
const { getRowParams, generateTableID } = require("../../../db/utils")
const { FieldTypes } = require("../../../constants")
const { TableSaveFunctions, hasTypeChanged } = require("./utils")
const {
TableSaveFunctions,
hasTypeChanged,
getTable,
handleDataImport,
} = require("./utils")
exports.save = async function (ctx) {
const appId = ctx.appId
@ -140,3 +145,11 @@ exports.destroy = async function (ctx) {
return tableToDelete
}
exports.bulkImport = async function (ctx) {
const appId = ctx.appId
const table = await getTable(appId, ctx.params.tableId)
const { dataImport } = ctx.request.body
await handleDataImport(appId, ctx.user, table, dataImport)
return table
}

64
packages/server/src/api/controllers/table/utils.js
View File

@ -72,43 +72,47 @@ exports.makeSureTableUpToDate = (table, tableToSave) => {
}
exports.handleDataImport = async (appId, user, table, dataImport) => {
if (!dataImport || !dataImport.csvString) {
return table
}
const db = new CouchDB(appId)
if (dataImport && dataImport.csvString) {
// Populate the table with rows imported from CSV in a bulk update
const data = await csvParser.transform(dataImport)
// Populate the table with rows imported from CSV in a bulk update
const data = await csvParser.transform({
...dataImport,
existingTable: table,
})
let finalData = []
for (let i = 0; i < data.length; i++) {
let row = data[i]
row._id = generateRowID(table._id)
row.tableId = table._id
const processed = inputProcessing(user, table, row, {
noAutoRelationships: true,
})
table = processed.table
row = processed.row
let finalData = []
for (let i = 0; i < data.length; i++) {
let row = data[i]
row._id = generateRowID(table._id)
row.tableId = table._id
const processed = inputProcessing(user, table, row, {
noAutoRelationships: true,
})
table = processed.table
row = processed.row
for (let [fieldName, schema] of Object.entries(table.schema)) {
// check whether the options need to be updated for inclusion as part of the data import
if (
schema.type === FieldTypes.OPTIONS &&
(!schema.constraints.inclusion ||
schema.constraints.inclusion.indexOf(row[fieldName]) === -1)
) {
schema.constraints.inclusion = [
...schema.constraints.inclusion,
row[fieldName],
]
}
for (let [fieldName, schema] of Object.entries(table.schema)) {
// check whether the options need to be updated for inclusion as part of the data import
if (
schema.type === FieldTypes.OPTIONS &&
(!schema.constraints.inclusion ||
schema.constraints.inclusion.indexOf(row[fieldName]) === -1)
) {
schema.constraints.inclusion = [
...schema.constraints.inclusion,
row[fieldName],
]
}
finalData.push(row)
}
await db.bulkDocs(finalData)
let response = await db.put(table)
table._rev = response._rev
finalData.push(row)
}
await db.bulkDocs(finalData)
let response = await db.put(table)
table._rev = response._rev
return table
}

11
packages/server/src/api/routes/table.js
View File

@ -53,5 +53,16 @@ router
authorized(BUILDER),
tableController.destroy
)
// this is currently builder only, but in the future
// it could be carried out by an end user in app,
// however some thought will need to be had about
// implications for automations (triggers)
// new trigger type, bulk rows created
.post(
"/api/tables/:tableId/import",
paramResource("tableId"),
authorized(BUILDER),
tableController.bulkImport
)
module.exports = router

6
packages/server/src/api/routes/tests/misc.spec.js
View File

@ -75,7 +75,11 @@ describe("run misc tests", () => {
},
})
const dataImport = {
csvString: "a,b,c,d\n1,2,3,4"
csvString: "a,b,c,d\n1,2,3,4",
schema: {},
}
for (let col of ["a", "b", "c", "d"]) {
dataImport.schema[col] = { type: "string" }
}
await tableUtils.handleDataImport(
config.getAppId(),

1
packages/server/src/constants/index.js
View File

@ -69,6 +69,7 @@ exports.DataSourceOperation = {
READ: "READ",
UPDATE: "UPDATE",
DELETE: "DELETE",
BULK_CREATE: "BULK_CREATE",
CREATE_TABLE: "CREATE_TABLE",
UPDATE_TABLE: "UPDATE_TABLE",
DELETE_TABLE: "DELETE_TABLE",

5
packages/server/src/definitions/datasource.ts
View File

@ -1,10 +1,11 @@
import { Table } from "./common"
import { Row, Table } from "./common"
export enum Operation {
CREATE = "CREATE",
READ = "READ",
UPDATE = "UPDATE",
DELETE = "DELETE",
BULK_CREATE = "BULK_CREATE",
CREATE_TABLE = "CREATE_TABLE",
UPDATE_TABLE = "UPDATE_TABLE",
DELETE_TABLE = "DELETE_TABLE",
@ -144,7 +145,7 @@ export interface QueryJson {
filters?: SearchFilters
sort?: SortJson
paginate?: PaginationJson
body?: object
body?: Row | Row[]
table?: Table
meta?: {
table?: Table

13
packages/server/src/integrations/base/sql.ts
View File

@ -179,6 +179,16 @@ class InternalBuilder {
}
}
bulkCreate(knex: Knex, json: QueryJson): KnexQuery {
const { endpoint, body } = json
let query: KnexQuery = knex(endpoint.entityId)
if (!Array.isArray(body)) {
return query
}
const parsedBody = body.map(row => parseBody(row))
return query.insert(parsedBody)
}
read(knex: Knex, json: QueryJson, limit: number): KnexQuery {
let { endpoint, resource, filters, sort, paginate, relationships } = json
const tableName = endpoint.entityId
@ -294,6 +304,9 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
case Operation.DELETE:
query = builder.delete(client, json, opts)
break
case Operation.BULK_CREATE:
query = builder.bulkCreate(client, json)
break
case Operation.CREATE_TABLE:
case Operation.UPDATE_TABLE:
case Operation.DELETE_TABLE:

5
packages/server/src/integrations/base/sqlTable.ts
View File

@ -29,10 +29,7 @@ function generateSchema(
for (let [key, column] of Object.entries(table.schema)) {
// skip things that are already correct
const oldColumn = oldTable ? oldTable.schema[key] : null
if (
(oldColumn && oldColumn.type) ||
(primaryKey === key && !isJunction)
) {
if ((oldColumn && oldColumn.type) || (primaryKey === key && !isJunction)) {
continue
}
switch (column.type) {

10
packages/server/src/integrations/postgres.ts
View File

@ -130,7 +130,7 @@ module PostgresModule {
public tables: Record<string, Table> = {}
public schemaErrors: Record<string, string> = {}
COLUMNS_SQL!: string
COLUMNS_SQL!: string
PRIMARY_KEYS_SQL = `
select tc.table_schema, tc.table_name, kc.column_name as primary_key
@ -165,11 +165,11 @@ module PostgresModule {
setSchema() {
if (!this.config.schema) {
this.config.schema = 'public'
this.config.schema = "public"
}
this.client.on('connect', (client: any) => {
client.query(`SET search_path TO ${this.config.schema}`);
});
this.client.on("connect", (client: any) => {
client.query(`SET search_path TO ${this.config.schema}`)
})
this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'`
}

53
packages/server/src/utilities/csvParser.js
View File

@ -51,7 +51,7 @@ function parse(csvString, parsers) {
})
result.subscribe(row => {
// For each CSV row parse all the columns that need parsed
for (let key in parsers) {
for (let key of Object.keys(parsers)) {
if (!schema[key] || schema[key].success) {
// get the validator for the column type
const validator = VALIDATORS[parsers[key].type]
@ -76,16 +76,58 @@ function parse(csvString, parsers) {
})
}
async function transform({ schema, csvString }) {
function updateSchema({ schema, existingTable }) {
if (!schema) {
return schema
}
const finalSchema = {}
const schemaKeyMap = {}
Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
for (let [key, field] of Object.entries(existingTable.schema)) {
const lcKey = key.toLowerCase()
const foundKey = schemaKeyMap[lcKey]
if (foundKey) {
finalSchema[key] = schema[foundKey]
finalSchema[key].type = field.type
}
}
return finalSchema
}
async function transform({ schema, csvString, existingTable }) {
const colParser = {}
for (let key in schema) {
// make sure the table has all the columns required for import
if (existingTable) {
schema = updateSchema({ schema, existingTable })
}
for (let key of Object.keys(schema)) {
colParser[key] = PARSERS[schema[key].type] || schema[key].type
}
try {
const json = await csv({ colParser }).fromString(csvString)
return json
const data = await csv({ colParser }).fromString(csvString)
const schemaKeyMap = {}
Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
for (let element of data) {
if (!data) {
continue
}
for (let key of Object.keys(element)) {
const mappedKey = schemaKeyMap[key.toLowerCase()]
// isn't a column in the table, remove it
if (mappedKey == null) {
delete element[key]
}
// casing is different, fix it in row
else if (key !== mappedKey) {
element[mappedKey] = element[key]
delete element[key]
}
}
}
return data
} catch (err) {
console.error(`Error transforming CSV to JSON for data import`, err)
throw err
@ -95,4 +137,5 @@ async function transform({ schema, csvString }) {
module.exports = {
parse,
transform,
updateSchema,
}

6
packages/server/src/utilities/tests/__snapshots__/csvParser.spec.js.snap
View File

@ -3,19 +3,13 @@
exports[`CSV Parser transformation transforms a CSV file into JSON 1`] = `
Array [
Object {
"Address": "5 Sesame Street",
"Age": 4324,
"Name": "Bertå",
},
Object {
"Address": "1 World Trade Center",
"Age": 34,
"Name": "Ernie",
},
Object {
"Address": "44 Second Avenue",
"Age": 23423,
"Name": "Big Bird",
},
]
`;

3
packages/server/src/utilities/tests/csvParser.spec.js
View File

@ -24,6 +24,9 @@ const SCHEMAS = {
Age: {
type: "omit",
},
Name: {
type: "string",
},
},
BROKEN: {
Address: {