MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3356 from Budibase/feature/existing-table-import 

Import CSV to existing table from builder
This commit is contained in:
Michael Drury 2021-11-15 14:32:41 +00:00 committed by GitHub
parent b049d083ee d137cb46e8
commit e257b5ee8b
22 changed files with 277 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/bbui/src/InlineAlert/InlineAlert.svelte
View File

@ -47,5 +47,6 @@
--spectrum-semantic-positive-border-color: #2d9d78;
--spectrum-semantic-positive-icon-color: #2d9d78;
--spectrum-semantic-negative-icon-color: #e34850;
min-width: 100px;
}
</style>

5
packages/builder/src/components/backend/DataTable/DataTable.svelte
View File

@ -6,6 +6,7 @@
import CreateViewButton from "./buttons/CreateViewButton.svelte"
import ExistingRelationshipButton from "./buttons/ExistingRelationshipButton.svelte"
import ExportButton from "./buttons/ExportButton.svelte"
import ImportButton from "./buttons/ImportButton.svelte"
import EditRolesButton from "./buttons/EditRolesButton.svelte"
import ManageAccessButton from "./buttons/ManageAccessButton.svelte"
import HideAutocolumnButton from "./buttons/HideAutocolumnButton.svelte"
@ -124,6 +125,10 @@
<HideAutocolumnButton bind:hideAutocolumns />
<!-- always have the export last -->
<ExportButton view={$tables.selected?._id} />
<ImportButton
tableId={$tables.selected?._id}
on:updaterows={onUpdateRows}
/>
{#key id}
<TableFilterButton {schema} on:change={onFilter} />
{/key}

2
packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte
View File

@ -7,7 +7,7 @@
let modal
</script>
<ActionButton icon="Download" size="S" quiet on:click={modal.show}>
<ActionButton icon="DataDownload" size="S" quiet on:click={modal.show}>
Export
</ActionButton>
<Modal bind:this={modal}>

15
packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte Normal file
View File

@ -0,0 +1,15 @@
<script>
import { ActionButton, Modal } from "@budibase/bbui"
import ImportModal from "../modals/ImportModal.svelte"
export let tableId
let modal
</script>
<ActionButton icon="DataUpload" size="S" quiet on:click={modal.show}>
Import
</ActionButton>
<Modal bind:this={modal}>
<ImportModal {tableId} on:updaterows />
</Modal>

43
packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte Normal file
View File

@ -0,0 +1,43 @@
<script>
import { ModalContent, Label, notifications, Body } from "@budibase/bbui"
import TableDataImport from "../../TableNavigator/TableDataImport.svelte"
import api from "builderStore/api"
import { createEventDispatcher } from "svelte"
const dispatch = createEventDispatcher()
export let tableId
let dataImport
$: valid = dataImport?.csvString != null && dataImport?.valid
async function importData() {
const response = await api.post(`/api/tables/${tableId}/import`, {
dataImport,
})
if (response.status !== 200) {
const error = await response.text()
notifications.error(`Unable to import data - ${error}`)
} else {
notifications.success("Rows successfully imported.")
}
dispatch("updaterows")
}
</script>
<ModalContent
title="Import Data"
confirmText="Import"
onConfirm={importData}
disabled={!valid}
>
<Body
>Import rows to an existing table from a CSV. Only columns from the CSV
which exist in the table will be imported.</Body
>
<Label grey extraSmall>CSV to import</Label>
<TableDataImport bind:dataImport bind:existingTableId={tableId} />
</ModalContent>
<style>
</style>

40
packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
View File

@ -1,6 +1,5 @@
<script>
import { Select } from "@budibase/bbui"
import { notifications } from "@budibase/bbui"
import { Select, InlineAlert, notifications } from "@budibase/bbui"
import { FIELDS } from "constants/backend"
import api from "builderStore/api"
@ -12,11 +11,13 @@
valid: true,
schema: {},
}
export let existingTableId
let csvString
let primaryDisplay
let csvString = undefined
let primaryDisplay = undefined
let schema = {}
let fields = []
let hasValidated = false
$: valid = !schema || fields.every(column => schema[column].success)
$: dataImport = {
@ -25,6 +26,9 @@
csvString,
primaryDisplay,
}
$: noFieldsError = existingTableId
? "No columns in CSV match existing table schema"
: "Could not find any columns to import"
function buildTableSchema(schema) {
const tableSchema = {}
@ -46,6 +50,7 @@
const response = await api.post("/api/tables/csv/validate", {
csvString,
schema: schema || {},
tableId: existingTableId,
})
const parseResult = await response.json()
@ -63,6 +68,7 @@
notifications.error("CSV Invalid, please try another CSV file")
return []
}
hasValidated = true
}
async function handleFile(evt) {
@ -138,6 +144,7 @@
placeholder={null}
getOptionLabel={option => option.label}
getOptionValue={option => option.value}
disabled={!!existingTableId}
/>
<span class="field-status" class:error={!schema[columnName].success}>
{schema[columnName].success ? "Success" : "Failure"}
@ -149,15 +156,22 @@
</div>
{/each}
</div>
{/if}
{#if fields.length}
<div class="display-column">
<Select
label="Display Column"
bind:value={primaryDisplay}
options={fields}
sort
{#if !existingTableId}
<div class="display-column">
<Select
label="Display Column"
bind:value={primaryDisplay}
options={fields}
sort
/>
</div>
{/if}
{:else if hasValidated}
<div>
<InlineAlert
header="Invalid CSV"
bind:message={noFieldsError}
type="error"
/>
</div>
{/if}

8
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -33,6 +33,7 @@ interface RunConfig {
sort?: SortJson
paginate?: PaginationJson
row?: Row
rows?: Row[]
}
module External {
@ -600,7 +601,10 @@ module External {
throw `Unable to process query, table "${tableName}" not defined.`
}
// look for specific components of config which may not be considered acceptable
let { id, row, filters, sort, paginate } = cleanupConfig(config, table)
let { id, row, filters, sort, paginate, rows } = cleanupConfig(
config,
table
)
filters = buildFilters(id, filters || {}, table)
const relationships = this.buildRelationships(table)
// clean up row on ingress using schema
@ -626,7 +630,7 @@ module External {
sort,
paginate,
relationships,
body: row,
body: row || rows,
// pass an id filter into extra, purely for mysql/returning
extra: {
idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),

2
packages/server/src/api/controllers/row/external.js
View File

@ -30,6 +30,8 @@ async function handleRequest(appId, operation, tableId, opts = {}) {
)
}
exports.handleRequest = handleRequest
exports.patch = async ctx => {
const appId = ctx.appId
const inputs = ctx.request.body

19
packages/server/src/api/controllers/table/external.js
View File

@ -17,6 +17,8 @@ const {
} = require("../../../constants")
const { makeExternalQuery } = require("../../../integrations/base/utils")
const { cloneDeep } = require("lodash/fp")
const csvParser = require("../../../utilities/csvParser")
const { handleRequest } = require("../row/external")
async function makeTableRequest(
datasource,
@ -279,3 +281,20 @@ exports.destroy = async function (ctx) {
return tableToDelete
}
exports.bulkImport = async function (ctx) {
const appId = ctx.appId
const table = await getTable(appId, ctx.params.tableId)
const { dataImport } = ctx.request.body
if (!dataImport || !dataImport.schema || !dataImport.csvString) {
ctx.throw(400, "Provided data import information is invalid.")
}
const rows = await csvParser.transform({
...dataImport,
existingTable: table,
})
await handleRequest(appId, DataSourceOperation.BULK_CREATE, table._id, {
rows,
})
return table
}

22
packages/server/src/api/controllers/table/index.js
View File

@ -81,8 +81,26 @@ exports.destroy = async function (ctx) {
ctx.body = { message: `Table ${tableId} deleted.` }
}
exports.bulkImport = async function (ctx) {
const tableId = ctx.params.tableId
await pickApi({ tableId }).bulkImport(ctx)
// right now we don't trigger anything for bulk import because it
// can only be done in the builder, but in the future we may need to
// think about events for bulk items
ctx.status = 200
ctx.body = { message: `Bulk rows created.` }
}
exports.validateCSVSchema = async function (ctx) {
const { csvString, schema = {} } = ctx.request.body
const result = await csvParser.parse(csvString, schema)
// tableId being specified means its an import to an existing table
const { csvString, schema = {}, tableId } = ctx.request.body
let existingTable
if (tableId) {
existingTable = await getTable(ctx.appId, tableId)
}
let result = await csvParser.parse(csvString, schema)
if (existingTable) {
result = csvParser.updateSchema({ schema: result, existingTable })
}
ctx.body = { schema: result }
}

15
packages/server/src/api/controllers/table/internal.js
View File

@ -2,7 +2,12 @@ const CouchDB = require("../../../db")
const linkRows = require("../../../db/linkedRows")
const { getRowParams, generateTableID } = require("../../../db/utils")
const { FieldTypes } = require("../../../constants")
const { TableSaveFunctions, hasTypeChanged } = require("./utils")
const {
TableSaveFunctions,
hasTypeChanged,
getTable,
handleDataImport,
} = require("./utils")
exports.save = async function (ctx) {
const appId = ctx.appId
@ -140,3 +145,11 @@ exports.destroy = async function (ctx) {
return tableToDelete
}
exports.bulkImport = async function (ctx) {
const appId = ctx.appId
const table = await getTable(appId, ctx.params.tableId)
const { dataImport } = ctx.request.body
await handleDataImport(appId, ctx.user, table, dataImport)
return table
}

64
packages/server/src/api/controllers/table/utils.js
View File

@ -72,43 +72,47 @@ exports.makeSureTableUpToDate = (table, tableToSave) => {
}
exports.handleDataImport = async (appId, user, table, dataImport) => {
if (!dataImport || !dataImport.csvString) {
return table
}
const db = new CouchDB(appId)
if (dataImport && dataImport.csvString) {
// Populate the table with rows imported from CSV in a bulk update
const data = await csvParser.transform(dataImport)
// Populate the table with rows imported from CSV in a bulk update
const data = await csvParser.transform({
...dataImport,
existingTable: table,
})
let finalData = []
for (let i = 0; i < data.length; i++) {
let row = data[i]
row._id = generateRowID(table._id)
row.tableId = table._id
const processed = inputProcessing(user, table, row, {
noAutoRelationships: true,
})
table = processed.table
row = processed.row
let finalData = []
for (let i = 0; i < data.length; i++) {
let row = data[i]
row._id = generateRowID(table._id)
row.tableId = table._id
const processed = inputProcessing(user, table, row, {
noAutoRelationships: true,
})
table = processed.table
row = processed.row
for (let [fieldName, schema] of Object.entries(table.schema)) {
// check whether the options need to be updated for inclusion as part of the data import
if (
schema.type === FieldTypes.OPTIONS &&
(!schema.constraints.inclusion ||
schema.constraints.inclusion.indexOf(row[fieldName]) === -1)
) {
schema.constraints.inclusion = [
...schema.constraints.inclusion,
row[fieldName],
]
}
for (let [fieldName, schema] of Object.entries(table.schema)) {
// check whether the options need to be updated for inclusion as part of the data import
if (
schema.type === FieldTypes.OPTIONS &&
(!schema.constraints.inclusion ||
schema.constraints.inclusion.indexOf(row[fieldName]) === -1)
) {
schema.constraints.inclusion = [
...schema.constraints.inclusion,
row[fieldName],
]
}
finalData.push(row)
}
await db.bulkDocs(finalData)
let response = await db.put(table)
table._rev = response._rev
finalData.push(row)
}
await db.bulkDocs(finalData)
let response = await db.put(table)
table._rev = response._rev
return table
}

11
packages/server/src/api/routes/table.js
View File

@ -53,5 +53,16 @@ router
authorized(BUILDER),
tableController.destroy
)
// this is currently builder only, but in the future
// it could be carried out by an end user in app,
// however some thought will need to be had about
// implications for automations (triggers)
// new trigger type, bulk rows created
.post(
"/api/tables/:tableId/import",
paramResource("tableId"),
authorized(BUILDER),
tableController.bulkImport
)
module.exports = router

6
packages/server/src/api/routes/tests/misc.spec.js
View File

@ -75,7 +75,11 @@ describe("run misc tests", () => {
},
})
const dataImport = {
csvString: "a,b,c,d\n1,2,3,4"
csvString: "a,b,c,d\n1,2,3,4",
schema: {},
}
for (let col of ["a", "b", "c", "d"]) {
dataImport.schema[col] = { type: "string" }
}
await tableUtils.handleDataImport(
config.getAppId(),

1
packages/server/src/constants/index.js
View File

@ -69,6 +69,7 @@ exports.DataSourceOperation = {
READ: "READ",
UPDATE: "UPDATE",
DELETE: "DELETE",
BULK_CREATE: "BULK_CREATE",
CREATE_TABLE: "CREATE_TABLE",
UPDATE_TABLE: "UPDATE_TABLE",
DELETE_TABLE: "DELETE_TABLE",

5
packages/server/src/definitions/datasource.ts
View File

@ -1,10 +1,11 @@
import { Table } from "./common"
import { Row, Table } from "./common"
export enum Operation {
CREATE = "CREATE",
READ = "READ",
UPDATE = "UPDATE",
DELETE = "DELETE",
BULK_CREATE = "BULK_CREATE",
CREATE_TABLE = "CREATE_TABLE",
UPDATE_TABLE = "UPDATE_TABLE",
DELETE_TABLE = "DELETE_TABLE",
@ -144,7 +145,7 @@ export interface QueryJson {
filters?: SearchFilters
sort?: SortJson
paginate?: PaginationJson
body?: object
body?: Row | Row[]
table?: Table
meta?: {
table?: Table

13
packages/server/src/integrations/base/sql.ts
View File

@ -179,6 +179,16 @@ class InternalBuilder {
}
}
bulkCreate(knex: Knex, json: QueryJson): KnexQuery {
const { endpoint, body } = json
let query: KnexQuery = knex(endpoint.entityId)
if (!Array.isArray(body)) {
return query
}
const parsedBody = body.map(row => parseBody(row))
return query.insert(parsedBody)
}
read(knex: Knex, json: QueryJson, limit: number): KnexQuery {
let { endpoint, resource, filters, sort, paginate, relationships } = json
const tableName = endpoint.entityId
@ -294,6 +304,9 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
case Operation.DELETE:
query = builder.delete(client, json, opts)
break
case Operation.BULK_CREATE:
query = builder.bulkCreate(client, json)
break
case Operation.CREATE_TABLE:
case Operation.UPDATE_TABLE:
case Operation.DELETE_TABLE:

5
packages/server/src/integrations/base/sqlTable.ts
View File

@ -29,10 +29,7 @@ function generateSchema(
for (let [key, column] of Object.entries(table.schema)) {
// skip things that are already correct
const oldColumn = oldTable ? oldTable.schema[key] : null
if (
(oldColumn && oldColumn.type) ||
(primaryKey === key && !isJunction)
) {
if ((oldColumn && oldColumn.type) || (primaryKey === key && !isJunction)) {
continue
}
switch (column.type) {

10
packages/server/src/integrations/postgres.ts
View File

@ -130,7 +130,7 @@ module PostgresModule {
public tables: Record<string, Table> = {}
public schemaErrors: Record<string, string> = {}
COLUMNS_SQL!: string
COLUMNS_SQL!: string
PRIMARY_KEYS_SQL = `
select tc.table_schema, tc.table_name, kc.column_name as primary_key
@ -165,11 +165,11 @@ module PostgresModule {
setSchema() {
if (!this.config.schema) {
this.config.schema = 'public'
this.config.schema = "public"
}
this.client.on('connect', (client: any) => {
client.query(`SET search_path TO ${this.config.schema}`);
});
this.client.on("connect", (client: any) => {
client.query(`SET search_path TO ${this.config.schema}`)
})
this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'`
}

53
packages/server/src/utilities/csvParser.js
View File

@ -51,7 +51,7 @@ function parse(csvString, parsers) {
})
result.subscribe(row => {
// For each CSV row parse all the columns that need parsed
for (let key in parsers) {
for (let key of Object.keys(parsers)) {
if (!schema[key] || schema[key].success) {
// get the validator for the column type
const validator = VALIDATORS[parsers[key].type]
@ -76,16 +76,58 @@ function parse(csvString, parsers) {
})
}
async function transform({ schema, csvString }) {
function updateSchema({ schema, existingTable }) {
if (!schema) {
return schema
}
const finalSchema = {}
const schemaKeyMap = {}
Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
for (let [key, field] of Object.entries(existingTable.schema)) {
const lcKey = key.toLowerCase()
const foundKey = schemaKeyMap[lcKey]
if (foundKey) {
finalSchema[key] = schema[foundKey]
finalSchema[key].type = field.type
}
}
return finalSchema
}
async function transform({ schema, csvString, existingTable }) {
const colParser = {}
for (let key in schema) {
// make sure the table has all the columns required for import
if (existingTable) {
schema = updateSchema({ schema, existingTable })
}
for (let key of Object.keys(schema)) {
colParser[key] = PARSERS[schema[key].type] || schema[key].type
}
try {
const json = await csv({ colParser }).fromString(csvString)
return json
const data = await csv({ colParser }).fromString(csvString)
const schemaKeyMap = {}
Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
for (let element of data) {
if (!data) {
continue
}
for (let key of Object.keys(element)) {
const mappedKey = schemaKeyMap[key.toLowerCase()]
// isn't a column in the table, remove it
if (mappedKey == null) {
delete element[key]
}
// casing is different, fix it in row
else if (key !== mappedKey) {
element[mappedKey] = element[key]
delete element[key]
}
}
}
return data
} catch (err) {
console.error(`Error transforming CSV to JSON for data import`, err)
throw err
@ -95,4 +137,5 @@ async function transform({ schema, csvString }) {
module.exports = {
parse,
transform,
updateSchema,
}

6
packages/server/src/utilities/tests/__snapshots__/csvParser.spec.js.snap
View File

@ -3,19 +3,13 @@
exports[`CSV Parser transformation transforms a CSV file into JSON 1`] = `
Array [
Object {
"Address": "5 Sesame Street",
"Age": 4324,
"Name": "Bertå",
},
Object {
"Address": "1 World Trade Center",
"Age": 34,
"Name": "Ernie",
},
Object {
"Address": "44 Second Avenue",
"Age": 23423,
"Name": "Big Bird",
},
]
`;

3
packages/server/src/utilities/tests/csvParser.spec.js
View File

@ -24,6 +24,9 @@ const SCHEMAS = {
Age: {
type: "omit",
},
Name: {
type: "string",
},
},
BROKEN: {
Address: {