Merge branch 'master' of github.com:Budibase/budibase into backend-ag-grid
This commit is contained in:
commit
4ae78dd74d
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"version": "0.2.5",
|
||||
"version": "0.2.6",
|
||||
"npmClient": "yarn",
|
||||
"packages": [
|
||||
"packages/*"
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@budibase/builder",
|
||||
"version": "0.2.5",
|
||||
"version": "0.2.6",
|
||||
"license": "AGPL-3.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
|
@ -64,7 +64,7 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"@budibase/bbui": "^1.44.1",
|
||||
"@budibase/client": "^0.2.5",
|
||||
"@budibase/client": "^0.2.6",
|
||||
"@budibase/colorpicker": "^1.0.1",
|
||||
"@budibase/svelte-ag-grid": "^0.0.13",
|
||||
"@fortawesome/fontawesome-free": "^5.14.0",
|
||||
|
|
|
@ -86,10 +86,12 @@ const contextToBindables = (tables, walkResult) => context => {
|
|||
}
|
||||
|
||||
const newBindable = ([key, fieldSchema]) => {
|
||||
// Replace link bindings with a new property representing the count
|
||||
// Replace certain bindings with a new property to help display components
|
||||
let runtimeBoundKey = key
|
||||
if (fieldSchema.type === "link") {
|
||||
runtimeBoundKey = `${key}_count`
|
||||
} else if (fieldSchema.type === "attachment") {
|
||||
runtimeBoundKey = `${key}_first`
|
||||
}
|
||||
return {
|
||||
type: "context",
|
||||
|
|
|
@ -233,7 +233,7 @@ const createLink = store => async (url, title) => {
|
|||
|
||||
// Save page and regenerate all CSS because otherwise weird things happen
|
||||
nav._children = [...nav._children, newLink]
|
||||
setCurrentPage("main")
|
||||
state.currentPageName = "main"
|
||||
regenerateCssForScreen(state.pages.main)
|
||||
for (let screen of state.pages.main._screens) {
|
||||
regenerateCssForScreen(screen)
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
<script>
|
||||
import { Heading, Body, Button, Select } from "@budibase/bbui"
|
||||
import { Heading, Body, Button, Select, Label } from "@budibase/bbui"
|
||||
import { notifier } from "builderStore/store/notifications"
|
||||
import { FIELDS } from "constants/backend"
|
||||
import api from "builderStore/api"
|
||||
|
@ -14,15 +14,17 @@
|
|||
schema: {},
|
||||
}
|
||||
|
||||
let parseResult
|
||||
let csvString
|
||||
let primaryDisplay
|
||||
let schema = {}
|
||||
let fields = []
|
||||
|
||||
$: schema = parseResult && parseResult.schema
|
||||
$: valid =
|
||||
!schema || Object.keys(schema).every(column => schema[column].success)
|
||||
$: valid = !schema || fields.every(column => schema[column].success)
|
||||
$: dataImport = {
|
||||
valid,
|
||||
schema: buildTableSchema(schema),
|
||||
path: files[0] && files[0].path,
|
||||
csvString,
|
||||
primaryDisplay,
|
||||
}
|
||||
|
||||
function buildTableSchema(schema) {
|
||||
|
@ -43,11 +45,20 @@
|
|||
|
||||
async function validateCSV() {
|
||||
const response = await api.post("/api/tables/csv/validate", {
|
||||
file: files[0],
|
||||
csvString,
|
||||
schema: schema || {},
|
||||
})
|
||||
|
||||
parseResult = await response.json()
|
||||
const parseResult = await response.json()
|
||||
schema = parseResult && parseResult.schema
|
||||
fields = Object.keys(schema || {}).filter(
|
||||
key => schema[key].type !== "omit"
|
||||
)
|
||||
|
||||
// Check primary display is valid
|
||||
if (!primaryDisplay || fields.indexOf(primaryDisplay) === -1) {
|
||||
primaryDisplay = fields[0]
|
||||
}
|
||||
|
||||
if (response.status !== 200) {
|
||||
notifier.danger("CSV Invalid, please try another CSV file")
|
||||
|
@ -57,13 +68,7 @@
|
|||
|
||||
async function handleFile(evt) {
|
||||
const fileArray = Array.from(evt.target.files)
|
||||
const filesToProcess = fileArray.map(({ name, path, size }) => ({
|
||||
name,
|
||||
path,
|
||||
size,
|
||||
}))
|
||||
|
||||
if (filesToProcess.some(file => file.size >= FILE_SIZE_LIMIT)) {
|
||||
if (fileArray.some(file => file.size >= FILE_SIZE_LIMIT)) {
|
||||
notifier.danger(
|
||||
`Files cannot exceed ${FILE_SIZE_LIMIT /
|
||||
BYTES_IN_MB}MB. Please try again with smaller files.`
|
||||
|
@ -71,9 +76,14 @@
|
|||
return
|
||||
}
|
||||
|
||||
files = filesToProcess
|
||||
|
||||
await validateCSV()
|
||||
// Read CSV as plain text to upload alongside schema
|
||||
let reader = new FileReader()
|
||||
reader.addEventListener("load", function(e) {
|
||||
csvString = e.target.result
|
||||
files = fileArray
|
||||
validateCSV()
|
||||
})
|
||||
reader.readAsBinaryString(fileArray[0])
|
||||
}
|
||||
|
||||
async function omitColumn(columnName) {
|
||||
|
@ -94,8 +104,8 @@
|
|||
</label>
|
||||
</div>
|
||||
<div class="schema-fields">
|
||||
{#if schema}
|
||||
{#each Object.keys(schema).filter(key => schema[key].type !== 'omit') as columnName}
|
||||
{#if fields.length}
|
||||
{#each fields as columnName}
|
||||
<div class="field">
|
||||
<span>{columnName}</span>
|
||||
<Select
|
||||
|
@ -117,6 +127,16 @@
|
|||
{/each}
|
||||
{/if}
|
||||
</div>
|
||||
{#if fields.length}
|
||||
<div class="display-column">
|
||||
<Label extraSmall grey>Display Column</Label>
|
||||
<Select thin secondary bind:value={primaryDisplay}>
|
||||
{#each fields as field}
|
||||
<option value={field}>{field}</option>
|
||||
{/each}
|
||||
</Select>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<style>
|
||||
.dropzone {
|
||||
|
@ -188,4 +208,8 @@
|
|||
grid-gap: var(--spacing-m);
|
||||
font-size: var(--font-size-xs);
|
||||
}
|
||||
|
||||
.display-column {
|
||||
margin-top: var(--spacing-xl);
|
||||
}
|
||||
</style>
|
||||
|
|
|
@ -38,12 +38,19 @@
|
|||
}
|
||||
|
||||
async function saveTable() {
|
||||
// Create table
|
||||
const table = await backendUiStore.actions.tables.save({
|
||||
let newTable = {
|
||||
name,
|
||||
schema: dataImport.schema || {},
|
||||
dataImport,
|
||||
})
|
||||
}
|
||||
|
||||
// Only set primary display if defined
|
||||
if (dataImport.primaryDisplay && dataImport.primaryDisplay.length) {
|
||||
newTable.primaryDisplay = dataImport.primaryDisplay
|
||||
}
|
||||
|
||||
// Create table
|
||||
const table = await backendUiStore.actions.tables.save(newTable)
|
||||
notifier.success(`Table ${name} created successfully.`)
|
||||
analytics.captureEvent("Table Created", { name })
|
||||
|
||||
|
|
|
@ -19,7 +19,7 @@
|
|||
hour12: true,
|
||||
},
|
||||
}
|
||||
const POLL_INTERVAL = 1000
|
||||
const POLL_INTERVAL = 5000
|
||||
|
||||
export let appId
|
||||
|
||||
|
@ -68,7 +68,7 @@
|
|||
</span>
|
||||
</div>
|
||||
<div class="deployment-right">
|
||||
{#if deployment.status.toLowerCase() === "pending"}
|
||||
{#if deployment.status.toLowerCase() === 'pending'}
|
||||
<Spinner size="10" />
|
||||
{/if}
|
||||
<div class={`deployment-status ${deployment.status}`}>
|
||||
|
|
|
@ -135,6 +135,9 @@
|
|||
|
||||
.toprightnav {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
justify-content: flex-end;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.topleftnav {
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "budibase",
|
||||
"version": "0.2.5",
|
||||
"version": "0.2.6",
|
||||
"description": "Budibase CLI",
|
||||
"repository": "https://github.com/Budibase/Budibase",
|
||||
"homepage": "https://www.budibase.com",
|
||||
|
@ -17,7 +17,7 @@
|
|||
"author": "Budibase",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"dependencies": {
|
||||
"@budibase/server": "^0.2.5",
|
||||
"@budibase/server": "^0.2.6",
|
||||
"@inquirer/password": "^0.0.6-alpha.0",
|
||||
"chalk": "^2.4.2",
|
||||
"dotenv": "^8.2.0",
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@budibase/client",
|
||||
"version": "0.2.5",
|
||||
"version": "0.2.6",
|
||||
"license": "MPL-2.0",
|
||||
"main": "dist/budibase-client.js",
|
||||
"module": "dist/budibase-client.esm.mjs",
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@budibase/server",
|
||||
"version": "0.2.5",
|
||||
"version": "0.2.6",
|
||||
"description": "Budibase Web Server",
|
||||
"main": "src/electron.js",
|
||||
"repository": {
|
||||
|
@ -42,7 +42,7 @@
|
|||
"author": "Michael Shanks",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"dependencies": {
|
||||
"@budibase/client": "^0.2.5",
|
||||
"@budibase/client": "^0.2.6",
|
||||
"@koa/router": "^8.0.0",
|
||||
"@sendgrid/mail": "^7.1.1",
|
||||
"@sentry/node": "^5.19.2",
|
||||
|
@ -57,6 +57,7 @@
|
|||
"electron-updater": "^4.3.1",
|
||||
"fix-path": "^3.0.0",
|
||||
"fs-extra": "^8.1.0",
|
||||
"jimp": "^0.16.1",
|
||||
"joi": "^17.2.1",
|
||||
"jsonwebtoken": "^8.5.1",
|
||||
"koa": "^2.7.0",
|
||||
|
@ -73,7 +74,7 @@
|
|||
"pouchdb": "^7.2.1",
|
||||
"pouchdb-all-dbs": "^1.0.2",
|
||||
"pouchdb-replication-stream": "^1.2.9",
|
||||
"sharp": "^0.26.0",
|
||||
"sanitize-s3-objectkey": "^0.0.1",
|
||||
"squirrelly": "^7.5.0",
|
||||
"tar-fs": "^2.1.0",
|
||||
"uuid": "^3.3.2",
|
||||
|
|
|
@ -1,19 +1,13 @@
|
|||
const fs = require("fs")
|
||||
const { join } = require("../../../utilities/centralPath")
|
||||
let { wait } = require("../../../utilities")
|
||||
const AWS = require("aws-sdk")
|
||||
const fetch = require("node-fetch")
|
||||
const uuid = require("uuid")
|
||||
const sanitize = require("sanitize-s3-objectkey")
|
||||
const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
|
||||
const PouchDB = require("../../../db")
|
||||
const environment = require("../../../environment")
|
||||
|
||||
const MAX_INVALIDATE_WAIT_MS = 120000
|
||||
const INVALIDATE_WAIT_PERIODS_MS = 5000
|
||||
|
||||
// export so main deploy functions can use too
|
||||
exports.MAX_INVALIDATE_WAIT_MS = MAX_INVALIDATE_WAIT_MS
|
||||
|
||||
async function invalidateCDN(cfDistribution, appId) {
|
||||
const cf = new AWS.CloudFront({})
|
||||
const resp = await cf
|
||||
|
@ -28,28 +22,24 @@ async function invalidateCDN(cfDistribution, appId) {
|
|||
},
|
||||
})
|
||||
.promise()
|
||||
let totalWaitTimeMs = 0
|
||||
let complete = false
|
||||
do {
|
||||
try {
|
||||
const state = await cf
|
||||
return resp.Invalidation.Id
|
||||
}
|
||||
|
||||
exports.isInvalidationComplete = async function(
|
||||
distributionId,
|
||||
invalidationId
|
||||
) {
|
||||
if (distributionId == null || invalidationId == null) {
|
||||
return false
|
||||
}
|
||||
const cf = new AWS.CloudFront({})
|
||||
const resp = await cf
|
||||
.getInvalidation({
|
||||
DistributionId: cfDistribution,
|
||||
Id: resp.Invalidation.Id,
|
||||
DistributionId: distributionId,
|
||||
Id: invalidationId,
|
||||
})
|
||||
.promise()
|
||||
if (state.Invalidation.Status === "Completed") {
|
||||
complete = true
|
||||
}
|
||||
} catch (err) {
|
||||
console.log()
|
||||
}
|
||||
await wait(INVALIDATE_WAIT_PERIODS_MS)
|
||||
totalWaitTimeMs += INVALIDATE_WAIT_PERIODS_MS
|
||||
} while (totalWaitTimeMs <= MAX_INVALIDATE_WAIT_MS && !complete)
|
||||
if (!complete) {
|
||||
throw "Unable to invalidate old app version"
|
||||
}
|
||||
return resp.Invalidation.Status === "Completed"
|
||||
}
|
||||
|
||||
exports.updateDeploymentQuota = async function(quota) {
|
||||
|
@ -102,6 +92,18 @@ exports.verifyDeployment = async function({ instanceId, appId, quota }) {
|
|||
}
|
||||
|
||||
const json = await response.json()
|
||||
if (json.errors) {
|
||||
throw new Error(json.errors)
|
||||
}
|
||||
|
||||
// set credentials here, means any time we're verified we're ready to go
|
||||
if (json.credentials) {
|
||||
AWS.config.update({
|
||||
accessKeyId: json.credentials.AccessKeyId,
|
||||
secretAccessKey: json.credentials.SecretAccessKey,
|
||||
sessionToken: json.credentials.SessionToken,
|
||||
})
|
||||
}
|
||||
|
||||
return json
|
||||
}
|
||||
|
@ -136,7 +138,8 @@ async function prepareUploadForS3({ s3Key, metadata, s3, file }) {
|
|||
|
||||
const upload = await s3
|
||||
.upload({
|
||||
Key: s3Key,
|
||||
// windows filepaths need to be converted to forward slashes for s3
|
||||
Key: sanitize(s3Key).replace(/\\/g, "/"),
|
||||
Body: fileBytes,
|
||||
ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()],
|
||||
Metadata: metadata,
|
||||
|
@ -157,17 +160,10 @@ exports.prepareUploadForS3 = prepareUploadForS3
|
|||
exports.uploadAppAssets = async function({
|
||||
appId,
|
||||
instanceId,
|
||||
credentials,
|
||||
bucket,
|
||||
cfDistribution,
|
||||
accountId,
|
||||
}) {
|
||||
AWS.config.update({
|
||||
accessKeyId: credentials.AccessKeyId,
|
||||
secretAccessKey: credentials.SecretAccessKey,
|
||||
sessionToken: credentials.SessionToken,
|
||||
})
|
||||
|
||||
const s3 = new AWS.S3({
|
||||
params: {
|
||||
Bucket: bucket,
|
||||
|
@ -225,7 +221,7 @@ exports.uploadAppAssets = async function({
|
|||
|
||||
try {
|
||||
await Promise.all(uploads)
|
||||
await invalidateCDN(cfDistribution, appId)
|
||||
return await invalidateCDN(cfDistribution, appId)
|
||||
} catch (err) {
|
||||
console.error("Error uploading budibase app assets to s3", err)
|
||||
throw err
|
||||
|
|
|
@ -4,17 +4,73 @@ const {
|
|||
uploadAppAssets,
|
||||
verifyDeployment,
|
||||
updateDeploymentQuota,
|
||||
MAX_INVALIDATE_WAIT_MS,
|
||||
isInvalidationComplete,
|
||||
} = require("./aws")
|
||||
const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils")
|
||||
const newid = require("../../../db/newid")
|
||||
|
||||
// the max time we can wait for an invalidation to complete before considering it failed
|
||||
const MAX_PENDING_TIME_MS = 30 * 60000
|
||||
|
||||
const DeploymentStatus = {
|
||||
SUCCESS: "SUCCESS",
|
||||
PENDING: "PENDING",
|
||||
FAILURE: "FAILURE",
|
||||
}
|
||||
|
||||
// checks that deployments are in a good state, any pending will be updated
|
||||
async function checkAllDeployments(deployments, user) {
|
||||
let updated = false
|
||||
function update(deployment, status) {
|
||||
deployment.status = status
|
||||
delete deployment.invalidationId
|
||||
delete deployment.cfDistribution
|
||||
updated = true
|
||||
}
|
||||
|
||||
for (let deployment of Object.values(deployments.history)) {
|
||||
// check that no deployments have crashed etc and are now stuck
|
||||
if (
|
||||
deployment.status === DeploymentStatus.PENDING &&
|
||||
Date.now() - deployment.updatedAt > MAX_PENDING_TIME_MS
|
||||
) {
|
||||
update(deployment, DeploymentStatus.FAILURE)
|
||||
}
|
||||
// if pending but not past failure point need to update them
|
||||
else if (deployment.status === DeploymentStatus.PENDING) {
|
||||
let complete = false
|
||||
try {
|
||||
complete = await isInvalidationComplete(
|
||||
deployment.cfDistribution,
|
||||
deployment.invalidationId
|
||||
)
|
||||
} catch (err) {
|
||||
// system may have restarted, need to re-verify
|
||||
if (
|
||||
err !== undefined &&
|
||||
err.code === "InvalidClientTokenId" &&
|
||||
deployment.quota
|
||||
) {
|
||||
await verifyDeployment({
|
||||
...user,
|
||||
quota: deployment.quota,
|
||||
})
|
||||
complete = await isInvalidationComplete(
|
||||
deployment.cfDistribution,
|
||||
deployment.invalidationId
|
||||
)
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
if (complete) {
|
||||
update(deployment, DeploymentStatus.SUCCESS)
|
||||
}
|
||||
}
|
||||
}
|
||||
return { updated, deployments }
|
||||
}
|
||||
|
||||
function replicate(local, remote) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const replication = local.sync(remote)
|
||||
|
@ -102,7 +158,7 @@ async function storeLocalDeploymentHistory(deployment) {
|
|||
async function deployApp({ instanceId, appId, clientId, deploymentId }) {
|
||||
try {
|
||||
const instanceQuota = await getCurrentInstanceQuota(instanceId)
|
||||
const credentials = await verifyDeployment({
|
||||
const verification = await verifyDeployment({
|
||||
instanceId,
|
||||
appId,
|
||||
quota: instanceQuota,
|
||||
|
@ -110,31 +166,36 @@ async function deployApp({ instanceId, appId, clientId, deploymentId }) {
|
|||
|
||||
console.log(`Uploading assets for appID ${appId} assets to s3..`)
|
||||
|
||||
if (credentials.errors) throw new Error(credentials.errors)
|
||||
|
||||
await uploadAppAssets({ clientId, appId, instanceId, ...credentials })
|
||||
const invalidationId = await uploadAppAssets({
|
||||
clientId,
|
||||
appId,
|
||||
instanceId,
|
||||
...verification,
|
||||
})
|
||||
|
||||
// replicate the DB to the couchDB cluster in prod
|
||||
console.log("Replicating local PouchDB to remote..")
|
||||
await replicateCouch({
|
||||
instanceId,
|
||||
clientId,
|
||||
session: credentials.couchDbSession,
|
||||
session: verification.couchDbSession,
|
||||
})
|
||||
|
||||
await updateDeploymentQuota(credentials.quota)
|
||||
await updateDeploymentQuota(verification.quota)
|
||||
|
||||
await storeLocalDeploymentHistory({
|
||||
_id: deploymentId,
|
||||
instanceId,
|
||||
quota: credentials.quota,
|
||||
status: "SUCCESS",
|
||||
invalidationId,
|
||||
cfDistribution: verification.cfDistribution,
|
||||
quota: verification.quota,
|
||||
status: DeploymentStatus.PENDING,
|
||||
})
|
||||
} catch (err) {
|
||||
await storeLocalDeploymentHistory({
|
||||
_id: deploymentId,
|
||||
instanceId,
|
||||
status: "FAILURE",
|
||||
status: DeploymentStatus.FAILURE,
|
||||
err: err.message,
|
||||
})
|
||||
throw new Error(`Deployment Failed: ${err.message}`)
|
||||
|
@ -145,21 +206,14 @@ exports.fetchDeployments = async function(ctx) {
|
|||
try {
|
||||
const db = new PouchDB(ctx.user.instanceId)
|
||||
const deploymentDoc = await db.get("_local/deployments")
|
||||
// check that no deployments have crashed etc and are now stuck
|
||||
let changed = false
|
||||
for (let deployment of Object.values(deploymentDoc.history)) {
|
||||
if (
|
||||
deployment.status === DeploymentStatus.PENDING &&
|
||||
Date.now() - deployment.updatedAt > MAX_INVALIDATE_WAIT_MS
|
||||
) {
|
||||
deployment.status = DeploymentStatus.FAILURE
|
||||
changed = true
|
||||
const { updated, deployments } = await checkAllDeployments(
|
||||
deploymentDoc,
|
||||
ctx.user
|
||||
)
|
||||
if (updated) {
|
||||
await db.put(deployments)
|
||||
}
|
||||
}
|
||||
if (changed) {
|
||||
await db.put(deploymentDoc)
|
||||
}
|
||||
ctx.body = Object.values(deploymentDoc.history).reverse()
|
||||
ctx.body = Object.values(deployments.history).reverse()
|
||||
} catch (err) {
|
||||
ctx.body = []
|
||||
}
|
||||
|
@ -185,10 +239,10 @@ exports.deployApp = async function(ctx) {
|
|||
const deployment = await storeLocalDeploymentHistory({
|
||||
instanceId: ctx.user.instanceId,
|
||||
appId: ctx.user.appId,
|
||||
status: "PENDING",
|
||||
status: DeploymentStatus.PENDING,
|
||||
})
|
||||
|
||||
deployApp({
|
||||
await deployApp({
|
||||
...ctx.user,
|
||||
clientId,
|
||||
deploymentId: deployment._id,
|
||||
|
|
|
@ -163,15 +163,14 @@ exports.serveApp = async function(ctx) {
|
|||
|
||||
exports.serveAttachment = async function(ctx) {
|
||||
const appId = ctx.user.appId
|
||||
|
||||
const attachmentsPath = resolve(budibaseAppsDir(), appId, "attachments")
|
||||
|
||||
// Serve from CloudFront
|
||||
if (process.env.CLOUD) {
|
||||
const S3_URL = `https://cdn.app.budi.live/assets/${appId}/attachments/${ctx.file}`
|
||||
|
||||
const response = await fetch(S3_URL)
|
||||
const body = await response.text()
|
||||
ctx.set("Content-Type", response.headers.get("Content-Type"))
|
||||
ctx.body = body
|
||||
return
|
||||
}
|
||||
|
|
|
@ -109,7 +109,7 @@ exports.save = async function(ctx) {
|
|||
ctx.eventEmitter &&
|
||||
ctx.eventEmitter.emitTable(`table:save`, instanceId, tableToSave)
|
||||
|
||||
if (dataImport && dataImport.path) {
|
||||
if (dataImport && dataImport.csvString) {
|
||||
// Populate the table with rows imported from CSV in a bulk update
|
||||
const data = await csvParser.transform(dataImport)
|
||||
|
||||
|
@ -156,10 +156,7 @@ exports.destroy = async function(ctx) {
|
|||
}
|
||||
|
||||
exports.validateCSVSchema = async function(ctx) {
|
||||
const { file, schema = {} } = ctx.request.body
|
||||
const result = await csvParser.parse(file.path, schema)
|
||||
ctx.body = {
|
||||
schema: result,
|
||||
path: file.path,
|
||||
}
|
||||
const { csvString, schema = {} } = ctx.request.body
|
||||
const result = await csvParser.parse(csvString, schema)
|
||||
ctx.body = { schema: result }
|
||||
}
|
||||
|
|
|
@ -11,8 +11,8 @@ const PARSERS = {
|
|||
datetime: attribute => new Date(attribute).toISOString(),
|
||||
}
|
||||
|
||||
function parse(path, parsers) {
|
||||
const result = csv().fromFile(path)
|
||||
function parse(csvString, parsers) {
|
||||
const result = csv().fromString(csvString)
|
||||
|
||||
const schema = {}
|
||||
|
||||
|
@ -52,7 +52,7 @@ function parse(path, parsers) {
|
|||
})
|
||||
}
|
||||
|
||||
async function transform({ schema, path }) {
|
||||
async function transform({ schema, csvString }) {
|
||||
const colParser = {}
|
||||
|
||||
for (let key in schema) {
|
||||
|
@ -60,7 +60,7 @@ async function transform({ schema, path }) {
|
|||
}
|
||||
|
||||
try {
|
||||
const json = await csv({ colParser }).fromFile(path)
|
||||
const json = await csv({ colParser }).fromString(csvString)
|
||||
return json
|
||||
} catch (err) {
|
||||
console.error(`Error transforming CSV to JSON for data import`, err)
|
||||
|
|
|
@ -1,24 +1,21 @@
|
|||
const fs = require("fs")
|
||||
const sharp = require("sharp")
|
||||
const jimp = require("jimp")
|
||||
const fsPromises = fs.promises
|
||||
|
||||
const FORMATS = {
|
||||
IMAGES: ["png", "jpg", "jpeg", "gif", "svg", "tiff", "raw"],
|
||||
IMAGES: ["png", "jpg", "jpeg", "gif", "bmp", "tiff"],
|
||||
}
|
||||
|
||||
async function processImage(file) {
|
||||
const imgMeta = await sharp(file.path)
|
||||
.resize(300)
|
||||
.toFile(file.outputPath)
|
||||
return {
|
||||
...file,
|
||||
...imgMeta,
|
||||
}
|
||||
function processImage(file) {
|
||||
return jimp.read(file.path).then(img => {
|
||||
return img.resize(300, jimp.AUTO).write(file.outputPath)
|
||||
})
|
||||
}
|
||||
|
||||
async function process(file) {
|
||||
if (FORMATS.IMAGES.includes(file.extension.toLowerCase())) {
|
||||
return await processImage(file)
|
||||
await processImage(file)
|
||||
return file
|
||||
}
|
||||
|
||||
// No processing required
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
const csvParser = require("../csvParser");
|
||||
const fs = require("fs")
|
||||
const csvParser = require("../csvParser")
|
||||
|
||||
const CSV_PATH = __dirname + "/test.csv";
|
||||
const CSV_PATH = __dirname + "/test.csv"
|
||||
|
||||
const SCHEMAS = {
|
||||
VALID: {
|
||||
|
@ -27,16 +28,16 @@ const SCHEMAS = {
|
|||
BROKEN: {
|
||||
Address: {
|
||||
type: "datetime",
|
||||
}
|
||||
},
|
||||
};
|
||||
},
|
||||
}
|
||||
|
||||
describe("CSV Parser", () => {
|
||||
const csvString = fs.readFileSync(CSV_PATH, "utf8")
|
||||
|
||||
describe("parsing", () => {
|
||||
it("returns status and types for a valid CSV transformation", async () => {
|
||||
expect(
|
||||
await csvParser.parse(CSV_PATH, SCHEMAS.VALID)
|
||||
).toEqual({
|
||||
expect(await csvParser.parse(csvString, SCHEMAS.VALID)).toEqual({
|
||||
Address: {
|
||||
success: true,
|
||||
type: "string",
|
||||
|
@ -49,13 +50,11 @@ describe("CSV Parser", () => {
|
|||
success: true,
|
||||
type: "string",
|
||||
},
|
||||
});
|
||||
});
|
||||
})
|
||||
})
|
||||
|
||||
it("returns status and types for an invalid CSV transformation", async () => {
|
||||
expect(
|
||||
await csvParser.parse(CSV_PATH, SCHEMAS.INVALID)
|
||||
).toEqual({
|
||||
expect(await csvParser.parse(csvString, SCHEMAS.INVALID)).toEqual({
|
||||
Address: {
|
||||
success: false,
|
||||
type: "number",
|
||||
|
@ -68,41 +67,43 @@ describe("CSV Parser", () => {
|
|||
success: true,
|
||||
type: "string",
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe("transformation", () => {
|
||||
it("transforms a CSV file into JSON", async () => {
|
||||
expect(
|
||||
await csvParser.transform({
|
||||
schema: SCHEMAS.VALID,
|
||||
path: CSV_PATH,
|
||||
csvString,
|
||||
})
|
||||
).toMatchSnapshot()
|
||||
})
|
||||
).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it("transforms a CSV file into JSON ignoring certain fields", async () => {
|
||||
expect(
|
||||
await csvParser.transform({
|
||||
schema: SCHEMAS.IGNORE,
|
||||
path: CSV_PATH,
|
||||
csvString,
|
||||
})
|
||||
).toEqual([
|
||||
{
|
||||
Name: "Bert"
|
||||
Name: "Bert",
|
||||
},
|
||||
{
|
||||
Name: "Ernie"
|
||||
Name: "Ernie",
|
||||
},
|
||||
{
|
||||
Name: "Big Bird"
|
||||
}
|
||||
]);
|
||||
});
|
||||
Name: "Big Bird",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it("throws an error on invalid schema", async () => {
|
||||
await expect(csvParser.transform({ schema: SCHEMAS.BROKEN, path: CSV_PATH })).rejects.toThrow()
|
||||
});
|
||||
});
|
||||
});
|
||||
await expect(
|
||||
csvParser.transform({ schema: SCHEMAS.BROKEN, csvString })
|
||||
).rejects.toThrow()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -13,7 +13,7 @@
|
|||
"dev:builder": "rollup -cw"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@budibase/client": "^0.2.5",
|
||||
"@budibase/client": "^0.2.6",
|
||||
"@rollup/plugin-commonjs": "^11.1.0",
|
||||
"lodash": "^4.17.15",
|
||||
"rollup": "^2.11.2",
|
||||
|
@ -31,7 +31,7 @@
|
|||
"keywords": [
|
||||
"svelte"
|
||||
],
|
||||
"version": "0.2.5",
|
||||
"version": "0.2.6",
|
||||
"license": "MIT",
|
||||
"gitHead": "284cceb9b703c38566c6e6363c022f79a08d5691",
|
||||
"dependencies": {
|
||||
|
|
|
@ -51,8 +51,15 @@
|
|||
// Fetch table schema so we can check for linked rows
|
||||
const tableObj = await fetchTable(row.tableId)
|
||||
for (let key of Object.keys(tableObj.schema)) {
|
||||
if (tableObj.schema[key].type === "link") {
|
||||
const type = tableObj.schema[key].type
|
||||
if (type === "link") {
|
||||
row[`${key}_count`] = Array.isArray(row[key]) ? row[key].length : 0
|
||||
} else if (type === "attachment") {
|
||||
let url = null
|
||||
if (Array.isArray(row[key]) && row[key][0] != null) {
|
||||
url = row[key][0].url
|
||||
}
|
||||
row[`${key}_first`] = url
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -6,11 +6,11 @@ export default async function fetchData(datasource, store) {
|
|||
if (name) {
|
||||
let rows = []
|
||||
if (type === "table") {
|
||||
rows = fetchTableData()
|
||||
rows = await fetchTableData()
|
||||
} else if (type === "view") {
|
||||
rows = fetchViewData()
|
||||
rows = await fetchViewData()
|
||||
} else if (type === "link") {
|
||||
rows = fetchLinkedRowsData()
|
||||
rows = await fetchLinkedRowsData()
|
||||
}
|
||||
|
||||
// Fetch table schema so we can check for linked rows
|
||||
|
@ -19,8 +19,15 @@ export default async function fetchData(datasource, store) {
|
|||
const keys = Object.keys(table.schema)
|
||||
rows.forEach(row => {
|
||||
for (let key of keys) {
|
||||
if (table.schema[key].type === "link") {
|
||||
const type = table.schema[key].type
|
||||
if (type === "link") {
|
||||
row[`${key}_count`] = Array.isArray(row[key]) ? row[key].length : 0
|
||||
} else if (type === "attachment") {
|
||||
let url = null
|
||||
if (Array.isArray(row[key]) && row[key][0] != null) {
|
||||
url = row[key][0].url
|
||||
}
|
||||
row[`${key}_first`] = url
|
||||
}
|
||||
}
|
||||
})
|
||||
|
|
Loading…
Reference in New Issue