Merge branch 'master' of github.com:Budibase/budibase into backend-ag-grid

This commit is contained in:
Martin McKeaveney 2020-10-21 15:03:31 +01:00
commit bc58c8ebf5
23 changed files with 764 additions and 410 deletions

View File

@ -1,5 +1,5 @@
{
"version": "0.2.5",
"version": "0.2.6",
"npmClient": "yarn",
"packages": [
"packages/*"

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "0.2.5",
"version": "0.2.6",
"license": "AGPL-3.0",
"private": true,
"scripts": {
@ -64,7 +64,7 @@
},
"dependencies": {
"@budibase/bbui": "^1.44.1",
"@budibase/client": "^0.2.5",
"@budibase/client": "^0.2.6",
"@budibase/colorpicker": "^1.0.1",
"@budibase/svelte-ag-grid": "^0.0.13",
"@fortawesome/fontawesome-free": "^5.14.0",

View File

@ -86,10 +86,12 @@ const contextToBindables = (tables, walkResult) => context => {
}
const newBindable = ([key, fieldSchema]) => {
// Replace link bindings with a new property representing the count
// Replace certain bindings with a new property to help display components
let runtimeBoundKey = key
if (fieldSchema.type === "link") {
runtimeBoundKey = `${key}_count`
} else if (fieldSchema.type === "attachment") {
runtimeBoundKey = `${key}_first`
}
return {
type: "context",

View File

@ -233,7 +233,7 @@ const createLink = store => async (url, title) => {
// Save page and regenerate all CSS because otherwise weird things happen
nav._children = [...nav._children, newLink]
setCurrentPage("main")
state.currentPageName = "main"
regenerateCssForScreen(state.pages.main)
for (let screen of state.pages.main._screens) {
regenerateCssForScreen(screen)

View File

@ -1,5 +1,5 @@
<script>
import { Heading, Body, Button, Select } from "@budibase/bbui"
import { Heading, Body, Button, Select, Label } from "@budibase/bbui"
import { notifier } from "builderStore/store/notifications"
import { FIELDS } from "constants/backend"
import api from "builderStore/api"
@ -14,15 +14,17 @@
schema: {},
}
let parseResult
let csvString
let primaryDisplay
let schema = {}
let fields = []
$: schema = parseResult && parseResult.schema
$: valid =
!schema || Object.keys(schema).every(column => schema[column].success)
$: valid = !schema || fields.every(column => schema[column].success)
$: dataImport = {
valid,
schema: buildTableSchema(schema),
path: files[0] && files[0].path,
csvString,
primaryDisplay,
}
function buildTableSchema(schema) {
@ -43,11 +45,20 @@
async function validateCSV() {
const response = await api.post("/api/tables/csv/validate", {
file: files[0],
csvString,
schema: schema || {},
})
parseResult = await response.json()
const parseResult = await response.json()
schema = parseResult && parseResult.schema
fields = Object.keys(schema || {}).filter(
key => schema[key].type !== "omit"
)
// Check primary display is valid
if (!primaryDisplay || fields.indexOf(primaryDisplay) === -1) {
primaryDisplay = fields[0]
}
if (response.status !== 200) {
notifier.danger("CSV Invalid, please try another CSV file")
@ -57,13 +68,7 @@
async function handleFile(evt) {
const fileArray = Array.from(evt.target.files)
const filesToProcess = fileArray.map(({ name, path, size }) => ({
name,
path,
size,
}))
if (filesToProcess.some(file => file.size >= FILE_SIZE_LIMIT)) {
if (fileArray.some(file => file.size >= FILE_SIZE_LIMIT)) {
notifier.danger(
`Files cannot exceed ${FILE_SIZE_LIMIT /
BYTES_IN_MB}MB. Please try again with smaller files.`
@ -71,9 +76,14 @@
return
}
files = filesToProcess
await validateCSV()
// Read CSV as plain text to upload alongside schema
let reader = new FileReader()
reader.addEventListener("load", function(e) {
csvString = e.target.result
files = fileArray
validateCSV()
})
reader.readAsBinaryString(fileArray[0])
}
async function omitColumn(columnName) {
@ -94,8 +104,8 @@
</label>
</div>
<div class="schema-fields">
{#if schema}
{#each Object.keys(schema).filter(key => schema[key].type !== 'omit') as columnName}
{#if fields.length}
{#each fields as columnName}
<div class="field">
<span>{columnName}</span>
<Select
@ -117,6 +127,16 @@
{/each}
{/if}
</div>
{#if fields.length}
<div class="display-column">
<Label extraSmall grey>Display Column</Label>
<Select thin secondary bind:value={primaryDisplay}>
{#each fields as field}
<option value={field}>{field}</option>
{/each}
</Select>
</div>
{/if}
<style>
.dropzone {
@ -188,4 +208,8 @@
grid-gap: var(--spacing-m);
font-size: var(--font-size-xs);
}
.display-column {
margin-top: var(--spacing-xl);
}
</style>

View File

@ -38,12 +38,19 @@
}
async function saveTable() {
// Create table
const table = await backendUiStore.actions.tables.save({
let newTable = {
name,
schema: dataImport.schema || {},
dataImport,
})
}
// Only set primary display if defined
if (dataImport.primaryDisplay && dataImport.primaryDisplay.length) {
newTable.primaryDisplay = dataImport.primaryDisplay
}
// Create table
const table = await backendUiStore.actions.tables.save(newTable)
notifier.success(`Table ${name} created successfully.`)
analytics.captureEvent("Table Created", { name })

View File

@ -19,7 +19,7 @@
hour12: true,
},
}
const POLL_INTERVAL = 1000
const POLL_INTERVAL = 5000
export let appId
@ -68,7 +68,7 @@
</span>
</div>
<div class="deployment-right">
{#if deployment.status.toLowerCase() === "pending"}
{#if deployment.status.toLowerCase() === 'pending'}
<Spinner size="10" />
{/if}
<div class={`deployment-status ${deployment.status}`}>

View File

@ -135,6 +135,9 @@
.toprightnav {
display: flex;
flex-direction: row;
justify-content: flex-end;
align-items: center;
}
.topleftnav {

View File

@ -1,6 +1,6 @@
{
"name": "budibase",
"version": "0.2.5",
"version": "0.2.6",
"description": "Budibase CLI",
"repository": "https://github.com/Budibase/Budibase",
"homepage": "https://www.budibase.com",
@ -17,7 +17,7 @@
"author": "Budibase",
"license": "AGPL-3.0-or-later",
"dependencies": {
"@budibase/server": "^0.2.5",
"@budibase/server": "^0.2.6",
"@inquirer/password": "^0.0.6-alpha.0",
"chalk": "^2.4.2",
"dotenv": "^8.2.0",

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "0.2.5",
"version": "0.2.6",
"license": "MPL-2.0",
"main": "dist/budibase-client.js",
"module": "dist/budibase-client.esm.mjs",

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/server",
"version": "0.2.5",
"version": "0.2.6",
"description": "Budibase Web Server",
"main": "src/electron.js",
"repository": {
@ -42,7 +42,7 @@
"author": "Michael Shanks",
"license": "AGPL-3.0-or-later",
"dependencies": {
"@budibase/client": "^0.2.5",
"@budibase/client": "^0.2.6",
"@koa/router": "^8.0.0",
"@sendgrid/mail": "^7.1.1",
"@sentry/node": "^5.19.2",
@ -57,6 +57,7 @@
"electron-updater": "^4.3.1",
"fix-path": "^3.0.0",
"fs-extra": "^8.1.0",
"jimp": "^0.16.1",
"joi": "^17.2.1",
"jsonwebtoken": "^8.5.1",
"koa": "^2.7.0",
@ -73,7 +74,7 @@
"pouchdb": "^7.2.1",
"pouchdb-all-dbs": "^1.0.2",
"pouchdb-replication-stream": "^1.2.9",
"sharp": "^0.26.0",
"sanitize-s3-objectkey": "^0.0.1",
"squirrelly": "^7.5.0",
"tar-fs": "^2.1.0",
"uuid": "^3.3.2",

View File

@ -1,19 +1,13 @@
const fs = require("fs")
const { join } = require("../../../utilities/centralPath")
let { wait } = require("../../../utilities")
const AWS = require("aws-sdk")
const fetch = require("node-fetch")
const uuid = require("uuid")
const sanitize = require("sanitize-s3-objectkey")
const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
const PouchDB = require("../../../db")
const environment = require("../../../environment")
const MAX_INVALIDATE_WAIT_MS = 120000
const INVALIDATE_WAIT_PERIODS_MS = 5000
// export so main deploy functions can use too
exports.MAX_INVALIDATE_WAIT_MS = MAX_INVALIDATE_WAIT_MS
async function invalidateCDN(cfDistribution, appId) {
const cf = new AWS.CloudFront({})
const resp = await cf
@ -28,28 +22,24 @@ async function invalidateCDN(cfDistribution, appId) {
},
})
.promise()
let totalWaitTimeMs = 0
let complete = false
do {
try {
const state = await cf
return resp.Invalidation.Id
}
exports.isInvalidationComplete = async function(
distributionId,
invalidationId
) {
if (distributionId == null || invalidationId == null) {
return false
}
const cf = new AWS.CloudFront({})
const resp = await cf
.getInvalidation({
DistributionId: cfDistribution,
Id: resp.Invalidation.Id,
DistributionId: distributionId,
Id: invalidationId,
})
.promise()
if (state.Invalidation.Status === "Completed") {
complete = true
}
} catch (err) {
console.log()
}
await wait(INVALIDATE_WAIT_PERIODS_MS)
totalWaitTimeMs += INVALIDATE_WAIT_PERIODS_MS
} while (totalWaitTimeMs <= MAX_INVALIDATE_WAIT_MS && !complete)
if (!complete) {
throw "Unable to invalidate old app version"
}
return resp.Invalidation.Status === "Completed"
}
exports.updateDeploymentQuota = async function(quota) {
@ -102,6 +92,18 @@ exports.verifyDeployment = async function({ instanceId, appId, quota }) {
}
const json = await response.json()
if (json.errors) {
throw new Error(json.errors)
}
// set credentials here, means any time we're verified we're ready to go
if (json.credentials) {
AWS.config.update({
accessKeyId: json.credentials.AccessKeyId,
secretAccessKey: json.credentials.SecretAccessKey,
sessionToken: json.credentials.SessionToken,
})
}
return json
}
@ -136,7 +138,8 @@ async function prepareUploadForS3({ s3Key, metadata, s3, file }) {
const upload = await s3
.upload({
Key: s3Key,
// windows filepaths need to be converted to forward slashes for s3
Key: sanitize(s3Key).replace(/\\/g, "/"),
Body: fileBytes,
ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()],
Metadata: metadata,
@ -157,17 +160,10 @@ exports.prepareUploadForS3 = prepareUploadForS3
exports.uploadAppAssets = async function({
appId,
instanceId,
credentials,
bucket,
cfDistribution,
accountId,
}) {
AWS.config.update({
accessKeyId: credentials.AccessKeyId,
secretAccessKey: credentials.SecretAccessKey,
sessionToken: credentials.SessionToken,
})
const s3 = new AWS.S3({
params: {
Bucket: bucket,
@ -225,7 +221,7 @@ exports.uploadAppAssets = async function({
try {
await Promise.all(uploads)
await invalidateCDN(cfDistribution, appId)
return await invalidateCDN(cfDistribution, appId)
} catch (err) {
console.error("Error uploading budibase app assets to s3", err)
throw err

View File

@ -4,17 +4,73 @@ const {
uploadAppAssets,
verifyDeployment,
updateDeploymentQuota,
MAX_INVALIDATE_WAIT_MS,
isInvalidationComplete,
} = require("./aws")
const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils")
const newid = require("../../../db/newid")
// the max time we can wait for an invalidation to complete before considering it failed
const MAX_PENDING_TIME_MS = 30 * 60000
const DeploymentStatus = {
SUCCESS: "SUCCESS",
PENDING: "PENDING",
FAILURE: "FAILURE",
}
// checks that deployments are in a good state, any pending will be updated
async function checkAllDeployments(deployments, user) {
let updated = false
function update(deployment, status) {
deployment.status = status
delete deployment.invalidationId
delete deployment.cfDistribution
updated = true
}
for (let deployment of Object.values(deployments.history)) {
// check that no deployments have crashed etc and are now stuck
if (
deployment.status === DeploymentStatus.PENDING &&
Date.now() - deployment.updatedAt > MAX_PENDING_TIME_MS
) {
update(deployment, DeploymentStatus.FAILURE)
}
// if pending but not past failure point need to update them
else if (deployment.status === DeploymentStatus.PENDING) {
let complete = false
try {
complete = await isInvalidationComplete(
deployment.cfDistribution,
deployment.invalidationId
)
} catch (err) {
// system may have restarted, need to re-verify
if (
err !== undefined &&
err.code === "InvalidClientTokenId" &&
deployment.quota
) {
await verifyDeployment({
...user,
quota: deployment.quota,
})
complete = await isInvalidationComplete(
deployment.cfDistribution,
deployment.invalidationId
)
} else {
throw err
}
}
if (complete) {
update(deployment, DeploymentStatus.SUCCESS)
}
}
}
return { updated, deployments }
}
function replicate(local, remote) {
return new Promise((resolve, reject) => {
const replication = local.sync(remote)
@ -102,7 +158,7 @@ async function storeLocalDeploymentHistory(deployment) {
async function deployApp({ instanceId, appId, clientId, deploymentId }) {
try {
const instanceQuota = await getCurrentInstanceQuota(instanceId)
const credentials = await verifyDeployment({
const verification = await verifyDeployment({
instanceId,
appId,
quota: instanceQuota,
@ -110,31 +166,36 @@ async function deployApp({ instanceId, appId, clientId, deploymentId }) {
console.log(`Uploading assets for appID ${appId} assets to s3..`)
if (credentials.errors) throw new Error(credentials.errors)
await uploadAppAssets({ clientId, appId, instanceId, ...credentials })
const invalidationId = await uploadAppAssets({
clientId,
appId,
instanceId,
...verification,
})
// replicate the DB to the couchDB cluster in prod
console.log("Replicating local PouchDB to remote..")
await replicateCouch({
instanceId,
clientId,
session: credentials.couchDbSession,
session: verification.couchDbSession,
})
await updateDeploymentQuota(credentials.quota)
await updateDeploymentQuota(verification.quota)
await storeLocalDeploymentHistory({
_id: deploymentId,
instanceId,
quota: credentials.quota,
status: "SUCCESS",
invalidationId,
cfDistribution: verification.cfDistribution,
quota: verification.quota,
status: DeploymentStatus.PENDING,
})
} catch (err) {
await storeLocalDeploymentHistory({
_id: deploymentId,
instanceId,
status: "FAILURE",
status: DeploymentStatus.FAILURE,
err: err.message,
})
throw new Error(`Deployment Failed: ${err.message}`)
@ -145,21 +206,14 @@ exports.fetchDeployments = async function(ctx) {
try {
const db = new PouchDB(ctx.user.instanceId)
const deploymentDoc = await db.get("_local/deployments")
// check that no deployments have crashed etc and are now stuck
let changed = false
for (let deployment of Object.values(deploymentDoc.history)) {
if (
deployment.status === DeploymentStatus.PENDING &&
Date.now() - deployment.updatedAt > MAX_INVALIDATE_WAIT_MS
) {
deployment.status = DeploymentStatus.FAILURE
changed = true
const { updated, deployments } = await checkAllDeployments(
deploymentDoc,
ctx.user
)
if (updated) {
await db.put(deployments)
}
}
if (changed) {
await db.put(deploymentDoc)
}
ctx.body = Object.values(deploymentDoc.history).reverse()
ctx.body = Object.values(deployments.history).reverse()
} catch (err) {
ctx.body = []
}
@ -185,10 +239,10 @@ exports.deployApp = async function(ctx) {
const deployment = await storeLocalDeploymentHistory({
instanceId: ctx.user.instanceId,
appId: ctx.user.appId,
status: "PENDING",
status: DeploymentStatus.PENDING,
})
deployApp({
await deployApp({
...ctx.user,
clientId,
deploymentId: deployment._id,

View File

@ -163,15 +163,14 @@ exports.serveApp = async function(ctx) {
exports.serveAttachment = async function(ctx) {
const appId = ctx.user.appId
const attachmentsPath = resolve(budibaseAppsDir(), appId, "attachments")
// Serve from CloudFront
if (process.env.CLOUD) {
const S3_URL = `https://cdn.app.budi.live/assets/${appId}/attachments/${ctx.file}`
const response = await fetch(S3_URL)
const body = await response.text()
ctx.set("Content-Type", response.headers.get("Content-Type"))
ctx.body = body
return
}

View File

@ -109,7 +109,7 @@ exports.save = async function(ctx) {
ctx.eventEmitter &&
ctx.eventEmitter.emitTable(`table:save`, instanceId, tableToSave)
if (dataImport && dataImport.path) {
if (dataImport && dataImport.csvString) {
// Populate the table with rows imported from CSV in a bulk update
const data = await csvParser.transform(dataImport)
@ -156,10 +156,7 @@ exports.destroy = async function(ctx) {
}
exports.validateCSVSchema = async function(ctx) {
const { file, schema = {} } = ctx.request.body
const result = await csvParser.parse(file.path, schema)
ctx.body = {
schema: result,
path: file.path,
}
const { csvString, schema = {} } = ctx.request.body
const result = await csvParser.parse(csvString, schema)
ctx.body = { schema: result }
}

View File

@ -11,8 +11,8 @@ const PARSERS = {
datetime: attribute => new Date(attribute).toISOString(),
}
function parse(path, parsers) {
const result = csv().fromFile(path)
function parse(csvString, parsers) {
const result = csv().fromString(csvString)
const schema = {}
@ -52,7 +52,7 @@ function parse(path, parsers) {
})
}
async function transform({ schema, path }) {
async function transform({ schema, csvString }) {
const colParser = {}
for (let key in schema) {
@ -60,7 +60,7 @@ async function transform({ schema, path }) {
}
try {
const json = await csv({ colParser }).fromFile(path)
const json = await csv({ colParser }).fromString(csvString)
return json
} catch (err) {
console.error(`Error transforming CSV to JSON for data import`, err)

View File

@ -1,24 +1,21 @@
const fs = require("fs")
const sharp = require("sharp")
const jimp = require("jimp")
const fsPromises = fs.promises
const FORMATS = {
IMAGES: ["png", "jpg", "jpeg", "gif", "svg", "tiff", "raw"],
IMAGES: ["png", "jpg", "jpeg", "gif", "bmp", "tiff"],
}
async function processImage(file) {
const imgMeta = await sharp(file.path)
.resize(300)
.toFile(file.outputPath)
return {
...file,
...imgMeta,
}
function processImage(file) {
return jimp.read(file.path).then(img => {
return img.resize(300, jimp.AUTO).write(file.outputPath)
})
}
async function process(file) {
if (FORMATS.IMAGES.includes(file.extension.toLowerCase())) {
return await processImage(file)
await processImage(file)
return file
}
// No processing required

View File

@ -1,6 +1,7 @@
const csvParser = require("../csvParser");
const fs = require("fs")
const csvParser = require("../csvParser")
const CSV_PATH = __dirname + "/test.csv";
const CSV_PATH = __dirname + "/test.csv"
const SCHEMAS = {
VALID: {
@ -27,16 +28,16 @@ const SCHEMAS = {
BROKEN: {
Address: {
type: "datetime",
}
},
};
},
}
describe("CSV Parser", () => {
const csvString = fs.readFileSync(CSV_PATH, "utf8")
describe("parsing", () => {
it("returns status and types for a valid CSV transformation", async () => {
expect(
await csvParser.parse(CSV_PATH, SCHEMAS.VALID)
).toEqual({
expect(await csvParser.parse(csvString, SCHEMAS.VALID)).toEqual({
Address: {
success: true,
type: "string",
@ -49,13 +50,11 @@ describe("CSV Parser", () => {
success: true,
type: "string",
},
});
});
})
})
it("returns status and types for an invalid CSV transformation", async () => {
expect(
await csvParser.parse(CSV_PATH, SCHEMAS.INVALID)
).toEqual({
expect(await csvParser.parse(csvString, SCHEMAS.INVALID)).toEqual({
Address: {
success: false,
type: "number",
@ -68,41 +67,43 @@ describe("CSV Parser", () => {
success: true,
type: "string",
},
});
});
});
})
})
})
describe("transformation", () => {
it("transforms a CSV file into JSON", async () => {
expect(
await csvParser.transform({
schema: SCHEMAS.VALID,
path: CSV_PATH,
csvString,
})
).toMatchSnapshot()
})
).toMatchSnapshot();
});
it("transforms a CSV file into JSON ignoring certain fields", async () => {
expect(
await csvParser.transform({
schema: SCHEMAS.IGNORE,
path: CSV_PATH,
csvString,
})
).toEqual([
{
Name: "Bert"
Name: "Bert",
},
{
Name: "Ernie"
Name: "Ernie",
},
{
Name: "Big Bird"
}
]);
});
Name: "Big Bird",
},
])
})
it("throws an error on invalid schema", async () => {
await expect(csvParser.transform({ schema: SCHEMAS.BROKEN, path: CSV_PATH })).rejects.toThrow()
});
});
});
await expect(
csvParser.transform({ schema: SCHEMAS.BROKEN, csvString })
).rejects.toThrow()
})
})
})

File diff suppressed because it is too large Load Diff

View File

@ -13,7 +13,7 @@
"dev:builder": "rollup -cw"
},
"devDependencies": {
"@budibase/client": "^0.2.5",
"@budibase/client": "^0.2.6",
"@rollup/plugin-commonjs": "^11.1.0",
"lodash": "^4.17.15",
"rollup": "^2.11.2",
@ -31,7 +31,7 @@
"keywords": [
"svelte"
],
"version": "0.2.5",
"version": "0.2.6",
"license": "MIT",
"gitHead": "284cceb9b703c38566c6e6363c022f79a08d5691",
"dependencies": {

View File

@ -51,8 +51,15 @@
// Fetch table schema so we can check for linked rows
const tableObj = await fetchTable(row.tableId)
for (let key of Object.keys(tableObj.schema)) {
if (tableObj.schema[key].type === "link") {
const type = tableObj.schema[key].type
if (type === "link") {
row[`${key}_count`] = Array.isArray(row[key]) ? row[key].length : 0
} else if (type === "attachment") {
let url = null
if (Array.isArray(row[key]) && row[key][0] != null) {
url = row[key][0].url
}
row[`${key}_first`] = url
}
}

View File

@ -6,11 +6,11 @@ export default async function fetchData(datasource, store) {
if (name) {
let rows = []
if (type === "table") {
rows = fetchTableData()
rows = await fetchTableData()
} else if (type === "view") {
rows = fetchViewData()
rows = await fetchViewData()
} else if (type === "link") {
rows = fetchLinkedRowsData()
rows = await fetchLinkedRowsData()
}
// Fetch table schema so we can check for linked rows
@ -19,8 +19,15 @@ export default async function fetchData(datasource, store) {
const keys = Object.keys(table.schema)
rows.forEach(row => {
for (let key of keys) {
if (table.schema[key].type === "link") {
const type = table.schema[key].type
if (type === "link") {
row[`${key}_count`] = Array.isArray(row[key]) ? row[key].length : 0
} else if (type === "attachment") {
let url = null
if (Array.isArray(row[key]) && row[key][0] != null) {
url = row[key][0].url
}
row[`${key}_first`] = url
}
}
})