From 6b00fb2d8e675b78b14543340aea7c6ed653578e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 23 Sep 2020 16:15:09 +0100 Subject: [PATCH] local file upload from apps --- .../builder/src/builderStore/store/backend.js | 2 +- .../src/components/common/Dropzone.svelte | 3 +- packages/server/package.json | 2 +- .../server/src/api/controllers/deploy/aws.js | 20 +- packages/server/src/api/controllers/static.js | 69 +++- packages/server/src/api/routes/static.js | 1 + packages/server/yarn.lock | 9 +- .../standard-components/src/DataForm.svelte | 6 +- .../standard-components/src/DataTable.svelte | 7 +- packages/standard-components/src/api.js | 5 +- .../src/attachments/AttachmentList.svelte | 64 ++++ .../src/attachments/Dropzone.svelte | 299 ++++++++++++++++++ .../src/attachments/fileTypes.js | 5 + 13 files changed, 473 insertions(+), 19 deletions(-) create mode 100644 packages/standard-components/src/attachments/AttachmentList.svelte create mode 100644 packages/standard-components/src/attachments/Dropzone.svelte create mode 100644 packages/standard-components/src/attachments/fileTypes.js diff --git a/packages/builder/src/builderStore/store/backend.js b/packages/builder/src/builderStore/store/backend.js index 3835f44d64..be7dccde6c 100644 --- a/packages/builder/src/builderStore/store/backend.js +++ b/packages/builder/src/builderStore/store/backend.js @@ -28,7 +28,7 @@ export const getBackendUiStore = () => { }, }, records: { - save: () => + save: record => store.update(state => { state.selectedView = state.selectedView return state diff --git a/packages/builder/src/components/common/Dropzone.svelte b/packages/builder/src/components/common/Dropzone.svelte index a356ff811f..0af4aa5ffd 100644 --- a/packages/builder/src/components/common/Dropzone.svelte +++ b/packages/builder/src/components/common/Dropzone.svelte @@ -35,10 +35,11 @@ return } - const filesToProcess = fileArray.map(({ name, path, size }) => ({ + const filesToProcess = fileArray.map(({ name, path, size, type }) => ({ name, path, size, + type, })) const response = await api.post(`/api/attachments/process`, { diff --git a/packages/server/package.json b/packages/server/package.json index c73ade269f..520870d735 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -58,7 +58,7 @@ "joi": "^17.2.1", "jsonwebtoken": "^8.5.1", "koa": "^2.7.0", - "koa-body": "^4.1.0", + "koa-body": "^4.2.0", "koa-compress": "^4.0.1", "koa-pino-logger": "^3.0.0", "koa-send": "^5.0.0", diff --git a/packages/server/src/api/controllers/deploy/aws.js b/packages/server/src/api/controllers/deploy/aws.js index 91f28793da..2cdc419635 100644 --- a/packages/server/src/api/controllers/deploy/aws.js +++ b/packages/server/src/api/controllers/deploy/aws.js @@ -64,19 +64,30 @@ function walkDir(dirPath, callback) { } } -function prepareUploadForS3({ filePath, s3Key, metadata, s3 }) { - const fileExtension = [...filePath.split(".")].pop() +async function prepareUploadForS3({ filePath, s3Key, metadata, fileType, s3 }) { + const contentType = + fileType || CONTENT_TYPE_MAP[[...filePath.split(".")].pop().toLowerCase()] const fileBytes = fs.readFileSync(filePath) - return s3 + + const upload = await s3 .upload({ Key: s3Key, Body: fileBytes, - ContentType: CONTENT_TYPE_MAP[fileExtension.toLowerCase()], + ContentType: contentType, Metadata: metadata, }) .promise() + + return { + // TODO: return all the passed in file info + ...upload, + url: upload.Location, + key: upload.Key, + } } +exports.prepareUploadForS3 = prepareUploadForS3 + exports.uploadAppAssets = async function({ appId, instanceId, @@ -124,6 +135,7 @@ exports.uploadAppAssets = async function({ if (file.uploaded) continue const attachmentUpload = prepareUploadForS3({ + fileType: file.type, filePath: file.path, s3Key: `assets/${appId}/attachments/${file.name}`, s3, diff --git a/packages/server/src/api/controllers/static.js b/packages/server/src/api/controllers/static.js index fd4afb42e7..2ea4691038 100644 --- a/packages/server/src/api/controllers/static.js +++ b/packages/server/src/api/controllers/static.js @@ -4,6 +4,8 @@ const jwt = require("jsonwebtoken") const fetch = require("node-fetch") const fs = require("fs") const uuid = require("uuid") +const AWS = require("aws-sdk") +const { prepareUploadForS3 } = require("./deploy/aws") const { budibaseAppsDir, @@ -22,6 +24,65 @@ exports.serveBuilder = async function(ctx) { await send(ctx, ctx.file, { root: ctx.devPath || builderPath }) } +exports.uploadFile = async function(ctx) { + let files + files = + ctx.request.files.file.length > 1 + ? Array.from(ctx.request.files.file) + : [ctx.request.files.file] + + console.log(files) + + let uploads = [] + + const attachmentsPath = resolve( + budibaseAppsDir(), + ctx.user.appId, + "attachments" + ) + + if (process.env.CLOUD) { + // remote upload + const s3 = new AWS.S3({ + params: { + // TODO: Don't hardcode + Bucket: "", + }, + }) + + // TODO: probably need to UUID this too, so that we don't override by name + uploads = files.map(file => + prepareUploadForS3({ + fileType: file.type, + filePath: file.path, + s3Key: `assets/${ctx.user.appId}/attachments/${file.name}`, + s3, + }) + ) + } else { + uploads = files.map(file => { + const fileExtension = [...file.name.split(".")].pop() + const processedFileName = `${uuid.v4()}.${fileExtension}` + + return fileProcessor.process({ + format: file.format, + type: file.type, + name: file.name, + size: file.size, + path: file.path, + processedFileName, + extension: fileExtension, + outputPath: `${attachmentsPath}/${processedFileName}`, + url: `/attachments/${processedFileName}`, + }) + }) + } + + const responses = await Promise.all(uploads) + + ctx.body = responses +} + exports.processLocalFileUpload = async function(ctx) { const { files } = ctx.request.body @@ -38,14 +99,14 @@ exports.processLocalFileUpload = async function(ctx) { const filesToProcess = files.map(file => { const fileExtension = [...file.path.split(".")].pop() // filenames converted to UUIDs so they are unique - const fileName = `${uuid.v4()}.${fileExtension}` + const processedFileName = `${uuid.v4()}.${fileExtension}` return { ...file, - fileName, + processedFileName, extension: fileExtension, - outputPath: join(attachmentsPath, fileName), - url: join("/attachments", fileName), + outputPath: join(attachmentsPath, processedFileName), + url: join("/attachments", processedFileName), } }) diff --git a/packages/server/src/api/routes/static.js b/packages/server/src/api/routes/static.js index 0ce6a62668..ccb8ee2b57 100644 --- a/packages/server/src/api/routes/static.js +++ b/packages/server/src/api/routes/static.js @@ -28,6 +28,7 @@ router authorized(BUILDER), controller.processLocalFileUpload ) + .post("/api/attachments/upload", controller.uploadFile) .get("/componentlibrary", controller.serveComponentLibrary) .get("/assets/:file*", controller.serveAppAsset) .get("/attachments/:file*", controller.serveAttachment) diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index a13c7dd4d8..acaa908d81 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -534,10 +534,12 @@ "@types/events@*": version "3.0.0" resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7" + integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g== "@types/formidable@^1.0.31": version "1.0.31" resolved "https://registry.yarnpkg.com/@types/formidable/-/formidable-1.0.31.tgz#274f9dc2d0a1a9ce1feef48c24ca0859e7ec947b" + integrity sha512-dIhM5t8lRP0oWe2HF8MuPvdd1TpPTjhDMAqemcq6oIZQCBQTovhBAdTQ5L5veJB4pdQChadmHuxtB0YzqvfU3Q== dependencies: "@types/events" "*" "@types/node" "*" @@ -3836,9 +3838,10 @@ kleur@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" -koa-body@^4.1.0: - version "4.1.1" - resolved "https://registry.yarnpkg.com/koa-body/-/koa-body-4.1.1.tgz#50686d290891fc6f1acb986cf7cfcd605f855ef0" +koa-body@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/koa-body/-/koa-body-4.2.0.tgz#37229208b820761aca5822d14c5fc55cee31b26f" + integrity sha512-wdGu7b9amk4Fnk/ytH8GuWwfs4fsB5iNkY8kZPpgQVb04QZSv85T0M8reb+cJmvLE8cjPYvBzRikD3s6qz8OoA== dependencies: "@types/formidable" "^1.0.31" co-body "^5.1.1" diff --git a/packages/standard-components/src/DataForm.svelte b/packages/standard-components/src/DataForm.svelte index ba5998e5c6..0b8267c87f 100644 --- a/packages/standard-components/src/DataForm.svelte +++ b/packages/standard-components/src/DataForm.svelte @@ -2,6 +2,7 @@ import { onMount } from "svelte" import { fade } from "svelte/transition" import { Label, DatePicker } from "@budibase/bbui" + import Dropzone from "./attachments/Dropzone.svelte" import debounce from "lodash.debounce" export let _bb @@ -54,8 +55,9 @@ const save = debounce(async () => { for (let field of fields) { // Assign defaults to empty fields to prevent validation issues - if (!(field in record)) + if (!(field in record)) { record[field] = DEFAULTS_FOR_TYPE[schema[field].type] + } } const SAVE_RECORD_URL = `/api/${model}/records` @@ -132,6 +134,8 @@ {:else if schema[field].type === 'string'} + {:else if schema[field].type === 'attachment'} + {/if}
diff --git a/packages/standard-components/src/DataTable.svelte b/packages/standard-components/src/DataTable.svelte index 10f132d017..fe967338f5 100644 --- a/packages/standard-components/src/DataTable.svelte +++ b/packages/standard-components/src/DataTable.svelte @@ -6,6 +6,7 @@ import fsort from "fast-sort" import fetchData from "./fetchData.js" import { isEmpty } from "lodash/fp" + import AttachmentList from "./attachments/AttachmentList.svelte" export let backgroundColor export let color @@ -17,6 +18,7 @@ let headers = [] let sort = {} let sorted = [] + let schema = {} $: cssVariables = { backgroundColor, @@ -83,7 +85,10 @@ {#each sorted as row (row._id)} {#each headers as header} - {#if row[header]} + + {#if Array.isArray(row[header])} + + {:else if row[header]} {row[header]} {/if} {/each} diff --git a/packages/standard-components/src/api.js b/packages/standard-components/src/api.js index da29c70578..1b1535be97 100644 --- a/packages/standard-components/src/api.js +++ b/packages/standard-components/src/api.js @@ -1,7 +1,6 @@ -const apiCall = method => async (url, body) => { - const headers = { +const apiCall = method => async (url, body, headers = { "Content-Type": "application/json", - } +}) => { const response = await fetch(url, { method: method, body: body && JSON.stringify(body), diff --git a/packages/standard-components/src/attachments/AttachmentList.svelte b/packages/standard-components/src/attachments/AttachmentList.svelte new file mode 100644 index 0000000000..9a2813883d --- /dev/null +++ b/packages/standard-components/src/attachments/AttachmentList.svelte @@ -0,0 +1,64 @@ + + +
+ {#each files as file} + +
+ {#if FILE_TYPES.IMAGE.includes(file.extension.toLowerCase())} + + {:else} + + {/if} +
+ {file.name} + + {/each} +
+ + diff --git a/packages/standard-components/src/attachments/Dropzone.svelte b/packages/standard-components/src/attachments/Dropzone.svelte new file mode 100644 index 0000000000..ac83fa7bb6 --- /dev/null +++ b/packages/standard-components/src/attachments/Dropzone.svelte @@ -0,0 +1,299 @@ + + +
+
    + {#if selectedImage} +
  • +
    +
    + + {selectedImage.name} +
    +

    + {#if selectedImage.size <= BYTES_IN_MB} + {selectedImage.size / BYTES_IN_KB}KB + {:else}{selectedImage.size / BYTES_IN_MB}MB{/if} +

    +
    +
    + +
    + {#if selectedImageIdx !== 0} +
    + +
    + {/if} + + {#if selectedImageIdx !== files.length - 1} +
    + +
    + {/if} +
    • + {/if} +
+ + + +
+ + diff --git a/packages/standard-components/src/attachments/fileTypes.js b/packages/standard-components/src/attachments/fileTypes.js new file mode 100644 index 0000000000..2ce6958f2d --- /dev/null +++ b/packages/standard-components/src/attachments/fileTypes.js @@ -0,0 +1,5 @@ +export const FILE_TYPES = { + IMAGE: ["png", "tiff", "gif", "raw", "jpg", "jpeg"], + CODE: ["js", "rs", "py", "java", "rb", "hs", "yml"], + DOCUMENT: ["odf", "docx", "doc", "pdf", "csv"], +}