Merge pull request #633 from Budibase/attachment-components

Attachment components

packages/builder/package.json

@@ -63,7 +63,7 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.34.2",
+    "@budibase/bbui": "^1.34.6",
     "@budibase/client": "^0.1.21",
     "@budibase/colorpicker": "^1.0.1",
     "@fortawesome/fontawesome-free": "^5.14.0",

packages/builder/src/components/common/Dropzone.svelte

@@ -1,299 +1,32 @@
 <script>
   import { notifier } from "builderStore/store/notifications"
-  import { Heading, Body, Button } from "@budibase/bbui"
+  import { Heading, Body, Button, Dropzone } from "@budibase/bbui"
-  import { FILE_TYPES } from "constants/backend"
   import api from "builderStore/api"
 
-  const BYTES_IN_KB = 1000
-  const BYTES_IN_MB = 1000000
-
   export let files = []
-  export let fileSizeLimit = BYTES_IN_MB * 20
 
-  let selectedImageIdx = 0
+  function handleFileTooLarge() {
-  let fileDragged = false
+    notifier.danger(
-
+      `Files cannot exceed ${fileSizeLimit /
-  $: selectedImage = files[selectedImageIdx]
+        BYTES_IN_MB}MB. Please try again with smaller files.`
-
+    )
-  function determineFileIcon(extension) {
-    const ext = extension.toLowerCase()
-
-    if (FILE_TYPES.IMAGE.includes(ext)) return "ri-image-2-line"
-    if (FILE_TYPES.CODE.includes(ext)) return "ri-terminal-box-line"
-
-    return "ri-file-line"
   }
 
   async function processFiles(fileList) {
     const fileArray = Array.from(fileList)
 
-    if (fileArray.some(file => file.size >= fileSizeLimit)) {
+    const filesToProcess = fileArray.map(({ name, path, size, type }) => ({
-      notifier.danger(
-        `Files cannot exceed ${fileSizeLimit /
-          BYTES_IN_MB}MB. Please try again with smaller files.`
-      )
-      return
-    }
-
-    const filesToProcess = fileArray.map(({ name, path, size }) => ({
       name,
       path,
       size,
+      type,
     }))
 
     const response = await api.post(`/api/attachments/process`, {
       files: filesToProcess,
     })
-    const processedFiles = await response.json()
+    return await response.json()
-    files = [...processedFiles, ...files]
-    selectedImageIdx = 0
-  }
-
-  async function removeFile() {
-    files.splice(selectedImageIdx, 1)
-    files = files
-    selectedImageIdx = 0
-  }
-
-  function navigateLeft() {
-    selectedImageIdx -= 1
-  }
-
-  function navigateRight() {
-    selectedImageIdx += 1
-  }
-
-  function handleFile(evt) {
-    processFiles(evt.target.files)
-  }
-
-  function handleDragOver(evt) {
-    evt.preventDefault()
-    fileDragged = true
-  }
-
-  function handleDragLeave(evt) {
-    evt.preventDefault()
-    fileDragged = false
-  }
-
-  function handleDrop(evt) {
-    evt.preventDefault()
-    processFiles(evt.dataTransfer.files)
-    fileDragged = false
   }
 </script>
 
-<div
+<Dropzone bind:files {processFiles} {handleFileTooLarge} />
-  class="dropzone"
-  on:dragover={handleDragOver}
-  on:dragleave={handleDragLeave}
-  on:dragenter={handleDragOver}
-  on:drop={handleDrop}
-  class:fileDragged>
-  <ul>
-    {#if selectedImage}
-      <li>
-        <header>
-          <div>
-            <i
-              class={`file-icon ${determineFileIcon(selectedImage.extension)}`} />
-            <span class="filename">{selectedImage.name}</span>
-          </div>
-          <p>
-            {#if selectedImage.size <= BYTES_IN_MB}
-              {selectedImage.size / BYTES_IN_KB}KB
-            {:else}{selectedImage.size / BYTES_IN_MB}MB{/if}
-          </p>
-        </header>
-        <div class="delete-button" on:click={removeFile}>
-          <i class="ri-close-line" />
-        </div>
-        {#if selectedImageIdx !== 0}
-          <div class="nav left" on:click={navigateLeft}>
-            <i class="ri-arrow-left-line" />
-          </div>
-        {/if}
-        <img src={selectedImage.url} />
-        {#if selectedImageIdx !== files.length - 1}
-          <div class="nav right" on:click={navigateRight}>
-            <i class="ri-arrow-right-line" />
-          </div>
-        {/if}
-      </li>
-    {/if}
-  </ul>
-  <i class="ri-folder-upload-line" />
-  <input id="file-upload" type="file" multiple on:change={handleFile} />
-  <label for="file-upload">Upload</label>
-</div>
-
-<style>
-  .dropzone {
-    padding: var(--spacing-l);
-    border: 2px dashed var(--grey-7);
-    text-align: center;
-    display: flex;
-    align-items: center;
-    flex-direction: column;
-    border-radius: 10px;
-    transition: all 0.3s;
-  }
-
-  .fileDragged {
-    border: 2px dashed var(--grey-7);
-    transform: scale(1.03);
-    background: var(--blue-light);
-  }
-
-  input[type="file"] {
-    display: none;
-  }
-
-  label {
-    font-family: var(--font-sans);
-    cursor: pointer;
-    font-weight: 600;
-    box-sizing: border-box;
-    overflow: hidden;
-    border-radius: var(--border-radius-s);
-    color: var(--white);
-    padding: var(--spacing-s) var(--spacing-l);
-    transition: all 0.2s ease 0s;
-    display: inline-flex;
-    text-rendering: optimizeLegibility;
-    min-width: auto;
-    outline: none;
-    font-feature-settings: "case" 1, "rlig" 1, "calt" 0;
-    -webkit-box-align: center;
-    user-select: none;
-    flex-shrink: 0;
-    align-items: center;
-    justify-content: center;
-    margin-top: 10px;
-    width: 100%;
-    border: solid 1.5px var(--ink);
-    background-color: var(--ink);
-  }
-
-  div.nav {
-    position: absolute;
-    background: black;
-    color: var(--white);
-    display: flex;
-    align-items: center;
-    bottom: var(--spacing-s);
-    border-radius: 10px;
-    transition: 0.2s transform;
-  }
-
-  .nav:hover {
-    cursor: pointer;
-    transform: scale(1.1);
-  }
-
-  .left {
-    left: var(--spacing-s);
-  }
-
-  .right {
-    right: var(--spacing-s);
-  }
-
-  li {
-    position: relative;
-    height: 300px;
-    background: var(--grey-7);
-    display: flex;
-    justify-content: center;
-    border-radius: 10px;
-  }
-
-  img {
-    border-radius: 10px;
-    width: 100%;
-    box-shadow: 0 var(--spacing-s) 12px rgba(0, 0, 0, 0.15);
-    object-fit: contain;
-  }
-
-  i {
-    font-size: 3em;
-  }
-
-  .file-icon {
-    color: var(--white);
-    font-size: 2em;
-    margin-right: var(--spacing-s);
-  }
-
-  ul {
-    padding: 0;
-    display: grid;
-    grid-gap: var(--spacing-s);
-    list-style-type: none;
-    width: 100%;
-  }
-
-  header {
-    display: flex;
-    align-items: center;
-    justify-content: space-between;
-    position: absolute;
-    background: linear-gradient(
-      180deg,
-      rgba(12, 12, 12, 1),
-      rgba(60, 60, 60, 0)
-    );
-    width: 100%;
-    border-top-left-radius: 10px;
-    border-top-right-radius: 10px;
-    height: 60px;
-  }
-
-  header > div {
-    color: var(--white);
-    display: flex;
-    align-items: center;
-    font-size: 15px;
-    margin-left: var(--spacing-m);
-    width: 60%;
-    overflow: hidden;
-    white-space: nowrap;
-    text-overflow: ellipsis;
-  }
-
-  .filename {
-    overflow: hidden;
-    text-overflow: ellipsis;
-  }
-
-  header > p {
-    color: var(--grey-5);
-    margin-right: var(--spacing-m);
-  }
-
-  .delete-button {
-    position: absolute;
-    top: var(--spacing-s);
-    right: var(--spacing-s);
-    padding: var(--spacing-s);
-    border-radius: 10px;
-    opacity: 0;
-    transition: all 0.3s;
-    color: var(--white);
-  }
-
-  .delete-button i {
-    font-size: 2em;
-  }
-
-  .delete-button:hover {
-    opacity: 1;
-    cursor: pointer;
-    background: linear-gradient(
-      to top right,
-      rgba(60, 60, 60, 0),
-      rgba(255, 0, 0, 0.2)
-    );
-  }
-</style>

packages/builder/yarn.lock

@@ -709,10 +709,10 @@
     lodash "^4.17.13"
     to-fast-properties "^2.0.0"
 
-"@budibase/bbui@^1.34.2":
+"@budibase/bbui@^1.34.6":
-  version "1.34.2"
+  version "1.34.6"
-  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.34.2.tgz#e4fcc728dc8d51a918f8ebd5c3f0b0afacfa4047"
+  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.34.6.tgz#d94a9c0af52244ded20dfcd7c93dbd6b184460dc"
-  integrity sha512-6RusGPZnEpHx1gtGcjk/lFLgMgFdDpSIxB8v2MiA+kp+uP1pFlzegbaDh+/JXyqFwK7HO91I0yXXBoPjibi7Aw==
+  integrity sha512-FLYKst1WDjQWpZPOm5w31M5mpdc4FZaHNT5UPyE+LTOtVJquUPycyS1Y/lhGjt/QjwP/Gn8wSvwwsD0gCNJvvg==
   dependencies:
     sirv-cli "^0.4.6"
     svelte-flatpickr "^2.4.0"

packages/server/package.json

@@ -59,7 +59,7 @@
     "joi": "^17.2.1",
     "jsonwebtoken": "^8.5.1",
     "koa": "^2.7.0",
-    "koa-body": "^4.1.0",
+    "koa-body": "^4.2.0",
     "koa-compress": "^4.0.1",
     "koa-pino-logger": "^3.0.0",
     "koa-send": "^5.0.0",

packages/server/src/api/controllers/deploy/aws.js

@@ -64,18 +64,29 @@ function walkDir(dirPath, callback) {
   }
 }
 
-function prepareUploadForS3({ filePath, s3Key, metadata, s3 }) {
+async function prepareUploadForS3({ s3Key, metadata, s3, file }) {
-  const fileExtension = [...filePath.split(".")].pop()
+  const extension = [...file.name.split(".")].pop()
-  const fileBytes = fs.readFileSync(filePath)
+  const fileBytes = fs.readFileSync(file.path)
-  return s3
+
+  const upload = await s3
     .upload({
       Key: s3Key,
       Body: fileBytes,
-      ContentType: CONTENT_TYPE_MAP[fileExtension.toLowerCase()],
+      ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()],
       Metadata: metadata,
     })
     .promise()
+
+  return {
+    size: file.size,
+    name: file.name,
+    extension,
+    url: upload.Location,
+    key: upload.Key,
   }
+}
+
+exports.prepareUploadForS3 = prepareUploadForS3
 
 exports.uploadAppAssets = async function({
   appId,
@@ -107,7 +118,10 @@ exports.uploadAppAssets = async function({
     // Upload HTML, CSS and JS for each page of the web app
     walkDir(`${appAssetsPath}/${page}`, function(filePath) {
       const appAssetUpload = prepareUploadForS3({
-        filePath,
+        file: {
+          path: filePath,
+          name: [...filePath.split("/")].pop(),
+        },
         s3Key: filePath.replace(appAssetsPath, `assets/${appId}`),
         s3,
         metadata: { accountId },
@@ -124,8 +138,8 @@ exports.uploadAppAssets = async function({
       if (file.uploaded) continue
 
       const attachmentUpload = prepareUploadForS3({
-        filePath: file.path,
+        file,
-        s3Key: `assets/${appId}/attachments/${file.name}`,
+        s3Key: `assets/${appId}/attachments/${file.processedFileName}`,
         s3,
         metadata: { accountId },
       })

packages/server/src/api/controllers/static.js

@@ -4,6 +4,8 @@ const jwt = require("jsonwebtoken")
 const fetch = require("node-fetch")
 const fs = require("fs")
 const uuid = require("uuid")
+const AWS = require("aws-sdk")
+const { prepareUploadForS3 } = require("./deploy/aws")
 
 const {
   budibaseAppsDir,
@@ -22,8 +24,12 @@ exports.serveBuilder = async function(ctx) {
   await send(ctx, ctx.file, { root: ctx.devPath || builderPath })
 }
 
-exports.processLocalFileUpload = async function(ctx) {
+exports.uploadFile = async function(ctx) {
-  const { files } = ctx.request.body
+  let files
+  files =
+    ctx.request.files.file.length > 1
+      ? Array.from(ctx.request.files.file)
+      : [ctx.request.files.file]
 
   const attachmentsPath = resolve(
     budibaseAppsDir(),
@@ -31,36 +37,66 @@ exports.processLocalFileUpload = async function(ctx) {
     "attachments"
   )
 
+  if (process.env.CLOUD) {
+    // remote upload
+    const s3 = new AWS.S3({
+      params: {
+        Bucket: "prod-budi-app-assets",
+      },
+    })
+
+    const uploads = files.map(file => {
+      const fileExtension = [...file.name.split(".")].pop()
+      const processedFileName = `${uuid.v4()}.${fileExtension}`
+
+      return prepareUploadForS3({
+        file,
+        s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`,
+        s3,
+      })
+    })
+
+    ctx.body = await Promise.all(uploads)
+    return
+  }
+
+  ctx.body = await processLocalFileUploads({
+    files,
+    outputPath: attachmentsPath,
+    instanceId: ctx.user.instanceId,
+  })
+}
+
+async function processLocalFileUploads({ files, outputPath, instanceId }) {
   // create attachments dir if it doesnt exist
-  !fs.existsSync(attachmentsPath) &&
+  !fs.existsSync(outputPath) && fs.mkdirSync(outputPath, { recursive: true })
-    fs.mkdirSync(attachmentsPath, { recursive: true })
 
   const filesToProcess = files.map(file => {
-    const fileExtension = [...file.path.split(".")].pop()
+    const fileExtension = [...file.name.split(".")].pop()
     // filenames converted to UUIDs so they are unique
-    const fileName = `${uuid.v4()}.${fileExtension}`
+    const processedFileName = `${uuid.v4()}.${fileExtension}`
 
     return {
-      ...file,
+      name: file.name,
-      fileName,
+      path: file.path,
+      size: file.size,
+      type: file.type,
+      processedFileName,
       extension: fileExtension,
-      outputPath: join(attachmentsPath, fileName),
+      outputPath: join(outputPath, processedFileName),
-      url: join("/attachments", fileName),
+      url: join("/attachments", processedFileName),
     }
   })
 
-  const fileProcessOperations = filesToProcess.map(file =>
+  const fileProcessOperations = filesToProcess.map(fileProcessor.process)
-    fileProcessor.process(file)
-  )
 
-  try {
   const processedFiles = await Promise.all(fileProcessOperations)
 
   let pendingFileUploads
   // local document used to track which files need to be uploaded
   // db.get throws an error if the document doesn't exist
   // need to use a promise to default
-    const db = new CouchDB(ctx.user.instanceId)
+  const db = new CouchDB(instanceId)
   await db
     .get("_local/fileuploads")
     .then(data => {
@@ -76,7 +112,24 @@ exports.processLocalFileUpload = async function(ctx) {
   ]
   await db.put(pendingFileUploads)
 
-    ctx.body = processedFiles
+  return processedFiles
+}
+
+exports.performLocalFileProcessing = async function(ctx) {
+  const { files } = ctx.request.body
+
+  const processedFileOutputPath = resolve(
+    budibaseAppsDir(),
+    ctx.user.appId,
+    "attachments"
+  )
+
+  try {
+    ctx.body = await processLocalFileUploads({
+      files,
+      outputPath: processedFileOutputPath,
+      instanceId: ctx.user.instanceId,
+    })
   } catch (err) {
     ctx.throw(500, err)
   }

packages/server/src/api/routes/static.js

@@ -26,8 +26,9 @@ router
   .post(
     "/api/attachments/process",
     authorized(BUILDER),
-    controller.processLocalFileUpload
+    controller.performLocalFileProcessing
   )
+  .post("/api/attachments/upload", controller.uploadFile)
   .get("/componentlibrary", controller.serveComponentLibrary)
   .get("/assets/:file*", controller.serveAppAsset)
   .get("/attachments/:file*", controller.serveAttachment)

packages/server/yarn.lock

@@ -530,10 +530,12 @@
 "@types/events@*":
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7"
+  integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==
 
 "@types/formidable@^1.0.31":
   version "1.0.31"
   resolved "https://registry.yarnpkg.com/@types/formidable/-/formidable-1.0.31.tgz#274f9dc2d0a1a9ce1feef48c24ca0859e7ec947b"
+  integrity sha512-dIhM5t8lRP0oWe2HF8MuPvdd1TpPTjhDMAqemcq6oIZQCBQTovhBAdTQ5L5veJB4pdQChadmHuxtB0YzqvfU3Q==
   dependencies:
     "@types/events" "*"
     "@types/node" "*"
@@ -3946,9 +3948,10 @@ kleur@^3.0.3:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e"
 
-koa-body@^4.1.0:
+koa-body@^4.2.0:
-  version "4.1.1"
+  version "4.2.0"
-  resolved "https://registry.yarnpkg.com/koa-body/-/koa-body-4.1.1.tgz#50686d290891fc6f1acb986cf7cfcd605f855ef0"
+  resolved "https://registry.yarnpkg.com/koa-body/-/koa-body-4.2.0.tgz#37229208b820761aca5822d14c5fc55cee31b26f"
+  integrity sha512-wdGu7b9amk4Fnk/ytH8GuWwfs4fsB5iNkY8kZPpgQVb04QZSv85T0M8reb+cJmvLE8cjPYvBzRikD3s6qz8OoA==
   dependencies:
     "@types/formidable" "^1.0.31"
     co-body "^5.1.1"

packages/standard-components/package.json

@@ -36,6 +36,7 @@
   "gitHead": "284cceb9b703c38566c6e6363c022f79a08d5691",
   "dependencies": {
     "@beyonk/svelte-googlemaps": "^2.2.0",
+    "@budibase/bbui": "^1.34.6",
     "@fortawesome/fontawesome-free": "^5.14.0",
     "@budibase/bbui": "^1.34.2",
     "britecharts": "^2.16.1",

packages/standard-components/src/DataForm.svelte

@@ -2,6 +2,7 @@
   import { onMount } from "svelte"
   import { fade } from "svelte/transition"
   import { Label, DatePicker } from "@budibase/bbui"
+  import Dropzone from "./attachments/Dropzone.svelte"
   import debounce from "lodash.debounce"
 
   export let _bb
@@ -54,9 +55,10 @@
   const save = debounce(async () => {
     for (let field of fields) {
       // Assign defaults to empty fields to prevent validation issues
-      if (!(field in record))
+      if (!(field in record)) {
         record[field] = DEFAULTS_FOR_TYPE[schema[field].type]
       }
+    }
 
     const SAVE_RECORD_URL = `/api/${model}/records`
     const response = await _bb.api.post(SAVE_RECORD_URL, record)
@@ -132,6 +134,8 @@
           <input class="input" type="number" bind:value={record[field]} />
         {:else if schema[field].type === 'string'}
           <input class="input" type="text" bind:value={record[field]} />
+        {:else if schema[field].type === 'attachment'}
+          <Dropzone bind:files={record[field]} />
         {/if}
       </div>
       <hr />

packages/standard-components/src/DataTable.svelte

@@ -6,6 +6,7 @@
   import fsort from "fast-sort"
   import fetchData from "./fetchData.js"
   import { isEmpty } from "lodash/fp"
+  import AttachmentList from "./attachments/AttachmentList.svelte"
 
   export let backgroundColor
   export let color
@@ -17,6 +18,7 @@
   let headers = []
   let sort = {}
   let sorted = []
+  let schema = {}
 
   $: cssVariables = {
     backgroundColor,
@@ -83,7 +85,10 @@
     {#each sorted as row (row._id)}
       <tr>
         {#each headers as header}
-          {#if row[header]}
+          <!-- Rudimentary solution for attachments on array given this entire table will be replaced by AG Grid -->
+          {#if Array.isArray(row[header])}
+            <AttachmentList files={row[header]} />
+          {:else if row[header]}
             <td>{row[header]}</td>
           {/if}
         {/each}

packages/standard-components/src/api.js

@@ -1,7 +1,10 @@
-const apiCall = method => async (url, body) => {
+const apiCall = method => async (
-  const headers = {
+  url,
+  body,
+  headers = {
     "Content-Type": "application/json",
   }
+) => {
   const response = await fetch(url, {
     method: method,
     body: body && JSON.stringify(body),

packages/standard-components/src/attachments/AttachmentList.svelte

@@ -0,0 +1,64 @@
+<script>
+  import { FILE_TYPES } from "./fileTypes"
+
+  export let files
+  export let height = "70"
+  export let width = "70"
+</script>
+
+<div class="file-list">
+  {#each files as file}
+    <a href={file.url} target="_blank">
+      <div class="file">
+        {#if FILE_TYPES.IMAGE.includes(file.extension.toLowerCase())}
+          <img {width} {height} src={file.url} />
+        {:else}
+          <i class="far fa-file" />
+        {/if}
+      </div>
+      <span>{file.name}</span>
+    </a>
+  {/each}
+</div>
+
+<style>
+  .file-list {
+    display: grid;
+    grid-auto-flow: column;
+    grid-gap: var(--spacing-m);
+    grid-template-columns: repeat(10, 1fr);
+  }
+
+  img {
+    object-fit: contain;
+  }
+
+  i {
+    margin-bottom: var(--spacing-m);
+  }
+
+  a {
+    color: var(--ink);
+    text-decoration: none;
+  }
+
+  .file {
+    position: relative;
+    height: 75px;
+    width: 75px;
+    border: 2px dashed var(--grey-7);
+    padding: var(--spacing-xs);
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    justify-content: center;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
+
+  span {
+    width: 75px;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
+</style>

packages/standard-components/src/attachments/Dropzone.svelte

@@ -0,0 +1,35 @@
+<script>
+  import { Heading, Body, Button, Dropzone } from "@budibase/bbui"
+  import { FILE_TYPES } from "./fileTypes"
+
+  const BYTES_IN_KB = 1000
+
+  export let files = []
+
+  function handleFileTooLarge(fileSizeLimit) {
+    alert(
+      `Files cannot exceed ${fileSizeLimit /
+        BYTES_IN_MB}MB. Please try again with smaller files.`
+    )
+  }
+
+  async function processFiles(fileList) {
+    let data = new FormData()
+    for (var i = 0; i < fileList.length; i++) {
+      data.append("file", fileList[i])
+    }
+
+    const response = await fetch("/api/attachments/upload", {
+      method: "POST",
+      body: data,
+      headers: {
+        Accept: "application/json",
+      },
+    })
+
+    const processedFiles = await response.json()
+    return processedFiles
+  }
+</script>
+
+<Dropzone bind:files {processFiles} {handleFileTooLarge} />

packages/standard-components/src/attachments/fileTypes.js

@@ -0,0 +1,5 @@
+export const FILE_TYPES = {
+  IMAGE: ["png", "tiff", "gif", "raw", "jpg", "jpeg"],
+  CODE: ["js", "rs", "py", "java", "rb", "hs", "yml"],
+  DOCUMENT: ["odf", "docx", "doc", "pdf", "csv"],
+}