Merge branch 'develop' into feature/migrations-2.0

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.46-alpha.3",
+  "version": "1.0.46-alpha.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.46-alpha.3",
+  "version": "1.0.46-alpha.4",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",

packages/backend-core/src/constants.js

@@ -8,7 +8,6 @@ exports.Cookies = {
   Auth: "budibase:auth",
   Init: "budibase:init",
   OIDC_CONFIG: "budibase:oidc:config",
-  RETURN_URL: "budibase:returnurl",
 }
 
 exports.Headers = {

packages/backend-core/src/utils.js

@@ -93,12 +93,7 @@ exports.getCookie = (ctx, name) => {
  * @param {string|object} value The value of cookie which will be set.
  * @param {object} opts options like whether to sign.
  */
-exports.setCookie = (
+exports.setCookie = (ctx, value, name = "builder", opts = { sign: true }) => {
-  ctx,
-  value,
-  name = "builder",
-  opts = { sign: true, requestDomain: false }
-) => {
   if (value && opts && opts.sign) {
     value = jwt.sign(value, options.secretOrKey)
   }
@@ -110,7 +105,7 @@ exports.setCookie = (
     overwrite: true,
   }
 
-  if (environment.COOKIE_DOMAIN && !opts.requestDomain) {
+  if (environment.COOKIE_DOMAIN) {
     config.domain = environment.COOKIE_DOMAIN
   }
 

packages/backend-core/yarn.lock

@@ -3410,9 +3410,9 @@ node-fetch@2.6.0:
   integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
 
 node-fetch@^2.6.1:
-  version "2.6.6"
+  version "2.6.7"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.6.tgz#1751a7c01834e8e1697758732e9efb6eeadfaf89"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
-  integrity sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==
+  integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
   dependencies:
     whatwg-url "^5.0.0"
 

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.46-alpha.3",
+  "version": "1.0.46-alpha.4",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -147,7 +147,9 @@
           <img alt="preview" src={selectedUrl} />
         {:else}
           <div class="placeholder">
-            <div class="extension">{selectedImage.extension}</div>
+            <div class="extension">
+              {selectedImage.name || "Unknown file"}
+            </div>
             <div>Preview not supported</div>
           </div>
         {/if}
@@ -359,18 +361,21 @@
     white-space: nowrap;
     width: 0;
     margin-right: 10px;
+    user-select: all;
   }
   .placeholder {
     display: flex;
     flex-direction: column;
     justify-content: center;
     align-items: center;
+    text-align: center;
   }
   .extension {
     color: var(--spectrum-global-color-gray-600);
     text-transform: uppercase;
     font-weight: 600;
     margin-bottom: 5px;
+    user-select: all;
   }
 
   .nav {

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.46-alpha.3",
+  "version": "1.0.46-alpha.4",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.46-alpha.3",
+    "@budibase/bbui": "^1.0.46-alpha.4",
-    "@budibase/client": "^1.0.46-alpha.3",
+    "@budibase/client": "^1.0.46-alpha.4",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^1.0.46-alpha.3",
+    "@budibase/string-templates": "^1.0.46-alpha.4",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/components/design/AppPreview/AppThemeSelect.svelte

@@ -1,6 +1,7 @@
 <script>
   import { Select } from "@budibase/bbui"
   import { store } from "builderStore"
+  import { get } from "svelte/store"
 
   const themeOptions = [
     {
@@ -20,6 +21,17 @@
       value: "spectrum--darkest",
     },
   ]
+
+  const onChangeTheme = async theme => {
+    await store.actions.theme.save(theme)
+    await store.actions.customTheme.save({
+      ...get(store).customTheme,
+      navBackground:
+        theme === "spectrum--light"
+          ? "var(--spectrum-global-color-gray-50)"
+          : "var(--spectrum-global-color-gray-100)",
+    })
+  }
 </script>
 
 <div>
@@ -27,7 +39,7 @@
     value={$store.theme}
     options={themeOptions}
     placeholder={null}
-    on:change={e => store.actions.theme.save(e.detail)}
+    on:change={e => onChangeTheme(e.detail)}
   />
 </div>
 

packages/builder/src/components/design/AppPreview/ThemeEditor.svelte

@@ -19,7 +19,7 @@
     primaryColor: "var(--spectrum-global-color-blue-600)",
     primaryColorHover: "var(--spectrum-global-color-blue-500)",
     buttonBorderRadius: "16px",
-    navBackground: "var(--spectrum-global-color-gray-100)",
+    navBackground: "var(--spectrum-global-color-gray-50)",
     navTextColor: "var(--spectrum-global-color-gray-800)",
   }
 
@@ -52,7 +52,14 @@
   }
 
   const resetTheme = () => {
-    store.actions.customTheme.save(null)
+    const theme = get(store).theme
+    store.actions.customTheme.save({
+      ...defaultTheme,
+      navBackground:
+        theme === "spectrum--light"
+          ? "var(--spectrum-global-color-gray-50)"
+          : "var(--spectrum-global-color-gray-100)",
+    })
   }
 </script>
 

packages/builder/src/components/design/AppPreview/componentStructure.json

@@ -44,7 +44,8 @@
       "relationshipfield",
       "daterangepicker",
       "multifieldselect",
-      "jsonfield"
+      "jsonfield",
+      "s3upload"
     ]
   },
   {

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/S3Upload.svelte

@@ -0,0 +1,33 @@
+<script>
+  import { Select, Label } from "@budibase/bbui"
+  import { currentAsset } from "builderStore"
+  import { findAllMatchingComponents } from "builderStore/componentUtils"
+
+  export let parameters
+
+  $: components = findAllMatchingComponents($currentAsset.props, component =>
+    component._component.endsWith("s3upload")
+  )
+</script>
+
+<div class="root">
+  <Label small>S3 Upload Component</Label>
+  <Select
+    bind:value={parameters.componentId}
+    options={components}
+    getOptionLabel={x => x._instanceName}
+    getOptionValue={x => x._id}
+  />
+</div>
+
+<style>
+  .root {
+    display: grid;
+    column-gap: var(--spacing-l);
+    row-gap: var(--spacing-s);
+    grid-template-columns: 120px 1fr;
+    align-items: center;
+    max-width: 400px;
+    margin: 0 auto;
+  }
+</style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/index.js

@@ -11,3 +11,4 @@ export { default as ChangeFormStep } from "./ChangeFormStep.svelte"
 export { default as UpdateState } from "./UpdateState.svelte"
 export { default as RefreshDataProvider } from "./RefreshDataProvider.svelte"
 export { default as DuplicateRow } from "./DuplicateRow.svelte"
+export { default as S3Upload } from "./S3Upload.svelte"

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/manifest.json

@@ -70,6 +70,16 @@
       "name": "Update State",
       "component": "UpdateState",
       "dependsOnFeature": "state"
+    },
+    {
+      "name": "Upload File to S3",
+      "component": "S3Upload",
+      "context": [
+        {
+          "label": "File URL",
+          "value": "publicUrl"
+        }
+      ]
     }
   ]
 }

packages/builder/src/components/design/PropertiesPanel/PropertyControls/S3DataSourceSelect.svelte

@@ -0,0 +1,15 @@
+<script>
+  import { Select } from "@budibase/bbui"
+  import { datasources } from "stores/backend"
+
+  export let value = null
+
+  $: dataSources = $datasources.list
+    .filter(ds => ds.source === "S3" && !ds.config?.endpoint)
+    .map(ds => ({
+      label: ds.name,
+      value: ds._id,
+    }))
+</script>
+
+<Select options={dataSources} {value} on:change />

packages/builder/src/components/design/PropertiesPanel/PropertyControls/componentSettings.js

@@ -1,5 +1,6 @@
 import { Checkbox, Select, Stepper } from "@budibase/bbui"
 import DataSourceSelect from "./DataSourceSelect.svelte"
+import S3DataSourceSelect from "./S3DataSourceSelect.svelte"
 import DataProviderSelect from "./DataProviderSelect.svelte"
 import ButtonActionEditor from "./ButtonActionEditor/ButtonActionEditor.svelte"
 import TableSelect from "./TableSelect.svelte"
@@ -22,6 +23,7 @@ const componentMap = {
   text: DrawerBindableCombobox,
   select: Select,
   dataSource: DataSourceSelect,
+  "dataSource/s3": S3DataSourceSelect,
   dataProvider: DataProviderSelect,
   boolean: Checkbox,
   number: Stepper,

packages/builder/src/helpers/validation/yup/app.js

@@ -7,7 +7,10 @@ export const name = (validation, { apps, currentApp } = { apps: [] }) => {
     string()
       .trim()
       .required("Your application must have a name")
-      .matches(APP_NAME_REGEX, "App name must be letters and numbers only")
+      .matches(
+        APP_NAME_REGEX,
+        "App name must be letters, numbers and spaces only"
+      )
       .test(
         "non-existing-app-name",
         "Another app with the same name already exists",
@@ -48,7 +51,7 @@ export const url = (validation, { apps, currentApp } = { apps: [] }) => {
           }
           return !apps
             .map(app => app.url)
-            .some(appUrl => appUrl.toLowerCase() === value.toLowerCase())
+            .some(appUrl => appUrl?.toLowerCase() === value.toLowerCase())
         }
       )
       .test("valid-url", "Not a valid URL", value => {

packages/builder/src/helpers/validation/yup/index.js

@@ -1,6 +1,7 @@
 import { capitalise } from "helpers"
 import { object } from "yup"
 import { writable, get } from "svelte/store"
+import { notifications } from "@budibase/bbui"
 
 export const createValidationStore = () => {
   const DEFAULT = {
@@ -24,19 +25,26 @@ export const createValidationStore = () => {
     // clear the previous errors
     const properties = Object.keys(validator)
     properties.forEach(property => (get(validation).errors[property] = null))
+
+    let validationError = false
     try {
       await obj.validate(values, { abortEarly: false })
     } catch (error) {
-      error.inner.forEach(err => {
+      if (!error.inner) {
-        validation.update(store => {
+        notifications.error("Unexpected validation error", error)
-          store.errors[err.path] = capitalise(err.message)
+        validationError = true
-          return store
+      } else {
+        error.inner.forEach(err => {
+          validation.update(store => {
+            store.errors[err.path] = capitalise(err.message)
+            return store
+          })
         })
-      })
+      }
     }
 
     let valid
-    if (properties.length) {
+    if (properties.length && !validationError) {
       valid = await obj.isValid(values)
     } else {
       // don't say valid until validators have been loaded

packages/builder/src/pages/builder/apps/index.svelte

@@ -41,7 +41,11 @@
       )
 
   function getUrl(app) {
-    return `/app${app.url}`
+    if (app.url) {
+      return `/app${app.url}`
+    } else {
+      return `/${app.prodId}`
+    }
   }
 </script>
 

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -161,7 +161,11 @@
   }
 
   const viewApp = app => {
-    window.open(`/app${app.url}`)
+    if (app.url) {
+      window.open(`/app${app.url}`)
+    } else {
+      window.open(`/${app.prodId}`)
+    }
   }
 
   const editApp = app => {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.46-alpha.3",
+  "version": "1.0.46-alpha.4",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/manifest.json

@@ -3340,5 +3340,50 @@
         "suffix": "repeater"
       }
     ]
+  },
+  "s3upload": {
+    "name": "S3 File Upload",
+    "info": "This component can't be used with S3 datasources that use custom endpoints.",
+    "icon": "UploadToCloud",
+    "styles": ["size"],
+    "editable": true,
+    "settings": [
+      {
+        "type": "field/attachment",
+        "label": "Field",
+        "key": "field"
+      },
+      {
+        "type": "text",
+        "label": "Label",
+        "key": "label"
+      },
+      {
+        "type": "dataSource/s3",
+        "label": "S3 Datasource",
+        "key": "datasourceId"
+      },
+      {
+        "type": "text",
+        "label": "Bucket",
+        "key": "bucket"
+      },
+      {
+        "type": "text",
+        "label": "File Name",
+        "key": "key"
+      },
+      {
+        "type": "boolean",
+        "label": "Disabled",
+        "key": "disabled",
+        "defaultValue": false
+      },
+      {
+        "type": "validation/attachment",
+        "label": "Validation",
+        "key": "validation"
+      }
+    ]
   }
 }

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.46-alpha.3",
+  "version": "1.0.46-alpha.4",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,10 +19,11 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.46-alpha.3",
+    "@budibase/bbui": "^1.0.46-alpha.4",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^1.0.46-alpha.3",
+    "@budibase/string-templates": "^1.0.46-alpha.4",
     "regexparam": "^1.3.0",
+    "rollup-plugin-polyfill-node": "^0.8.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"
   },
@@ -45,8 +46,6 @@
     "postcss": "^8.2.10",
     "rollup": "^2.44.0",
     "rollup-plugin-json": "^4.0.0",
-    "rollup-plugin-node-builtins": "^2.1.2",
-    "rollup-plugin-node-globals": "^1.4.0",
     "rollup-plugin-postcss": "^4.0.0",
     "rollup-plugin-svelte": "^7.1.0",
     "rollup-plugin-svg": "^2.0.0",

packages/client/rollup.config.js

@@ -6,8 +6,7 @@ import { terser } from "rollup-plugin-terser"
 import postcss from "rollup-plugin-postcss"
 import svg from "rollup-plugin-svg"
 import json from "rollup-plugin-json"
-import builtins from "rollup-plugin-node-builtins"
+import nodePolyfills from "rollup-plugin-polyfill-node"
-import globals from "rollup-plugin-node-globals"
 import path from "path"
 
 const production = !process.env.ROLLUP_WATCH
@@ -75,8 +74,7 @@ export default {
     }),
     postcss(),
     commonjs(),
-    globals(),
+    nodePolyfills(),
-    builtins(),
     resolve({
       preferBuiltins: true,
       browser: true,

packages/client/src/api/api.js

@@ -36,7 +36,11 @@ const makeApiCall = async ({ method, url, body, json = true }) => {
     })
     switch (response.status) {
       case 200:
-        return response.json()
+        try {
+          return await response.json()
+        } catch (error) {
+          return null
+        }
       case 401:
         notificationStore.actions.error("Invalid credentials")
         return handleError(`Invalid credentials`)
@@ -82,14 +86,15 @@ const makeCachedApiCall = async params => {
  * Constructs an API call function for a particular HTTP method.
  */
 const requestApiCall = method => async params => {
-  const { url, cache = false } = params
+  const { external = false, url, cache = false } = params
-  const fixedUrl = `/${url}`.replace("//", "/")
+  const fixedUrl = external ? url : `/${url}`.replace("//", "/")
   const enrichedParams = { ...params, method, url: fixedUrl }
   return await (cache ? makeCachedApiCall : makeApiCall)(enrichedParams)
 }
 
 export default {
   post: requestApiCall("POST"),
+  put: requestApiCall("PUT"),
   get: requestApiCall("GET"),
   patch: requestApiCall("PATCH"),
   del: requestApiCall("DELETE"),

packages/client/src/api/attachments.js

@@ -10,3 +10,41 @@ export const uploadAttachment = async (data, tableId = "") => {
     json: false,
   })
 }
+
+/**
+ * Generates a signed URL to upload a file to an external datasource.
+ */
+export const getSignedDatasourceURL = async (datasourceId, bucket, key) => {
+  if (!datasourceId) {
+    return null
+  }
+  const res = await API.post({
+    url: `/api/attachments/${datasourceId}/url`,
+    body: { bucket, key },
+  })
+  if (res.error) {
+    throw "Could not generate signed upload URL"
+  }
+  return res
+}
+
+/**
+ * Uploads a file to an external datasource.
+ */
+export const externalUpload = async (datasourceId, bucket, key, data) => {
+  const { signedUrl, publicUrl } = await getSignedDatasourceURL(
+    datasourceId,
+    bucket,
+    key
+  )
+  const res = await API.put({
+    url: signedUrl,
+    body: data,
+    json: false,
+    external: true,
+  })
+  if (res?.error) {
+    throw "Could not upload file to signed URL"
+  }
+  return { publicUrl }
+}

packages/client/src/components/app/forms/S3Upload.svelte

@@ -0,0 +1,143 @@
+<script>
+  import Field from "./Field.svelte"
+  import { CoreDropzone, ProgressCircle } from "@budibase/bbui"
+  import { getContext, onMount, onDestroy } from "svelte"
+
+  export let datasourceId
+  export let bucket
+  export let key
+  export let field
+  export let label
+  export let disabled = false
+  export let validation
+
+  let fieldState
+  let fieldApi
+
+  const { API, notificationStore, uploadStore } = getContext("sdk")
+  const component = getContext("component")
+
+  // 5GB cap per item sent via S3 REST API
+  const MaxFileSize = 1000000000 * 5
+
+  // Actual file data to upload
+  let data
+  let loading = false
+
+  const handleFileTooLarge = () => {
+    notificationStore.actions.warning(
+      "Files cannot exceed 5GB. Please try again with a smaller file."
+    )
+  }
+
+  // Process the file input and return a serializable structure expected by
+  // the dropzone component to display the file
+  const processFiles = async fileList => {
+    return await new Promise(resolve => {
+      if (!fileList?.length) {
+        return []
+      }
+
+      // Don't read in non-image files
+      data = fileList[0]
+      if (!data.type?.startsWith("image")) {
+        resolve([
+          {
+            name: data.name,
+            type: data.type,
+          },
+        ])
+      }
+
+      // Read image files and display as preview
+      const reader = new FileReader()
+      reader.addEventListener(
+        "load",
+        () => {
+          resolve([
+            {
+              url: reader.result,
+              name: data.name,
+              type: data.type,
+            },
+          ])
+        },
+        false
+      )
+      reader.readAsDataURL(fileList[0])
+    })
+  }
+
+  const upload = async () => {
+    loading = true
+    try {
+      const res = await API.externalUpload(datasourceId, bucket, key, data)
+      notificationStore.actions.success("File uploaded successfully")
+      loading = false
+      return res
+    } catch (error) {
+      notificationStore.actions.error(`Error uploading file: ${error}`)
+    }
+  }
+
+  onMount(() => {
+    uploadStore.actions.registerFileUpload($component.id, upload)
+  })
+
+  onDestroy(() => {
+    uploadStore.actions.unregisterFileUpload($component.id)
+  })
+</script>
+
+<Field
+  {label}
+  {field}
+  {disabled}
+  {validation}
+  type="s3upload"
+  bind:fieldState
+  bind:fieldApi
+  defaultValue={[]}
+>
+  <div class="content">
+    {#if fieldState}
+      <CoreDropzone
+        value={fieldState.value}
+        disabled={loading || fieldState.disabled}
+        error={fieldState.error}
+        on:change={e => {
+          fieldApi.setValue(e.detail)
+        }}
+        {processFiles}
+        {handleFileTooLarge}
+        maximum={1}
+        fileSizeLimit={MaxFileSize}
+      />
+    {/if}
+    {#if loading}
+      <div class="overlay" />
+      <div class="loading">
+        <ProgressCircle />
+      </div>
+    {/if}
+  </div>
+</Field>
+
+<style>
+  .content {
+    position: relative;
+  }
+  .overlay,
+  .loading {
+    position: absolute;
+    top: 0;
+    height: 100%;
+    width: 100%;
+    display: grid;
+    place-items: center;
+  }
+  .overlay {
+    background-color: var(--spectrum-global-color-gray-50);
+    opacity: 0.5;
+  }
+</style>

packages/client/src/components/app/forms/index.js

@@ -12,3 +12,4 @@ export { default as relationshipfield } from "./RelationshipField.svelte"
 export { default as passwordfield } from "./PasswordField.svelte"
 export { default as formstep } from "./FormStep.svelte"
 export { default as jsonfield } from "./JSONField.svelte"
+export { default as s3upload } from "./S3Upload.svelte"

packages/client/src/sdk.js

@@ -5,6 +5,7 @@ import {
   routeStore,
   screenStore,
   builderStore,
+  uploadStore,
 } from "stores"
 import { styleable } from "utils/styleable"
 import { linkable } from "utils/linkable"
@@ -20,6 +21,7 @@ export default {
   routeStore,
   screenStore,
   builderStore,
+  uploadStore,
   styleable,
   linkable,
   getAction,

packages/client/src/stores/index.js

@@ -9,6 +9,7 @@ export { confirmationStore } from "./confirmation"
 export { peekStore } from "./peek"
 export { stateStore } from "./state"
 export { themeStore } from "./theme"
+export { uploadStore } from "./uploads.js"
 
 // Context stores are layered and duplicated, so it is not a singleton
 export { createContextStore } from "./context"

packages/client/src/stores/uploads.js

@@ -0,0 +1,42 @@
+import { writable, get } from "svelte/store"
+
+export const createUploadStore = () => {
+  const store = writable([])
+
+  // Registers a new file upload component
+  const registerFileUpload = (componentId, callback) => {
+    if (!componentId || !callback) {
+      return
+    }
+
+    store.update(state => {
+      state.push({
+        componentId,
+        callback,
+      })
+      return state
+    })
+  }
+
+  // Unregisters a file upload component
+  const unregisterFileUpload = componentId => {
+    store.update(state => state.filter(c => c.componentId !== componentId))
+  }
+
+  // Processes a file upload for a given component ID
+  const processFileUpload = async componentId => {
+    if (!componentId) {
+      return
+    }
+
+    const component = get(store).find(c => c.componentId === componentId)
+    return await component?.callback()
+  }
+
+  return {
+    subscribe: store.subscribe,
+    actions: { registerFileUpload, unregisterFileUpload, processFileUpload },
+  }
+}
+
+export const uploadStore = createUploadStore()

packages/client/src/utils/buttonActions.js

@@ -5,6 +5,7 @@ import {
   confirmationStore,
   authStore,
   stateStore,
+  uploadStore,
 } from "stores"
 import { saveRow, deleteRow, executeQuery, triggerAutomation } from "api"
 import { ActionTypes } from "constants"
@@ -169,6 +170,17 @@ const updateStateHandler = action => {
   }
 }
 
+const s3UploadHandler = async action => {
+  const { componentId } = action.parameters
+  if (!componentId) {
+    return
+  }
+  const res = await uploadStore.actions.processFileUpload(componentId)
+  return {
+    publicUrl: res?.publicUrl,
+  }
+}
+
 const handlerMap = {
   ["Save Row"]: saveRowHandler,
   ["Duplicate Row"]: duplicateRowHandler,
@@ -183,6 +195,7 @@ const handlerMap = {
   ["Close Screen Modal"]: closeScreenModalHandler,
   ["Change Form Step"]: changeFormStepHandler,
   ["Update State"]: updateStateHandler,
+  ["Upload File to S3"]: s3UploadHandler,
 }
 
 const confirmTextMap = {

packages/server/nodemon.json

@@ -1,5 +1,5 @@
 {
-  "watch": ["src", "../auth"],
+  "watch": ["src", "../backend-core"],
   "ext": "js,ts,json",
   "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js"],
   "exec": "ts-node src/index.ts"

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.46-alpha.3",
+  "version": "1.0.46-alpha.4",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.46-alpha.3",
+    "@budibase/backend-core": "^1.0.46-alpha.4",
-    "@budibase/client": "^1.0.46-alpha.3",
+    "@budibase/client": "^1.0.46-alpha.4",
-    "@budibase/string-templates": "^1.0.46-alpha.3",
+    "@budibase/string-templates": "^1.0.46-alpha.4",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -110,7 +110,7 @@
     "mongodb": "3.6.3",
     "mssql": "6.2.3",
     "mysql2": "^2.3.1",
-    "node-fetch": "2.6.0",
+    "node-fetch": "2.6.7",
     "open": "^8.4.0",
     "pg": "8.5.1",
     "pino-pretty": "4.0.0",

packages/server/src/api/controllers/static/index.js

@@ -17,6 +17,8 @@ const { clientLibraryPath } = require("../../../utilities")
 const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentTypes } = require("../../../db/utils")
+const AWS = require("aws-sdk")
+const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"
 
 async function prepareUpload({ s3Key, bucket, metadata, file }) {
   const response = await upload({
@@ -43,7 +45,9 @@ async function getAppIdFromUrl(ctx) {
 
   // search prod apps for a url that matches, exclude dev where id is always used
   const apps = await getAllApps(CouchDB, { dev: false })
-  const app = apps.filter(a => a.url.toLowerCase() === possibleAppUrl)[0]
+  const app = apps.filter(
+    a => a.url && a.url.toLowerCase() === possibleAppUrl
+  )[0]
 
   if (app && app.appId) {
     return app.appId
@@ -105,3 +109,51 @@ exports.serveClientLibrary = async function (ctx) {
     root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
   })
 }
+
+exports.getSignedUploadURL = async function (ctx) {
+  const database = new CouchDB(ctx.appId)
+
+  // Ensure datasource is valid
+  let datasource
+  try {
+    const { datasourceId } = ctx.params
+    datasource = await database.get(datasourceId)
+    if (!datasource) {
+      ctx.throw(400, "The specified datasource could not be found")
+    }
+  } catch (error) {
+    ctx.throw(400, "The specified datasource could not be found")
+  }
+
+  // Ensure we aren't using a custom endpoint
+  if (datasource?.config?.endpoint) {
+    ctx.throw(400, "S3 datasources with custom endpoints are not supported")
+  }
+
+  // Determine type of datasource and generate signed URL
+  let signedUrl
+  let publicUrl
+  if (datasource.source === "S3") {
+    const { bucket, key } = ctx.request.body || {}
+    if (!bucket || !key) {
+      ctx.throw(400, "bucket and key values are required")
+      return
+    }
+    try {
+      const s3 = new AWS.S3({
+        region: AWS_REGION,
+        accessKeyId: datasource?.config?.accessKeyId,
+        secretAccessKey: datasource?.config?.secretAccessKey,
+        apiVersion: "2006-03-01",
+        signatureVersion: "v4",
+      })
+      const params = { Bucket: bucket, Key: key }
+      signedUrl = s3.getSignedUrl("putObject", params)
+      publicUrl = `https://${bucket}.s3.${AWS_REGION}.amazonaws.com/${key}`
+    } catch (error) {
+      ctx.throw(400, error)
+    }
+  }
+
+  ctx.body = { signedUrl, publicUrl }
+}

packages/server/src/api/routes/static.js

@@ -46,5 +46,10 @@ router
   )
   // TODO: this likely needs to be secured in some way
   .get("/:appId/:path*", controller.serveApp)
+  .post(
+    "/api/attachments/:datasourceId/url",
+    authorized(PermissionTypes.TABLE, PermissionLevels.READ),
+    controller.getSignedUploadURL
+  )
 
 module.exports = router

packages/server/src/api/routes/tests/static.spec.js

@@ -0,0 +1,98 @@
+jest.mock("node-fetch")
+jest.mock("aws-sdk", () => ({
+  config: {
+    update: jest.fn(),
+  },
+  DynamoDB: {
+    DocumentClient: jest.fn(),
+  },
+  S3: jest.fn(() => ({
+    getSignedUrl: jest.fn(() => {
+      return "my-url"
+    }),
+  })),
+}))
+
+const setup = require("./utilities")
+
+describe("/attachments", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+  let app
+
+  afterAll(setup.afterAll)
+
+  beforeEach(async () => {
+    app = await config.init()
+  })
+
+  describe("generateSignedUrls", () => {
+    let datasource
+
+    beforeEach(async () => {
+      datasource = await config.createDatasource({
+        datasource: {
+          type: "datasource",
+          name: "Test",
+          source: "S3",
+          config: {},
+        },
+      })
+    })
+
+    it("should be able to generate a signed upload URL", async () => {
+      const bucket = "foo"
+      const key = "bar"
+      const res = await request
+        .post(`/api/attachments/${datasource._id}/url`)
+        .send({ bucket, key })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      expect(res.body.signedUrl).toEqual("my-url")
+      expect(res.body.publicUrl).toEqual(
+        `https://${bucket}.s3.eu-west-1.amazonaws.com/${key}`
+      )
+    })
+
+    it("should handle an invalid datasource ID", async () => {
+      const res = await request
+        .post(`/api/attachments/foo/url`)
+        .send({
+          bucket: "foo",
+          key: "bar",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+      expect(res.body.message).toEqual(
+        "The specified datasource could not be found"
+      )
+    })
+
+    it("should require a bucket parameter", async () => {
+      const res = await request
+        .post(`/api/attachments/${datasource._id}/url`)
+        .send({
+          bucket: undefined,
+          key: "bar",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+      expect(res.body.message).toEqual("bucket and key values are required")
+    })
+
+    it("should require a key parameter", async () => {
+      const res = await request
+        .post(`/api/attachments/${datasource._id}/url`)
+        .send({
+          bucket: "foo",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+      expect(res.body.message).toEqual("bucket and key values are required")
+    })
+  })
+})

packages/server/src/integrations/rest.ts

@@ -43,8 +43,8 @@ const coreFields = {
     enum: Object.values(BodyTypes),
   },
   pagination: {
-    type: DatasourceFieldTypes.OBJECT,
+    type: DatasourceFieldTypes.OBJECT
-  },
+  }
 }
 
 module RestModule {
@@ -178,17 +178,12 @@ module RestModule {
           headers,
         },
         pagination: {
-          cursor: nextCursor,
+          cursor: nextCursor
-        },
+        }
       }
     }
 
-    getUrl(
+    getUrl(path: string, queryString: string, pagination: PaginationConfig | null, paginationValues: PaginationValues | null): string {
-      path: string,
-      queryString: string,
-      pagination: PaginationConfig | null,
-      paginationValues: PaginationValues | null
-    ): string {
       // Add pagination params to query string if required
       if (pagination?.location === "query" && paginationValues) {
         const { pageParam, sizeParam } = pagination
@@ -222,22 +217,14 @@ module RestModule {
       return complete
     }
 
-    addBody(
+    addBody(bodyType: string, body: string | any, input: any, pagination: PaginationConfig | null, paginationValues: PaginationValues | null) {
-      bodyType: string,
-      body: string | any,
-      input: any,
-      pagination: PaginationConfig | null,
-      paginationValues: PaginationValues | null
-    ) {
       if (!input.headers) {
         input.headers = {}
       }
       if (bodyType === BodyTypes.NONE) {
         return input
       }
-      let error,
+      let error, object: any = {}, string = ""
-        object: any = {},
-        string = ""
       try {
         if (body) {
           string = typeof body !== "string" ? JSON.stringify(body) : body
@@ -346,7 +333,7 @@ module RestModule {
         requestBody,
         authConfigId,
         pagination,
-        paginationValues,
+        paginationValues
       } = query
       const authHeaders = this.getAuthHeaders(authConfigId)
 
@@ -365,13 +352,7 @@ module RestModule {
       }
 
       let input: any = { method, headers: this.headers }
-      input = this.addBody(
+      input = this.addBody(bodyType, requestBody, input, pagination, paginationValues)
-        bodyType,
-        requestBody,
-        input,
-        pagination,
-        paginationValues
-      )
 
       this.startTimeMs = performance.now()
       const url = this.getUrl(path, queryString, pagination, paginationValues)

packages/server/src/integrations/s3.ts

@@ -38,7 +38,7 @@ module S3Module {
       signatureVersion: {
         type: "string",
         required: false,
-        default: "v4",
+        default: "v4"
       },
     },
     query: {

packages/server/src/middleware/currentapp.js

@@ -47,15 +47,6 @@ module.exports = async (ctx, next) => {
     (!ctx.user || !ctx.user.builder || !ctx.user.builder.global)
   ) {
     clearCookie(ctx, Cookies.CurrentApp)
-    // have to set the return url on the server side as client side is not available
-    setCookie(ctx, ctx.url, Cookies.RETURN_URL, {
-      // don't sign so the browser can easily read
-      sign: false,
-      // use the request domain to match how ui handles the return url cookie.
-      // it's important we don't use the shared domain here as the builder
-      // can't delete from it without awareness of the domain.
-      requestDomain: true,
-    })
     return ctx.redirect("/")
   }
 

packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js

@@ -27,7 +27,7 @@ describe("syncRows", () => {
     await config.createTable()
     await config.createRow()
     // app 2
-    await config.createApp()
+    await config.createApp("second-app")
     await config.createTable()
     await config.createRow()
     await config.createRow()

packages/server/yarn.lock

@@ -983,10 +983,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@^1.0.27-alpha.13":
+"@budibase/backend-core@^1.0.46-alpha.3":
-  version "1.0.27-alpha.13"
+  version "1.0.46"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.27-alpha.13.tgz#89f46e081eb7b342f483fd0eccd72c42b2b2fa6c"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.46.tgz#795e80038e11c054bb1aa313c16716a7035f3000"
-  integrity sha512-NiasBvZ5wTpvANG9AjuO34DHMTqWQWSpabLcgwBY0tNG4ekh+wvSCPjCcUvN/bBpOzrVMQ8C4hmS4pvv342BhQ==
+  integrity sha512-vXDjTOMlTaGx1Vm6ste7D7ZXwC+NgLzzu+8Ji7T0Pz2WXj+05vWpPha6L5CkNxRYTwUGoU1BAOvMYrChbGOftQ==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
@@ -1056,10 +1056,10 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/bbui@^1.0.35":
+"@budibase/bbui@^1.0.46":
-  version "1.0.35"
+  version "1.0.46"
-  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.35.tgz#a51886886772257d31e2c6346dbec46fe0c9fd85"
+  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.46.tgz#7306d4eda7f2c827577a4affa1fd314b38ba1198"
-  integrity sha512-8qeAzTujtO7uvhj+dMiyW4BTkQ7dC4xF1CNIwyuTnDwIeFDlXYgNb09VVRs3+nWcX2e2eC53EUs1RnLUoSlTsw==
+  integrity sha512-padm0qq2SBNIslXEQW+HIv32pkIHFzloR93FDzSXh0sO43Q+/d2gbAhjI9ZUSAVncx9JNc46dolL1CwrvHFElg==
   dependencies:
     "@adobe/spectrum-css-workflow-icons" "^1.2.1"
     "@spectrum-css/actionbutton" "^1.0.1"
@@ -1106,14 +1106,14 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/client@^1.0.27-alpha.13":
+"@budibase/client@^1.0.46-alpha.3":
-  version "1.0.35"
+  version "1.0.46"
-  resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.35.tgz#b832e7e7e35032fb35fe5492fbb721db1da15394"
+  resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.46.tgz#e6ef8945b9d7046b6e6d6761628aa1d85387acca"
-  integrity sha512-maL3V29PQb9VjgnPZq44GSDZCuamAGp01bheUeJxEeskjQqZUdf8QC7Frf1mT+ZjgKJf3gU6qtFOxmWRbVzVbw==
+  integrity sha512-jI3z1G/EsfJNCQCvrqzsR4vR1zLoVefzCXCEASIPg9BPzdiAFSwuUJVLijLFIIKfuDVeveUll94fgu7XNY8U2w==
   dependencies:
-    "@budibase/bbui" "^1.0.35"
+    "@budibase/bbui" "^1.0.46"
     "@budibase/standard-components" "^0.9.139"
-    "@budibase/string-templates" "^1.0.35"
+    "@budibase/string-templates" "^1.0.46"
     regexparam "^1.3.0"
     shortid "^2.2.15"
     svelte-spa-router "^3.0.5"
@@ -1163,10 +1163,10 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
-"@budibase/string-templates@^1.0.27-alpha.13", "@budibase/string-templates@^1.0.35":
+"@budibase/string-templates@^1.0.46", "@budibase/string-templates@^1.0.46-alpha.3":
-  version "1.0.35"
+  version "1.0.46"
-  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.35.tgz#a888f1e9327bb36416336a91a95a43cb34e6a42d"
+  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.46.tgz#5beef1687b451e4512a465b4e143c8ab46234006"
-  integrity sha512-8HxSv0ru+cgSmphqtOm1pmBM8rc0TRC/6RQGzQefmFFQFfm/SBLAVLLWRmZxAOYTxt4mittGWeL4y05FqEuocg==
+  integrity sha512-t4ZAUkSz2XatjAN0faex5ovmD3mFz672lV/aBk7tfLFzZiKlWjngqdwpLLQNnsqeGvYo75JP2J06j86SX6O83w==
   dependencies:
     "@budibase/handlebars-helpers" "^0.11.7"
     dayjs "^1.10.4"
@@ -9313,15 +9313,15 @@ node-fetch@2.4.1:
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.4.1.tgz#b2e38f1117b8acbedbe0524f041fb3177188255d"
   integrity sha512-P9UbpFK87NyqBZzUuDBDz4f6Yiys8xm8j7ACDbi6usvFm6KItklQUKjeoqTrYS/S1k6I8oaOC2YLLDr/gg26Mw==
 
-node-fetch@2.6.0, node-fetch@^2.6.0:
+node-fetch@2.6.0:
   version "2.6.0"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
   integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
 
-node-fetch@^2.6.1:
+node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1:
-  version "2.6.6"
+  version "2.6.7"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.6.tgz#1751a7c01834e8e1697758732e9efb6eeadfaf89"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
-  integrity sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==
+  integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
   dependencies:
     whatwg-url "^5.0.0"
 

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "1.0.46-alpha.3",
+  "version": "1.0.46-alpha.4",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/worker/nodemon.json

@@ -1,3 +1,3 @@
 {
-  "watch": ["src", "../auth"]
+  "watch": ["src", "../backend-core"]
 }

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.46-alpha.3",
+  "version": "1.0.46-alpha.4",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -29,8 +29,8 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^1.0.46-alpha.3",
+    "@budibase/backend-core": "^1.0.46-alpha.4",
-    "@budibase/string-templates": "^1.0.46-alpha.3",
+    "@budibase/string-templates": "^1.0.46-alpha.4",
     "@koa/router": "^8.0.0",
     "@sentry/node": "^6.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",