Merge branch 'develop' into feature/migrations-2.0

This commit is contained in:
Rory Powell 2022-01-26 15:00:52 +00:00
commit e5d04d2830
48 changed files with 2099 additions and 832 deletions

View File

@ -1,5 +1,5 @@
{
"version": "1.0.46-alpha.3",
"version": "1.0.46-alpha.4",
"npmClient": "yarn",
"packages": [
"packages/*"

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/backend-core",
"version": "1.0.46-alpha.3",
"version": "1.0.46-alpha.4",
"description": "Budibase backend core libraries used in server and worker",
"main": "src/index.js",
"author": "Budibase",

View File

@ -8,7 +8,6 @@ exports.Cookies = {
Auth: "budibase:auth",
Init: "budibase:init",
OIDC_CONFIG: "budibase:oidc:config",
RETURN_URL: "budibase:returnurl",
}
exports.Headers = {

View File

@ -93,12 +93,7 @@ exports.getCookie = (ctx, name) => {
* @param {string|object} value The value of cookie which will be set.
* @param {object} opts options like whether to sign.
*/
exports.setCookie = (
ctx,
value,
name = "builder",
opts = { sign: true, requestDomain: false }
) => {
exports.setCookie = (ctx, value, name = "builder", opts = { sign: true }) => {
if (value && opts && opts.sign) {
value = jwt.sign(value, options.secretOrKey)
}
@ -110,7 +105,7 @@ exports.setCookie = (
overwrite: true,
}
if (environment.COOKIE_DOMAIN && !opts.requestDomain) {
if (environment.COOKIE_DOMAIN) {
config.domain = environment.COOKIE_DOMAIN
}

View File

@ -3410,9 +3410,9 @@ node-fetch@2.6.0:
integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
node-fetch@^2.6.1:
version "2.6.6"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.6.tgz#1751a7c01834e8e1697758732e9efb6eeadfaf89"
integrity sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==
version "2.6.7"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
dependencies:
whatwg-url "^5.0.0"

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
"version": "1.0.46-alpha.3",
"version": "1.0.46-alpha.4",
"license": "MPL-2.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",

View File

@ -147,7 +147,9 @@
<img alt="preview" src={selectedUrl} />
{:else}
<div class="placeholder">
<div class="extension">{selectedImage.extension}</div>
<div class="extension">
{selectedImage.name || "Unknown file"}
</div>
<div>Preview not supported</div>
</div>
{/if}
@ -359,18 +361,21 @@
white-space: nowrap;
width: 0;
margin-right: 10px;
user-select: all;
}
.placeholder {
display: flex;
flex-direction: column;
justify-content: center;
align-items: center;
text-align: center;
}
.extension {
color: var(--spectrum-global-color-gray-600);
text-transform: uppercase;
font-weight: 600;
margin-bottom: 5px;
user-select: all;
}
.nav {

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "1.0.46-alpha.3",
"version": "1.0.46-alpha.4",
"license": "GPL-3.0",
"private": true,
"scripts": {
@ -65,10 +65,10 @@
}
},
"dependencies": {
"@budibase/bbui": "^1.0.46-alpha.3",
"@budibase/client": "^1.0.46-alpha.3",
"@budibase/bbui": "^1.0.46-alpha.4",
"@budibase/client": "^1.0.46-alpha.4",
"@budibase/colorpicker": "1.1.2",
"@budibase/string-templates": "^1.0.46-alpha.3",
"@budibase/string-templates": "^1.0.46-alpha.4",
"@sentry/browser": "5.19.1",
"@spectrum-css/page": "^3.0.1",
"@spectrum-css/vars": "^3.0.1",

View File

@ -1,6 +1,7 @@
<script>
import { Select } from "@budibase/bbui"
import { store } from "builderStore"
import { get } from "svelte/store"
const themeOptions = [
{
@ -20,6 +21,17 @@
value: "spectrum--darkest",
},
]
const onChangeTheme = async theme => {
await store.actions.theme.save(theme)
await store.actions.customTheme.save({
...get(store).customTheme,
navBackground:
theme === "spectrum--light"
? "var(--spectrum-global-color-gray-50)"
: "var(--spectrum-global-color-gray-100)",
})
}
</script>
<div>
@ -27,7 +39,7 @@
value={$store.theme}
options={themeOptions}
placeholder={null}
on:change={e => store.actions.theme.save(e.detail)}
on:change={e => onChangeTheme(e.detail)}
/>
</div>

View File

@ -19,7 +19,7 @@
primaryColor: "var(--spectrum-global-color-blue-600)",
primaryColorHover: "var(--spectrum-global-color-blue-500)",
buttonBorderRadius: "16px",
navBackground: "var(--spectrum-global-color-gray-100)",
navBackground: "var(--spectrum-global-color-gray-50)",
navTextColor: "var(--spectrum-global-color-gray-800)",
}
@ -52,7 +52,14 @@
}
const resetTheme = () => {
store.actions.customTheme.save(null)
const theme = get(store).theme
store.actions.customTheme.save({
...defaultTheme,
navBackground:
theme === "spectrum--light"
? "var(--spectrum-global-color-gray-50)"
: "var(--spectrum-global-color-gray-100)",
})
}
</script>

View File

@ -44,7 +44,8 @@
"relationshipfield",
"daterangepicker",
"multifieldselect",
"jsonfield"
"jsonfield",
"s3upload"
]
},
{

View File

@ -0,0 +1,33 @@
<script>
import { Select, Label } from "@budibase/bbui"
import { currentAsset } from "builderStore"
import { findAllMatchingComponents } from "builderStore/componentUtils"
export let parameters
$: components = findAllMatchingComponents($currentAsset.props, component =>
component._component.endsWith("s3upload")
)
</script>
<div class="root">
<Label small>S3 Upload Component</Label>
<Select
bind:value={parameters.componentId}
options={components}
getOptionLabel={x => x._instanceName}
getOptionValue={x => x._id}
/>
</div>
<style>
.root {
display: grid;
column-gap: var(--spacing-l);
row-gap: var(--spacing-s);
grid-template-columns: 120px 1fr;
align-items: center;
max-width: 400px;
margin: 0 auto;
}
</style>

View File

@ -11,3 +11,4 @@ export { default as ChangeFormStep } from "./ChangeFormStep.svelte"
export { default as UpdateState } from "./UpdateState.svelte"
export { default as RefreshDataProvider } from "./RefreshDataProvider.svelte"
export { default as DuplicateRow } from "./DuplicateRow.svelte"
export { default as S3Upload } from "./S3Upload.svelte"

View File

@ -70,6 +70,16 @@
"name": "Update State",
"component": "UpdateState",
"dependsOnFeature": "state"
},
{
"name": "Upload File to S3",
"component": "S3Upload",
"context": [
{
"label": "File URL",
"value": "publicUrl"
}
]
}
]
}

View File

@ -0,0 +1,15 @@
<script>
import { Select } from "@budibase/bbui"
import { datasources } from "stores/backend"
export let value = null
$: dataSources = $datasources.list
.filter(ds => ds.source === "S3" && !ds.config?.endpoint)
.map(ds => ({
label: ds.name,
value: ds._id,
}))
</script>
<Select options={dataSources} {value} on:change />

View File

@ -1,5 +1,6 @@
import { Checkbox, Select, Stepper } from "@budibase/bbui"
import DataSourceSelect from "./DataSourceSelect.svelte"
import S3DataSourceSelect from "./S3DataSourceSelect.svelte"
import DataProviderSelect from "./DataProviderSelect.svelte"
import ButtonActionEditor from "./ButtonActionEditor/ButtonActionEditor.svelte"
import TableSelect from "./TableSelect.svelte"
@ -22,6 +23,7 @@ const componentMap = {
text: DrawerBindableCombobox,
select: Select,
dataSource: DataSourceSelect,
"dataSource/s3": S3DataSourceSelect,
dataProvider: DataProviderSelect,
boolean: Checkbox,
number: Stepper,

View File

@ -7,7 +7,10 @@ export const name = (validation, { apps, currentApp } = { apps: [] }) => {
string()
.trim()
.required("Your application must have a name")
.matches(APP_NAME_REGEX, "App name must be letters and numbers only")
.matches(
APP_NAME_REGEX,
"App name must be letters, numbers and spaces only"
)
.test(
"non-existing-app-name",
"Another app with the same name already exists",
@ -48,7 +51,7 @@ export const url = (validation, { apps, currentApp } = { apps: [] }) => {
}
return !apps
.map(app => app.url)
.some(appUrl => appUrl.toLowerCase() === value.toLowerCase())
.some(appUrl => appUrl?.toLowerCase() === value.toLowerCase())
}
)
.test("valid-url", "Not a valid URL", value => {

View File

@ -1,6 +1,7 @@
import { capitalise } from "helpers"
import { object } from "yup"
import { writable, get } from "svelte/store"
import { notifications } from "@budibase/bbui"
export const createValidationStore = () => {
const DEFAULT = {
@ -24,9 +25,15 @@ export const createValidationStore = () => {
// clear the previous errors
const properties = Object.keys(validator)
properties.forEach(property => (get(validation).errors[property] = null))
let validationError = false
try {
await obj.validate(values, { abortEarly: false })
} catch (error) {
if (!error.inner) {
notifications.error("Unexpected validation error", error)
validationError = true
} else {
error.inner.forEach(err => {
validation.update(store => {
store.errors[err.path] = capitalise(err.message)
@ -34,9 +41,10 @@ export const createValidationStore = () => {
})
})
}
}
let valid
if (properties.length) {
if (properties.length && !validationError) {
valid = await obj.isValid(values)
} else {
// don't say valid until validators have been loaded

View File

@ -41,7 +41,11 @@
)
function getUrl(app) {
if (app.url) {
return `/app${app.url}`
} else {
return `/${app.prodId}`
}
}
</script>

View File

@ -161,7 +161,11 @@
}
const viewApp = app => {
if (app.url) {
window.open(`/app${app.url}`)
} else {
window.open(`/${app.prodId}`)
}
}
const editApp = app => {

File diff suppressed because it is too large Load Diff

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
"version": "1.0.46-alpha.3",
"version": "1.0.46-alpha.4",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "src/index.js",
"bin": {

View File

@ -3340,5 +3340,50 @@
"suffix": "repeater"
}
]
},
"s3upload": {
"name": "S3 File Upload",
"info": "This component can't be used with S3 datasources that use custom endpoints.",
"icon": "UploadToCloud",
"styles": ["size"],
"editable": true,
"settings": [
{
"type": "field/attachment",
"label": "Field",
"key": "field"
},
{
"type": "text",
"label": "Label",
"key": "label"
},
{
"type": "dataSource/s3",
"label": "S3 Datasource",
"key": "datasourceId"
},
{
"type": "text",
"label": "Bucket",
"key": "bucket"
},
{
"type": "text",
"label": "File Name",
"key": "key"
},
{
"type": "boolean",
"label": "Disabled",
"key": "disabled",
"defaultValue": false
},
{
"type": "validation/attachment",
"label": "Validation",
"key": "validation"
}
]
}
}

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "1.0.46-alpha.3",
"version": "1.0.46-alpha.4",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@ -19,10 +19,11 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
"@budibase/bbui": "^1.0.46-alpha.3",
"@budibase/bbui": "^1.0.46-alpha.4",
"@budibase/standard-components": "^0.9.139",
"@budibase/string-templates": "^1.0.46-alpha.3",
"@budibase/string-templates": "^1.0.46-alpha.4",
"regexparam": "^1.3.0",
"rollup-plugin-polyfill-node": "^0.8.0",
"shortid": "^2.2.15",
"svelte-spa-router": "^3.0.5"
},
@ -45,8 +46,6 @@
"postcss": "^8.2.10",
"rollup": "^2.44.0",
"rollup-plugin-json": "^4.0.0",
"rollup-plugin-node-builtins": "^2.1.2",
"rollup-plugin-node-globals": "^1.4.0",
"rollup-plugin-postcss": "^4.0.0",
"rollup-plugin-svelte": "^7.1.0",
"rollup-plugin-svg": "^2.0.0",

View File

@ -6,8 +6,7 @@ import { terser } from "rollup-plugin-terser"
import postcss from "rollup-plugin-postcss"
import svg from "rollup-plugin-svg"
import json from "rollup-plugin-json"
import builtins from "rollup-plugin-node-builtins"
import globals from "rollup-plugin-node-globals"
import nodePolyfills from "rollup-plugin-polyfill-node"
import path from "path"
const production = !process.env.ROLLUP_WATCH
@ -75,8 +74,7 @@ export default {
}),
postcss(),
commonjs(),
globals(),
builtins(),
nodePolyfills(),
resolve({
preferBuiltins: true,
browser: true,

View File

@ -36,7 +36,11 @@ const makeApiCall = async ({ method, url, body, json = true }) => {
})
switch (response.status) {
case 200:
return response.json()
try {
return await response.json()
} catch (error) {
return null
}
case 401:
notificationStore.actions.error("Invalid credentials")
return handleError(`Invalid credentials`)
@ -82,14 +86,15 @@ const makeCachedApiCall = async params => {
* Constructs an API call function for a particular HTTP method.
*/
const requestApiCall = method => async params => {
const { url, cache = false } = params
const fixedUrl = `/${url}`.replace("//", "/")
const { external = false, url, cache = false } = params
const fixedUrl = external ? url : `/${url}`.replace("//", "/")
const enrichedParams = { ...params, method, url: fixedUrl }
return await (cache ? makeCachedApiCall : makeApiCall)(enrichedParams)
}
export default {
post: requestApiCall("POST"),
put: requestApiCall("PUT"),
get: requestApiCall("GET"),
patch: requestApiCall("PATCH"),
del: requestApiCall("DELETE"),

View File

@ -10,3 +10,41 @@ export const uploadAttachment = async (data, tableId = "") => {
json: false,
})
}
/**
* Generates a signed URL to upload a file to an external datasource.
*/
export const getSignedDatasourceURL = async (datasourceId, bucket, key) => {
if (!datasourceId) {
return null
}
const res = await API.post({
url: `/api/attachments/${datasourceId}/url`,
body: { bucket, key },
})
if (res.error) {
throw "Could not generate signed upload URL"
}
return res
}
/**
* Uploads a file to an external datasource.
*/
export const externalUpload = async (datasourceId, bucket, key, data) => {
const { signedUrl, publicUrl } = await getSignedDatasourceURL(
datasourceId,
bucket,
key
)
const res = await API.put({
url: signedUrl,
body: data,
json: false,
external: true,
})
if (res?.error) {
throw "Could not upload file to signed URL"
}
return { publicUrl }
}

View File

@ -0,0 +1,143 @@
<script>
import Field from "./Field.svelte"
import { CoreDropzone, ProgressCircle } from "@budibase/bbui"
import { getContext, onMount, onDestroy } from "svelte"
export let datasourceId
export let bucket
export let key
export let field
export let label
export let disabled = false
export let validation
let fieldState
let fieldApi
const { API, notificationStore, uploadStore } = getContext("sdk")
const component = getContext("component")
// 5GB cap per item sent via S3 REST API
const MaxFileSize = 1000000000 * 5
// Actual file data to upload
let data
let loading = false
const handleFileTooLarge = () => {
notificationStore.actions.warning(
"Files cannot exceed 5GB. Please try again with a smaller file."
)
}
// Process the file input and return a serializable structure expected by
// the dropzone component to display the file
const processFiles = async fileList => {
return await new Promise(resolve => {
if (!fileList?.length) {
return []
}
// Don't read in non-image files
data = fileList[0]
if (!data.type?.startsWith("image")) {
resolve([
{
name: data.name,
type: data.type,
},
])
}
// Read image files and display as preview
const reader = new FileReader()
reader.addEventListener(
"load",
() => {
resolve([
{
url: reader.result,
name: data.name,
type: data.type,
},
])
},
false
)
reader.readAsDataURL(fileList[0])
})
}
const upload = async () => {
loading = true
try {
const res = await API.externalUpload(datasourceId, bucket, key, data)
notificationStore.actions.success("File uploaded successfully")
loading = false
return res
} catch (error) {
notificationStore.actions.error(`Error uploading file: ${error}`)
}
}
onMount(() => {
uploadStore.actions.registerFileUpload($component.id, upload)
})
onDestroy(() => {
uploadStore.actions.unregisterFileUpload($component.id)
})
</script>
<Field
{label}
{field}
{disabled}
{validation}
type="s3upload"
bind:fieldState
bind:fieldApi
defaultValue={[]}
>
<div class="content">
{#if fieldState}
<CoreDropzone
value={fieldState.value}
disabled={loading || fieldState.disabled}
error={fieldState.error}
on:change={e => {
fieldApi.setValue(e.detail)
}}
{processFiles}
{handleFileTooLarge}
maximum={1}
fileSizeLimit={MaxFileSize}
/>
{/if}
{#if loading}
<div class="overlay" />
<div class="loading">
<ProgressCircle />
</div>
{/if}
</div>
</Field>
<style>
.content {
position: relative;
}
.overlay,
.loading {
position: absolute;
top: 0;
height: 100%;
width: 100%;
display: grid;
place-items: center;
}
.overlay {
background-color: var(--spectrum-global-color-gray-50);
opacity: 0.5;
}
</style>

View File

@ -12,3 +12,4 @@ export { default as relationshipfield } from "./RelationshipField.svelte"
export { default as passwordfield } from "./PasswordField.svelte"
export { default as formstep } from "./FormStep.svelte"
export { default as jsonfield } from "./JSONField.svelte"
export { default as s3upload } from "./S3Upload.svelte"

View File

@ -5,6 +5,7 @@ import {
routeStore,
screenStore,
builderStore,
uploadStore,
} from "stores"
import { styleable } from "utils/styleable"
import { linkable } from "utils/linkable"
@ -20,6 +21,7 @@ export default {
routeStore,
screenStore,
builderStore,
uploadStore,
styleable,
linkable,
getAction,

View File

@ -9,6 +9,7 @@ export { confirmationStore } from "./confirmation"
export { peekStore } from "./peek"
export { stateStore } from "./state"
export { themeStore } from "./theme"
export { uploadStore } from "./uploads.js"
// Context stores are layered and duplicated, so it is not a singleton
export { createContextStore } from "./context"

View File

@ -0,0 +1,42 @@
import { writable, get } from "svelte/store"
export const createUploadStore = () => {
const store = writable([])
// Registers a new file upload component
const registerFileUpload = (componentId, callback) => {
if (!componentId || !callback) {
return
}
store.update(state => {
state.push({
componentId,
callback,
})
return state
})
}
// Unregisters a file upload component
const unregisterFileUpload = componentId => {
store.update(state => state.filter(c => c.componentId !== componentId))
}
// Processes a file upload for a given component ID
const processFileUpload = async componentId => {
if (!componentId) {
return
}
const component = get(store).find(c => c.componentId === componentId)
return await component?.callback()
}
return {
subscribe: store.subscribe,
actions: { registerFileUpload, unregisterFileUpload, processFileUpload },
}
}
export const uploadStore = createUploadStore()

View File

@ -5,6 +5,7 @@ import {
confirmationStore,
authStore,
stateStore,
uploadStore,
} from "stores"
import { saveRow, deleteRow, executeQuery, triggerAutomation } from "api"
import { ActionTypes } from "constants"
@ -169,6 +170,17 @@ const updateStateHandler = action => {
}
}
const s3UploadHandler = async action => {
const { componentId } = action.parameters
if (!componentId) {
return
}
const res = await uploadStore.actions.processFileUpload(componentId)
return {
publicUrl: res?.publicUrl,
}
}
const handlerMap = {
["Save Row"]: saveRowHandler,
["Duplicate Row"]: duplicateRowHandler,
@ -183,6 +195,7 @@ const handlerMap = {
["Close Screen Modal"]: closeScreenModalHandler,
["Change Form Step"]: changeFormStepHandler,
["Update State"]: updateStateHandler,
["Upload File to S3"]: s3UploadHandler,
}
const confirmTextMap = {

File diff suppressed because it is too large Load Diff

View File

@ -1,5 +1,5 @@
{
"watch": ["src", "../auth"],
"watch": ["src", "../backend-core"],
"ext": "js,ts,json",
"ignore": ["src/**/*.spec.ts", "src/**/*.spec.js"],
"exec": "ts-node src/index.ts"

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/server",
"email": "hi@budibase.com",
"version": "1.0.46-alpha.3",
"version": "1.0.46-alpha.4",
"description": "Budibase Web Server",
"main": "src/index.ts",
"repository": {
@ -70,9 +70,9 @@
"license": "GPL-3.0",
"dependencies": {
"@apidevtools/swagger-parser": "^10.0.3",
"@budibase/backend-core": "^1.0.46-alpha.3",
"@budibase/client": "^1.0.46-alpha.3",
"@budibase/string-templates": "^1.0.46-alpha.3",
"@budibase/backend-core": "^1.0.46-alpha.4",
"@budibase/client": "^1.0.46-alpha.4",
"@budibase/string-templates": "^1.0.46-alpha.4",
"@bull-board/api": "^3.7.0",
"@bull-board/koa": "^3.7.0",
"@elastic/elasticsearch": "7.10.0",
@ -110,7 +110,7 @@
"mongodb": "3.6.3",
"mssql": "6.2.3",
"mysql2": "^2.3.1",
"node-fetch": "2.6.0",
"node-fetch": "2.6.7",
"open": "^8.4.0",
"pg": "8.5.1",
"pino-pretty": "4.0.0",

View File

@ -17,6 +17,8 @@ const { clientLibraryPath } = require("../../../utilities")
const { upload } = require("../../../utilities/fileSystem")
const { attachmentsRelativeURL } = require("../../../utilities")
const { DocumentTypes } = require("../../../db/utils")
const AWS = require("aws-sdk")
const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"
async function prepareUpload({ s3Key, bucket, metadata, file }) {
const response = await upload({
@ -43,7 +45,9 @@ async function getAppIdFromUrl(ctx) {
// search prod apps for a url that matches, exclude dev where id is always used
const apps = await getAllApps(CouchDB, { dev: false })
const app = apps.filter(a => a.url.toLowerCase() === possibleAppUrl)[0]
const app = apps.filter(
a => a.url && a.url.toLowerCase() === possibleAppUrl
)[0]
if (app && app.appId) {
return app.appId
@ -105,3 +109,51 @@ exports.serveClientLibrary = async function (ctx) {
root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
})
}
exports.getSignedUploadURL = async function (ctx) {
const database = new CouchDB(ctx.appId)
// Ensure datasource is valid
let datasource
try {
const { datasourceId } = ctx.params
datasource = await database.get(datasourceId)
if (!datasource) {
ctx.throw(400, "The specified datasource could not be found")
}
} catch (error) {
ctx.throw(400, "The specified datasource could not be found")
}
// Ensure we aren't using a custom endpoint
if (datasource?.config?.endpoint) {
ctx.throw(400, "S3 datasources with custom endpoints are not supported")
}
// Determine type of datasource and generate signed URL
let signedUrl
let publicUrl
if (datasource.source === "S3") {
const { bucket, key } = ctx.request.body || {}
if (!bucket || !key) {
ctx.throw(400, "bucket and key values are required")
return
}
try {
const s3 = new AWS.S3({
region: AWS_REGION,
accessKeyId: datasource?.config?.accessKeyId,
secretAccessKey: datasource?.config?.secretAccessKey,
apiVersion: "2006-03-01",
signatureVersion: "v4",
})
const params = { Bucket: bucket, Key: key }
signedUrl = s3.getSignedUrl("putObject", params)
publicUrl = `https://${bucket}.s3.${AWS_REGION}.amazonaws.com/${key}`
} catch (error) {
ctx.throw(400, error)
}
}
ctx.body = { signedUrl, publicUrl }
}

View File

@ -46,5 +46,10 @@ router
)
// TODO: this likely needs to be secured in some way
.get("/:appId/:path*", controller.serveApp)
.post(
"/api/attachments/:datasourceId/url",
authorized(PermissionTypes.TABLE, PermissionLevels.READ),
controller.getSignedUploadURL
)
module.exports = router

View File

@ -0,0 +1,98 @@
jest.mock("node-fetch")
jest.mock("aws-sdk", () => ({
config: {
update: jest.fn(),
},
DynamoDB: {
DocumentClient: jest.fn(),
},
S3: jest.fn(() => ({
getSignedUrl: jest.fn(() => {
return "my-url"
}),
})),
}))
const setup = require("./utilities")
describe("/attachments", () => {
let request = setup.getRequest()
let config = setup.getConfig()
let app
afterAll(setup.afterAll)
beforeEach(async () => {
app = await config.init()
})
describe("generateSignedUrls", () => {
let datasource
beforeEach(async () => {
datasource = await config.createDatasource({
datasource: {
type: "datasource",
name: "Test",
source: "S3",
config: {},
},
})
})
it("should be able to generate a signed upload URL", async () => {
const bucket = "foo"
const key = "bar"
const res = await request
.post(`/api/attachments/${datasource._id}/url`)
.send({ bucket, key })
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body.signedUrl).toEqual("my-url")
expect(res.body.publicUrl).toEqual(
`https://${bucket}.s3.eu-west-1.amazonaws.com/${key}`
)
})
it("should handle an invalid datasource ID", async () => {
const res = await request
.post(`/api/attachments/foo/url`)
.send({
bucket: "foo",
key: "bar",
})
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(400)
expect(res.body.message).toEqual(
"The specified datasource could not be found"
)
})
it("should require a bucket parameter", async () => {
const res = await request
.post(`/api/attachments/${datasource._id}/url`)
.send({
bucket: undefined,
key: "bar",
})
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(400)
expect(res.body.message).toEqual("bucket and key values are required")
})
it("should require a key parameter", async () => {
const res = await request
.post(`/api/attachments/${datasource._id}/url`)
.send({
bucket: "foo",
})
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(400)
expect(res.body.message).toEqual("bucket and key values are required")
})
})
})

View File

@ -43,8 +43,8 @@ const coreFields = {
enum: Object.values(BodyTypes),
},
pagination: {
type: DatasourceFieldTypes.OBJECT,
},
type: DatasourceFieldTypes.OBJECT
}
}
module RestModule {
@ -178,17 +178,12 @@ module RestModule {
headers,
},
pagination: {
cursor: nextCursor,
},
cursor: nextCursor
}
}
}
getUrl(
path: string,
queryString: string,
pagination: PaginationConfig | null,
paginationValues: PaginationValues | null
): string {
getUrl(path: string, queryString: string, pagination: PaginationConfig | null, paginationValues: PaginationValues | null): string {
// Add pagination params to query string if required
if (pagination?.location === "query" && paginationValues) {
const { pageParam, sizeParam } = pagination
@ -222,22 +217,14 @@ module RestModule {
return complete
}
addBody(
bodyType: string,
body: string | any,
input: any,
pagination: PaginationConfig | null,
paginationValues: PaginationValues | null
) {
addBody(bodyType: string, body: string | any, input: any, pagination: PaginationConfig | null, paginationValues: PaginationValues | null) {
if (!input.headers) {
input.headers = {}
}
if (bodyType === BodyTypes.NONE) {
return input
}
let error,
object: any = {},
string = ""
let error, object: any = {}, string = ""
try {
if (body) {
string = typeof body !== "string" ? JSON.stringify(body) : body
@ -346,7 +333,7 @@ module RestModule {
requestBody,
authConfigId,
pagination,
paginationValues,
paginationValues
} = query
const authHeaders = this.getAuthHeaders(authConfigId)
@ -365,13 +352,7 @@ module RestModule {
}
let input: any = { method, headers: this.headers }
input = this.addBody(
bodyType,
requestBody,
input,
pagination,
paginationValues
)
input = this.addBody(bodyType, requestBody, input, pagination, paginationValues)
this.startTimeMs = performance.now()
const url = this.getUrl(path, queryString, pagination, paginationValues)

View File

@ -38,7 +38,7 @@ module S3Module {
signatureVersion: {
type: "string",
required: false,
default: "v4",
default: "v4"
},
},
query: {

View File

@ -47,15 +47,6 @@ module.exports = async (ctx, next) => {
(!ctx.user || !ctx.user.builder || !ctx.user.builder.global)
) {
clearCookie(ctx, Cookies.CurrentApp)
// have to set the return url on the server side as client side is not available
setCookie(ctx, ctx.url, Cookies.RETURN_URL, {
// don't sign so the browser can easily read
sign: false,
// use the request domain to match how ui handles the return url cookie.
// it's important we don't use the shared domain here as the builder
// can't delete from it without awareness of the domain.
requestDomain: true,
})
return ctx.redirect("/")
}

View File

@ -27,7 +27,7 @@ describe("syncRows", () => {
await config.createTable()
await config.createRow()
// app 2
await config.createApp()
await config.createApp("second-app")
await config.createTable()
await config.createRow()
await config.createRow()

View File

@ -983,10 +983,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@^1.0.27-alpha.13":
version "1.0.27-alpha.13"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.27-alpha.13.tgz#89f46e081eb7b342f483fd0eccd72c42b2b2fa6c"
integrity sha512-NiasBvZ5wTpvANG9AjuO34DHMTqWQWSpabLcgwBY0tNG4ekh+wvSCPjCcUvN/bBpOzrVMQ8C4hmS4pvv342BhQ==
"@budibase/backend-core@^1.0.46-alpha.3":
version "1.0.46"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.46.tgz#795e80038e11c054bb1aa313c16716a7035f3000"
integrity sha512-vXDjTOMlTaGx1Vm6ste7D7ZXwC+NgLzzu+8Ji7T0Pz2WXj+05vWpPha6L5CkNxRYTwUGoU1BAOvMYrChbGOftQ==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -1056,10 +1056,10 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/bbui@^1.0.35":
version "1.0.35"
resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.35.tgz#a51886886772257d31e2c6346dbec46fe0c9fd85"
integrity sha512-8qeAzTujtO7uvhj+dMiyW4BTkQ7dC4xF1CNIwyuTnDwIeFDlXYgNb09VVRs3+nWcX2e2eC53EUs1RnLUoSlTsw==
"@budibase/bbui@^1.0.46":
version "1.0.46"
resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.46.tgz#7306d4eda7f2c827577a4affa1fd314b38ba1198"
integrity sha512-padm0qq2SBNIslXEQW+HIv32pkIHFzloR93FDzSXh0sO43Q+/d2gbAhjI9ZUSAVncx9JNc46dolL1CwrvHFElg==
dependencies:
"@adobe/spectrum-css-workflow-icons" "^1.2.1"
"@spectrum-css/actionbutton" "^1.0.1"
@ -1106,14 +1106,14 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/client@^1.0.27-alpha.13":
version "1.0.35"
resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.35.tgz#b832e7e7e35032fb35fe5492fbb721db1da15394"
integrity sha512-maL3V29PQb9VjgnPZq44GSDZCuamAGp01bheUeJxEeskjQqZUdf8QC7Frf1mT+ZjgKJf3gU6qtFOxmWRbVzVbw==
"@budibase/client@^1.0.46-alpha.3":
version "1.0.46"
resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.46.tgz#e6ef8945b9d7046b6e6d6761628aa1d85387acca"
integrity sha512-jI3z1G/EsfJNCQCvrqzsR4vR1zLoVefzCXCEASIPg9BPzdiAFSwuUJVLijLFIIKfuDVeveUll94fgu7XNY8U2w==
dependencies:
"@budibase/bbui" "^1.0.35"
"@budibase/bbui" "^1.0.46"
"@budibase/standard-components" "^0.9.139"
"@budibase/string-templates" "^1.0.35"
"@budibase/string-templates" "^1.0.46"
regexparam "^1.3.0"
shortid "^2.2.15"
svelte-spa-router "^3.0.5"
@ -1163,10 +1163,10 @@
svelte-apexcharts "^1.0.2"
svelte-flatpickr "^3.1.0"
"@budibase/string-templates@^1.0.27-alpha.13", "@budibase/string-templates@^1.0.35":
version "1.0.35"
resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.35.tgz#a888f1e9327bb36416336a91a95a43cb34e6a42d"
integrity sha512-8HxSv0ru+cgSmphqtOm1pmBM8rc0TRC/6RQGzQefmFFQFfm/SBLAVLLWRmZxAOYTxt4mittGWeL4y05FqEuocg==
"@budibase/string-templates@^1.0.46", "@budibase/string-templates@^1.0.46-alpha.3":
version "1.0.46"
resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.46.tgz#5beef1687b451e4512a465b4e143c8ab46234006"
integrity sha512-t4ZAUkSz2XatjAN0faex5ovmD3mFz672lV/aBk7tfLFzZiKlWjngqdwpLLQNnsqeGvYo75JP2J06j86SX6O83w==
dependencies:
"@budibase/handlebars-helpers" "^0.11.7"
dayjs "^1.10.4"
@ -9313,15 +9313,15 @@ node-fetch@2.4.1:
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.4.1.tgz#b2e38f1117b8acbedbe0524f041fb3177188255d"
integrity sha512-P9UbpFK87NyqBZzUuDBDz4f6Yiys8xm8j7ACDbi6usvFm6KItklQUKjeoqTrYS/S1k6I8oaOC2YLLDr/gg26Mw==
node-fetch@2.6.0, node-fetch@^2.6.0:
node-fetch@2.6.0:
version "2.6.0"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
node-fetch@^2.6.1:
version "2.6.6"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.6.tgz#1751a7c01834e8e1697758732e9efb6eeadfaf89"
integrity sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==
node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1:
version "2.6.7"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
dependencies:
whatwg-url "^5.0.0"

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/string-templates",
"version": "1.0.46-alpha.3",
"version": "1.0.46-alpha.4",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"module": "dist/bundle.mjs",

View File

@ -1,3 +1,3 @@
{
"watch": ["src", "../auth"]
"watch": ["src", "../backend-core"]
}

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/worker",
"email": "hi@budibase.com",
"version": "1.0.46-alpha.3",
"version": "1.0.46-alpha.4",
"description": "Budibase background service",
"main": "src/index.js",
"repository": {
@ -29,8 +29,8 @@
"author": "Budibase",
"license": "GPL-3.0",
"dependencies": {
"@budibase/backend-core": "^1.0.46-alpha.3",
"@budibase/string-templates": "^1.0.46-alpha.3",
"@budibase/backend-core": "^1.0.46-alpha.4",
"@budibase/string-templates": "^1.0.46-alpha.4",
"@koa/router": "^8.0.0",
"@sentry/node": "^6.0.0",
"@techpass/passport-openidconnect": "^0.3.0",

File diff suppressed because it is too large Load Diff