Merge branch 'master' of github.com:Budibase/budibase into feature/sqs-table-cleanup

This commit is contained in:
mike12345567 2024-05-16 12:47:45 +01:00
commit 94b85eeed0
30 changed files with 622 additions and 800 deletions

3
.github/AUTHORS.md vendored
View File

@ -8,4 +8,5 @@ Contributors
* Andrew Kingston - [@aptkingston](https://github.com/aptkingston)
* Michael Drury - [@mike12345567](https://github.com/mike12345567)
* Peter Clement - [@PClmnt](https://github.com/PClmnt)
* Rory Powell - [@Rory-Powell](https://github.com/Rory-Powell)
* Rory Powell - [@Rory-Powell](https://github.com/Rory-Powell)
* Michaël St-Georges [@CSLTech](https://github.com/CSLTech)

View File

@ -1,5 +1,5 @@
{
"version": "2.26.3",
"version": "2.26.4",
"npmClient": "yarn",
"packages": [
"packages/*",

View File

@ -155,6 +155,8 @@ export default function positionDropdown(element, opts) {
applyXStrategy(Strategies.StartToEnd)
} else if (align === "left-outside") {
applyXStrategy(Strategies.EndToStart)
} else if (align === "center") {
applyXStrategy(Strategies.MidPoint)
} else {
applyXStrategy(Strategies.StartToStart)
}

View File

@ -14,9 +14,8 @@
import LinkedRowSelector from "components/common/LinkedRowSelector.svelte"
import Editor from "../../integration/QueryEditor.svelte"
export let defaultValue
export let meta
export let value = defaultValue || (meta.type === "boolean" ? false : "")
export let value
export let readonly
export let error

View File

@ -1,5 +1,6 @@
<script>
import { datasources, tables, integrations, appStore } from "stores/builder"
import { admin } from "stores/portal"
import EditRolesButton from "./buttons/EditRolesButton.svelte"
import { TableNames } from "constants"
import { Grid } from "@budibase/frontend-core"
@ -63,6 +64,7 @@
schemaOverrides={isUsersTable ? userSchemaOverrides : null}
showAvatars={false}
on:updatedatasource={handleGridTableUpdate}
isCloud={$admin.cloud}
>
<svelte:fragment slot="filter">
{#if isUsersTable && $appStore.features.disableUserMetadata}

View File

@ -1,5 +1,6 @@
<script>
import { viewsV2 } from "stores/builder"
import { admin } from "stores/portal"
import { Grid } from "@budibase/frontend-core"
import { API } from "api"
import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
@ -26,6 +27,7 @@
allowDeleteRows
showAvatars={false}
on:updatedatasource={handleGridViewUpdate}
isCloud={$admin.cloud}
>
<svelte:fragment slot="filter">
<GridFilterButton />

View File

@ -0,0 +1,50 @@
import { it, expect, describe, vi } from "vitest"
import Dropzone from "./Dropzone.svelte"
import { render, fireEvent } from "@testing-library/svelte"
import { notifications } from "@budibase/bbui"
import { admin } from "stores/portal"
vi.spyOn(notifications, "error").mockImplementation(() => {})
describe("Dropzone", () => {
let instance = null
afterEach(() => {
vi.restoreAllMocks()
})
it("that the Dropzone is rendered", () => {
instance = render(Dropzone, {})
expect(instance).toBeDefined()
})
it("Ensure the correct error message is shown when uploading the file in cloud", async () => {
admin.subscribe = vi.fn().mockImplementation(callback => {
callback({ cloud: true })
return () => {}
})
instance = render(Dropzone, { props: { fileSizeLimit: 1000000 } }) // 1MB
const fileInput = instance.getByLabelText("Select a file to upload")
const file = new File(["hello".repeat(2000000)], "hello.png", {
type: "image/png",
})
await fireEvent.change(fileInput, { target: { files: [file] } })
expect(notifications.error).toHaveBeenCalledWith(
"Files cannot exceed 1MB. Please try again with smaller files."
)
})
it("Ensure the file size error message is not shown when running on self host", async () => {
admin.subscribe = vi.fn().mockImplementation(callback => {
callback({ cloud: false })
return () => {}
})
instance = render(Dropzone, { props: { fileSizeLimit: 1000000 } }) // 1MB
const fileInput = instance.getByLabelText("Select a file to upload")
const file = new File(["hello".repeat(2000000)], "hello.png", {
type: "image/png",
})
await fireEvent.change(fileInput, { target: { files: [file] } })
expect(notifications.error).not.toHaveBeenCalled()
})
})

View File

@ -1,9 +1,11 @@
<script>
import { Dropzone, notifications } from "@budibase/bbui"
import { admin } from "stores/portal"
import { API } from "api"
export let value = []
export let label
export let fileSizeLimit = undefined
const BYTES_IN_MB = 1000000
@ -34,5 +36,6 @@
{label}
{...$$restProps}
{processFiles}
{handleFileTooLarge}
handleFileTooLarge={$admin.cloud ? handleFileTooLarge : null}
{fileSizeLimit}
/>

View File

@ -7,6 +7,7 @@
export let app
export let color
export let autoSave = false
export let disabled = false
let modal
</script>
@ -14,12 +15,16 @@
<!-- svelte-ignore a11y-click-events-have-key-events -->
<!-- svelte-ignore a11y-no-static-element-interactions -->
<div class="editable-icon">
<div class="hover" on:click={modal.show}>
<Icon name="Edit" {size} color="var(--spectrum-global-color-gray-600)" />
</div>
<div class="normal">
{#if !disabled}
<div class="hover" on:click={modal.show}>
<Icon name="Edit" {size} color="var(--spectrum-global-color-gray-600)" />
</div>
<div class="normal">
<Icon name={name || "Apps"} {size} {color} />
</div>
{:else}
<Icon {name} {size} {color} />
</div>
{/if}
</div>
<Modal bind:this={modal}>

View File

@ -0,0 +1,214 @@
<script>
import { Button, Label, Icon, Input, notifications } from "@budibase/bbui"
import { AppStatus } from "constants"
import { appStore, initialise } from "stores/builder"
import { appsStore } from "stores/portal"
import { API } from "api"
import { writable } from "svelte/store"
import { createValidationStore } from "helpers/validation/yup"
import * as appValidation from "helpers/validation/yup/app"
import EditableIcon from "components/common/EditableIcon.svelte"
import { isEqual } from "lodash"
import { createEventDispatcher } from "svelte"
export let alignActions = "left"
const values = writable({})
const validation = createValidationStore()
const dispatch = createEventDispatcher()
let updating = false
let edited = false
let initialised = false
$: filteredApps = $appsStore.apps.filter(app => app.devId == $appStore.appId)
$: app = filteredApps.length ? filteredApps[0] : {}
$: appDeployed = app?.status === AppStatus.DEPLOYED
$: appName = $appStore.name
$: appURL = $appStore.url
$: appIconName = $appStore.icon?.name
$: appIconColor = $appStore.icon?.color
$: appMeta = {
name: appName,
url: appURL,
iconName: appIconName,
iconColor: appIconColor,
}
const initForm = appMeta => {
edited = false
values.set({
...appMeta,
})
if (!initialised) {
setupValidation()
initialised = true
}
}
const validate = (vals, appMeta) => {
const { url } = vals || {}
validation.check({
...vals,
url: url?.[0] === "/" ? url.substring(1, url.length) : url,
})
edited = !isEqual(vals, appMeta)
}
// On app/apps update, reset the state.
$: initForm(appMeta)
$: validate($values, appMeta)
const resolveAppUrl = (template, name) => {
let parsedName
const resolvedName = resolveAppName(null, name)
parsedName = resolvedName ? resolvedName.toLowerCase() : ""
const parsedUrl = parsedName ? parsedName.replace(/\s+/g, "-") : ""
return encodeURI(parsedUrl)
}
const nameToUrl = appName => {
let resolvedUrl = resolveAppUrl(null, appName)
tidyUrl(resolvedUrl)
}
const resolveAppName = (template, name) => {
if (template && !name) {
return template.name
}
return name ? name.trim() : null
}
const tidyUrl = url => {
if (url && !url.startsWith("/")) {
url = `/${url}`
}
$values.url = url === "" ? null : url
}
const updateIcon = e => {
const { name, color } = e.detail
$values.iconColor = color
$values.iconName = name
}
const setupValidation = async () => {
appValidation.name(validation, {
apps: $appsStore.apps,
currentApp: app,
})
appValidation.url(validation, {
apps: $appsStore.apps,
currentApp: app,
})
}
async function updateApp() {
try {
await appsStore.save($appStore.appId, {
name: $values.name?.trim(),
url: $values.url?.trim(),
icon: {
name: $values.iconName,
color: $values.iconColor,
},
})
await initialiseApp()
notifications.success("App update successful")
} catch (error) {
console.error(error)
notifications.error("Error updating app")
}
}
const initialiseApp = async () => {
const applicationPkg = await API.fetchAppPackage($appStore.appId)
await initialise(applicationPkg)
}
</script>
<div class="form">
<div class="fields">
<div class="field">
<Label size="L">Name</Label>
<Input
bind:value={$values.name}
error={$validation.touched.name && $validation.errors.name}
on:blur={() => ($validation.touched.name = true)}
on:change={nameToUrl($values.name)}
disabled={appDeployed}
/>
</div>
<div class="field">
<Label size="L">URL</Label>
<Input
bind:value={$values.url}
error={$validation.touched.url && $validation.errors.url}
on:blur={() => ($validation.touched.url = true)}
on:change={tidyUrl($values.url)}
placeholder={$values.url
? $values.url
: `/${resolveAppUrl(null, $values.name)}`}
disabled={appDeployed}
/>
</div>
<div class="field">
<Label size="L">Icon</Label>
<EditableIcon
{app}
size="XL"
name={$values.iconName}
color={$values.iconColor}
on:change={updateIcon}
disabled={appDeployed}
/>
</div>
<div class="actions" class:right={alignActions === "right"}>
{#if !appDeployed}
<Button
cta
on:click={async () => {
updating = true
await updateApp()
updating = false
dispatch("updated")
}}
disabled={appDeployed || updating || !edited || !$validation.valid}
>
Save
</Button>
{:else}
<div class="edit-info">
<Icon size="S" name="Info" /> Unpublish your app to edit name and URL
</div>
{/if}
</div>
</div>
</div>
<style>
.actions {
display: flex;
}
.actions.right {
justify-content: end;
}
.fields {
display: grid;
grid-gap: var(--spacing-l);
}
.field {
display: grid;
grid-template-columns: 80px 220px;
grid-gap: var(--spacing-l);
align-items: center;
}
.edit-info {
display: flex;
gap: var(--spacing-s);
}
</style>

View File

@ -0,0 +1,68 @@
<script>
import { Popover, Layout, Icon } from "@budibase/bbui"
import UpdateAppForm from "./UpdateAppForm.svelte"
let formPopover
let formPopoverAnchor
let formPopoverOpen = false
</script>
<div bind:this={formPopoverAnchor}>
<!-- svelte-ignore a11y-click-events-have-key-events -->
<!-- svelte-ignore a11y-no-static-element-interactions -->
<div
class="app-heading"
class:editing={formPopoverOpen}
on:click={() => {
formPopover.show()
}}
>
<slot />
<span class="edit-icon">
<Icon size="S" name="Edit" color={"var(--grey-7)"} />
</span>
</div>
</div>
<Popover
customZindex={998}
bind:this={formPopover}
align="center"
anchor={formPopoverAnchor}
offset={20}
on:close={() => {
formPopoverOpen = false
}}
on:open={() => {
formPopoverOpen = true
}}
>
<Layout noPadding gap="M">
<div class="popover-content">
<UpdateAppForm
on:updated={() => {
formPopover.hide()
}}
/>
</div>
</Layout>
</Popover>
<style>
.popover-content {
padding: var(--spacing-xl);
}
.app-heading {
display: flex;
cursor: pointer;
align-items: center;
gap: var(--spacing-s);
}
.edit-icon {
display: none;
}
.app-heading:hover .edit-icon,
.app-heading.editing .edit-icon {
display: inline;
}
</style>

View File

@ -8,13 +8,11 @@
ActionButton,
Icon,
Link,
Modal,
StatusLight,
AbsTooltip,
} from "@budibase/bbui"
import RevertModal from "components/deploy/RevertModal.svelte"
import VersionModal from "components/deploy/VersionModal.svelte"
import UpdateAppModal from "components/start/UpdateAppModal.svelte"
import { processStringSync } from "@budibase/string-templates"
import ConfirmDialog from "components/common/ConfirmDialog.svelte"
import analytics, { Events, EventSource } from "analytics"
@ -26,7 +24,6 @@
isOnlyUser,
appStore,
deploymentStore,
initialise,
sortedScreens,
} from "stores/builder"
import TourWrap from "components/portal/onboarding/TourWrap.svelte"
@ -37,7 +34,6 @@
export let loaded
let unpublishModal
let updateAppModal
let revertModal
let versionModal
let appActionPopover
@ -61,11 +57,6 @@
$: canPublish = !publishing && loaded && $sortedScreens.length > 0
$: lastDeployed = getLastDeployedString($deploymentStore, lastOpened)
const initialiseApp = async () => {
const applicationPkg = await API.fetchAppPackage($appStore.devId)
await initialise(applicationPkg)
}
const getLastDeployedString = deployments => {
return deployments?.length
? processStringSync("Published {{ duration time 'millisecond' }} ago", {
@ -247,16 +238,12 @@
appActionPopover.hide()
if (isPublished) {
viewApp()
} else {
updateAppModal.show()
}
}}
>
{$appStore.url}
{#if isPublished}
<Icon size="S" name="LinkOut" />
{:else}
<Icon size="S" name="Edit" />
{/if}
</span>
</Body>
@ -330,20 +317,6 @@
Are you sure you want to unpublish the app <b>{selectedApp?.name}</b>?
</ConfirmDialog>
<Modal bind:this={updateAppModal} padding={false} width="600px">
<UpdateAppModal
app={{
name: $appStore.name,
url: $appStore.url,
icon: $appStore.icon,
appId: $appStore.appId,
}}
onUpdateComplete={async () => {
await initialiseApp()
}}
/>
</Modal>
<RevertModal bind:this={revertModal} />
<VersionModal hideIcon bind:this={versionModal} />

View File

@ -5,7 +5,7 @@
export let value = null
$: dataSources = $datasources.list
.filter(ds => ds.source === "S3" && !ds.config?.endpoint)
.filter(ds => ds.source === "S3")
.map(ds => ({
label: ds.name,
value: ds._id,

View File

@ -1,151 +0,0 @@
<script>
import { writable, get as svelteGet } from "svelte/store"
import {
notifications,
Input,
ModalContent,
Layout,
Label,
} from "@budibase/bbui"
import { appsStore } from "stores/portal"
import { onMount } from "svelte"
import { createValidationStore } from "helpers/validation/yup"
import * as appValidation from "helpers/validation/yup/app"
import EditableIcon from "../common/EditableIcon.svelte"
export let app
export let onUpdateComplete
$: appIdParts = app.appId ? app.appId?.split("_") : []
$: appId = appIdParts.slice(-1)[0]
const values = writable({
name: app.name,
url: app.url,
iconName: app.icon?.name,
iconColor: app.icon?.color,
})
const validation = createValidationStore()
$: {
const { url } = $values
validation.check({
...$values,
url: url?.[0] === "/" ? url.substring(1, url.length) : url,
})
}
const setupValidation = async () => {
const applications = svelteGet(appsStore).apps
appValidation.name(validation, {
apps: applications,
currentApp: {
...app,
appId,
},
})
appValidation.url(validation, {
apps: applications,
currentApp: {
...app,
appId,
},
})
// init validation
const { url } = $values
validation.check({
...$values,
url: url?.[0] === "/" ? url.substring(1, url.length) : url,
})
}
async function updateApp() {
try {
await appsStore.save(app.appId, {
name: $values.name?.trim(),
url: $values.url?.trim(),
icon: {
name: $values.iconName,
color: $values.iconColor,
},
})
if (typeof onUpdateComplete == "function") {
onUpdateComplete()
}
} catch (error) {
console.error(error)
notifications.error("Error updating app")
}
}
const resolveAppUrl = (template, name) => {
let parsedName
const resolvedName = resolveAppName(null, name)
parsedName = resolvedName ? resolvedName.toLowerCase() : ""
const parsedUrl = parsedName ? parsedName.replace(/\s+/g, "-") : ""
return encodeURI(parsedUrl)
}
const resolveAppName = (template, name) => {
if (template && !name) {
return template.name
}
return name ? name.trim() : null
}
const tidyUrl = url => {
if (url && !url.startsWith("/")) {
url = `/${url}`
}
$values.url = url === "" ? null : url
}
const nameToUrl = appName => {
let resolvedUrl = resolveAppUrl(null, appName)
tidyUrl(resolvedUrl)
}
const updateIcon = e => {
const { name, color } = e.detail
$values.iconColor = color
$values.iconName = name
}
onMount(setupValidation)
</script>
<ModalContent
title="Edit name and URL"
confirmText="Save"
onConfirm={updateApp}
disabled={!$validation.valid}
>
<Input
bind:value={$values.name}
error={$validation.touched.name && $validation.errors.name}
on:blur={() => ($validation.touched.name = true)}
on:change={nameToUrl($values.name)}
label="Name"
/>
<Layout noPadding gap="XS">
<Label>Icon</Label>
<EditableIcon
{app}
size="XL"
name={$values.iconName}
color={$values.iconColor}
on:change={updateIcon}
/>
</Layout>
<Input
bind:value={$values.url}
error={$validation.touched.url && $validation.errors.url}
on:blur={() => ($validation.touched.url = true)}
on:change={tidyUrl($values.url)}
label="URL"
placeholder={$values.url
? $values.url
: `/${resolveAppUrl(null, $values.name)}`}
/>
</ModalContent>

View File

@ -19,11 +19,10 @@ export const name = (validation, { apps, currentApp } = { apps: [] }) => {
// exit early, above validator will fail
return true
}
if (currentApp) {
// filter out the current app if present
apps = apps.filter(app => app.appId !== currentApp.appId)
}
return !apps
.filter(app => {
return app.appId !== currentApp?.appId
})
.map(app => app.name)
.some(appName => appName.toLowerCase() === value.toLowerCase())
}

View File

@ -33,6 +33,7 @@
import { TOUR_KEYS } from "components/portal/onboarding/tours.js"
import PreviewOverlay from "./_components/PreviewOverlay.svelte"
import EnterpriseBasicTrialModal from "components/portal/onboarding/EnterpriseBasicTrialModal.svelte"
import UpdateAppTopNav from "components/common/UpdateAppTopNav.svelte"
export let application
@ -164,7 +165,11 @@
</Tabs>
</div>
<div class="topcenternav">
<Heading size="XS">{$appStore.name}</Heading>
<div class="app-name">
<UpdateAppTopNav {application}>
<Heading noPadding size="XS">{$appStore.name}</Heading>
</UpdateAppTopNav>
</div>
</div>
<div class="toprightnav">
<span>
@ -253,7 +258,6 @@
font-weight: 600;
overflow: hidden;
text-overflow: ellipsis;
padding: 0px var(--spacing-m);
}
.topleftnav {

View File

@ -1,30 +1,6 @@
<script>
import {
Layout,
Divider,
Heading,
Body,
Button,
Label,
Modal,
Icon,
} from "@budibase/bbui"
import { AppStatus } from "constants"
import { appStore, initialise } from "stores/builder"
import { appsStore } from "stores/portal"
import UpdateAppModal from "components/start/UpdateAppModal.svelte"
import { API } from "api"
let updatingModal
$: filteredApps = $appsStore.apps.filter(app => app.devId == $appStore.appId)
$: app = filteredApps.length ? filteredApps[0] : {}
$: appDeployed = app?.status === AppStatus.DEPLOYED
const initialiseApp = async () => {
const applicationPkg = await API.fetchAppPackage($appStore.appId)
await initialise(applicationPkg)
}
import { Layout, Divider, Heading, Body } from "@budibase/bbui"
import UpdateAppForm from "components/common/UpdateAppForm.svelte"
</script>
<Layout noPadding>
@ -33,61 +9,5 @@
<Body>Edit your app's name and URL</Body>
</Layout>
<Divider />
<Layout noPadding gap="XXS">
<Label size="L">Name</Label>
<Body>{$appStore?.name}</Body>
</Layout>
<Layout noPadding gap="XS">
<Label size="L">Icon</Label>
<div class="icon">
<Icon
size="L"
name={$appStore?.icon?.name || "Apps"}
color={$appStore?.icon?.color}
/>
</div>
</Layout>
<Layout noPadding gap="XXS">
<Label size="L">URL</Label>
<Body>{$appStore.url}</Body>
</Layout>
<div>
<Button
cta
on:click={() => {
updatingModal.show()
}}
disabled={appDeployed}
tooltip={appDeployed
? "You must unpublish your app to make changes"
: null}
>
Edit
</Button>
</div>
<UpdateAppForm />
</Layout>
<Modal bind:this={updatingModal} padding={false} width="600px">
<UpdateAppModal
app={{
name: $appStore.name,
url: $appStore.url,
icon: $appStore.icon,
appId: $appStore.appId,
}}
onUpdateComplete={async () => {
await initialiseApp()
}}
/>
</Modal>
<style>
.icon {
display: flex;
justify-content: flex-start;
}
</style>

View File

@ -131,7 +131,7 @@ export class AppsStore extends BudiStore {
if (updatedAppIndex !== -1) {
let updatedApp = state.apps[updatedAppIndex]
updatedApp = { ...updatedApp, ...value }
state.apps = state.apps.splice(updatedAppIndex, 1, updatedApp)
state.apps.splice(updatedAppIndex, 1, updatedApp)
}
return state
})

View File

@ -22,6 +22,7 @@
const context = getContext("context")
const component = getContext("component")
const { environmentStore } = getContext("sdk")
const {
styleable,
API,
@ -168,6 +169,7 @@
notifySuccess={notificationStore.actions.success}
notifyError={notificationStore.actions.error}
buttons={enrichedButtons}
isCloud={$environmentStore.cloud}
on:rowclick={e => onRowClick?.({ row: e.detail })}
/>
</span>

View File

@ -25,7 +25,7 @@
let fieldState
let fieldApi
const { API, notificationStore } = getContext("sdk")
const { API, notificationStore, environmentStore } = getContext("sdk")
const formContext = getContext("form")
const BYTES_IN_MB = 1000000
@ -87,7 +87,7 @@
error={fieldState.error}
on:change={handleChange}
{processFiles}
{handleFileTooLarge}
handleFileTooLarge={$environmentStore.cloud ? handleFileTooLarge : null}
{handleTooManyFiles}
{maximum}
{extensions}

View File

@ -12,7 +12,7 @@
export let schema
export let maximum
const { API, notifications } = getContext("grid")
const { API, notifications, props } = getContext("grid")
const imageExtensions = ["png", "tiff", "gif", "raw", "jpg", "jpeg"]
let isOpen = false
@ -106,7 +106,7 @@
on:change={e => onChange(e.detail)}
maximum={maximum || schema.constraints?.length?.maximum}
{processFiles}
{handleFileTooLarge}
handleFileTooLarge={$props.isCloud ? handleFileTooLarge : null}
/>
</div>
</GridPopover>

View File

@ -54,6 +54,7 @@
export let notifySuccess = null
export let notifyError = null
export let buttons = null
export let isCloud = null
// Unique identifier for DOM nodes inside this instance
const gridID = `grid-${Math.random().toString().slice(2)}`
@ -108,6 +109,7 @@
notifySuccess,
notifyError,
buttons,
isCloud,
})
$: minHeight =
Padding + SmallRowHeight + $rowHeight + (showControls ? ControlsHeight : 0)

View File

@ -17,8 +17,10 @@ module FetchMock {
raw: () => {
return { "content-type": ["application/json"] }
},
get: () => {
return ["application/json"]
get: (name: string) => {
if (name.toLowerCase() === "content-type") {
return ["application/json"]
}
},
},
json: async () => {

View File

@ -292,11 +292,6 @@ export const getSignedUploadURL = async function (ctx: Ctx) {
ctx.throw(400, "The specified datasource could not be found")
}
// Ensure we aren't using a custom endpoint
if (datasource?.config?.endpoint) {
ctx.throw(400, "S3 datasources with custom endpoints are not supported")
}
// Determine type of datasource and generate signed URL
let signedUrl
let publicUrl
@ -309,6 +304,7 @@ export const getSignedUploadURL = async function (ctx: Ctx) {
try {
const s3 = new AWS.S3({
region: awsRegion,
endpoint: datasource?.config?.endpoint as string,
accessKeyId: datasource?.config?.accessKeyId as string,
secretAccessKey: datasource?.config?.secretAccessKey as string,
apiVersion: "2006-03-01",
@ -316,7 +312,11 @@ export const getSignedUploadURL = async function (ctx: Ctx) {
})
const params = { Bucket: bucket, Key: key }
signedUrl = s3.getSignedUrl("putObject", params)
publicUrl = `https://${bucket}.s3.${awsRegion}.amazonaws.com/${key}`
if (datasource?.config?.endpoint) {
publicUrl = `${datasource.config.endpoint}/${bucket}/${key}`
} else {
publicUrl = `https://${bucket}.s3.${awsRegion}.amazonaws.com/${key}`
}
} catch (error: any) {
ctx.throw(400, error)
}

View File

@ -16,6 +16,7 @@ import get from "lodash/get"
import * as https from "https"
import qs from "querystring"
import fetch from "node-fetch"
import type { Response } from "node-fetch"
import { formatBytes } from "../utilities"
import { performance } from "perf_hooks"
import FormData from "form-data"
@ -25,6 +26,7 @@ import { handleFileResponse, handleXml } from "./utils"
import { parse } from "content-disposition"
import path from "path"
import { Builder as XmlBuilder } from "xml2js"
import { getAttachmentHeaders } from "./utils/restUtils"
enum BodyType {
NONE = "none",
@ -130,14 +132,15 @@ class RestIntegration implements IntegrationBase {
this.config = config
}
async parseResponse(response: any, pagination: PaginationConfig | null) {
async parseResponse(response: Response, pagination: PaginationConfig | null) {
let data: any[] | string | undefined,
raw: string | undefined,
headers: Record<string, string> = {},
headers: Record<string, string[] | string> = {},
filename: string | undefined
const contentType = response.headers.get("content-type") || ""
const contentDisposition = response.headers.get("content-disposition") || ""
const { contentType, contentDisposition } = getAttachmentHeaders(
response.headers
)
if (
contentDisposition.includes("filename") ||
contentDisposition.includes("attachment") ||
@ -172,7 +175,7 @@ class RestIntegration implements IntegrationBase {
throw `Failed to parse response body: ${err}`
}
let contentLength: string = response.headers.get("content-length")
let contentLength = response.headers.get("content-length")
if (!contentLength && raw) {
contentLength = Buffer.byteLength(raw, "utf8").toString()
}

View File

@ -4,7 +4,11 @@ jest.mock("node-fetch", () => {
raw: () => {
return { "content-type": ["application/json"] }
},
get: () => ["application/json"],
get: (name: string) => {
if (name.toLowerCase() === "content-type") {
return ["application/json"]
}
},
},
json: jest.fn(() => ({
my_next_cursor: 123,
@ -211,7 +215,16 @@ describe("REST Integration", () => {
json: json ? async () => json : undefined,
text: text ? async () => text : undefined,
headers: {
get: (key: any) => (key === "content-length" ? 100 : header),
get: (key: string) => {
switch (key.toLowerCase()) {
case "content-length":
return 100
case "content-type":
return header
default:
return ""
}
},
raw: () => ({ "content-type": header }),
},
}

View File

@ -0,0 +1,38 @@
import { getAttachmentHeaders } from "../utils/restUtils"
import type { Headers } from "node-fetch"
function headers(dispositionValue: string) {
return {
get: (name: string) => {
if (name.toLowerCase() === "content-disposition") {
return dispositionValue
} else {
return "application/pdf"
}
},
set: () => {},
} as unknown as Headers
}
describe("getAttachmentHeaders", () => {
it("should be able to correctly handle a broken content-disposition", () => {
const { contentDisposition } = getAttachmentHeaders(
headers(`filename="report.pdf"`)
)
expect(contentDisposition).toBe(`attachment; filename="report.pdf"`)
})
it("should be able to correctly with a filename that could cause problems", () => {
const { contentDisposition } = getAttachmentHeaders(
headers(`filename="report;.pdf"`)
)
expect(contentDisposition).toBe(`attachment; filename="report;.pdf"`)
})
it("should not touch a valid content-disposition", () => {
const { contentDisposition } = getAttachmentHeaders(
headers(`inline; filename="report.pdf"`)
)
expect(contentDisposition).toBe(`inline; filename="report.pdf"`)
})
})

View File

@ -0,0 +1,28 @@
import type { Headers } from "node-fetch"
export function getAttachmentHeaders(headers: Headers) {
const contentType = headers.get("content-type") || ""
let contentDisposition = headers.get("content-disposition") || ""
// the API does not follow the requirements of https://www.ietf.org/rfc/rfc2183.txt
// all content-disposition headers should be format disposition-type; parameters
// but some APIs do not provide a type, causing the parse below to fail - add one to fix this
if (contentDisposition) {
const quotesRegex = /"(?:[^"\\]|\\.)*"|;/g
let match: RegExpMatchArray | null = null,
found = false
while ((match = quotesRegex.exec(contentDisposition)) !== null) {
if (match[0] === ";") {
found = true
}
}
if (!found) {
return {
contentDisposition: `attachment; ${contentDisposition}`,
contentType,
}
}
}
return { contentDisposition, contentType }
}

View File

@ -242,7 +242,7 @@ export async function outputProcessing<T extends Row[] | Row>(
}
return attachment
}
if (typeof row[property] === "string") {
if (typeof row[property] === "string" && row[property].length) {
row[property] = JSON.parse(row[property])
}
if (Array.isArray(row[property])) {

645
yarn.lock

File diff suppressed because it is too large Load Diff