MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13835 from Budibase/fix/rest-download-issue 

Fixing some REST file download issues
This commit is contained in:
Michael Drury 2024-06-03 12:24:32 +01:00 committed by GitHub
parent 792970a93e 38ff7debb4
commit f109c1be64
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 102 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
packages/backend-core/src/objectStore/objectStore.ts
View File

@ -14,6 +14,7 @@ import { v4 } from "uuid"
import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
import fsp from "fs/promises"
import { HeadObjectOutput } from "aws-sdk/clients/s3"
import { ReadableStream } from "stream/web"
const streamPipeline = promisify(stream.pipeline)
// use this as a temporary store of buckets that are being created
@ -41,10 +42,7 @@ type UploadParams = BaseUploadParams & {
path?: string | PathLike
}
export type StreamTypes =
| ReadStream
| NodeJS.ReadableStream
| ReadableStream<Uint8Array>
export type StreamTypes = ReadStream | NodeJS.ReadableStream
export type StreamUploadParams = BaseUploadParams & {
stream?: StreamTypes
@ -222,6 +220,9 @@ export async function streamUpload({
extra,
ttl,
}: StreamUploadParams) {
if (!stream) {
throw new Error("Stream to upload is invalid/undefined")
}
const extension = filename.split(".").pop()
const objectStore = ObjectStore(bucketName)
const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
@ -251,14 +252,27 @@ export async function streamUpload({
: CONTENT_TYPE_MAP.txt
}
const bucket = sanitizeBucket(bucketName),
objKey = sanitizeKey(filename)
const params = {
Bucket: sanitizeBucket(bucketName),
Key: sanitizeKey(filename),
Bucket: bucket,
Key: objKey,
Body: stream,
ContentType: contentType,
...extra,
}
return objectStore.upload(params).promise()
const details = await objectStore.upload(params).promise()
const headDetails = await objectStore
.headObject({
Bucket: bucket,
Key: objKey,
})
.promise()
return {
...details,
ContentLength: headDetails.ContentLength,
}
}
/**

5
packages/server/package.json
View File

@ -68,7 +68,6 @@
"aws-sdk": "2.1030.0",
"bcrypt": "5.1.0",
"bcryptjs": "2.4.3",
"bl": "^6.0.12",
"bull": "4.10.1",
"chokidar": "3.5.3",
"content-disposition": "^0.5.4",
@ -116,7 +115,8 @@
"uuid": "^8.3.2",
"validate.js": "0.13.1",
"worker-farm": "1.7.0",
"xml2js": "0.5.0"
"xml2js": "0.5.0",
"tmp": "0.2.3"
},
"devDependencies": {
"@babel/preset-env": "7.16.11",
@ -137,6 +137,7 @@
"@types/supertest": "2.0.14",
"@types/tar": "6.1.5",
"@types/uuid": "8.3.4",
"@types/tmp": "0.2.6",
"copyfiles": "2.4.1",
"docker-compose": "0.23.17",
"jest": "29.7.0",

14
packages/server/src/integrations/rest.ts
View File

@ -149,13 +149,12 @@ class RestIntegration implements IntegrationBase {
{ downloadImages: this.config.downloadImages }
)
let contentLength = response.headers.get("content-length")
if (!contentLength && raw) {
contentLength = Buffer.byteLength(raw, "utf8").toString()
}
let isSuccess = response.status >= 200 && response.status < 300
if (
contentDisposition.includes("filename") ||
contentDisposition.includes("attachment") ||
contentDisposition.includes("form-data")
(contentDisposition.includes("filename") ||
contentDisposition.includes("attachment") ||
contentDisposition.includes("form-data")) &&
isSuccess
) {
filename =
path.basename(parse(contentDisposition).parameters?.filename) || ""
@ -168,6 +167,9 @@ class RestIntegration implements IntegrationBase {
return handleFileResponse(response, filename, this.startTimeMs)
} else {
responseTxt = response.text ? await response.text() : ""
if (!contentLength && responseTxt) {
contentLength = Buffer.byteLength(responseTxt, "utf8").toString()
}
const hasContent =
(contentLength && parseInt(contentLength) > 0) ||
responseTxt.length > 0

2
packages/server/src/integrations/tests/rest.spec.ts
View File

@ -657,6 +657,7 @@ describe("REST Integration", () => {
mockReadable.push(null)
;(fetch as unknown as jest.Mock).mockImplementationOnce(() =>
Promise.resolve({
status: 200,
headers: {
raw: () => ({
"content-type": [contentType],
@ -700,6 +701,7 @@ describe("REST Integration", () => {
mockReadable.push(null)
;(fetch as unknown as jest.Mock).mockImplementationOnce(() =>
Promise.resolve({
status: 200,
headers: {
raw: () => ({
"content-type": [contentType],

69
packages/server/src/integrations/utils/utils.ts
View File

@ -9,10 +9,12 @@ import { context, objectStore, sql } from "@budibase/backend-core"
import { v4 } from "uuid"
import { parseStringPromise as xmlParser } from "xml2js"
import { formatBytes } from "../../utilities"
import bl from "bl"
import env from "../../environment"
import { InvalidColumns } from "../../constants"
import { helpers, utils } from "@budibase/shared-core"
import { pipeline } from "stream/promises"
import tmp from "tmp"
import fs from "fs"
type PrimitiveTypes =
| FieldType.STRING
@ -360,35 +362,44 @@ export async function handleFileResponse(
const key = `${context.getProdAppId()}/${processedFileName}`
const bucket = objectStore.ObjectStoreBuckets.TEMP
const stream = response.body.pipe(bl((error, data) => data))
// put the response stream to disk temporarily as a buffer
const tmpObj = tmp.fileSync()
try {
await pipeline(response.body, fs.createWriteStream(tmpObj.name))
if (response.body) {
const contentLength = response.headers.get("content-length")
if (contentLength) {
size = parseInt(contentLength, 10)
}
if (response.body) {
const contentLength = response.headers.get("content-length")
if (contentLength) {
size = parseInt(contentLength, 10)
const details = await objectStore.streamUpload({
bucket,
filename: key,
stream: fs.createReadStream(tmpObj.name),
ttl: 1,
type: response.headers["content-type"],
})
if (!size && details.ContentLength) {
size = details.ContentLength
}
}
await objectStore.streamUpload({
bucket,
filename: key,
stream,
ttl: 1,
type: response.headers["content-type"],
})
}
presignedUrl = objectStore.getPresignedUrl(bucket, key)
return {
data: {
size,
name: processedFileName,
url: presignedUrl,
extension: fileExtension,
key: key,
},
info: {
code: response.status,
size: formatBytes(size.toString()),
time: `${Math.round(performance.now() - startTime)}ms`,
},
presignedUrl = objectStore.getPresignedUrl(bucket, key)
return {
data: {
size,
name: processedFileName,
url: presignedUrl,
extension: fileExtension,
key: key,
},
info: {
code: response.status,
size: formatBytes(size.toString()),
time: `${Math.round(performance.now() - startTime)}ms`,
},
}
} finally {
// cleanup tmp
tmpObj.removeCallback()
}
}

14
packages/server/src/threads/query.ts
View File

@ -196,12 +196,22 @@ class QueryRunner {
return { rows, keys, info, extra, pagination }
}
async runAnotherQuery(queryId: string, parameters: any) {
async runAnotherQuery(
queryId: string,
currentParameters: Record<string, any>
) {
const db = context.getAppDB()
const query = await db.get<Query>(queryId)
const datasource = await sdk.datasources.get(query.datasourceId, {
enriched: true,
})
// enrich parameters with dynamic queries defaults
const defaultParams = query.parameters || []
for (let param of defaultParams) {
if (!currentParameters[param.name]) {
currentParameters[param.name] = param.default
}
}
return new QueryRunner(
{
schema: query.schema,
@ -210,7 +220,7 @@ class QueryRunner {
transformer: query.transformer,
nullDefaultSupport: query.nullDefaultSupport,
ctx: this.ctx,
parameters,
parameters: currentParameters,
datasource,
queryId,
},

2
packages/types/src/documents/app/automation.ts
View File

@ -245,7 +245,7 @@ export type AutomationAttachment = {
export type AutomationAttachmentContent = {
filename: string
content: ReadStream | NodeJS.ReadableStream | ReadableStream<Uint8Array>
content: ReadStream | NodeJS.ReadableStream
}
export type BucketedContent = AutomationAttachmentContent & {

20
yarn.lock
View File

@ -6348,6 +6348,11 @@
dependencies:
"@types/estree" "*"
"@types/tmp@0.2.6":
version "0.2.6"
resolved "https://registry.yarnpkg.com/@types/tmp/-/tmp-0.2.6.tgz#d785ee90c52d7cc020e249c948c36f7b32d1e217"
integrity sha512-chhaNf2oKHlRkDGt+tiKE2Z5aJ6qalm7Z9rlLdBwmOiAAf09YQvvoLXjWK4HWPF1xU/fqvMgfNfpVoBscA/tKA==
"@types/tough-cookie@*", "@types/tough-cookie@^4.0.2":
version "4.0.2"
resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz#6286b4c7228d58ab7866d19716f3696e03a09397"
@ -7700,7 +7705,7 @@ bl@^4.0.3, bl@^4.1.0:
inherits "^2.0.4"
readable-stream "^3.4.0"
bl@^6.0.12, bl@^6.0.3:
bl@^6.0.3:
version "6.0.12"
resolved "https://registry.yarnpkg.com/bl/-/bl-6.0.12.tgz#77c35b96e13aeff028496c798b75389ddee9c7f8"
integrity sha512-EnEYHilP93oaOa2MnmNEjAcovPS3JlQZOyzGXi3EyEpPhm9qWvdDp7BmAVEVusGzp8LlwQK56Av+OkDoRjzE0w==
@ -16065,10 +16070,10 @@ mute-stream@~1.0.0:
resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-1.0.0.tgz#e31bd9fe62f0aed23520aa4324ea6671531e013e"
integrity sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==
mysql2@3.9.7:
version "3.9.7"
resolved "https://registry.yarnpkg.com/mysql2/-/mysql2-3.9.7.tgz#843755daf65b5ef08afe545fe14b8fb62824741a"
integrity sha512-KnJT8vYRcNAZv73uf9zpXqNbvBG7DJrs+1nACsjZP1HMJ1TgXEy8wnNilXAn/5i57JizXKtrUtwDB7HxT9DDpw==
mysql2@3.9.8:
version "3.9.8"
resolved "https://registry.yarnpkg.com/mysql2/-/mysql2-3.9.8.tgz#fe8a0f975f2c495ed76ca988ddc5505801dc49ce"
integrity sha512-+5JKNjPuks1FNMoy9TYpl77f+5frbTklz7eb3XDwbpsERRLEeXiW2PDEkakYF50UuKU2qwfGnyXpKYvukv8mGA==
dependencies:
denque "^2.1.0"
generate-function "^2.3.1"
@ -21283,6 +21288,11 @@ tlhunter-sorted-set@^0.1.0:
resolved "https://registry.yarnpkg.com/tlhunter-sorted-set/-/tlhunter-sorted-set-0.1.0.tgz#1c3eae28c0fa4dff97e9501d2e3c204b86406f4b"
integrity sha512-eGYW4bjf1DtrHzUYxYfAcSytpOkA44zsr7G2n3PV7yOUR23vmkGe3LL4R+1jL9OsXtbsFOwe8XtbCrabeaEFnw==
tmp@0.2.3:
version "0.2.3"
resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.3.tgz#eb783cc22bc1e8bebd0671476d46ea4eb32a79ae"
integrity sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==
tmp@^0.0.33:
version "0.0.33"
resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9"