From 52d8d7d1dcfd52ca922c28ad11ce1755a0d86d96 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Feb 2025 18:15:48 +0000 Subject: [PATCH 1/3] Updating configuration for S3 upload/read components, in new SDK the format for these has changed slightly, updated the types to correctly respect the new types. --- .../src/api/controllers/static/index.ts | 11 ++++--- packages/server/src/integrations/s3.ts | 26 +++++++++++----- .../src/sdk/app/datasources/datasources.ts | 30 +++++++++++-------- packages/types/src/sdk/datasources.ts | 2 +- 4 files changed, 44 insertions(+), 25 deletions(-) diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index 97406d677d..327ce5369f 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -339,10 +339,13 @@ export const getSignedUploadURL = async function ( ctx.throw(400, "bucket and key values are required") } try { + let endpoint = datasource?.config?.endpoint + if (endpoint && !endpoint.startsWith("http")) { + endpoint = `https://${endpoint}` + } const s3 = new S3({ region: awsRegion, - endpoint: datasource?.config?.endpoint || undefined, - + endpoint: endpoint, credentials: { accessKeyId: datasource?.config?.accessKeyId as string, secretAccessKey: datasource?.config?.secretAccessKey as string, @@ -350,8 +353,8 @@ export const getSignedUploadURL = async function ( }) const params = { Bucket: bucket, Key: key } signedUrl = await getSignedUrl(s3, new PutObjectCommand(params)) - if (datasource?.config?.endpoint) { - publicUrl = `${datasource.config.endpoint}/${bucket}/${key}` + if (endpoint) { + publicUrl = `${endpoint}/${bucket}/${key}` } else { publicUrl = `https://${bucket}.s3.${awsRegion}.amazonaws.com/${key}` } diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index 488c22835a..52830fb7c1 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -7,9 +7,10 @@ import { ConnectionInfo, } from "@budibase/types" -import { S3 } from "@aws-sdk/client-s3" +import { S3, S3ClientConfig } from "@aws-sdk/client-s3" import csv from "csvtojson" import stream from "stream" +import { NodeJsClient } from "@smithy/types" interface S3Config { region: string @@ -157,18 +158,25 @@ const SCHEMA: Integration = { } class S3Integration implements IntegrationBase { - private readonly config: S3Config - private client + private readonly config: S3ClientConfig + private client: NodeJsClient constructor(config: S3Config) { - this.config = config - if (this.config.endpoint) { - this.config.s3ForcePathStyle = true + this.config = { + forcePathStyle: config.s3ForcePathStyle || true, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + region: config.region, + } + if (config.endpoint) { + this.config.forcePathStyle = true } else { delete this.config.endpoint } - this.client = new S3(this.config) + this.client = new S3(this.config) as NodeJsClient } async testConnection() { @@ -176,7 +184,9 @@ class S3Integration implements IntegrationBase { connected: false, } try { - await this.client.listBuckets() + await this.client.listBuckets({ + MaxBuckets: 1, + }) response.connected = true } catch (e: any) { response.error = e.message as string diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index 84e1601152..5699f31ee5 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -119,19 +119,23 @@ export function areRESTVariablesValid(datasource: Datasource) { } export function checkDatasourceTypes(schema: Integration, config: any) { - for (let key of Object.keys(config)) { - if (!schema.datasource[key]) { - continue - } - const type = schema.datasource[key].type - if ( - type === DatasourceFieldType.NUMBER && - typeof config[key] === "string" - ) { - config[key] = parseFloat(config[key]) + try { + for (let key of Object.keys(config)) { + if (!schema.datasource?.[key]) { + continue + } + const type = schema.datasource[key].type + if ( + type === DatasourceFieldType.NUMBER && + typeof config[key] === "string" + ) { + config[key] = parseFloat(config[key]) + } } + return config + } catch (err) { + console.log(err) } - return config } async function enrichDatasourceWithValues( @@ -149,7 +153,9 @@ async function enrichDatasourceWithValues( ) as Datasource processed.entities = entities const definition = await getDefinition(processed.source) - processed.config = checkDatasourceTypes(definition!, processed.config) + if (definition) { + processed.config = checkDatasourceTypes(definition, processed.config) + } return { datasource: processed, envVars: env as Record, diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 1f2a4d2127..349e9fbe0e 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -157,7 +157,7 @@ export interface Integration { friendlyName: string type?: string iconUrl?: string - datasource: DatasourceConfig + datasource?: DatasourceConfig query: { [key: string]: QueryDefinition } From e7f42a0075fcec1227a029f946045ab4b791c023 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Feb 2025 18:17:07 +0000 Subject: [PATCH 2/3] Removing try catch wrap. --- .../src/sdk/app/datasources/datasources.ts | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index 5699f31ee5..d87ef69205 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -119,23 +119,19 @@ export function areRESTVariablesValid(datasource: Datasource) { } export function checkDatasourceTypes(schema: Integration, config: any) { - try { - for (let key of Object.keys(config)) { - if (!schema.datasource?.[key]) { - continue - } - const type = schema.datasource[key].type - if ( - type === DatasourceFieldType.NUMBER && - typeof config[key] === "string" - ) { - config[key] = parseFloat(config[key]) - } + for (let key of Object.keys(config)) { + if (!schema.datasource?.[key]) { + continue + } + const type = schema.datasource[key].type + if ( + type === DatasourceFieldType.NUMBER && + typeof config[key] === "string" + ) { + config[key] = parseFloat(config[key]) } - return config - } catch (err) { - console.log(err) } + return config } async function enrichDatasourceWithValues( From 8ea25418b9b6140906353a4cb2d3cf0f43ae9986 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Feb 2025 18:26:04 +0000 Subject: [PATCH 3/3] Adding has protocol function and using that. --- packages/backend-core/src/utils/utils.ts | 4 ++++ packages/server/package.json | 1 + packages/server/src/api/controllers/static/index.ts | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index 30cf55b149..7f2e25b6d4 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -247,3 +247,7 @@ export function hasCircularStructure(json: any) { } return false } + +export function urlHasProtocol(url: string): boolean { + return !!url.match(/^.+:\/\/.+$/) +} diff --git a/packages/server/package.json b/packages/server/package.json index 9a70ecba9c..37df7138d7 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -159,6 +159,7 @@ "@types/tar": "6.1.5", "@types/tmp": "0.2.6", "@types/uuid": "8.3.4", + "@smithy/types": "4.0.0", "chance": "^1.1.12", "copyfiles": "2.4.1", "docker-compose": "0.23.17", diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index 327ce5369f..6b8ecda0d9 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -340,7 +340,7 @@ export const getSignedUploadURL = async function ( } try { let endpoint = datasource?.config?.endpoint - if (endpoint && !endpoint.startsWith("http")) { + if (endpoint && !utils.urlHasProtocol(endpoint)) { endpoint = `https://${endpoint}` } const s3 = new S3({