MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7521 from Budibase/feature/s3-bucket-connector 

S3 bucket connector improvements
This commit is contained in:
melohagan 2022-11-01 09:56:47 +00:00 committed by GitHub
parent 83056d5cad 86fef02567
commit cc44430638
5 changed files with 280 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
packages/server/__mocks__/aws-sdk.ts
View File

@ -37,6 +37,20 @@ module AwsMock {
Contents: {},
})
)
// @ts-ignore
this.createBucket = jest.fn(
response({
Contents: {},
})
)
// @ts-ignore
this.deleteObjects = jest.fn(
response({
Contents: {},
})
)
}
aws.DynamoDB = { DocumentClient }

2
packages/server/package.json
View File

@ -200,4 +200,4 @@
"oracledb": "5.3.0"
},
"gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
}
}

182
packages/server/src/integrations/s3.ts
View File

@ -1,5 +1,12 @@
import { Integration, QueryType, IntegrationBase } from "@budibase/types"
import {
Integration,
QueryType,
IntegrationBase,
DatasourceFieldType,
} from "@budibase/types"
const AWS = require("aws-sdk")
const csv = require("csvtojson")
interface S3Config {
region: string
@ -40,13 +47,103 @@ const SCHEMA: Integration = {
},
},
query: {
create: {
type: QueryType.FIELDS,
fields: {
bucket: {
display: "New Bucket",
type: DatasourceFieldType.STRING,
required: true,
},
location: {
required: true,
default: "us-east-1",
type: DatasourceFieldType.STRING,
},
grantFullControl: {
display: "Grant full control",
type: DatasourceFieldType.STRING,
},
grantRead: {
display: "Grant read",
type: DatasourceFieldType.STRING,
},
grantReadAcp: {
display: "Grant read ACP",
type: DatasourceFieldType.STRING,
},
grantWrite: {
display: "Grant write",
type: DatasourceFieldType.STRING,
},
grantWriteAcp: {
display: "Grant write ACP",
type: DatasourceFieldType.STRING,
},
},
},
read: {
type: QueryType.FIELDS,
fields: {
bucket: {
type: "string",
type: DatasourceFieldType.STRING,
required: true,
},
delimiter: {
type: DatasourceFieldType.STRING,
},
marker: {
type: DatasourceFieldType.STRING,
},
maxKeys: {
type: DatasourceFieldType.NUMBER,
display: "Max Keys",
},
prefix: {
type: DatasourceFieldType.STRING,
},
},
},
readCsv: {
displayName: "Read CSV",
type: QueryType.FIELDS,
fields: {
bucket: {
type: DatasourceFieldType.STRING,
required: true,
},
key: {
type: DatasourceFieldType.STRING,
required: true,
},
},
},
delete: {
type: QueryType.FIELDS,
fields: {
bucket: {
type: DatasourceFieldType.STRING,
required: true,
},
delete: {
type: DatasourceFieldType.JSON,
required: true,
},
},
},
},
extra: {
acl: {
required: false,
displayName: "ACL",
type: DatasourceFieldType.LIST,
data: {
create: [
"private",
"public-read",
"public-read-write",
"authenticated-read",
],
},
},
},
@ -67,14 +164,93 @@ class S3Integration implements IntegrationBase {
this.client = new AWS.S3(this.config)
}
async read(query: { bucket: string }) {
async create(query: {
bucket: string
location: string
grantFullControl: string
grantRead: string
grantReadAcp: string
grantWrite: string
grantWriteAcp: string
extra: {
acl: string
}
}) {
let params: any = {
Bucket: query.bucket,
ACL: query.extra?.acl,
GrantFullControl: query.grantFullControl,
GrantRead: query.grantRead,
GrantReadACP: query.grantReadAcp,
GrantWrite: query.grantWrite,
GrantWriteACP: query.grantWriteAcp,
}
if (query.location) {
params["CreateBucketConfiguration"] = {
LocationConstraint: query.location,
}
}
return await this.client.createBucket(params).promise()
}
async read(query: {
bucket: string
delimiter: string
expectedBucketOwner: string
marker: string
maxKeys: number
prefix: string
}) {
const response = await this.client
.listObjects({
Bucket: query.bucket,
Delimiter: query.delimiter,
Marker: query.marker,
MaxKeys: query.maxKeys,
Prefix: query.prefix,
})
.promise()
return response.Contents
}
async readCsv(query: { bucket: string; key: string }) {
const stream = this.client
.getObject({
Bucket: query.bucket,
Key: query.key,
})
.createReadStream()
let csvError = false
return new Promise((resolve, reject) => {
stream.on("error", (err: Error) => {
reject(err)
})
const response = csv()
.fromStream(stream)
.on("error", () => {
csvError = true
})
stream.on("finish", () => {
resolve(response)
})
}).catch(err => {
if (csvError) {
throw new Error("Could not read CSV")
} else {
throw err
}
})
}
async delete(query: { bucket: string; delete: string }) {
return await this.client
.deleteObjects({
Bucket: query.bucket,
Delete: JSON.parse(query.delete),
})
.promise()
}
}
export default {

86
packages/server/src/integrations/tests/s3.spec.ts
View File

@ -18,11 +18,95 @@ describe("S3 Integration", () => {
})
it("calls the read method with the correct params", async () => {
const response = await config.integration.read({
await config.integration.read({
bucket: "test",
delimiter: "/",
marker: "file.txt",
maxKeys: 999,
prefix: "directory/",
})
expect(config.integration.client.listObjects).toHaveBeenCalledWith({
Bucket: "test",
Delimiter: "/",
Marker: "file.txt",
MaxKeys: 999,
Prefix: "directory/",
})
})
it("calls the create method with the correct params", async () => {
await config.integration.create({
bucket: "test",
location: "af-south-1",
grantFullControl: "me",
grantRead: "him",
grantReadAcp: "her",
grantWrite: "she",
grantWriteAcp: "he",
objectLockEnabledForBucket: true,
extra: {
acl: "private",
},
})
expect(config.integration.client.createBucket).toHaveBeenCalledWith({
Bucket: "test",
CreateBucketConfiguration: {
LocationConstraint: "af-south-1",
},
GrantFullControl: "me",
GrantRead: "him",
GrantReadACP: "her",
GrantWrite: "she",
GrantWriteACP: "he",
ACL: "private",
})
})
it("does not add undefined location constraint when calling the create method", async () => {
await config.integration.create({
bucket: "test",
})
expect(config.integration.client.createBucket).toHaveBeenCalledWith({
Bucket: "test",
GrantFullControl: undefined,
GrantRead: undefined,
GrantReadACP: undefined,
GrantWrite: undefined,
GrantWriteACP: undefined,
ACL: undefined,
})
})
it("calls the delete method with the correct params ", async () => {
await config.integration.delete({
bucket: "test",
delete: `{
"Objects": [
{
"Key": "HappyFace.jpg",
"VersionId": "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b"
},
{
"Key": "HappyFace.jpg",
"VersionId": "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd"
}
]
}`,
})
expect(config.integration.client.deleteObjects).toHaveBeenCalledWith({
Bucket: "test",
Delete: {
Objects: [
{
Key: "HappyFace.jpg",
VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b",
},
{
Key: "HappyFace.jpg",
VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd",
},
],
},
})
})
})

2
packages/server/yarn.lock
View File

@ -5172,7 +5172,7 @@ cssstyle@^2.3.0:
dependencies:
cssom "~0.3.6"
csvtojson@2.0.10:
csvtojson@^2.0.10:
version "2.0.10"
resolved "https://registry.yarnpkg.com/csvtojson/-/csvtojson-2.0.10.tgz#11e7242cc630da54efce7958a45f443210357574"
integrity sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==