Merge pull request #7521 from Budibase/feature/s3-bucket-connector

S3 bucket connector improvements

packages/server/__mocks__/aws-sdk.ts

@@ -37,6 +37,20 @@ module AwsMock {
         Contents: {},
       })
     )
+
+    // @ts-ignore
+    this.createBucket = jest.fn(
+      response({
+        Contents: {},
+      })
+    )
+
+    // @ts-ignore
+    this.deleteObjects = jest.fn(
+      response({
+        Contents: {},
+      })
+    )
   }
 
   aws.DynamoDB = { DocumentClient }

packages/server/package.json

@@ -200,4 +200,4 @@
     "oracledb": "5.3.0"
   },
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
-}
+}

packages/server/src/integrations/s3.ts

@@ -1,5 +1,12 @@
-import { Integration, QueryType, IntegrationBase } from "@budibase/types"
+import {
+  Integration,
+  QueryType,
+  IntegrationBase,
+  DatasourceFieldType,
+} from "@budibase/types"
+
 const AWS = require("aws-sdk")
+const csv = require("csvtojson")
 
 interface S3Config {
   region: string
@@ -40,13 +47,103 @@ const SCHEMA: Integration = {
     },
   },
   query: {
+    create: {
+      type: QueryType.FIELDS,
+      fields: {
+        bucket: {
+          display: "New Bucket",
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        location: {
+          required: true,
+          default: "us-east-1",
+          type: DatasourceFieldType.STRING,
+        },
+        grantFullControl: {
+          display: "Grant full control",
+          type: DatasourceFieldType.STRING,
+        },
+        grantRead: {
+          display: "Grant read",
+          type: DatasourceFieldType.STRING,
+        },
+        grantReadAcp: {
+          display: "Grant read ACP",
+          type: DatasourceFieldType.STRING,
+        },
+        grantWrite: {
+          display: "Grant write",
+          type: DatasourceFieldType.STRING,
+        },
+        grantWriteAcp: {
+          display: "Grant write ACP",
+          type: DatasourceFieldType.STRING,
+        },
+      },
+    },
     read: {
       type: QueryType.FIELDS,
       fields: {
         bucket: {
-          type: "string",
+          type: DatasourceFieldType.STRING,
           required: true,
         },
+        delimiter: {
+          type: DatasourceFieldType.STRING,
+        },
+        marker: {
+          type: DatasourceFieldType.STRING,
+        },
+        maxKeys: {
+          type: DatasourceFieldType.NUMBER,
+          display: "Max Keys",
+        },
+        prefix: {
+          type: DatasourceFieldType.STRING,
+        },
+      },
+    },
+    readCsv: {
+      displayName: "Read CSV",
+      type: QueryType.FIELDS,
+      fields: {
+        bucket: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        key: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+      },
+    },
+    delete: {
+      type: QueryType.FIELDS,
+      fields: {
+        bucket: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        delete: {
+          type: DatasourceFieldType.JSON,
+          required: true,
+        },
+      },
+    },
+  },
+  extra: {
+    acl: {
+      required: false,
+      displayName: "ACL",
+      type: DatasourceFieldType.LIST,
+      data: {
+        create: [
+          "private",
+          "public-read",
+          "public-read-write",
+          "authenticated-read",
+        ],
       },
     },
   },
@@ -67,14 +164,93 @@ class S3Integration implements IntegrationBase {
     this.client = new AWS.S3(this.config)
   }
 
-  async read(query: { bucket: string }) {
+  async create(query: {
+    bucket: string
+    location: string
+    grantFullControl: string
+    grantRead: string
+    grantReadAcp: string
+    grantWrite: string
+    grantWriteAcp: string
+    extra: {
+      acl: string
+    }
+  }) {
+    let params: any = {
+      Bucket: query.bucket,
+      ACL: query.extra?.acl,
+      GrantFullControl: query.grantFullControl,
+      GrantRead: query.grantRead,
+      GrantReadACP: query.grantReadAcp,
+      GrantWrite: query.grantWrite,
+      GrantWriteACP: query.grantWriteAcp,
+    }
+    if (query.location) {
+      params["CreateBucketConfiguration"] = {
+        LocationConstraint: query.location,
+      }
+    }
+    return await this.client.createBucket(params).promise()
+  }
+
+  async read(query: {
+    bucket: string
+    delimiter: string
+    expectedBucketOwner: string
+    marker: string
+    maxKeys: number
+    prefix: string
+  }) {
     const response = await this.client
       .listObjects({
         Bucket: query.bucket,
+        Delimiter: query.delimiter,
+        Marker: query.marker,
+        MaxKeys: query.maxKeys,
+        Prefix: query.prefix,
       })
       .promise()
     return response.Contents
   }
+
+  async readCsv(query: { bucket: string; key: string }) {
+    const stream = this.client
+      .getObject({
+        Bucket: query.bucket,
+        Key: query.key,
+      })
+      .createReadStream()
+
+    let csvError = false
+    return new Promise((resolve, reject) => {
+      stream.on("error", (err: Error) => {
+        reject(err)
+      })
+      const response = csv()
+        .fromStream(stream)
+        .on("error", () => {
+          csvError = true
+        })
+      stream.on("finish", () => {
+        resolve(response)
+      })
+    }).catch(err => {
+      if (csvError) {
+        throw new Error("Could not read CSV")
+      } else {
+        throw err
+      }
+    })
+  }
+
+  async delete(query: { bucket: string; delete: string }) {
+    return await this.client
+      .deleteObjects({
+        Bucket: query.bucket,
+        Delete: JSON.parse(query.delete),
+      })
+      .promise()
+  }
 }
 
 export default {

packages/server/src/integrations/tests/s3.spec.ts

@@ -18,11 +18,95 @@ describe("S3 Integration", () => {
   })
 
   it("calls the read method with the correct params", async () => {
-    const response = await config.integration.read({
+    await config.integration.read({
       bucket: "test",
+      delimiter: "/",
+      marker: "file.txt",
+      maxKeys: 999,
+      prefix: "directory/",
     })
     expect(config.integration.client.listObjects).toHaveBeenCalledWith({
       Bucket: "test",
+      Delimiter: "/",
+      Marker: "file.txt",
+      MaxKeys: 999,
+      Prefix: "directory/",
+    })
+  })
+
+  it("calls the create method with the correct params", async () => {
+    await config.integration.create({
+      bucket: "test",
+      location: "af-south-1",
+      grantFullControl: "me",
+      grantRead: "him",
+      grantReadAcp: "her",
+      grantWrite: "she",
+      grantWriteAcp: "he",
+      objectLockEnabledForBucket: true,
+      extra: {
+        acl: "private",
+      },
+    })
+    expect(config.integration.client.createBucket).toHaveBeenCalledWith({
+      Bucket: "test",
+      CreateBucketConfiguration: {
+        LocationConstraint: "af-south-1",
+      },
+      GrantFullControl: "me",
+      GrantRead: "him",
+      GrantReadACP: "her",
+      GrantWrite: "she",
+      GrantWriteACP: "he",
+      ACL: "private",
+    })
+  })
+
+  it("does not add undefined location constraint when calling the create method", async () => {
+    await config.integration.create({
+      bucket: "test",
+    })
+    expect(config.integration.client.createBucket).toHaveBeenCalledWith({
+      Bucket: "test",
+      GrantFullControl: undefined,
+      GrantRead: undefined,
+      GrantReadACP: undefined,
+      GrantWrite: undefined,
+      GrantWriteACP: undefined,
+      ACL: undefined,
+    })
+  })
+
+  it("calls the delete method with the correct params ", async () => {
+    await config.integration.delete({
+      bucket: "test",
+      delete: `{
+        "Objects": [
+          {
+         "Key": "HappyFace.jpg", 
+         "VersionId": "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b"
+        }, 
+          {
+         "Key": "HappyFace.jpg", 
+         "VersionId": "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd"
+        }
+       ]
+      }`,
+    })
+    expect(config.integration.client.deleteObjects).toHaveBeenCalledWith({
+      Bucket: "test",
+      Delete: {
+        Objects: [
+          {
+            Key: "HappyFace.jpg",
+            VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b",
+          },
+          {
+            Key: "HappyFace.jpg",
+            VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd",
+          },
+        ],
+      },
     })
   })
 })

packages/server/yarn.lock

@@ -5172,7 +5172,7 @@ cssstyle@^2.3.0:
   dependencies:
     cssom "~0.3.6"
 
-csvtojson@2.0.10:
+csvtojson@^2.0.10:
   version "2.0.10"
   resolved "https://registry.yarnpkg.com/csvtojson/-/csvtojson-2.0.10.tgz#11e7242cc630da54efce7958a45f443210357574"
   integrity sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==