import { Integration, QueryType, IntegrationBase, DatasourceFieldType, } from "@budibase/types" module S3Module { const AWS = require("aws-sdk") const csv = require("csvtojson") interface S3Config { region: string accessKeyId: string secretAccessKey: string s3ForcePathStyle: boolean endpoint?: string } const SCHEMA: Integration = { docs: "https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html", description: "Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.", friendlyName: "Amazon S3", type: "Object store", datasource: { region: { type: "string", required: false, default: "us-east-1", }, accessKeyId: { type: "password", required: true, }, secretAccessKey: { type: "password", required: true, }, endpoint: { type: "string", required: false, }, signatureVersion: { type: "string", required: false, default: "v4", }, }, query: { read: { type: QueryType.FIELDS, fields: { bucket: { type: DatasourceFieldType.STRING, required: true, }, delimiter: { type: DatasourceFieldType.STRING, }, marker: { type: DatasourceFieldType.STRING, }, maxKeys: { type: DatasourceFieldType.NUMBER, display: "Max Keys", }, prefix: { type: DatasourceFieldType.STRING, }, }, }, readCsv: { displayName: "Read CSV", type: QueryType.FIELDS, fields: { bucket: { type: DatasourceFieldType.STRING, required: true, }, key: { type: DatasourceFieldType.STRING, required: true, }, }, }, }, } class S3Integration implements IntegrationBase { private readonly config: S3Config private client: any constructor(config: S3Config) { this.config = config if (this.config.endpoint) { this.config.s3ForcePathStyle = true } else { delete this.config.endpoint } this.client = new AWS.S3(this.config) } async read(query: { bucket: string delimiter: string expectedBucketOwner: string marker: string maxKeys: number prefix: string }) { const response = await this.client .listObjects({ Bucket: query.bucket, Delimiter: query.delimiter, Marker: query.marker, MaxKeys: query.maxKeys, Prefix: query.prefix, }) .promise() return response.Contents } async readCsv(query: { bucket: string; key: string }) { const stream = this.client .getObject({ Bucket: query.bucket, Key: query.key, }) .createReadStream() let csvError = false return new Promise(async (resolve, reject) => { stream.on("error", (err: Error) => { reject(err) }) const response = csv() .fromStream(stream) .on("error", () => { csvError = true }) stream.on("finish", () => { resolve(response) }) }).catch(err => { if (csvError) { throw new Error("Could not read CSV") } else { throw err } }) } } module.exports = { schema: SCHEMA, integration: S3Integration, } }