From 6518976825e3ebfa1c33af70b4e393fdfd9e5bee Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martin@shogunsystems.co.uk>
Date: Thu, 26 Nov 2020 21:23:20 +0000
Subject: [PATCH] browse contents of s3 buckets from within budibase

---
 .../modals/EditIntegrationConfig.svelte       | 21 +++++---
 .../TableNavigator/TableNavigator.svelte      |  2 +-
 .../popovers/EditTablePopover.svelte          |  2 +-
 .../data/table/[selectedTable]/index.svelte   |  2 +-
 packages/server/src/api/controllers/row.js    |  2 +-
 packages/server/src/integrations/dynamodb.js  |  4 +-
 .../server/src/integrations/elasticsearch.js  |  2 +-
 packages/server/src/integrations/index.js     |  4 +-
 packages/server/src/integrations/mongodb.js   |  8 ++++
 packages/server/src/integrations/s3.js        | 48 +++++++++++++++++++
 10 files changed, 80 insertions(+), 15 deletions(-)
 create mode 100644 packages/server/src/integrations/s3.js

diff --git a/packages/builder/src/components/backend/DataTable/modals/EditIntegrationConfig.svelte b/packages/builder/src/components/backend/DataTable/modals/EditIntegrationConfig.svelte
index 4789b6f8f0..9becd0593f 100644
--- a/packages/builder/src/components/backend/DataTable/modals/EditIntegrationConfig.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/EditIntegrationConfig.svelte
@@ -36,7 +36,7 @@
   }
 </script>
 
-<form>
+<section>
   <div class="config">
     <h6>Schema</h6>
     {#each fields as field, idx}
@@ -60,15 +60,22 @@
   <div class="config">
     <h6>Datasource</h6>
     {#each Object.keys(table.integration) as configKey}
-      <Input
-        thin
-        type={configKey.type}
-        label={configKey}
-        bind:value={table.integration[configKey]} />
+      {#if configKey === 'query'}
+        <TextArea
+          thin
+          label={configKey}
+          bind:value={table.integration[configKey]} />
+      {:else}
+        <Input
+          thin
+          type={configKey.type}
+          label={configKey}
+          bind:value={table.integration[configKey]} />
+      {/if}
       <Spacer small />
     {/each}
   </div>
-</form>
+</section>
 
 <style>
   .field {
diff --git a/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte b/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte
index eb5cfffa6d..699155e1bf 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte
@@ -43,7 +43,7 @@
     {#each $backendUiStore.tables as table, idx}
       <NavItem
         border={idx > 0}
-        icon={table.integration ? 'ri-database-2-line' : 'ri-table-line'}
+        icon={table.integration?.type ? 'ri-database-2-line' : 'ri-table-line'}
         text={table.name}
         selected={selectedView === `all_${table._id}`}
         on:click={() => selectTable(table)}>
diff --git a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte
index bd9a58e874..05302788cb 100644
--- a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte
@@ -79,7 +79,7 @@
           bind:value={table.name}
           on:input={checkValid}
           {error} />
-        {#if table.integration}
+        {#if table.integration?.type}
           <IntegrationConfigForm integration={table.integration} />
         {/if}
         <footer>
diff --git a/packages/builder/src/pages/[application]/data/table/[selectedTable]/index.svelte b/packages/builder/src/pages/[application]/data/table/[selectedTable]/index.svelte
index 89a3604dc1..475271bb31 100644
--- a/packages/builder/src/pages/[application]/data/table/[selectedTable]/index.svelte
+++ b/packages/builder/src/pages/[application]/data/table/[selectedTable]/index.svelte
@@ -7,7 +7,7 @@
 </script>
 
 {#if $backendUiStore.selectedDatabase._id && selectedTable.name}
-  {#if selectedTable.integration}
+  {#if selectedTable.integration?.type}
     <ExternalDataSourceTable />
   {:else}
     <TableDataTable />
diff --git a/packages/server/src/api/controllers/row.js b/packages/server/src/api/controllers/row.js
index 443cb298af..2aa539a8f0 100644
--- a/packages/server/src/api/controllers/row.js
+++ b/packages/server/src/api/controllers/row.js
@@ -190,7 +190,7 @@ exports.fetchTableRows = async function(ctx) {
 
   const table = await db.get(ctx.params.tableId)
 
-  if (table.integration) {
+  if (table.integration && table.integration.type) {
     const Integration = integrations[table.integration.type]
     ctx.body = await new Integration(table.integration).query()
     return
diff --git a/packages/server/src/integrations/dynamodb.js b/packages/server/src/integrations/dynamodb.js
index 3ed8e36aa1..a5220decf7 100644
--- a/packages/server/src/integrations/dynamodb.js
+++ b/packages/server/src/integrations/dynamodb.js
@@ -27,11 +27,11 @@ const DYNAMODB_OPTIONS = {
     required: true,
   },
   attributeNames: {
-    type: "object",
+    type: "json",
     required: true,
   },
   attributeValues: {
-    type: "object",
+    type: "json",
     required: true,
   },
 }
diff --git a/packages/server/src/integrations/elasticsearch.js b/packages/server/src/integrations/elasticsearch.js
index d5074bb26f..d7b8f8cbd6 100644
--- a/packages/server/src/integrations/elasticsearch.js
+++ b/packages/server/src/integrations/elasticsearch.js
@@ -11,7 +11,7 @@ const ELASTICSEARCH_OPTIONS = {
     required: true,
   },
   query: {
-    type: "query",
+    type: "json",
     required: true,
   },
 }
diff --git a/packages/server/src/integrations/index.js b/packages/server/src/integrations/index.js
index 4cdb78f703..23179feb60 100644
--- a/packages/server/src/integrations/index.js
+++ b/packages/server/src/integrations/index.js
@@ -4,7 +4,7 @@ const mongodb = require("./mongodb")
 const elasticsearch = require("./elasticsearch")
 const couchdb = require("./couchdb")
 // const redis = require("./redis")
-// const s3 = require("./s3")
+const s3 = require("./s3")
 
 const DEFINITIONS = {
   POSTGRES: postgres.schema,
@@ -12,6 +12,7 @@ const DEFINITIONS = {
   MONGODB: mongodb.schema,
   ELASTICSEARCH: elasticsearch.schema,
   COUCHDB: couchdb.schema,
+  S3: s3.schema,
 }
 
 const INTEGRATIONS = {
@@ -20,6 +21,7 @@ const INTEGRATIONS = {
   MONGODB: mongodb.integration,
   ELASTICSEARCH: elasticsearch.integration,
   COUCHDB: couchdb.integration,
+  S3: s3.integration,
 }
 
 module.exports = {
diff --git a/packages/server/src/integrations/mongodb.js b/packages/server/src/integrations/mongodb.js
index 1eb351bcb5..1d1d7e3918 100644
--- a/packages/server/src/integrations/mongodb.js
+++ b/packages/server/src/integrations/mongodb.js
@@ -24,6 +24,11 @@ class MongoIntegration {
   constructor(config) {
     this.config = config
     this.client = new MongoClient(config.connectionString)
+    try {
+      this.config.query = JSON.parse(this.config.query)
+    } catch (err) {
+      this.config.query = {}
+    }
   }
 
   async connect() {
@@ -37,6 +42,9 @@ class MongoIntegration {
       const collection = db.collection(this.config.collection)
       const result = await collection.find(this.config.query).toArray()
       return result
+    } catch (err) {
+      console.error("Error querying mongodb", err)
+      throw err
     } finally {
       await this.client.close()
     }
diff --git a/packages/server/src/integrations/s3.js b/packages/server/src/integrations/s3.js
new file mode 100644
index 0000000000..24e4edfa1d
--- /dev/null
+++ b/packages/server/src/integrations/s3.js
@@ -0,0 +1,48 @@
+const AWS = require("aws-sdk")
+
+const S3_OPTIONS = {
+  bucket: {
+    type: "string",
+    required: true,
+  },
+  region: {
+    type: "string",
+    required: true,
+    default: "us-east-1",
+  },
+  accessKeyId: {
+    type: "string",
+    required: true,
+  },
+  secretAccessKey: {
+    type: "secretKey",
+    required: true,
+    default: 5432,
+  },
+}
+
+class S3Integration {
+  constructor(config) {
+    this.config = config
+    this.connect()
+    this.client = new AWS.S3()
+  }
+
+  async connect() {
+    AWS.config.update(this.config)
+  }
+
+  async query() {
+    const response = await this.client
+      .listObjects({
+        Bucket: this.config.bucket,
+      })
+      .promise()
+    return response.Contents
+  }
+}
+
+module.exports = {
+  schema: S3_OPTIONS,
+  integration: S3Integration,
+}