Merge pull request #1070 from Budibase/bug/integration-fixes

Some data source/integration fixes

packages/builder/src/builderStore/store/backend.js

@@ -119,6 +119,18 @@ export const getBackendUiStore = () => {
         return json
       },
       save: async (datasourceId, query) => {
+        const integrations = get(store).integrations
+        const dataSource = get(store).datasources.filter(
+          ds => ds._id === datasourceId
+        )
+        // check if readable attribute is found
+        if (dataSource.length !== 0) {
+          const integration = integrations[dataSource[0].source]
+          const readable = integration.query[query.queryVerb].readable
+          if (readable) {
+            query.readable = readable
+          }
+        }
         query.datasourceId = datasourceId
         const response = await api.post(`/api/queries`, query)
         const json = await response.json()

packages/builder/src/components/design/PropertiesPanel/PropertyControls/TableViewSelect.svelte

@@ -35,7 +35,7 @@
     return [...acc, ...viewsArr]
   }, [])
   $: queries = $backendUiStore.queries
-    .filter(query => query.queryVerb === "read")
+    .filter(query => query.queryVerb === "read" || query.readable)
     .map(query => ({
       label: query.name,
       name: query.name,

packages/server/src/api/controllers/query.js

@@ -1,8 +1,18 @@
-const handlebars = require("handlebars")
+const { processString } = require("@budibase/string-templates")
 const CouchDB = require("../../db")
 const { generateQueryID, getQueryParams } = require("../../db/utils")
 const { integrations } = require("../../integrations")
 
+function formatResponse(resp) {
+  if (typeof resp === "string") {
+    resp = JSON.parse(resp)
+  }
+  if (!Array.isArray(resp)) {
+    resp = [resp]
+  }
+  return resp
+}
+
 exports.fetch = async function(ctx) {
   const db = new CouchDB(ctx.user.appId)
 
@@ -29,19 +39,22 @@ exports.save = async function(ctx) {
   ctx.message = `Query ${query.name} saved successfully.`
 }
 
-function enrichQueryFields(fields, parameters) {
+async function enrichQueryFields(fields, parameters) {
   const enrichedQuery = {}
 
   // enrich the fields with dynamic parameters
-  for (let key in fields) {
+  for (let key of Object.keys(fields)) {
-    const template = handlebars.compile(fields[key])
+    enrichedQuery[key] = await processString(fields[key], parameters)
-    enrichedQuery[key] = template(parameters)
   }
 
   if (enrichedQuery.json || enrichedQuery.customData) {
-    enrichedQuery.json = JSON.parse(
+    try {
-      enrichedQuery.json || enrichedQuery.customData
+      enrichedQuery.json = JSON.parse(
-    )
+        enrichedQuery.json || enrichedQuery.customData
+      )
+    } catch (err) {
+      throw { message: `JSON Invalid - error: ${err}` }
+    }
     delete enrichedQuery.customData
   }
 
@@ -62,9 +75,11 @@ exports.preview = async function(ctx) {
 
   const { fields, parameters, queryVerb } = ctx.request.body
 
-  const enrichedQuery = enrichQueryFields(fields, parameters)
+  const enrichedQuery = await enrichQueryFields(fields, parameters)
 
-  ctx.body = await new Integration(datasource.config)[queryVerb](enrichedQuery)
+  ctx.body = formatResponse(
+    await new Integration(datasource.config)[queryVerb](enrichedQuery)
+  )
 }
 
 exports.execute = async function(ctx) {
@@ -80,17 +95,15 @@ exports.execute = async function(ctx) {
     return
   }
 
-  const enrichedQuery = enrichQueryFields(
+  const enrichedQuery = await enrichQueryFields(
     query.fields,
     ctx.request.body.parameters
   )
 
   // call the relevant CRUD method on the integration class
-  const response = await new Integration(datasource.config)[query.queryVerb](
+  ctx.body = formatResponse(
-    enrichedQuery
+    await new Integration(datasource.config)[query.queryVerb](enrichedQuery)
   )
-
-  ctx.body = response
 }
 
 exports.destroy = async function(ctx) {

packages/server/src/api/routes/query.js

@@ -26,6 +26,7 @@ function generateQueryValidation() {
     name: Joi.string().required(),
     fields: Joi.object().required(),
     datasourceId: Joi.string().required(),
+    readable: Joi.boolean(),
     parameters: Joi.array().items(Joi.object({
       name: Joi.string(),
       default: Joi.string()

packages/server/src/integrations/dynamodb.js

@@ -17,20 +17,27 @@ const SCHEMA = {
       type: FIELD_TYPES.PASSWORD,
       required: true,
     },
+    endpoint: {
+      type: FIELD_TYPES.STRING,
+      required: false,
+      default: "https://dynamodb.us-east-1.amazonaws.com",
+    },
   },
   query: {
     create: {
       type: QUERY_TYPES.FIELDS,
+      customisable: true,
       fields: {
         table: {
           type: FIELD_TYPES.STRING,
           required: true,
         },
-        customisable: true,
       },
     },
     read: {
       type: QUERY_TYPES.FIELDS,
+      customisable: true,
+      readable: true,
       fields: {
         table: {
           type: FIELD_TYPES.STRING,
@@ -39,30 +46,51 @@ const SCHEMA = {
         index: {
           type: FIELD_TYPES.STRING,
         },
-        customisable: true,
+      },
+    },
+    scan: {
+      type: QUERY_TYPES.FIELDS,
+      customisable: true,
+      readable: true,
+      fields: {
+        table: {
+          type: FIELD_TYPES.STRING,
+          required: true,
+        },
+        index: {
+          type: FIELD_TYPES.STRING,
+        },
+      },
+    },
+    get: {
+      type: QUERY_TYPES.FIELDS,
+      customisable: true,
+      readable: true,
+      fields: {
+        table: {
+          type: FIELD_TYPES.STRING,
+          required: true,
+        },
       },
     },
     update: {
       type: QUERY_TYPES.FIELDS,
+      customisable: true,
       fields: {
         table: {
           type: FIELD_TYPES.STRING,
           required: true,
         },
-        customisable: true,
       },
     },
     delete: {
       type: QUERY_TYPES.FIELDS,
+      customisable: true,
       fields: {
         table: {
           type: FIELD_TYPES.STRING,
           required: true,
         },
-        key: {
-          type: FIELD_TYPES.STRING,
-          required: true,
-        },
       },
     },
   },
@@ -72,7 +100,15 @@ class DynamoDBIntegration {
   constructor(config) {
     this.config = config
     this.connect()
-    this.client = new AWS.DynamoDB.DocumentClient()
+    let options = {
+      correctClockSkew: true,
+    }
+    if (config.endpoint) {
+      options.endpoint = config.endpoint
+    }
+    this.client = new AWS.DynamoDB.DocumentClient({
+      correctClockSkew: true,
+    })
   }
 
   async connect() {
@@ -80,37 +116,65 @@ class DynamoDBIntegration {
   }
 
   async create(query) {
-    const response = await this.client.query({
+    const params = {
       TableName: query.table,
-      Item: query.json,
+      ...query.json,
-    })
+    }
-    return response
+    return this.client.put(params).promise()
   }
 
   async read(query) {
-    const response = await this.client.query({
+    const params = {
-      TableName: query.Table,
+      TableName: query.table,
       ...query.json,
-    })
+    }
+    if (query.index) {
+      params.IndexName = query.index
+    }
+    const response = await this.client.query(params).promise()
+    if (response.Items) {
+      return response.Items
+    }
     return response
   }
 
+  async scan(query) {
+    const params = {
+      TableName: query.table,
+      ...query.json,
+    }
+    if (query.index) {
+      params.IndexName = query.index
+    }
+    const response = await this.client.scan(params).promise()
+    if (response.Items) {
+      return response.Items
+    }
+    return response
+  }
+
+  async get(query) {
+    const params = {
+      TableName: query.table,
+      ...query.json,
+    }
+    return this.client.get(params).promise()
+  }
+
   async update(query) {
-    const response = await this.client.query({
+    const params = {
       TableName: query.Table,
       ...query.json,
-    })
+    }
-    return response
+    return this.client.update(params).promise()
   }
 
   async delete(query) {
-    const response = await this.client.query({
+    const params = {
-      TableName: query.Table,
+      TableName: query.table,
-      Key: {
+      ...query.json,
-        id: query.key,
+    }
-      },
+    return this.client.delete(params).promise()
-    })
-    return response
   }
 }
 

packages/standard-components/yarn.lock

@@ -40,9 +40,9 @@
     to-fast-properties "^2.0.0"
 
 "@budibase/bbui@^1.55.1":
-  version "1.55.1"
+  version "1.56.2"
-  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.55.1.tgz#291fb6fa10479b49f078d3a911ad0ed42c2e6b12"
+  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.56.2.tgz#bb8f7d9b9b5ed06a22df877fbe028780d7602471"
-  integrity sha512-bxsHBwkOqCtuFz89e0hAXwvwycfS4xPPrEge5PxK1Lh3uqetO4bXoIxYaIDjfi2Ku7CYIzEmOwSloNaQWeTF4g==
+  integrity sha512-cWYkT1FNwNGTjisxtC5/MlQ1zeu7MYbMJsD6UyCEW3Ku6JIQZ6jyOkV6HKrmNND8VzVfddEGpzR37q+NoDpDFQ==
   dependencies:
     markdown-it "^12.0.2"
     quill "^1.3.7"
@@ -1034,7 +1034,12 @@ fill-range@^7.0.1:
   dependencies:
     to-regex-range "^5.0.1"
 
-flatpickr@^4.5.2, flatpickr@^4.6.6:
+flatpickr@^4.5.2:
+  version "4.6.9"
+  resolved "https://registry.yarnpkg.com/flatpickr/-/flatpickr-4.6.9.tgz#9a13383e8a6814bda5d232eae3fcdccb97dc1499"
+  integrity sha512-F0azNNi8foVWKSF+8X+ZJzz8r9sE1G4hl06RyceIaLvyltKvDl6vqk9Lm/6AUUCi5HWaIjiUbk7UpeE/fOXOpw==
+
+flatpickr@^4.6.6:
   version "4.6.6"
   resolved "https://registry.yarnpkg.com/flatpickr/-/flatpickr-4.6.6.tgz#34d2ad80adfa34254e62583a34264d472f1038d6"
   integrity sha512-EZ48CJMttMg3maMhJoX+GvTuuEhX/RbA1YeuI19attP3pwBdbYy6+yqAEVm0o0hSBFYBiLbVxscLW6gJXq6H3A==
@@ -1558,9 +1563,9 @@ loader-utils@^1.1.0:
     json5 "^1.0.1"
 
 local-access@^1.0.1:
-  version "1.0.1"
+  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/local-access/-/local-access-1.0.1.tgz#5121258146d64e869046c642ea4f1dd39ff942bb"
+  resolved "https://registry.yarnpkg.com/local-access/-/local-access-1.1.0.tgz#e007c76ba2ca83d5877ba1a125fc8dfe23ba4798"
-  integrity sha512-ykt2pgN0aqIy6KQC1CqdWTWkmUwNgaOS6dcpHVjyBJONA+Xi7AtSB1vuxC/U/0tjIP3wcRudwQk1YYzUvzk2bA==
+  integrity sha512-XfegD5pyTAfb+GY6chk283Ox5z8WexG56OvM06RWLpAc/UHozO8X6xAxEkIitZOtsSMM1Yr3DkHgW5W+onLhCw==
 
 lodash.camelcase@^4.3.0:
   version "4.3.0"
@@ -1648,9 +1653,9 @@ miller-rabin@^4.0.0:
     brorand "^1.0.1"
 
 mime@^2.3.1:
-  version "2.4.6"
+  version "2.5.0"
-  resolved "https://registry.yarnpkg.com/mime/-/mime-2.4.6.tgz#e5b407c90db442f2beb5b162373d07b69affa4d1"
+  resolved "https://registry.yarnpkg.com/mime/-/mime-2.5.0.tgz#2b4af934401779806ee98026bb42e8c1ae1876b1"
-  integrity sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==
+  integrity sha512-ft3WayFSFUVBuJj7BMLKAQcSlItKtfjsKDDsii3rqFDAZ7t11zRe8ASw/GlmivGwVUYtwkQrxiGGpL6gFvB0ag==
 
 minimalistic-assert@^1.0.0, minimalistic-assert@^1.0.1:
   version "1.0.1"
@@ -2481,9 +2486,9 @@ rollup@^2.11.2:
     fsevents "~2.1.2"
 
 sade@^1.4.0:
-  version "1.7.3"
+  version "1.7.4"
-  resolved "https://registry.yarnpkg.com/sade/-/sade-1.7.3.tgz#a217ccc4fb4abb2d271648bf48f6628b2636fa1b"
+  resolved "https://registry.yarnpkg.com/sade/-/sade-1.7.4.tgz#ea681e0c65d248d2095c90578c03ca0bb1b54691"
-  integrity sha512-m4BctppMvJ60W1dXnHq7jMmFe3hPJZDAH85kQ3ACTo7XZNVUuTItCQ+2HfyaMeV5cKrbw7l4vD/6We3GBxvdJw==
+  integrity sha512-y5yauMD93rX840MwUJr7C1ysLFBgMspsdTo4UVrDg3fXDvtwOyIqykhVAAm6fk/3au77773itJStObgK+LKaiA==
   dependencies:
     mri "^1.1.0"
 

packages/string-templates/test/helpers.spec.js

@@ -302,6 +302,11 @@ describe("Cover a few complex use cases", () => {
     expect(validity).toBe(true)
   })
 
+  it("should make sure object functions check out valid", () => {
+    const validity = isValid("{{ JSONstringify obj }}")
+    expect(validity).toBe(true)
+  })
+
   it("should be able to solve an example from docs", async () => {
     const output = await processString(`{{first ( split "a-b-c" "-") 2}}`, {})
     expect(output).toBe(`a,b`)