Merge pull request #15664 from Budibase/demock-dynamodb

De-mock dynamodb

.github/workflows/budibase_ci.yml

@@ -165,6 +165,7 @@ jobs:
             oracle,
             sqs,
             elasticsearch,
+            dynamodb,
             none,
           ]
     steps:
@@ -205,6 +206,8 @@ jobs:
             docker pull postgres:9.5.25
           elif [ "${{ matrix.datasource }}" == "elasticsearch" ]; then
             docker pull elasticsearch@${{ steps.dotenv.outputs.ELASTICSEARCH_SHA }}
+          elif [ "${{ matrix.datasource }}" == "dynamodb" ]; then
+            docker pull amazon/dynamodb-local@${{ steps.dotenv.outputs.DYNAMODB_SHA }}
           fi
           docker pull minio/minio &
           docker pull redis &

packages/server/datasource-sha.env

@@ -3,4 +3,5 @@ MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588eb
 POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e
 MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d
 MARIADB_SHA=sha256:e59ba8783bf7bc02a4779f103bb0d8751ac0e10f9471089709608377eded7aa8
-ELASTICSEARCH_SHA=sha256:9a6443f55243f6acbfeb4a112d15eb3b9aac74bf25e0e39fa19b3ddd3a6879d0
+ELASTICSEARCH_SHA=sha256:9a6443f55243f6acbfeb4a112d15eb3b9aac74bf25e0e39fa19b3ddd3a6879d0
+DYNAMODB_SHA=sha256:cf8cebd061f988628c02daff10fdb950a54478feff9c52f6ddf84710fe3c3906

packages/server/src/integrations/dynamodb.ts

@@ -14,15 +14,14 @@ import {
   UpdateCommandInput,
   DeleteCommandInput,
 } from "@aws-sdk/lib-dynamodb"
-import { DynamoDB } from "@aws-sdk/client-dynamodb"
+import { DynamoDB, DynamoDBClientConfig } from "@aws-sdk/client-dynamodb"
 import { AWS_REGION } from "../constants"
 
-interface DynamoDBConfig {
+export interface DynamoDBConfig {
   region: string
   accessKeyId: string
   secretAccessKey: string
   endpoint?: string
-  currentClockSkew?: boolean
 }
 
 const SCHEMA: Integration = {
@@ -138,22 +137,16 @@ const SCHEMA: Integration = {
   },
 }
 
-class DynamoDBIntegration implements IntegrationBase {
-  private config: DynamoDBConfig
-  private client
+export class DynamoDBIntegration implements IntegrationBase {
+  private config: DynamoDBClientConfig
+  private client: DynamoDBDocument
 
   constructor(config: DynamoDBConfig) {
-    this.config = config
-
-    // User is using a local dynamoDB endpoint, don't auth with remote
-    if (this.config?.endpoint?.includes("localhost")) {
-      // @ts-ignore
-      this.config = {}
-    }
-
     this.config = {
-      ...this.config,
-      currentClockSkew: true,
+      credentials: {
+        accessKeyId: config.accessKeyId,
+        secretAccessKey: config.secretAccessKey,
+      },
       region: config.region || AWS_REGION,
       endpoint: config.endpoint || undefined,
     }

packages/server/src/integrations/tests/dynamodb.spec.ts

@@ -1,167 +1,108 @@
-jest.mock("@aws-sdk/lib-dynamodb", () => ({
-  DynamoDBDocument: {
-    from: jest.fn(() => ({
-      update: jest.fn(),
-      put: jest.fn(),
-      query: jest.fn(() => ({
-        Items: [],
-      })),
-      scan: jest.fn(() => ({
-        Items: [],
-      })),
-      delete: jest.fn(),
-      get: jest.fn(),
-    })),
-  },
-}))
-jest.mock("@aws-sdk/client-dynamodb")
-import { default as DynamoDBIntegration } from "../dynamodb"
+import { Datasource } from "@budibase/types"
+import { DynamoDBConfig, DynamoDBIntegration } from "../dynamodb"
+import { DatabaseName, datasourceDescribe } from "./utils"
+import {
+  CreateTableCommandInput,
+  DynamoDB,
+  DynamoDBClientConfig,
+} from "@aws-sdk/client-dynamodb"
 
-class TestConfiguration {
-  integration: any
+const describes = datasourceDescribe({ only: [DatabaseName.DYNAMODB] })
 
-  constructor(config: any = {}) {
-    this.integration = new DynamoDBIntegration.integration(config)
+async function createTable(client: DynamoDB, req: CreateTableCommandInput) {
+  try {
+    await client.deleteTable({ TableName: req.TableName })
+  } catch (e: any) {
+    if (e.name !== "ResourceNotFoundException") {
+      throw e
+    }
   }
+
+  return await client.createTable(req)
 }
 
-describe("DynamoDB Integration", () => {
-  let config: any
-  let tableName = "Users"
+if (describes.length > 0) {
+  describe.each(describes)("DynamoDB Integration", ({ dsProvider }) => {
+    let table = "Users"
+    let rawDatasource: Datasource
+    let dynamodb: DynamoDBIntegration
 
-  beforeEach(() => {
-    config = new TestConfiguration()
-  })
-
-  it("calls the create method with the correct params", async () => {
-    await config.integration.create({
-      table: tableName,
-      json: {
-        Name: "John",
-      },
-    })
-    expect(config.integration.client.put).toHaveBeenCalledWith({
-      TableName: tableName,
-      Name: "John",
-    })
-  })
-
-  it("calls the read method with the correct params", async () => {
-    const indexName = "Test"
-
-    const response = await config.integration.read({
-      table: tableName,
-      index: indexName,
-      json: {},
-    })
-    expect(config.integration.client.query).toHaveBeenCalledWith({
-      TableName: tableName,
-      IndexName: indexName,
-    })
-    expect(response).toEqual([])
-  })
-
-  it("calls the scan method with the correct params", async () => {
-    const indexName = "Test"
-
-    const response = await config.integration.scan({
-      table: tableName,
-      index: indexName,
-      json: {},
-    })
-    expect(config.integration.client.scan).toHaveBeenCalledWith({
-      TableName: tableName,
-      IndexName: indexName,
-    })
-    expect(response).toEqual([])
-  })
-
-  it("calls the get method with the correct params", async () => {
-    await config.integration.get({
-      table: tableName,
-      json: {
-        Id: 123,
-      },
-    })
-
-    expect(config.integration.client.get).toHaveBeenCalledWith({
-      TableName: tableName,
-      Id: 123,
-    })
-  })
-
-  it("calls the update method with the correct params", async () => {
-    await config.integration.update({
-      table: tableName,
-      json: {
-        Name: "John",
-      },
-    })
-    expect(config.integration.client.update).toHaveBeenCalledWith({
-      TableName: tableName,
-      Name: "John",
-    })
-  })
-
-  it("calls the delete method with the correct params", async () => {
-    await config.integration.delete({
-      table: tableName,
-      json: {
-        Name: "John",
-      },
-    })
-    expect(config.integration.client.delete).toHaveBeenCalledWith({
-      TableName: tableName,
-      Name: "John",
-    })
-  })
-
-  it("configures the dynamoDB constructor based on an empty endpoint parameter", async () => {
-    const config = {
-      region: "us-east-1",
-      accessKeyId: "test",
-      secretAccessKey: "test",
+    function item(json: Record<string, any>) {
+      return { table, json: { Item: json } }
     }
 
-    const integration: any = new DynamoDBIntegration.integration(config)
-
-    expect(integration.config).toEqual({
-      currentClockSkew: true,
-      ...config,
-    })
-  })
-
-  it("configures the dynamoDB constructor based on a localhost endpoint parameter", async () => {
-    const config = {
-      region: "us-east-1",
-      accessKeyId: "test",
-      secretAccessKey: "test",
-      endpoint: "localhost:8080",
+    function key(json: Record<string, any>) {
+      return { table, json: { Key: json } }
     }
 
-    const integration: any = new DynamoDBIntegration.integration(config)
+    beforeEach(async () => {
+      const ds = await dsProvider()
+      rawDatasource = ds.rawDatasource!
+      dynamodb = new DynamoDBIntegration(
+        rawDatasource.config! as DynamoDBConfig
+      )
 
-    expect(integration.config).toEqual({
-      region: "us-east-1",
-      currentClockSkew: true,
-      endpoint: "localhost:8080",
+      const config: DynamoDBClientConfig = {
+        credentials: {
+          accessKeyId: "test",
+          secretAccessKey: "test",
+        },
+        region: "us-east-1",
+        endpoint: rawDatasource.config!.endpoint,
+      }
+
+      const client = new DynamoDB(config)
+      await createTable(client, {
+        TableName: table,
+        KeySchema: [{ AttributeName: "Id", KeyType: "HASH" }],
+        AttributeDefinitions: [{ AttributeName: "Id", AttributeType: "N" }],
+        ProvisionedThroughput: { ReadCapacityUnits: 1, WriteCapacityUnits: 1 },
+      })
+    })
+
+    it("can create and read a record", async () => {
+      await dynamodb.create(item({ Id: 1, Name: "John" }))
+
+      const resp = await dynamodb.get(key({ Id: 1 }))
+      expect(resp.Item).toEqual({ Id: 1, Name: "John" })
+    })
+
+    it("can scan", async () => {
+      await dynamodb.create(item({ Id: 1, Name: "John" }))
+      await dynamodb.create(item({ Id: 2, Name: "Jane" }))
+      await dynamodb.create(item({ Id: 3, Name: "Jack" }))
+
+      const resp = await dynamodb.scan({ table, json: {}, index: null })
+      expect(resp).toEqual(
+        expect.arrayContaining([
+          { Id: 1, Name: "John" },
+          { Id: 2, Name: "Jane" },
+          { Id: 3, Name: "Jack" },
+        ])
+      )
+    })
+
+    it("can update", async () => {
+      await dynamodb.create(item({ Id: 1, Foo: "John" }))
+      await dynamodb.update({
+        table,
+        json: {
+          Key: { Id: 1 },
+          UpdateExpression: "SET Foo = :foo",
+          ExpressionAttributeValues: { ":foo": "Jane" },
+        },
+      })
+
+      const updatedRecord = await dynamodb.get(key({ Id: 1 }))
+      expect(updatedRecord.Item).toEqual({ Id: 1, Foo: "Jane" })
+    })
+
+    it("can delete", async () => {
+      await dynamodb.create(item({ Id: 1, Name: "John" }))
+      await dynamodb.delete(key({ Id: 1 }))
+
+      const deletedRecord = await dynamodb.get(key({ Id: 1 }))
+      expect(deletedRecord.Item).toBeUndefined()
     })
   })
-
-  it("configures the dynamoDB constructor based on a remote endpoint parameter", async () => {
-    const config = {
-      region: "us-east-1",
-      accessKeyId: "test",
-      secretAccessKey: "test",
-      endpoint: "dynamodb.aws.foo.net",
-    }
-
-    const integration = new DynamoDBIntegration.integration(config)
-
-    // @ts-ignore
-    expect(integration.config).toEqual({
-      currentClockSkew: true,
-      ...config,
-    })
-  })
-})
+}

packages/server/src/integrations/tests/utils/dynamodb.ts

@@ -0,0 +1,41 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait } from "testcontainers"
+import { testContainerUtils } from "@budibase/backend-core/tests"
+import { startContainer } from "."
+import { DYNAMODB_IMAGE } from "./images"
+import { DynamoDBConfig } from "../../dynamodb"
+
+let ports: Promise<testContainerUtils.Port[]>
+
+export async function getDatasource(): Promise<Datasource> {
+  if (!ports) {
+    ports = startContainer(
+      new GenericContainer(DYNAMODB_IMAGE)
+        .withExposedPorts(8000)
+        .withWaitStrategy(
+          Wait.forSuccessfulCommand(
+            // https://stackoverflow.com/a/77373799
+            `if [ "$(curl -s -o /dev/null -I -w ''%{http_code}'' http://localhost:8000)" == "400" ]; then exit 0; else exit 1; fi`
+          ).withStartupTimeout(60000)
+        )
+    )
+  }
+
+  const port = (await ports).find(x => x.container === 8000)?.host
+  if (!port) {
+    throw new Error("DynamoDB port not found")
+  }
+
+  const config: DynamoDBConfig = {
+    accessKeyId: "test",
+    secretAccessKey: "test",
+    region: "us-east-1",
+    endpoint: `http://127.0.0.1:${port}`,
+  }
+
+  return {
+    type: "datasource",
+    source: SourceName.DYNAMODB,
+    config,
+  }
+}

packages/server/src/integrations/tests/utils/images.ts

@@ -13,3 +13,4 @@ export const POSTGRES_LEGACY_IMAGE = `postgres:9.5.25`
 export const MONGODB_IMAGE = `mongo@${process.env.MONGODB_SHA}`
 export const MARIADB_IMAGE = `mariadb@${process.env.MARIADB_SHA}`
 export const ELASTICSEARCH_IMAGE = `elasticsearch@${process.env.ELASTICSEARCH_SHA}`
+export const DYNAMODB_IMAGE = `amazon/dynamodb-local@${process.env.DYNAMODB_SHA}`

packages/server/src/integrations/tests/utils/index.ts

@@ -7,6 +7,7 @@ import * as mssql from "./mssql"
 import * as mariadb from "./mariadb"
 import * as oracle from "./oracle"
 import * as elasticsearch from "./elasticsearch"
+import * as dynamodb from "./dynamodb"
 import { testContainerUtils } from "@budibase/backend-core/tests"
 import { Knex } from "knex"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
@@ -25,6 +26,7 @@ export enum DatabaseName {
   ORACLE = "oracle",
   SQS = "sqs",
   ELASTICSEARCH = "elasticsearch",
+  DYNAMODB = "dynamodb",
 }
 
 const DATASOURCE_PLUS = [
@@ -50,6 +52,7 @@ const providers: Record<DatabaseName, DatasourceProvider> = {
   // rest
   [DatabaseName.ELASTICSEARCH]: elasticsearch.getDatasource,
   [DatabaseName.MONGODB]: mongodb.getDatasource,
+  [DatabaseName.DYNAMODB]: dynamodb.getDatasource,
 }
 
 export interface DatasourceDescribeReturnPromise {

packages/worker/src/tests/jestSetup.ts

@@ -12,8 +12,7 @@ nock.enableNetConnect(host => {
   return (
     host.includes("localhost") ||
     host.includes("127.0.0.1") ||
-    host.includes("::1") ||
-    host.includes("ethereal.email") // used in realEmail.spec.ts
+    host.includes("::1")
   )
 })