diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 2e7851b338..c8bdfe9655 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -165,6 +165,7 @@ jobs: oracle, sqs, elasticsearch, + dynamodb, none, ] steps: @@ -205,6 +206,8 @@ jobs: docker pull postgres:9.5.25 elif [ "${{ matrix.datasource }}" == "elasticsearch" ]; then docker pull elasticsearch@${{ steps.dotenv.outputs.ELASTICSEARCH_SHA }} + elif [ "${{ matrix.datasource }}" == "dynamodb" ]; then + docker pull amazon/dynamodb-local@${{ steps.dotenv.outputs.DYNAMODB_SHA }} fi docker pull minio/minio & docker pull redis & diff --git a/packages/server/datasource-sha.env b/packages/server/datasource-sha.env index 69750793ce..13413dcb59 100644 --- a/packages/server/datasource-sha.env +++ b/packages/server/datasource-sha.env @@ -3,4 +3,5 @@ MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588eb POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d MARIADB_SHA=sha256:e59ba8783bf7bc02a4779f103bb0d8751ac0e10f9471089709608377eded7aa8 -ELASTICSEARCH_SHA=sha256:9a6443f55243f6acbfeb4a112d15eb3b9aac74bf25e0e39fa19b3ddd3a6879d0 \ No newline at end of file +ELASTICSEARCH_SHA=sha256:9a6443f55243f6acbfeb4a112d15eb3b9aac74bf25e0e39fa19b3ddd3a6879d0 +DYNAMODB_SHA=sha256:cf8cebd061f988628c02daff10fdb950a54478feff9c52f6ddf84710fe3c3906 \ No newline at end of file diff --git a/packages/server/src/integrations/dynamodb.ts b/packages/server/src/integrations/dynamodb.ts index 96941ebb0e..cfaa851cea 100644 --- a/packages/server/src/integrations/dynamodb.ts +++ b/packages/server/src/integrations/dynamodb.ts @@ -14,15 +14,14 @@ import { UpdateCommandInput, DeleteCommandInput, } from "@aws-sdk/lib-dynamodb" -import { DynamoDB } from "@aws-sdk/client-dynamodb" +import { DynamoDB, DynamoDBClientConfig } from "@aws-sdk/client-dynamodb" import { AWS_REGION } from "../constants" -interface DynamoDBConfig { +export interface DynamoDBConfig { region: string accessKeyId: string secretAccessKey: string endpoint?: string - currentClockSkew?: boolean } const SCHEMA: Integration = { @@ -138,22 +137,16 @@ const SCHEMA: Integration = { }, } -class DynamoDBIntegration implements IntegrationBase { - private config: DynamoDBConfig - private client +export class DynamoDBIntegration implements IntegrationBase { + private config: DynamoDBClientConfig + private client: DynamoDBDocument constructor(config: DynamoDBConfig) { - this.config = config - - // User is using a local dynamoDB endpoint, don't auth with remote - if (this.config?.endpoint?.includes("localhost")) { - // @ts-ignore - this.config = {} - } - this.config = { - ...this.config, - currentClockSkew: true, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, region: config.region || AWS_REGION, endpoint: config.endpoint || undefined, } diff --git a/packages/server/src/integrations/tests/dynamodb.spec.ts b/packages/server/src/integrations/tests/dynamodb.spec.ts index 75fb84ae60..e6b1ed405c 100644 --- a/packages/server/src/integrations/tests/dynamodb.spec.ts +++ b/packages/server/src/integrations/tests/dynamodb.spec.ts @@ -1,167 +1,108 @@ -jest.mock("@aws-sdk/lib-dynamodb", () => ({ - DynamoDBDocument: { - from: jest.fn(() => ({ - update: jest.fn(), - put: jest.fn(), - query: jest.fn(() => ({ - Items: [], - })), - scan: jest.fn(() => ({ - Items: [], - })), - delete: jest.fn(), - get: jest.fn(), - })), - }, -})) -jest.mock("@aws-sdk/client-dynamodb") -import { default as DynamoDBIntegration } from "../dynamodb" +import { Datasource } from "@budibase/types" +import { DynamoDBConfig, DynamoDBIntegration } from "../dynamodb" +import { DatabaseName, datasourceDescribe } from "./utils" +import { + CreateTableCommandInput, + DynamoDB, + DynamoDBClientConfig, +} from "@aws-sdk/client-dynamodb" -class TestConfiguration { - integration: any +const describes = datasourceDescribe({ only: [DatabaseName.DYNAMODB] }) - constructor(config: any = {}) { - this.integration = new DynamoDBIntegration.integration(config) +async function createTable(client: DynamoDB, req: CreateTableCommandInput) { + try { + await client.deleteTable({ TableName: req.TableName }) + } catch (e: any) { + if (e.name !== "ResourceNotFoundException") { + throw e + } } + + return await client.createTable(req) } -describe("DynamoDB Integration", () => { - let config: any - let tableName = "Users" +if (describes.length > 0) { + describe.each(describes)("DynamoDB Integration", ({ dsProvider }) => { + let table = "Users" + let rawDatasource: Datasource + let dynamodb: DynamoDBIntegration - beforeEach(() => { - config = new TestConfiguration() - }) - - it("calls the create method with the correct params", async () => { - await config.integration.create({ - table: tableName, - json: { - Name: "John", - }, - }) - expect(config.integration.client.put).toHaveBeenCalledWith({ - TableName: tableName, - Name: "John", - }) - }) - - it("calls the read method with the correct params", async () => { - const indexName = "Test" - - const response = await config.integration.read({ - table: tableName, - index: indexName, - json: {}, - }) - expect(config.integration.client.query).toHaveBeenCalledWith({ - TableName: tableName, - IndexName: indexName, - }) - expect(response).toEqual([]) - }) - - it("calls the scan method with the correct params", async () => { - const indexName = "Test" - - const response = await config.integration.scan({ - table: tableName, - index: indexName, - json: {}, - }) - expect(config.integration.client.scan).toHaveBeenCalledWith({ - TableName: tableName, - IndexName: indexName, - }) - expect(response).toEqual([]) - }) - - it("calls the get method with the correct params", async () => { - await config.integration.get({ - table: tableName, - json: { - Id: 123, - }, - }) - - expect(config.integration.client.get).toHaveBeenCalledWith({ - TableName: tableName, - Id: 123, - }) - }) - - it("calls the update method with the correct params", async () => { - await config.integration.update({ - table: tableName, - json: { - Name: "John", - }, - }) - expect(config.integration.client.update).toHaveBeenCalledWith({ - TableName: tableName, - Name: "John", - }) - }) - - it("calls the delete method with the correct params", async () => { - await config.integration.delete({ - table: tableName, - json: { - Name: "John", - }, - }) - expect(config.integration.client.delete).toHaveBeenCalledWith({ - TableName: tableName, - Name: "John", - }) - }) - - it("configures the dynamoDB constructor based on an empty endpoint parameter", async () => { - const config = { - region: "us-east-1", - accessKeyId: "test", - secretAccessKey: "test", + function item(json: Record) { + return { table, json: { Item: json } } } - const integration: any = new DynamoDBIntegration.integration(config) - - expect(integration.config).toEqual({ - currentClockSkew: true, - ...config, - }) - }) - - it("configures the dynamoDB constructor based on a localhost endpoint parameter", async () => { - const config = { - region: "us-east-1", - accessKeyId: "test", - secretAccessKey: "test", - endpoint: "localhost:8080", + function key(json: Record) { + return { table, json: { Key: json } } } - const integration: any = new DynamoDBIntegration.integration(config) + beforeEach(async () => { + const ds = await dsProvider() + rawDatasource = ds.rawDatasource! + dynamodb = new DynamoDBIntegration( + rawDatasource.config! as DynamoDBConfig + ) - expect(integration.config).toEqual({ - region: "us-east-1", - currentClockSkew: true, - endpoint: "localhost:8080", + const config: DynamoDBClientConfig = { + credentials: { + accessKeyId: "test", + secretAccessKey: "test", + }, + region: "us-east-1", + endpoint: rawDatasource.config!.endpoint, + } + + const client = new DynamoDB(config) + await createTable(client, { + TableName: table, + KeySchema: [{ AttributeName: "Id", KeyType: "HASH" }], + AttributeDefinitions: [{ AttributeName: "Id", AttributeType: "N" }], + ProvisionedThroughput: { ReadCapacityUnits: 1, WriteCapacityUnits: 1 }, + }) + }) + + it("can create and read a record", async () => { + await dynamodb.create(item({ Id: 1, Name: "John" })) + + const resp = await dynamodb.get(key({ Id: 1 })) + expect(resp.Item).toEqual({ Id: 1, Name: "John" }) + }) + + it("can scan", async () => { + await dynamodb.create(item({ Id: 1, Name: "John" })) + await dynamodb.create(item({ Id: 2, Name: "Jane" })) + await dynamodb.create(item({ Id: 3, Name: "Jack" })) + + const resp = await dynamodb.scan({ table, json: {}, index: null }) + expect(resp).toEqual( + expect.arrayContaining([ + { Id: 1, Name: "John" }, + { Id: 2, Name: "Jane" }, + { Id: 3, Name: "Jack" }, + ]) + ) + }) + + it("can update", async () => { + await dynamodb.create(item({ Id: 1, Foo: "John" })) + await dynamodb.update({ + table, + json: { + Key: { Id: 1 }, + UpdateExpression: "SET Foo = :foo", + ExpressionAttributeValues: { ":foo": "Jane" }, + }, + }) + + const updatedRecord = await dynamodb.get(key({ Id: 1 })) + expect(updatedRecord.Item).toEqual({ Id: 1, Foo: "Jane" }) + }) + + it("can delete", async () => { + await dynamodb.create(item({ Id: 1, Name: "John" })) + await dynamodb.delete(key({ Id: 1 })) + + const deletedRecord = await dynamodb.get(key({ Id: 1 })) + expect(deletedRecord.Item).toBeUndefined() }) }) - - it("configures the dynamoDB constructor based on a remote endpoint parameter", async () => { - const config = { - region: "us-east-1", - accessKeyId: "test", - secretAccessKey: "test", - endpoint: "dynamodb.aws.foo.net", - } - - const integration = new DynamoDBIntegration.integration(config) - - // @ts-ignore - expect(integration.config).toEqual({ - currentClockSkew: true, - ...config, - }) - }) -}) +} diff --git a/packages/server/src/integrations/tests/utils/dynamodb.ts b/packages/server/src/integrations/tests/utils/dynamodb.ts new file mode 100644 index 0000000000..aae6358ece --- /dev/null +++ b/packages/server/src/integrations/tests/utils/dynamodb.ts @@ -0,0 +1,41 @@ +import { Datasource, SourceName } from "@budibase/types" +import { GenericContainer, Wait } from "testcontainers" +import { testContainerUtils } from "@budibase/backend-core/tests" +import { startContainer } from "." +import { DYNAMODB_IMAGE } from "./images" +import { DynamoDBConfig } from "../../dynamodb" + +let ports: Promise + +export async function getDatasource(): Promise { + if (!ports) { + ports = startContainer( + new GenericContainer(DYNAMODB_IMAGE) + .withExposedPorts(8000) + .withWaitStrategy( + Wait.forSuccessfulCommand( + // https://stackoverflow.com/a/77373799 + `if [ "$(curl -s -o /dev/null -I -w ''%{http_code}'' http://localhost:8000)" == "400" ]; then exit 0; else exit 1; fi` + ).withStartupTimeout(60000) + ) + ) + } + + const port = (await ports).find(x => x.container === 8000)?.host + if (!port) { + throw new Error("DynamoDB port not found") + } + + const config: DynamoDBConfig = { + accessKeyId: "test", + secretAccessKey: "test", + region: "us-east-1", + endpoint: `http://127.0.0.1:${port}`, + } + + return { + type: "datasource", + source: SourceName.DYNAMODB, + config, + } +} diff --git a/packages/server/src/integrations/tests/utils/images.ts b/packages/server/src/integrations/tests/utils/images.ts index c09b130ea5..68d219ac7d 100644 --- a/packages/server/src/integrations/tests/utils/images.ts +++ b/packages/server/src/integrations/tests/utils/images.ts @@ -13,3 +13,4 @@ export const POSTGRES_LEGACY_IMAGE = `postgres:9.5.25` export const MONGODB_IMAGE = `mongo@${process.env.MONGODB_SHA}` export const MARIADB_IMAGE = `mariadb@${process.env.MARIADB_SHA}` export const ELASTICSEARCH_IMAGE = `elasticsearch@${process.env.ELASTICSEARCH_SHA}` +export const DYNAMODB_IMAGE = `amazon/dynamodb-local@${process.env.DYNAMODB_SHA}` diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index 08777cab89..138861a9e6 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -7,6 +7,7 @@ import * as mssql from "./mssql" import * as mariadb from "./mariadb" import * as oracle from "./oracle" import * as elasticsearch from "./elasticsearch" +import * as dynamodb from "./dynamodb" import { testContainerUtils } from "@budibase/backend-core/tests" import { Knex } from "knex" import TestConfiguration from "../../../tests/utilities/TestConfiguration" @@ -25,6 +26,7 @@ export enum DatabaseName { ORACLE = "oracle", SQS = "sqs", ELASTICSEARCH = "elasticsearch", + DYNAMODB = "dynamodb", } const DATASOURCE_PLUS = [ @@ -50,6 +52,7 @@ const providers: Record = { // rest [DatabaseName.ELASTICSEARCH]: elasticsearch.getDatasource, [DatabaseName.MONGODB]: mongodb.getDatasource, + [DatabaseName.DYNAMODB]: dynamodb.getDatasource, } export interface DatasourceDescribeReturnPromise { diff --git a/packages/worker/src/tests/jestSetup.ts b/packages/worker/src/tests/jestSetup.ts index 6a98031d34..77501b2f06 100644 --- a/packages/worker/src/tests/jestSetup.ts +++ b/packages/worker/src/tests/jestSetup.ts @@ -12,8 +12,7 @@ nock.enableNetConnect(host => { return ( host.includes("localhost") || host.includes("127.0.0.1") || - host.includes("::1") || - host.includes("ethereal.email") // used in realEmail.spec.ts + host.includes("::1") ) })