From 4096f4857c9b0ae15a90ba4d2bb89788d3b01657 Mon Sep 17 00:00:00 2001 From: Christo Date: Thu, 8 Jul 2021 14:38:49 +0200 Subject: [PATCH 1/2] Feat: Add collection fields as "extra" query config for MongoDB --- .../integration/ExtraQueryConfig.svelte | 48 +++++++ .../components/integration/QueryViewer.svelte | 17 +++ packages/server/src/api/routes/query.js | 2 + packages/server/src/definitions/datasource.ts | 10 ++ packages/server/src/integrations/mongodb.ts | 125 ++++++++++++++++-- 5 files changed, 191 insertions(+), 11 deletions(-) create mode 100644 packages/builder/src/components/integration/ExtraQueryConfig.svelte diff --git a/packages/builder/src/components/integration/ExtraQueryConfig.svelte b/packages/builder/src/components/integration/ExtraQueryConfig.svelte new file mode 100644 index 0000000000..303beb1e4b --- /dev/null +++ b/packages/builder/src/components/integration/ExtraQueryConfig.svelte @@ -0,0 +1,48 @@ + + +{#each extraFields as { key, displayName, type }} +
+ + {#if type === "string"} + populateExtraQuery(extraQueryFields)} + bind:value={extraQueryFields[key]} + /> + {/if} + + {#if type === "list"} + queryConfig[verb]?.displayName || capitalise(verb)} />
+ {#if integrationInfo?.extra && query.queryVerb} + + {/if} {/if} diff --git a/packages/server/src/api/routes/query.js b/packages/server/src/api/routes/query.js index dd307c9444..2bbadcb40a 100644 --- a/packages/server/src/api/routes/query.js +++ b/packages/server/src/api/routes/query.js @@ -30,6 +30,7 @@ function generateQueryValidation() { default: Joi.string().allow(""), })), queryVerb: Joi.string().allow().required(), + extra: Joi.object().optional(), schema: Joi.object({}).required().unknown(true) })) } @@ -39,6 +40,7 @@ function generateQueryPreviewValidation() { return joiValidator.body(Joi.object({ fields: Joi.object().required(), queryVerb: Joi.string().allow().required(), + extra: Joi.object().optional(), datasourceId: Joi.string().required(), parameters: Joi.object({}).required().unknown(true) })) diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index 22f1998601..24d8144940 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -49,6 +49,15 @@ export interface QueryDefinition { urlDisplay?: boolean } +export interface ExtraQueryConfig { + [key: string]: { + displayName: string, + type: string, + required: boolean + data?: object + } +} + export interface Integration { docs: string plus?: boolean @@ -58,6 +67,7 @@ export interface Integration { query: { [key: string]: QueryDefinition } + extra?: ExtraQueryConfig } export interface SearchFilters { diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index af7b49153d..ab21da6fd0 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -10,7 +10,7 @@ module MongoDBModule { interface MongoDBConfig { connectionString: string db: string - collection: string + // collection: string } const SCHEMA: Integration = { @@ -28,10 +28,6 @@ module MongoDBModule { type: DatasourceFieldTypes.STRING, required: true, }, - collection: { - type: DatasourceFieldTypes.STRING, - required: true, - }, }, query: { create: { @@ -40,7 +36,31 @@ module MongoDBModule { read: { type: QueryTypes.JSON, }, + update: { + type: QueryTypes.JSON, + }, + delete: { + type: QueryTypes.JSON, + } }, + extra: { + collection: { + displayName: "Collection", + type: DatasourceFieldTypes.STRING, + required: true, + }, + actionTypes: { + displayName: "Action Types", + type: DatasourceFieldTypes.LIST, + required: true, + data: { + read: ['find', 'findOne', 'findOneAndUpdate', "count", "distinct"], + create: ['insertOne', 'insertMany'], + update: ['updateOne', 'updateMany'], + delete: ['deleteOne', 'deleteMany'] + } + } + } } class MongoIntegration { @@ -56,12 +76,25 @@ module MongoDBModule { return this.client.connect() } - async create(query: { json: object }) { + async create(query: { json: object, extra: { [key: string]: string } }) { try { await this.connect() const db = this.client.db(this.config.db) - const collection = db.collection(this.config.collection) - return collection.insertOne(query.json) + const collection = db.collection(query.extra.collection) + + // For mongodb we add an extra actionType to specify + // which method we want to call on the collection + switch(query.extra.actionTypes) { + case 'insertOne': { + return collection.insertOne(query.json) + } + case 'insertMany': { + return collection.insertOne(query.json).toArray() + } + default: { + throw new Error(`actionType ${query.extra.actionTypes} does not exist on DB for create`) + } + } } catch (err) { console.error("Error writing to mongodb", err) throw err @@ -70,12 +103,32 @@ module MongoDBModule { } } - async read(query: { json: object }) { + async read(query: { json: object, extra: { [key: string]: string } }) { try { await this.connect() const db = this.client.db(this.config.db) - const collection = db.collection(this.config.collection) - return collection.find(query.json).toArray() + const collection = db.collection(query.extra.collection) + + switch(query.extra.actionTypes) { + case 'find': { + return collection.find(query.json).toArray() + } + case 'findOne': { + return collection.findOne(query.json) + } + case 'findOneAndUpdate': { + return collection.findOneAndUpdate(query.json) + } + case 'count': { + return collection.countDocuments(query.json) + } + case 'distinct': { + return collection.distinct(query.json) + } + default: { + throw new Error(`actionType ${query.extra.actionTypes} does not exist on DB for read`) + } + } } catch (err) { console.error("Error querying mongodb", err) throw err @@ -83,6 +136,56 @@ module MongoDBModule { await this.client.close() } } + + async update(query: { json: object, extra: { [key: string]: string } }) { + try { + await this.connect() + const db = this.client.db(this.config.db) + const collection = db.collection(query.extra.collection) + + switch(query.extra.actionTypes) { + case 'updateOne': { + return collection.updateOne(query.json) + } + case 'updateMany': { + return collection.updateMany(query.json).toArray() + } + default: { + throw new Error(`actionType ${query.extra.actionTypes} does not exist on DB for update`) + } + } + } catch (err) { + console.error("Error writing to mongodb", err) + throw err + } finally { + await this.client.close() + } + } + + async delete(query: { json: object, extra: { [key: string]: string } }) { + try { + await this.connect() + const db = this.client.db(this.config.db) + const collection = db.collection(query.extra.collection) + + switch(query.extra.actionTypes) { + case 'deleteOne': { + return collection.deleteOne(query.json) + } + case 'deleteMany': { + return collection.deleteMany(query.json).toArray() + } + default: { + throw new Error(`actionType ${query.extra.actionTypes} does not exist on DB for delete`) + } + } + } catch (err) { + console.error("Error writing to mongodb", err) + throw err + } finally { + await this.client.close() + } + } } module.exports = { From 8579d1872c1f44fe3490e1b6e6099b80643d8163 Mon Sep 17 00:00:00 2001 From: Christo Date: Fri, 9 Jul 2021 20:08:26 +0200 Subject: [PATCH 2/2] Test: Add some basic tests for mongodb queries --- packages/server/__mocks__/mongodb.ts | 15 ++++++ .../src/integrations/tests/mongo.spec.js | 48 ++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/packages/server/__mocks__/mongodb.ts b/packages/server/__mocks__/mongodb.ts index d7d8f852c1..06eefefea6 100644 --- a/packages/server/__mocks__/mongodb.ts +++ b/packages/server/__mocks__/mongodb.ts @@ -5,11 +5,26 @@ module MongoMock { this.connect = jest.fn() this.close = jest.fn() this.insertOne = jest.fn() + this.insertMany = jest.fn(() => ({toArray: () => []})) this.find = jest.fn(() => ({toArray: () => []})) + this.findOne = jest.fn() + this.count = jest.fn() + this.deleteOne = jest.fn() + this.deleteMany = jest.fn(() => ({toArray: () => []})) + this.updateOne = jest.fn() + this.updateMany = jest.fn(() => ({toArray: () => []})) + this.collection = jest.fn(() => ({ insertOne: this.insertOne, find: this.find, + insertMany: this.insertMany, + findOne: this.findOne, + count: this.count, + deleteOne: this.deleteOne, + deleteMany: this.deleteMany, + updateOne: this.updateOne, + updateMany: this.updateMany, })) this.db = () => ({ diff --git a/packages/server/src/integrations/tests/mongo.spec.js b/packages/server/src/integrations/tests/mongo.spec.js index 1e37d5dd70..ce44617eb1 100644 --- a/packages/server/src/integrations/tests/mongo.spec.js +++ b/packages/server/src/integrations/tests/mongo.spec.js @@ -8,6 +8,13 @@ class TestConfiguration { } } +function disableConsole() { + jest.spyOn(console, 'error'); + console.error.mockImplementation(() => {}); + + return console.error.mockRestore; +} + describe("MongoDB Integration", () => { let config let indexName = "Users" @@ -22,7 +29,8 @@ describe("MongoDB Integration", () => { } const response = await config.integration.create({ index: indexName, - json: body + json: body, + extra: { collection: 'testCollection', actionTypes: 'insertOne'} }) expect(config.integration.client.insertOne).toHaveBeenCalledWith(body) }) @@ -31,10 +39,46 @@ describe("MongoDB Integration", () => { const query = { json: { address: "test" - } + }, + extra: { collection: 'testCollection', actionTypes: 'find'} } const response = await config.integration.read(query) expect(config.integration.client.find).toHaveBeenCalledWith(query.json) expect(response).toEqual(expect.any(Array)) }) + + it("calls the delete method with the correct params", async () => { + const query = { + json: { + id: "test" + }, + extra: { collection: 'testCollection', actionTypes: 'deleteOne'} + } + const response = await config.integration.delete(query) + expect(config.integration.client.deleteOne).toHaveBeenCalledWith(query.json) + }) + + it("calls the update method with the correct params", async () => { + const query = { + json: { + id: "test" + }, + extra: { collection: 'testCollection', actionTypes: 'updateOne'} + } + const response = await config.integration.update(query) + expect(config.integration.client.updateOne).toHaveBeenCalledWith(query.json) + }) + + it("throws an error when an invalid query.extra.actionType is passed for each method", async () => { + const restore = disableConsole() + + const query = { + extra: { collection: 'testCollection', actionTypes: 'deleteOne'} + } + // Weird, need to do an IIFE for jest to recognize that it throws + expect(() => config.integration.read(query)()).toThrow(expect.any(Object)) + + restore() + }) + }) \ No newline at end of file