From 0d8ec8e03a4263afe01f871e8f15fea51bb1bbcb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 22 Oct 2020 17:48:32 +0100 Subject: [PATCH] Some initial work towards webhooks, that generates schema similar to integromat. --- .../BlockList/AutomationBlock.svelte | 13 ++- .../SetupPanel/AutomationBlockSetup.svelte | 47 +++++++++- packages/server/package.json | 2 + .../server/src/api/controllers/automation.js | 76 ++++++++++++++- .../server/src/api/controllers/webhook.js | 94 +++++++++++++++++++ packages/server/src/api/index.js | 4 + packages/server/src/api/routes/automation.js | 10 +- packages/server/src/api/routes/index.js | 2 + packages/server/src/api/routes/webhook.js | 45 +++++++++ packages/server/src/automations/triggers.js | 31 ++++++ packages/server/src/db/utils.js | 16 ++++ packages/server/src/middleware/authorized.js | 6 ++ packages/server/src/utilities/accessLevels.js | 1 + packages/server/yarn.lock | 42 +++++++++ 14 files changed, 382 insertions(+), 7 deletions(-) create mode 100644 packages/server/src/api/controllers/webhook.js create mode 100644 packages/server/src/api/routes/webhook.js diff --git a/packages/builder/src/components/automation/AutomationPanel/BlockList/AutomationBlock.svelte b/packages/builder/src/components/automation/AutomationPanel/BlockList/AutomationBlock.svelte index 2419999475..45d3b17940 100644 --- a/packages/builder/src/components/automation/AutomationPanel/BlockList/AutomationBlock.svelte +++ b/packages/builder/src/components/automation/AutomationPanel/BlockList/AutomationBlock.svelte @@ -1,11 +1,15 @@
@@ -64,6 +79,13 @@ {:else if value.customType === 'row'} + {:else if value.customType === 'webhookUrl'} +
+ + copyToClipboard(fullWebhookURL(block.inputs[key]))}> + + +
{:else if value.type === 'string' || value.type === 'number'} diff --git a/packages/server/package.json b/packages/server/package.json index 03002a0579..87a91da198 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -59,6 +59,7 @@ "fs-extra": "^8.1.0", "jimp": "^0.16.1", "joi": "^17.2.1", + "jsonschema": "^1.4.0", "jsonwebtoken": "^8.5.1", "koa": "^2.7.0", "koa-body": "^4.2.0", @@ -77,6 +78,7 @@ "sanitize-s3-objectkey": "^0.0.1", "squirrelly": "^7.5.0", "tar-fs": "^2.1.0", + "to-json-schema": "^0.2.5", "uuid": "^3.3.2", "validate.js": "^0.13.1", "worker-farm": "^1.7.0", diff --git a/packages/server/src/api/controllers/automation.js b/packages/server/src/api/controllers/automation.js index 92391da7e9..1647d59413 100644 --- a/packages/server/src/api/controllers/automation.js +++ b/packages/server/src/api/controllers/automation.js @@ -2,8 +2,10 @@ const CouchDB = require("../../db") const actions = require("../../automations/actions") const logic = require("../../automations/logic") const triggers = require("../../automations/triggers") +const webhooks = require("./webhook") const { getAutomationParams, generateAutomationID } = require("../../db/utils") +const WH_STEP_ID = triggers.BUILTIN_DEFINITIONS.WEBHOOK.stepId /************************* * * * BUILDER FUNCTIONS * @@ -30,6 +32,68 @@ function cleanAutomationInputs(automation) { return automation } +/** + * This function handles checking if any webhooks need to be created or deleted for automations. + * @param {object} user The user object, including all auth info + * @param {object|undefined} oldAuto The old automation object if updating/deleting + * @param {object|undefined} newAuto The new automation object if creating/updating + * @returns {Promise} After this is complete the new automation object may have been updated and should be + * written to DB (this does not write to DB as it would be wasteful to repeat). + */ +async function checkForWebhooks({ user, oldAuto, newAuto }) { + function isWebhookTrigger(auto) { + return ( + auto && + auto.definition.trigger && + auto.definition.trigger.stepId === WH_STEP_ID + ) + } + // need to delete webhook + if ( + isWebhookTrigger(oldAuto) && + !isWebhookTrigger(newAuto) && + oldAuto.definition.trigger.webhook + ) { + const ctx = { + user, + params: { + id: oldAuto.definition.trigger.webhook.id, + rev: oldAuto.definition.trigger.webhook.rev, + }, + } + // reset the inputs to remove the URLs + if (newAuto && newAuto.definition.trigger) { + const trigger = newAuto.definition.trigger + delete trigger.webhook + delete trigger.inputs.schemaUrl + delete trigger.inputs.triggerUrl + } + await webhooks.destroy(ctx) + } + // need to create webhook + else if (!isWebhookTrigger(oldAuto) && isWebhookTrigger(newAuto)) { + const ctx = { + user, + request: { + body: new webhooks.Webhook( + "Automation webhook", + webhooks.WebhookType.AUTOMATION, + newAuto._id + ), + }, + } + await webhooks.save(ctx) + const id = ctx.body.webhook._id, + rev = ctx.body.webhook._rev + newAuto.definition.trigger.webhook = { id, rev } + newAuto.definition.trigger.inputs = { + schemaUrl: `api/webhooks/schema/${user.instanceId}/${id}`, + triggerUrl: `api/webhooks/trigger/${user.instanceId}/${id}`, + } + } + return newAuto +} + exports.create = async function(ctx) { const db = new CouchDB(ctx.user.instanceId) let automation = ctx.request.body @@ -39,7 +103,8 @@ exports.create = async function(ctx) { automation.type = "automation" automation = cleanAutomationInputs(automation) - const response = await db.post(automation) + automation = await checkForWebhooks({ user: ctx.user, newAuto: automation }) + const response = await db.put(automation) automation._rev = response.rev ctx.status = 200 @@ -56,8 +121,13 @@ exports.update = async function(ctx) { const db = new CouchDB(ctx.user.instanceId) let automation = ctx.request.body automation.appId = ctx.user.appId - + const oldAutomation = await db.get(automation._id) automation = cleanAutomationInputs(automation) + automation = await checkForWebhooks({ + user: ctx.user, + oldAuto: oldAutomation, + newAuto: automation, + }) const response = await db.put(automation) automation._rev = response.rev @@ -89,6 +159,8 @@ exports.find = async function(ctx) { exports.destroy = async function(ctx) { const db = new CouchDB(ctx.user.instanceId) + const oldAutomation = await db.get(ctx.params.id) + await checkForWebhooks({ user: ctx.user, oldAuto: oldAutomation }) ctx.body = await db.remove(ctx.params.id, ctx.params.rev) } diff --git a/packages/server/src/api/controllers/webhook.js b/packages/server/src/api/controllers/webhook.js new file mode 100644 index 0000000000..a593133a47 --- /dev/null +++ b/packages/server/src/api/controllers/webhook.js @@ -0,0 +1,94 @@ +const CouchDB = require("../../db") +const { generateWebhookID, getWebhookParams } = require("../../db/utils") +const toJsonSchema = require("to-json-schema") +const validate = require("jsonschema").validate +const triggers = require("../../automations/triggers") + +const AUTOMATION_DESCRIPTION = "Generated from Webhook Schema" + +function Webhook(name, type, target) { + this.live = true + this.name = name + this.action = { + type, + target, + } +} + +exports.Webhook = Webhook + +exports.WebhookType = { + AUTOMATION: "automation", +} + +exports.fetch = async ctx => { + const db = new CouchDB(ctx.user.instanceId) + const response = await db.allDocs( + getWebhookParams(null, { + include_docs: true, + }) + ) + ctx.body = response.rows.map(row => row.doc) +} + +exports.save = async ctx => { + const db = new CouchDB(ctx.user.instanceId) + const webhook = ctx.request.body + webhook.appId = ctx.user.appId + + // check that the webhook exists + if (webhook._id) { + await db.get(webhook._id) + } else { + webhook._id = generateWebhookID() + } + const response = await db.put(webhook) + ctx.body = { + message: "Webhook created successfully", + webhook: { + ...webhook, + ...response, + }, + } +} + +exports.destroy = async ctx => { + const db = new CouchDB(ctx.user.instanceId) + ctx.body = await db.remove(ctx.params.id, ctx.params.rev) +} + +exports.buildSchema = async ctx => { + const db = new CouchDB(ctx.params.instance) + const webhook = await db.get(ctx.params.id) + webhook.bodySchema = toJsonSchema(ctx.request.body) + // update the automation outputs + if (webhook.action.type === exports.WebhookType.AUTOMATION) { + let automation = await db.get(webhook.action.target) + const autoOutputs = automation.definition.trigger.schema.outputs + let properties = webhook.bodySchema.properties + for (let prop of Object.keys(properties)) { + autoOutputs.properties[prop] = { + type: properties[prop].type, + description: AUTOMATION_DESCRIPTION, + } + } + await db.put(automation) + } + ctx.body = await db.put(webhook) +} + +exports.trigger = async ctx => { + const db = new CouchDB(ctx.params.instance) + const webhook = await db.get(ctx.params.id) + // validate against the schema + if (!webhook.bodySchema) { + ctx.throw(400, "Webhook has not been fully configured, no schema created") + } + validate(ctx.request.body, webhook.bodySchema) + const target = await db.get(webhook.action.target) + if (webhook.action.type === exports.WebhookType.AUTOMATION) { + await triggers.externalTrigger(target, ctx.request.body) + } + ctx.status = 200 + ctx.body = "Webhook trigger fired successfully" +} diff --git a/packages/server/src/api/index.js b/packages/server/src/api/index.js index 0a4de71135..26e7498d2e 100644 --- a/packages/server/src/api/index.js +++ b/packages/server/src/api/index.js @@ -21,6 +21,7 @@ const { apiKeysRoutes, templatesRoutes, analyticsRoutes, + webhookRoutes, } = require("./routes") const router = new Router() @@ -90,6 +91,9 @@ router.use(instanceRoutes.allowedMethods()) router.use(automationRoutes.routes()) router.use(automationRoutes.allowedMethods()) +router.use(webhookRoutes.routes()) +router.use(webhookRoutes.allowedMethods()) + router.use(deployRoutes.routes()) router.use(deployRoutes.allowedMethods()) diff --git a/packages/server/src/api/routes/automation.js b/packages/server/src/api/routes/automation.js index 84b429be66..3ac7937da2 100644 --- a/packages/server/src/api/routes/automation.js +++ b/packages/server/src/api/routes/automation.js @@ -2,7 +2,7 @@ const Router = require("@koa/router") const controller = require("../controllers/automation") const authorized = require("../../middleware/authorized") const joiValidator = require("../../middleware/joi-validator") -const { BUILDER } = require("../../utilities/accessLevels") +const { BUILDER, EXECUTE_AUTOMATION } = require("../../utilities/accessLevels") const Joi = require("joi") const router = Router() @@ -33,7 +33,7 @@ function generateValidator(existing = false) { type: Joi.string().valid("automation").required(), definition: Joi.object({ steps: Joi.array().required().items(generateStepSchema(["ACTION", "LOGIC"])), - trigger: generateStepSchema(["TRIGGER"]), + trigger: generateStepSchema(["TRIGGER"]).allow(null), }).required().unknown(true), }).unknown(true)) } @@ -73,7 +73,11 @@ router generateValidator(false), controller.create ) - .post("/api/automations/:id/trigger", controller.trigger) + .post( + "/api/automations/:id/trigger", + authorized(EXECUTE_AUTOMATION), + controller.trigger + ) .delete("/api/automations/:id/:rev", authorized(BUILDER), controller.destroy) module.exports = router diff --git a/packages/server/src/api/routes/index.js b/packages/server/src/api/routes/index.js index 2b60ddc894..72688a7de5 100644 --- a/packages/server/src/api/routes/index.js +++ b/packages/server/src/api/routes/index.js @@ -10,6 +10,7 @@ const viewRoutes = require("./view") const staticRoutes = require("./static") const componentRoutes = require("./component") const automationRoutes = require("./automation") +const webhookRoutes = require("./webhook") const accesslevelRoutes = require("./accesslevel") const deployRoutes = require("./deploy") const apiKeysRoutes = require("./apikeys") @@ -34,4 +35,5 @@ module.exports = { apiKeysRoutes, templatesRoutes, analyticsRoutes, + webhookRoutes, } diff --git a/packages/server/src/api/routes/webhook.js b/packages/server/src/api/routes/webhook.js new file mode 100644 index 0000000000..a7072904ed --- /dev/null +++ b/packages/server/src/api/routes/webhook.js @@ -0,0 +1,45 @@ +const Router = require("@koa/router") +const controller = require("../controllers/webhook") +const authorized = require("../../middleware/authorized") +const joiValidator = require("../../middleware/joi-validator") +const { BUILDER, EXECUTE_WEBHOOK } = require("../../utilities/accessLevels") +const Joi = require("joi") + +const router = Router() + +function generateSaveValidator() { + // prettier-ignore + return joiValidator.body(Joi.object({ + live: Joi.bool(), + _id: Joi.string().optional(), + _rev: Joi.string().optional(), + name: Joi.string().required(), + bodySchema: Joi.object().optional(), + action: Joi.object({ + type: Joi.string().required().valid(controller.WebhookType.AUTOMATION), + target: Joi.string().required(), + }).required(), + }).unknown(true)) +} + +router + .get("/api/webhooks", authorized(BUILDER), controller.fetch) + .put( + "/api/webhooks", + authorized(BUILDER), + generateSaveValidator(), + controller.save + ) + .delete("/api/webhooks/:id/:rev", authorized(BUILDER), controller.destroy) + .post( + "/api/webhooks/schema/:instance/:id", + authorized(BUILDER), + controller.buildSchema + ) + .post( + "/api/webhooks/trigger/:instance/:id", + authorized(EXECUTE_WEBHOOK), + controller.trigger + ) + +module.exports = router diff --git a/packages/server/src/automations/triggers.js b/packages/server/src/automations/triggers.js index c95b317f16..4870cfd051 100644 --- a/packages/server/src/automations/triggers.js +++ b/packages/server/src/automations/triggers.js @@ -83,6 +83,37 @@ const BUILTIN_DEFINITIONS = { }, type: "TRIGGER", }, + WEBHOOK: { + name: "Webhook", + event: "web:trigger", + icon: "ri-global-line", + tagline: "Webhook endpoint is hit", + description: "Trigger an automation when a HTTP POST webhook is hit", + stepId: "WEBHOOK", + inputs: {}, + schema: { + inputs: { + properties: { + schemaUrl: { + type: "string", + customType: "webhookUrl", + title: "Schema URL", + }, + triggerUrl: { + type: "string", + customType: "webhookUrl", + title: "Trigger URL", + }, + }, + required: ["schemaUrl", "triggerUrl"], + }, + outputs: { + properties: {}, + required: [], + }, + }, + type: "TRIGGER", + }, } async function queueRelevantRowAutomations(event, eventType) { diff --git a/packages/server/src/db/utils.js b/packages/server/src/db/utils.js index f44173bb25..673a340417 100644 --- a/packages/server/src/db/utils.js +++ b/packages/server/src/db/utils.js @@ -11,6 +11,7 @@ const DocumentTypes = { LINK: "li", APP: "app", ACCESS_LEVEL: "ac", + WEBHOOK: "wh", } exports.DocumentTypes = DocumentTypes @@ -164,3 +165,18 @@ exports.generateAccessLevelID = () => { exports.getAccessLevelParams = (accessLevelId = null, otherProps = {}) => { return getDocParams(DocumentTypes.ACCESS_LEVEL, accessLevelId, otherProps) } + +/** + * Generates a new webhook ID. + * @returns {string} The new webhook ID which the webhook doc can be stored under. + */ +exports.generateWebhookID = () => { + return `${DocumentTypes.WEBHOOK}${SEPARATOR}${newid()}` +} + +/** + * Gets parameters for retrieving a webhook, this is a utility function for the getDocParams function. + */ +exports.getWebhookParams = (webhookId = null, otherProps = {}) => { + return getDocParams(DocumentTypes.WEBHOOK, webhookId, otherProps) +} diff --git a/packages/server/src/middleware/authorized.js b/packages/server/src/middleware/authorized.js index 46f68c1564..9f9337b959 100644 --- a/packages/server/src/middleware/authorized.js +++ b/packages/server/src/middleware/authorized.js @@ -9,7 +9,13 @@ const environment = require("../environment") const { apiKeyTable } = require("../db/dynamoClient") const { AuthTypes } = require("../constants") +const LOCAL_PASS = new RegExp(["webhooks/trigger", "webhooks/schema"].join("|")) + module.exports = (permName, getItemId) => async (ctx, next) => { + // webhooks can pass locally + if (!environment.CLOUD && LOCAL_PASS.test(ctx.request.url)) { + return next() + } if ( environment.CLOUD && ctx.headers["x-api-key"] && diff --git a/packages/server/src/utilities/accessLevels.js b/packages/server/src/utilities/accessLevels.js index ada3f36880..e38a7cf23f 100644 --- a/packages/server/src/utilities/accessLevels.js +++ b/packages/server/src/utilities/accessLevels.js @@ -3,6 +3,7 @@ module.exports.READ_TABLE = "read-table" module.exports.WRITE_TABLE = "write-table" module.exports.READ_VIEW = "read-view" module.exports.EXECUTE_AUTOMATION = "execute-automation" +module.exports.EXECUTE_WEBHOOK = "execute-webhook" module.exports.USER_MANAGEMENT = "user-management" module.exports.BUILDER = "builder" module.exports.LIST_USERS = "list-users" diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 839939282d..4c3d84e7f6 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -4714,6 +4714,11 @@ jsonfile@^6.0.1: optionalDependencies: graceful-fs "^4.1.6" +jsonschema@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/jsonschema/-/jsonschema-1.4.0.tgz#1afa34c4bc22190d8e42271ec17ac8b3404f87b2" + integrity sha512-/YgW6pRMr6M7C+4o8kS+B/2myEpHCrxO4PEWnqJNBFMjn7EWXqlQ4tGwL6xTHeRplwuZmcAncdvfOad1nT2yMw== + jsonwebtoken@^8.5.1: version "8.5.1" resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz#00e71e0b8df54c2121a1f26137df2280673bcc0d" @@ -5160,6 +5165,21 @@ lodash.isstring@^4.0.1: resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451" integrity sha1-1SfftUVuynzJu5XV2ur4i6VKVFE= +lodash.keys@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-4.2.0.tgz#a08602ac12e4fb83f91fc1fb7a360a4d9ba35205" + integrity sha1-oIYCrBLk+4P5H8H7ejYKTZujUgU= + +lodash.merge@^4.6.2: + version "4.6.2" + resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" + integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== + +lodash.omit@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/lodash.omit/-/lodash.omit-4.5.0.tgz#6eb19ae5a1ee1dd9df0b969e66ce0b7fa30b5e60" + integrity sha1-brGa5aHuHdnfC5aeZs4Lf6MLXmA= + lodash.once@^4.0.0: version "4.1.1" resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac" @@ -5175,6 +5195,16 @@ lodash.sortby@^4.7.0: resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438" integrity sha1-7dFMgk4sycHgsKG0K7UhBRakJDg= +lodash.without@^4.4.0: + version "4.4.0" + resolved "https://registry.yarnpkg.com/lodash.without/-/lodash.without-4.4.0.tgz#3cd4574a00b67bae373a94b748772640507b7aac" + integrity sha1-PNRXSgC2e643OpS3SHcmQFB7eqw= + +lodash.xor@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/lodash.xor/-/lodash.xor-4.5.0.tgz#4d48ed7e98095b0632582ba714d3ff8ae8fb1db6" + integrity sha1-TUjtfpgJWwYyWCunFNP/iuj7HbY= + lodash@^4.17.10, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.19, lodash@^4.17.3: version "4.17.20" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" @@ -7471,6 +7501,18 @@ to-fast-properties@^2.0.0: resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" integrity sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4= +to-json-schema@^0.2.5: + version "0.2.5" + resolved "https://registry.yarnpkg.com/to-json-schema/-/to-json-schema-0.2.5.tgz#ef3c3f11ad64460dcfbdbafd0fd525d69d62a98f" + integrity sha512-jP1ievOee8pec3tV9ncxLSS48Bnw7DIybgy112rhMCEhf3K4uyVNZZHr03iQQBzbV5v5Hos+dlZRRyk6YSMNDw== + dependencies: + lodash.isequal "^4.5.0" + lodash.keys "^4.2.0" + lodash.merge "^4.6.2" + lodash.omit "^4.5.0" + lodash.without "^4.4.0" + lodash.xor "^4.5.0" + to-object-path@^0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/to-object-path/-/to-object-path-0.3.0.tgz#297588b7b0e7e0ac08e04e672f85c1f4999e17af"