From 68cc60d0151bb241bc3b53ac93a2c10b5628fd7c Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Tue, 11 Jan 2022 10:35:53 +0000 Subject: [PATCH] Invalidate dynamic variables when they are removed from datasource --- .../server/src/api/controllers/datasource.js | 34 ++++++++++ .../src/api/routes/tests/datasource.spec.js | 45 +++++++++++++ .../server/src/api/routes/tests/query.spec.js | 67 +++++++------------ .../src/tests/utilities/TestConfiguration.js | 47 +++++++++++++ 4 files changed, 150 insertions(+), 43 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.js b/packages/server/src/api/controllers/datasource.js index b99b1a5048..f08b622c3e 100644 --- a/packages/server/src/api/controllers/datasource.js +++ b/packages/server/src/api/controllers/datasource.js @@ -10,6 +10,7 @@ const { const { BuildSchemaErrors, InvalidColumns } = require("../../constants") const { integrations } = require("../../integrations") const { getDatasourceAndQuery } = require("./row/utils") +const { invalidateDynamicVariables } = require("../../threads/utils") exports.fetch = async function (ctx) { const database = new CouchDB(ctx.appId) @@ -57,10 +58,43 @@ exports.buildSchemaFromDb = async function (ctx) { ctx.body = response } +/** + * Check for variables that have been updated or removed and invalidate them. + */ +const invalidateVariables = async (existingDatasource, updatedDatasource) => { + const existingVariables = existingDatasource.config.dynamicVariables + const updatedVariables = updatedDatasource.config.dynamicVariables + const toInvalidate = [] + + if (!existingVariables) { + return + } + + if (!updatedVariables) { + // invalidate all + toInvalidate.push(...existingVariables) + } else { + // invaldate changed / removed + existingVariables.forEach(existing => { + const unchanged = updatedVariables.find( + updated => + existing.name === updated.name && + existing.queryId === updated.queryId && + existing.value === updated.value + ) + if (!unchanged) { + toInvalidate.push(existing) + } + }) + } + await invalidateDynamicVariables(toInvalidate) +} + exports.update = async function (ctx) { const db = new CouchDB(ctx.appId) const datasourceId = ctx.params.datasourceId let datasource = await db.get(datasourceId) + await invalidateVariables(datasource, ctx.request.body) datasource = { ...datasource, ...ctx.request.body } const response = await db.put(datasource) diff --git a/packages/server/src/api/routes/tests/datasource.spec.js b/packages/server/src/api/routes/tests/datasource.spec.js index 04f4dd77b8..880cc338a2 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.js +++ b/packages/server/src/api/routes/tests/datasource.spec.js @@ -4,6 +4,7 @@ let setup = require("./utilities") let { basicDatasource } = setup.structures let { checkBuilderEndpoint } = require("./utilities/TestFunctions") const pg = require("pg") +const { checkCacheForDynamicVariable } = require("../../../threads/utils") describe("/datasources", () => { let request = setup.getRequest() @@ -31,6 +32,50 @@ describe("/datasources", () => { }) }) + describe("update", () => { + it("should update an existing datasource", async () => { + datasource.name = "Updated Test" + const res = await request + .put(`/api/datasources/${datasource._id}`) + .send(datasource) + .set(config.defaultHeaders()) + .expect('Content-Type', /json/) + .expect(200) + + expect(res.body.datasource.name).toEqual("Updated Test") + expect(res.body.errors).toBeUndefined() + }) + + describe("dynamic variables", () => { + async function preview(datasource, fields) { + return config.previewQuery(request, config, datasource, fields) + } + + it("should invalidate changed or removed variables", async () => { + const { datasource, query } = await config.dynamicVariableDatasource() + // preview once to cache variables + await preview(datasource, { path: "www.test.com", queryString: "test={{ variable3 }}" }) + // check variables in cache + let contents = await checkCacheForDynamicVariable(query._id, "variable3") + expect(contents.rows.length).toEqual(1) + + // update the datasource to remove the variables + datasource.config.dynamicVariables = [] + const res = await request + .put(`/api/datasources/${datasource._id}`) + .send(datasource) + .set(config.defaultHeaders()) + .expect('Content-Type', /json/) + .expect(200) + expect(res.body.errors).toBeUndefined() + + // check variables no longer in cache + contents = await checkCacheForDynamicVariable(query._id, "variable3") + expect(contents).toBe(null) + }) + }) + }) + describe("fetch", () => { it("returns all the datasources from the server", async () => { const res = await request diff --git a/packages/server/src/api/routes/tests/query.spec.js b/packages/server/src/api/routes/tests/query.spec.js index c8f842e584..50d91a736d 100644 --- a/packages/server/src/api/routes/tests/query.spec.js +++ b/packages/server/src/api/routes/tests/query.spec.js @@ -229,52 +229,14 @@ describe("/queries", () => { }) }) - describe("test variables", () => { - async function restDatasource(cfg) { - return await config.createDatasource({ - datasource: { - ...basicDatasource().datasource, - source: "REST", - config: cfg || {}, - }, - }) - } - - async function dynamicVariableDatasource() { - const datasource = await restDatasource() - const basedOnQuery = await config.createQuery({ - ...basicQuery(datasource._id), - fields: { - path: "www.google.com", - }, - }) - await config.updateDatasource({ - ...datasource, - config: { - dynamicVariables: [ - { queryId: basedOnQuery._id, name: "variable3", value: "{{ data.0.[value] }}" } - ] - } - }) - return { datasource, query: basedOnQuery } - } + describe("variables", () => { async function preview(datasource, fields) { - return await request - .post(`/api/queries/preview`) - .send({ - datasourceId: datasource._id, - parameters: {}, - fields, - queryVerb: "read", - }) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) + return config.previewQuery(request, config, datasource, fields) } it("should work with static variables", async () => { - const datasource = await restDatasource({ + const datasource = await config.restDatasource({ staticVariables: { variable: "google", variable2: "1", @@ -290,7 +252,7 @@ describe("/queries", () => { }) it("should work with dynamic variables", async () => { - const { datasource } = await dynamicVariableDatasource() + const { datasource } = await config.dynamicVariableDatasource() const res = await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}", @@ -300,7 +262,7 @@ describe("/queries", () => { }) it("check that it automatically retries on fail with cached dynamics", async () => { - const { datasource, query: base } = await dynamicVariableDatasource() + const { datasource, query: base } = await config.dynamicVariableDatasource() // preview once to cache await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" }) // check its in cache @@ -313,5 +275,24 @@ describe("/queries", () => { expect(res.body.schemaFields).toEqual(["fails", "url", "opts"]) expect(res.body.rows[0].fails).toEqual(1) }) + + it("deletes variables when linked query is deleted", async () => { + const { datasource, query: base } = await config.dynamicVariableDatasource() + // preview once to cache + await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" }) + // check its in cache + let contents = await checkCacheForDynamicVariable(base._id, "variable3") + expect(contents.rows.length).toEqual(1) + + // delete the query + await request + .delete(`/api/queries/${base._id}/${base._rev}`) + .set(config.defaultHeaders()) + .expect(200) + + // check variables no longer in cache + contents = await checkCacheForDynamicVariable(base._id, "variable3") + expect(contents).toBe(null) + }) }) }) diff --git a/packages/server/src/tests/utilities/TestConfiguration.js b/packages/server/src/tests/utilities/TestConfiguration.js index f003a8a78d..2fbddee7fd 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.js +++ b/packages/server/src/tests/utilities/TestConfiguration.js @@ -326,6 +326,53 @@ class TestConfiguration { return this.datasource } + async restDatasource(cfg) { + return this.createDatasource({ + datasource: { + ...basicDatasource().datasource, + source: "REST", + config: cfg || {}, + }, + }) + } + + async dynamicVariableDatasource() { + let datasource = await this.restDatasource() + const basedOnQuery = await this.createQuery({ + ...basicQuery(datasource._id), + fields: { + path: "www.google.com", + }, + }) + datasource = await this.updateDatasource({ + ...datasource, + config: { + dynamicVariables: [ + { + queryId: basedOnQuery._id, + name: "variable3", + value: "{{ data.0.[value] }}", + }, + ], + }, + }) + return { datasource, query: basedOnQuery } + } + + async previewQuery(request, config, datasource, fields) { + return request + .post(`/api/queries/preview`) + .send({ + datasourceId: datasource._id, + parameters: {}, + fields, + queryVerb: "read", + }) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + } + async createQuery(config = null) { if (!this.datasource && !config) { throw "No data source created for query."