From 4960ad760ebaf21f590eade289205672e8a60413 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 16 Dec 2021 16:58:15 +0000 Subject: [PATCH] Adding start to redis caching. --- packages/auth/src/redis/utils.js | 1 + .../server/src/api/controllers/query/index.js | 54 +------------- packages/server/src/threads/query.js | 74 ++++++++++++++++++- packages/server/src/threads/utils.js | 20 +++++ 4 files changed, 97 insertions(+), 52 deletions(-) diff --git a/packages/auth/src/redis/utils.js b/packages/auth/src/redis/utils.js index 466b117e96..3461d2a511 100644 --- a/packages/auth/src/redis/utils.js +++ b/packages/auth/src/redis/utils.js @@ -16,6 +16,7 @@ exports.Databases = { USER_CACHE: "users", FLAGS: "flags", APP_METADATA: "appMetadata", + QUERY_VARS: "queryVars", } exports.SEPARATOR = SEPARATOR diff --git a/packages/server/src/api/controllers/query/index.js b/packages/server/src/api/controllers/query/index.js index 198d68d081..ddb49b7b3d 100644 --- a/packages/server/src/api/controllers/query/index.js +++ b/packages/server/src/api/controllers/query/index.js @@ -1,4 +1,3 @@ -const { processString } = require("@budibase/string-templates") const CouchDB = require("../../../db") const { generateQueryID, @@ -90,47 +89,6 @@ exports.save = async function (ctx) { ctx.message = `Query ${query.name} saved successfully.` } -async function enrichQueryFields(fields, parameters = {}) { - const enrichedQuery = {} - - // enrich the fields with dynamic parameters - for (let key of Object.keys(fields)) { - if (fields[key] == null) { - continue - } - if (typeof fields[key] === "object") { - // enrich nested fields object - enrichedQuery[key] = await enrichQueryFields(fields[key], parameters) - } else if (typeof fields[key] === "string") { - // enrich string value as normal - enrichedQuery[key] = await processString(fields[key], parameters, { - noHelpers: true, - }) - } else { - enrichedQuery[key] = fields[key] - } - } - - if ( - enrichedQuery.json || - enrichedQuery.customData || - enrichedQuery.requestBody - ) { - try { - enrichedQuery.json = JSON.parse( - enrichedQuery.json || - enrichedQuery.customData || - enrichedQuery.requestBody - ) - } catch (err) { - // no json found, ignore - } - delete enrichedQuery.customData - } - - return enrichedQuery -} - exports.find = async function (ctx) { const db = new CouchDB(ctx.appId) const query = enrichQueries(await db.get(ctx.params.queryId)) @@ -148,13 +106,13 @@ exports.preview = async function (ctx) { const datasource = await db.get(ctx.request.body.datasourceId) const { fields, parameters, queryVerb, transformer } = ctx.request.body - const enrichedQuery = await enrichQueryFields(fields, parameters) try { const { rows, keys, info, extra } = await Runner.run({ datasource, queryVerb, - query: enrichedQuery, + fields, + parameters, transformer, }) @@ -175,17 +133,13 @@ async function execute(ctx, opts = { rowsOnly: false }) { const query = await db.get(ctx.params.queryId) const datasource = await db.get(query.datasourceId) - const enrichedQuery = await enrichQueryFields( - query.fields, - ctx.request.body.parameters - ) - // call the relevant CRUD method on the integration class try { const { rows, extra } = await Runner.run({ datasource, queryVerb: query.queryVerb, - query: enrichedQuery, + fields: query.fields, + parameters: ctx.request.body.parameter, transformer: query.transformer, }) if (opts && opts.rowsOnly) { diff --git a/packages/server/src/threads/query.js b/packages/server/src/threads/query.js index f8d713e6a2..d60812f5fd 100644 --- a/packages/server/src/threads/query.js +++ b/packages/server/src/threads/query.js @@ -1,6 +1,66 @@ require("./utils").threadSetup() const ScriptRunner = require("../utilities/scriptRunner") const { integrations } = require("../integrations") +const { processStringSync } = require("@budibase/string-templates") + +async function addDatasourceVariables(datasource, parameters) { + if (!datasource || !datasource.config) { + return parameters + } + const staticVars = datasource.config.staticVariables || {} + const dynamicVars = datasource.config.dynamicVariables || [] + for (let [key, value] of Object.entries(staticVars)) { + if (!parameters[key]) { + parameters[key] = value + } + } + for (let variable of dynamicVars) { + console.log(variable) + // TODO: get the variable from query + } + return parameters +} + +function enrichQueryFields(fields, parameters = {}) { + const enrichedQuery = {} + + // enrich the fields with dynamic parameters + for (let key of Object.keys(fields)) { + if (fields[key] == null) { + continue + } + if (typeof fields[key] === "object") { + // enrich nested fields object + enrichedQuery[key] = enrichQueryFields(fields[key], parameters) + } else if (typeof fields[key] === "string") { + // enrich string value as normal + enrichedQuery[key] = processStringSync(fields[key], parameters, { + noHelpers: true, + }) + } else { + enrichedQuery[key] = fields[key] + } + } + + if ( + enrichedQuery.json || + enrichedQuery.customData || + enrichedQuery.requestBody + ) { + try { + enrichedQuery.json = JSON.parse( + enrichedQuery.json || + enrichedQuery.customData || + enrichedQuery.requestBody + ) + } catch (err) { + // no json found, ignore + } + delete enrichedQuery.customData + } + + return enrichedQuery +} function formatResponse(resp) { if (typeof resp === "string") { @@ -22,7 +82,16 @@ function hasExtraData(response) { ) } -async function runAndTransform(datasource, queryVerb, query, transformer) { +async function runAndTransform( + datasource, + queryVerb, + fields, + parameters, + transformer +) { + // pre-query, make sure datasource variables are added to parameters + parameters = await addDatasourceVariables(datasource, parameters) + const query = enrichQueryFields(fields, parameters) const Integration = integrations[datasource.source] if (!Integration) { throw "Integration type does not exist." @@ -69,7 +138,8 @@ module.exports = (input, callback) => { runAndTransform( input.datasource, input.queryVerb, - input.query, + input.fields, + input.parameters, input.transformer ) .then(response => { diff --git a/packages/server/src/threads/utils.js b/packages/server/src/threads/utils.js index 48b40d7002..a0fb9170a0 100644 --- a/packages/server/src/threads/utils.js +++ b/packages/server/src/threads/utils.js @@ -1,6 +1,18 @@ const env = require("../environment") const CouchDB = require("../db") const { init } = require("@budibase/auth") +const redis = require("@budibase/auth/redis") +const { SEPARATOR } = require("@budibase/auth/db") + +let client + +if (!client) { + client = await new redis.Client(redis.utils.Databases.QUERY_VARS).init() +} + +process.on("exit", async () => { + if (client) await client.finish() +}) exports.threadSetup = () => { // don't run this if not threading @@ -11,3 +23,11 @@ exports.threadSetup = () => { env.setInThread() init(CouchDB) } + +function makeVariableKey(queryId, variable) { + return `${queryId}${SEPARATOR}${variable}` +} + +exports.checkCacheForDynamicVariable = async (queryId, variable) => { + await client.get(makeVariableKey(queryId, variable)) +}