MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/query-variables' of github.com:Budibase/budibase into rest-pagination

This commit is contained in:
Andrew Kingston 2021-12-17 18:56:16 +00:00
parent 6539f90b51 ab77c081bd
commit c1feb935cd
13 changed files with 114 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/auth/src/constants.js
View File

@ -16,6 +16,7 @@ exports.Headers = {
APP_ID: "x-budibase-app-id",
TYPE: "x-budibase-type",
TENANT_ID: "x-budibase-tenant-id",
TOKEN: "x-budibase-token",
}
exports.GlobalRoles = {

7
packages/auth/src/middleware/authenticated.js
View File

@ -1,5 +1,5 @@
const { Cookies, Headers } = require("../constants")
const { getCookie, clearCookie } = require("../utils")
const { getCookie, clearCookie, openJwt } = require("../utils")
const { getUser } = require("../cache/user")
const { getSession, updateSessionTTL } = require("../security/sessions")
const { buildMatcherRegex, matches } = require("./matchers")
@ -35,8 +35,9 @@ module.exports = (
publicEndpoint = true
}
try {
// check the actual user is authenticated first
const authCookie = getCookie(ctx, Cookies.Auth)
// check the actual user is authenticated first, try header or cookie
const headerToken = ctx.request.headers[Headers.TOKEN]
const authCookie = getCookie(ctx, Cookies.Auth) || openJwt(headerToken)
let authenticated = false,
user = null,
internal = false

13
packages/auth/src/utils.js
View File

@ -63,6 +63,17 @@ exports.getAppId = ctx => {
return appId
}
/**
* opens the contents of the specified encrypted JWT.
* @return {object} the contents of the token.
*/
exports.openJwt = token => {
if (!token) {
return token
}
return jwt.verify(token, options.secretOrKey)
}
/**
* Get a cookie from context, and decrypt if necessary.
* @param {object} ctx The request which is to be manipulated.
@ -75,7 +86,7 @@ exports.getCookie = (ctx, name) => {
return cookie
}
return jwt.verify(cookie, options.secretOrKey)
return exports.openJwt(cookie)
}
/**

8
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte
View File

@ -60,9 +60,9 @@
</div>
</div>
<Body size="S"
>Variables enabled you to store and reuse values in queries. Static variables
use constant values while dynamic values can be bound to the response headers
or body of a query</Body
>Variables enable you to store and re-use values in queries, with the choice
of a static value such as a token using static variables, or a value from a
query response using dynamic variables.</Body
>
<Heading size="XS">Static</Heading>
<Layout noPadding gap="XS">
@ -78,7 +78,7 @@
<Heading size="XS">Dynamic</Heading>
<Body size="S">
Dynamic variables are evaluated when a dependant query is executed. The value
is cached for 24 hours and will re-evaluate if the dependendent query fails.
is cached for a period of time and will be refreshed if a query fails.
</Body>
<ViewDynamicVariables {queries} {datasource} />

25
packages/builder/src/helpers/data/utils.js
View File

@ -121,10 +121,13 @@ export function flipHeaderState(headersActivity) {
}
// convert dynamic variables list to simple key/val object
export function variablesToObject(datasource) {
export function getDynamicVariables(datasource, queryId) {
const variablesList = datasource?.config?.dynamicVariables
if (variablesList && variablesList.length > 0) {
return variablesList.reduce(
const filtered = queryId
? variablesList.filter(variable => variable.queryId === queryId)
: variablesList
return filtered.reduce(
(acc, next) => ({ ...acc, [next.name]: next.value }),
{}
)
@ -133,10 +136,10 @@ export function variablesToObject(datasource) {
}
// convert dynamic variables object back to a list, enrich with query id
export function rebuildVariables(queryId, variables) {
let vars = []
export function rebuildVariables(datasource, queryId, variables) {
let newVariables = []
if (variables) {
vars = Object.entries(variables).map(entry => {
newVariables = Object.entries(variables).map(entry => {
return {
name: entry[0],
value: entry[1],
@ -144,7 +147,15 @@ export function rebuildVariables(queryId, variables) {
}
})
}
return vars
let existing = datasource?.config?.dynamicVariables || []
// filter out any by same name
existing = existing.filter(
variable =>
!newVariables.find(
newVar => newVar.name.toLowerCase() === variable.name.toLowerCase()
)
)
return [...existing, ...newVariables]
}
export function shouldShowVariables(dynamicVariables, variablesReadOnly) {
@ -173,7 +184,7 @@ export default {
keyValueToQueryParameters,
queryParametersToKeyValue,
schemaToFields,
variablesToObject,
getDynamicVariables,
rebuildVariables,
shouldShowVariables,
buildAuthConfigs,

17
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte
View File

@ -1,10 +1,11 @@
<script>
import { Input, ModalContent, Modal } from "@budibase/bbui"
import { Input, ModalContent, Modal, Body } from "@budibase/bbui"
export let dynamicVariables
export let datasource
export let binding
let name, modal, valid
let name, modal, valid, allVariableNames
export const show = () => {
modal.show()
@ -17,11 +18,15 @@
if (!name) {
return false
}
const varKeys = Object.keys(vars || {})
return varKeys.find(key => key.toLowerCase() === name.toLowerCase()) == null
return !allVariableNames.find(
varName => varName.toLowerCase() === name.toLowerCase()
)
}
$: valid = checkValid(dynamicVariables, name)
$: allVariableNames = (datasource?.config?.dynamicVariables || []).map(
variable => variable.name
)
$: error = name && !valid ? "Variable name is already in use." : null
async function saveVariable() {
@ -40,6 +45,10 @@
onConfirm={saveVariable}
disabled={!valid}
>
<Body size="S"
>Specify a name for your new dynamic variable, this must be unique across
your datasource.</Body
>
<Input label="Variable name" bind:value={name} on:input {error} />
</ModalContent>
</Modal>

10
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
View File

@ -118,6 +118,7 @@
if (dynamicVariables) {
datasource.config.dynamicVariables = restUtils.rebuildVariables(
datasource,
saveId,
dynamicVariables
)
@ -218,7 +219,7 @@
if (query && !query.fields.pagination) {
query.fields.pagination = {}
}
dynamicVariables = restUtils.variablesToObject(datasource)
dynamicVariables = restUtils.getDynamicVariables(datasource, query._id)
})
</script>
@ -438,9 +439,10 @@
{#if showVariablesTab}
<Tab title="Dynamic Variables">
<Layout noPadding gap="S">
<Body size="S"
>{"Create dynamic variables to use body and headers results in other queries"}</Body
>
<Body size="S">
Create dynamic variables based on response body or headers
from other queries.
</Body>
<KeyValueBuilder
bind:object={dynamicVariables}
name="Variable"

1
packages/builder/src/stores/backend/queries.js
View File

@ -91,6 +91,7 @@ export function createQueriesStore() {
{}
),
datasourceId: query.datasourceId,
queryId: query._id || undefined,
})
if (response.status !== 200) {

8
packages/server/src/api/controllers/query/index.js
View File

@ -104,8 +104,10 @@ exports.preview = async function (ctx) {
const db = new CouchDB(ctx.appId)
const datasource = await db.get(ctx.request.body.datasourceId)
const { fields, parameters, queryVerb, transformer } = ctx.request.body
// preview may not have a queryId as it hasn't been saved, but if it does
// this stops dynamic variables from calling the same query
const { fields, parameters, queryVerb, transformer, queryId } =
ctx.request.body
try {
const { rows, keys, info, extra } = await Runner.run({
@ -115,6 +117,7 @@ exports.preview = async function (ctx) {
fields,
parameters,
transformer,
queryId,
})
ctx.body = {
@ -143,6 +146,7 @@ async function execute(ctx, opts = { rowsOnly: false }) {
fields: query.fields,
parameters: ctx.request.body.parameter,
transformer: query.transformer,
queryId: ctx.params.queryId,
})
if (opts && opts.rowsOnly) {
ctx.body = rows

3
packages/server/src/api/controllers/query/validation.js
View File

@ -37,6 +37,7 @@ exports.generateQueryPreviewValidation = () => {
extra: Joi.object().optional(),
datasourceId: Joi.string().required(),
transformer: Joi.string().optional(),
parameters: Joi.object({}).required().unknown(true)
parameters: Joi.object({}).required().unknown(true),
queryId: Joi.string().optional(),
}))
}

34
packages/server/src/threads/query.js
View File

@ -13,7 +13,14 @@ class QueryRunner {
this.fields = input.fields
this.parameters = input.parameters
this.transformer = input.transformer
this.queryId = input.queryId
this.noRecursiveQuery = flags.noRecursiveQuery
this.cachedVariables = []
// allows the response from a query to be stored throughout this
// execution so that if it needs to be re-used for another variable
// it can be
this.queryResponse = {}
this.hasRerun = false
}
async execute() {
@ -43,6 +50,19 @@ class QueryRunner {
rows = runner.execute()
}
// if the request fails we retry once, invalidating the cached value
if (
info &&
info.code >= 400 &&
this.cachedVariables.length > 0 &&
!this.hasRerun
) {
this.hasRerun = true
// invalidate the cache value
await threadUtils.invalidateDynamicVariables(this.cachedVariables)
return this.execute()
}
// needs to an array for next step
if (!Array.isArray(rows)) {
rows = [rows]
@ -86,8 +106,14 @@ class QueryRunner {
name = variable.name
let value = await threadUtils.checkCacheForDynamicVariable(queryId, name)
if (!value) {
value = await this.runAnotherQuery(queryId, parameters)
value = this.queryResponse[queryId]
? this.queryResponse[queryId]
: await this.runAnotherQuery(queryId, parameters)
// store incase this query is to be called again
this.queryResponse[queryId] = value
await threadUtils.storeDynamicVariable(queryId, name, value)
} else {
this.cachedVariables.push({ queryId, name })
}
return value
}
@ -108,6 +134,10 @@ class QueryRunner {
// need to see if this uses any variables
const stringFields = JSON.stringify(fields)
const foundVars = dynamicVars.filter(variable => {
// don't allow a query to use its own dynamic variable (loop)
if (variable.queryId === this.queryId) {
return false
}
// look for {{ variable }} but allow spaces between handlebars
const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`)
return regex.test(stringFields)
@ -120,6 +150,8 @@ class QueryRunner {
data: responses[i].rows,
info: responses[i].extra,
})
// make sure its known that this uses dynamic variables in case it fails
this.hasDynamicVariables = true
}
}
return parameters

10
packages/server/src/threads/utils.js
View File

@ -38,6 +38,16 @@ exports.checkCacheForDynamicVariable = async (queryId, variable) => {
return cache.get(makeVariableKey(queryId, variable))
}
exports.invalidateDynamicVariables = async cachedVars => {
let promises = []
for (let variable of cachedVars) {
promises.push(
client.delete(makeVariableKey(variable.queryId, variable.name))
)
}
await Promise.all(promises)
}
exports.storeDynamicVariable = async (queryId, variable, value) => {
const cache = await getClient()
await cache.store(

5
packages/worker/src/api/controllers/global/auth.js
View File

@ -12,7 +12,7 @@ const {
hash,
platformLogout,
} = authPkg.utils
const { Cookies } = authPkg.constants
const { Cookies, Headers } = authPkg.constants
const { passport } = authPkg.auth
const { checkResetPasswordCode } = require("../../../utilities/redis")
const {
@ -60,7 +60,10 @@ async function authInternal(ctx, user, err = null, info = null) {
return ctx.throw(403, info ? info : "Unauthorized")
}
// set a cookie for browser access
setCookie(ctx, user.token, Cookies.Auth, { sign: false })
// set the token in a header as well for APIs
ctx.set(Headers.TOKEN, user.token)
// get rid of any app cookies on login
// have to check test because this breaks cypress
if (!env.isTest()) {