MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/query-variables' of github.com:Budibase/budibase into rest-pagination

This commit is contained in:
Andrew Kingston 2021-12-17 18:56:16 +00:00
parent 3b9b6bdde0 ae510760c3
commit 932a63de7f
13 changed files with 114 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/auth/src/constants.js
View File

@ -16,6 +16,7 @@ exports.Headers = {
APP_ID: "x-budibase-app-id", APP_ID: "x-budibase-app-id",
TYPE: "x-budibase-type", TYPE: "x-budibase-type",
TENANT_ID: "x-budibase-tenant-id", TENANT_ID: "x-budibase-tenant-id",
TOKEN: "x-budibase-token",
} }
exports.GlobalRoles = { exports.GlobalRoles = {

7
packages/auth/src/middleware/authenticated.js
View File

@ -1,5 +1,5 @@
const { Cookies, Headers } = require("../constants") const { Cookies, Headers } = require("../constants")
const { getCookie, clearCookie } = require("../utils") const { getCookie, clearCookie, openJwt } = require("../utils")
const { getUser } = require("../cache/user") const { getUser } = require("../cache/user")
const { getSession, updateSessionTTL } = require("../security/sessions") const { getSession, updateSessionTTL } = require("../security/sessions")
const { buildMatcherRegex, matches } = require("./matchers") const { buildMatcherRegex, matches } = require("./matchers")
@ -35,8 +35,9 @@ module.exports = (
publicEndpoint = true publicEndpoint = true
} }
try { try {
// check the actual user is authenticated first // check the actual user is authenticated first, try header or cookie
const authCookie = getCookie(ctx, Cookies.Auth) const headerToken = ctx.request.headers[Headers.TOKEN]
const authCookie = getCookie(ctx, Cookies.Auth) || openJwt(headerToken)
let authenticated = false, let authenticated = false,
user = null, user = null,
internal = false internal = false

13
packages/auth/src/utils.js
View File

@ -63,6 +63,17 @@ exports.getAppId = ctx => {
return appId return appId
} }
/**
* opens the contents of the specified encrypted JWT.
* @return {object} the contents of the token.
*/
exports.openJwt = token => {
if (!token) {
return token
}
return jwt.verify(token, options.secretOrKey)
}
/** /**
* Get a cookie from context, and decrypt if necessary. * Get a cookie from context, and decrypt if necessary.
* @param {object} ctx The request which is to be manipulated. * @param {object} ctx The request which is to be manipulated.
@ -75,7 +86,7 @@ exports.getCookie = (ctx, name) => {
return cookie return cookie
} }
return jwt.verify(cookie, options.secretOrKey) return exports.openJwt(cookie)
} }
/** /**

8
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte
View File

@ -60,9 +60,9 @@
</div> </div>
</div> </div>
<Body size="S" <Body size="S"
>Variables enabled you to store and reuse values in queries. Static variables >Variables enable you to store and re-use values in queries, with the choice
use constant values while dynamic values can be bound to the response headers of a static value such as a token using static variables, or a value from a
or body of a query</Body query response using dynamic variables.</Body
> >
<Heading size="XS">Static</Heading> <Heading size="XS">Static</Heading>
<Layout noPadding gap="XS"> <Layout noPadding gap="XS">
@ -78,7 +78,7 @@
<Heading size="XS">Dynamic</Heading> <Heading size="XS">Dynamic</Heading>
<Body size="S"> <Body size="S">
Dynamic variables are evaluated when a dependant query is executed. The value Dynamic variables are evaluated when a dependant query is executed. The value
is cached for 24 hours and will re-evaluate if the dependendent query fails. is cached for a period of time and will be refreshed if a query fails.
</Body> </Body>
<ViewDynamicVariables {queries} {datasource} /> <ViewDynamicVariables {queries} {datasource} />

25
packages/builder/src/helpers/data/utils.js
View File

@ -121,10 +121,13 @@ export function flipHeaderState(headersActivity) {
} }
// convert dynamic variables list to simple key/val object // convert dynamic variables list to simple key/val object
export function variablesToObject(datasource) { export function getDynamicVariables(datasource, queryId) {
const variablesList = datasource?.config?.dynamicVariables const variablesList = datasource?.config?.dynamicVariables
if (variablesList && variablesList.length > 0) { if (variablesList && variablesList.length > 0) {
return variablesList.reduce( const filtered = queryId
? variablesList.filter(variable => variable.queryId === queryId)
: variablesList
return filtered.reduce(
(acc, next) => ({ ...acc, [next.name]: next.value }), (acc, next) => ({ ...acc, [next.name]: next.value }),
{} {}
) )
@ -133,10 +136,10 @@ export function variablesToObject(datasource) {
} }
// convert dynamic variables object back to a list, enrich with query id // convert dynamic variables object back to a list, enrich with query id
export function rebuildVariables(queryId, variables) { export function rebuildVariables(datasource, queryId, variables) {
let vars = [] let newVariables = []
if (variables) { if (variables) {
vars = Object.entries(variables).map(entry => { newVariables = Object.entries(variables).map(entry => {
return { return {
name: entry[0], name: entry[0],
value: entry[1], value: entry[1],
@ -144,7 +147,15 @@ export function rebuildVariables(queryId, variables) {
} }
}) })
} }
return vars let existing = datasource?.config?.dynamicVariables || []
// filter out any by same name
existing = existing.filter(
variable =>
!newVariables.find(
newVar => newVar.name.toLowerCase() === variable.name.toLowerCase()
)
)
return [...existing, ...newVariables]
} }
export function shouldShowVariables(dynamicVariables, variablesReadOnly) { export function shouldShowVariables(dynamicVariables, variablesReadOnly) {
@ -173,7 +184,7 @@ export default {
keyValueToQueryParameters, keyValueToQueryParameters,
queryParametersToKeyValue, queryParametersToKeyValue,
schemaToFields, schemaToFields,
variablesToObject, getDynamicVariables,
rebuildVariables, rebuildVariables,
shouldShowVariables, shouldShowVariables,
buildAuthConfigs, buildAuthConfigs,

17
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte
View File

@ -1,10 +1,11 @@
<script> <script>
import { Input, ModalContent, Modal } from "@budibase/bbui" import { Input, ModalContent, Modal, Body } from "@budibase/bbui"
export let dynamicVariables export let dynamicVariables
export let datasource
export let binding export let binding
let name, modal, valid let name, modal, valid, allVariableNames
export const show = () => { export const show = () => {
modal.show() modal.show()
@ -17,11 +18,15 @@
if (!name) { if (!name) {
return false return false
} }
const varKeys = Object.keys(vars || {}) return !allVariableNames.find(
return varKeys.find(key => key.toLowerCase() === name.toLowerCase()) == null varName => varName.toLowerCase() === name.toLowerCase()
)
} }
$: valid = checkValid(dynamicVariables, name) $: valid = checkValid(dynamicVariables, name)
$: allVariableNames = (datasource?.config?.dynamicVariables || []).map(
variable => variable.name
)
$: error = name && !valid ? "Variable name is already in use." : null $: error = name && !valid ? "Variable name is already in use." : null
async function saveVariable() { async function saveVariable() {
@ -40,6 +45,10 @@
onConfirm={saveVariable} onConfirm={saveVariable}
disabled={!valid} disabled={!valid}
> >
<Body size="S"
>Specify a name for your new dynamic variable, this must be unique across
your datasource.</Body
>
<Input label="Variable name" bind:value={name} on:input {error} /> <Input label="Variable name" bind:value={name} on:input {error} />
</ModalContent> </ModalContent>
</Modal> </Modal>

10
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
View File

@ -118,6 +118,7 @@
if (dynamicVariables) { if (dynamicVariables) {
datasource.config.dynamicVariables = restUtils.rebuildVariables( datasource.config.dynamicVariables = restUtils.rebuildVariables(
datasource,
saveId, saveId,
dynamicVariables dynamicVariables
) )
@ -218,7 +219,7 @@
if (query && !query.fields.pagination) { if (query && !query.fields.pagination) {
query.fields.pagination = {} query.fields.pagination = {}
} }
dynamicVariables = restUtils.variablesToObject(datasource) dynamicVariables = restUtils.getDynamicVariables(datasource, query._id)
}) })
</script> </script>
@ -438,9 +439,10 @@
{#if showVariablesTab} {#if showVariablesTab}
<Tab title="Dynamic Variables"> <Tab title="Dynamic Variables">
<Layout noPadding gap="S"> <Layout noPadding gap="S">
<Body size="S" <Body size="S">
>{"Create dynamic variables to use body and headers results in other queries"}</Body Create dynamic variables based on response body or headers
> from other queries.
</Body>
<KeyValueBuilder <KeyValueBuilder
bind:object={dynamicVariables} bind:object={dynamicVariables}
name="Variable" name="Variable"

1
packages/builder/src/stores/backend/queries.js
View File

@ -91,6 +91,7 @@ export function createQueriesStore() {
{} {}
), ),
datasourceId: query.datasourceId, datasourceId: query.datasourceId,
queryId: query._id || undefined,
}) })
if (response.status !== 200) { if (response.status !== 200) {

8
packages/server/src/api/controllers/query/index.js
View File

@ -104,8 +104,10 @@ exports.preview = async function (ctx) {
const db = new CouchDB(ctx.appId) const db = new CouchDB(ctx.appId)
const datasource = await db.get(ctx.request.body.datasourceId) const datasource = await db.get(ctx.request.body.datasourceId)
// preview may not have a queryId as it hasn't been saved, but if it does
const { fields, parameters, queryVerb, transformer } = ctx.request.body // this stops dynamic variables from calling the same query
const { fields, parameters, queryVerb, transformer, queryId } =
ctx.request.body
try { try {
const { rows, keys, info, extra } = await Runner.run({ const { rows, keys, info, extra } = await Runner.run({
@ -115,6 +117,7 @@ exports.preview = async function (ctx) {
fields, fields,
parameters, parameters,
transformer, transformer,
queryId,
}) })
ctx.body = { ctx.body = {
@ -143,6 +146,7 @@ async function execute(ctx, opts = { rowsOnly: false }) {
fields: query.fields, fields: query.fields,
parameters: ctx.request.body.parameter, parameters: ctx.request.body.parameter,
transformer: query.transformer, transformer: query.transformer,
queryId: ctx.params.queryId,
}) })
if (opts && opts.rowsOnly) { if (opts && opts.rowsOnly) {
ctx.body = rows ctx.body = rows

3
packages/server/src/api/controllers/query/validation.js
View File

@ -37,6 +37,7 @@ exports.generateQueryPreviewValidation = () => {
extra: Joi.object().optional(), extra: Joi.object().optional(),
datasourceId: Joi.string().required(), datasourceId: Joi.string().required(),
transformer: Joi.string().optional(), transformer: Joi.string().optional(),
parameters: Joi.object({}).required().unknown(true) parameters: Joi.object({}).required().unknown(true),
queryId: Joi.string().optional(),
})) }))
} }

34
packages/server/src/threads/query.js
View File

@ -13,7 +13,14 @@ class QueryRunner {
this.fields = input.fields this.fields = input.fields
this.parameters = input.parameters this.parameters = input.parameters
this.transformer = input.transformer this.transformer = input.transformer
this.queryId = input.queryId
this.noRecursiveQuery = flags.noRecursiveQuery this.noRecursiveQuery = flags.noRecursiveQuery
this.cachedVariables = []
// allows the response from a query to be stored throughout this
// execution so that if it needs to be re-used for another variable
// it can be
this.queryResponse = {}
this.hasRerun = false
} }
async execute() { async execute() {
@ -43,6 +50,19 @@ class QueryRunner {
rows = runner.execute() rows = runner.execute()
} }
// if the request fails we retry once, invalidating the cached value
if (
info &&
info.code >= 400 &&
this.cachedVariables.length > 0 &&
!this.hasRerun
) {
this.hasRerun = true
// invalidate the cache value
await threadUtils.invalidateDynamicVariables(this.cachedVariables)
return this.execute()
}
// needs to an array for next step // needs to an array for next step
if (!Array.isArray(rows)) { if (!Array.isArray(rows)) {
rows = [rows] rows = [rows]
@ -86,8 +106,14 @@ class QueryRunner {
name = variable.name name = variable.name
let value = await threadUtils.checkCacheForDynamicVariable(queryId, name) let value = await threadUtils.checkCacheForDynamicVariable(queryId, name)
if (!value) { if (!value) {
value = await this.runAnotherQuery(queryId, parameters) value = this.queryResponse[queryId]
? this.queryResponse[queryId]
: await this.runAnotherQuery(queryId, parameters)
// store incase this query is to be called again
this.queryResponse[queryId] = value
await threadUtils.storeDynamicVariable(queryId, name, value) await threadUtils.storeDynamicVariable(queryId, name, value)
} else {
this.cachedVariables.push({ queryId, name })
} }
return value return value
} }
@ -108,6 +134,10 @@ class QueryRunner {
// need to see if this uses any variables // need to see if this uses any variables
const stringFields = JSON.stringify(fields) const stringFields = JSON.stringify(fields)
const foundVars = dynamicVars.filter(variable => { const foundVars = dynamicVars.filter(variable => {
// don't allow a query to use its own dynamic variable (loop)
if (variable.queryId === this.queryId) {
return false
}
// look for {{ variable }} but allow spaces between handlebars // look for {{ variable }} but allow spaces between handlebars
const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`) const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`)
return regex.test(stringFields) return regex.test(stringFields)
@ -120,6 +150,8 @@ class QueryRunner {
data: responses[i].rows, data: responses[i].rows,
info: responses[i].extra, info: responses[i].extra,
}) })
// make sure its known that this uses dynamic variables in case it fails
this.hasDynamicVariables = true
} }
} }
return parameters return parameters

10
packages/server/src/threads/utils.js
View File

@ -38,6 +38,16 @@ exports.checkCacheForDynamicVariable = async (queryId, variable) => {
return cache.get(makeVariableKey(queryId, variable)) return cache.get(makeVariableKey(queryId, variable))
} }
exports.invalidateDynamicVariables = async cachedVars => {
let promises = []
for (let variable of cachedVars) {
promises.push(
client.delete(makeVariableKey(variable.queryId, variable.name))
)
}
await Promise.all(promises)
}
exports.storeDynamicVariable = async (queryId, variable, value) => { exports.storeDynamicVariable = async (queryId, variable, value) => {
const cache = await getClient() const cache = await getClient()
await cache.store( await cache.store(

5
packages/worker/src/api/controllers/global/auth.js
View File

@ -12,7 +12,7 @@ const {
hash, hash,
platformLogout, platformLogout,
} = authPkg.utils } = authPkg.utils
const { Cookies } = authPkg.constants const { Cookies, Headers } = authPkg.constants
const { passport } = authPkg.auth const { passport } = authPkg.auth
const { checkResetPasswordCode } = require("../../../utilities/redis") const { checkResetPasswordCode } = require("../../../utilities/redis")
const { const {
@ -60,7 +60,10 @@ async function authInternal(ctx, user, err = null, info = null) {
return ctx.throw(403, info ? info : "Unauthorized") return ctx.throw(403, info ? info : "Unauthorized")
} }
// set a cookie for browser access
setCookie(ctx, user.token, Cookies.Auth, { sign: false }) setCookie(ctx, user.token, Cookies.Auth, { sign: false })
// set the token in a header as well for APIs
ctx.set(Headers.TOKEN, user.token)
// get rid of any app cookies on login // get rid of any app cookies on login
// have to check test because this breaks cypress // have to check test because this breaks cypress
if (!env.isTest()) { if (!env.isTest()) {