MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3830 from Budibase/feature/query-variables 

Datasource static and dynamic variables
This commit is contained in:
Michael Drury 2022-01-05 15:22:30 +00:00 committed by GitHub
parent ba5586e601 8d9db5c846
commit fe40e3e85d
36 changed files with 1828 additions and 242 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/auth/src/constants.js
View File

@ -16,6 +16,7 @@ exports.Headers = {
APP_ID: "x-budibase-app-id", APP_ID: "x-budibase-app-id",
TYPE: "x-budibase-type", TYPE: "x-budibase-type",
TENANT_ID: "x-budibase-tenant-id", TENANT_ID: "x-budibase-tenant-id",
TOKEN: "x-budibase-token",
} }
exports.GlobalRoles = { exports.GlobalRoles = {

7
packages/auth/src/middleware/authenticated.js
View File

@ -1,5 +1,5 @@
const { Cookies, Headers } = require("../constants") const { Cookies, Headers } = require("../constants")
const { getCookie, clearCookie } = require("../utils") const { getCookie, clearCookie, openJwt } = require("../utils")
const { getUser } = require("../cache/user") const { getUser } = require("../cache/user")
const { getSession, updateSessionTTL } = require("../security/sessions") const { getSession, updateSessionTTL } = require("../security/sessions")
const { buildMatcherRegex, matches } = require("./matchers") const { buildMatcherRegex, matches } = require("./matchers")
@ -35,8 +35,9 @@ module.exports = (
publicEndpoint = true publicEndpoint = true
} }
try { try {
// check the actual user is authenticated first // check the actual user is authenticated first, try header or cookie
const authCookie = getCookie(ctx, Cookies.Auth) const headerToken = ctx.request.headers[Headers.TOKEN]
const authCookie = getCookie(ctx, Cookies.Auth) || openJwt(headerToken)
let authenticated = false, let authenticated = false,
user = null, user = null,
internal = false internal = false

1
packages/auth/src/redis/utils.js
View File

@ -16,6 +16,7 @@ exports.Databases = {
USER_CACHE: "users", USER_CACHE: "users",
FLAGS: "flags", FLAGS: "flags",
APP_METADATA: "appMetadata", APP_METADATA: "appMetadata",
QUERY_VARS: "queryVars",
} }
exports.SEPARATOR = SEPARATOR exports.SEPARATOR = SEPARATOR

13
packages/auth/src/utils.js
View File

@ -63,6 +63,17 @@ exports.getAppId = ctx => {
return appId return appId
} }
/**
* opens the contents of the specified encrypted JWT.
* @return {object} the contents of the token.
*/
exports.openJwt = token => {
if (!token) {
return token
}
return jwt.verify(token, options.secretOrKey)
}
/** /**
* Get a cookie from context, and decrypt if necessary. * Get a cookie from context, and decrypt if necessary.
* @param {object} ctx The request which is to be manipulated. * @param {object} ctx The request which is to be manipulated.
@ -75,7 +86,7 @@ exports.getCookie = (ctx, name) => {
return cookie return cookie
} }
return jwt.verify(cookie, options.secretOrKey) return exports.openJwt(cookie)
} }
/** /**

11
packages/bbui/src/Table/BoldRenderer.svelte Normal file
View File

@ -0,0 +1,11 @@
<script>
export let value
</script>
<div class="bold">{value}</div>
<style>
.bold {
font-weight: bold;
}
</style>

5
packages/bbui/src/Table/CodeRenderer.svelte Normal file
View File

@ -0,0 +1,5 @@
<script>
export let value
</script>
<code>{value}</code>

4
packages/bbui/src/index.js
View File

@ -61,6 +61,10 @@ export { default as ColorPicker } from "./ColorPicker/ColorPicker.svelte"
export { default as InlineAlert } from "./InlineAlert/InlineAlert.svelte" export { default as InlineAlert } from "./InlineAlert/InlineAlert.svelte"
export { default as Banner } from "./Banner/Banner.svelte" export { default as Banner } from "./Banner/Banner.svelte"
// Renderers
export { default as BoldRenderer } from "./Table/BoldRenderer.svelte"
export { default as CodeRenderer } from "./Table/CodeRenderer.svelte"
// Typography // Typography
export { default as Body } from "./Typography/Body.svelte" export { default as Body } from "./Typography/Body.svelte"
export { default as Heading } from "./Typography/Heading.svelte" export { default as Heading } from "./Typography/Heading.svelte"

16
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte
View File

@ -175,7 +175,8 @@
onConfirm={datasources.removeSchemaError} onConfirm={datasources.removeSchemaError}
/> />
{/if} {/if}
<Table {#if plusTables && Object.values(plusTables).length > 0}
<Table
on:click={({ detail }) => onClickTable(detail)} on:click={({ detail }) => onClickTable(detail)}
schema={tableSchema} schema={tableSchema}
data={Object.values(plusTables)} data={Object.values(plusTables)}
@ -183,7 +184,10 @@
allowEditRows={false} allowEditRows={false}
allowSelectRows={false} allowSelectRows={false}
customRenderers={[{ column: "primary", component: ArrayRenderer }]} customRenderers={[{ column: "primary", component: ArrayRenderer }]}
/> />
{:else}
<Body size="S"><i>No tables found.</i></Body>
{/if}
{#if plusTables?.length !== 0} {#if plusTables?.length !== 0}
<Divider size="S" /> <Divider size="S" />
<div class="query-header"> <div class="query-header">
@ -196,14 +200,18 @@
Tell budibase how your tables are related to get even more smart features. Tell budibase how your tables are related to get even more smart features.
</Body> </Body>
{/if} {/if}
<Table {#if relationshipInfo && relationshipInfo.length > 0}
<Table
on:click={({ detail }) => openRelationshipModal(detail.from, detail.to)} on:click={({ detail }) => openRelationshipModal(detail.from, detail.to)}
schema={relationshipSchema} schema={relationshipSchema}
data={relationshipInfo} data={relationshipInfo}
allowEditColumns={false} allowEditColumns={false}
allowEditRows={false} allowEditRows={false}
allowSelectRows={false} allowSelectRows={false}
/> />
{:else}
<Body size="S"><i>No relationships configured.</i></Body>
{/if}
<style> <style>
.query-header { .query-header {

41
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte
View File

@ -1,9 +1,18 @@
<script> <script>
import { Divider, Heading, ActionButton, Badge, Body } from "@budibase/bbui" import {
Divider,
Heading,
ActionButton,
Badge,
Body,
Layout,
} from "@budibase/bbui"
import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte" import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
import RestAuthenticationBuilder from "./auth/RestAuthenticationBuilder.svelte" import RestAuthenticationBuilder from "./auth/RestAuthenticationBuilder.svelte"
import ViewDynamicVariables from "./variables/ViewDynamicVariables.svelte"
export let datasource export let datasource
export let queries
let addHeader let addHeader
</script> </script>
@ -43,6 +52,36 @@
</Body> </Body>
<RestAuthenticationBuilder bind:configs={datasource.config.authConfigs} /> <RestAuthenticationBuilder bind:configs={datasource.config.authConfigs} />
<Divider size="S" />
<div class="section-header">
<div class="badge">
<Heading size="S">Variables</Heading>
<Badge quiet grey>Optional</Badge>
</div>
</div>
<Body size="S"
>Variables enable you to store and re-use values in queries, with the choice
of a static value such as a token using static variables, or a value from a
query response using dynamic variables.</Body
>
<Heading size="XS">Static</Heading>
<Layout noPadding gap="XS">
<KeyValueBuilder
name="Variable"
keyPlaceholder="Name"
headings
bind:object={datasource.config.staticVariables}
on:change
/>
</Layout>
<div />
<Heading size="XS">Dynamic</Heading>
<Body size="S">
Dynamic variables are evaluated when a dependant query is executed. The value
is cached for a period of time and will be refreshed if a query fails.
</Body>
<ViewDynamicVariables {queries} {datasource} />
<style> <style>
.section-header { .section-header {
display: flex; display: flex;

54
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/variables/ViewDynamicVariables.svelte Normal file
View File

@ -0,0 +1,54 @@
<script>
import { Body, Table, BoldRenderer, CodeRenderer } from "@budibase/bbui"
import { queries as queriesStore } from "stores/backend"
import { goto } from "@roxi/routify"
export let datasource
export let queries
let dynamicVariables = []
$: enrichDynamicVariables(datasource, queries)
const dynamicVariableSchema = {
name: "",
query: "",
value: "",
}
const onClick = dynamicVariable => {
const queryId = dynamicVariable.queryId
queriesStore.select({ _id: queryId })
$goto(`./${queryId}`)
}
/**
* Add the query name to the dynamic variables
*/
function enrichDynamicVariables(ds, possibleQueries) {
dynamicVariables = []
ds.config.dynamicVariables?.forEach(dv => {
const query = possibleQueries.find(query => query._id === dv.queryId)
if (query) {
dynamicVariables.push({ ...dv, query: query.name })
}
})
}
</script>
{#if dynamicVariables && dynamicVariables.length > 0}
<Table
on:click={({ detail }) => onClick(detail)}
schema={dynamicVariableSchema}
data={dynamicVariables}
allowEditColumns={false}
allowEditRows={false}
allowSelectRows={false}
customRenderers={[
{ column: "name", component: BoldRenderer },
{ column: "value", component: CodeRenderer },
]}
/>
{:else}
<Body size="S"><i>No dynamic variables specified.</i></Body>
{/if}

41
packages/builder/src/components/integration/KeyValueBuilder.svelte
View File

@ -6,6 +6,8 @@
Label, Label,
Toggle, Toggle,
Select, Select,
ActionMenu,
MenuItem,
} from "@budibase/bbui" } from "@budibase/bbui"
import { createEventDispatcher } from "svelte" import { createEventDispatcher } from "svelte"
import { lowercase } from "helpers" import { lowercase } from "helpers"
@ -23,7 +25,11 @@
export let toggle export let toggle
export let keyPlaceholder = "Key" export let keyPlaceholder = "Key"
export let valuePlaceholder = "Value" export let valuePlaceholder = "Value"
export let valueHeading
export let keyHeading
export let tooltip export let tooltip
export let menuItems
export let showMenu = false
let fields = Object.entries(object).map(([name, value]) => ({ name, value })) let fields = Object.entries(object).map(([name, value]) => ({ name, value }))
let fieldActivity = buildFieldActivity(activity) let fieldActivity = buildFieldActivity(activity)
@ -76,17 +82,24 @@
{#if Object.keys(object || {}).length > 0} {#if Object.keys(object || {}).length > 0}
{#if headings} {#if headings}
<div class="container" class:container-active={toggle}> <div class="container" class:container-active={toggle}>
<Label {tooltip}>{keyPlaceholder}</Label> <Label {tooltip}>{keyHeading || keyPlaceholder}</Label>
<Label>{valuePlaceholder}</Label> <Label>{valueHeading || valuePlaceholder}</Label>
{#if toggle} {#if toggle}
<Label>Active</Label> <Label>Active</Label>
{/if} {/if}
</div> </div>
{/if} {/if}
<div class="container" class:container-active={toggle} class:readOnly> <div
class="container"
class:container-active={toggle}
class:container-menu={showMenu}
class:readOnly
class:readOnly-menu={readOnly && showMenu}
>
{#each fields as field, idx} {#each fields as field, idx}
<Input <Input
placeholder={keyPlaceholder} placeholder={keyPlaceholder}
readonly={readOnly}
bind:value={field.name} bind:value={field.name}
on:change={changed} on:change={changed}
/> />
@ -95,6 +108,7 @@
{:else} {:else}
<Input <Input
placeholder={valuePlaceholder} placeholder={valuePlaceholder}
readonly={readOnly}
bind:value={field.value} bind:value={field.value}
on:change={changed} on:change={changed}
/> />
@ -105,6 +119,18 @@
{#if !readOnly} {#if !readOnly}
<Icon hoverable name="Close" on:click={() => deleteEntry(idx)} /> <Icon hoverable name="Close" on:click={() => deleteEntry(idx)} />
{/if} {/if}
{#if menuItems?.length > 0 && showMenu}
<ActionMenu>
<div slot="control" class="control icon">
<Icon size="S" hoverable name="MoreSmallList" />
</div>
{#each menuItems as item}
<MenuItem on:click={() => item.onClick(field)}>
{item.text}
</MenuItem>
{/each}
</ActionMenu>
{/if}
{/each} {/each}
</div> </div>
{/if} {/if}
@ -127,7 +153,16 @@
.container-active { .container-active {
grid-template-columns: 1fr 1fr 50px 20px; grid-template-columns: 1fr 1fr 50px 20px;
} }
.container-menu {
grid-template-columns: 1fr 1fr 20px 20px;
}
.readOnly { .readOnly {
grid-template-columns: 1fr 1fr; grid-template-columns: 1fr 1fr;
} }
.readOnly-menu {
grid-template-columns: 1fr 1fr 20px;
}
.control {
margin-top: 4px;
}
</style> </style>

10
packages/builder/src/helpers/data/utils.js
View File

@ -119,3 +119,13 @@ export function flipHeaderState(headersActivity) {
}) })
return enabled return enabled
} }
export default {
breakQueryString,
buildQueryString,
fieldsToSchema,
flipHeaderState,
keyValueToQueryParameters,
queryParametersToKeyValue,
schemaToFields,
}

54
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte Normal file
View File

@ -0,0 +1,54 @@
<script>
import { Input, ModalContent, Modal, Body } from "@budibase/bbui"
export let dynamicVariables
export let datasource
export let binding
let name, modal, valid, allVariableNames
export const show = () => {
modal.show()
}
export const hide = () => {
modal.hide()
}
function checkValid(vars, name) {
if (!name) {
return false
}
return !allVariableNames.find(
varName => varName.toLowerCase() === name.toLowerCase()
)
}
$: valid = checkValid(dynamicVariables, name)
$: allVariableNames = (datasource?.config?.dynamicVariables || []).map(
variable => variable.name
)
$: error = name && !valid ? "Variable name is already in use." : null
async function saveVariable() {
const copiedName = name,
copiedBinding = binding
name = null
binding = null
dynamicVariables[copiedName] = copiedBinding
}
</script>
<Modal bind:this={modal}>
<ModalContent
title="Add dynamic variable"
confirmText="Save"
onConfirm={saveVariable}
disabled={!valid}
>
<Body size="S"
>Specify a name for your new dynamic variable, this must be unique across
your datasource.</Body
>
<Input label="Variable name" bind:value={name} on:input {error} />
</ModalContent>
</Modal>

32
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/RestBodyInput.svelte
View File

@ -16,17 +16,33 @@
export let query export let query
export let bodyType export let bodyType
let text = ""
let json = ""
$: checkRequestBody(bodyType) $: checkRequestBody(bodyType)
$: updateRequestBody(bodyType, text, json)
function checkRequestBody(type) { function checkRequestBody(type) {
if (!bodyType || !query) { if (!bodyType || !query) {
return return
} }
const currentType = typeof query?.fields.requestBody const currentType = typeof query?.fields.requestBody
if (objectTypes.includes(type) && currentType !== "object") { const isObject = objectTypes.includes(type)
query.fields.requestBody = {} const isText = textTypes.includes(type)
} else if (textTypes.includes(type) && currentType !== "string") { if (isText && currentType === "string") {
query.fields.requestBody = "" text = query.fields.requestBody
} else if (isObject && currentType === "object") {
json = query.fields.requestBody
}
}
function updateRequestBody(type, text, json) {
if (type === RawRestBodyTypes.NONE) {
query.fields.requestBody = null
} else if (objectTypes.includes(type)) {
query.fields.requestBody = json
} else {
query.fields.requestBody = text
} }
} }
@ -49,16 +65,12 @@
<Body size="S" weight="800">THE REQUEST DOES NOT HAVE A BODY</Body> <Body size="S" weight="800">THE REQUEST DOES NOT HAVE A BODY</Body>
</div> </div>
{:else if objectTypes.includes(bodyType)} {:else if objectTypes.includes(bodyType)}
<KeyValueBuilder <KeyValueBuilder bind:object={json} name="param" headings />
bind:object={query.fields.requestBody}
name="param"
headings
/>
{:else if textTypes.includes(bodyType)} {:else if textTypes.includes(bodyType)}
<CodeMirrorEditor <CodeMirrorEditor
height={200} height={200}
mode={editorMode(bodyType)} mode={editorMode(bodyType)}
value={query.fields.requestBody} value={text}
resize="vertical" resize="vertical"
on:change={e => (query.fields.requestBody = e.detail)} on:change={e => (query.fields.requestBody = e.detail)}
/> />

22
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte
View File

@ -21,11 +21,11 @@
import { cloneDeep } from "lodash/fp" import { cloneDeep } from "lodash/fp"
import ImportRestQueriesModal from "components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte" import ImportRestQueriesModal from "components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte"
import { onMount } from "svelte"
let importQueriesModal let importQueriesModal
let changed let changed
let datasource let integration, baseDatasource, datasource
let queryList
const querySchema = { const querySchema = {
name: {}, name: {},
queryVerb: { displayName: "Method" }, queryVerb: { displayName: "Method" },
@ -34,11 +34,12 @@
$: baseDatasource = $datasources.list.find( $: baseDatasource = $datasources.list.find(
ds => ds._id === $datasources.selected ds => ds._id === $datasources.selected
) )
$: integration = datasource && $integrations[datasource.source]
$: queryList = $queries.list.filter( $: queryList = $queries.list.filter(
query => query.datasourceId === datasource?._id query => query.datasourceId === datasource?._id
) )
$: hasChanged(baseDatasource, datasource) $: hasChanged(baseDatasource, datasource)
$: updateDatasource(baseDatasource)
function hasChanged(base, ds) { function hasChanged(base, ds) {
if (base && ds) { if (base && ds) {
@ -66,9 +67,12 @@
$goto(`./${query._id}`) $goto(`./${query._id}`)
} }
onMount(() => { function updateDatasource(base) {
datasource = cloneDeep(baseDatasource) if (base) {
}) datasource = cloneDeep(base)
integration = $integrations[datasource.source]
}
}
</script> </script>
<Modal bind:this={importQueriesModal}> <Modal bind:this={importQueriesModal}>
@ -142,7 +146,11 @@
</div> </div>
{/if} {/if}
{#if datasource?.source === IntegrationTypes.REST} {#if datasource?.source === IntegrationTypes.REST}
<RestExtraConfigForm bind:datasource on:change={hasChanged} /> <RestExtraConfigForm
queries={queryList}
bind:datasource
on:change={hasChanged}
/>
{/if} {/if}
</Layout> </Layout>
</section> </section>

210
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
View File

@ -1,22 +1,22 @@
<script> <script>
import { params } from "@roxi/routify" import { params } from "@roxi/routify"
import { datasources, integrations, queries, flags } from "stores/backend" import { datasources, flags, integrations, queries } from "stores/backend"
import { import {
Layout,
Input,
Select,
Tabs,
Tab,
Banner, Banner,
Divider,
Button,
Heading,
RadioGroup,
Label,
Body, Body,
TextArea, Button,
Table, Divider,
Heading,
Input,
Label,
Layout,
notifications, notifications,
RadioGroup,
Select,
Tab,
Table,
Tabs,
TextArea,
} from "@budibase/bbui" } from "@budibase/bbui"
import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte" import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
import EditableLabel from "components/common/inputs/EditableLabel.svelte" import EditableLabel from "components/common/inputs/EditableLabel.svelte"
@ -26,42 +26,37 @@
import RestBodyInput from "../../_components/RestBodyInput.svelte" import RestBodyInput from "../../_components/RestBodyInput.svelte"
import { capitalise } from "helpers" import { capitalise } from "helpers"
import { onMount } from "svelte" import { onMount } from "svelte"
import { import restUtils from "helpers/data/utils"
fieldsToSchema,
schemaToFields,
breakQueryString,
buildQueryString,
keyValueToQueryParameters,
queryParametersToKeyValue,
flipHeaderState,
} from "helpers/data/utils"
import { import {
RestBodyTypes as bodyTypes, RestBodyTypes as bodyTypes,
SchemaTypeOptions, SchemaTypeOptions,
} from "constants/backend" } from "constants/backend"
import JSONPreview from "components/integration/JSONPreview.svelte" import JSONPreview from "components/integration/JSONPreview.svelte"
import AccessLevelSelect from "components/integration/AccessLevelSelect.svelte" import AccessLevelSelect from "components/integration/AccessLevelSelect.svelte"
import DynamicVariableModal from "../../_components/DynamicVariableModal.svelte"
import Placeholder from "assets/bb-spaceship.svg" import Placeholder from "assets/bb-spaceship.svg"
import { cloneDeep } from "lodash/fp" import { cloneDeep } from "lodash/fp"
import { RawRestBodyTypes } from "constants/backend"
let query, datasource let query, datasource
let breakQs = {}, let breakQs = {},
bindings = {} bindings = {}
let url = "" let saveId, url
let saveId, isGet
let response, schema, enabledHeaders let response, schema, enabledHeaders
let datasourceType, integrationInfo, queryConfig, responseSuccess
let authConfigId let authConfigId
let dynamicVariables, addVariableModal, varBinding
$: datasourceType = datasource?.source $: datasourceType = datasource?.source
$: integrationInfo = $integrations[datasourceType] $: integrationInfo = $integrations[datasourceType]
$: queryConfig = integrationInfo?.query $: queryConfig = integrationInfo?.query
$: url = buildUrl(url, breakQs) $: url = buildUrl(url, breakQs)
$: checkQueryName(url) $: checkQueryName(url)
$: responseSuccess = response?.info?.code >= 200 && response?.info?.code < 400
$: isGet = query?.queryVerb === "read" $: isGet = query?.queryVerb === "read"
$: responseSuccess =
response?.info?.code >= 200 && response?.info?.code <= 206
$: authConfigs = buildAuthConfigs(datasource) $: authConfigs = buildAuthConfigs(datasource)
$: schemaReadOnly = !responseSuccess
$: variablesReadOnly = !responseSuccess
$: showVariablesTab = shouldShowVariables(dynamicVariables, variablesReadOnly)
function getSelectedQuery() { function getSelectedQuery() {
return cloneDeep( return cloneDeep(
@ -89,7 +84,7 @@
if (!base) { if (!base) {
return base return base
} }
const qs = buildQueryString(qsObj) const qs = restUtils.buildQueryString(qsObj)
let newUrl = base let newUrl = base
if (base.includes("?")) { if (base.includes("?")) {
newUrl = base.split("?")[0] newUrl = base.split("?")[0]
@ -97,29 +92,15 @@
return qs.length > 0 ? `${newUrl}?${qs}` : newUrl return qs.length > 0 ? `${newUrl}?${qs}` : newUrl
} }
const buildAuthConfigs = datasource => {
if (datasource?.config?.authConfigs) {
return datasource.config.authConfigs.map(c => ({
label: c.name,
value: c._id,
}))
}
return []
}
function learnMoreBanner() {
window.open("https://docs.budibase.com/building-apps/data/transformers")
}
function buildQuery() { function buildQuery() {
const newQuery = { ...query } const newQuery = { ...query }
const queryString = buildQueryString(breakQs) const queryString = restUtils.buildQueryString(breakQs)
newQuery.fields.path = url.split("?")[0] newQuery.fields.path = url.split("?")[0]
newQuery.fields.queryString = queryString newQuery.fields.queryString = queryString
newQuery.fields.authConfigId = authConfigId newQuery.fields.authConfigId = authConfigId
newQuery.fields.disabledHeaders = flipHeaderState(enabledHeaders) newQuery.fields.disabledHeaders = restUtils.flipHeaderState(enabledHeaders)
newQuery.schema = fieldsToSchema(schema) newQuery.schema = restUtils.fieldsToSchema(schema)
newQuery.parameters = keyValueToQueryParameters(bindings) newQuery.parameters = restUtils.keyValueToQueryParameters(bindings)
return newQuery return newQuery
} }
@ -130,8 +111,13 @@
saveId = _id saveId = _id
query = getSelectedQuery() query = getSelectedQuery()
notifications.success(`Request saved successfully.`) notifications.success(`Request saved successfully.`)
if (dynamicVariables) {
datasource.config.dynamicVariables = rebuildVariables(saveId)
datasource = await datasources.save(datasource)
}
} catch (err) { } catch (err) {
notifications.error(`Error creating query. ${err.message}`) notifications.error(`Error saving query. ${err.message}`)
} }
} }
@ -166,6 +152,78 @@
return id return id
} }
const buildAuthConfigs = datasource => {
if (datasource?.config?.authConfigs) {
return datasource.config.authConfigs.map(c => ({
label: c.name,
value: c._id,
}))
}
return []
}
const schemaMenuItems = [
{
text: "Create dynamic variable",
onClick: input => {
varBinding = `{{ data.0.[${input.name}] }}`
addVariableModal.show()
},
},
]
const responseHeadersMenuItems = [
{
text: "Create dynamic variable",
onClick: input => {
varBinding = `{{ info.headers.[${input.name}] }}`
addVariableModal.show()
},
},
]
// convert dynamic variables list to simple key/val object
const getDynamicVariables = (datasource, queryId) => {
const variablesList = datasource?.config?.dynamicVariables
if (variablesList && variablesList.length > 0) {
const filtered = queryId
? variablesList.filter(variable => variable.queryId === queryId)
: variablesList
return filtered.reduce(
(acc, next) => ({ ...acc, [next.name]: next.value }),
{}
)
}
return {}
}
// convert dynamic variables object back to a list, enrich with query id
const rebuildVariables = queryId => {
let variables = []
if (dynamicVariables) {
variables = Object.entries(dynamicVariables).map(entry => {
return {
name: entry[0],
value: entry[1],
queryId,
}
})
}
let existing = datasource?.config?.dynamicVariables || []
// remove existing query variables (for changes and deletions)
existing = existing.filter(variable => variable.queryId !== queryId)
// re-add the new query variables
return [...existing, ...variables]
}
const shouldShowVariables = (dynamicVariables, variablesReadOnly) => {
return !!(
dynamicVariables &&
// show when editable or when read only and not empty
(!variablesReadOnly || Object.keys(dynamicVariables).length > 0)
)
}
onMount(async () => { onMount(async () => {
query = getSelectedQuery() query = getSelectedQuery()
// clear any unsaved changes to the datasource // clear any unsaved changes to the datasource
@ -173,14 +231,14 @@
datasource = $datasources.list.find(ds => ds._id === query?.datasourceId) datasource = $datasources.list.find(ds => ds._id === query?.datasourceId)
const datasourceUrl = datasource?.config.url const datasourceUrl = datasource?.config.url
const qs = query?.fields.queryString const qs = query?.fields.queryString
breakQs = breakQueryString(qs) breakQs = restUtils.breakQueryString(qs)
if (datasourceUrl && !query.fields.path?.startsWith(datasourceUrl)) { if (datasourceUrl && !query.fields.path?.startsWith(datasourceUrl)) {
const path = query.fields.path const path = query.fields.path
query.fields.path = `${datasource.config.url}/${path ? path : ""}` query.fields.path = `${datasource.config.url}/${path ? path : ""}`
} }
url = buildUrl(query.fields.path, breakQs) url = buildUrl(query.fields.path, breakQs)
schema = schemaToFields(query.schema) schema = restUtils.schemaToFields(query.schema)
bindings = queryParametersToKeyValue(query.parameters) bindings = restUtils.queryParametersToKeyValue(query.parameters)
authConfigId = getAuthConfigId() authConfigId = getAuthConfigId()
if (!query.fields.disabledHeaders) { if (!query.fields.disabledHeaders) {
query.fields.disabledHeaders = {} query.fields.disabledHeaders = {}
@ -191,7 +249,7 @@
query.fields.disabledHeaders[header] = false query.fields.disabledHeaders[header] = false
} }
} }
enabledHeaders = flipHeaderState(query.fields.disabledHeaders) enabledHeaders = restUtils.flipHeaderState(query.fields.disabledHeaders)
if (query && !query.transformer) { if (query && !query.transformer) {
query.transformer = "return data" query.transformer = "return data"
} }
@ -201,11 +259,22 @@
} }
} }
if (query && !query.fields.bodyType) { if (query && !query.fields.bodyType) {
query.fields.bodyType = "none" if (query.fields.requestBody) {
query.fields.bodyType = RawRestBodyTypes.JSON
} else {
query.fields.bodyType = RawRestBodyTypes.NONE
} }
}
dynamicVariables = getDynamicVariables(datasource, query._id)
}) })
</script> </script>
<DynamicVariableModal
{datasource}
{dynamicVariables}
bind:binding={varBinding}
bind:this={addVariableModal}
/>
{#if query && queryConfig} {#if query && queryConfig}
<div class="inner"> <div class="inner">
<div class="top"> <div class="top">
@ -275,7 +344,10 @@
{#if !$flags.queryTransformerBanner} {#if !$flags.queryTransformerBanner}
<Banner <Banner
extraButtonText="Learn more" extraButtonText="Learn more"
extraButtonAction={learnMoreBanner} extraButtonAction={() =>
window.open(
"https://docs.budibase.com/building-apps/data/transformers"
)}
on:change={() => on:change={() =>
flags.updateFlag("queryTransformerBanner", true)} flags.updateFlag("queryTransformerBanner", true)}
> >
@ -341,6 +413,9 @@
name="schema" name="schema"
headings headings
options={SchemaTypeOptions} options={SchemaTypeOptions}
menuItems={schemaMenuItems}
showMenu={!schemaReadOnly}
readOnly={schemaReadOnly}
/> />
</Tab> </Tab>
{/if} {/if}
@ -349,7 +424,12 @@
<TextArea disabled value={response.extra?.raw} height="300" /> <TextArea disabled value={response.extra?.raw} height="300" />
</Tab> </Tab>
<Tab title="Headers"> <Tab title="Headers">
<KeyValueBuilder object={response.extra?.headers} readOnly /> <KeyValueBuilder
object={response.extra?.headers}
readOnly
menuItems={responseHeadersMenuItems}
showMenu={true}
/>
</Tab> </Tab>
<Tab title="Preview"> <Tab title="Preview">
<div class="table"> <div class="table">
@ -364,6 +444,28 @@
{/if} {/if}
</div> </div>
</Tab> </Tab>
{/if}
{#if showVariablesTab}
<Tab title="Dynamic Variables">
<Layout noPadding gap="S">
<Body size="S">
Create dynamic variables based on response body or headers
from other queries.
</Body>
<KeyValueBuilder
bind:object={dynamicVariables}
name="Variable"
headings
keyHeading="Name"
keyPlaceholder="Variable name"
valueHeading={`Value`}
valuePlaceholder={`{{ value }}`}
readOnly={variablesReadOnly}
/>
</Layout>
</Tab>
{/if}
{#if response}
<div class="stats"> <div class="stats">
<Label size="L"> <Label size="L">
Status: <span class={responseSuccess ? "green" : "red"} Status: <span class={responseSuccess ? "green" : "red"}

1
packages/builder/src/stores/backend/queries.js
View File

@ -91,6 +91,7 @@ export function createQueriesStore() {
{} {}
), ),
datasourceId: query.datasourceId, datasourceId: query.datasourceId,
queryId: query._id || undefined,
}) })
if (response.status !== 200) { if (response.status !== 200) {

8
packages/client/manifest.json
View File

@ -2557,6 +2557,10 @@
"label": "Rows", "label": "Rows",
"key": "rows" "key": "rows"
}, },
{
"label": "Extra Info",
"key": "info"
},
{ {
"label": "Rows Length", "label": "Rows Length",
"key": "rowsLength" "key": "rowsLength"
@ -3278,6 +3282,10 @@
"label": "Rows", "label": "Rows",
"key": "rows" "key": "rows"
}, },
{
"label": "Extra Info",
"key": "info"
},
{ {
"label": "Rows Length", "label": "Rows Length",
"key": "rowsLength" "key": "rowsLength"

12
packages/client/src/api/datasources.js
View File

@ -19,7 +19,8 @@ export const fetchDatasource = async dataSource => {
// Fetch all rows in data source // Fetch all rows in data source
const { type, tableId, fieldName } = dataSource const { type, tableId, fieldName } = dataSource
let rows = [] let rows = [],
info = {}
if (type === "table") { if (type === "table") {
rows = await fetchTableData(tableId) rows = await fetchTableData(tableId)
} else if (type === "view") { } else if (type === "view") {
@ -32,7 +33,12 @@ export const fetchDatasource = async dataSource => {
parameters[param.name] = param.default parameters[param.name] = param.default
} }
} }
rows = await executeQuery({ queryId: dataSource._id, parameters }) const { data, ...rest } = await executeQuery({
queryId: dataSource._id,
parameters,
})
info = rest
rows = data
} else if (type === FieldTypes.LINK) { } else if (type === FieldTypes.LINK) {
rows = await fetchRelationshipData({ rows = await fetchRelationshipData({
rowId: dataSource.rowId, rowId: dataSource.rowId,
@ -42,7 +48,7 @@ export const fetchDatasource = async dataSource => {
} }
// Enrich the result is always an array // Enrich the result is always an array
return Array.isArray(rows) ? rows : [] return { rows: Array.isArray(rows) ? rows : [], info }
} }
/** /**

2
packages/client/src/api/queries.js
View File

@ -11,7 +11,7 @@ export const executeQuery = async ({ queryId, parameters }) => {
return return
} }
const res = await API.post({ const res = await API.post({
url: `/api/queries/${queryId}`, url: `/api/v2/queries/${queryId}`,
body: { body: {
parameters, parameters,
}, },

8
packages/client/src/components/app/DataProvider.svelte
View File

@ -30,6 +30,7 @@
// Provider state // Provider state
let rows = [] let rows = []
let allRows = [] let allRows = []
let info = {}
let schema = {} let schema = {}
let bookmarks = [null] let bookmarks = [null]
let pageNumber = 0 let pageNumber = 0
@ -120,8 +121,9 @@
// Build our data context // Build our data context
$: dataContext = { $: dataContext = {
rows, rows,
info,
schema, schema,
rowsLength: rows.length, rowsLength: rows?.length,
// Undocumented properties. These aren't supposed to be used in builder // Undocumented properties. These aren't supposed to be used in builder
// bindings, but are used internally by other components // bindings, but are used internally by other components
@ -209,7 +211,9 @@
} else { } else {
// For other data sources like queries or views, fetch all rows from the // For other data sources like queries or views, fetch all rows from the
// server // server
allRows = await API.fetchDatasource(dataSource) const data = await API.fetchDatasource(dataSource)
allRows = data.rows
info = data.info
} }
loading = false loading = false
loaded = true loaded = true

18
packages/server/__mocks__/node-fetch.ts
View File

@ -1,5 +1,6 @@
module FetchMock { module FetchMock {
const fetch = jest.requireActual("node-fetch") const fetch = jest.requireActual("node-fetch")
let failCount = 0
module.exports = async (url: any, opts: any) => { module.exports = async (url: any, opts: any) => {
function json(body: any, status = 200) { function json(body: any, status = 200) {
@ -57,6 +58,23 @@ module FetchMock {
], ],
bookmark: "test", bookmark: "test",
}) })
} else if (url.includes("google.com")) {
return json({
url,
opts,
value: "<!doctype html><html itemscope=\"\" itemtype=\"http://schema.org/WebPage\" lang=\"en-GB\"></html>",
})
} else if (url.includes("failonce.com")) {
failCount++
if (failCount === 1) {
return json({ message: "error" }, 500)
} else {
return json({
fails: failCount - 1,
url,
opts,
})
}
} }
return fetch(url, opts) return fetch(url, opts)
} }

1
packages/server/package.json
View File

@ -122,6 +122,7 @@
"pouchdb-replication-stream": "1.2.9", "pouchdb-replication-stream": "1.2.9",
"server-destroy": "1.0.1", "server-destroy": "1.0.1",
"svelte": "^3.38.2", "svelte": "^3.38.2",
"swagger-parser": "^10.0.3",
"to-json-schema": "0.2.5", "to-json-schema": "0.2.5",
"uuid": "3.3.2", "uuid": "3.3.2",
"validate.js": "0.13.1", "validate.js": "0.13.1",

6
packages/server/src/api/controllers/query/import/sources/curl.ts
View File

@ -17,8 +17,12 @@ const parseBody = (curl: any) => {
if (curl.data) { if (curl.data) {
const keys = Object.keys(curl.data) const keys = Object.keys(curl.data)
if (keys.length) { if (keys.length) {
const key = keys[0] let key = keys[0]
try { try {
// filter out the dollar syntax used by curl for shell support
if (key.startsWith("$")) {
key = key.substring(1)
}
return JSON.parse(key) return JSON.parse(key)
} catch (e) { } catch (e) {
// do nothing // do nothing

80
packages/server/src/api/controllers/query/index.js
View File

@ -1,4 +1,3 @@
const { processString } = require("@budibase/string-templates")
const CouchDB = require("../../../db") const CouchDB = require("../../../db")
const { const {
generateQueryID, generateQueryID,
@ -90,47 +89,6 @@ exports.save = async function (ctx) {
ctx.message = `Query ${query.name} saved successfully.` ctx.message = `Query ${query.name} saved successfully.`
} }
async function enrichQueryFields(fields, parameters = {}) {
const enrichedQuery = {}
// enrich the fields with dynamic parameters
for (let key of Object.keys(fields)) {
if (fields[key] == null) {
continue
}
if (typeof fields[key] === "object") {
// enrich nested fields object
enrichedQuery[key] = await enrichQueryFields(fields[key], parameters)
} else if (typeof fields[key] === "string") {
// enrich string value as normal
enrichedQuery[key] = await processString(fields[key], parameters, {
noHelpers: true,
})
} else {
enrichedQuery[key] = fields[key]
}
}
if (
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
) {
try {
enrichedQuery.json = JSON.parse(
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
)
} catch (err) {
// no json found, ignore
}
delete enrichedQuery.customData
}
return enrichedQuery
}
exports.find = async function (ctx) { exports.find = async function (ctx) {
const db = new CouchDB(ctx.appId) const db = new CouchDB(ctx.appId)
const query = enrichQueries(await db.get(ctx.params.queryId)) const query = enrichQueries(await db.get(ctx.params.queryId))
@ -146,16 +104,20 @@ exports.preview = async function (ctx) {
const db = new CouchDB(ctx.appId) const db = new CouchDB(ctx.appId)
const datasource = await db.get(ctx.request.body.datasourceId) const datasource = await db.get(ctx.request.body.datasourceId)
// preview may not have a queryId as it hasn't been saved, but if it does
const { fields, parameters, queryVerb, transformer } = ctx.request.body // this stops dynamic variables from calling the same query
const enrichedQuery = await enrichQueryFields(fields, parameters) const { fields, parameters, queryVerb, transformer, queryId } =
ctx.request.body
try { try {
const { rows, keys, info, extra } = await Runner.run({ const { rows, keys, info, extra } = await Runner.run({
appId: ctx.appId,
datasource, datasource,
queryVerb, queryVerb,
query: enrichedQuery, fields,
parameters,
transformer, transformer,
queryId,
}) })
ctx.body = { ctx.body = {
@ -169,31 +131,41 @@ exports.preview = async function (ctx) {
} }
} }
exports.execute = async function (ctx) { async function execute(ctx, opts = { rowsOnly: false }) {
const db = new CouchDB(ctx.appId) const db = new CouchDB(ctx.appId)
const query = await db.get(ctx.params.queryId) const query = await db.get(ctx.params.queryId)
const datasource = await db.get(query.datasourceId) const datasource = await db.get(query.datasourceId)
const enrichedQuery = await enrichQueryFields(
query.fields,
ctx.request.body.parameters
)
// call the relevant CRUD method on the integration class // call the relevant CRUD method on the integration class
try { try {
const { rows } = await Runner.run({ const { rows, extra } = await Runner.run({
appId: ctx.appId,
datasource, datasource,
queryVerb: query.queryVerb, queryVerb: query.queryVerb,
query: enrichedQuery, fields: query.fields,
parameters: ctx.request.body.parameters,
transformer: query.transformer, transformer: query.transformer,
queryId: ctx.params.queryId,
}) })
if (opts && opts.rowsOnly) {
ctx.body = rows ctx.body = rows
} else {
ctx.body = { data: rows, ...extra }
}
} catch (err) { } catch (err) {
ctx.throw(400, err) ctx.throw(400, err)
} }
} }
exports.executeV1 = async function (ctx) {
return execute(ctx, { rowsOnly: true })
}
exports.executeV2 = async function (ctx) {
return execute(ctx, { rowsOnly: false })
}
exports.destroy = async function (ctx) { exports.destroy = async function (ctx) {
const db = new CouchDB(ctx.appId) const db = new CouchDB(ctx.appId)
await db.remove(ctx.params.queryId, ctx.params.revId) await db.remove(ctx.params.queryId, ctx.params.revId)

3
packages/server/src/api/controllers/query/validation.js
View File

@ -36,6 +36,7 @@ exports.generateQueryPreviewValidation = () => {
extra: Joi.object().optional(), extra: Joi.object().optional(),
datasourceId: Joi.string().required(), datasourceId: Joi.string().required(),
transformer: Joi.string().optional(), transformer: Joi.string().optional(),
parameters: Joi.object({}).required().unknown(true) parameters: Joi.object({}).required().unknown(true),
queryId: Joi.string().optional(),
})) }))
} }

9
packages/server/src/api/routes/query.js
View File

@ -41,11 +41,18 @@ router
authorized(PermissionTypes.QUERY, PermissionLevels.READ), authorized(PermissionTypes.QUERY, PermissionLevels.READ),
queryController.find queryController.find
) )
// DEPRECATED - use new query endpoint for future work
.post( .post(
"/api/queries/:queryId", "/api/queries/:queryId",
paramResource("queryId"), paramResource("queryId"),
authorized(PermissionTypes.QUERY, PermissionLevels.WRITE), authorized(PermissionTypes.QUERY, PermissionLevels.WRITE),
queryController.execute queryController.executeV1
)
.post(
"/api/v2/queries/:queryId",
paramResource("queryId"),
authorized(PermissionTypes.QUERY, PermissionLevels.WRITE),
queryController.executeV2
) )
.delete( .delete(
"/api/queries/:queryId/:revId", "/api/queries/:queryId/:revId",

88
packages/server/src/api/routes/tests/query.spec.js
View File

@ -1,5 +1,6 @@
// Mock out postgres for this // Mock out postgres for this
jest.mock("pg") jest.mock("pg")
jest.mock("node-fetch")
// Mock isProdAppID to we can later mock the implementation and pretend we are // Mock isProdAppID to we can later mock the implementation and pretend we are
// using prod app IDs // using prod app IDs
@ -10,6 +11,7 @@ authDb.isProdAppID = mockIsProdAppID
const setup = require("./utilities") const setup = require("./utilities")
const { checkBuilderEndpoint } = require("./utilities/TestFunctions") const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
const { checkCacheForDynamicVariable } = require("../../../threads/utils")
const { basicQuery, basicDatasource } = setup.structures const { basicQuery, basicDatasource } = setup.structures
describe("/queries", () => { describe("/queries", () => {
@ -226,4 +228,90 @@ describe("/queries", () => {
.expect(400) .expect(400)
}) })
}) })
describe("test variables", () => {
async function restDatasource(cfg) {
return await config.createDatasource({
datasource: {
...basicDatasource().datasource,
source: "REST",
config: cfg || {},
},
})
}
async function dynamicVariableDatasource() {
const datasource = await restDatasource()
const basedOnQuery = await config.createQuery({
...basicQuery(datasource._id),
fields: {
path: "www.google.com",
},
})
await config.updateDatasource({
...datasource,
config: {
dynamicVariables: [
{ queryId: basedOnQuery._id, name: "variable3", value: "{{ data.0.[value] }}" }
]
}
})
return { datasource, query: basedOnQuery }
}
async function preview(datasource, fields) {
return await request
.post(`/api/queries/preview`)
.send({
datasourceId: datasource._id,
parameters: {},
fields,
queryVerb: "read",
})
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
}
it("should work with static variables", async () => {
const datasource = await restDatasource({
staticVariables: {
variable: "google",
variable2: "1",
},
})
const res = await preview(datasource, {
path: "www.{{ variable }}.com",
queryString: "test={{ variable2 }}",
})
// these responses come from the mock
expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
expect(res.body.rows[0].url).toEqual("http://www.google.com?test=1")
})
it("should work with dynamic variables", async () => {
const { datasource } = await dynamicVariableDatasource()
const res = await preview(datasource, {
path: "www.google.com",
queryString: "test={{ variable3 }}",
})
expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
expect(res.body.rows[0].url).toContain("doctype html")
})
it("check that it automatically retries on fail with cached dynamics", async () => {
const { datasource, query: base } = await dynamicVariableDatasource()
// preview once to cache
await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
// check its in cache
const contents = await checkCacheForDynamicVariable(base._id, "variable3")
expect(contents.rows.length).toEqual(1)
const res = await preview(datasource, {
path: "www.failonce.com",
queryString: "test={{ variable3 }}",
})
expect(res.body.schemaFields).toEqual(["fails", "url", "opts"])
expect(res.body.rows[0].fails).toEqual(1)
})
})
}) })

10
packages/server/src/automations/steps/executeQuery.js
View File

@ -35,6 +35,11 @@ exports.definition = {
type: "object", type: "object",
description: "The response from the datasource execution", description: "The response from the datasource execution",
}, },
info: {
type: "object",
description:
"Some query types may return extra data, like headers from a REST query",
},
success: { success: {
type: "boolean", type: "boolean",
description: "Whether the action was successful", description: "Whether the action was successful",
@ -68,13 +73,16 @@ exports.run = async function ({ inputs, appId, emitter }) {
try { try {
await queryController.execute(ctx) await queryController.execute(ctx)
const { data, ...rest } = ctx.body
return { return {
response: ctx.body, response: data,
info: rest,
success: true, success: true,
} }
} catch (err) { } catch (err) {
return { return {
success: false, success: false,
info: {},
response: automationUtils.getError(err), response: automationUtils.getError(err),
} }
} }

10
packages/server/src/definitions/datasource.ts
View File

@ -240,6 +240,16 @@ export interface RestConfig {
[key: string]: any [key: string]: any
} }
authConfigs: AuthConfig[] authConfigs: AuthConfig[]
staticVariables: {
[key: string]: string
}
dynamicVariables: [
{
name: string
queryId: string
value: string
}
]
} }
export interface Query { export interface Query {

7
packages/server/src/integrations/rest.ts
View File

@ -48,7 +48,10 @@ module RestModule {
const { performance } = require("perf_hooks") const { performance } = require("perf_hooks")
const FormData = require("form-data") const FormData = require("form-data")
const { URLSearchParams } = require("url") const { URLSearchParams } = require("url")
const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js") const {
parseStringPromise: xmlParser,
Builder: XmlBuilder,
} = require("xml2js")
const SCHEMA: Integration = { const SCHEMA: Integration = {
docs: "https://github.com/node-fetch/node-fetch", docs: "https://github.com/node-fetch/node-fetch",
@ -211,7 +214,7 @@ module RestModule {
break break
case BodyTypes.XML: case BodyTypes.XML:
if (object != null) { if (object != null) {
string = (new XmlBuilder()).buildObject(object) string = new XmlBuilder().buildObject(object)
} }
input.body = string input.body = string
input.headers["Content-Type"] = "application/xml" input.headers["Content-Type"] = "application/xml"

10
packages/server/src/tests/utilities/TestConfiguration.js
View File

@ -316,6 +316,16 @@ class TestConfiguration {
return this.datasource return this.datasource
} }
async updateDatasource(datasource) {
const response = await this._req(
datasource,
{ datasourceId: datasource._id },
controllers.datasource.update
)
this.datasource = response.datasource
return this.datasource
}
async createQuery(config = null) { async createQuery(config = null) {
if (!this.datasource && !config) { if (!this.datasource && !config) {
throw "No data source created for query." throw "No data source created for query."

145
packages/server/src/threads/query.js
View File

@ -1,39 +1,44 @@
require("./utils").threadSetup() const threadUtils = require("./utils")
threadUtils.threadSetup()
const ScriptRunner = require("../utilities/scriptRunner") const ScriptRunner = require("../utilities/scriptRunner")
const { integrations } = require("../integrations") const { integrations } = require("../integrations")
const { processStringSync } = require("@budibase/string-templates")
const CouchDB = require("../db")
function formatResponse(resp) { class QueryRunner {
if (typeof resp === "string") { constructor(input, flags = { noRecursiveQuery: false }) {
try { this.appId = input.appId
resp = JSON.parse(resp) this.datasource = input.datasource
} catch (err) { this.queryVerb = input.queryVerb
resp = { response: resp } this.fields = input.fields
this.parameters = input.parameters
this.transformer = input.transformer
this.queryId = input.queryId
this.noRecursiveQuery = flags.noRecursiveQuery
this.cachedVariables = []
// allows the response from a query to be stored throughout this
// execution so that if it needs to be re-used for another variable
// it can be
this.queryResponse = {}
this.hasRerun = false
} }
}
return resp
}
function hasExtraData(response) { async execute() {
return ( let { datasource, fields, queryVerb, transformer } = this
typeof response === "object" && // pre-query, make sure datasource variables are added to parameters
!Array.isArray(response) && const parameters = await this.addDatasourceVariables()
response.data != null && const query = threadUtils.enrichQueryFields(fields, parameters)
response.info != null
)
}
async function runAndTransform(datasource, queryVerb, query, transformer) {
const Integration = integrations[datasource.source] const Integration = integrations[datasource.source]
if (!Integration) { if (!Integration) {
throw "Integration type does not exist." throw "Integration type does not exist."
} }
const integration = new Integration(datasource.config) const integration = new Integration(datasource.config)
let output = formatResponse(await integration[queryVerb](query)) let output = threadUtils.formatResponse(await integration[queryVerb](query))
let rows = output, let rows = output,
info = undefined, info = undefined,
extra = undefined extra = undefined
if (hasExtraData(output)) { if (threadUtils.hasExtraData(output)) {
rows = output.data rows = output.data
info = output.info info = output.info
extra = output.extra extra = output.extra
@ -45,6 +50,19 @@ async function runAndTransform(datasource, queryVerb, query, transformer) {
rows = runner.execute() rows = runner.execute()
} }
// if the request fails we retry once, invalidating the cached value
if (
info &&
info.code >= 400 &&
this.cachedVariables.length > 0 &&
!this.hasRerun
) {
this.hasRerun = true
// invalidate the cache value
await threadUtils.invalidateDynamicVariables(this.cachedVariables)
return this.execute()
}
// needs to an array for next step // needs to an array for next step
if (!Array.isArray(rows)) { if (!Array.isArray(rows)) {
rows = [rows] rows = [rows]
@ -63,15 +81,86 @@ async function runAndTransform(datasource, queryVerb, query, transformer) {
} }
return { rows, keys, info, extra } return { rows, keys, info, extra }
}
async runAnotherQuery(queryId, parameters) {
const db = new CouchDB(this.appId)
const query = await db.get(queryId)
const datasource = await db.get(query.datasourceId)
return new QueryRunner(
{
appId: this.appId,
datasource,
queryVerb: query.queryVerb,
fields: query.fields,
parameters,
transformer: query.transformer,
},
{ noRecursiveQuery: true }
).execute()
}
async getDynamicVariable(variable) {
let { parameters } = this
const queryId = variable.queryId,
name = variable.name
let value = await threadUtils.checkCacheForDynamicVariable(queryId, name)
if (!value) {
value = this.queryResponse[queryId]
? this.queryResponse[queryId]
: await this.runAnotherQuery(queryId, parameters)
// store incase this query is to be called again
this.queryResponse[queryId] = value
await threadUtils.storeDynamicVariable(queryId, name, value)
} else {
this.cachedVariables.push({ queryId, name })
}
return value
}
async addDatasourceVariables() {
let { datasource, parameters, fields } = this
if (!datasource || !datasource.config) {
return parameters
}
const staticVars = datasource.config.staticVariables || {}
const dynamicVars = datasource.config.dynamicVariables || []
for (let [key, value] of Object.entries(staticVars)) {
if (!parameters[key]) {
parameters[key] = value
}
}
if (!this.noRecursiveQuery) {
// need to see if this uses any variables
const stringFields = JSON.stringify(fields)
const foundVars = dynamicVars.filter(variable => {
// don't allow a query to use its own dynamic variable (loop)
if (variable.queryId === this.queryId) {
return false
}
// look for {{ variable }} but allow spaces between handlebars
const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`)
return regex.test(stringFields)
})
const dynamics = foundVars.map(dynVar => this.getDynamicVariable(dynVar))
const responses = await Promise.all(dynamics)
for (let i = 0; i < foundVars.length; i++) {
const variable = foundVars[i]
parameters[variable.name] = processStringSync(variable.value, {
data: responses[i].rows,
info: responses[i].extra,
})
// make sure its known that this uses dynamic variables in case it fails
this.hasDynamicVariables = true
}
}
return parameters
}
} }
module.exports = (input, callback) => { module.exports = (input, callback) => {
runAndTransform( const Runner = new QueryRunner(input)
input.datasource, Runner.execute()
input.queryVerb,
input.query,
input.transformer
)
.then(response => { .then(response => {
callback(null, response) callback(null, response)
}) })

114
packages/server/src/threads/utils.js
View File

@ -1,6 +1,26 @@
const env = require("../environment") const env = require("../environment")
const CouchDB = require("../db") const CouchDB = require("../db")
const { init } = require("@budibase/auth") const { init } = require("@budibase/auth")
const redis = require("@budibase/auth/redis")
const { SEPARATOR } = require("@budibase/auth/db")
const { processStringSync } = require("@budibase/string-templates")
const VARIABLE_TTL_SECONDS = 3600
let client
const IS_TRIPLE_BRACE = new RegExp(/^{{3}.*}{3}$/)
const IS_HANDLEBARS = new RegExp(/^{{2}.*}{2}$/)
async function getClient() {
if (!client) {
client = await new redis.Client(redis.utils.Databases.QUERY_VARS).init()
}
return client
}
process.on("exit", async () => {
if (client) await client.finish()
})
exports.threadSetup = () => { exports.threadSetup = () => {
// don't run this if not threading // don't run this if not threading
@ -11,3 +31,97 @@ exports.threadSetup = () => {
env.setInThread() env.setInThread()
init(CouchDB) init(CouchDB)
} }
function makeVariableKey(queryId, variable) {
return `${queryId}${SEPARATOR}${variable}`
}
exports.checkCacheForDynamicVariable = async (queryId, variable) => {
const cache = await getClient()
return cache.get(makeVariableKey(queryId, variable))
}
exports.invalidateDynamicVariables = async cachedVars => {
let promises = []
for (let variable of cachedVars) {
promises.push(
client.delete(makeVariableKey(variable.queryId, variable.name))
)
}
await Promise.all(promises)
}
exports.storeDynamicVariable = async (queryId, variable, value) => {
const cache = await getClient()
await cache.store(
makeVariableKey(queryId, variable),
value,
VARIABLE_TTL_SECONDS
)
}
exports.formatResponse = resp => {
if (typeof resp === "string") {
try {
resp = JSON.parse(resp)
} catch (err) {
resp = { response: resp }
}
}
return resp
}
exports.hasExtraData = response => {
return (
typeof response === "object" &&
!Array.isArray(response) &&
response.data != null &&
response.info != null
)
}
exports.enrichQueryFields = (fields, parameters = {}) => {
const enrichedQuery = {}
// enrich the fields with dynamic parameters
for (let key of Object.keys(fields)) {
if (fields[key] == null) {
continue
}
if (typeof fields[key] === "object") {
// enrich nested fields object
enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters)
} else if (typeof fields[key] === "string") {
// enrich string value as normal
let value = fields[key]
// add triple brace to avoid escaping e.g. '=' in cookie header
if (IS_HANDLEBARS.test(value) && !IS_TRIPLE_BRACE.test(value)) {
value = `{${value}}`
}
enrichedQuery[key] = processStringSync(value, parameters, {
noHelpers: true,
})
} else {
enrichedQuery[key] = fields[key]
}
}
if (
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
) {
try {
enrichedQuery.json = JSON.parse(
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
)
} catch (err) {
// no json found, ignore
}
delete enrichedQuery.customData
}
return enrichedQuery
}

925
packages/server/yarn.lock
View File

File diff suppressed because it is too large Load Diff

5
packages/worker/src/api/controllers/global/auth.js
View File

@ -12,7 +12,7 @@ const {
hash, hash,
platformLogout, platformLogout,
} = authPkg.utils } = authPkg.utils
const { Cookies } = authPkg.constants const { Cookies, Headers } = authPkg.constants
const { passport } = authPkg.auth const { passport } = authPkg.auth
const { checkResetPasswordCode } = require("../../../utilities/redis") const { checkResetPasswordCode } = require("../../../utilities/redis")
const { const {
@ -60,7 +60,10 @@ async function authInternal(ctx, user, err = null, info = null) {
return ctx.throw(403, info ? info : "Unauthorized") return ctx.throw(403, info ? info : "Unauthorized")
} }
// set a cookie for browser access
setCookie(ctx, user.token, Cookies.Auth, { sign: false }) setCookie(ctx, user.token, Cookies.Auth, { sign: false })
// set the token in a header as well for APIs
ctx.set(Headers.TOKEN, user.token)
// get rid of any app cookies on login // get rid of any app cookies on login
// have to check test because this breaks cypress // have to check test because this breaks cypress
if (!env.isTest()) { if (!env.isTest()) {