MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3830 from Budibase/feature/query-variables 

Datasource static and dynamic variables
This commit is contained in:
Michael Drury 2022-01-05 15:22:30 +00:00 committed by GitHub
parent ba5586e601 8d9db5c846
commit fe40e3e85d
36 changed files with 1828 additions and 242 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/auth/src/constants.js
View File

@ -16,6 +16,7 @@ exports.Headers = {
APP_ID: "x-budibase-app-id",
TYPE: "x-budibase-type",
TENANT_ID: "x-budibase-tenant-id",
TOKEN: "x-budibase-token",
}
exports.GlobalRoles = {

7
packages/auth/src/middleware/authenticated.js
View File

@ -1,5 +1,5 @@
const { Cookies, Headers } = require("../constants")
const { getCookie, clearCookie } = require("../utils")
const { getCookie, clearCookie, openJwt } = require("../utils")
const { getUser } = require("../cache/user")
const { getSession, updateSessionTTL } = require("../security/sessions")
const { buildMatcherRegex, matches } = require("./matchers")
@ -35,8 +35,9 @@ module.exports = (
publicEndpoint = true
}
try {
// check the actual user is authenticated first
const authCookie = getCookie(ctx, Cookies.Auth)
// check the actual user is authenticated first, try header or cookie
const headerToken = ctx.request.headers[Headers.TOKEN]
const authCookie = getCookie(ctx, Cookies.Auth) || openJwt(headerToken)
let authenticated = false,
user = null,
internal = false

1
packages/auth/src/redis/utils.js
View File

@ -16,6 +16,7 @@ exports.Databases = {
USER_CACHE: "users",
FLAGS: "flags",
APP_METADATA: "appMetadata",
QUERY_VARS: "queryVars",
}
exports.SEPARATOR = SEPARATOR

13
packages/auth/src/utils.js
View File

@ -63,6 +63,17 @@ exports.getAppId = ctx => {
return appId
}
/**
* opens the contents of the specified encrypted JWT.
* @return {object} the contents of the token.
*/
exports.openJwt = token => {
if (!token) {
return token
}
return jwt.verify(token, options.secretOrKey)
}
/**
* Get a cookie from context, and decrypt if necessary.
* @param {object} ctx The request which is to be manipulated.
@ -75,7 +86,7 @@ exports.getCookie = (ctx, name) => {
return cookie
}
return jwt.verify(cookie, options.secretOrKey)
return exports.openJwt(cookie)
}
/**

11
packages/bbui/src/Table/BoldRenderer.svelte Normal file
View File

@ -0,0 +1,11 @@
<script>
export let value
</script>
<div class="bold">{value}</div>
<style>
.bold {
font-weight: bold;
}
</style>

5
packages/bbui/src/Table/CodeRenderer.svelte Normal file
View File

@ -0,0 +1,5 @@
<script>
export let value
</script>
<code>{value}</code>

4
packages/bbui/src/index.js
View File

@ -61,6 +61,10 @@ export { default as ColorPicker } from "./ColorPicker/ColorPicker.svelte"
export { default as InlineAlert } from "./InlineAlert/InlineAlert.svelte"
export { default as Banner } from "./Banner/Banner.svelte"
// Renderers
export { default as BoldRenderer } from "./Table/BoldRenderer.svelte"
export { default as CodeRenderer } from "./Table/CodeRenderer.svelte"
// Typography
export { default as Body } from "./Typography/Body.svelte"
export { default as Heading } from "./Typography/Heading.svelte"

8
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte
View File

@ -175,6 +175,7 @@
onConfirm={datasources.removeSchemaError}
/>
{/if}
{#if plusTables && Object.values(plusTables).length > 0}
<Table
on:click={({ detail }) => onClickTable(detail)}
schema={tableSchema}
@ -184,6 +185,9 @@
allowSelectRows={false}
customRenderers={[{ column: "primary", component: ArrayRenderer }]}
/>
{:else}
<Body size="S"><i>No tables found.</i></Body>
{/if}
{#if plusTables?.length !== 0}
<Divider size="S" />
<div class="query-header">
@ -196,6 +200,7 @@
Tell budibase how your tables are related to get even more smart features.
</Body>
{/if}
{#if relationshipInfo && relationshipInfo.length > 0}
<Table
on:click={({ detail }) => openRelationshipModal(detail.from, detail.to)}
schema={relationshipSchema}
@ -204,6 +209,9 @@
allowEditRows={false}
allowSelectRows={false}
/>
{:else}
<Body size="S"><i>No relationships configured.</i></Body>
{/if}
<style>
.query-header {

41
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte
View File

@ -1,9 +1,18 @@
<script>
import { Divider, Heading, ActionButton, Badge, Body } from "@budibase/bbui"
import {
Divider,
Heading,
ActionButton,
Badge,
Body,
Layout,
} from "@budibase/bbui"
import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
import RestAuthenticationBuilder from "./auth/RestAuthenticationBuilder.svelte"
import ViewDynamicVariables from "./variables/ViewDynamicVariables.svelte"
export let datasource
export let queries
let addHeader
</script>
@ -43,6 +52,36 @@
</Body>
<RestAuthenticationBuilder bind:configs={datasource.config.authConfigs} />
<Divider size="S" />
<div class="section-header">
<div class="badge">
<Heading size="S">Variables</Heading>
<Badge quiet grey>Optional</Badge>
</div>
</div>
<Body size="S"
>Variables enable you to store and re-use values in queries, with the choice
of a static value such as a token using static variables, or a value from a
query response using dynamic variables.</Body
>
<Heading size="XS">Static</Heading>
<Layout noPadding gap="XS">
<KeyValueBuilder
name="Variable"
keyPlaceholder="Name"
headings
bind:object={datasource.config.staticVariables}
on:change
/>
</Layout>
<div />
<Heading size="XS">Dynamic</Heading>
<Body size="S">
Dynamic variables are evaluated when a dependant query is executed. The value
is cached for a period of time and will be refreshed if a query fails.
</Body>
<ViewDynamicVariables {queries} {datasource} />
<style>
.section-header {
display: flex;

54
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/variables/ViewDynamicVariables.svelte Normal file
View File

@ -0,0 +1,54 @@
<script>
import { Body, Table, BoldRenderer, CodeRenderer } from "@budibase/bbui"
import { queries as queriesStore } from "stores/backend"
import { goto } from "@roxi/routify"
export let datasource
export let queries
let dynamicVariables = []
$: enrichDynamicVariables(datasource, queries)
const dynamicVariableSchema = {
name: "",
query: "",
value: "",
}
const onClick = dynamicVariable => {
const queryId = dynamicVariable.queryId
queriesStore.select({ _id: queryId })
$goto(`./${queryId}`)
}
/**
* Add the query name to the dynamic variables
*/
function enrichDynamicVariables(ds, possibleQueries) {
dynamicVariables = []
ds.config.dynamicVariables?.forEach(dv => {
const query = possibleQueries.find(query => query._id === dv.queryId)
if (query) {
dynamicVariables.push({ ...dv, query: query.name })
}
})
}
</script>
{#if dynamicVariables && dynamicVariables.length > 0}
<Table
on:click={({ detail }) => onClick(detail)}
schema={dynamicVariableSchema}
data={dynamicVariables}
allowEditColumns={false}
allowEditRows={false}
allowSelectRows={false}
customRenderers={[
{ column: "name", component: BoldRenderer },
{ column: "value", component: CodeRenderer },
]}
/>
{:else}
<Body size="S"><i>No dynamic variables specified.</i></Body>
{/if}

41
packages/builder/src/components/integration/KeyValueBuilder.svelte
View File

@ -6,6 +6,8 @@
Label,
Toggle,
Select,
ActionMenu,
MenuItem,
} from "@budibase/bbui"
import { createEventDispatcher } from "svelte"
import { lowercase } from "helpers"
@ -23,7 +25,11 @@
export let toggle
export let keyPlaceholder = "Key"
export let valuePlaceholder = "Value"
export let valueHeading
export let keyHeading
export let tooltip
export let menuItems
export let showMenu = false
let fields = Object.entries(object).map(([name, value]) => ({ name, value }))
let fieldActivity = buildFieldActivity(activity)
@ -76,17 +82,24 @@
{#if Object.keys(object || {}).length > 0}
{#if headings}
<div class="container" class:container-active={toggle}>
<Label {tooltip}>{keyPlaceholder}</Label>
<Label>{valuePlaceholder}</Label>
<Label {tooltip}>{keyHeading || keyPlaceholder}</Label>
<Label>{valueHeading || valuePlaceholder}</Label>
{#if toggle}
<Label>Active</Label>
{/if}
</div>
{/if}
<div class="container" class:container-active={toggle} class:readOnly>
<div
class="container"
class:container-active={toggle}
class:container-menu={showMenu}
class:readOnly
class:readOnly-menu={readOnly && showMenu}
>
{#each fields as field, idx}
<Input
placeholder={keyPlaceholder}
readonly={readOnly}
bind:value={field.name}
on:change={changed}
/>
@ -95,6 +108,7 @@
{:else}
<Input
placeholder={valuePlaceholder}
readonly={readOnly}
bind:value={field.value}
on:change={changed}
/>
@ -105,6 +119,18 @@
{#if !readOnly}
<Icon hoverable name="Close" on:click={() => deleteEntry(idx)} />
{/if}
{#if menuItems?.length > 0 && showMenu}
<ActionMenu>
<div slot="control" class="control icon">
<Icon size="S" hoverable name="MoreSmallList" />
</div>
{#each menuItems as item}
<MenuItem on:click={() => item.onClick(field)}>
{item.text}
</MenuItem>
{/each}
</ActionMenu>
{/if}
{/each}
</div>
{/if}
@ -127,7 +153,16 @@
.container-active {
grid-template-columns: 1fr 1fr 50px 20px;
}
.container-menu {
grid-template-columns: 1fr 1fr 20px 20px;
}
.readOnly {
grid-template-columns: 1fr 1fr;
}
.readOnly-menu {
grid-template-columns: 1fr 1fr 20px;
}
.control {
margin-top: 4px;
}
</style>

10
packages/builder/src/helpers/data/utils.js
View File

@ -119,3 +119,13 @@ export function flipHeaderState(headersActivity) {
})
return enabled
}
export default {
breakQueryString,
buildQueryString,
fieldsToSchema,
flipHeaderState,
keyValueToQueryParameters,
queryParametersToKeyValue,
schemaToFields,
}

54
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte Normal file
View File

@ -0,0 +1,54 @@
<script>
import { Input, ModalContent, Modal, Body } from "@budibase/bbui"
export let dynamicVariables
export let datasource
export let binding
let name, modal, valid, allVariableNames
export const show = () => {
modal.show()
}
export const hide = () => {
modal.hide()
}
function checkValid(vars, name) {
if (!name) {
return false
}
return !allVariableNames.find(
varName => varName.toLowerCase() === name.toLowerCase()
)
}
$: valid = checkValid(dynamicVariables, name)
$: allVariableNames = (datasource?.config?.dynamicVariables || []).map(
variable => variable.name
)
$: error = name && !valid ? "Variable name is already in use." : null
async function saveVariable() {
const copiedName = name,
copiedBinding = binding
name = null
binding = null
dynamicVariables[copiedName] = copiedBinding
}
</script>
<Modal bind:this={modal}>
<ModalContent
title="Add dynamic variable"
confirmText="Save"
onConfirm={saveVariable}
disabled={!valid}
>
<Body size="S"
>Specify a name for your new dynamic variable, this must be unique across
your datasource.</Body
>
<Input label="Variable name" bind:value={name} on:input {error} />
</ModalContent>
</Modal>

32
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/RestBodyInput.svelte
View File

@ -16,17 +16,33 @@
export let query
export let bodyType
let text = ""
let json = ""
$: checkRequestBody(bodyType)
$: updateRequestBody(bodyType, text, json)
function checkRequestBody(type) {
if (!bodyType || !query) {
return
}
const currentType = typeof query?.fields.requestBody
if (objectTypes.includes(type) && currentType !== "object") {
query.fields.requestBody = {}
} else if (textTypes.includes(type) && currentType !== "string") {
query.fields.requestBody = ""
const isObject = objectTypes.includes(type)
const isText = textTypes.includes(type)
if (isText && currentType === "string") {
text = query.fields.requestBody
} else if (isObject && currentType === "object") {
json = query.fields.requestBody
}
}
function updateRequestBody(type, text, json) {
if (type === RawRestBodyTypes.NONE) {
query.fields.requestBody = null
} else if (objectTypes.includes(type)) {
query.fields.requestBody = json
} else {
query.fields.requestBody = text
}
}
@ -49,16 +65,12 @@
<Body size="S" weight="800">THE REQUEST DOES NOT HAVE A BODY</Body>
</div>
{:else if objectTypes.includes(bodyType)}
<KeyValueBuilder
bind:object={query.fields.requestBody}
name="param"
headings
/>
<KeyValueBuilder bind:object={json} name="param" headings />
{:else if textTypes.includes(bodyType)}
<CodeMirrorEditor
height={200}
mode={editorMode(bodyType)}
value={query.fields.requestBody}
value={text}
resize="vertical"
on:change={e => (query.fields.requestBody = e.detail)}
/>

22
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte
View File

@ -21,11 +21,11 @@
import { cloneDeep } from "lodash/fp"
import ImportRestQueriesModal from "components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte"
import { onMount } from "svelte"
let importQueriesModal
let changed
let datasource
let integration, baseDatasource, datasource
let queryList
const querySchema = {
name: {},
queryVerb: { displayName: "Method" },
@ -34,11 +34,12 @@
$: baseDatasource = $datasources.list.find(
ds => ds._id === $datasources.selected
)
$: integration = datasource && $integrations[datasource.source]
$: queryList = $queries.list.filter(
query => query.datasourceId === datasource?._id
)
$: hasChanged(baseDatasource, datasource)
$: updateDatasource(baseDatasource)
function hasChanged(base, ds) {
if (base && ds) {
@ -66,9 +67,12 @@
$goto(`./${query._id}`)
}
onMount(() => {
datasource = cloneDeep(baseDatasource)
})
function updateDatasource(base) {
if (base) {
datasource = cloneDeep(base)
integration = $integrations[datasource.source]
}
}
</script>
<Modal bind:this={importQueriesModal}>
@ -142,7 +146,11 @@
</div>
{/if}
{#if datasource?.source === IntegrationTypes.REST}
<RestExtraConfigForm bind:datasource on:change={hasChanged} />
<RestExtraConfigForm
queries={queryList}
bind:datasource
on:change={hasChanged}
/>
{/if}
</Layout>
</section>

210
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
View File

@ -1,22 +1,22 @@
<script>
import { params } from "@roxi/routify"
import { datasources, integrations, queries, flags } from "stores/backend"
import { datasources, flags, integrations, queries } from "stores/backend"
import {
Layout,
Input,
Select,
Tabs,
Tab,
Banner,
Divider,
Button,
Heading,
RadioGroup,
Label,
Body,
TextArea,
Table,
Button,
Divider,
Heading,
Input,
Label,
Layout,
notifications,
RadioGroup,
Select,
Tab,
Table,
Tabs,
TextArea,
} from "@budibase/bbui"
import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
import EditableLabel from "components/common/inputs/EditableLabel.svelte"
@ -26,42 +26,37 @@
import RestBodyInput from "../../_components/RestBodyInput.svelte"
import { capitalise } from "helpers"
import { onMount } from "svelte"
import {
fieldsToSchema,
schemaToFields,
breakQueryString,
buildQueryString,
keyValueToQueryParameters,
queryParametersToKeyValue,
flipHeaderState,
} from "helpers/data/utils"
import restUtils from "helpers/data/utils"
import {
RestBodyTypes as bodyTypes,
SchemaTypeOptions,
} from "constants/backend"
import JSONPreview from "components/integration/JSONPreview.svelte"
import AccessLevelSelect from "components/integration/AccessLevelSelect.svelte"
import DynamicVariableModal from "../../_components/DynamicVariableModal.svelte"
import Placeholder from "assets/bb-spaceship.svg"
import { cloneDeep } from "lodash/fp"
import { RawRestBodyTypes } from "constants/backend"
let query, datasource
let breakQs = {},
bindings = {}
let url = ""
let saveId, isGet
let saveId, url
let response, schema, enabledHeaders
let datasourceType, integrationInfo, queryConfig, responseSuccess
let authConfigId
let dynamicVariables, addVariableModal, varBinding
$: datasourceType = datasource?.source
$: integrationInfo = $integrations[datasourceType]
$: queryConfig = integrationInfo?.query
$: url = buildUrl(url, breakQs)
$: checkQueryName(url)
$: responseSuccess = response?.info?.code >= 200 && response?.info?.code < 400
$: isGet = query?.queryVerb === "read"
$: responseSuccess =
response?.info?.code >= 200 && response?.info?.code <= 206
$: authConfigs = buildAuthConfigs(datasource)
$: schemaReadOnly = !responseSuccess
$: variablesReadOnly = !responseSuccess
$: showVariablesTab = shouldShowVariables(dynamicVariables, variablesReadOnly)
function getSelectedQuery() {
return cloneDeep(
@ -89,7 +84,7 @@
if (!base) {
return base
}
const qs = buildQueryString(qsObj)
const qs = restUtils.buildQueryString(qsObj)
let newUrl = base
if (base.includes("?")) {
newUrl = base.split("?")[0]
@ -97,29 +92,15 @@
return qs.length > 0 ? `${newUrl}?${qs}` : newUrl
}
const buildAuthConfigs = datasource => {
if (datasource?.config?.authConfigs) {
return datasource.config.authConfigs.map(c => ({
label: c.name,
value: c._id,
}))
}
return []
}
function learnMoreBanner() {
window.open("https://docs.budibase.com/building-apps/data/transformers")
}
function buildQuery() {
const newQuery = { ...query }
const queryString = buildQueryString(breakQs)
const queryString = restUtils.buildQueryString(breakQs)
newQuery.fields.path = url.split("?")[0]
newQuery.fields.queryString = queryString
newQuery.fields.authConfigId = authConfigId
newQuery.fields.disabledHeaders = flipHeaderState(enabledHeaders)
newQuery.schema = fieldsToSchema(schema)
newQuery.parameters = keyValueToQueryParameters(bindings)
newQuery.fields.disabledHeaders = restUtils.flipHeaderState(enabledHeaders)
newQuery.schema = restUtils.fieldsToSchema(schema)
newQuery.parameters = restUtils.keyValueToQueryParameters(bindings)
return newQuery
}
@ -130,8 +111,13 @@
saveId = _id
query = getSelectedQuery()
notifications.success(`Request saved successfully.`)
if (dynamicVariables) {
datasource.config.dynamicVariables = rebuildVariables(saveId)
datasource = await datasources.save(datasource)
}
} catch (err) {
notifications.error(`Error creating query. ${err.message}`)
notifications.error(`Error saving query. ${err.message}`)
}
}
@ -166,6 +152,78 @@
return id
}
const buildAuthConfigs = datasource => {
if (datasource?.config?.authConfigs) {
return datasource.config.authConfigs.map(c => ({
label: c.name,
value: c._id,
}))
}
return []
}
const schemaMenuItems = [
{
text: "Create dynamic variable",
onClick: input => {
varBinding = `{{ data.0.[${input.name}] }}`
addVariableModal.show()
},
},
]
const responseHeadersMenuItems = [
{
text: "Create dynamic variable",
onClick: input => {
varBinding = `{{ info.headers.[${input.name}] }}`
addVariableModal.show()
},
},
]
// convert dynamic variables list to simple key/val object
const getDynamicVariables = (datasource, queryId) => {
const variablesList = datasource?.config?.dynamicVariables
if (variablesList && variablesList.length > 0) {
const filtered = queryId
? variablesList.filter(variable => variable.queryId === queryId)
: variablesList
return filtered.reduce(
(acc, next) => ({ ...acc, [next.name]: next.value }),
{}
)
}
return {}
}
// convert dynamic variables object back to a list, enrich with query id
const rebuildVariables = queryId => {
let variables = []
if (dynamicVariables) {
variables = Object.entries(dynamicVariables).map(entry => {
return {
name: entry[0],
value: entry[1],
queryId,
}
})
}
let existing = datasource?.config?.dynamicVariables || []
// remove existing query variables (for changes and deletions)
existing = existing.filter(variable => variable.queryId !== queryId)
// re-add the new query variables
return [...existing, ...variables]
}
const shouldShowVariables = (dynamicVariables, variablesReadOnly) => {
return !!(
dynamicVariables &&
// show when editable or when read only and not empty
(!variablesReadOnly || Object.keys(dynamicVariables).length > 0)
)
}
onMount(async () => {
query = getSelectedQuery()
// clear any unsaved changes to the datasource
@ -173,14 +231,14 @@
datasource = $datasources.list.find(ds => ds._id === query?.datasourceId)
const datasourceUrl = datasource?.config.url
const qs = query?.fields.queryString
breakQs = breakQueryString(qs)
breakQs = restUtils.breakQueryString(qs)
if (datasourceUrl && !query.fields.path?.startsWith(datasourceUrl)) {
const path = query.fields.path
query.fields.path = `${datasource.config.url}/${path ? path : ""}`
}
url = buildUrl(query.fields.path, breakQs)
schema = schemaToFields(query.schema)
bindings = queryParametersToKeyValue(query.parameters)
schema = restUtils.schemaToFields(query.schema)
bindings = restUtils.queryParametersToKeyValue(query.parameters)
authConfigId = getAuthConfigId()
if (!query.fields.disabledHeaders) {
query.fields.disabledHeaders = {}
@ -191,7 +249,7 @@
query.fields.disabledHeaders[header] = false
}
}
enabledHeaders = flipHeaderState(query.fields.disabledHeaders)
enabledHeaders = restUtils.flipHeaderState(query.fields.disabledHeaders)
if (query && !query.transformer) {
query.transformer = "return data"
}
@ -201,11 +259,22 @@
}
}
if (query && !query.fields.bodyType) {
query.fields.bodyType = "none"
if (query.fields.requestBody) {
query.fields.bodyType = RawRestBodyTypes.JSON
} else {
query.fields.bodyType = RawRestBodyTypes.NONE
}
}
dynamicVariables = getDynamicVariables(datasource, query._id)
})
</script>
<DynamicVariableModal
{datasource}
{dynamicVariables}
bind:binding={varBinding}
bind:this={addVariableModal}
/>
{#if query && queryConfig}
<div class="inner">
<div class="top">
@ -275,7 +344,10 @@
{#if !$flags.queryTransformerBanner}
<Banner
extraButtonText="Learn more"
extraButtonAction={learnMoreBanner}
extraButtonAction={() =>
window.open(
"https://docs.budibase.com/building-apps/data/transformers"
)}
on:change={() =>
flags.updateFlag("queryTransformerBanner", true)}
>
@ -341,6 +413,9 @@
name="schema"
headings
options={SchemaTypeOptions}
menuItems={schemaMenuItems}
showMenu={!schemaReadOnly}
readOnly={schemaReadOnly}
/>
</Tab>
{/if}
@ -349,7 +424,12 @@
<TextArea disabled value={response.extra?.raw} height="300" />
</Tab>
<Tab title="Headers">
<KeyValueBuilder object={response.extra?.headers} readOnly />
<KeyValueBuilder
object={response.extra?.headers}
readOnly
menuItems={responseHeadersMenuItems}
showMenu={true}
/>
</Tab>
<Tab title="Preview">
<div class="table">
@ -364,6 +444,28 @@
{/if}
</div>
</Tab>
{/if}
{#if showVariablesTab}
<Tab title="Dynamic Variables">
<Layout noPadding gap="S">
<Body size="S">
Create dynamic variables based on response body or headers
from other queries.
</Body>
<KeyValueBuilder
bind:object={dynamicVariables}
name="Variable"
headings
keyHeading="Name"
keyPlaceholder="Variable name"
valueHeading={`Value`}
valuePlaceholder={`{{ value }}`}
readOnly={variablesReadOnly}
/>
</Layout>
</Tab>
{/if}
{#if response}
<div class="stats">
<Label size="L">
Status: <span class={responseSuccess ? "green" : "red"}

1
packages/builder/src/stores/backend/queries.js
View File

@ -91,6 +91,7 @@ export function createQueriesStore() {
{}
),
datasourceId: query.datasourceId,
queryId: query._id || undefined,
})
if (response.status !== 200) {

8
packages/client/manifest.json
View File

@ -2557,6 +2557,10 @@
"label": "Rows",
"key": "rows"
},
{
"label": "Extra Info",
"key": "info"
},
{
"label": "Rows Length",
"key": "rowsLength"
@ -3278,6 +3282,10 @@
"label": "Rows",
"key": "rows"
},
{
"label": "Extra Info",
"key": "info"
},
{
"label": "Rows Length",
"key": "rowsLength"

12
packages/client/src/api/datasources.js
View File

@ -19,7 +19,8 @@ export const fetchDatasource = async dataSource => {
// Fetch all rows in data source
const { type, tableId, fieldName } = dataSource
let rows = []
let rows = [],
info = {}
if (type === "table") {
rows = await fetchTableData(tableId)
} else if (type === "view") {
@ -32,7 +33,12 @@ export const fetchDatasource = async dataSource => {
parameters[param.name] = param.default
}
}
rows = await executeQuery({ queryId: dataSource._id, parameters })
const { data, ...rest } = await executeQuery({
queryId: dataSource._id,
parameters,
})
info = rest
rows = data
} else if (type === FieldTypes.LINK) {
rows = await fetchRelationshipData({
rowId: dataSource.rowId,
@ -42,7 +48,7 @@ export const fetchDatasource = async dataSource => {
}
// Enrich the result is always an array
return Array.isArray(rows) ? rows : []
return { rows: Array.isArray(rows) ? rows : [], info }
}
/**

2
packages/client/src/api/queries.js
View File

@ -11,7 +11,7 @@ export const executeQuery = async ({ queryId, parameters }) => {
return
}
const res = await API.post({
url: `/api/queries/${queryId}`,
url: `/api/v2/queries/${queryId}`,
body: {
parameters,
},

8
packages/client/src/components/app/DataProvider.svelte
View File

@ -30,6 +30,7 @@
// Provider state
let rows = []
let allRows = []
let info = {}
let schema = {}
let bookmarks = [null]
let pageNumber = 0
@ -120,8 +121,9 @@
// Build our data context
$: dataContext = {
rows,
info,
schema,
rowsLength: rows.length,
rowsLength: rows?.length,
// Undocumented properties. These aren't supposed to be used in builder
// bindings, but are used internally by other components
@ -209,7 +211,9 @@
} else {
// For other data sources like queries or views, fetch all rows from the
// server
allRows = await API.fetchDatasource(dataSource)
const data = await API.fetchDatasource(dataSource)
allRows = data.rows
info = data.info
}
loading = false
loaded = true

18
packages/server/__mocks__/node-fetch.ts
View File

@ -1,5 +1,6 @@
module FetchMock {
const fetch = jest.requireActual("node-fetch")
let failCount = 0
module.exports = async (url: any, opts: any) => {
function json(body: any, status = 200) {
@ -57,6 +58,23 @@ module FetchMock {
],
bookmark: "test",
})
} else if (url.includes("google.com")) {
return json({
url,
opts,
value: "<!doctype html><html itemscope=\"\" itemtype=\"http://schema.org/WebPage\" lang=\"en-GB\"></html>",
})
} else if (url.includes("failonce.com")) {
failCount++
if (failCount === 1) {
return json({ message: "error" }, 500)
} else {
return json({
fails: failCount - 1,
url,
opts,
})
}
}
return fetch(url, opts)
}

1
packages/server/package.json
View File

@ -122,6 +122,7 @@
"pouchdb-replication-stream": "1.2.9",
"server-destroy": "1.0.1",
"svelte": "^3.38.2",
"swagger-parser": "^10.0.3",
"to-json-schema": "0.2.5",
"uuid": "3.3.2",
"validate.js": "0.13.1",

6
packages/server/src/api/controllers/query/import/sources/curl.ts
View File

@ -17,8 +17,12 @@ const parseBody = (curl: any) => {
if (curl.data) {
const keys = Object.keys(curl.data)
if (keys.length) {
const key = keys[0]
let key = keys[0]
try {
// filter out the dollar syntax used by curl for shell support
if (key.startsWith("$")) {
key = key.substring(1)
}
return JSON.parse(key)
} catch (e) {
// do nothing

80
packages/server/src/api/controllers/query/index.js
View File

@ -1,4 +1,3 @@
const { processString } = require("@budibase/string-templates")
const CouchDB = require("../../../db")
const {
generateQueryID,
@ -90,47 +89,6 @@ exports.save = async function (ctx) {
ctx.message = `Query ${query.name} saved successfully.`
}
async function enrichQueryFields(fields, parameters = {}) {
const enrichedQuery = {}
// enrich the fields with dynamic parameters
for (let key of Object.keys(fields)) {
if (fields[key] == null) {
continue
}
if (typeof fields[key] === "object") {
// enrich nested fields object
enrichedQuery[key] = await enrichQueryFields(fields[key], parameters)
} else if (typeof fields[key] === "string") {
// enrich string value as normal
enrichedQuery[key] = await processString(fields[key], parameters, {
noHelpers: true,
})
} else {
enrichedQuery[key] = fields[key]
}
}
if (
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
) {
try {
enrichedQuery.json = JSON.parse(
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
)
} catch (err) {
// no json found, ignore
}
delete enrichedQuery.customData
}
return enrichedQuery
}
exports.find = async function (ctx) {
const db = new CouchDB(ctx.appId)
const query = enrichQueries(await db.get(ctx.params.queryId))
@ -146,16 +104,20 @@ exports.preview = async function (ctx) {
const db = new CouchDB(ctx.appId)
const datasource = await db.get(ctx.request.body.datasourceId)
const { fields, parameters, queryVerb, transformer } = ctx.request.body
const enrichedQuery = await enrichQueryFields(fields, parameters)
// preview may not have a queryId as it hasn't been saved, but if it does
// this stops dynamic variables from calling the same query
const { fields, parameters, queryVerb, transformer, queryId } =
ctx.request.body
try {
const { rows, keys, info, extra } = await Runner.run({
appId: ctx.appId,
datasource,
queryVerb,
query: enrichedQuery,
fields,
parameters,
transformer,
queryId,
})
ctx.body = {
@ -169,31 +131,41 @@ exports.preview = async function (ctx) {
}
}
exports.execute = async function (ctx) {
async function execute(ctx, opts = { rowsOnly: false }) {
const db = new CouchDB(ctx.appId)
const query = await db.get(ctx.params.queryId)
const datasource = await db.get(query.datasourceId)
const enrichedQuery = await enrichQueryFields(
query.fields,
ctx.request.body.parameters
)
// call the relevant CRUD method on the integration class
try {
const { rows } = await Runner.run({
const { rows, extra } = await Runner.run({
appId: ctx.appId,
datasource,
queryVerb: query.queryVerb,
query: enrichedQuery,
fields: query.fields,
parameters: ctx.request.body.parameters,
transformer: query.transformer,
queryId: ctx.params.queryId,
})
if (opts && opts.rowsOnly) {
ctx.body = rows
} else {
ctx.body = { data: rows, ...extra }
}
} catch (err) {
ctx.throw(400, err)
}
}
exports.executeV1 = async function (ctx) {
return execute(ctx, { rowsOnly: true })
}
exports.executeV2 = async function (ctx) {
return execute(ctx, { rowsOnly: false })
}
exports.destroy = async function (ctx) {
const db = new CouchDB(ctx.appId)
await db.remove(ctx.params.queryId, ctx.params.revId)

3
packages/server/src/api/controllers/query/validation.js
View File

@ -36,6 +36,7 @@ exports.generateQueryPreviewValidation = () => {
extra: Joi.object().optional(),
datasourceId: Joi.string().required(),
transformer: Joi.string().optional(),
parameters: Joi.object({}).required().unknown(true)
parameters: Joi.object({}).required().unknown(true),
queryId: Joi.string().optional(),
}))
}

9
packages/server/src/api/routes/query.js
View File

@ -41,11 +41,18 @@ router
authorized(PermissionTypes.QUERY, PermissionLevels.READ),
queryController.find
)
// DEPRECATED - use new query endpoint for future work
.post(
"/api/queries/:queryId",
paramResource("queryId"),
authorized(PermissionTypes.QUERY, PermissionLevels.WRITE),
queryController.execute
queryController.executeV1
)
.post(
"/api/v2/queries/:queryId",
paramResource("queryId"),
authorized(PermissionTypes.QUERY, PermissionLevels.WRITE),
queryController.executeV2
)
.delete(
"/api/queries/:queryId/:revId",

88
packages/server/src/api/routes/tests/query.spec.js
View File

@ -1,5 +1,6 @@
// Mock out postgres for this
jest.mock("pg")
jest.mock("node-fetch")
// Mock isProdAppID to we can later mock the implementation and pretend we are
// using prod app IDs
@ -10,6 +11,7 @@ authDb.isProdAppID = mockIsProdAppID
const setup = require("./utilities")
const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
const { checkCacheForDynamicVariable } = require("../../../threads/utils")
const { basicQuery, basicDatasource } = setup.structures
describe("/queries", () => {
@ -226,4 +228,90 @@ describe("/queries", () => {
.expect(400)
})
})
describe("test variables", () => {
async function restDatasource(cfg) {
return await config.createDatasource({
datasource: {
...basicDatasource().datasource,
source: "REST",
config: cfg || {},
},
})
}
async function dynamicVariableDatasource() {
const datasource = await restDatasource()
const basedOnQuery = await config.createQuery({
...basicQuery(datasource._id),
fields: {
path: "www.google.com",
},
})
await config.updateDatasource({
...datasource,
config: {
dynamicVariables: [
{ queryId: basedOnQuery._id, name: "variable3", value: "{{ data.0.[value] }}" }
]
}
})
return { datasource, query: basedOnQuery }
}
async function preview(datasource, fields) {
return await request
.post(`/api/queries/preview`)
.send({
datasourceId: datasource._id,
parameters: {},
fields,
queryVerb: "read",
})
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
}
it("should work with static variables", async () => {
const datasource = await restDatasource({
staticVariables: {
variable: "google",
variable2: "1",
},
})
const res = await preview(datasource, {
path: "www.{{ variable }}.com",
queryString: "test={{ variable2 }}",
})
// these responses come from the mock
expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
expect(res.body.rows[0].url).toEqual("http://www.google.com?test=1")
})
it("should work with dynamic variables", async () => {
const { datasource } = await dynamicVariableDatasource()
const res = await preview(datasource, {
path: "www.google.com",
queryString: "test={{ variable3 }}",
})
expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
expect(res.body.rows[0].url).toContain("doctype html")
})
it("check that it automatically retries on fail with cached dynamics", async () => {
const { datasource, query: base } = await dynamicVariableDatasource()
// preview once to cache
await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
// check its in cache
const contents = await checkCacheForDynamicVariable(base._id, "variable3")
expect(contents.rows.length).toEqual(1)
const res = await preview(datasource, {
path: "www.failonce.com",
queryString: "test={{ variable3 }}",
})
expect(res.body.schemaFields).toEqual(["fails", "url", "opts"])
expect(res.body.rows[0].fails).toEqual(1)
})
})
})

10
packages/server/src/automations/steps/executeQuery.js
View File

@ -35,6 +35,11 @@ exports.definition = {
type: "object",
description: "The response from the datasource execution",
},
info: {
type: "object",
description:
"Some query types may return extra data, like headers from a REST query",
},
success: {
type: "boolean",
description: "Whether the action was successful",
@ -68,13 +73,16 @@ exports.run = async function ({ inputs, appId, emitter }) {
try {
await queryController.execute(ctx)
const { data, ...rest } = ctx.body
return {
response: ctx.body,
response: data,
info: rest,
success: true,
}
} catch (err) {
return {
success: false,
info: {},
response: automationUtils.getError(err),
}
}

10
packages/server/src/definitions/datasource.ts
View File

@ -240,6 +240,16 @@ export interface RestConfig {
[key: string]: any
}
authConfigs: AuthConfig[]
staticVariables: {
[key: string]: string
}
dynamicVariables: [
{
name: string
queryId: string
value: string
}
]
}
export interface Query {

7
packages/server/src/integrations/rest.ts
View File

@ -48,7 +48,10 @@ module RestModule {
const { performance } = require("perf_hooks")
const FormData = require("form-data")
const { URLSearchParams } = require("url")
const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js")
const {
parseStringPromise: xmlParser,
Builder: XmlBuilder,
} = require("xml2js")
const SCHEMA: Integration = {
docs: "https://github.com/node-fetch/node-fetch",
@ -211,7 +214,7 @@ module RestModule {
break
case BodyTypes.XML:
if (object != null) {
string = (new XmlBuilder()).buildObject(object)
string = new XmlBuilder().buildObject(object)
}
input.body = string
input.headers["Content-Type"] = "application/xml"

10
packages/server/src/tests/utilities/TestConfiguration.js
View File

@ -316,6 +316,16 @@ class TestConfiguration {
return this.datasource
}
async updateDatasource(datasource) {
const response = await this._req(
datasource,
{ datasourceId: datasource._id },
controllers.datasource.update
)
this.datasource = response.datasource
return this.datasource
}
async createQuery(config = null) {
if (!this.datasource && !config) {
throw "No data source created for query."

145
packages/server/src/threads/query.js
View File

@ -1,39 +1,44 @@
require("./utils").threadSetup()
const threadUtils = require("./utils")
threadUtils.threadSetup()
const ScriptRunner = require("../utilities/scriptRunner")
const { integrations } = require("../integrations")
const { processStringSync } = require("@budibase/string-templates")
const CouchDB = require("../db")
function formatResponse(resp) {
if (typeof resp === "string") {
try {
resp = JSON.parse(resp)
} catch (err) {
resp = { response: resp }
}
}
return resp
class QueryRunner {
constructor(input, flags = { noRecursiveQuery: false }) {
this.appId = input.appId
this.datasource = input.datasource
this.queryVerb = input.queryVerb
this.fields = input.fields
this.parameters = input.parameters
this.transformer = input.transformer
this.queryId = input.queryId
this.noRecursiveQuery = flags.noRecursiveQuery
this.cachedVariables = []
// allows the response from a query to be stored throughout this
// execution so that if it needs to be re-used for another variable
// it can be
this.queryResponse = {}
this.hasRerun = false
}
function hasExtraData(response) {
return (
typeof response === "object" &&
!Array.isArray(response) &&
response.data != null &&
response.info != null
)
}
async function runAndTransform(datasource, queryVerb, query, transformer) {
async execute() {
let { datasource, fields, queryVerb, transformer } = this
// pre-query, make sure datasource variables are added to parameters
const parameters = await this.addDatasourceVariables()
const query = threadUtils.enrichQueryFields(fields, parameters)
const Integration = integrations[datasource.source]
if (!Integration) {
throw "Integration type does not exist."
}
const integration = new Integration(datasource.config)
let output = formatResponse(await integration[queryVerb](query))
let output = threadUtils.formatResponse(await integration[queryVerb](query))
let rows = output,
info = undefined,
extra = undefined
if (hasExtraData(output)) {
if (threadUtils.hasExtraData(output)) {
rows = output.data
info = output.info
extra = output.extra
@ -45,6 +50,19 @@ async function runAndTransform(datasource, queryVerb, query, transformer) {
rows = runner.execute()
}
// if the request fails we retry once, invalidating the cached value
if (
info &&
info.code >= 400 &&
this.cachedVariables.length > 0 &&
!this.hasRerun
) {
this.hasRerun = true
// invalidate the cache value
await threadUtils.invalidateDynamicVariables(this.cachedVariables)
return this.execute()
}
// needs to an array for next step
if (!Array.isArray(rows)) {
rows = [rows]
@ -65,13 +83,84 @@ async function runAndTransform(datasource, queryVerb, query, transformer) {
return { rows, keys, info, extra }
}
async runAnotherQuery(queryId, parameters) {
const db = new CouchDB(this.appId)
const query = await db.get(queryId)
const datasource = await db.get(query.datasourceId)
return new QueryRunner(
{
appId: this.appId,
datasource,
queryVerb: query.queryVerb,
fields: query.fields,
parameters,
transformer: query.transformer,
},
{ noRecursiveQuery: true }
).execute()
}
async getDynamicVariable(variable) {
let { parameters } = this
const queryId = variable.queryId,
name = variable.name
let value = await threadUtils.checkCacheForDynamicVariable(queryId, name)
if (!value) {
value = this.queryResponse[queryId]
? this.queryResponse[queryId]
: await this.runAnotherQuery(queryId, parameters)
// store incase this query is to be called again
this.queryResponse[queryId] = value
await threadUtils.storeDynamicVariable(queryId, name, value)
} else {
this.cachedVariables.push({ queryId, name })
}
return value
}
async addDatasourceVariables() {
let { datasource, parameters, fields } = this
if (!datasource || !datasource.config) {
return parameters
}
const staticVars = datasource.config.staticVariables || {}
const dynamicVars = datasource.config.dynamicVariables || []
for (let [key, value] of Object.entries(staticVars)) {
if (!parameters[key]) {
parameters[key] = value
}
}
if (!this.noRecursiveQuery) {
// need to see if this uses any variables
const stringFields = JSON.stringify(fields)
const foundVars = dynamicVars.filter(variable => {
// don't allow a query to use its own dynamic variable (loop)
if (variable.queryId === this.queryId) {
return false
}
// look for {{ variable }} but allow spaces between handlebars
const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`)
return regex.test(stringFields)
})
const dynamics = foundVars.map(dynVar => this.getDynamicVariable(dynVar))
const responses = await Promise.all(dynamics)
for (let i = 0; i < foundVars.length; i++) {
const variable = foundVars[i]
parameters[variable.name] = processStringSync(variable.value, {
data: responses[i].rows,
info: responses[i].extra,
})
// make sure its known that this uses dynamic variables in case it fails
this.hasDynamicVariables = true
}
}
return parameters
}
}
module.exports = (input, callback) => {
runAndTransform(
input.datasource,
input.queryVerb,
input.query,
input.transformer
)
const Runner = new QueryRunner(input)
Runner.execute()
.then(response => {
callback(null, response)
})

114
packages/server/src/threads/utils.js
View File

@ -1,6 +1,26 @@
const env = require("../environment")
const CouchDB = require("../db")
const { init } = require("@budibase/auth")
const redis = require("@budibase/auth/redis")
const { SEPARATOR } = require("@budibase/auth/db")
const { processStringSync } = require("@budibase/string-templates")
const VARIABLE_TTL_SECONDS = 3600
let client
const IS_TRIPLE_BRACE = new RegExp(/^{{3}.*}{3}$/)
const IS_HANDLEBARS = new RegExp(/^{{2}.*}{2}$/)
async function getClient() {
if (!client) {
client = await new redis.Client(redis.utils.Databases.QUERY_VARS).init()
}
return client
}
process.on("exit", async () => {
if (client) await client.finish()
})
exports.threadSetup = () => {
// don't run this if not threading
@ -11,3 +31,97 @@ exports.threadSetup = () => {
env.setInThread()
init(CouchDB)
}
function makeVariableKey(queryId, variable) {
return `${queryId}${SEPARATOR}${variable}`
}
exports.checkCacheForDynamicVariable = async (queryId, variable) => {
const cache = await getClient()
return cache.get(makeVariableKey(queryId, variable))
}
exports.invalidateDynamicVariables = async cachedVars => {
let promises = []
for (let variable of cachedVars) {
promises.push(
client.delete(makeVariableKey(variable.queryId, variable.name))
)
}
await Promise.all(promises)
}
exports.storeDynamicVariable = async (queryId, variable, value) => {
const cache = await getClient()
await cache.store(
makeVariableKey(queryId, variable),
value,
VARIABLE_TTL_SECONDS
)
}
exports.formatResponse = resp => {
if (typeof resp === "string") {
try {
resp = JSON.parse(resp)
} catch (err) {
resp = { response: resp }
}
}
return resp
}
exports.hasExtraData = response => {
return (
typeof response === "object" &&
!Array.isArray(response) &&
response.data != null &&
response.info != null
)
}
exports.enrichQueryFields = (fields, parameters = {}) => {
const enrichedQuery = {}
// enrich the fields with dynamic parameters
for (let key of Object.keys(fields)) {
if (fields[key] == null) {
continue
}
if (typeof fields[key] === "object") {
// enrich nested fields object
enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters)
} else if (typeof fields[key] === "string") {
// enrich string value as normal
let value = fields[key]
// add triple brace to avoid escaping e.g. '=' in cookie header
if (IS_HANDLEBARS.test(value) && !IS_TRIPLE_BRACE.test(value)) {
value = `{${value}}`
}
enrichedQuery[key] = processStringSync(value, parameters, {
noHelpers: true,
})
} else {
enrichedQuery[key] = fields[key]
}
}
if (
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
) {
try {
enrichedQuery.json = JSON.parse(
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
)
} catch (err) {
// no json found, ignore
}
delete enrichedQuery.customData
}
return enrichedQuery
}

925
packages/server/yarn.lock
View File

File diff suppressed because it is too large Load Diff

5
packages/worker/src/api/controllers/global/auth.js
View File

@ -12,7 +12,7 @@ const {
hash,
platformLogout,
} = authPkg.utils
const { Cookies } = authPkg.constants
const { Cookies, Headers } = authPkg.constants
const { passport } = authPkg.auth
const { checkResetPasswordCode } = require("../../../utilities/redis")
const {
@ -60,7 +60,10 @@ async function authInternal(ctx, user, err = null, info = null) {
return ctx.throw(403, info ? info : "Unauthorized")
}
// set a cookie for browser access
setCookie(ctx, user.token, Cookies.Auth, { sign: false })
// set the token in a header as well for APIs
ctx.set(Headers.TOKEN, user.token)
// get rid of any app cookies on login
// have to check test because this breaks cypress
if (!env.isTest()) {