MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into qa-core-changes

This commit is contained in:
Martin McKeaveney 2024-02-21 23:01:05 +02:00 committed by GitHub
parent 3fad3e98aa 722b81cc90
commit 53a93e7439
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 418 additions and 224 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,5 +1,5 @@
{ {
"version": "2.20.5", "version": "2.20.7",
"npmClient": "yarn", "npmClient": "yarn",
"packages": [ "packages": [
"packages/*", "packages/*",

1
packages/bbui/src/Layout/Page.svelte
View File

@ -43,6 +43,7 @@
flex-direction: row; flex-direction: row;
justify-content: flex-start; justify-content: flex-start;
align-items: stretch; align-items: stretch;
overflow-y: scroll !important;
flex: 1 1 auto; flex: 1 1 auto;
overflow-x: hidden; overflow-x: hidden;
} }

6
packages/bbui/src/Typography/Body.svelte
View File

@ -20,3 +20,9 @@
> >
<slot /> <slot />
</p> </p>
<style>
p {
text-wrap: pretty;
}
</style>

4
packages/bbui/src/Typography/Heading.svelte
View File

@ -21,4 +21,8 @@
h1 { h1 {
font-family: var(--font-accent); font-family: var(--font-accent);
} }
h1 {
text-wrap: balance;
}
</style> </style>

1
packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte
View File

@ -130,6 +130,7 @@
flex-grow: 1; flex-grow: 1;
padding: 23px 23px 80px; padding: 23px 23px 80px;
box-sizing: border-box; box-sizing: border-box;
overflow-x: hidden;
} }
.header.scrolling { .header.scrolling {

3
packages/builder/src/components/integration/RestQueryViewer.svelte
View File

@ -60,6 +60,7 @@
let authConfigId let authConfigId
let dynamicVariables, addVariableModal, varBinding, globalDynamicBindings let dynamicVariables, addVariableModal, varBinding, globalDynamicBindings
let restBindings = getRestBindings() let restBindings = getRestBindings()
let nestedSchemaFields = {}
$: staticVariables = datasource?.config?.staticVariables || {} $: staticVariables = datasource?.config?.staticVariables || {}
@ -160,6 +161,7 @@
newQuery.fields.authConfigId = authConfigId newQuery.fields.authConfigId = authConfigId
newQuery.fields.disabledHeaders = restUtils.flipHeaderState(enabledHeaders) newQuery.fields.disabledHeaders = restUtils.flipHeaderState(enabledHeaders)
newQuery.schema = schema || {} newQuery.schema = schema || {}
newQuery.nestedSchemaFields = nestedSchemaFields || {}
return newQuery return newQuery
} }
@ -238,6 +240,7 @@
} }
} }
schema = response.schema schema = response.schema
nestedSchemaFields = response.nestedSchemaFields
notifications.success("Request sent successfully") notifications.success("Request sent successfully")
} }
} catch (error) { } catch (error) {

2
packages/builder/src/pages/builder/app/[application]/data/new.svelte
View File

@ -77,7 +77,7 @@
</DatasourceOption> </DatasourceOption>
<DatasourceOption <DatasourceOption
on:click={() => internalTableModal.show({ promptUpload: true })} on:click={() => internalTableModal.show({ promptUpload: true })}
title="Upload data" title="Upload CSV / JSON"
description="Non-relational" description="Non-relational"
{disabled} {disabled}
> >

4
packages/builder/src/pages/builder/portal/_components/UpgradeButton.svelte
View File

@ -10,7 +10,7 @@
{#if $admin.cloud && $auth?.user?.accountPortalAccess} {#if $admin.cloud && $auth?.user?.accountPortalAccess}
<Button <Button
cta cta
size="S" size="M"
on:click on:click
on:click={() => { on:click={() => {
window.open($admin.accountPortalUrl + "/portal/upgrade", "_blank") window.open($admin.accountPortalUrl + "/portal/upgrade", "_blank")
@ -21,7 +21,7 @@
{:else if !$admin.cloud && sdk.users.isAdmin($auth.user)} {:else if !$admin.cloud && sdk.users.isAdmin($auth.user)}
<Button <Button
cta cta
size="S" size="M"
on:click={() => $goto("/builder/portal/account/upgrade")} on:click={() => $goto("/builder/portal/account/upgrade")}
on:click on:click
> >

9
packages/builder/src/pages/builder/portal/settings/diagnostics.svelte
View File

@ -49,10 +49,13 @@
{#if sdk.users.isAdmin($auth.user) && diagnosticInfo} {#if sdk.users.isAdmin($auth.user) && diagnosticInfo}
<Layout noPadding> <Layout noPadding>
<Layout gap="XS"> <Layout gap="XS" noPadding>
<Heading size="M">Diagnostics</Heading> <Heading size="M">Diagnostics</Heading>
Please include this diagnostic information in support requests and github issues <Body>
by clicking the button on the top right to copy to clipboard. Please include this diagnostic information in support requests and
github issues by clicking the button on the top right to copy to
clipboard.
</Body>
<Divider /> <Divider />
<Body size="M"> <Body size="M">
<section> <section>

12
packages/builder/src/stores/builder/queries.js
View File

@ -75,17 +75,7 @@ export function createQueriesStore() {
} }
const preview = async query => { const preview = async query => {
const parameters = query.parameters.reduce( const result = await API.previewQuery(query)
(acc, next) => ({
...acc,
[next.name]: next.default,
}),
{}
)
const result = await API.previewQuery({
...query,
parameters,
})
// Assume all the fields are strings and create a basic schema from the // Assume all the fields are strings and create a basic schema from the
// unique fields returned by the server // unique fields returned by the server
const schema = {} const schema = {}

26
packages/client/manifest.json
View File

@ -89,13 +89,13 @@
{ {
"label": "Column", "label": "Column",
"value": "column", "value": "column",
"barIcon": "ViewColumn", "barIcon": "TableSelectColumn",
"barTitle": "Column layout" "barTitle": "Column layout"
}, },
{ {
"label": "Row", "label": "Row",
"value": "row", "value": "row",
"barIcon": "ViewRow", "barIcon": "TableSelectRow",
"barTitle": "Row layout" "barTitle": "Row layout"
} }
], ],
@ -298,13 +298,13 @@
{ {
"label": "Column", "label": "Column",
"value": "column", "value": "column",
"barIcon": "ViewColumn", "barIcon": "TableSelectColumn",
"barTitle": "Column layout" "barTitle": "Column layout"
}, },
{ {
"label": "Row", "label": "Row",
"value": "row", "value": "row",
"barIcon": "ViewRow", "barIcon": "TableSelectRow",
"barTitle": "Row layout" "barTitle": "Row layout"
} }
], ],
@ -460,6 +460,10 @@
"label": "Variant", "label": "Variant",
"key": "type", "key": "type",
"options": [ "options": [
{
"label": "Action",
"value": "cta"
},
{ {
"label": "Primary", "label": "Primary",
"value": "primary" "value": "primary"
@ -468,10 +472,6 @@
"label": "Secondary", "label": "Secondary",
"value": "secondary" "value": "secondary"
}, },
{
"label": "Action",
"value": "cta"
},
{ {
"label": "Warning", "label": "Warning",
"value": "warning" "value": "warning"
@ -481,7 +481,7 @@
"value": "overBackground" "value": "overBackground"
} }
], ],
"defaultValue": "primary" "defaultValue": "cta"
}, },
{ {
"type": "select", "type": "select",
@ -602,13 +602,13 @@
{ {
"label": "Column", "label": "Column",
"value": "column", "value": "column",
"barIcon": "ViewColumn", "barIcon": "TableSelectColumn",
"barTitle": "Column layout" "barTitle": "Column layout"
}, },
{ {
"label": "Row", "label": "Row",
"value": "row", "value": "row",
"barIcon": "ViewRow", "barIcon": "TableSelectRow",
"barTitle": "Row layout" "barTitle": "Row layout"
} }
], ],
@ -5917,13 +5917,13 @@
{ {
"label": "Column", "label": "Column",
"value": "column", "value": "column",
"barIcon": "ViewRow", "barIcon": "TableSelectColumn",
"barTitle": "Column layout" "barTitle": "Column layout"
}, },
{ {
"label": "Row", "label": "Row",
"value": "row", "value": "row",
"barIcon": "ViewColumn", "barIcon": "TableSelectRow",
"barTitle": "Row layout" "barTitle": "Row layout"
} }
], ],

2
packages/client/src/components/app/Button.svelte
View File

@ -11,7 +11,7 @@
export let text = "" export let text = ""
export let onClick export let onClick
export let size = "M" export let size = "M"
export let type = "primary" export let type = "cta"
export let quiet = false export let quiet = false
// For internal use only for now - not defined in the manifest // For internal use only for now - not defined in the manifest

145
packages/server/src/api/controllers/query/index.ts
View File

@ -20,6 +20,7 @@ import {
type ExecuteQueryRequest, type ExecuteQueryRequest,
type ExecuteQueryResponse, type ExecuteQueryResponse,
type Row, type Row,
QueryParameter,
} from "@budibase/types" } from "@budibase/types"
import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core" import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core"
@ -118,6 +119,21 @@ function getAuthConfig(ctx: UserCtx) {
return authConfigCtx return authConfigCtx
} }
function enrichParameters(
queryParameters: QueryParameter[],
requestParameters: { [key: string]: string } = {}
): {
[key: string]: string
} {
// make sure parameters are fully enriched with defaults
for (let parameter of queryParameters) {
if (!requestParameters[parameter.name]) {
requestParameters[parameter.name] = parameter.default
}
}
return requestParameters
}
export async function preview(ctx: UserCtx) { export async function preview(ctx: UserCtx) {
const { datasource, envVars } = await sdk.datasources.getWithEnvVars( const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
ctx.request.body.datasourceId ctx.request.body.datasourceId
@ -142,6 +158,68 @@ export async function preview(ctx: UserCtx) {
const authConfigCtx: any = getAuthConfig(ctx) const authConfigCtx: any = getAuthConfig(ctx)
function getFieldMetadata(field: any, key: string): QuerySchema {
const makeQuerySchema = (
type: FieldType,
name: string,
subtype?: string
): QuerySchema => ({
type,
name,
subtype,
})
// Because custom queries have no fixed schema, we dynamically determine the schema,
// however types cannot be determined from null. We have no 'unknown' type, so we default to string.
let type = typeof field,
fieldMetadata = makeQuerySchema(FieldType.STRING, key)
if (field != null)
switch (type) {
case "boolean":
fieldMetadata = makeQuerySchema(FieldType.BOOLEAN, key)
break
case "object":
if (field instanceof Date) {
fieldMetadata = makeQuerySchema(FieldType.DATETIME, key)
} else if (Array.isArray(field)) {
if (field.some(item => JsonUtils.hasSchema(item))) {
fieldMetadata = makeQuerySchema(
FieldType.JSON,
key,
JsonFieldSubType.ARRAY
)
} else {
fieldMetadata = makeQuerySchema(FieldType.ARRAY, key)
}
} else {
fieldMetadata = makeQuerySchema(FieldType.JSON, key)
}
break
case "number":
fieldMetadata = makeQuerySchema(FieldType.NUMBER, key)
break
}
return fieldMetadata
}
function buildNestedSchema(
nestedSchemaFields: {
[key: string]: Record<string, string | QuerySchema>
},
key: string,
fieldArray: any[]
) {
let schema: { [key: string]: any } = {}
// build the schema by aggregating all row objects in the array
for (const item of fieldArray) {
if (JsonUtils.hasSchema(item)) {
for (const [key, value] of Object.entries(item)) {
schema[key] = getFieldMetadata(value, key)
}
}
}
nestedSchemaFields[key] = schema
}
function getSchemaFields( function getSchemaFields(
rows: any[], rows: any[],
keys: string[] keys: string[]
@ -155,51 +233,16 @@ export async function preview(ctx: UserCtx) {
const nestedSchemaFields: { const nestedSchemaFields: {
[key: string]: Record<string, string | QuerySchema> [key: string]: Record<string, string | QuerySchema>
} = {} } = {}
const makeQuerySchema = (
type: FieldType,
name: string,
subtype?: string
): QuerySchema => ({
type,
name,
subtype,
})
if (rows?.length > 0) { if (rows?.length > 0) {
for (let key of [...new Set(keys)] as string[]) { for (let key of new Set(keys)) {
const field = rows[0][key] const fieldMetadata = getFieldMetadata(rows[0][key], key)
let type = typeof field,
fieldMetadata = makeQuerySchema(FieldType.STRING, key)
if (field)
switch (type) {
case "boolean":
fieldMetadata = makeQuerySchema(FieldType.BOOLEAN, key)
break
case "object":
if (field instanceof Date) {
fieldMetadata = makeQuerySchema(FieldType.DATETIME, key)
} else if (Array.isArray(field)) {
if (JsonUtils.hasSchema(field[0])) {
fieldMetadata = makeQuerySchema(
FieldType.JSON,
key,
JsonFieldSubType.ARRAY
)
} else {
fieldMetadata = makeQuerySchema(FieldType.ARRAY, key)
}
nestedSchemaFields[key] = getSchemaFields(
field,
Object.keys(field[0])
).previewSchema
} else {
fieldMetadata = makeQuerySchema(FieldType.JSON, key)
}
break
case "number":
fieldMetadata = makeQuerySchema(FieldType.NUMBER, key)
break
}
previewSchema[key] = fieldMetadata previewSchema[key] = fieldMetadata
if (
fieldMetadata.type === FieldType.JSON &&
fieldMetadata.subtype === JsonFieldSubType.ARRAY
) {
buildNestedSchema(nestedSchemaFields, key, rows[0][key])
}
} }
} }
return { previewSchema, nestedSchemaFields } return { previewSchema, nestedSchemaFields }
@ -211,7 +254,7 @@ export async function preview(ctx: UserCtx) {
datasource, datasource,
queryVerb, queryVerb,
fields, fields,
parameters, parameters: enrichParameters(parameters),
transformer, transformer,
queryId, queryId,
schema, schema,
@ -266,15 +309,6 @@ async function execute(
if (!opts.isAutomation) { if (!opts.isAutomation) {
authConfigCtx = getAuthConfig(ctx) authConfigCtx = getAuthConfig(ctx)
} }
const enrichedParameters = ctx.request.body.parameters || {}
// make sure parameters are fully enriched with defaults
if (query && query.parameters) {
for (let parameter of query.parameters) {
if (!enrichedParameters[parameter.name]) {
enrichedParameters[parameter.name] = parameter.default
}
}
}
// call the relevant CRUD method on the integration class // call the relevant CRUD method on the integration class
try { try {
@ -284,7 +318,10 @@ async function execute(
queryVerb: query.queryVerb, queryVerb: query.queryVerb,
fields: query.fields, fields: query.fields,
pagination: ctx.request.body.pagination, pagination: ctx.request.body.pagination,
parameters: enrichedParameters, parameters: enrichParameters(
query.parameters,
ctx.request.body.parameters
),
transformer: query.transformer, transformer: query.transformer,
queryId: ctx.params.queryId, queryId: ctx.params.queryId,
// have to pass down to the thread runner - can't put into context now // have to pass down to the thread runner - can't put into context now

38
packages/server/src/api/controllers/query/validation.ts
View File

@ -3,11 +3,10 @@ import Joi from "joi"
const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("") const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("")
export function queryValidation() { function baseQueryValidation() {
return Joi.object({ return {
_id: Joi.string(), _id: OPTIONAL_STRING,
_rev: Joi.string(), _rev: OPTIONAL_STRING,
name: Joi.string().required(),
fields: Joi.object().required(), fields: Joi.object().required(),
datasourceId: Joi.string().required(), datasourceId: Joi.string().required(),
readable: Joi.boolean(), readable: Joi.boolean(),
@ -17,11 +16,19 @@ export function queryValidation() {
default: Joi.string().allow(""), default: Joi.string().allow(""),
}) })
), ),
queryVerb: Joi.string().allow().required(), queryVerb: Joi.string().required(),
extra: Joi.object().optional(), extra: Joi.object().optional(),
schema: Joi.object({}).required().unknown(true), schema: Joi.object({}).required().unknown(true),
transformer: OPTIONAL_STRING, transformer: OPTIONAL_STRING,
flags: Joi.object().optional(), flags: Joi.object().optional(),
queryId: OPTIONAL_STRING,
}
}
export function queryValidation() {
return Joi.object({
...baseQueryValidation(),
name: Joi.string().required(),
}).unknown(true) }).unknown(true)
} }
@ -32,19 +39,10 @@ export function generateQueryValidation() {
export function generateQueryPreviewValidation() { export function generateQueryPreviewValidation() {
// prettier-ignore // prettier-ignore
return auth.joiValidator.body(Joi.object({ return auth.joiValidator.body(
_id: OPTIONAL_STRING, Joi.object({
_rev: OPTIONAL_STRING, ...baseQueryValidation(),
readable: Joi.boolean().optional(),
fields: Joi.object().required(),
queryVerb: Joi.string().required(),
name: OPTIONAL_STRING, name: OPTIONAL_STRING,
flags: Joi.object().optional(), }).unknown(true)
schema: Joi.object().optional(), )
extra: Joi.object().optional(),
datasourceId: Joi.string().required(),
transformer: OPTIONAL_STRING,
parameters: Joi.object({}).required().unknown(true),
queryId: OPTIONAL_STRING,
}).unknown(true))
} }

2
packages/server/src/api/routes/query.ts
View File

@ -8,8 +8,8 @@ import {
paramResource, paramResource,
} from "../../middleware/resourceId" } from "../../middleware/resourceId"
import { import {
generateQueryPreviewValidation,
generateQueryValidation, generateQueryValidation,
generateQueryPreviewValidation,
} from "../controllers/query/validation" } from "../controllers/query/validation"
const { BUILDER, PermissionType, PermissionLevel } = permissions const { BUILDER, PermissionType, PermissionLevel } = permissions

18
packages/server/src/api/routes/tests/datasource.spec.ts
View File

@ -7,6 +7,7 @@ import sdk from "../../../sdk"
import tk from "timekeeper" import tk from "timekeeper"
import { mocks } from "@budibase/backend-core/tests" import { mocks } from "@budibase/backend-core/tests"
import { QueryPreview } from "@budibase/types"
tk.freeze(mocks.date.MOCK_DATE) tk.freeze(mocks.date.MOCK_DATE)
@ -63,14 +64,17 @@ describe("/datasources", () => {
datasource: any, datasource: any,
fields: { path: string; queryString: string } fields: { path: string; queryString: string }
) { ) {
return config.previewQuery( const queryPreview: QueryPreview = {
request,
config,
datasource,
fields, fields,
undefined, datasourceId: datasource._id,
"" parameters: [],
) transformer: null,
queryVerb: "read",
name: datasource.name,
schema: {},
readable: true,
}
return config.api.query.previewQuery(queryPreview)
} }
it("should invalidate changed or removed variables", async () => { it("should invalidate changed or removed variables", async () => {

12
packages/server/src/api/routes/tests/environmentVariables.spec.ts
View File

@ -14,6 +14,7 @@ jest.mock("pg", () => {
import * as setup from "./utilities" import * as setup from "./utilities"
import { mocks } from "@budibase/backend-core/tests" import { mocks } from "@budibase/backend-core/tests"
import { env, events } from "@budibase/backend-core" import { env, events } from "@budibase/backend-core"
import { QueryPreview } from "@budibase/types"
const structures = setup.structures const structures = setup.structures
@ -120,16 +121,19 @@ describe("/api/env/variables", () => {
.expect(200) .expect(200)
expect(response.body.datasource._id).toBeDefined() expect(response.body.datasource._id).toBeDefined()
const query = { const queryPreview: QueryPreview = {
datasourceId: response.body.datasource._id, datasourceId: response.body.datasource._id,
parameters: {}, parameters: [],
fields: {}, fields: {},
queryVerb: "read", queryVerb: "read",
name: response.body.datasource.name, name: response.body.datasource.name,
transformer: null,
schema: {},
readable: true,
} }
const res = await request const res = await request
.post(`/api/queries/preview`) .post(`/api/queries/preview`)
.send(query) .send(queryPreview)
.set(config.defaultHeaders()) .set(config.defaultHeaders())
.expect("Content-Type", /json/) .expect("Content-Type", /json/)
.expect(200) .expect(200)
@ -139,7 +143,7 @@ describe("/api/env/variables", () => {
delete response.body.datasource.config delete response.body.datasource.config
expect(events.query.previewed).toBeCalledWith( expect(events.query.previewed).toBeCalledWith(
response.body.datasource, response.body.datasource,
query queryPreview
) )
expect(pg.Client).toHaveBeenCalledWith({ password: "test", ssl: undefined }) expect(pg.Client).toHaveBeenCalledWith({ password: "test", ssl: undefined })
}) })

305
packages/server/src/api/routes/tests/queries/query.seq.spec.ts
View File

@ -1,5 +1,7 @@
import tk from "timekeeper" import tk from "timekeeper"
const pg = require("pg")
// Mock out postgres for this // Mock out postgres for this
jest.mock("pg") jest.mock("pg")
jest.mock("node-fetch") jest.mock("node-fetch")
@ -22,7 +24,13 @@ import { checkCacheForDynamicVariable } from "../../../../threads/utils"
const { basicQuery, basicDatasource } = setup.structures const { basicQuery, basicDatasource } = setup.structures
import { events, db as dbCore } from "@budibase/backend-core" import { events, db as dbCore } from "@budibase/backend-core"
import { Datasource, Query, SourceName } from "@budibase/types" import {
Datasource,
Query,
SourceName,
QueryPreview,
QueryParameter,
} from "@budibase/types"
tk.freeze(Date.now()) tk.freeze(Date.now())
@ -218,28 +226,26 @@ describe("/queries", () => {
describe("preview", () => { describe("preview", () => {
it("should be able to preview the query", async () => { it("should be able to preview the query", async () => {
const query = { const queryPreview: QueryPreview = {
datasourceId: datasource._id, datasourceId: datasource._id,
parameters: {},
fields: {},
queryVerb: "read", queryVerb: "read",
name: datasource.name, fields: {},
parameters: [],
transformer: "return data",
name: datasource.name!,
schema: {},
readable: true,
} }
const res = await request const responseBody = await config.api.query.previewQuery(queryPreview)
.post(`/api/queries/preview`)
.send(query)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
// these responses come from the mock // these responses come from the mock
expect(res.body.schema).toEqual({ expect(responseBody.schema).toEqual({
a: { type: "string", name: "a" }, a: { type: "string", name: "a" },
b: { type: "number", name: "b" }, b: { type: "number", name: "b" },
}) })
expect(res.body.rows.length).toEqual(1) expect(responseBody.rows.length).toEqual(1)
expect(events.query.previewed).toBeCalledTimes(1) expect(events.query.previewed).toBeCalledTimes(1)
delete datasource.config delete datasource.config
expect(events.query.previewed).toBeCalledWith(datasource, query) expect(events.query.previewed).toBeCalledWith(datasource, queryPreview)
}) })
it("should apply authorization to endpoint", async () => { it("should apply authorization to endpoint", async () => {
@ -249,6 +255,128 @@ describe("/queries", () => {
url: `/api/queries/preview`, url: `/api/queries/preview`,
}) })
}) })
it("should not error when trying to generate a nested schema for an empty array", async () => {
const queryPreview: QueryPreview = {
datasourceId: datasource._id,
parameters: [],
fields: {},
queryVerb: "read",
name: datasource.name!,
transformer: "return data",
schema: {},
readable: true,
}
const rows = [
{
contacts: [],
},
]
pg.queryMock.mockImplementation(() => ({
rows,
}))
const responseBody = await config.api.query.previewQuery(queryPreview)
expect(responseBody).toEqual({
nestedSchemaFields: {},
rows,
schema: {
contacts: { type: "array", name: "contacts" },
},
})
expect(responseBody.rows.length).toEqual(1)
delete datasource.config
})
it("should generate a nested schema based on all the nested items", async () => {
const queryPreview: QueryPreview = {
datasourceId: datasource._id,
parameters: [],
fields: {},
queryVerb: "read",
name: datasource.name!,
transformer: "return data",
schema: {},
readable: true,
}
const rows = [
{
contacts: [
{
address: "123 Lane",
},
{
address: "456 Drive",
},
{
postcode: "BT1 12N",
lat: 54.59,
long: -5.92,
},
{
city: "Belfast",
},
{
address: "789 Avenue",
phoneNumber: "0800-999-5555",
},
{
name: "Name",
isActive: false,
},
],
},
]
pg.queryMock.mockImplementation(() => ({
rows,
}))
const responseBody = await config.api.query.previewQuery(queryPreview)
expect(responseBody).toEqual({
nestedSchemaFields: {
contacts: {
address: {
type: "string",
name: "address",
},
postcode: {
type: "string",
name: "postcode",
},
lat: {
type: "number",
name: "lat",
},
long: {
type: "number",
name: "long",
},
city: {
type: "string",
name: "city",
},
phoneNumber: {
type: "string",
name: "phoneNumber",
},
name: {
type: "string",
name: "name",
},
isActive: {
type: "boolean",
name: "isActive",
},
},
},
rows,
schema: {
contacts: { type: "json", name: "contacts", subtype: "array" },
},
})
expect(responseBody.rows.length).toEqual(1)
delete datasource.config
})
}) })
describe("execute", () => { describe("execute", () => {
@ -283,7 +411,17 @@ describe("/queries", () => {
describe("variables", () => { describe("variables", () => {
async function preview(datasource: Datasource, fields: any) { async function preview(datasource: Datasource, fields: any) {
return config.previewQuery(request, config, datasource, fields, undefined) const queryPreview: QueryPreview = {
datasourceId: datasource._id!,
parameters: [],
fields,
queryVerb: "read",
name: datasource.name!,
transformer: "return data",
schema: {},
readable: true,
}
return await config.api.query.previewQuery(queryPreview)
} }
it("should work with static variables", async () => { it("should work with static variables", async () => {
@ -293,31 +431,31 @@ describe("/queries", () => {
variable2: "1", variable2: "1",
}, },
}) })
const res = await preview(datasource, { const responseBody = await preview(datasource, {
path: "www.{{ variable }}.com", path: "www.{{ variable }}.com",
queryString: "test={{ variable2 }}", queryString: "test={{ variable2 }}",
}) })
// these responses come from the mock // these responses come from the mock
expect(res.body.schema).toEqual({ expect(responseBody.schema).toEqual({
opts: { type: "json", name: "opts" }, opts: { type: "json", name: "opts" },
url: { type: "string", name: "url" }, url: { type: "string", name: "url" },
value: { type: "string", name: "value" }, value: { type: "string", name: "value" },
}) })
expect(res.body.rows[0].url).toEqual("http://www.google.com?test=1") expect(responseBody.rows[0].url).toEqual("http://www.google.com?test=1")
}) })
it("should work with dynamic variables", async () => { it("should work with dynamic variables", async () => {
const { datasource } = await config.dynamicVariableDatasource() const { datasource } = await config.dynamicVariableDatasource()
const res = await preview(datasource, { const responseBody = await preview(datasource, {
path: "www.google.com", path: "www.google.com",
queryString: "test={{ variable3 }}", queryString: "test={{ variable3 }}",
}) })
expect(res.body.schema).toEqual({ expect(responseBody.schema).toEqual({
opts: { type: "json", name: "opts" }, opts: { type: "json", name: "opts" },
url: { type: "string", name: "url" }, url: { type: "string", name: "url" },
value: { type: "string", name: "value" }, value: { type: "string", name: "value" },
}) })
expect(res.body.rows[0].url).toContain("doctype%20html") expect(responseBody.rows[0].url).toContain("doctype%20html")
}) })
it("check that it automatically retries on fail with cached dynamics", async () => { it("check that it automatically retries on fail with cached dynamics", async () => {
@ -331,16 +469,16 @@ describe("/queries", () => {
// check its in cache // check its in cache
const contents = await checkCacheForDynamicVariable(base._id, "variable3") const contents = await checkCacheForDynamicVariable(base._id, "variable3")
expect(contents.rows.length).toEqual(1) expect(contents.rows.length).toEqual(1)
const res = await preview(datasource, { const responseBody = await preview(datasource, {
path: "www.failonce.com", path: "www.failonce.com",
queryString: "test={{ variable3 }}", queryString: "test={{ variable3 }}",
}) })
expect(res.body.schema).toEqual({ expect(responseBody.schema).toEqual({
fails: { type: "number", name: "fails" }, fails: { type: "number", name: "fails" },
opts: { type: "json", name: "opts" }, opts: { type: "json", name: "opts" },
url: { type: "string", name: "url" }, url: { type: "string", name: "url" },
}) })
expect(res.body.rows[0].fails).toEqual(1) expect(responseBody.rows[0].fails).toEqual(1)
}) })
it("deletes variables when linked query is deleted", async () => { it("deletes variables when linked query is deleted", async () => {
@ -371,24 +509,37 @@ describe("/queries", () => {
async function previewGet( async function previewGet(
datasource: Datasource, datasource: Datasource,
fields: any, fields: any,
params: any params: QueryParameter[]
) { ) {
return config.previewQuery(request, config, datasource, fields, params) const queryPreview: QueryPreview = {
datasourceId: datasource._id!,
parameters: params,
fields,
queryVerb: "read",
name: datasource.name!,
transformer: "return data",
schema: {},
readable: true,
}
return await config.api.query.previewQuery(queryPreview)
} }
async function previewPost( async function previewPost(
datasource: Datasource, datasource: Datasource,
fields: any, fields: any,
params: any params: QueryParameter[]
) { ) {
return config.previewQuery( const queryPreview: QueryPreview = {
request, datasourceId: datasource._id!,
config, parameters: params,
datasource,
fields, fields,
params, queryVerb: "create",
"create" name: datasource.name!,
) transformer: null,
schema: {},
readable: false,
}
return await config.api.query.previewQuery(queryPreview)
} }
it("should parse global and query level header mappings", async () => { it("should parse global and query level header mappings", async () => {
@ -400,7 +551,7 @@ describe("/queries", () => {
emailHdr: "{{[user].[email]}}", emailHdr: "{{[user].[email]}}",
}, },
}) })
const res = await previewGet( const responseBody = await previewGet(
datasource, datasource,
{ {
path: "www.google.com", path: "www.google.com",
@ -410,17 +561,17 @@ describe("/queries", () => {
secondHdr: "1234", secondHdr: "1234",
}, },
}, },
undefined []
) )
const parsedRequest = JSON.parse(res.body.extra.raw) const parsedRequest = JSON.parse(responseBody.extra.raw)
expect(parsedRequest.opts.headers).toEqual({ expect(parsedRequest.opts.headers).toEqual({
test: "headerVal", test: "headerVal",
emailHdr: userDetails.email, emailHdr: userDetails.email,
queryHdr: userDetails.firstName, queryHdr: userDetails.firstName,
secondHdr: "1234", secondHdr: "1234",
}) })
expect(res.body.rows[0].url).toEqual( expect(responseBody.rows[0].url).toEqual(
"http://www.google.com?email=" + userDetails.email.replace("@", "%40") "http://www.google.com?email=" + userDetails.email.replace("@", "%40")
) )
}) })
@ -430,21 +581,21 @@ describe("/queries", () => {
const datasource = await config.restDatasource() const datasource = await config.restDatasource()
const res = await previewGet( const responseBody = await previewGet(
datasource, datasource,
{ {
path: "www.google.com", path: "www.google.com",
queryString: queryString:
"test={{myEmail}}&testName={{myName}}&testParam={{testParam}}", "test={{myEmail}}&testName={{myName}}&testParam={{testParam}}",
}, },
{ [
myEmail: "{{[user].[email]}}", { name: "myEmail", default: "{{[user].[email]}}" },
myName: "{{[user].[firstName]}}", { name: "myName", default: "{{[user].[firstName]}}" },
testParam: "1234", { name: "testParam", default: "1234" },
} ]
) )
expect(res.body.rows[0].url).toEqual( expect(responseBody.rows[0].url).toEqual(
"http://www.google.com?test=" + "http://www.google.com?test=" +
userDetails.email.replace("@", "%40") + userDetails.email.replace("@", "%40") +
"&testName=" + "&testName=" +
@ -457,7 +608,7 @@ describe("/queries", () => {
const userDetails = config.getUserDetails() const userDetails = config.getUserDetails()
const datasource = await config.restDatasource() const datasource = await config.restDatasource()
const res = await previewPost( const responseBody = await previewPost(
datasource, datasource,
{ {
path: "www.google.com", path: "www.google.com",
@ -466,16 +617,14 @@ describe("/queries", () => {
"This is plain text and this is my email: {{[user].[email]}}. This is a test param: {{testParam}}", "This is plain text and this is my email: {{[user].[email]}}. This is a test param: {{testParam}}",
bodyType: "text", bodyType: "text",
}, },
{ [{ name: "testParam", default: "1234" }]
testParam: "1234",
}
) )
const parsedRequest = JSON.parse(res.body.extra.raw) const parsedRequest = JSON.parse(responseBody.extra.raw)
expect(parsedRequest.opts.body).toEqual( expect(parsedRequest.opts.body).toEqual(
`This is plain text and this is my email: ${userDetails.email}. This is a test param: 1234` `This is plain text and this is my email: ${userDetails.email}. This is a test param: 1234`
) )
expect(res.body.rows[0].url).toEqual( expect(responseBody.rows[0].url).toEqual(
"http://www.google.com?testParam=1234" "http://www.google.com?testParam=1234"
) )
}) })
@ -484,7 +633,7 @@ describe("/queries", () => {
const userDetails = config.getUserDetails() const userDetails = config.getUserDetails()
const datasource = await config.restDatasource() const datasource = await config.restDatasource()
const res = await previewPost( const responseBody = await previewPost(
datasource, datasource,
{ {
path: "www.google.com", path: "www.google.com",
@ -493,16 +642,16 @@ describe("/queries", () => {
'{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}', '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}',
bodyType: "json", bodyType: "json",
}, },
{ [
testParam: "1234", { name: "testParam", default: "1234" },
userRef: "{{[user].[firstName]}}", { name: "userRef", default: "{{[user].[firstName]}}" },
} ]
) )
const parsedRequest = JSON.parse(res.body.extra.raw) const parsedRequest = JSON.parse(responseBody.extra.raw)
const test = `{"email":"${userDetails.email}","queryCode":1234,"userRef":"${userDetails.firstName}"}` const test = `{"email":"${userDetails.email}","queryCode":1234,"userRef":"${userDetails.firstName}"}`
expect(parsedRequest.opts.body).toEqual(test) expect(parsedRequest.opts.body).toEqual(test)
expect(res.body.rows[0].url).toEqual( expect(responseBody.rows[0].url).toEqual(
"http://www.google.com?testParam=1234" "http://www.google.com?testParam=1234"
) )
}) })
@ -511,7 +660,7 @@ describe("/queries", () => {
const userDetails = config.getUserDetails() const userDetails = config.getUserDetails()
const datasource = await config.restDatasource() const datasource = await config.restDatasource()
const res = await previewPost( const responseBody = await previewPost(
datasource, datasource,
{ {
path: "www.google.com", path: "www.google.com",
@ -521,17 +670,17 @@ describe("/queries", () => {
"<ref>{{userId}}</ref> <somestring>testing</somestring> </note>", "<ref>{{userId}}</ref> <somestring>testing</somestring> </note>",
bodyType: "xml", bodyType: "xml",
}, },
{ [
testParam: "1234", { name: "testParam", default: "1234" },
userId: "{{[user].[firstName]}}", { name: "userId", default: "{{[user].[firstName]}}" },
} ]
) )
const parsedRequest = JSON.parse(res.body.extra.raw) const parsedRequest = JSON.parse(responseBody.extra.raw)
const test = `<note> <email>${userDetails.email}</email> <code>1234</code> <ref>${userDetails.firstName}</ref> <somestring>testing</somestring> </note>` const test = `<note> <email>${userDetails.email}</email> <code>1234</code> <ref>${userDetails.firstName}</ref> <somestring>testing</somestring> </note>`
expect(parsedRequest.opts.body).toEqual(test) expect(parsedRequest.opts.body).toEqual(test)
expect(res.body.rows[0].url).toEqual( expect(responseBody.rows[0].url).toEqual(
"http://www.google.com?testParam=1234" "http://www.google.com?testParam=1234"
) )
}) })
@ -540,7 +689,7 @@ describe("/queries", () => {
const userDetails = config.getUserDetails() const userDetails = config.getUserDetails()
const datasource = await config.restDatasource() const datasource = await config.restDatasource()
const res = await previewPost( const responseBody = await previewPost(
datasource, datasource,
{ {
path: "www.google.com", path: "www.google.com",
@ -549,13 +698,13 @@ describe("/queries", () => {
'{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}', '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}',
bodyType: "form", bodyType: "form",
}, },
{ [
testParam: "1234", { name: "testParam", default: "1234" },
userRef: "{{[user].[firstName]}}", { name: "userRef", default: "{{[user].[firstName]}}" },
} ]
) )
const parsedRequest = JSON.parse(res.body.extra.raw) const parsedRequest = JSON.parse(responseBody.extra.raw)
const emailData = parsedRequest.opts.body._streams[1] const emailData = parsedRequest.opts.body._streams[1]
expect(emailData).toEqual(userDetails.email) expect(emailData).toEqual(userDetails.email)
@ -566,7 +715,7 @@ describe("/queries", () => {
const userRef = parsedRequest.opts.body._streams[7] const userRef = parsedRequest.opts.body._streams[7]
expect(userRef).toEqual(userDetails.firstName) expect(userRef).toEqual(userDetails.firstName)
expect(res.body.rows[0].url).toEqual( expect(responseBody.rows[0].url).toEqual(
"http://www.google.com?testParam=1234" "http://www.google.com?testParam=1234"
) )
}) })
@ -575,7 +724,7 @@ describe("/queries", () => {
const userDetails = config.getUserDetails() const userDetails = config.getUserDetails()
const datasource = await config.restDatasource() const datasource = await config.restDatasource()
const res = await previewPost( const responseBody = await previewPost(
datasource, datasource,
{ {
path: "www.google.com", path: "www.google.com",
@ -584,12 +733,12 @@ describe("/queries", () => {
'{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}', '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}',
bodyType: "encoded", bodyType: "encoded",
}, },
{ [
testParam: "1234", { name: "testParam", default: "1234" },
userRef: "{{[user].[firstName]}}", { name: "userRef", default: "{{[user].[firstName]}}" },
} ]
) )
const parsedRequest = JSON.parse(res.body.extra.raw) const parsedRequest = JSON.parse(responseBody.extra.raw)
expect(parsedRequest.opts.body.email).toEqual(userDetails.email) expect(parsedRequest.opts.body.email).toEqual(userDetails.email)
expect(parsedRequest.opts.body.queryCode).toEqual("1234") expect(parsedRequest.opts.body.queryCode).toEqual("1234")

22
packages/server/src/tests/utilities/TestConfiguration.ts
View File

@ -866,28 +866,6 @@ export default class TestConfiguration {
// QUERY // QUERY
async previewQuery(
request: any,
config: any,
datasource: any,
fields: any,
params: any,
verb?: string
) {
return request
.post(`/api/queries/preview`)
.send({
datasourceId: datasource._id,
parameters: params || {},
fields,
queryVerb: verb || "read",
name: datasource.name,
})
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
}
async createQuery(config?: any) { async createQuery(config?: any) {
if (!this.datasource && !config) { if (!this.datasource && !config) {
throw "No datasource created for query." throw "No datasource created for query."

16
packages/server/src/tests/utilities/api/query.ts
View File

@ -1,6 +1,7 @@
import TestConfiguration from "../TestConfiguration" import TestConfiguration from "../TestConfiguration"
import { import {
Query, Query,
QueryPreview,
type ExecuteQueryRequest, type ExecuteQueryRequest,
type ExecuteQueryResponse, type ExecuteQueryResponse,
} from "@budibase/types" } from "@budibase/types"
@ -41,4 +42,19 @@ export class QueryAPI extends TestAPI {
return res.body return res.body
} }
previewQuery = async (queryPreview: QueryPreview) => {
const res = await this.request
.post(`/api/queries/preview`)
.send(queryPreview)
.set(this.config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
if (res.status !== 200) {
throw new Error(JSON.stringify(res.body))
}
return res.body
}
} }

2
packages/server/src/tests/utilities/structures.ts
View File

@ -366,7 +366,7 @@ export function basicDatasource(): { datasource: Datasource } {
export function basicQuery(datasourceId: string): Query { export function basicQuery(datasourceId: string): Query {
return { return {
datasourceId: datasourceId, datasourceId,
name: "New Query", name: "New Query",
parameters: [], parameters: [],
fields: {}, fields: {},

4
packages/server/src/threads/definitions.ts
View File

@ -7,10 +7,10 @@ export interface QueryEvent {
datasource: Datasource datasource: Datasource
queryVerb: string queryVerb: string
fields: { [key: string]: any } fields: { [key: string]: any }
parameters: { [key: string]: any } parameters: { [key: string]: unknown }
pagination?: any pagination?: any
transformer: any transformer: any
queryId: string queryId?: string
environmentVariables?: Record<string, string> environmentVariables?: Record<string, string>
ctx?: any ctx?: any
schema?: Record<string, QuerySchema | string> schema?: Record<string, QuerySchema | string>

2
packages/server/src/threads/query.ts
View File

@ -43,7 +43,7 @@ class QueryRunner {
this.parameters = input.parameters this.parameters = input.parameters
this.pagination = input.pagination this.pagination = input.pagination
this.transformer = input.transformer this.transformer = input.transformer
this.queryId = input.queryId this.queryId = input.queryId!
this.schema = input.schema this.schema = input.schema
this.noRecursiveQuery = flags.noRecursiveQuery this.noRecursiveQuery = flags.noRecursiveQuery
this.cachedVariables = [] this.cachedVariables = []

2
packages/types/src/documents/app/query.ts
View File

@ -19,7 +19,7 @@ export interface Query extends Document {
} }
export interface QueryPreview extends Omit<Query, "_id"> { export interface QueryPreview extends Omit<Query, "_id"> {
queryId: string queryId?: string
} }
export interface QueryParameter { export interface QueryParameter {