Feature/query array schema (#13118)

* Revert "Revert "Add nestedSchemaFields to RestQueryViewer (#13102)""

This reverts commit 6545af12fa.

* Revert "Revert "Dynamic schema generation for query arrays: bug fix and refactor (#13…""

This reverts commit ae4bc362c3.

packages/builder/src/components/integration/RestQueryViewer.svelte

@@ -60,6 +60,7 @@
   let authConfigId
   let dynamicVariables, addVariableModal, varBinding, globalDynamicBindings
   let restBindings = getRestBindings()
+  let nestedSchemaFields = {}
 
   $: staticVariables = datasource?.config?.staticVariables || {}
 
@@ -160,6 +161,7 @@
     newQuery.fields.authConfigId = authConfigId
     newQuery.fields.disabledHeaders = restUtils.flipHeaderState(enabledHeaders)
     newQuery.schema = schema || {}
+    newQuery.nestedSchemaFields = nestedSchemaFields || {}
 
     return newQuery
   }
@@ -238,6 +240,7 @@
           }
         }
         schema = response.schema
+        nestedSchemaFields = response.nestedSchemaFields
         notifications.success("Request sent successfully")
       }
     } catch (error) {

packages/builder/src/stores/builder/queries.js

@@ -76,17 +76,7 @@ export function createQueriesStore() {
   }
 
   const preview = async query => {
-    const parameters = query.parameters.reduce(
+    const result = await API.previewQuery(query)
-      (acc, next) => ({
-        ...acc,
-        [next.name]: next.default,
-      }),
-      {}
-    )
-    const result = await API.previewQuery({
-      ...query,
-      parameters,
-    })
     // Assume all the fields are strings and create a basic schema from the
     // unique fields returned by the server
     const schema = {}

packages/server/src/api/controllers/query/index.ts

@@ -20,6 +20,7 @@ import {
   type ExecuteQueryRequest,
   type ExecuteQueryResponse,
   type Row,
+  QueryParameter,
 } from "@budibase/types"
 import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core"
 
@@ -118,6 +119,21 @@ function getAuthConfig(ctx: UserCtx) {
   return authConfigCtx
 }
 
+function enrichParameters(
+  queryParameters: QueryParameter[],
+  requestParameters: { [key: string]: string } = {}
+): {
+  [key: string]: string
+} {
+  // make sure parameters are fully enriched with defaults
+  for (let parameter of queryParameters) {
+    if (!requestParameters[parameter.name]) {
+      requestParameters[parameter.name] = parameter.default
+    }
+  }
+  return requestParameters
+}
+
 export async function preview(ctx: UserCtx) {
   const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
     ctx.request.body.datasourceId
@@ -142,6 +158,68 @@ export async function preview(ctx: UserCtx) {
 
   const authConfigCtx: any = getAuthConfig(ctx)
 
+  function getFieldMetadata(field: any, key: string): QuerySchema {
+    const makeQuerySchema = (
+      type: FieldType,
+      name: string,
+      subtype?: string
+    ): QuerySchema => ({
+      type,
+      name,
+      subtype,
+    })
+    // Because custom queries have no fixed schema, we dynamically determine the schema,
+    // however types cannot be determined from null. We have no 'unknown' type, so we default to string.
+    let type = typeof field,
+      fieldMetadata = makeQuerySchema(FieldType.STRING, key)
+    if (field != null)
+      switch (type) {
+        case "boolean":
+          fieldMetadata = makeQuerySchema(FieldType.BOOLEAN, key)
+          break
+        case "object":
+          if (field instanceof Date) {
+            fieldMetadata = makeQuerySchema(FieldType.DATETIME, key)
+          } else if (Array.isArray(field)) {
+            if (field.some(item => JsonUtils.hasSchema(item))) {
+              fieldMetadata = makeQuerySchema(
+                FieldType.JSON,
+                key,
+                JsonFieldSubType.ARRAY
+              )
+            } else {
+              fieldMetadata = makeQuerySchema(FieldType.ARRAY, key)
+            }
+          } else {
+            fieldMetadata = makeQuerySchema(FieldType.JSON, key)
+          }
+          break
+        case "number":
+          fieldMetadata = makeQuerySchema(FieldType.NUMBER, key)
+          break
+      }
+    return fieldMetadata
+  }
+
+  function buildNestedSchema(
+    nestedSchemaFields: {
+      [key: string]: Record<string, string | QuerySchema>
+    },
+    key: string,
+    fieldArray: any[]
+  ) {
+    let schema: { [key: string]: any } = {}
+    // build the schema by aggregating all row objects in the array
+    for (const item of fieldArray) {
+      if (JsonUtils.hasSchema(item)) {
+        for (const [key, value] of Object.entries(item)) {
+          schema[key] = getFieldMetadata(value, key)
+        }
+      }
+    }
+    nestedSchemaFields[key] = schema
+  }
+
   function getSchemaFields(
     rows: any[],
     keys: string[]
@@ -155,51 +233,16 @@ export async function preview(ctx: UserCtx) {
     const nestedSchemaFields: {
       [key: string]: Record<string, string | QuerySchema>
     } = {}
-    const makeQuerySchema = (
-      type: FieldType,
-      name: string,
-      subtype?: string
-    ): QuerySchema => ({
-      type,
-      name,
-      subtype,
-    })
     if (rows?.length > 0) {
-      for (let key of [...new Set(keys)] as string[]) {
+      for (let key of new Set(keys)) {
-        const field = rows[0][key]
+        const fieldMetadata = getFieldMetadata(rows[0][key], key)
-        let type = typeof field,
-          fieldMetadata = makeQuerySchema(FieldType.STRING, key)
-        if (field)
-          switch (type) {
-            case "boolean":
-              fieldMetadata = makeQuerySchema(FieldType.BOOLEAN, key)
-              break
-            case "object":
-              if (field instanceof Date) {
-                fieldMetadata = makeQuerySchema(FieldType.DATETIME, key)
-              } else if (Array.isArray(field)) {
-                if (JsonUtils.hasSchema(field[0])) {
-                  fieldMetadata = makeQuerySchema(
-                    FieldType.JSON,
-                    key,
-                    JsonFieldSubType.ARRAY
-                  )
-                } else {
-                  fieldMetadata = makeQuerySchema(FieldType.ARRAY, key)
-                }
-                nestedSchemaFields[key] = getSchemaFields(
-                  field,
-                  Object.keys(field[0])
-                ).previewSchema
-              } else {
-                fieldMetadata = makeQuerySchema(FieldType.JSON, key)
-              }
-              break
-            case "number":
-              fieldMetadata = makeQuerySchema(FieldType.NUMBER, key)
-              break
-          }
         previewSchema[key] = fieldMetadata
+        if (
+          fieldMetadata.type === FieldType.JSON &&
+          fieldMetadata.subtype === JsonFieldSubType.ARRAY
+        ) {
+          buildNestedSchema(nestedSchemaFields, key, rows[0][key])
+        }
       }
     }
     return { previewSchema, nestedSchemaFields }
@@ -211,7 +254,7 @@ export async function preview(ctx: UserCtx) {
       datasource,
       queryVerb,
       fields,
-      parameters,
+      parameters: enrichParameters(parameters),
       transformer,
       queryId,
       schema,
@@ -266,15 +309,6 @@ async function execute(
   if (!opts.isAutomation) {
     authConfigCtx = getAuthConfig(ctx)
   }
-  const enrichedParameters = ctx.request.body.parameters || {}
-  // make sure parameters are fully enriched with defaults
-  if (query && query.parameters) {
-    for (let parameter of query.parameters) {
-      if (!enrichedParameters[parameter.name]) {
-        enrichedParameters[parameter.name] = parameter.default
-      }
-    }
-  }
 
   // call the relevant CRUD method on the integration class
   try {
@@ -284,7 +318,10 @@ async function execute(
       queryVerb: query.queryVerb,
       fields: query.fields,
       pagination: ctx.request.body.pagination,
-      parameters: enrichedParameters,
+      parameters: enrichParameters(
+        query.parameters,
+        ctx.request.body.parameters
+      ),
       transformer: query.transformer,
       queryId: ctx.params.queryId,
       // have to pass down to the thread runner - can't put into context now

packages/server/src/api/controllers/query/validation.ts

@@ -3,11 +3,10 @@ import Joi from "joi"
 
 const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("")
 
-export function queryValidation() {
+function baseQueryValidation() {
-  return Joi.object({
+  return {
-    _id: Joi.string(),
+    _id: OPTIONAL_STRING,
-    _rev: Joi.string(),
+    _rev: OPTIONAL_STRING,
-    name: Joi.string().required(),
     fields: Joi.object().required(),
     datasourceId: Joi.string().required(),
     readable: Joi.boolean(),
@@ -17,11 +16,19 @@ export function queryValidation() {
         default: Joi.string().allow(""),
       })
     ),
-    queryVerb: Joi.string().allow().required(),
+    queryVerb: Joi.string().required(),
     extra: Joi.object().optional(),
     schema: Joi.object({}).required().unknown(true),
     transformer: OPTIONAL_STRING,
     flags: Joi.object().optional(),
+    queryId: OPTIONAL_STRING,
+  }
+}
+
+export function queryValidation() {
+  return Joi.object({
+    ...baseQueryValidation(),
+    name: Joi.string().required(),
   }).unknown(true)
 }
 
@@ -32,19 +39,10 @@ export function generateQueryValidation() {
 
 export function generateQueryPreviewValidation() {
   // prettier-ignore
-  return auth.joiValidator.body(Joi.object({
+  return auth.joiValidator.body(
-    _id: OPTIONAL_STRING,
+    Joi.object({
-    _rev: OPTIONAL_STRING,
+      ...baseQueryValidation(),
-    readable: Joi.boolean().optional(),
+      name: OPTIONAL_STRING,
-    fields: Joi.object().required(),
+    }).unknown(true)
-    queryVerb: Joi.string().required(),
+  )
-    name: OPTIONAL_STRING,
-    flags: Joi.object().optional(),
-    schema: Joi.object().optional(),
-    extra: Joi.object().optional(),
-    datasourceId: Joi.string().required(),
-    transformer: OPTIONAL_STRING,
-    parameters: Joi.object({}).required().unknown(true),
-    queryId: OPTIONAL_STRING,
-  }).unknown(true))
 }

packages/server/src/api/routes/query.ts

@@ -8,8 +8,8 @@ import {
   paramResource,
 } from "../../middleware/resourceId"
 import {
-  generateQueryPreviewValidation,
   generateQueryValidation,
+  generateQueryPreviewValidation,
 } from "../controllers/query/validation"
 
 const { BUILDER, PermissionType, PermissionLevel } = permissions

packages/server/src/api/routes/tests/datasource.spec.ts

@@ -7,6 +7,7 @@ import sdk from "../../../sdk"
 
 import tk from "timekeeper"
 import { mocks } from "@budibase/backend-core/tests"
+import { QueryPreview } from "@budibase/types"
 
 tk.freeze(mocks.date.MOCK_DATE)
 
@@ -63,14 +64,17 @@ describe("/datasources", () => {
         datasource: any,
         fields: { path: string; queryString: string }
       ) {
-        return config.previewQuery(
+        const queryPreview: QueryPreview = {
-          request,
-          config,
-          datasource,
           fields,
-          undefined,
+          datasourceId: datasource._id,
-          ""
+          parameters: [],
-        )
+          transformer: null,
+          queryVerb: "read",
+          name: datasource.name,
+          schema: {},
+          readable: true,
+        }
+        return config.api.query.previewQuery(queryPreview)
       }
 
       it("should invalidate changed or removed variables", async () => {

packages/server/src/api/routes/tests/environmentVariables.spec.ts

@@ -14,6 +14,7 @@ jest.mock("pg", () => {
 import * as setup from "./utilities"
 import { mocks } from "@budibase/backend-core/tests"
 import { env, events } from "@budibase/backend-core"
+import { QueryPreview } from "@budibase/types"
 
 const structures = setup.structures
 
@@ -120,16 +121,19 @@ describe("/api/env/variables", () => {
       .expect(200)
     expect(response.body.datasource._id).toBeDefined()
 
-    const query = {
+    const queryPreview: QueryPreview = {
       datasourceId: response.body.datasource._id,
-      parameters: {},
+      parameters: [],
       fields: {},
       queryVerb: "read",
       name: response.body.datasource.name,
+      transformer: null,
+      schema: {},
+      readable: true,
     }
     const res = await request
       .post(`/api/queries/preview`)
-      .send(query)
+      .send(queryPreview)
       .set(config.defaultHeaders())
       .expect("Content-Type", /json/)
       .expect(200)
@@ -139,7 +143,7 @@ describe("/api/env/variables", () => {
     delete response.body.datasource.config
     expect(events.query.previewed).toBeCalledWith(
       response.body.datasource,
-      query
+      queryPreview
     )
     expect(pg.Client).toHaveBeenCalledWith({ password: "test", ssl: undefined })
   })

packages/server/src/api/routes/tests/queries/query.seq.spec.ts

@@ -1,5 +1,7 @@
 import tk from "timekeeper"
 
+const pg = require("pg")
+
 // Mock out postgres for this
 jest.mock("pg")
 jest.mock("node-fetch")
@@ -22,7 +24,13 @@ import { checkCacheForDynamicVariable } from "../../../../threads/utils"
 
 const { basicQuery, basicDatasource } = setup.structures
 import { events, db as dbCore } from "@budibase/backend-core"
-import { Datasource, Query, SourceName } from "@budibase/types"
+import {
+  Datasource,
+  Query,
+  SourceName,
+  QueryPreview,
+  QueryParameter,
+} from "@budibase/types"
 
 tk.freeze(Date.now())
 
@@ -218,28 +226,26 @@ describe("/queries", () => {
 
   describe("preview", () => {
     it("should be able to preview the query", async () => {
-      const query = {
+      const queryPreview: QueryPreview = {
         datasourceId: datasource._id,
-        parameters: {},
-        fields: {},
         queryVerb: "read",
-        name: datasource.name,
+        fields: {},
+        parameters: [],
+        transformer: "return data",
+        name: datasource.name!,
+        schema: {},
+        readable: true,
       }
-      const res = await request
+      const responseBody = await config.api.query.previewQuery(queryPreview)
-        .post(`/api/queries/preview`)
-        .send(query)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
       // these responses come from the mock
-      expect(res.body.schema).toEqual({
+      expect(responseBody.schema).toEqual({
         a: { type: "string", name: "a" },
         b: { type: "number", name: "b" },
       })
-      expect(res.body.rows.length).toEqual(1)
+      expect(responseBody.rows.length).toEqual(1)
       expect(events.query.previewed).toBeCalledTimes(1)
       delete datasource.config
-      expect(events.query.previewed).toBeCalledWith(datasource, query)
+      expect(events.query.previewed).toBeCalledWith(datasource, queryPreview)
     })
 
     it("should apply authorization to endpoint", async () => {
@@ -249,6 +255,128 @@ describe("/queries", () => {
         url: `/api/queries/preview`,
       })
     })
+
+    it("should not error when trying to generate a nested schema for an empty array", async () => {
+      const queryPreview: QueryPreview = {
+        datasourceId: datasource._id,
+        parameters: [],
+        fields: {},
+        queryVerb: "read",
+        name: datasource.name!,
+        transformer: "return data",
+        schema: {},
+        readable: true,
+      }
+      const rows = [
+        {
+          contacts: [],
+        },
+      ]
+      pg.queryMock.mockImplementation(() => ({
+        rows,
+      }))
+
+      const responseBody = await config.api.query.previewQuery(queryPreview)
+      expect(responseBody).toEqual({
+        nestedSchemaFields: {},
+        rows,
+        schema: {
+          contacts: { type: "array", name: "contacts" },
+        },
+      })
+      expect(responseBody.rows.length).toEqual(1)
+      delete datasource.config
+    })
+
+    it("should generate a nested schema based on all the nested items", async () => {
+      const queryPreview: QueryPreview = {
+        datasourceId: datasource._id,
+        parameters: [],
+        fields: {},
+        queryVerb: "read",
+        name: datasource.name!,
+        transformer: "return data",
+        schema: {},
+        readable: true,
+      }
+      const rows = [
+        {
+          contacts: [
+            {
+              address: "123 Lane",
+            },
+            {
+              address: "456 Drive",
+            },
+            {
+              postcode: "BT1 12N",
+              lat: 54.59,
+              long: -5.92,
+            },
+            {
+              city: "Belfast",
+            },
+            {
+              address: "789 Avenue",
+              phoneNumber: "0800-999-5555",
+            },
+            {
+              name: "Name",
+              isActive: false,
+            },
+          ],
+        },
+      ]
+      pg.queryMock.mockImplementation(() => ({
+        rows,
+      }))
+
+      const responseBody = await config.api.query.previewQuery(queryPreview)
+      expect(responseBody).toEqual({
+        nestedSchemaFields: {
+          contacts: {
+            address: {
+              type: "string",
+              name: "address",
+            },
+            postcode: {
+              type: "string",
+              name: "postcode",
+            },
+            lat: {
+              type: "number",
+              name: "lat",
+            },
+            long: {
+              type: "number",
+              name: "long",
+            },
+            city: {
+              type: "string",
+              name: "city",
+            },
+            phoneNumber: {
+              type: "string",
+              name: "phoneNumber",
+            },
+            name: {
+              type: "string",
+              name: "name",
+            },
+            isActive: {
+              type: "boolean",
+              name: "isActive",
+            },
+          },
+        },
+        rows,
+        schema: {
+          contacts: { type: "json", name: "contacts", subtype: "array" },
+        },
+      })
+      expect(responseBody.rows.length).toEqual(1)
+      delete datasource.config
+    })
   })
 
   describe("execute", () => {
@@ -283,7 +411,17 @@ describe("/queries", () => {
 
   describe("variables", () => {
     async function preview(datasource: Datasource, fields: any) {
-      return config.previewQuery(request, config, datasource, fields, undefined)
+      const queryPreview: QueryPreview = {
+        datasourceId: datasource._id!,
+        parameters: [],
+        fields,
+        queryVerb: "read",
+        name: datasource.name!,
+        transformer: "return data",
+        schema: {},
+        readable: true,
+      }
+      return await config.api.query.previewQuery(queryPreview)
     }
 
     it("should work with static variables", async () => {
@@ -293,31 +431,31 @@ describe("/queries", () => {
           variable2: "1",
         },
       })
-      const res = await preview(datasource, {
+      const responseBody = await preview(datasource, {
         path: "www.{{ variable }}.com",
         queryString: "test={{ variable2 }}",
       })
       // these responses come from the mock
-      expect(res.body.schema).toEqual({
+      expect(responseBody.schema).toEqual({
         opts: { type: "json", name: "opts" },
         url: { type: "string", name: "url" },
         value: { type: "string", name: "value" },
       })
-      expect(res.body.rows[0].url).toEqual("http://www.google.com?test=1")
+      expect(responseBody.rows[0].url).toEqual("http://www.google.com?test=1")
     })
 
     it("should work with dynamic variables", async () => {
       const { datasource } = await config.dynamicVariableDatasource()
-      const res = await preview(datasource, {
+      const responseBody = await preview(datasource, {
         path: "www.google.com",
         queryString: "test={{ variable3 }}",
       })
-      expect(res.body.schema).toEqual({
+      expect(responseBody.schema).toEqual({
         opts: { type: "json", name: "opts" },
         url: { type: "string", name: "url" },
         value: { type: "string", name: "value" },
       })
-      expect(res.body.rows[0].url).toContain("doctype%20html")
+      expect(responseBody.rows[0].url).toContain("doctype%20html")
     })
 
     it("check that it automatically retries on fail with cached dynamics", async () => {
@@ -331,16 +469,16 @@ describe("/queries", () => {
       // check its in cache
       const contents = await checkCacheForDynamicVariable(base._id, "variable3")
       expect(contents.rows.length).toEqual(1)
-      const res = await preview(datasource, {
+      const responseBody = await preview(datasource, {
         path: "www.failonce.com",
         queryString: "test={{ variable3 }}",
       })
-      expect(res.body.schema).toEqual({
+      expect(responseBody.schema).toEqual({
         fails: { type: "number", name: "fails" },
         opts: { type: "json", name: "opts" },
         url: { type: "string", name: "url" },
       })
-      expect(res.body.rows[0].fails).toEqual(1)
+      expect(responseBody.rows[0].fails).toEqual(1)
     })
 
     it("deletes variables when linked query is deleted", async () => {
@@ -371,24 +509,37 @@ describe("/queries", () => {
     async function previewGet(
       datasource: Datasource,
       fields: any,
-      params: any
+      params: QueryParameter[]
     ) {
-      return config.previewQuery(request, config, datasource, fields, params)
+      const queryPreview: QueryPreview = {
+        datasourceId: datasource._id!,
+        parameters: params,
+        fields,
+        queryVerb: "read",
+        name: datasource.name!,
+        transformer: "return data",
+        schema: {},
+        readable: true,
+      }
+      return await config.api.query.previewQuery(queryPreview)
     }
 
     async function previewPost(
       datasource: Datasource,
       fields: any,
-      params: any
+      params: QueryParameter[]
     ) {
-      return config.previewQuery(
+      const queryPreview: QueryPreview = {
-        request,
+        datasourceId: datasource._id!,
-        config,
+        parameters: params,
-        datasource,
         fields,
-        params,
+        queryVerb: "create",
-        "create"
+        name: datasource.name!,
-      )
+        transformer: null,
+        schema: {},
+        readable: false,
+      }
+      return await config.api.query.previewQuery(queryPreview)
     }
 
     it("should parse global and query level header mappings", async () => {
@@ -400,7 +551,7 @@ describe("/queries", () => {
           emailHdr: "{{[user].[email]}}",
         },
       })
-      const res = await previewGet(
+      const responseBody = await previewGet(
         datasource,
         {
           path: "www.google.com",
@@ -410,17 +561,17 @@ describe("/queries", () => {
             secondHdr: "1234",
           },
         },
-        undefined
+        []
       )
 
-      const parsedRequest = JSON.parse(res.body.extra.raw)
+      const parsedRequest = JSON.parse(responseBody.extra.raw)
       expect(parsedRequest.opts.headers).toEqual({
         test: "headerVal",
         emailHdr: userDetails.email,
         queryHdr: userDetails.firstName,
         secondHdr: "1234",
       })
-      expect(res.body.rows[0].url).toEqual(
+      expect(responseBody.rows[0].url).toEqual(
         "http://www.google.com?email=" + userDetails.email.replace("@", "%40")
       )
     })
@@ -430,21 +581,21 @@ describe("/queries", () => {
 
       const datasource = await config.restDatasource()
 
-      const res = await previewGet(
+      const responseBody = await previewGet(
         datasource,
         {
           path: "www.google.com",
           queryString:
             "test={{myEmail}}&testName={{myName}}&testParam={{testParam}}",
         },
-        {
+        [
-          myEmail: "{{[user].[email]}}",
+          { name: "myEmail", default: "{{[user].[email]}}" },
-          myName: "{{[user].[firstName]}}",
+          { name: "myName", default: "{{[user].[firstName]}}" },
-          testParam: "1234",
+          { name: "testParam", default: "1234" },
-        }
+        ]
       )
 
-      expect(res.body.rows[0].url).toEqual(
+      expect(responseBody.rows[0].url).toEqual(
         "http://www.google.com?test=" +
           userDetails.email.replace("@", "%40") +
           "&testName=" +
@@ -457,7 +608,7 @@ describe("/queries", () => {
       const userDetails = config.getUserDetails()
       const datasource = await config.restDatasource()
 
-      const res = await previewPost(
+      const responseBody = await previewPost(
         datasource,
         {
           path: "www.google.com",
@@ -466,16 +617,14 @@ describe("/queries", () => {
             "This is plain text and this is my email: {{[user].[email]}}. This is a test param: {{testParam}}",
           bodyType: "text",
         },
-        {
+        [{ name: "testParam", default: "1234" }]
-          testParam: "1234",
-        }
       )
 
-      const parsedRequest = JSON.parse(res.body.extra.raw)
+      const parsedRequest = JSON.parse(responseBody.extra.raw)
       expect(parsedRequest.opts.body).toEqual(
         `This is plain text and this is my email: ${userDetails.email}. This is a test param: 1234`
       )
-      expect(res.body.rows[0].url).toEqual(
+      expect(responseBody.rows[0].url).toEqual(
         "http://www.google.com?testParam=1234"
       )
     })
@@ -484,7 +633,7 @@ describe("/queries", () => {
       const userDetails = config.getUserDetails()
       const datasource = await config.restDatasource()
 
-      const res = await previewPost(
+      const responseBody = await previewPost(
         datasource,
         {
           path: "www.google.com",
@@ -493,16 +642,16 @@ describe("/queries", () => {
             '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}',
           bodyType: "json",
         },
-        {
+        [
-          testParam: "1234",
+          { name: "testParam", default: "1234" },
-          userRef: "{{[user].[firstName]}}",
+          { name: "userRef", default: "{{[user].[firstName]}}" },
-        }
+        ]
       )
 
-      const parsedRequest = JSON.parse(res.body.extra.raw)
+      const parsedRequest = JSON.parse(responseBody.extra.raw)
       const test = `{"email":"${userDetails.email}","queryCode":1234,"userRef":"${userDetails.firstName}"}`
       expect(parsedRequest.opts.body).toEqual(test)
-      expect(res.body.rows[0].url).toEqual(
+      expect(responseBody.rows[0].url).toEqual(
         "http://www.google.com?testParam=1234"
       )
     })
@@ -511,7 +660,7 @@ describe("/queries", () => {
       const userDetails = config.getUserDetails()
       const datasource = await config.restDatasource()
 
-      const res = await previewPost(
+      const responseBody = await previewPost(
         datasource,
         {
           path: "www.google.com",
@@ -521,17 +670,17 @@ describe("/queries", () => {
             "<ref>{{userId}}</ref> <somestring>testing</somestring> </note>",
           bodyType: "xml",
         },
-        {
+        [
-          testParam: "1234",
+          { name: "testParam", default: "1234" },
-          userId: "{{[user].[firstName]}}",
+          { name: "userId", default: "{{[user].[firstName]}}" },
-        }
+        ]
       )
 
-      const parsedRequest = JSON.parse(res.body.extra.raw)
+      const parsedRequest = JSON.parse(responseBody.extra.raw)
       const test = `<note> <email>${userDetails.email}</email> <code>1234</code> <ref>${userDetails.firstName}</ref> <somestring>testing</somestring> </note>`
 
       expect(parsedRequest.opts.body).toEqual(test)
-      expect(res.body.rows[0].url).toEqual(
+      expect(responseBody.rows[0].url).toEqual(
         "http://www.google.com?testParam=1234"
       )
     })
@@ -540,7 +689,7 @@ describe("/queries", () => {
       const userDetails = config.getUserDetails()
       const datasource = await config.restDatasource()
 
-      const res = await previewPost(
+      const responseBody = await previewPost(
         datasource,
         {
           path: "www.google.com",
@@ -549,13 +698,13 @@ describe("/queries", () => {
             '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}',
           bodyType: "form",
         },
-        {
+        [
-          testParam: "1234",
+          { name: "testParam", default: "1234" },
-          userRef: "{{[user].[firstName]}}",
+          { name: "userRef", default: "{{[user].[firstName]}}" },
-        }
+        ]
       )
 
-      const parsedRequest = JSON.parse(res.body.extra.raw)
+      const parsedRequest = JSON.parse(responseBody.extra.raw)
 
       const emailData = parsedRequest.opts.body._streams[1]
       expect(emailData).toEqual(userDetails.email)
@@ -566,7 +715,7 @@ describe("/queries", () => {
       const userRef = parsedRequest.opts.body._streams[7]
       expect(userRef).toEqual(userDetails.firstName)
 
-      expect(res.body.rows[0].url).toEqual(
+      expect(responseBody.rows[0].url).toEqual(
         "http://www.google.com?testParam=1234"
       )
     })
@@ -575,7 +724,7 @@ describe("/queries", () => {
       const userDetails = config.getUserDetails()
       const datasource = await config.restDatasource()
 
-      const res = await previewPost(
+      const responseBody = await previewPost(
         datasource,
         {
           path: "www.google.com",
@@ -584,12 +733,12 @@ describe("/queries", () => {
             '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}',
           bodyType: "encoded",
         },
-        {
+        [
-          testParam: "1234",
+          { name: "testParam", default: "1234" },
-          userRef: "{{[user].[firstName]}}",
+          { name: "userRef", default: "{{[user].[firstName]}}" },
-        }
+        ]
       )
-      const parsedRequest = JSON.parse(res.body.extra.raw)
+      const parsedRequest = JSON.parse(responseBody.extra.raw)
 
       expect(parsedRequest.opts.body.email).toEqual(userDetails.email)
       expect(parsedRequest.opts.body.queryCode).toEqual("1234")

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -866,28 +866,6 @@ export default class TestConfiguration {
 
   // QUERY
 
-  async previewQuery(
-    request: any,
-    config: any,
-    datasource: any,
-    fields: any,
-    params: any,
-    verb?: string
-  ) {
-    return request
-      .post(`/api/queries/preview`)
-      .send({
-        datasourceId: datasource._id,
-        parameters: params || {},
-        fields,
-        queryVerb: verb || "read",
-        name: datasource.name,
-      })
-      .set(config.defaultHeaders())
-      .expect("Content-Type", /json/)
-      .expect(200)
-  }
-
   async createQuery(config?: any) {
     if (!this.datasource && !config) {
       throw "No datasource created for query."

packages/server/src/tests/utilities/api/query.ts

@@ -1,6 +1,7 @@
 import TestConfiguration from "../TestConfiguration"
 import {
   Query,
+  QueryPreview,
   type ExecuteQueryRequest,
   type ExecuteQueryResponse,
 } from "@budibase/types"
@@ -41,4 +42,19 @@ export class QueryAPI extends TestAPI {
 
     return res.body
   }
+
+  previewQuery = async (queryPreview: QueryPreview) => {
+    const res = await this.request
+      .post(`/api/queries/preview`)
+      .send(queryPreview)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+
+    if (res.status !== 200) {
+      throw new Error(JSON.stringify(res.body))
+    }
+
+    return res.body
+  }
 }

packages/server/src/tests/utilities/structures.ts

@@ -366,7 +366,7 @@ export function basicDatasource(): { datasource: Datasource } {
 
 export function basicQuery(datasourceId: string): Query {
   return {
-    datasourceId: datasourceId,
+    datasourceId,
     name: "New Query",
     parameters: [],
     fields: {},

packages/server/src/threads/definitions.ts

@@ -7,10 +7,10 @@ export interface QueryEvent {
   datasource: Datasource
   queryVerb: string
   fields: { [key: string]: any }
-  parameters: { [key: string]: any }
+  parameters: { [key: string]: unknown }
   pagination?: any
   transformer: any
-  queryId: string
+  queryId?: string
   environmentVariables?: Record<string, string>
   ctx?: any
   schema?: Record<string, QuerySchema | string>

packages/server/src/threads/query.ts

@@ -43,7 +43,7 @@ class QueryRunner {
     this.parameters = input.parameters
     this.pagination = input.pagination
     this.transformer = input.transformer
-    this.queryId = input.queryId
+    this.queryId = input.queryId!
     this.schema = input.schema
     this.noRecursiveQuery = flags.noRecursiveQuery
     this.cachedVariables = []

packages/types/src/documents/app/query.ts

@@ -19,7 +19,7 @@ export interface Query extends Document {
 }
 
 export interface QueryPreview extends Omit<Query, "_id"> {
-  queryId: string
+  queryId?: string
 }
 
 export interface QueryParameter {