From c12e5fd196a38696a3a8a3f0a778e966caef5b97 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Mon, 19 Feb 2024 09:13:03 +0000 Subject: [PATCH] Fix array type missing from query schema selector (#12772) * Tidy MongoDB aggregation pipeline view * Remove unused code * WIP * Add bindings for bindings drawer * Is not external table if it's a query * Add QueryArrayFetch * Bug fix * JavaScript is the worst * refactor * Add array label to query schema * Remove console log * type fix * Don't include Array in SchemaTypeOptions, but show label * Fix bindings * refactor * Rename isObject to hasSchema * WIP * Typing WIP * Type not Types * Unused import * type fix * Handle json array subtype * Support queryarray datasource type * refactor * yarn lock * update account portal --------- Co-authored-by: Sam Rose --- .../src/events/publishers/query.ts | 5 +- .../DataSourceSelect/DataSourceSelect.svelte | 11 +- .../integration/KeyValueBuilder.svelte | 14 +- .../components/integration/QueryViewer.svelte | 5 +- .../src/components/integration/index.svelte | 2 - .../builder/src/constants/backend/index.js | 1 + packages/builder/src/dataBinding.js | 15 ++- .../src/components/app/forms/Form.svelte | 2 +- packages/client/src/utils/schema.js | 2 + .../src/fetch/QueryArrayFetch.js | 25 ++++ packages/frontend-core/src/fetch/index.js | 2 + packages/frontend-core/src/utils/json.js | 32 +++++ .../server/src/api/controllers/query/index.ts | 126 +++++++++++------- packages/shared-core/src/utils.ts | 10 ++ packages/types/src/documents/app/query.ts | 13 ++ .../src/documents/app/table/constants.ts | 4 + .../types/src/documents/app/table/schema.ts | 7 + yarn.lock | 8 +- 18 files changed, 219 insertions(+), 65 deletions(-) create mode 100644 packages/frontend-core/src/fetch/QueryArrayFetch.js diff --git a/packages/backend-core/src/events/publishers/query.ts b/packages/backend-core/src/events/publishers/query.ts index 7d28129cf6..48603257d2 100644 --- a/packages/backend-core/src/events/publishers/query.ts +++ b/packages/backend-core/src/events/publishers/query.ts @@ -3,6 +3,7 @@ import { Event, Datasource, Query, + QueryPreview, QueryCreatedEvent, QueryUpdatedEvent, QueryDeletedEvent, @@ -68,9 +69,9 @@ const run = async (count: number, timestamp?: string | number) => { await publishEvent(Event.QUERIES_RUN, properties, timestamp) } -const previewed = async (datasource: Datasource, query: Query) => { +const previewed = async (datasource: Datasource, query: QueryPreview) => { const properties: QueryPreviewedEvent = { - queryId: query._id, + queryId: query.queryId, datasourceId: datasource._id as string, source: datasource.source, queryVerb: query.queryVerb, diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index 14cbc973a1..0e01c264fc 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -127,10 +127,14 @@ } }) $: jsonArrays = bindings - .filter(x => x.fieldSchema?.type === "jsonarray") + .filter( + x => + x.fieldSchema?.type === "jsonarray" || + (x.fieldSchema?.type === "json" && x.fieldSchema?.subtype === "array") + ) .map(binding => { const { providerId, readableBinding, runtimeBinding, tableId } = binding - const { name, type, prefixKeys } = binding.fieldSchema + const { name, type, prefixKeys, subtype } = binding.fieldSchema return { providerId, label: readableBinding, @@ -138,7 +142,8 @@ fieldType: type, tableId, prefixKeys, - type: "jsonarray", + type: type === "jsonarray" ? "jsonarray" : "queryarray", + subtype, value: `{{ literal ${runtimeBinding} }}`, } }) diff --git a/packages/builder/src/components/integration/KeyValueBuilder.svelte b/packages/builder/src/components/integration/KeyValueBuilder.svelte index 8dac07bcec..e71090f613 100644 --- a/packages/builder/src/components/integration/KeyValueBuilder.svelte +++ b/packages/builder/src/components/integration/KeyValueBuilder.svelte @@ -85,6 +85,16 @@ activity = newActivity dispatch("change", fields) } + + function isJsonArray(value) { + if (!value || typeof value === "string") { + return false + } + if (value.type === "array") { + return true + } + return value.type === "json" && value.subtype === "array" + } @@ -112,7 +122,9 @@ bind:value={field.name} on:blur={changed} /> - {#if options} + {#if isJsonArray(field.value)} +
-
Stage {index + 1} diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index eb47ac97fe..f1e3e1e2c2 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -310,6 +310,7 @@ export const BannedSearchTypes = [ "formula", "json", "jsonarray", + "queryarray", ] export const DatasourceTypes = { diff --git a/packages/builder/src/dataBinding.js b/packages/builder/src/dataBinding.js index 8c71631b09..0442a67da9 100644 --- a/packages/builder/src/dataBinding.js +++ b/packages/builder/src/dataBinding.js @@ -425,7 +425,7 @@ const generateComponentContextBindings = (asset, componentContext) => { table = info.table // Determine what to prefix bindings with - if (datasource.type === "jsonarray") { + if (datasource.type === "jsonarray" || datasource.type === "queryarray") { // For JSON arrays, use the array name as the readable prefix const split = datasource.label.split(".") readablePrefix = split[split.length - 1] @@ -904,6 +904,19 @@ export const getSchemaForDatasource = (asset, datasource, options) => { schema = JSONUtils.getJSONArrayDatasourceSchema(tableSchema, datasource) } + // "queryarray" datasources are arrays inside JSON responses + else if (type === "queryarray") { + const queries = get(queriesStores).list + table = queries.find(query => query._id === datasource.tableId) + let tableSchema = table?.schema + let nestedSchemaFields = table?.nestedSchemaFields + schema = JSONUtils.generateQueryArraySchemas( + tableSchema, + nestedSchemaFields + ) + schema = JSONUtils.getJSONArrayDatasourceSchema(schema, datasource) + } + // Otherwise we assume we're targeting an internal table or a plus // datasource, and we can treat it as a table with a schema else { diff --git a/packages/client/src/components/app/forms/Form.svelte b/packages/client/src/components/app/forms/Form.svelte index 464ca95829..5522bd4b46 100644 --- a/packages/client/src/components/app/forms/Form.svelte +++ b/packages/client/src/components/app/forms/Form.svelte @@ -84,7 +84,7 @@ // Fetches the form schema from this form's dataSource const fetchSchema = async dataSource => { - if (dataSource?.tableId && dataSource?.type !== "query") { + if (dataSource?.tableId && !dataSource?.type?.startsWith("query")) { try { table = await API.fetchTableDefinition(dataSource.tableId) } catch (error) { diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js index f20e724a6e..e2399e8738 100644 --- a/packages/client/src/utils/schema.js +++ b/packages/client/src/utils/schema.js @@ -7,6 +7,7 @@ import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProvide import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch.js" import JSONArrayFetch from "@budibase/frontend-core/src/fetch/JSONArrayFetch.js" import ViewV2Fetch from "@budibase/frontend-core/src/fetch/ViewV2Fetch.js" +import QueryArrayFetch from "@budibase/frontend-core/src/fetch/QueryArrayFetch" /** * Fetches the schema of any kind of datasource. @@ -28,6 +29,7 @@ export const fetchDatasourceSchema = async ( provider: NestedProviderFetch, field: FieldFetch, jsonarray: JSONArrayFetch, + queryarray: QueryArrayFetch, }[datasource?.type] if (!handler) { return null diff --git a/packages/frontend-core/src/fetch/QueryArrayFetch.js b/packages/frontend-core/src/fetch/QueryArrayFetch.js new file mode 100644 index 0000000000..0b36b640a6 --- /dev/null +++ b/packages/frontend-core/src/fetch/QueryArrayFetch.js @@ -0,0 +1,25 @@ +import FieldFetch from "./FieldFetch.js" +import { + getJSONArrayDatasourceSchema, + generateQueryArraySchemas, +} from "../utils/json" + +export default class QueryArrayFetch extends FieldFetch { + async getDefinition(datasource) { + if (!datasource?.tableId) { + return null + } + // JSON arrays need their table definitions fetched. + // We can then extract their schema as a subset of the table schema. + try { + const table = await this.API.fetchQueryDefinition(datasource.tableId) + const schema = generateQueryArraySchemas( + table?.schema, + table?.nestedSchemaFields + ) + return { schema: getJSONArrayDatasourceSchema(schema, datasource) } + } catch (error) { + return null + } + } +} diff --git a/packages/frontend-core/src/fetch/index.js b/packages/frontend-core/src/fetch/index.js index a41a859351..903810ac25 100644 --- a/packages/frontend-core/src/fetch/index.js +++ b/packages/frontend-core/src/fetch/index.js @@ -9,6 +9,7 @@ import JSONArrayFetch from "./JSONArrayFetch.js" import UserFetch from "./UserFetch.js" import GroupUserFetch from "./GroupUserFetch.js" import CustomFetch from "./CustomFetch.js" +import QueryArrayFetch from "./QueryArrayFetch.js" const DataFetchMap = { table: TableFetch, @@ -24,6 +25,7 @@ const DataFetchMap = { provider: NestedProviderFetch, field: FieldFetch, jsonarray: JSONArrayFetch, + queryarray: QueryArrayFetch, } // Constructs a new fetch model for a certain datasource diff --git a/packages/frontend-core/src/utils/json.js b/packages/frontend-core/src/utils/json.js index 29bf2df34e..8cd37f9ad1 100644 --- a/packages/frontend-core/src/utils/json.js +++ b/packages/frontend-core/src/utils/json.js @@ -1,3 +1,5 @@ +import { utils } from "@budibase/shared-core" + /** * Gets the schema for a datasource which is targeting a JSON array, including * nested JSON arrays. The returned schema is a squashed, table-like schema @@ -119,3 +121,33 @@ const extractJSONSchemaKeys = (jsonSchema, squashObjects = false) => { }) return keys } + +export const generateQueryArraySchemas = (schema, nestedSchemaFields) => { + for (let key in schema) { + if ( + schema[key]?.type === "json" && + schema[key]?.subtype === "array" && + utils.hasSchema(nestedSchemaFields[key]) + ) { + schema[key] = { + schema: { + schema: Object.entries(nestedSchemaFields[key] || {}).reduce( + (acc, [nestedKey, fieldSchema]) => { + acc[nestedKey] = { + name: nestedKey, + type: fieldSchema.type, + subtype: fieldSchema.subtype, + } + return acc + }, + {} + ), + type: "json", + }, + type: "json", + subtype: "array", + } + } + } + return schema +} diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 8dabe5b3cc..89330f3216 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -1,5 +1,4 @@ import { generateQueryID } from "../../../db/utils" -import { BaseQueryVerbs } from "../../../constants" import { Thread, ThreadType } from "../../../threads" import { save as saveDatasource } from "../datasource" import { RestImporter } from "./import" @@ -7,36 +6,27 @@ import { invalidateDynamicVariables } from "../../../threads/utils" import env from "../../../environment" import { events, context, utils, constants } from "@budibase/backend-core" import sdk from "../../../sdk" -import { QueryEvent, QueryResponse } from "../../../threads/definitions" +import { QueryEvent } from "../../../threads/definitions" import { ConfigType, Query, UserCtx, SessionCookie, + JsonFieldSubType, + QueryResponse, + QueryPreview, QuerySchema, FieldType, type ExecuteQueryRequest, type ExecuteQueryResponse, type Row, } from "@budibase/types" -import { ValidQueryNameRegex } from "@budibase/shared-core" +import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core" const Runner = new Thread(ThreadType.QUERY, { timeoutMs: env.QUERY_THREAD_TIMEOUT, }) -// simple function to append "readable" to all read queries -function enrichQueries(input: any) { - const wasArray = Array.isArray(input) - const queries = wasArray ? input : [input] - for (let query of queries) { - if (query.queryVerb === BaseQueryVerbs.READ) { - query.readable = true - } - } - return wasArray ? queries : queries[0] -} - export async function fetch(ctx: UserCtx) { ctx.body = await sdk.queries.fetch() } @@ -84,7 +74,7 @@ export { _import as import } export async function save(ctx: UserCtx) { const db = context.getAppDB() - const query = ctx.request.body + const query: Query = ctx.request.body // Validate query name if (!query?.name.match(ValidQueryNameRegex)) { @@ -100,7 +90,6 @@ export async function save(ctx: UserCtx) { } else { eventFn = () => events.query.updated(datasource, query) } - const response = await db.put(query) await eventFn() query._rev = response.rev @@ -133,7 +122,7 @@ export async function preview(ctx: UserCtx) { const { datasource, envVars } = await sdk.datasources.getWithEnvVars( ctx.request.body.datasourceId ) - const query = ctx.request.body + const query: QueryPreview = ctx.request.body // preview may not have a queryId as it hasn't been saved, but if it does // this stops dynamic variables from calling the same query const { fields, parameters, queryVerb, transformer, queryId, schema } = query @@ -153,6 +142,69 @@ export async function preview(ctx: UserCtx) { const authConfigCtx: any = getAuthConfig(ctx) + function getSchemaFields( + rows: any[], + keys: string[] + ): { + previewSchema: Record + nestedSchemaFields: { + [key: string]: Record + } + } { + const previewSchema: Record = {} + const nestedSchemaFields: { + [key: string]: Record + } = {} + const makeQuerySchema = ( + type: FieldType, + name: string, + subtype?: string + ): QuerySchema => ({ + type, + name, + subtype, + }) + if (rows?.length > 0) { + for (let key of [...new Set(keys)] as string[]) { + const field = rows[0][key] + let type = typeof field, + fieldMetadata = makeQuerySchema(FieldType.STRING, key) + if (field) + switch (type) { + case "boolean": + fieldMetadata = makeQuerySchema(FieldType.BOOLEAN, key) + break + case "object": + if (field instanceof Date) { + fieldMetadata = makeQuerySchema(FieldType.DATETIME, key) + } else if (Array.isArray(field)) { + if (JsonUtils.hasSchema(field[0])) { + fieldMetadata = makeQuerySchema( + FieldType.JSON, + key, + JsonFieldSubType.ARRAY + ) + } else { + fieldMetadata = makeQuerySchema(FieldType.ARRAY, key) + } + nestedSchemaFields[key] = getSchemaFields( + field, + Object.keys(field[0]) + ).previewSchema + } else { + fieldMetadata = makeQuerySchema(FieldType.JSON, key) + } + break + case "number": + fieldMetadata = makeQuerySchema(FieldType.NUMBER, key) + break + } + previewSchema[key] = fieldMetadata + } + } + return { previewSchema, nestedSchemaFields } + } + try { const inputs: QueryEvent = { appId: ctx.appId, @@ -171,38 +223,11 @@ export async function preview(ctx: UserCtx) { }, } - const { rows, keys, info, extra } = await Runner.run(inputs) - const previewSchema: Record = {} - const makeQuerySchema = (type: FieldType, name: string): QuerySchema => ({ - type, - name, - }) - if (rows?.length > 0) { - for (let key of [...new Set(keys)] as string[]) { - const field = rows[0][key] - let type = typeof field, - fieldMetadata = makeQuerySchema(FieldType.STRING, key) - if (field) - switch (type) { - case "boolean": - fieldMetadata = makeQuerySchema(FieldType.BOOLEAN, key) - break - case "object": - if (field instanceof Date) { - fieldMetadata = makeQuerySchema(FieldType.DATETIME, key) - } else if (Array.isArray(field)) { - fieldMetadata = makeQuerySchema(FieldType.ARRAY, key) - } else { - fieldMetadata = makeQuerySchema(FieldType.JSON, key) - } - break - case "number": - fieldMetadata = makeQuerySchema(FieldType.NUMBER, key) - break - } - previewSchema[key] = fieldMetadata - } - } + const { rows, keys, info, extra } = (await Runner.run( + inputs + )) as QueryResponse + const { previewSchema, nestedSchemaFields } = getSchemaFields(rows, keys) + // if existing schema, update to include any previous schema keys if (existingSchema) { for (let key of Object.keys(previewSchema)) { @@ -216,6 +241,7 @@ export async function preview(ctx: UserCtx) { await events.query.previewed(datasource, query) ctx.body = { rows, + nestedSchemaFields, schema: previewSchema, info, extra, diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index c775010909..cc5646a00e 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -57,3 +57,13 @@ export function filterValueToLabel() { {} ) } + +export function hasSchema(test: any) { + return ( + typeof test === "object" && + !Array.isArray(test) && + test !== null && + !(test instanceof Date) && + Object.keys(test).length > 0 + ) +} diff --git a/packages/types/src/documents/app/query.ts b/packages/types/src/documents/app/query.ts index 81aa90b807..f4547b9774 100644 --- a/packages/types/src/documents/app/query.ts +++ b/packages/types/src/documents/app/query.ts @@ -4,6 +4,7 @@ import type { Row } from "./row" export interface QuerySchema { name?: string type: string + subtype?: string } export interface Query extends Document { @@ -17,11 +18,23 @@ export interface Query extends Document { queryVerb: string } +export interface QueryPreview extends Omit { + queryId: string +} + export interface QueryParameter { name: string default: string } +export interface QueryResponse { + rows: any[] + keys: string[] + info: any + extra: any + pagination: any +} + export interface RestQueryFields { path: string queryString?: string diff --git a/packages/types/src/documents/app/table/constants.ts b/packages/types/src/documents/app/table/constants.ts index fc831e7e7c..1d9d14695a 100644 --- a/packages/types/src/documents/app/table/constants.ts +++ b/packages/types/src/documents/app/table/constants.ts @@ -16,6 +16,10 @@ export enum AutoFieldSubType { AUTO_ID = "autoID", } +export enum JsonFieldSubType { + ARRAY = "array", +} + export enum FormulaType { STATIC = "static", DYNAMIC = "dynamic", diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts index 47ec303b66..17abf747b2 100644 --- a/packages/types/src/documents/app/table/schema.ts +++ b/packages/types/src/documents/app/table/schema.ts @@ -5,6 +5,7 @@ import { AutoFieldSubType, AutoReason, FormulaType, + JsonFieldSubType, RelationshipType, } from "./constants" @@ -81,6 +82,11 @@ export interface NumberFieldMetadata extends Omit { } } +export interface JsonFieldMetadata extends Omit { + type: FieldType.JSON + subtype?: JsonFieldSubType.ARRAY +} + export interface DateFieldMetadata extends Omit { type: FieldType.DATETIME ignoreTimezones?: boolean @@ -162,6 +168,7 @@ export type FieldSchema = | NumberFieldMetadata | LongFormFieldMetadata | BBReferenceFieldMetadata + | JsonFieldMetadata export interface TableSchema { [key: string]: FieldSchema diff --git a/yarn.lock b/yarn.lock index 1937482837..c0a11b9bf7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5572,9 +5572,9 @@ integrity sha512-7GgtHCs/QZrBrDzgIJnQtuSvhFSwhyYSI2uafSwZoNt1iOGhEN5fwNrQMjtONyHm9+/LoA4453jH0CMYcr06Pg== "@types/node@^18.11.18": - version "18.19.10" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.10.tgz#4de314ab66faf6bc8ba691021a091ddcdf13a158" - integrity sha512-IZD8kAM02AW1HRDTPOlz3npFava678pr8Ie9Vp8uRhBROXAv8MXT2pCnGZZAKYdromsNQLHQcfWQ6EOatVLtqA== + version "18.19.13" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.13.tgz#c3e989ca967b862a1f6c8c4148fe31865eedaf1a" + integrity sha512-kgnbRDj8ioDyGxoiaXsiu1Ybm/K14ajCgMOkwiqpHrnF7d7QiYRoRqHIpglMMs3DwXinlK4qJ8TZGlj4hfleJg== dependencies: undici-types "~5.26.4" @@ -10763,7 +10763,7 @@ fetch-cookie@0.11.0: dependencies: tough-cookie "^2.3.3 || ^3.0.1 || ^4.0.0" -fflate@^0.4.1: +fflate@^0.4.1, fflate@^0.4.8: version "0.4.8" resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.4.8.tgz#f90b82aefbd8ac174213abb338bd7ef848f0f5ae" integrity sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==