Merge pull request #6043 from Budibase/fix/json-schema

Enrich data fetch subscriptions with additional JSON schema fields

packages/client/src/utils/schema.js

@@ -1,5 +1,4 @@
 import { API } from "api"
-import { JSONUtils } from "@budibase/frontend-core"
 import TableFetch from "@budibase/frontend-core/src/fetch/TableFetch.js"
 import ViewFetch from "@budibase/frontend-core/src/fetch/ViewFetch.js"
 import QueryFetch from "@budibase/frontend-core/src/fetch/QueryFetch.js"
@@ -40,26 +39,27 @@ export const fetchDatasourceSchema = async (
     return null
   }
 
-  // Check for any JSON fields so we can add any top level properties
+  // Enrich schema with relationships if required
-  let jsonAdditions = {}
+  if (definition?.sql && options?.enrichRelationships) {
-  Object.keys(schema).forEach(fieldKey => {
+    const relationshipAdditions = await getRelationshipSchemaAdditions(schema)
-    const fieldSchema = schema[fieldKey]
+    schema = {
-    if (fieldSchema?.type === "json") {
+      ...schema,
-      const jsonSchema = JSONUtils.convertJSONSchemaToTableSchema(fieldSchema, {
+      ...relationshipAdditions,
-        squashObjects: true,
-      })
-      Object.keys(jsonSchema).forEach(jsonKey => {
-        jsonAdditions[`${fieldKey}.${jsonKey}`] = {
-          type: jsonSchema[jsonKey].type,
-          nestedJSON: true,
     }
-      })
   }
-  })
-  schema = { ...schema, ...jsonAdditions }
 
-  // Check for any relationship fields if required
+  // Ensure schema is in the correct structure
-  if (options?.enrichRelationships && definition.sql) {
+  return instance.enrichSchema(schema)
+}
+
+/**
+ * Fetches the schema of relationship fields for a SQL table schema
+ * @param schema the schema to enrich
+ */
+export const getRelationshipSchemaAdditions = async schema => {
+  if (!schema) {
+    return null
+  }
   let relationshipAdditions = {}
   for (let fieldKey of Object.keys(schema)) {
     const fieldSchema = schema[fieldKey]
@@ -75,9 +75,5 @@ export const fetchDatasourceSchema = async (
       })
     }
   }
-    schema = { ...schema, ...relationshipAdditions }
+  return relationshipAdditions
-  }
-
-  // Ensure schema structure is correct
-  return instance.enrichSchema(schema)
 }

packages/frontend-core/src/fetch/DataFetch.js

@@ -6,6 +6,7 @@ import {
   runLuceneQuery,
   luceneSort,
 } from "../utils/lucene"
+import { convertJSONSchemaToTableSchema } from "../utils/json"
 
 /**
  * Parent class which handles the implementation of fetching data from an
@@ -248,7 +249,8 @@ export default class DataFetch {
   }
 
   /**
-   * Enriches the schema and ensures that entries are objects with names
+   * Enriches a datasource schema with nested fields and ensures the structure
+   * is correct.
    * @param schema the datasource schema
    * @return {object} the enriched datasource schema
    */
@@ -256,6 +258,26 @@ export default class DataFetch {
     if (schema == null) {
       return null
     }
+
+    // Check for any JSON fields so we can add any top level properties
+    let jsonAdditions = {}
+    Object.keys(schema).forEach(fieldKey => {
+      const fieldSchema = schema[fieldKey]
+      if (fieldSchema?.type === "json") {
+        const jsonSchema = convertJSONSchemaToTableSchema(fieldSchema, {
+          squashObjects: true,
+        })
+        Object.keys(jsonSchema).forEach(jsonKey => {
+          jsonAdditions[`${fieldKey}.${jsonKey}`] = {
+            type: jsonSchema[jsonKey].type,
+            nestedJSON: true,
+          }
+        })
+      }
+    })
+    schema = { ...schema, ...jsonAdditions }
+
+    // Ensure schema is in the correct structure
     let enrichedSchema = {}
     Object.entries(schema).forEach(([fieldName, fieldSchema]) => {
       if (typeof fieldSchema === "string") {
@@ -270,6 +292,7 @@ export default class DataFetch {
         }
       }
     })
+
     return enrichedSchema
   }