From ed28bf664dfe460490bcaa3bf82818d54613cbc6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 26 Nov 2021 17:39:18 +0000 Subject: [PATCH 1/8] Adding server functionality to determine schema for JSON data type, some basic UI around an editor for getting JSON to determine schema from and the key/value mechanism for flat structures. --- .../DataTable/modals/CreateEditColumn.svelte | 5 +- .../DataTable/modals/JSONSchemaModal.svelte | 54 +++++++++++++------ .../server/src/api/controllers/table/index.js | 33 ++++++++++++ packages/server/src/api/routes/table.js | 18 +++++++ 4 files changed, 93 insertions(+), 17 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index fc597b658e..f4c0422304 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -452,7 +452,10 @@ - console.log(detail)} /> + (field.schema = detail)} + /> { - if (!schema) { - schema = {} - } - let i = 0 - for (let [key, value] of Object.entries(schema)) { - fieldKeys[i] = key - fieldTypes[i] = value.type - i++ - } - fieldCount = i + updateCounts() }) @@ -56,7 +78,7 @@ title={"Key/Value Schema Editor"} confirmText="Save Column" onConfirm={saveSchema} - disabled={invalid} + bind:disabled={invalid} size="L" > diff --git a/packages/server/src/api/controllers/table/index.js b/packages/server/src/api/controllers/table/index.js index abbb4d6ff9..bd404f3da8 100644 --- a/packages/server/src/api/controllers/table/index.js +++ b/packages/server/src/api/controllers/table/index.js @@ -9,6 +9,7 @@ const { BudibaseInternalDB, } = require("../../../db/utils") const { getTable } = require("./utils") +const { FieldTypes } = require("../../../constants") function pickApi({ tableId, table }) { if (table && !tableId) { @@ -81,6 +82,38 @@ exports.destroy = async function (ctx) { ctx.body = { message: `Table ${tableId} deleted.` } } +exports.schemaGenerate = function (ctx) { + const { json } = ctx.request.body + function recurse(schemaLevel, objectLevel) { + for (let [key, value] of Object.entries(objectLevel)) { + const type = typeof value + // check array first, since arrays are objects + if (Array.isArray(value)) { + schemaLevel[key] = { + type: FieldTypes.ARRAY, + } + } else if (type === "object") { + schemaLevel[key] = recurse(schemaLevel[key], objectLevel) + } else if (type === "string") { + schemaLevel[key] = { + type: FieldTypes.STRING, + } + } else if (type === "boolean") { + schemaLevel[key] = { + type: FieldTypes.BOOLEAN, + } + } else if (type === "number") { + schemaLevel[key] = { + type: FieldTypes.NUMBER, + } + } + } + return schemaLevel + } + + ctx.body = recurse({}, json) || {} +} + exports.bulkImport = async function (ctx) { const tableId = ctx.params.tableId await pickApi({ tableId }).bulkImport(ctx) diff --git a/packages/server/src/api/routes/table.js b/packages/server/src/api/routes/table.js index d8ddbe8133..a4575b3572 100644 --- a/packages/server/src/api/routes/table.js +++ b/packages/server/src/api/routes/table.js @@ -139,6 +139,24 @@ router generateSaveValidator(), tableController.save ) + /** + * @api {post} /api/tables/schema/generate Generate schema from JSON + * @apiName Generate schema from JSON + * @apiGroup tables + * @apiPermission builder + * @apiDescription Given a JSON structure this will generate a nested schema that can be used for a key/value data + * type in a table. + * + * @apiParam (Body) {object} json The JSON structure from which a nest schema should be generated. + * + * @apiSuccess {object} schema The response body will contain the schema, which can now be used for a key/value + * data type. + */ + .post( + "/api/tables/schema/generate", + authorized(BUILDER), + tableController.schemaGenerate + ) /** * @api {post} /api/tables/csv/validate Validate a CSV for a table * @apiName Validate a CSV for a table From 0da0002bc5400265b518ddf9215aa29f469a763a Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 29 Nov 2021 08:30:52 +0000 Subject: [PATCH 2/8] Preserve bindings when duplicating components --- packages/builder/src/builderStore/store/frontend.js | 4 ++-- .../design/NavigationPanel/ComponentDropdownMenu.svelte | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 1f1fb035a4..c94c759792 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -524,7 +524,7 @@ export const getFrontendStore = () => { } } }, - paste: async (targetComponent, mode) => { + paste: async (targetComponent, mode, preserveBindings = false) => { let promises = [] store.update(state => { // Stop if we have nothing to paste @@ -536,7 +536,7 @@ export const getFrontendStore = () => { const cut = state.componentToPaste.isCut // immediately need to remove bindings, currently these aren't valid when pasted - if (!cut) { + if (!cut && !preserveBindings) { state.componentToPaste = removeBindings(state.componentToPaste) } diff --git a/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte b/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte index 06293e4168..56c5eef2ad 100644 --- a/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte +++ b/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte @@ -53,7 +53,7 @@ const duplicateComponent = () => { storeComponentForCopy(false) - pasteComponent("below") + pasteComponent("below", true) } const deleteComponent = async () => { @@ -69,9 +69,9 @@ store.actions.components.copy(component, cut) } - const pasteComponent = mode => { + const pasteComponent = (mode, preserveBindings = false) => { // lives in store - also used by drag drop - store.actions.components.paste(component, mode) + store.actions.components.paste(component, mode, preserveBindings) } From 785ff452409c26c6097404ad6d1c973c375261f8 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 29 Nov 2021 08:58:49 +0000 Subject: [PATCH 3/8] Fix issue with navigation links editor mutating real component structure --- .../NavigationEditor/NavigationEditor.svelte | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/NavigationEditor/NavigationEditor.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/NavigationEditor/NavigationEditor.svelte index b7a272e608..ea02b4184d 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/NavigationEditor/NavigationEditor.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/NavigationEditor/NavigationEditor.svelte @@ -2,13 +2,15 @@ import { Button, ActionButton, Drawer } from "@budibase/bbui" import { createEventDispatcher } from "svelte" import NavigationDrawer from "./NavigationDrawer.svelte" + import { cloneDeep } from "lodash/fp" export let value = [] let drawer + let links = cloneDeep(value) const dispatch = createEventDispatcher() const save = () => { - dispatch("change", value) + dispatch("change", links) drawer.hide() } @@ -19,5 +21,5 @@ Configure the links in your navigation bar. - + From b44772b13680be893d9359b4f9e68f1a77ad64df Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 29 Nov 2021 09:05:46 +0000 Subject: [PATCH 4/8] Fix layout navigation not scrolling when required --- packages/client/src/components/app/Layout.svelte | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/client/src/components/app/Layout.svelte b/packages/client/src/components/app/Layout.svelte index 87e5ac3b5b..59765f9305 100644 --- a/packages/client/src/components/app/Layout.svelte +++ b/packages/client/src/components/app/Layout.svelte @@ -313,6 +313,9 @@ height: 100%; overflow: auto; } + .desktop.layout--left .links { + overflow-y: auto; + } .desktop .nav--left { width: 250px; @@ -379,6 +382,7 @@ justify-content: flex-start; align-items: stretch; padding: var(--spacing-xl); + overflow-y: auto; } .mobile .link { width: calc(100% - 30px); From de0b23dd9f931a87379f5bba7c85439a83ab61c2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 29 Nov 2021 17:11:08 +0000 Subject: [PATCH 5/8] Moving generation to builder because it reduces API calls and has no reason to be carried out server-side, handling array/object schema generation correctly. --- .../src/builderStore/schemaGenerator.js | 56 +++++++++++++++++++ .../DataTable/modals/CreateEditColumn.svelte | 11 +++- .../DataTable/modals/JSONSchemaModal.svelte | 40 ++++++------- .../server/src/api/controllers/table/index.js | 33 ----------- packages/server/src/api/routes/table.js | 18 ------ 5 files changed, 86 insertions(+), 72 deletions(-) create mode 100644 packages/builder/src/builderStore/schemaGenerator.js diff --git a/packages/builder/src/builderStore/schemaGenerator.js b/packages/builder/src/builderStore/schemaGenerator.js new file mode 100644 index 0000000000..33115fc997 --- /dev/null +++ b/packages/builder/src/builderStore/schemaGenerator.js @@ -0,0 +1,56 @@ +import { FIELDS } from "constants/backend" + +function baseConversion(type) { + if (type === "string") { + return { + type: FIELDS.STRING.type, + } + } else if (type === "boolean") { + return { + type: FIELDS.BOOLEAN.type, + } + } else if (type === "number") { + return { + type: FIELDS.NUMBER.type, + } + } +} + +function recurse(schemaLevel = {}, objectLevel) { + if (!objectLevel) { + return null + } + const baseType = typeof objectLevel + if (baseType !== "object") { + return baseConversion(baseType) + } + for (let [key, value] of Object.entries(objectLevel)) { + const type = typeof value + // check array first, since arrays are objects + if (Array.isArray(value)) { + const schema = recurse(schemaLevel[key], value[0]) + if (schema) { + schemaLevel[key] = { + type: FIELDS.ARRAY.type, + schema, + } + } + } else if (type === "object") { + const schema = recurse(schemaLevel[key], objectLevel[key]) + if (schema) { + schemaLevel[key] = schema + } + } else { + schemaLevel[key] = baseConversion(type) + } + } + if (!schemaLevel.type) { + return { type: FIELDS.JSON.type, schema: schemaLevel } + } else { + return schemaLevel + } +} + +export function generate(object) { + return recurse({}, object).schema +} diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index f4c0422304..752f291019 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -87,7 +87,10 @@ field.subtype !== AUTO_COLUMN_SUB_TYPES.CREATED_BY && field.subtype !== AUTO_COLUMN_SUB_TYPES.UPDATED_BY && field.type !== FORMULA_TYPE - $: canBeDisplay = field.type !== LINK_TYPE && field.type !== AUTO_TYPE + $: canBeDisplay = + field.type !== LINK_TYPE && + field.type !== AUTO_TYPE && + field.type !== JSON_TYPE $: canBeRequired = field.type !== LINK_TYPE && !uneditable && field.type !== AUTO_TYPE $: relationshipOptions = getRelationshipOptions(field) @@ -454,7 +457,11 @@ (field.schema = detail)} + json={field.json} + on:save={({ detail }) => { + field.schema = detail.schema + field.json = detail.json + }} /> { @@ -90,7 +91,8 @@ label="Type" options={keyValueOptions} bind:value={fieldTypes[i]} - getOptionValue={field => field.toLowerCase()} + getOptionValue={field => field.value} + getOptionLabel={field => field.label} /> {/each} diff --git a/packages/server/src/api/controllers/table/index.js b/packages/server/src/api/controllers/table/index.js index bd404f3da8..abbb4d6ff9 100644 --- a/packages/server/src/api/controllers/table/index.js +++ b/packages/server/src/api/controllers/table/index.js @@ -9,7 +9,6 @@ const { BudibaseInternalDB, } = require("../../../db/utils") const { getTable } = require("./utils") -const { FieldTypes } = require("../../../constants") function pickApi({ tableId, table }) { if (table && !tableId) { @@ -82,38 +81,6 @@ exports.destroy = async function (ctx) { ctx.body = { message: `Table ${tableId} deleted.` } } -exports.schemaGenerate = function (ctx) { - const { json } = ctx.request.body - function recurse(schemaLevel, objectLevel) { - for (let [key, value] of Object.entries(objectLevel)) { - const type = typeof value - // check array first, since arrays are objects - if (Array.isArray(value)) { - schemaLevel[key] = { - type: FieldTypes.ARRAY, - } - } else if (type === "object") { - schemaLevel[key] = recurse(schemaLevel[key], objectLevel) - } else if (type === "string") { - schemaLevel[key] = { - type: FieldTypes.STRING, - } - } else if (type === "boolean") { - schemaLevel[key] = { - type: FieldTypes.BOOLEAN, - } - } else if (type === "number") { - schemaLevel[key] = { - type: FieldTypes.NUMBER, - } - } - } - return schemaLevel - } - - ctx.body = recurse({}, json) || {} -} - exports.bulkImport = async function (ctx) { const tableId = ctx.params.tableId await pickApi({ tableId }).bulkImport(ctx) diff --git a/packages/server/src/api/routes/table.js b/packages/server/src/api/routes/table.js index a4575b3572..d8ddbe8133 100644 --- a/packages/server/src/api/routes/table.js +++ b/packages/server/src/api/routes/table.js @@ -139,24 +139,6 @@ router generateSaveValidator(), tableController.save ) - /** - * @api {post} /api/tables/schema/generate Generate schema from JSON - * @apiName Generate schema from JSON - * @apiGroup tables - * @apiPermission builder - * @apiDescription Given a JSON structure this will generate a nested schema that can be used for a key/value data - * type in a table. - * - * @apiParam (Body) {object} json The JSON structure from which a nest schema should be generated. - * - * @apiSuccess {object} schema The response body will contain the schema, which can now be used for a key/value - * data type. - */ - .post( - "/api/tables/schema/generate", - authorized(BUILDER), - tableController.schemaGenerate - ) /** * @api {post} /api/tables/csv/validate Validate a CSV for a table * @apiName Validate a CSV for a table From ec12d6a045bdb0a1c6f73bd1aafce9088546a581 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 29 Nov 2021 17:54:09 +0000 Subject: [PATCH 6/8] Fixing issue with updating row validation to allow empty objects. --- packages/server/src/api/controllers/row/utils.js | 12 ++++-------- packages/server/src/utilities/rowProcessor/index.js | 12 ++++++++++++ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils.js b/packages/server/src/api/controllers/row/utils.js index f7a4b13304..83c97ceb78 100644 --- a/packages/server/src/api/controllers/row/utils.js +++ b/packages/server/src/api/controllers/row/utils.js @@ -50,10 +50,10 @@ exports.validate = async ({ appId, tableId, row, table }) => { const errors = {} for (let fieldName of Object.keys(table.schema)) { const constraints = cloneDeep(table.schema[fieldName].constraints) + const type = table.schema[fieldName].type // special case for options, need to always allow unselected (null) if ( - table.schema[fieldName].type === - (FieldTypes.OPTIONS || FieldTypes.ARRAY) && + (type === FieldTypes.OPTIONS || type === FieldTypes.ARRAY) && constraints.inclusion ) { constraints.inclusion.push(null) @@ -61,17 +61,13 @@ exports.validate = async ({ appId, tableId, row, table }) => { let res // Validate.js doesn't seem to handle array - if ( - table.schema[fieldName].type === FieldTypes.ARRAY && - row[fieldName] && - row[fieldName].length - ) { + if (type === FieldTypes.ARRAY && row[fieldName] && row[fieldName].length) { row[fieldName].map(val => { if (!constraints.inclusion.includes(val)) { errors[fieldName] = "Field not in list" } }) - } else if (table.schema[fieldName].type === FieldTypes.FORMULA) { + } else if (type === FieldTypes.FORMULA) { res = validateJs.single( processStringSync(table.schema[fieldName].formula, row), constraints diff --git a/packages/server/src/utilities/rowProcessor/index.js b/packages/server/src/utilities/rowProcessor/index.js index ea63c23f7d..860063f173 100644 --- a/packages/server/src/utilities/rowProcessor/index.js +++ b/packages/server/src/utilities/rowProcessor/index.js @@ -81,6 +81,18 @@ const TYPE_TRANSFORM_MAP = { [FieldTypes.AUTO]: { parse: () => undefined, }, + [FieldTypes.JSON]: { + parse: input => { + try { + if (input === "") { + return undefined + } + return JSON.parse(input) + } catch (err) { + return input + } + }, + }, } /** From 1cee1b78e62cb1d6011c4d960af6f79f612387bb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 29 Nov 2021 18:16:44 +0000 Subject: [PATCH 7/8] Adding validation around invalid JSON inputs and allowing input via a code mirror editor in data UI. --- .../backend/DataTable/RowFieldControl.svelte | 12 ++++++++++++ .../backend/DataTable/modals/JSONSchemaModal.svelte | 1 + packages/server/src/api/controllers/row/utils.js | 7 +++++++ 3 files changed, 20 insertions(+) diff --git a/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte b/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte index 25ad67b52e..0d9ca3644b 100644 --- a/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte +++ b/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte @@ -6,16 +6,20 @@ Toggle, TextArea, Multiselect, + Label, } from "@budibase/bbui" import Dropzone from "components/common/Dropzone.svelte" import { capitalise } from "helpers" import LinkedRowSelector from "components/common/LinkedRowSelector.svelte" + import Editor from "../../integration/QueryEditor.svelte" export let defaultValue export let meta export let value = defaultValue || (meta.type === "boolean" ? false : "") export let readonly + $: stringVal = + typeof value === "object" ? JSON.stringify(value, null, 2) : value $: type = meta?.type $: label = meta.name ? capitalise(meta.name) : "" @@ -40,6 +44,14 @@ {:else if type === "longform"}