Refactor how grid schema mutations are managed to support custom orders, widths and visibility of columns that are still user-overridable

packages/client/src/components/app/GridBlock.svelte

@@ -38,11 +38,10 @@
   let grid
   let gridContext
   let minHeight
-  let resizedColumns = {}
 
   $: parsedColumns = getParsedColumns(columns)
   $: columnWhitelist = parsedColumns.filter(x => x.active).map(x => x.field)
-  $: schemaOverrides = getSchemaOverrides(parsedColumns, resizedColumns)
+  $: schemaOverrides = getSchemaOverrides(parsedColumns)
   $: enrichedButtons = enrichButtons(buttons)
   $: selectedRows = deriveSelectedRows(gridContext)
   $: styles = patchStyles($component.styles, minHeight)
@@ -84,17 +83,13 @@
     }))
   }
 
-  const getSchemaOverrides = (columns, resizedColumns) => {
+  const getSchemaOverrides = columns => {
     let overrides = {}
-    columns.forEach(column => {
+    columns.forEach((column, idx) => {
       overrides[column.field] = {
         displayName: column.label,
-      }
+        width: column.width,
-
+        order: idx,
-      // Only use the specified width until we resize the column, at which point
-      // we no longer want to override it
-      if (!resizedColumns[column.field]) {
-        overrides[column.field].width = column.width
       }
     })
     return overrides
@@ -146,13 +141,6 @@
     }
   }
 
-  const onColumnResize = e => {
-    // Mark that we've resized this column so we can remove this width from
-    // schema overrides if present
-    const { column } = e.detail
-    resizedColumns = { ...resizedColumns, [column]: true }
-  }
-
   onMount(() => {
     gridContext = grid.getContext()
     gridContext.minHeight.subscribe($height => (minHeight = $height))
@@ -185,7 +173,6 @@
     buttons={enrichedButtons}
     isCloud={$environmentStore.cloud}
     on:rowclick={e => onRowClick?.({ row: e.detail })}
-    on:columnresize={onColumnResize}
   />
 </div>
 

packages/frontend-core/src/components/grid/cells/HeaderCell.svelte

@@ -23,7 +23,6 @@
     subscribe,
     config,
     ui,
-    columns,
     definition,
     datasource,
     schema,
@@ -158,17 +157,13 @@
   }
 
   const makeDisplayColumn = () => {
-    columns.actions.changePrimaryDisplay(column.name)
+    datasource.actions.changePrimaryDisplay(column.name)
     open = false
   }
 
   const hideColumn = () => {
-    columns.update(state => {
+    datasource.actions.addSchemaMutation(column.name, { visible: false })
-      const index = state.findIndex(col => col.name === column.name)
+    datasource.actions.saveSchemaMutations()
-      state[index].visible = false
-      return state.slice()
-    })
-    columns.actions.saveChanges()
     open = false
   }
 

packages/frontend-core/src/components/grid/controls/HideColumnsButton.svelte

@@ -3,7 +3,7 @@
   import { ActionButton, Popover, Toggle, Icon } from "@budibase/bbui"
   import { getColumnIcon } from "../lib/utils"
 
-  const { columns, stickyColumn, dispatch } = getContext("grid")
+  const { columns, datasource, stickyColumn, dispatch } = getContext("grid")
 
   let open = false
   let anchor
@@ -12,34 +12,28 @@
   $: text = getText($columns)
 
   const toggleVisibility = async (column, visible) => {
-    columns.update(state => {
+    datasource.actions.addSchemaMutation(column, { visible })
-      const index = state.findIndex(col => col.name === column.name)
+    await datasource.actions.saveSchemaMutations()
-      state[index].visible = visible
-      return state.slice()
-    })
-    await columns.actions.saveChanges()
     dispatch(visible ? "show-column" : "hide-column")
   }
 
   const showAll = async () => {
-    columns.update(state => {
+    let mutations = {}
-      return state.map(col => ({
+    columns.forEach(column => {
-        ...col,
+      mutations[column.name] = { visible: true }
-        visible: true,
-      }))
     })
-    await columns.actions.saveChanges()
+    datasource.actions.addSchemaMutations(mutations)
+    await datasource.actions.saveSchemaMutations()
     dispatch("show-column")
   }
 
   const hideAll = async () => {
-    columns.update(state => {
+    let mutations = {}
-      return state.map(col => ({
+    columns.forEach(column => {
-        ...col,
+      mutations[column.name] = { visible: false }
-        visible: false,
-      }))
     })
-    await columns.actions.saveChanges()
+    datasource.actions.addSchemaMutations(mutations)
+    await datasource.actions.saveSchemaMutations()
     dispatch("hide-column")
   }
 

packages/frontend-core/src/components/grid/stores/columns.js

@@ -1,5 +1,4 @@
 import { derived, get, writable } from "svelte/store"
-import { cloneDeep } from "lodash/fp"
 import { GutterWidth, DefaultColumnWidth } from "../lib/constants"
 
 export const createStores = () => {
@@ -75,72 +74,23 @@ export const deriveStores = context => {
 }
 
 export const createActions = context => {
-  const { columns, stickyColumn, datasource, definition, schema } = context
+  const { columns, datasource, schema } = context
-
-  // Updates the datasources primary display column
-  const changePrimaryDisplay = async column => {
-    return await datasource.actions.saveDefinition({
-      ...get(definition),
-      primaryDisplay: column,
-    })
-  }
 
   // Updates the width of all columns
   const changeAllColumnWidths = async width => {
-    columns.update(state => {
+    const $schema = get(schema)
-      return state.map(col => ({
+    let mutations = {}
-        ...col,
+    Object.keys($schema).forEach(field => {
-        width,
+      mutations[field] = { width }
-      }))
-    })
-    if (get(stickyColumn)) {
-      stickyColumn.update(state => ({
-        ...state,
-        width,
-      }))
-    }
-    await saveChanges()
-  }
-
-  // Persists column changes by saving metadata against datasource schema
-  const saveChanges = async () => {
-    const $columns = get(columns)
-    const $definition = get(definition)
-    const $stickyColumn = get(stickyColumn)
-    let newSchema = cloneDeep(get(schema)) || {}
-
-    // Build new updated datasource schema
-    Object.keys(newSchema).forEach(column => {
-      // Respect order specified by columns
-      const index = $columns.findIndex(x => x.name === column)
-      if (index !== -1) {
-        newSchema[column].order = index
-      } else {
-        delete newSchema[column].order
-      }
-
-      // Copy over metadata
-      if (column === $stickyColumn?.name) {
-        newSchema[column].visible = true
-        newSchema[column].width = $stickyColumn.width || DefaultColumnWidth
-      } else {
-        newSchema[column].visible = $columns[index]?.visible ?? true
-        newSchema[column].width = $columns[index]?.width || DefaultColumnWidth
-      }
-    })
-
-    await datasource.actions.saveDefinition({
-      ...$definition,
-      schema: newSchema,
     })
+    datasource.actions.addSchemaMutations(mutations)
+    await datasource.actions.saveSchemaMutations()
   }
 
   return {
     columns: {
       ...columns,
       actions: {
-        saveChanges,
-        changePrimaryDisplay,
         changeAllColumnWidths,
       },
     },

packages/frontend-core/src/components/grid/stores/datasource.js

@@ -4,15 +4,25 @@ import { memo } from "../../../utils"
 
 export const createStores = () => {
   const definition = memo(null)
+  const schemaMutations = memo({})
+
+  definition.subscribe(console.log)
 
   return {
     definition,
+    schemaMutations,
   }
 }
 
 export const deriveStores = context => {
-  const { API, definition, schemaOverrides, columnWhitelist, datasource } =
+  const {
-    context
+    API,
+    definition,
+    schemaOverrides,
+    columnWhitelist,
+    datasource,
+    schemaMutations,
+  } = context
 
   const schema = derived(definition, $definition => {
     let schema = getDatasourceSchema({
@@ -35,42 +45,26 @@ export const deriveStores = context => {
     return schema
   })
 
+  // Derives the total enriched schema, made up of the saved schema and any
+  // prop and user overrides
   const enrichedSchema = derived(
-    [schema, schemaOverrides, columnWhitelist],
+    [schema, schemaOverrides, schemaMutations, columnWhitelist],
-    ([$schema, $schemaOverrides, $columnWhitelist]) => {
+    ([$schema, $schemaOverrides, schemaMutations, $columnWhitelist]) => {
       if (!$schema) {
         return null
       }
-      let enrichedSchema = { ...$schema }
+      let enrichedSchema = {}
-
+      Object.keys($schema).forEach(field => {
-      // Apply schema overrides
+        // Apply whitelist if provided
-      Object.keys($schemaOverrides || {}).forEach(field => {
+        if ($columnWhitelist?.length && !$columnWhitelist.includes(field)) {
-        if (enrichedSchema[field]) {
+          return
+        }
         enrichedSchema[field] = {
-            ...enrichedSchema[field],
+          ...$schema[field],
           ...$schemaOverrides[field],
-          }
+          ...schemaMutations[field],
         }
       })
-
-      // Apply whitelist if specified
-      if ($columnWhitelist?.length) {
-        const sortedColumns = {}
-
-        $columnWhitelist.forEach((columnKey, idx) => {
-          const enrichedColumn = enrichedSchema[columnKey]
-          if (enrichedColumn) {
-            sortedColumns[columnKey] = {
-              ...enrichedColumn,
-              order: idx,
-              visible: true,
-            }
-          }
-        })
-
-        return sortedColumns
-      }
-
       return enrichedSchema
     }
   )
@@ -100,6 +94,8 @@ export const createActions = context => {
     table,
     viewV2,
     nonPlus,
+    schemaMutations,
+    schema,
   } = context
 
   // Gets the appropriate API for the configured datasource type
@@ -136,11 +132,20 @@ export const createActions = context => {
     // Update server
     if (get(config).canSaveSchema) {
       await getAPI()?.actions.saveDefinition(newDefinition)
+
+      // Broadcast change so external state can be updated, as this change
+      // will not be received by the builder websocket because we caused it
+      // ourselves
+      dispatch("updatedatasource", newDefinition)
+    }
   }
 
-    // Broadcast change to external state can be updated, as this change
+  // Updates the datasources primary display column
-    // will not be received by the builder websocket because we caused it ourselves
+  const changePrimaryDisplay = async column => {
-    dispatch("updatedatasource", newDefinition)
+    return await saveDefinition({
+      ...get(definition),
+      primaryDisplay: column,
+    })
   }
 
   // Adds a row to the datasource
@@ -173,6 +178,67 @@ export const createActions = context => {
     return getAPI()?.actions.canUseColumn(name)
   }
 
+  // Adds a schema mutation for a single field
+  const addSchemaMutation = (field, mutation) => {
+    if (!field || !mutation) {
+      return
+    }
+    schemaMutations.update($schemaMutations => {
+      return {
+        ...$schemaMutations,
+        [field]: {
+          ...$schemaMutations[field],
+          ...mutation,
+        },
+      }
+    })
+  }
+
+  // Adds schema mutations for multiple fields at once
+  const addSchemaMutations = mutations => {
+    const fields = Object.keys(mutations || {})
+    if (!fields.length) {
+      return
+    }
+    schemaMutations.update($schemaMutations => {
+      let newSchemaMutations = { ...$schemaMutations }
+      fields.forEach(field => {
+        newSchemaMutations[field] = {
+          ...newSchemaMutations[field],
+          ...mutations[field],
+        }
+      })
+      return newSchemaMutations
+    })
+  }
+
+  // Saves schema changes to the server, if possible
+  const saveSchemaMutations = async () => {
+    // If we can't save schema changes then we just want to keep this in memory
+    if (!get(config).canSaveSchema) {
+      return
+    }
+    const $definition = get(definition)
+    const $schemaMutations = get(schemaMutations)
+    const $schema = get(schema)
+    let newSchema = {}
+
+    // Build new updated datasource schema
+    Object.keys($schema).forEach(column => {
+      newSchema[column] = {
+        ...$schema[column],
+        ...$schemaMutations[column],
+      }
+    })
+
+    // Save the changes, then reset our local mutations
+    await saveDefinition({
+      ...$definition,
+      schema: newSchema,
+    })
+    schemaMutations.set({})
+  }
+
   return {
     datasource: {
       ...datasource,
@@ -185,6 +251,10 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
+        changePrimaryDisplay,
+        addSchemaMutation,
+        addSchemaMutations,
+        saveSchemaMutations,
       },
     },
   }

packages/frontend-core/src/components/grid/stores/reorder.js

@@ -34,6 +34,7 @@ export const createActions = context => {
     stickyColumn,
     maxScrollLeft,
     width,
+    datasource,
   } = context
 
   let autoScrollInterval
@@ -176,8 +177,7 @@ export const createActions = context => {
     // Ensure there's actually a change
     let { sourceColumn, targetColumn } = get(reorder)
     if (sourceColumn !== targetColumn) {
-      moveColumn(sourceColumn, targetColumn)
+      await moveColumn(sourceColumn, targetColumn)
-      await columns.actions.saveChanges()
     }
 
     // Reset state
@@ -186,7 +186,7 @@ export const createActions = context => {
 
   // Moves a column after another columns.
   // An undefined target column will move the source to index 0.
-  const moveColumn = (sourceColumn, targetColumn) => {
+  const moveColumn = async (sourceColumn, targetColumn) => {
     let $columns = get(columns)
     let sourceIdx = $columns.findIndex(x => x.name === sourceColumn)
     let targetIdx = $columns.findIndex(x => x.name === targetColumn)
@@ -198,14 +198,21 @@ export const createActions = context => {
       }
       return state.toSpliced(targetIdx, 0, removed[0])
     })
+
+    // Extract new orders as schema mutations
+    let mutations = {}
+    get(columns).forEach((column, idx) => {
+      mutations[column.name] = { order: idx }
+    })
+    datasource.actions.addSchemaMutations(mutations)
+    await datasource.actions.saveSchemaMutations()
   }
 
   // Moves a column one place left (as appears visually)
   const moveColumnLeft = async column => {
     const $visibleColumns = get(visibleColumns)
     const sourceIdx = $visibleColumns.findIndex(x => x.name === column)
-    moveColumn(column, $visibleColumns[sourceIdx - 2]?.name)
+    await moveColumn(column, $visibleColumns[sourceIdx - 2]?.name)
-    await columns.actions.saveChanges()
   }
 
   // Moves a column one place right (as appears visually)
@@ -215,8 +222,7 @@ export const createActions = context => {
     if (sourceIdx === $visibleColumns.length - 1) {
       return
     }
-    moveColumn(column, $visibleColumns[sourceIdx + 1]?.name)
+    await moveColumn(column, $visibleColumns[sourceIdx + 1]?.name)
-    await columns.actions.saveChanges()
   }
 
   return {

packages/frontend-core/src/components/grid/stores/resize.js

@@ -6,7 +6,6 @@ const initialState = {
   initialMouseX: null,
   initialWidth: null,
   column: null,
-  columnIdx: null,
   width: 0,
   left: 0,
 }
@@ -21,7 +20,7 @@ export const createStores = () => {
 }
 
 export const createActions = context => {
-  const { resize, columns, stickyColumn, ui, dispatch } = context
+  const { resize, ui, datasource } = context
 
   // Starts resizing a certain column
   const startResizing = (column, e) => {
@@ -32,12 +31,6 @@ export const createActions = context => {
     e.preventDefault()
     ui.actions.blur()
 
-    // Find and cache index
-    let columnIdx = get(columns).findIndex(col => col.name === column.name)
-    if (columnIdx === -1) {
-      columnIdx = "sticky"
-    }
-
     // Set initial store state
     resize.set({
       width: column.width,
@@ -45,7 +38,6 @@ export const createActions = context => {
       initialWidth: column.width,
       initialMouseX: x,
       column: column.name,
-      columnIdx,
     })
 
     // Add mouse event listeners to handle resizing
@@ -58,7 +50,7 @@ export const createActions = context => {
 
   // Handler for moving the mouse to resize columns
   const onResizeMouseMove = e => {
-    const { initialMouseX, initialWidth, width, columnIdx } = get(resize)
+    const { initialMouseX, initialWidth, width, column } = get(resize)
     const { x } = parseEventLocation(e)
     const dx = x - initialMouseX
     const newWidth = Math.round(Math.max(MinColumnWidth, initialWidth + dx))
@@ -69,17 +61,7 @@ export const createActions = context => {
     }
 
     // Update column state
-    if (columnIdx === "sticky") {
+    datasource.actions.addSchemaMutation(column, { width })
-      stickyColumn.update(state => ({
-        ...state,
-        width: newWidth,
-      }))
-    } else {
-      columns.update(state => {
-        state[columnIdx].width = newWidth
-        return [...state]
-      })
-    }
 
     // Update state
     resize.update(state => ({
@@ -101,28 +83,14 @@ export const createActions = context => {
 
     // Persist width if it changed
     if ($resize.width !== $resize.initialWidth) {
-      await columns.actions.saveChanges()
+      await datasource.actions.saveSchemaMutations()
-      const { column, width } = $resize
-      dispatch("columnresize", { column, width })
     }
   }
 
   // Resets a column size back to default
   const resetSize = async column => {
-    const $stickyColumn = get(stickyColumn)
+    datasource.actions.addSchemaMutation(column, { width: DefaultColumnWidth })
-    if (column.name === $stickyColumn?.name) {
+    await datasource.actions.saveSchemaMutations()
-      stickyColumn.update(state => ({
-        ...state,
-        width: DefaultColumnWidth,
-      }))
-    } else {
-      columns.update(state => {
-        const columnIdx = state.findIndex(x => x.name === column.name)
-        state[columnIdx].width = DefaultColumnWidth
-        return [...state]
-      })
-    }
-    await columns.actions.saveChanges()
   }
 
   return {