Refactor grid to split up stores and provide better separation of datasource-specific logic

packages/builder/src/components/backend/DataTable/TableDataTable.svelte

@@ -61,7 +61,7 @@
     allowDeleteRows={!isUsersTable}
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
-    on:updatetable={handleGridTableUpdate}
+    on:updatedatasource={handleGridTableUpdate}
   >
     <svelte:fragment slot="filter">
       <GridFilterButton />

packages/builder/src/components/backend/DataTable/buttons/grid/GridFilterButton.svelte

@@ -2,22 +2,22 @@
   import TableFilterButton from "../TableFilterButton.svelte"
   import { getContext } from "svelte"
 
-  const { columns, tableId, filter, table } = getContext("grid")
+  const { columns, datasource, filter, definition } = getContext("grid")
 
   // Wipe filter whenever table ID changes to avoid using stale filters
-  $: $tableId, filter.set([])
+  $: $datasource, filter.set([])
 
   const onFilter = e => {
     filter.set(e.detail || [])
   }
 </script>
 
-{#key $tableId}
+{#key $datasource}
   <TableFilterButton
-    schema={$table?.schema}
+    schema={$definition?.schema}
     filters={$filter}
     on:change={onFilter}
     disabled={!$columns.length}
-    tableId={$tableId}
+    tableId={$datasource.tableId}
   />
 {/key}

packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte

@@ -4,12 +4,12 @@
 
   export let disabled = false
 
-  const { rows, datasource, table } = getContext("grid")
+  const { rows, datasource, definition } = getContext("grid")
 </script>
 
 <ImportButton
   {disabled}
-  tableId={$datasource.tableId}
-  tableType={$table?.type}
+  tableId={$datasource?.tableId}
+  tableType={$definition?.type}
   on:importrows={rows.actions.refreshData}
 />

packages/builder/src/components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte

@@ -2,7 +2,7 @@
   import ManageAccessButton from "../ManageAccessButton.svelte"
   import { getContext } from "svelte"
 
-  const { tableId } = getContext("grid")
+  const { datasource } = getContext("grid")
 </script>
 
-<ManageAccessButton resourceId={$tableId} />
+<ManageAccessButton resourceId={$datasource?.tableId} />

packages/builder/src/components/backend/DataTable/buttons/grid/GridRelationshipButton.svelte

@@ -2,12 +2,12 @@
   import ExistingRelationshipButton from "../ExistingRelationshipButton.svelte"
   import { getContext } from "svelte"
 
-  const { table, rows } = getContext("grid")
+  const { definition, rows } = getContext("grid")
 </script>
 
-{#if $table}
+{#if $definition}
   <ExistingRelationshipButton
-    table={$table}
+    table={$definition}
     on:updatecolumns={() => rows.actions.refreshData()}
   />
 {/if}

packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte

@@ -2,7 +2,7 @@
   import { getContext } from "svelte"
   import CreateEditColumn from "components/backend/DataTable/modals/CreateEditColumn.svelte"
 
-  const { rows } = getContext("grid")
+  const { datasource } = getContext("grid")
 </script>
 
-<CreateEditColumn on:updatecolumns={rows.actions.refreshDatasourceDefinition} />
+<CreateEditColumn on:updatecolumns={datasource.actions.refreshDefinition} />

packages/frontend-core/src/components/grid/controls/SizeButton.svelte

@@ -8,8 +8,14 @@
     SmallRowHeight,
   } from "../lib/constants"
 
-  const { stickyColumn, columns, rowHeight, table, fixedRowHeight } =
-    getContext("grid")
+  const {
+    stickyColumn,
+    columns,
+    rowHeight,
+    definition,
+    fixedRowHeight,
+    datasource,
+  } = getContext("grid")
 
   // Some constants for column width options
   const smallColSize = 120
@@ -60,8 +66,8 @@
   ]
 
   const changeRowHeight = height => {
-    columns.actions.saveTable({
-      ...$table,
+    datasource.actions.saveDefinition({
+      ...$definition,
       rowHeight: height,
     })
   }

packages/frontend-core/src/components/grid/index.js

@@ -1 +1,2 @@
 export { default as Grid } from "./layout/Grid.svelte"
+export { DatasourceType } from "./lib/constants"

packages/frontend-core/src/components/grid/lib/constants.js

@@ -1,3 +1,7 @@
+export const DatasourceType = {
+  Table: "table",
+  ViewV2: "viewV2",
+}
 export const Padding = 246
 export const MaxCellRenderHeight = 222
 export const ScrollBarSize = 8

packages/frontend-core/src/components/grid/stores/columns.js

@@ -69,8 +69,7 @@ export const deriveStores = context => {
 }
 
 export const createActions = context => {
-  const { table, columns, stickyColumn, API, dispatch, config, datasource } =
-    context
+  const { columns, stickyColumn, config, datasource, definition } = context
 
   // Checks if we have a certain column by name
   const hasColumn = column => {
@@ -79,13 +78,13 @@ export const createActions = context => {
     return $columns.some(col => col.name === column) || $sticky?.name === column
   }
 
-  // Updates the tables primary display column
+  // Updates the datasources primary display column
   const changePrimaryDisplay = async column => {
     if (!get(config).canEditPrimaryDisplay) {
       return
     }
-    return await saveTable({
-      ...get(table),
+    return await datasource.actions.saveDefinition({
+      ...get(definition),
       primaryDisplay: column,
     })
   }
@@ -107,14 +106,14 @@ export const createActions = context => {
     await saveChanges()
   }
 
-  // Persists column changes by saving metadata against table schema
+  // Persists column changes by saving metadata against datasource schema
   const saveChanges = async () => {
     const $columns = get(columns)
-    const $table = get(table)
+    const $definition = get(definition)
     const $stickyColumn = get(stickyColumn)
-    const newSchema = cloneDeep($table.schema)
+    const newSchema = cloneDeep($definition.schema)
 
-    // Build new updated table schema
+    // Build new updated datasource schema
     Object.keys(newSchema).forEach(column => {
       // Respect order specified by columns
       const index = $columns.findIndex(x => x.name === column)
@@ -134,28 +133,10 @@ export const createActions = context => {
       }
     })
 
-    await saveTable({ ...$table, schema: newSchema })
-  }
-
-  const saveTable = async newTable => {
-    const $config = get(config)
-    const $datasource = get(datasource)
-
-    // Update local state
-    table.set(newTable)
-
-    // Update server
-    if ($config.canSaveSchema) {
-      if ($datasource.type === "table") {
-        await API.saveTable(newTable)
-      } else if ($datasource.type === "viewV2") {
-        await API.viewV2.update({ ...newTable })
-      }
-    }
-
-    // Broadcast change to external state can be updated, as this change
-    // will not be received by the builder websocket because we caused it ourselves
-    dispatch("updatetable", newTable)
+    await datasource.actions.saveDefinition({
+      ...$definition,
+      schema: newSchema,
+    })
   }
 
   return {
@@ -164,7 +145,6 @@ export const createActions = context => {
       actions: {
         hasColumn,
         saveChanges,
-        saveTable,
         changePrimaryDisplay,
         changeAllColumnWidths,
       },
@@ -173,51 +153,7 @@ export const createActions = context => {
 }
 
 export const initialise = context => {
-  const { table, columns, stickyColumn, schemaOverrides, columnWhitelist } =
-    context
-
-  const schema = derived(
-    [table, schemaOverrides, columnWhitelist],
-    ([$table, $schemaOverrides, $columnWhitelist]) => {
-      if (!$table?.schema) {
-        return null
-      }
-      let newSchema = { ...$table?.schema }
-
-      // Edge case to temporarily allow deletion of duplicated user
-      // fields that were saved with the "disabled" flag set.
-      // By overriding the saved schema we ensure only overrides can
-      // set the disabled flag.
-      // TODO: remove in future
-      Object.keys(newSchema).forEach(field => {
-        newSchema[field] = {
-          ...newSchema[field],
-          disabled: false,
-        }
-      })
-
-      // Apply schema overrides
-      Object.keys($schemaOverrides || {}).forEach(field => {
-        if (newSchema[field]) {
-          newSchema[field] = {
-            ...newSchema[field],
-            ...$schemaOverrides[field],
-          }
-        }
-      })
-
-      // Apply whitelist if specified
-      if ($columnWhitelist?.length) {
-        Object.keys(newSchema).forEach(key => {
-          if (!$columnWhitelist.includes(key)) {
-            delete newSchema[key]
-          }
-        })
-      }
-
-      return newSchema
-    }
-  )
+  const { definition, columns, stickyColumn, schema } = context
 
   // Merge new schema fields with existing schema in order to preserve widths
   schema.subscribe($schema => {
@@ -226,12 +162,12 @@ export const initialise = context => {
       stickyColumn.set(null)
       return
     }
-    const $table = get(table)
+    const $definition = get(definition)
 
     // Find primary display
     let primaryDisplay
-    if ($table.primaryDisplay && $schema[$table.primaryDisplay]) {
-      primaryDisplay = $table.primaryDisplay
+    if ($definition.primaryDisplay && $schema[$definition.primaryDisplay]) {
+      primaryDisplay = $definition.primaryDisplay
     }
 
     // Get field list

packages/frontend-core/src/components/grid/stores/config.js

@@ -1,5 +1,6 @@
 import { derivedMemo } from "../../../utils"
 import { derived } from "svelte/store"
+import { DatasourceType } from "../lib/constants"
 
 export const deriveStores = context => {
   const { props, hasNonAutoColumn } = context
@@ -28,8 +29,7 @@ export const deriveStores = context => {
       }
 
       // Disable some features if we're editing a view
-      if ($props.datasource?.type === "viewV2") {
-        config.canEditPrimaryDisplay = false
+      if ($props.datasource?.type === DatasourceType.ViewV2) {
         config.canEditColumns = false
       }
 

packages/frontend-core/src/components/grid/stores/datsource.js

@@ -0,0 +1,107 @@
+import { derived, get, writable } from "svelte/store"
+
+export const createStores = () => {
+  const definition = writable(null)
+
+  return {
+    definition,
+  }
+}
+
+export const deriveStores = context => {
+  const { definition, schemaOverrides, columnWhitelist } = context
+
+  const schema = derived(
+    [definition, schemaOverrides, columnWhitelist],
+    ([$definition, $schemaOverrides, $columnWhitelist]) => {
+      if (!$definition?.schema) {
+        return null
+      }
+      let newSchema = { ...$definition?.schema }
+
+      // Edge case to temporarily allow deletion of duplicated user
+      // fields that were saved with the "disabled" flag set.
+      // By overriding the saved schema we ensure only overrides can
+      // set the disabled flag.
+      // TODO: remove in future
+      Object.keys(newSchema).forEach(field => {
+        newSchema[field] = {
+          ...newSchema[field],
+          disabled: false,
+        }
+      })
+
+      // Apply schema overrides
+      Object.keys($schemaOverrides || {}).forEach(field => {
+        if (newSchema[field]) {
+          newSchema[field] = {
+            ...newSchema[field],
+            ...$schemaOverrides[field],
+          }
+        }
+      })
+
+      // Apply whitelist if specified
+      if ($columnWhitelist?.length) {
+        Object.keys(newSchema).forEach(key => {
+          if (!$columnWhitelist.includes(key)) {
+            delete newSchema[key]
+          }
+        })
+      }
+
+      return newSchema
+    }
+  )
+
+  return {
+    schema,
+  }
+}
+
+export const createActions = context => {
+  const { datasource, definition, API, config, dispatch } = context
+
+  // Refreshes the datasource definition
+  const refreshDefinition = async () => {
+    const $datasource = get(datasource)
+    if ($datasource.type === "table") {
+      definition.set(await API.fetchTableDefinition($datasource.tableId))
+    } else if ($datasource.type === "viewV2") {
+      // const definition = await API.viewsV2.(get(tableId))
+      // table.set(definition)
+    }
+  }
+
+  // Saves the datasource definition
+  const saveDefinition = async newDefinition => {
+    const $config = get(config)
+    const $datasource = get(datasource)
+
+    // Update local state
+    definition.set(newDefinition)
+
+    // Update server
+    if ($config.canSaveSchema) {
+      if ($datasource.type === "table") {
+        await API.saveTable(newDefinition)
+      } else if ($datasource.type === "viewV2") {
+        await API.viewV2.update({ ...newDefinition })
+      }
+    }
+
+    // Broadcast change to external state can be updated, as this change
+    // will not be received by the builder websocket because we caused it ourselves
+    dispatch("updatedefinition", newDefinition)
+  }
+
+  return {
+    datasource: {
+      ...datasource,
+      actions: {
+        refreshDefinition,
+        saveDefinition,
+      },
+    },
+  }
+}

packages/frontend-core/src/components/grid/stores/index.js

@@ -15,13 +15,18 @@ import * as Config from "./config"
 import * as Sort from "./sort"
 import * as Filter from "./filter"
 import * as Notifications from "./notifications"
+import * as Table from "./table"
+import * as ViewV2 from "./viewV2"
+import * as Datasource from "./datsource"
 
 const DependencyOrderedStores = [
+  // Common stores
   Notifications,
   Sort,
   Filter,
   Bounds,
   Scroll,
+  Datasource,
   Columns,
   Rows,
   UI,
@@ -34,6 +39,10 @@ const DependencyOrderedStores = [
   Pagination,
   Clipboard,
   Config,
+
+  // Datasource specific stores
+  Table,
+  ViewV2,
 ]
 
 export const attachStores = context => {

packages/frontend-core/src/components/grid/stores/rows.js

@@ -7,13 +7,13 @@ const SuppressErrors = true
 
 export const createStores = () => {
   const rows = writable([])
-  const table = writable(null)
   const loading = writable(false)
   const loaded = writable(false)
   const rowChangeCache = writable({})
   const inProgressChanges = writable({})
   const hasNextPage = writable(false)
   const error = writable(null)
+  const fetch = writable(null)
 
   // Generate a lookup map to quick find a row by ID
   const rowLookupMap = derived(rows, $rows => {
@@ -51,8 +51,8 @@ export const createStores = () => {
       ...rows,
       subscribe: enrichedRows.subscribe,
     },
+    fetch,
     rowLookupMap,
-    table,
     loaded,
     loading,
     rowChangeCache,
@@ -66,7 +66,7 @@ export const createActions = context => {
   const {
     rows,
     rowLookupMap,
-    table,
+    definition,
     filter,
     loading,
     sort,
@@ -82,14 +82,14 @@ export const createActions = context => {
     hasNextPage,
     error,
     notifications,
+    fetch,
   } = context
   const instanceLoaded = writable(false)
-  const fetch = writable(null)
 
   // Local cache of row IDs to speed up checking if a row exists
   let rowCacheMap = {}
 
-  // Reset everything when table ID changes
+  // Reset everything when datasource changes
   let unsubscribe = null
   let lastResetKey = null
   datasource.subscribe(async $datasource => {
@@ -100,11 +100,11 @@ export const createActions = context => {
     loading.set(true)
 
     // Abandon if we don't have a valid datasource
-    if (!$datasource?.tableId) {
+    if (!$datasource) {
       return
     }
 
-    // Tick to allow other reactive logic to update stores when table ID changes
+    // Tick to allow other reactive logic to update stores when datasource changes
     // before proceeding. This allows us to wipe filters etc if needed.
     await tick()
     const $filter = get(filter)
@@ -142,7 +142,7 @@ export const createActions = context => {
         const previousResetKey = lastResetKey
         lastResetKey = $fetch.resetKey
 
-        // If resetting rows due to a table change, wipe data and wait for
+        // If resetting rows due to a datasource change, wipe data and wait for
         // derived stores to compute. This prevents stale data being passed
         // to cells when we save the new schema.
         if (!$instanceLoaded && previousResetKey) {
@@ -152,16 +152,17 @@ export const createActions = context => {
 
         // Reset state properties when dataset changes
         if (!$instanceLoaded || resetRows) {
-          table.set($fetch.definition)
-          sort.set({
-            column: $fetch.sortColumn,
-            order: $fetch.sortOrder,
-          })
+          definition.set($fetch.definition)
+
+          // sort.set({
+          //   column: $fetch.sortColumn,
+          //   order: $fetch.sortOrder,
+          // })
         }
 
         // Reset scroll state when data changes
         if (!$instanceLoaded) {
-          // Reset both top and left for a new table ID
+          // Reset both top and left for a new datasource ID
           instanceLoaded.set(true)
           scroll.set({ top: 0, left: 0 })
         } else if (resetRows) {
@@ -169,8 +170,6 @@ export const createActions = context => {
           scroll.update(state => ({ ...state, top: 0 }))
         }
 
-        // For views we always update the filter to match the definition
-
         // Process new rows
         handleNewRows($fetch.rows, resetRows)
 
@@ -182,23 +181,6 @@ export const createActions = context => {
     fetch.set(newFetch)
   })
 
-  // Update fetch when filter or sort config changes
-  filter.subscribe($filter => {
-    if (get(datasource)?.type === "table") {
-      get(fetch)?.update({
-        filter: $filter,
-      })
-    }
-  })
-  sort.subscribe($sort => {
-    if (get(datasource)?.type === "table") {
-      get(fetch)?.update({
-        sortOrder: $sort.order,
-        sortColumn: $sort.column,
-      })
-    }
-  })
-
   // Gets a row by ID
   const getRow = id => {
     const index = get(rowLookupMap)[id]
@@ -506,17 +488,6 @@ export const createActions = context => {
     get(fetch)?.nextPage()
   }
 
-  // Refreshes the schema of the data fetch subscription
-  const refreshDatasourceDefinition = async () => {
-    const $datasource = get(datasource)
-    if ($datasource.type === "table") {
-      table.set(await API.fetchTableDefinition($datasource.tableId))
-    } else if ($datasource.type === "viewV2") {
-      // const definition = await API.viewsV2.(get(tableId))
-      // table.set(definition)
-    }
-  }
-
   // Checks if we have a row with a certain ID
   const hasRow = id => {
     if (id === NewRowID) {
@@ -550,21 +521,7 @@ export const createActions = context => {
         refreshRow,
         replaceRow,
         refreshData,
-        refreshDatasourceDefinition,
       },
     },
   }
 }
-
-export const initialise = context => {
-  const { table, filter, datasource } = context
-
-  // For views, always keep the UI for filter and sorting up to date with the
-  // latest view definition
-  table.subscribe($definition => {
-    if (!$definition || get(datasource)?.type !== "viewV2") {
-      return
-    }
-    filter.set($definition.query)
-  })
-}

packages/frontend-core/src/components/grid/stores/sort.js

@@ -16,7 +16,8 @@ export const createStores = context => {
 }
 
 export const initialise = context => {
-  const { sort, initialSortColumn, initialSortOrder } = context
+  const { sort, initialSortColumn, initialSortOrder, table, datasource } =
+    context
 
   // Reset sort when initial sort props change
   initialSortColumn.subscribe(newSortColumn => {

packages/frontend-core/src/components/grid/stores/table.js

@@ -0,0 +1,24 @@
+import { get } from "svelte/store"
+
+export const initialise = context => {
+  const { datasource, fetch, filter, sort } = context
+
+  // Update fetch when filter changes
+  filter.subscribe($filter => {
+    if (get(datasource)?.type === "table") {
+      get(fetch)?.update({
+        filter: $filter,
+      })
+    }
+  })
+
+  // Update fetch when sorting changes
+  sort.subscribe($sort => {
+    if (get(datasource)?.type === "table") {
+      get(fetch)?.update({
+        sortOrder: $sort.order,
+        sortColumn: $sort.column,
+      })
+    }
+  })
+}

packages/frontend-core/src/components/grid/stores/ui.js

@@ -131,7 +131,7 @@ export const initialise = context => {
     focusedCellId,
     selectedRows,
     hoveredRowId,
-    table,
+    definition,
     rowHeight,
     fixedRowHeight,
   } = context
@@ -187,9 +187,9 @@ export const initialise = context => {
   })
 
   // Pull row height from table as long as we don't have a fixed height
-  table.subscribe($table => {
+  definition.subscribe($definition => {
     if (!get(fixedRowHeight)) {
-      rowHeight.set($table?.rowHeight || DefaultRowHeight)
+      rowHeight.set($definition?.rowHeight || DefaultRowHeight)
     }
   })
 
@@ -198,7 +198,7 @@ export const initialise = context => {
     if (height) {
       rowHeight.set(height)
     } else {
-      rowHeight.set(get(table)?.rowHeight || DefaultRowHeight)
+      rowHeight.set(get(definition)?.rowHeight || DefaultRowHeight)
     }
   })
 }

packages/frontend-core/src/components/grid/stores/viewV2.js

@@ -0,0 +1,43 @@
+import { get } from "svelte/store"
+
+export const initialise = context => {
+  const { definition, datasource, sort, rows } = context
+
+  // For views, keep sort state in line with the view definition
+  definition.subscribe($definition => {
+    if (!$definition || get(datasource)?.type !== "viewV2") {
+      return
+    }
+    const $sort = get(sort)
+    if (
+      $definition.sort?.field !== $sort?.column ||
+      $definition.sort?.order !== $sort?.order
+    ) {
+      sort.set({
+        column: $definition.sort?.field,
+        order: $definition.sort?.order,
+      })
+    }
+  })
+
+  // When sorting changes, ensure view definition is kept up to date
+  sort.subscribe(async $sort => {
+    const $view = get(definition)
+    if (!$view || get(datasource)?.type !== "viewV2") {
+      return
+    }
+    if (
+      $sort?.column !== $view.sort?.field ||
+      $sort?.order !== $view.sort?.order
+    ) {
+      await datasource.actions.saveDefinition({
+        ...$view,
+        sort: {
+          field: $sort.column,
+          order: $sort.order,
+        },
+      })
+      await rows.actions.refreshData()
+    }
+  })
+}

packages/frontend-core/src/fetch/DataFetch.js

@@ -116,8 +116,16 @@ export default class DataFetch {
   async getInitialData() {
     const { datasource, filter, paginate } = this.options
 
-    // Fetch datasource definition and determine feature flags
+    // Fetch datasource definition and extract filter and sort if configured
     const definition = await this.getDefinition(datasource)
+    if (definition?.sort?.field) {
+      this.options.sortColumn = definition.sort.field
+    }
+    if (definition?.sort?.order) {
+      this.options.sortOrder = definition.sort.order
+    }
+
+    // Determine feature flags
     const features = this.determineFeatureFlags(definition)
     this.features = {
       supportsSearch: !!features?.supportsSearch,