Merge branch 'views-v2-frontend' of github.com:Budibase/budibase into views-v2-frontend

packages/builder/src/builderStore/dataBinding.js

@@ -734,9 +734,19 @@ export const getSchemaForDatasource = (asset, datasource, options) => {
     // Determine the schema from the backing entity if not already determined
     if (table && !schema) {
       if (type === "view") {
-        // For views, the schema is pulled from the `views` property of the
-        // table
+        // Old views
         schema = cloneDeep(table.views?.[datasource.name]?.schema)
+      } else if (type === "viewV2") {
+        // New views which are DS+
+        const view = table.views?.[datasource.name]
+        schema = cloneDeep(view?.schema)
+
+        // Strip hidden fields
+        Object.keys(schema || {}).forEach(field => {
+          if (!schema[field].visible) {
+            delete schema[field]
+          }
+        })
       } else if (
         type === "query" &&
         (options.formSchema || options.searchableSchema)

packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte

@@ -13,7 +13,7 @@
   }
 
   const handleGridViewUpdate = async e => {
-    viewsV2.replace(id, e.detail)
+    viewsV2.replaceView(id, e.detail)
   }
 </script>
 

packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateViewButton.svelte

@@ -1,7 +1,7 @@
 <script>
   import { getContext } from "svelte"
   import { Modal, ActionButton } from "@budibase/bbui"
-  import CreateViewModal from "../../modals/CreateViewModal.svelte"
+  import GridCreateViewModal from "../../modals/grid/GridCreateViewModal.svelte"
 
   const { rows, columns } = getContext("grid")
 
@@ -14,5 +14,5 @@
   Add view
 </ActionButton>
 <Modal bind:this={modal}>
-  <CreateViewModal />
+  <GridCreateViewModal />
 </Modal>

packages/builder/src/components/backend/DataTable/buttons/grid/GridFilterButton.svelte

@@ -4,9 +4,6 @@
 
   const { columns, datasource, filter, definition } = getContext("grid")
 
-  // Wipe filter whenever table ID changes to avoid using stale filters
-  $: $datasource, filter.set([])
-
   const onFilter = e => {
     filter.set(e.detail || [])
   }

packages/builder/src/components/backend/DataTable/modals/grid/GridCreateViewModal.svelte

@@ -3,16 +3,12 @@
   import { Input, notifications, ModalContent } from "@budibase/bbui"
   import { goto } from "@roxi/routify"
   import { viewsV2 } from "stores/backend"
-  import { LuceneUtils } from "@budibase/frontend-core"
 
-  const { filter, sort, table } = getContext("grid")
-
-  $: query = LuceneUtils.buildLuceneQuery($filter)
+  const { filter, sort, definition } = getContext("grid")
 
   let name
 
-  $: console.log($table)
-  $: views = Object.keys($table?.views || {})
+  $: views = Object.keys($definition?.views || {})
   $: nameExists = views.includes(name?.trim())
 
   const saveView = async () => {
@@ -20,14 +16,14 @@
     try {
       const newView = await viewsV2.create({
         name,
-        tableId: $table._id,
-        query,
+        tableId: $definition._id,
+        query: $filter,
         sort: {
           field: $sort.column,
           order: $sort.order,
         },
-        schema: $table.schema,
-        primaryDisplay: $table.primaryDisplay,
+        schema: $definition.schema,
+        primaryDisplay: $definition.primaryDisplay,
       })
       notifications.success(`View ${name} created`)
       $goto(`../../view/v2/${newView.id}`)

packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte

@@ -58,7 +58,8 @@
               $goto(`./view/v1/${encodeURIComponent(name)}`)
             }
           }}
-          selectedBy={$userSelectedResourceMap[name]}
+          selectedBy={$userSelectedResourceMap[name] ||
+            $userSelectedResourceMap[view.id]}
         >
           <EditViewPopover {view} />
         </NavItem>

packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte

@@ -39,15 +39,33 @@
     tableId: m._id,
     type: "table",
   }))
-  $: views = $tablesStore.list.reduce((acc, cur) => {
-    let viewsArr = Object.entries(cur.views || {}).map(([key, value]) => ({
-      label: key,
-      name: key,
-      ...value,
-      type: "view",
-    }))
-    return [...acc, ...viewsArr]
-  }, [])
+  $: viewsV1 = $tablesStore.list.reduce(
+    (acc, table) => [
+      ...acc,
+      ...Object.values(table.views || {})
+        .filter(view => view.version !== 2)
+        .map(view => ({
+          ...view,
+          label: view.name,
+          type: "view",
+        })),
+    ],
+    []
+  )
+  $: viewsV2 = $tablesStore.list.reduce(
+    (acc, table) => [
+      ...acc,
+      ...Object.values(table.views || {})
+        .filter(view => view.version === 2)
+        .map(view => ({
+          ...view,
+          label: view.name,
+          type: "viewV2",
+        })),
+    ],
+    []
+  )
+  $: views = [...(viewsV1 || []), ...(viewsV2 || [])]
   $: queries = $queriesStore.list
     .filter(q => showAllQueries || q.queryVerb === "read" || q.readable)
     .map(query => ({

packages/builder/src/components/design/settings/controls/TableSelect.svelte

@@ -7,22 +7,50 @@
 
   const dispatch = createEventDispatcher()
 
-  $: tables = $tablesStore.list.map(m => ({
-    label: m.name,
-    tableId: m._id,
+  $: tables = $tablesStore.list.map(table => ({
+    ...table,
     type: "table",
+    label: table.name,
+    key: table._id,
   }))
+  $: views = $tablesStore.list.reduce(
+    (acc, table) => [
+      ...acc,
+      ...Object.values(table.views || {})
+        .filter(view => view.version === 2)
+        .map(view => ({
+          ...view,
+          type: "viewV2",
+          label: view.name,
+          key: view.id,
+        })),
+    ],
+    []
+  )
+  $: options = [...(tables || []), ...(views || [])]
+  $: {
+    // Migrate old table values before "key" existed
+    if (value && !value.key) {
+      console.log("migrate")
+      dispatch(
+        "change",
+        tables.find(x => x.tableId === value.tableId)
+      )
+    }
+  }
 
   const onChange = e => {
-    const dataSource = tables?.find(x => x.tableId === e.detail)
-    dispatch("change", dataSource)
+    dispatch(
+      "change",
+      options.find(x => x.key === e.detail)
+    )
   }
 </script>
 
 <Select
   on:change={onChange}
-  value={value?.tableId}
-  options={tables}
-  getOptionValue={x => x.tableId}
+  value={value?.key}
+  {options}
+  getOptionValue={x => x.key}
   getOptionLabel={x => x.label}
 />

packages/builder/src/stores/backend/viewsV2.js

@@ -1,4 +1,4 @@
-import { writable, derived } from "svelte/store"
+import { writable, derived, get } from "svelte/store"
 import { tables } from "./"
 import { API } from "api"
 
@@ -30,44 +30,63 @@ export function createViewsV2Store() {
 
   const deleteView = async view => {
     await API.viewV2.delete(view.id)
-
-    // Update tables
-    tables.update(state => {
-      const table = state.list.find(table => table._id === view.tableId)
-      delete table.views[view.name]
-      return { ...state }
-    })
+    replaceView(view.id, null)
   }
 
   const create = async view => {
     const savedViewResponse = await API.viewV2.create(view)
     const savedView = savedViewResponse.data
-
-    // Update tables
-    tables.update(state => {
-      const table = state.list.find(table => table._id === view.tableId)
-      table.views[view.name] = savedView
-      return { ...state }
-    })
-
+    replaceView(savedView.id, savedView)
     return savedView
   }
 
   const save = async view => {
     const res = await API.viewV2.update(view)
     const savedView = res?.data
+    replaceView(view.id, savedView)
+  }
 
-    // Update tables
-    tables.update(state => {
-      const table = state.list.find(table => table._id === view.tableId)
-      if (table) {
-        if (view.originalName) {
-          delete table.views[view.originalName]
-        }
-        table.views[view.name] = savedView
-      }
-      return { ...state }
+  // Handles external updates of tables
+  const replaceView = (viewId, view) => {
+    if (!viewId) {
+      return
+    }
+    const existingView = get(derivedStore).list.find(view => view.id === viewId)
+    const tableIndex = get(tables).list.findIndex(table => {
+      return table._id === view?.tableId || table._id === existingView?.tableId
     })
+    if (tableIndex === -1) {
+      return
+    }
+
+    // Handle deletion
+    if (!view) {
+      tables.update(state => {
+        delete state.list[tableIndex].views[existingView.name]
+        return state
+      })
+      return
+    }
+
+    // Add new view
+    if (!existingView) {
+      tables.update(state => {
+        state.list[tableIndex].views[view.name] = view
+        return state
+      })
+    }
+
+    // Update existing view
+    else {
+      tables.update(state => {
+        // Remove old view
+        delete state.list[tableIndex].views[existingView.name]
+
+        // Add new view
+        state.list[tableIndex].views[view.name] = view
+        return state
+      })
+    }
   }
 
   return {
@@ -76,6 +95,7 @@ export function createViewsV2Store() {
     delete: deleteView,
     create,
     save,
+    replaceView,
   }
 }
 

packages/client/manifest.json

@@ -5274,7 +5274,7 @@
       },
       {
         "type": "table",
-        "label": "Table",
+        "label": "Data",
         "key": "dataSource"
       },
       {
@@ -5549,7 +5549,7 @@
     "settings": [
       {
         "type": "table",
-        "label": "Table",
+        "label": "Data",
         "key": "table",
         "required": true
       },

packages/client/src/components/app/GridBlock.svelte

@@ -38,11 +38,8 @@
   class:in-builder={$builderStore.inBuilder}
 >
   <Grid
-    tableId={table?.tableId}
+    datasource={table}
     {API}
-    {allowAddRows}
-    {allowEditRows}
-    {allowDeleteRows}
     {stripeRows}
     {initialFilter}
     {initialSortColumn}
@@ -50,9 +47,12 @@
     {fixedRowHeight}
     {columnWhitelist}
     {schemaOverrides}
+    canAddRows={allowAddRows}
+    canEditRows={allowEditRows}
+    canDeleteRows={allowDeleteRows}
+    canExpandRows={false}
+    canSaveSchema={false}
     showControls={false}
-    allowExpandRows={false}
-    allowSchemaChanges={false}
     notifySuccess={notificationStore.actions.success}
     notifyError={notificationStore.actions.error}
   />

packages/client/src/utils/schema.js

@@ -6,6 +6,7 @@ import RelationshipFetch from "@budibase/frontend-core/src/fetch/RelationshipFet
 import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProviderFetch.js"
 import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch.js"
 import JSONArrayFetch from "@budibase/frontend-core/src/fetch/JSONArrayFetch.js"
+import ViewV2Fetch from "@budibase/frontend-core/src/fetch/ViewV2Fetch.js"
 
 /**
  * Fetches the schema of any kind of datasource.
@@ -21,6 +22,7 @@ export const fetchDatasourceSchema = async (
   const handler = {
     table: TableFetch,
     view: ViewFetch,
+    viewV2: ViewV2Fetch,
     query: QueryFetch,
     link: RelationshipFetch,
     provider: NestedProviderFetch,
@@ -49,6 +51,15 @@ export const fetchDatasourceSchema = async (
     return null
   }
 
+  // Strip hidden fields from views
+  if (datasource.type === "viewV2") {
+    Object.keys(schema).forEach(field => {
+      if (!schema[field].visible) {
+        delete schema[field]
+      }
+    })
+  }
+
   // Enrich schema with relationships if required
   if (definition?.sql && options?.enrichRelationships) {
     const relationshipAdditions = await getRelationshipSchemaAdditions(schema)

packages/frontend-core/src/api/viewsV2.js

@@ -22,9 +22,33 @@ export const buildViewV2Endpoints = API => ({
   /**
    * Fetches all rows in a view
    * @param viewId the id of the view
+   * @param paginate whether to paginate or not
+   * @param limit page size
+   * @param bookmark pagination cursor
+   * @param sort sort column
+   * @param sortOrder sort order
+   * @param sortType sort type (text or numeric)
    */
-  fetch: async viewId => {
-    return await API.get({ url: `/api/v2/views/${viewId}/search` })
+  fetch: async ({
+    viewId,
+    paginate,
+    limit,
+    bookmark,
+    sort,
+    sortOrder,
+    sortType,
+  }) => {
+    return await API.post({
+      url: `/api/v2/views/${viewId}/search`,
+      body: {
+        paginate,
+        limit,
+        bookmark,
+        sort,
+        sortOrder,
+        sortType,
+      },
+    })
   },
   /**
    * Delete a view

packages/frontend-core/src/components/grid/stores/datasource.js

@@ -80,7 +80,7 @@ export const createActions = context => {
 
     // Broadcast change to external state can be updated, as this change
     // will not be received by the builder websocket because we caused it ourselves
-    dispatch("updatedefinition", newDefinition)
+    dispatch("updatedatasource", newDefinition)
   }
 
   // Adds a row to the datasource
@@ -98,10 +98,16 @@ export const createActions = context => {
     return await getAPI()?.actions.deleteRows(rows)
   }
 
+  // Gets a single row from a datasource
   const getRow = async id => {
     return await getAPI()?.actions.getRow(id)
   }
 
+  // Checks if a certain datasource config is valid
+  const isDatasourceValid = datasource => {
+    return getAPI()?.actions.isDatasourceValid(datasource)
+  }
+
   return {
     datasource: {
       ...datasource,
@@ -112,6 +118,7 @@ export const createActions = context => {
         updateRow,
         deleteRows,
         getRow,
+        isDatasourceValid,
       },
     },
   }

packages/frontend-core/src/components/grid/stores/rows.js

@@ -98,7 +98,7 @@ export const createActions = context => {
     loading.set(true)
 
     // Abandon if we don't have a valid datasource
-    if (!$datasource) {
+    if (!datasource.actions.isDatasourceValid($datasource)) {
       return
     }
 

packages/frontend-core/src/components/grid/stores/table.js

@@ -25,6 +25,10 @@ export const createActions = context => {
     })
   }
 
+  const isDatasourceValid = datasource => {
+    return datasource?.type === "table" && datasource?.tableId
+  }
+
   const getRow = async id => {
     const res = await API.searchTable({
       tableId: get(datasource).tableId,
@@ -48,6 +52,7 @@ export const createActions = context => {
         updateRow: saveRow,
         deleteRows,
         getRow,
+        isDatasourceValid,
       },
     },
   }
@@ -56,38 +61,49 @@ export const createActions = context => {
 export const initialise = context => {
   const { datasource, fetch, filter, sort, definition } = context
 
+  // Wipe filter whenever table ID changes to avoid using stale filters
+  definition.subscribe(() => {
+    if (get(datasource)?.type !== "table") {
+      return
+    }
+    filter.set([])
+  })
+
   // Update fetch when filter changes
   filter.subscribe($filter => {
-    if (get(datasource)?.type === "table") {
-      get(fetch)?.update({
-        filter: $filter,
-      })
+    if (get(datasource)?.type !== "table") {
+      return
     }
+    get(fetch)?.update({
+      filter: $filter,
+    })
   })
 
   // Update fetch when sorting changes
   sort.subscribe($sort => {
-    if (get(datasource)?.type === "table") {
-      get(fetch)?.update({
-        sortOrder: $sort.order,
-        sortColumn: $sort.column,
-      })
+    if (get(datasource)?.type !== "table") {
+      return
     }
+    get(fetch)?.update({
+      sortOrder: $sort.order,
+      sortColumn: $sort.column,
+    })
   })
 
   // Ensure sorting UI reflects the fetch state whenever we reset the fetch,
   // which triggers a new definition
   definition.subscribe(() => {
-    if (get(datasource)?.type === "table") {
-      const $fetch = get(fetch)
-      if (!$fetch) {
-        return
-      }
-      const { sortColumn, sortOrder } = get($fetch)
-      sort.set({
-        column: sortColumn,
-        order: sortOrder,
-      })
+    if (get(datasource)?.type !== "table") {
+      return
     }
+    const $fetch = get(fetch)
+    if (!$fetch) {
+      return
+    }
+    const { sortColumn, sortOrder } = get($fetch)
+    sort.set({
+      column: sortColumn,
+      order: sortOrder,
+    })
   })
 }

packages/frontend-core/src/components/grid/stores/viewV2.js

@@ -60,6 +60,12 @@ export const createActions = context => {
     return res?.rows?.[0]
   }
 
+  const isDatasourceValid = datasource => {
+    return (
+      datasource?.type === "viewV2" && datasource?.id && datasource?.tableId
+    )
+  }
+
   return {
     viewV2: {
       actions: {
@@ -69,15 +75,16 @@ export const createActions = context => {
         updateRow,
         deleteRows,
         getRow,
+        isDatasourceValid,
       },
     },
   }
 }
 
 export const initialise = context => {
-  const { definition, datasource, sort, rows } = context
+  const { definition, datasource, sort, rows, filter } = context
 
-  // For views, keep sort state in line with the view definition
+  // Keep sort and filter state in line with the view definition
   definition.subscribe($definition => {
     if (!$definition || get(datasource)?.type !== "viewV2") {
       return
@@ -86,6 +93,7 @@ export const initialise = context => {
       column: $definition.sort?.field,
       order: $definition.sort?.order,
     })
+    filter.set($definition.query || [])
   })
 
   // When sorting changes, ensure view definition is kept up to date
@@ -108,4 +116,19 @@ export const initialise = context => {
       await rows.actions.refreshData()
     }
   })
+
+  // When filters change, ensure view definition is kept up to date
+  filter.subscribe(async $filter => {
+    const $view = get(definition)
+    if (!$view || get(datasource)?.type !== "viewV2") {
+      return
+    }
+    if (JSON.stringify($filter) !== JSON.stringify($view.query)) {
+      await datasource.actions.saveDefinition({
+        ...$view,
+        query: $filter,
+      })
+      await rows.actions.refreshData()
+    }
+  })
 }

packages/frontend-core/src/fetch/DataFetch.js

@@ -116,7 +116,7 @@ export default class DataFetch {
   async getInitialData() {
     const { datasource, filter, paginate } = this.options
 
-    // Fetch datasource definition and extract filter and sort if configured
+    // Fetch datasource definition and extract sort properties if configured
     const definition = await this.getDefinition(datasource)
     if (definition?.sort?.field) {
       this.options.sortColumn = definition.sort.field

packages/frontend-core/src/fetch/ViewV2Fetch.js

@@ -1,15 +1,19 @@
 import DataFetch from "./DataFetch.js"
+import { get } from "svelte/store"
 
 export default class ViewV2Fetch extends DataFetch {
   determineFeatureFlags() {
     return {
+      // The API does not actually support dynamic filtering, but since views
+      // have filters built in we don't want to perform client side filtering
+      // which would happen if we marked this as false
       supportsSearch: true,
       supportsSort: true,
       supportsPagination: true,
     }
   }
 
-  async getSchema(datasource, definition) {
+  getSchema(datasource, definition) {
     return definition?.schema
   }
 
@@ -29,12 +33,30 @@ export default class ViewV2Fetch extends DataFetch {
   }
 
   async getData() {
-    const { datasource } = this.options
+    const { datasource, limit, sortColumn, sortOrder, sortType, paginate } =
+      this.options
+    const { cursor } = get(this.store)
     try {
-      const res = await this.API.viewV2.fetch(datasource.id)
-      return { rows: res?.rows || [] }
+      const res = await this.API.viewV2.fetch({
+        viewId: datasource.id,
+        paginate,
+        limit,
+        bookmark: cursor,
+        sort: sortColumn,
+        sortOrder,
+        sortType,
+      })
+      return {
+        rows: res?.rows || [],
+        hasNextPage: res?.hasNextPage || false,
+        cursor: res?.bookmark || null,
+      }
     } catch (error) {
-      return { rows: [] }
+      return {
+        rows: [],
+        hasNextPage: false,
+        error,
+      }
     }
   }
 }

packages/server/src/api/controllers/table/internal.ts

@@ -9,7 +9,7 @@ import {
   fixAutoColumnSubType,
 } from "../../../utilities/rowProcessor"
 import { runStaticFormulaChecks } from "./bulkFormula"
-import { Table } from "@budibase/types"
+import { Table, ViewStatisticsSchema } from "@budibase/types"
 import { quotas } from "@budibase/pro"
 import isEqual from "lodash/isEqual"
 import { cloneDeep } from "lodash/fp"
@@ -36,7 +36,9 @@ function checkAutoColumns(table: Table, oldTable?: Table) {
 export async function save(ctx: any) {
   const db = context.getAppDB()
   const { rows, ...rest } = ctx.request.body
-  let tableToSave = {
+  let tableToSave: Table & {
+    _rename?: { old: string; updated: string } | null
+  } = {
     type: "table",
     _id: generateTableID(),
     views: {},
@@ -44,7 +46,7 @@ export async function save(ctx: any) {
   }
 
   // if the table obj had an _id then it will have been retrieved
-  let oldTable
+  let oldTable: Table | undefined
   if (ctx.request.body && ctx.request.body._id) {
     oldTable = await sdk.tables.getTable(ctx.request.body._id)
   }
@@ -97,7 +99,17 @@ export async function save(ctx: any) {
     const tableView = tableToSave.views[view]
     if (!tableView) continue
 
-    if (tableView.schema.group || tableView.schema.field) continue
+    if (sdk.views.isV2(tableView)) {
+      // We don't want to modify views from the tables controller
+      tableToSave.views[view] = oldTable!.views![view]
+      continue
+    }
+
+    if (
+      (tableView.schema as ViewStatisticsSchema).group ||
+      tableView.schema.field
+    )
+      continue
     tableView.schema = tableToSave.schema
   }
 

packages/server/src/api/controllers/view/viewsV2.ts

@@ -8,6 +8,7 @@ import {
   ViewV2,
   RequiredKeys,
 } from "@budibase/types"
+import { builderSocket } from "../../../websockets"
 
 async function parseSchemaUI(ctx: Ctx, view: CreateViewRequest) {
   if (!view.schema) {
@@ -86,6 +87,9 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
   ctx.body = {
     data: result,
   }
+
+  const table = await sdk.tables.getTable(tableId)
+  builderSocket?.emitTableUpdate(ctx, table)
 }
 
 export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
@@ -118,11 +122,17 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
   ctx.body = {
     data: result,
   }
+
+  const table = await sdk.tables.getTable(tableId)
+  builderSocket?.emitTableUpdate(ctx, table)
 }
 
 export async function remove(ctx: Ctx) {
   const { viewId } = ctx.params
 
-  await sdk.views.remove(viewId)
+  const view = await sdk.views.remove(viewId)
   ctx.status = 204
+
+  const table = await sdk.tables.getTable(view.tableId)
+  builderSocket?.emitTableUpdate(ctx, table)
 }

packages/server/src/sdk/app/views/index.ts

@@ -53,7 +53,7 @@ export function isV2(view: View | ViewV2): view is ViewV2 {
   return (view as ViewV2).version === 2
 }
 
-export async function remove(viewId: string): Promise<void> {
+export async function remove(viewId: string): Promise<ViewV2> {
   const db = context.getAppDB()
 
   const view = await get(viewId)
@@ -64,6 +64,7 @@ export async function remove(viewId: string): Promise<void> {
 
   delete table.views![view?.name]
   await db.put(table)
+  return view
 }
 
 export function enrichSchema(view: View | ViewV2, tableSchema: TableSchema) {